1 /* 2 * Copyright (c) 2006 ARM Ltd. 3 * Copyright (c) 2010 ST-Ericsson SA 4 * Copyirght (c) 2017 Linaro Ltd. 5 * 6 * Author: Peter Pearse <peter.pearse@arm.com> 7 * Author: Linus Walleij <linus.walleij@linaro.org> 8 * 9 * This program is free software; you can redistribute it and/or modify it 10 * under the terms of the GNU General Public License as published by the Free 11 * Software Foundation; either version 2 of the License, or (at your option) 12 * any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but WITHOUT 15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 17 * more details. 18 * 19 * The full GNU General Public License is in this distribution in the file 20 * called COPYING. 21 * 22 * Documentation: ARM DDI 0196G == PL080 23 * Documentation: ARM DDI 0218E == PL081 24 * Documentation: S3C6410 User's Manual == PL080S 25 * 26 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any 27 * channel. 28 * 29 * The PL080 has 8 channels available for simultaneous use, and the PL081 30 * has only two channels. So on these DMA controllers the number of channels 31 * and the number of incoming DMA signals are two totally different things. 32 * It is usually not possible to theoretically handle all physical signals, 33 * so a multiplexing scheme with possible denial of use is necessary. 34 * 35 * The PL080 has a dual bus master, PL081 has a single master. 36 * 37 * PL080S is a version modified by Samsung and used in S3C64xx SoCs. 38 * It differs in following aspects: 39 * - CH_CONFIG register at different offset, 40 * - separate CH_CONTROL2 register for transfer size, 41 * - bigger maximum transfer size, 42 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, 43 * - no support for peripheral flow control. 44 * 45 * Memory to peripheral transfer may be visualized as 46 * Get data from memory to DMAC 47 * Until no data left 48 * On burst request from peripheral 49 * Destination burst from DMAC to peripheral 50 * Clear burst request 51 * Raise terminal count interrupt 52 * 53 * For peripherals with a FIFO: 54 * Source burst size == half the depth of the peripheral FIFO 55 * Destination burst size == the depth of the peripheral FIFO 56 * 57 * (Bursts are irrelevant for mem to mem transfers - there are no burst 58 * signals, the DMA controller will simply facilitate its AHB master.) 59 * 60 * ASSUMES default (little) endianness for DMA transfers 61 * 62 * The PL08x has two flow control settings: 63 * - DMAC flow control: the transfer size defines the number of transfers 64 * which occur for the current LLI entry, and the DMAC raises TC at the 65 * end of every LLI entry. Observed behaviour shows the DMAC listening 66 * to both the BREQ and SREQ signals (contrary to documented), 67 * transferring data if either is active. The LBREQ and LSREQ signals 68 * are ignored. 69 * 70 * - Peripheral flow control: the transfer size is ignored (and should be 71 * zero). The data is transferred from the current LLI entry, until 72 * after the final transfer signalled by LBREQ or LSREQ. The DMAC 73 * will then move to the next LLI entry. Unsupported by PL080S. 74 */ 75 #include <linux/amba/bus.h> 76 #include <linux/amba/pl08x.h> 77 #include <linux/debugfs.h> 78 #include <linux/delay.h> 79 #include <linux/device.h> 80 #include <linux/dmaengine.h> 81 #include <linux/dmapool.h> 82 #include <linux/dma-mapping.h> 83 #include <linux/export.h> 84 #include <linux/init.h> 85 #include <linux/interrupt.h> 86 #include <linux/module.h> 87 #include <linux/of.h> 88 #include <linux/of_dma.h> 89 #include <linux/pm_runtime.h> 90 #include <linux/seq_file.h> 91 #include <linux/slab.h> 92 #include <linux/amba/pl080.h> 93 94 #include "dmaengine.h" 95 #include "virt-dma.h" 96 97 #define DRIVER_NAME "pl08xdmac" 98 99 #define PL80X_DMA_BUSWIDTHS \ 100 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 101 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 102 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 103 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 104 105 static struct amba_driver pl08x_amba_driver; 106 struct pl08x_driver_data; 107 108 /** 109 * struct vendor_data - vendor-specific config parameters for PL08x derivatives 110 * @config_offset: offset to the configuration register 111 * @channels: the number of channels available in this variant 112 * @signals: the number of request signals available from the hardware 113 * @dualmaster: whether this version supports dual AHB masters or not. 114 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the 115 * channels have Nomadik security extension bits that need to be checked 116 * for permission before use and some registers are missing 117 * @pl080s: whether this variant is a Samsung PL080S, which has separate 118 * register and LLI word for transfer size. 119 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020 120 * @max_transfer_size: the maximum single element transfer size for this 121 * PL08x variant. 122 */ 123 struct vendor_data { 124 u8 config_offset; 125 u8 channels; 126 u8 signals; 127 bool dualmaster; 128 bool nomadik; 129 bool pl080s; 130 bool ftdmac020; 131 u32 max_transfer_size; 132 }; 133 134 /** 135 * struct pl08x_bus_data - information of source or destination 136 * busses for a transfer 137 * @addr: current address 138 * @maxwidth: the maximum width of a transfer on this bus 139 * @buswidth: the width of this bus in bytes: 1, 2 or 4 140 */ 141 struct pl08x_bus_data { 142 dma_addr_t addr; 143 u8 maxwidth; 144 u8 buswidth; 145 }; 146 147 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) 148 149 /** 150 * struct pl08x_phy_chan - holder for the physical channels 151 * @id: physical index to this channel 152 * @base: memory base address for this physical channel 153 * @reg_config: configuration address for this physical channel 154 * @reg_control: control address for this physical channel 155 * @reg_src: transfer source address register 156 * @reg_dst: transfer destination address register 157 * @reg_lli: transfer LLI address register 158 * @reg_busy: if the variant has a special per-channel busy register, 159 * this contains a pointer to it 160 * @lock: a lock to use when altering an instance of this struct 161 * @serving: the virtual channel currently being served by this physical 162 * channel 163 * @locked: channel unavailable for the system, e.g. dedicated to secure 164 * world 165 * @ftdmac020: channel is on a FTDMAC020 166 * @pl080s: channel is on a PL08s 167 */ 168 struct pl08x_phy_chan { 169 unsigned int id; 170 void __iomem *base; 171 void __iomem *reg_config; 172 void __iomem *reg_control; 173 void __iomem *reg_src; 174 void __iomem *reg_dst; 175 void __iomem *reg_lli; 176 void __iomem *reg_busy; 177 spinlock_t lock; 178 struct pl08x_dma_chan *serving; 179 bool locked; 180 bool ftdmac020; 181 bool pl080s; 182 }; 183 184 /** 185 * struct pl08x_sg - structure containing data per sg 186 * @src_addr: src address of sg 187 * @dst_addr: dst address of sg 188 * @len: transfer len in bytes 189 * @node: node for txd's dsg_list 190 */ 191 struct pl08x_sg { 192 dma_addr_t src_addr; 193 dma_addr_t dst_addr; 194 size_t len; 195 struct list_head node; 196 }; 197 198 /** 199 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor 200 * @vd: virtual DMA descriptor 201 * @dsg_list: list of children sg's 202 * @llis_bus: DMA memory address (physical) start for the LLIs 203 * @llis_va: virtual memory address start for the LLIs 204 * @cctl: control reg values for current txd 205 * @ccfg: config reg values for current txd 206 * @done: this marks completed descriptors, which should not have their 207 * mux released. 208 * @cyclic: indicate cyclic transfers 209 */ 210 struct pl08x_txd { 211 struct virt_dma_desc vd; 212 struct list_head dsg_list; 213 dma_addr_t llis_bus; 214 u32 *llis_va; 215 /* Default cctl value for LLIs */ 216 u32 cctl; 217 /* 218 * Settings to be put into the physical channel when we 219 * trigger this txd. Other registers are in llis_va[0]. 220 */ 221 u32 ccfg; 222 bool done; 223 bool cyclic; 224 }; 225 226 /** 227 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel 228 * states 229 * @PL08X_CHAN_IDLE: the channel is idle 230 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport 231 * channel and is running a transfer on it 232 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport 233 * channel, but the transfer is currently paused 234 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport 235 * channel to become available (only pertains to memcpy channels) 236 */ 237 enum pl08x_dma_chan_state { 238 PL08X_CHAN_IDLE, 239 PL08X_CHAN_RUNNING, 240 PL08X_CHAN_PAUSED, 241 PL08X_CHAN_WAITING, 242 }; 243 244 /** 245 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 246 * @vc: wrappped virtual channel 247 * @phychan: the physical channel utilized by this channel, if there is one 248 * @name: name of channel 249 * @cd: channel platform data 250 * @cfg: slave configuration 251 * @at: active transaction on this channel 252 * @host: a pointer to the host (internal use) 253 * @state: whether the channel is idle, paused, running etc 254 * @slave: whether this channel is a device (slave) or for memcpy 255 * @signal: the physical DMA request signal which this channel is using 256 * @mux_use: count of descriptors using this DMA request signal setting 257 */ 258 struct pl08x_dma_chan { 259 struct virt_dma_chan vc; 260 struct pl08x_phy_chan *phychan; 261 const char *name; 262 struct pl08x_channel_data *cd; 263 struct dma_slave_config cfg; 264 struct pl08x_txd *at; 265 struct pl08x_driver_data *host; 266 enum pl08x_dma_chan_state state; 267 bool slave; 268 int signal; 269 unsigned mux_use; 270 }; 271 272 /** 273 * struct pl08x_driver_data - the local state holder for the PL08x 274 * @slave: optional slave engine for this instance 275 * @memcpy: memcpy engine for this instance 276 * @has_slave: the PL08x has a slave engine (routed signals) 277 * @base: virtual memory base (remapped) for the PL08x 278 * @adev: the corresponding AMBA (PrimeCell) bus entry 279 * @vd: vendor data for this PL08x variant 280 * @pd: platform data passed in from the platform/machine 281 * @phy_chans: array of data for the physical channels 282 * @pool: a pool for the LLI descriptors 283 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI 284 * fetches 285 * @mem_buses: set to indicate memory transfers on AHB2. 286 * @lli_words: how many words are used in each LLI item for this variant 287 */ 288 struct pl08x_driver_data { 289 struct dma_device slave; 290 struct dma_device memcpy; 291 bool has_slave; 292 void __iomem *base; 293 struct amba_device *adev; 294 const struct vendor_data *vd; 295 struct pl08x_platform_data *pd; 296 struct pl08x_phy_chan *phy_chans; 297 struct dma_pool *pool; 298 u8 lli_buses; 299 u8 mem_buses; 300 u8 lli_words; 301 }; 302 303 /* 304 * PL08X specific defines 305 */ 306 307 /* The order of words in an LLI. */ 308 #define PL080_LLI_SRC 0 309 #define PL080_LLI_DST 1 310 #define PL080_LLI_LLI 2 311 #define PL080_LLI_CCTL 3 312 #define PL080S_LLI_CCTL2 4 313 314 /* Total words in an LLI. */ 315 #define PL080_LLI_WORDS 4 316 #define PL080S_LLI_WORDS 8 317 318 /* 319 * Number of LLIs in each LLI buffer allocated for one transfer 320 * (maximum times we call dma_pool_alloc on this pool without freeing) 321 */ 322 #define MAX_NUM_TSFR_LLIS 512 323 #define PL08X_ALIGN 8 324 325 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 326 { 327 return container_of(chan, struct pl08x_dma_chan, vc.chan); 328 } 329 330 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) 331 { 332 return container_of(tx, struct pl08x_txd, vd.tx); 333 } 334 335 /* 336 * Mux handling. 337 * 338 * This gives us the DMA request input to the PL08x primecell which the 339 * peripheral described by the channel data will be routed to, possibly 340 * via a board/SoC specific external MUX. One important point to note 341 * here is that this does not depend on the physical channel. 342 */ 343 static int pl08x_request_mux(struct pl08x_dma_chan *plchan) 344 { 345 const struct pl08x_platform_data *pd = plchan->host->pd; 346 int ret; 347 348 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { 349 ret = pd->get_xfer_signal(plchan->cd); 350 if (ret < 0) { 351 plchan->mux_use = 0; 352 return ret; 353 } 354 355 plchan->signal = ret; 356 } 357 return 0; 358 } 359 360 static void pl08x_release_mux(struct pl08x_dma_chan *plchan) 361 { 362 const struct pl08x_platform_data *pd = plchan->host->pd; 363 364 if (plchan->signal >= 0) { 365 WARN_ON(plchan->mux_use == 0); 366 367 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { 368 pd->put_xfer_signal(plchan->cd, plchan->signal); 369 plchan->signal = -1; 370 } 371 } 372 } 373 374 /* 375 * Physical channel handling 376 */ 377 378 /* Whether a certain channel is busy or not */ 379 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) 380 { 381 unsigned int val; 382 383 /* If we have a special busy register, take a shortcut */ 384 if (ch->reg_busy) { 385 val = readl(ch->reg_busy); 386 return !!(val & BIT(ch->id)); 387 } 388 val = readl(ch->reg_config); 389 return val & PL080_CONFIG_ACTIVE; 390 } 391 392 /* 393 * pl08x_write_lli() - Write an LLI into the DMA controller. 394 * 395 * The PL08x derivatives support linked lists, but the first item of the 396 * list containing the source, destination, control word and next LLI is 397 * ignored. Instead the driver has to write those values directly into the 398 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE 399 * register need to be set up for the first transfer. 400 */ 401 static void pl08x_write_lli(struct pl08x_driver_data *pl08x, 402 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) 403 { 404 if (pl08x->vd->pl080s) 405 dev_vdbg(&pl08x->adev->dev, 406 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 407 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", 408 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 409 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], 410 lli[PL080S_LLI_CCTL2], ccfg); 411 else 412 dev_vdbg(&pl08x->adev->dev, 413 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 414 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", 415 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 416 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); 417 418 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); 419 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); 420 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); 421 422 /* 423 * The FTMAC020 has a different layout in the CCTL word of the LLI 424 * and the CCTL register which is split in CSR and SIZE registers. 425 * Convert the LLI item CCTL into the proper values to write into 426 * the CSR and SIZE registers. 427 */ 428 if (phychan->ftdmac020) { 429 u32 llictl = lli[PL080_LLI_CCTL]; 430 u32 val = 0; 431 432 /* Write the transfer size (12 bits) to the size register */ 433 writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK, 434 phychan->base + FTDMAC020_CH_SIZE); 435 /* 436 * Then write the control bits 28..16 to the control register 437 * by shuffleing the bits around to where they are in the 438 * main register. The mapping is as follows: 439 * Bit 28: TC_MSK - mask on all except last LLI 440 * Bit 27..25: SRC_WIDTH 441 * Bit 24..22: DST_WIDTH 442 * Bit 21..20: SRCAD_CTRL 443 * Bit 19..17: DSTAD_CTRL 444 * Bit 17: SRC_SEL 445 * Bit 16: DST_SEL 446 */ 447 if (llictl & FTDMAC020_LLI_TC_MSK) 448 val |= FTDMAC020_CH_CSR_TC_MSK; 449 val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 450 (FTDMAC020_LLI_SRC_WIDTH_SHIFT - 451 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT)); 452 val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 453 (FTDMAC020_LLI_DST_WIDTH_SHIFT - 454 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT)); 455 val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >> 456 (FTDMAC020_LLI_SRCAD_CTL_SHIFT - 457 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT)); 458 val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >> 459 (FTDMAC020_LLI_DSTAD_CTL_SHIFT - 460 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT)); 461 if (llictl & FTDMAC020_LLI_SRC_SEL) 462 val |= FTDMAC020_CH_CSR_SRC_SEL; 463 if (llictl & FTDMAC020_LLI_DST_SEL) 464 val |= FTDMAC020_CH_CSR_DST_SEL; 465 466 /* 467 * Set up the bits that exist in the CSR but are not 468 * part the LLI, i.e. only gets written to the control 469 * register right here. 470 * 471 * FIXME: do not just handle memcpy, also handle slave DMA. 472 */ 473 switch (pl08x->pd->memcpy_burst_size) { 474 default: 475 case PL08X_BURST_SZ_1: 476 val |= PL080_BSIZE_1 << 477 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 478 break; 479 case PL08X_BURST_SZ_4: 480 val |= PL080_BSIZE_4 << 481 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 482 break; 483 case PL08X_BURST_SZ_8: 484 val |= PL080_BSIZE_8 << 485 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 486 break; 487 case PL08X_BURST_SZ_16: 488 val |= PL080_BSIZE_16 << 489 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 490 break; 491 case PL08X_BURST_SZ_32: 492 val |= PL080_BSIZE_32 << 493 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 494 break; 495 case PL08X_BURST_SZ_64: 496 val |= PL080_BSIZE_64 << 497 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 498 break; 499 case PL08X_BURST_SZ_128: 500 val |= PL080_BSIZE_128 << 501 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 502 break; 503 case PL08X_BURST_SZ_256: 504 val |= PL080_BSIZE_256 << 505 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 506 break; 507 } 508 509 /* Protection flags */ 510 if (pl08x->pd->memcpy_prot_buff) 511 val |= FTDMAC020_CH_CSR_PROT2; 512 if (pl08x->pd->memcpy_prot_cache) 513 val |= FTDMAC020_CH_CSR_PROT3; 514 /* We are the kernel, so we are in privileged mode */ 515 val |= FTDMAC020_CH_CSR_PROT1; 516 517 writel_relaxed(val, phychan->reg_control); 518 } else { 519 /* Bits are just identical */ 520 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); 521 } 522 523 /* Second control word on the PL080s */ 524 if (pl08x->vd->pl080s) 525 writel_relaxed(lli[PL080S_LLI_CCTL2], 526 phychan->base + PL080S_CH_CONTROL2); 527 528 writel(ccfg, phychan->reg_config); 529 } 530 531 /* 532 * Set the initial DMA register values i.e. those for the first LLI 533 * The next LLI pointer and the configuration interrupt bit have 534 * been set when the LLIs were constructed. Poke them into the hardware 535 * and start the transfer. 536 */ 537 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) 538 { 539 struct pl08x_driver_data *pl08x = plchan->host; 540 struct pl08x_phy_chan *phychan = plchan->phychan; 541 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); 542 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 543 u32 val; 544 545 list_del(&txd->vd.node); 546 547 plchan->at = txd; 548 549 /* Wait for channel inactive */ 550 while (pl08x_phy_channel_busy(phychan)) 551 cpu_relax(); 552 553 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); 554 555 /* Enable the DMA channel */ 556 /* Do not access config register until channel shows as disabled */ 557 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) 558 cpu_relax(); 559 560 /* Do not access config register until channel shows as inactive */ 561 if (phychan->ftdmac020) { 562 val = readl(phychan->reg_config); 563 while (val & FTDMAC020_CH_CFG_BUSY) 564 val = readl(phychan->reg_config); 565 566 val = readl(phychan->reg_control); 567 while (val & FTDMAC020_CH_CSR_EN) 568 val = readl(phychan->reg_control); 569 570 writel(val | FTDMAC020_CH_CSR_EN, 571 phychan->reg_control); 572 } else { 573 val = readl(phychan->reg_config); 574 while ((val & PL080_CONFIG_ACTIVE) || 575 (val & PL080_CONFIG_ENABLE)) 576 val = readl(phychan->reg_config); 577 578 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); 579 } 580 } 581 582 /* 583 * Pause the channel by setting the HALT bit. 584 * 585 * For M->P transfers, pause the DMAC first and then stop the peripheral - 586 * the FIFO can only drain if the peripheral is still requesting data. 587 * (note: this can still timeout if the DMAC FIFO never drains of data.) 588 * 589 * For P->M transfers, disable the peripheral first to stop it filling 590 * the DMAC FIFO, and then pause the DMAC. 591 */ 592 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) 593 { 594 u32 val; 595 int timeout; 596 597 if (ch->ftdmac020) { 598 /* Use the enable bit on the FTDMAC020 */ 599 val = readl(ch->reg_control); 600 val &= ~FTDMAC020_CH_CSR_EN; 601 writel(val, ch->reg_control); 602 return; 603 } 604 605 /* Set the HALT bit and wait for the FIFO to drain */ 606 val = readl(ch->reg_config); 607 val |= PL080_CONFIG_HALT; 608 writel(val, ch->reg_config); 609 610 /* Wait for channel inactive */ 611 for (timeout = 1000; timeout; timeout--) { 612 if (!pl08x_phy_channel_busy(ch)) 613 break; 614 udelay(1); 615 } 616 if (pl08x_phy_channel_busy(ch)) 617 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); 618 } 619 620 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 621 { 622 u32 val; 623 624 /* Use the enable bit on the FTDMAC020 */ 625 if (ch->ftdmac020) { 626 val = readl(ch->reg_control); 627 val |= FTDMAC020_CH_CSR_EN; 628 writel(val, ch->reg_control); 629 return; 630 } 631 632 /* Clear the HALT bit */ 633 val = readl(ch->reg_config); 634 val &= ~PL080_CONFIG_HALT; 635 writel(val, ch->reg_config); 636 } 637 638 /* 639 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and 640 * clears any pending interrupt status. This should not be used for 641 * an on-going transfer, but as a method of shutting down a channel 642 * (eg, when it's no longer used) or terminating a transfer. 643 */ 644 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, 645 struct pl08x_phy_chan *ch) 646 { 647 u32 val; 648 649 /* The layout for the FTDMAC020 is different */ 650 if (ch->ftdmac020) { 651 /* Disable all interrupts */ 652 val = readl(ch->reg_config); 653 val |= (FTDMAC020_CH_CFG_INT_ABT_MASK | 654 FTDMAC020_CH_CFG_INT_ERR_MASK | 655 FTDMAC020_CH_CFG_INT_TC_MASK); 656 writel(val, ch->reg_config); 657 658 /* Abort and disable channel */ 659 val = readl(ch->reg_control); 660 val &= ~FTDMAC020_CH_CSR_EN; 661 val |= FTDMAC020_CH_CSR_ABT; 662 writel(val, ch->reg_control); 663 664 /* Clear ABT and ERR interrupt flags */ 665 writel(BIT(ch->id) | BIT(ch->id + 16), 666 pl08x->base + PL080_ERR_CLEAR); 667 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 668 669 return; 670 } 671 672 val = readl(ch->reg_config); 673 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | 674 PL080_CONFIG_TC_IRQ_MASK); 675 writel(val, ch->reg_config); 676 677 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); 678 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 679 } 680 681 static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch) 682 { 683 u32 val; 684 u32 bytes; 685 686 if (ch->ftdmac020) { 687 bytes = readl(ch->base + FTDMAC020_CH_SIZE); 688 689 val = readl(ch->reg_control); 690 val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK; 691 val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT; 692 } else if (ch->pl080s) { 693 val = readl(ch->base + PL080S_CH_CONTROL2); 694 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 695 696 val = readl(ch->reg_control); 697 val &= PL080_CONTROL_SWIDTH_MASK; 698 val >>= PL080_CONTROL_SWIDTH_SHIFT; 699 } else { 700 /* Plain PL08x */ 701 val = readl(ch->reg_control); 702 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 703 704 val &= PL080_CONTROL_SWIDTH_MASK; 705 val >>= PL080_CONTROL_SWIDTH_SHIFT; 706 } 707 708 switch (val) { 709 case PL080_WIDTH_8BIT: 710 break; 711 case PL080_WIDTH_16BIT: 712 bytes *= 2; 713 break; 714 case PL080_WIDTH_32BIT: 715 bytes *= 4; 716 break; 717 } 718 return bytes; 719 } 720 721 static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va) 722 { 723 u32 val; 724 u32 bytes; 725 726 if (ch->ftdmac020) { 727 val = llis_va[PL080_LLI_CCTL]; 728 bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK; 729 730 val = llis_va[PL080_LLI_CCTL]; 731 val &= FTDMAC020_LLI_SRC_WIDTH_MSK; 732 val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT; 733 } else if (ch->pl080s) { 734 val = llis_va[PL080S_LLI_CCTL2]; 735 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 736 737 val = llis_va[PL080_LLI_CCTL]; 738 val &= PL080_CONTROL_SWIDTH_MASK; 739 val >>= PL080_CONTROL_SWIDTH_SHIFT; 740 } else { 741 /* Plain PL08x */ 742 val = llis_va[PL080_LLI_CCTL]; 743 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 744 745 val &= PL080_CONTROL_SWIDTH_MASK; 746 val >>= PL080_CONTROL_SWIDTH_SHIFT; 747 } 748 749 switch (val) { 750 case PL080_WIDTH_8BIT: 751 break; 752 case PL080_WIDTH_16BIT: 753 bytes *= 2; 754 break; 755 case PL080_WIDTH_32BIT: 756 bytes *= 4; 757 break; 758 } 759 return bytes; 760 } 761 762 /* The channel should be paused when calling this */ 763 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 764 { 765 struct pl08x_driver_data *pl08x = plchan->host; 766 const u32 *llis_va, *llis_va_limit; 767 struct pl08x_phy_chan *ch; 768 dma_addr_t llis_bus; 769 struct pl08x_txd *txd; 770 u32 llis_max_words; 771 size_t bytes; 772 u32 clli; 773 774 ch = plchan->phychan; 775 txd = plchan->at; 776 777 if (!ch || !txd) 778 return 0; 779 780 /* 781 * Follow the LLIs to get the number of remaining 782 * bytes in the currently active transaction. 783 */ 784 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; 785 786 /* First get the remaining bytes in the active transfer */ 787 bytes = get_bytes_in_phy_channel(ch); 788 789 if (!clli) 790 return bytes; 791 792 llis_va = txd->llis_va; 793 llis_bus = txd->llis_bus; 794 795 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; 796 BUG_ON(clli < llis_bus || clli >= llis_bus + 797 sizeof(u32) * llis_max_words); 798 799 /* 800 * Locate the next LLI - as this is an array, 801 * it's simple maths to find. 802 */ 803 llis_va += (clli - llis_bus) / sizeof(u32); 804 805 llis_va_limit = llis_va + llis_max_words; 806 807 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { 808 bytes += get_bytes_in_lli(ch, llis_va); 809 810 /* 811 * A LLI pointer going backward terminates the LLI list 812 */ 813 if (llis_va[PL080_LLI_LLI] <= clli) 814 break; 815 } 816 817 return bytes; 818 } 819 820 /* 821 * Allocate a physical channel for a virtual channel 822 * 823 * Try to locate a physical channel to be used for this transfer. If all 824 * are taken return NULL and the requester will have to cope by using 825 * some fallback PIO mode or retrying later. 826 */ 827 static struct pl08x_phy_chan * 828 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, 829 struct pl08x_dma_chan *virt_chan) 830 { 831 struct pl08x_phy_chan *ch = NULL; 832 unsigned long flags; 833 int i; 834 835 for (i = 0; i < pl08x->vd->channels; i++) { 836 ch = &pl08x->phy_chans[i]; 837 838 spin_lock_irqsave(&ch->lock, flags); 839 840 if (!ch->locked && !ch->serving) { 841 ch->serving = virt_chan; 842 spin_unlock_irqrestore(&ch->lock, flags); 843 break; 844 } 845 846 spin_unlock_irqrestore(&ch->lock, flags); 847 } 848 849 if (i == pl08x->vd->channels) { 850 /* No physical channel available, cope with it */ 851 return NULL; 852 } 853 854 return ch; 855 } 856 857 /* Mark the physical channel as free. Note, this write is atomic. */ 858 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, 859 struct pl08x_phy_chan *ch) 860 { 861 ch->serving = NULL; 862 } 863 864 /* 865 * Try to allocate a physical channel. When successful, assign it to 866 * this virtual channel, and initiate the next descriptor. The 867 * virtual channel lock must be held at this point. 868 */ 869 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) 870 { 871 struct pl08x_driver_data *pl08x = plchan->host; 872 struct pl08x_phy_chan *ch; 873 874 ch = pl08x_get_phy_channel(pl08x, plchan); 875 if (!ch) { 876 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); 877 plchan->state = PL08X_CHAN_WAITING; 878 return; 879 } 880 881 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", 882 ch->id, plchan->name); 883 884 plchan->phychan = ch; 885 plchan->state = PL08X_CHAN_RUNNING; 886 pl08x_start_next_txd(plchan); 887 } 888 889 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, 890 struct pl08x_dma_chan *plchan) 891 { 892 struct pl08x_driver_data *pl08x = plchan->host; 893 894 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", 895 ch->id, plchan->name); 896 897 /* 898 * We do this without taking the lock; we're really only concerned 899 * about whether this pointer is NULL or not, and we're guaranteed 900 * that this will only be called when it _already_ is non-NULL. 901 */ 902 ch->serving = plchan; 903 plchan->phychan = ch; 904 plchan->state = PL08X_CHAN_RUNNING; 905 pl08x_start_next_txd(plchan); 906 } 907 908 /* 909 * Free a physical DMA channel, potentially reallocating it to another 910 * virtual channel if we have any pending. 911 */ 912 static void pl08x_phy_free(struct pl08x_dma_chan *plchan) 913 { 914 struct pl08x_driver_data *pl08x = plchan->host; 915 struct pl08x_dma_chan *p, *next; 916 917 retry: 918 next = NULL; 919 920 /* Find a waiting virtual channel for the next transfer. */ 921 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) 922 if (p->state == PL08X_CHAN_WAITING) { 923 next = p; 924 break; 925 } 926 927 if (!next && pl08x->has_slave) { 928 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) 929 if (p->state == PL08X_CHAN_WAITING) { 930 next = p; 931 break; 932 } 933 } 934 935 /* Ensure that the physical channel is stopped */ 936 pl08x_terminate_phy_chan(pl08x, plchan->phychan); 937 938 if (next) { 939 bool success; 940 941 /* 942 * Eww. We know this isn't going to deadlock 943 * but lockdep probably doesn't. 944 */ 945 spin_lock(&next->vc.lock); 946 /* Re-check the state now that we have the lock */ 947 success = next->state == PL08X_CHAN_WAITING; 948 if (success) 949 pl08x_phy_reassign_start(plchan->phychan, next); 950 spin_unlock(&next->vc.lock); 951 952 /* If the state changed, try to find another channel */ 953 if (!success) 954 goto retry; 955 } else { 956 /* No more jobs, so free up the physical channel */ 957 pl08x_put_phy_channel(pl08x, plchan->phychan); 958 } 959 960 plchan->phychan = NULL; 961 plchan->state = PL08X_CHAN_IDLE; 962 } 963 964 /* 965 * LLI handling 966 */ 967 968 static inline unsigned int 969 pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x, 970 u32 cctl, 971 bool source) 972 { 973 u32 val; 974 975 if (pl08x->vd->ftdmac020) { 976 if (source) 977 val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 978 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 979 else 980 val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 981 FTDMAC020_LLI_DST_WIDTH_SHIFT; 982 } else { 983 if (source) 984 val = (cctl & PL080_CONTROL_SWIDTH_MASK) >> 985 PL080_CONTROL_SWIDTH_SHIFT; 986 else 987 val = (cctl & PL080_CONTROL_DWIDTH_MASK) >> 988 PL080_CONTROL_DWIDTH_SHIFT; 989 } 990 991 switch (val) { 992 case PL080_WIDTH_8BIT: 993 return 1; 994 case PL080_WIDTH_16BIT: 995 return 2; 996 case PL080_WIDTH_32BIT: 997 return 4; 998 default: 999 break; 1000 } 1001 BUG(); 1002 return 0; 1003 } 1004 1005 static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x, 1006 u32 cctl, 1007 u8 srcwidth, u8 dstwidth, 1008 size_t tsize) 1009 { 1010 u32 retbits = cctl; 1011 1012 /* 1013 * Remove all src, dst and transfer size bits, then set the 1014 * width and size according to the parameters. The bit offsets 1015 * are different in the FTDMAC020 so we need to accound for this. 1016 */ 1017 if (pl08x->vd->ftdmac020) { 1018 retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK; 1019 retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK; 1020 retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1021 1022 switch (srcwidth) { 1023 case 1: 1024 retbits |= PL080_WIDTH_8BIT << 1025 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1026 break; 1027 case 2: 1028 retbits |= PL080_WIDTH_16BIT << 1029 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1030 break; 1031 case 4: 1032 retbits |= PL080_WIDTH_32BIT << 1033 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1034 break; 1035 default: 1036 BUG(); 1037 break; 1038 } 1039 1040 switch (dstwidth) { 1041 case 1: 1042 retbits |= PL080_WIDTH_8BIT << 1043 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1044 break; 1045 case 2: 1046 retbits |= PL080_WIDTH_16BIT << 1047 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1048 break; 1049 case 4: 1050 retbits |= PL080_WIDTH_32BIT << 1051 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1052 break; 1053 default: 1054 BUG(); 1055 break; 1056 } 1057 1058 tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1059 retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT; 1060 } else { 1061 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 1062 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 1063 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 1064 1065 switch (srcwidth) { 1066 case 1: 1067 retbits |= PL080_WIDTH_8BIT << 1068 PL080_CONTROL_SWIDTH_SHIFT; 1069 break; 1070 case 2: 1071 retbits |= PL080_WIDTH_16BIT << 1072 PL080_CONTROL_SWIDTH_SHIFT; 1073 break; 1074 case 4: 1075 retbits |= PL080_WIDTH_32BIT << 1076 PL080_CONTROL_SWIDTH_SHIFT; 1077 break; 1078 default: 1079 BUG(); 1080 break; 1081 } 1082 1083 switch (dstwidth) { 1084 case 1: 1085 retbits |= PL080_WIDTH_8BIT << 1086 PL080_CONTROL_DWIDTH_SHIFT; 1087 break; 1088 case 2: 1089 retbits |= PL080_WIDTH_16BIT << 1090 PL080_CONTROL_DWIDTH_SHIFT; 1091 break; 1092 case 4: 1093 retbits |= PL080_WIDTH_32BIT << 1094 PL080_CONTROL_DWIDTH_SHIFT; 1095 break; 1096 default: 1097 BUG(); 1098 break; 1099 } 1100 1101 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; 1102 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; 1103 } 1104 1105 return retbits; 1106 } 1107 1108 struct pl08x_lli_build_data { 1109 struct pl08x_txd *txd; 1110 struct pl08x_bus_data srcbus; 1111 struct pl08x_bus_data dstbus; 1112 size_t remainder; 1113 u32 lli_bus; 1114 }; 1115 1116 /* 1117 * Autoselect a master bus to use for the transfer. Slave will be the chosen as 1118 * victim in case src & dest are not similarly aligned. i.e. If after aligning 1119 * masters address with width requirements of transfer (by sending few byte by 1120 * byte data), slave is still not aligned, then its width will be reduced to 1121 * BYTE. 1122 * - prefers the destination bus if both available 1123 * - prefers bus with fixed address (i.e. peripheral) 1124 */ 1125 static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x, 1126 struct pl08x_lli_build_data *bd, 1127 struct pl08x_bus_data **mbus, 1128 struct pl08x_bus_data **sbus, 1129 u32 cctl) 1130 { 1131 bool dst_incr; 1132 bool src_incr; 1133 1134 /* 1135 * The FTDMAC020 only supports memory-to-memory transfer, so 1136 * source and destination always increase. 1137 */ 1138 if (pl08x->vd->ftdmac020) { 1139 dst_incr = true; 1140 src_incr = true; 1141 } else { 1142 dst_incr = !!(cctl & PL080_CONTROL_DST_INCR); 1143 src_incr = !!(cctl & PL080_CONTROL_SRC_INCR); 1144 } 1145 1146 /* 1147 * If either bus is not advancing, i.e. it is a peripheral, that 1148 * one becomes master 1149 */ 1150 if (!dst_incr) { 1151 *mbus = &bd->dstbus; 1152 *sbus = &bd->srcbus; 1153 } else if (!src_incr) { 1154 *mbus = &bd->srcbus; 1155 *sbus = &bd->dstbus; 1156 } else { 1157 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { 1158 *mbus = &bd->dstbus; 1159 *sbus = &bd->srcbus; 1160 } else { 1161 *mbus = &bd->srcbus; 1162 *sbus = &bd->dstbus; 1163 } 1164 } 1165 } 1166 1167 /* 1168 * Fills in one LLI for a certain transfer descriptor and advance the counter 1169 */ 1170 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 1171 struct pl08x_lli_build_data *bd, 1172 int num_llis, int len, u32 cctl, u32 cctl2) 1173 { 1174 u32 offset = num_llis * pl08x->lli_words; 1175 u32 *llis_va = bd->txd->llis_va + offset; 1176 dma_addr_t llis_bus = bd->txd->llis_bus; 1177 1178 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 1179 1180 /* Advance the offset to next LLI. */ 1181 offset += pl08x->lli_words; 1182 1183 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; 1184 llis_va[PL080_LLI_DST] = bd->dstbus.addr; 1185 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); 1186 llis_va[PL080_LLI_LLI] |= bd->lli_bus; 1187 llis_va[PL080_LLI_CCTL] = cctl; 1188 if (pl08x->vd->pl080s) 1189 llis_va[PL080S_LLI_CCTL2] = cctl2; 1190 1191 if (pl08x->vd->ftdmac020) { 1192 /* FIXME: only memcpy so far so both increase */ 1193 bd->srcbus.addr += len; 1194 bd->dstbus.addr += len; 1195 } else { 1196 if (cctl & PL080_CONTROL_SRC_INCR) 1197 bd->srcbus.addr += len; 1198 if (cctl & PL080_CONTROL_DST_INCR) 1199 bd->dstbus.addr += len; 1200 } 1201 1202 BUG_ON(bd->remainder < len); 1203 1204 bd->remainder -= len; 1205 } 1206 1207 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, 1208 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, 1209 int num_llis, size_t *total_bytes) 1210 { 1211 *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len); 1212 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); 1213 (*total_bytes) += len; 1214 } 1215 1216 #if 1 1217 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1218 const u32 *llis_va, int num_llis) 1219 { 1220 int i; 1221 1222 if (pl08x->vd->pl080s) { 1223 dev_vdbg(&pl08x->adev->dev, 1224 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", 1225 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); 1226 for (i = 0; i < num_llis; i++) { 1227 dev_vdbg(&pl08x->adev->dev, 1228 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 1229 i, llis_va, llis_va[PL080_LLI_SRC], 1230 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1231 llis_va[PL080_LLI_CCTL], 1232 llis_va[PL080S_LLI_CCTL2]); 1233 llis_va += pl08x->lli_words; 1234 } 1235 } else { 1236 dev_vdbg(&pl08x->adev->dev, 1237 "%-3s %-9s %-10s %-10s %-10s %s\n", 1238 "lli", "", "csrc", "cdst", "clli", "cctl"); 1239 for (i = 0; i < num_llis; i++) { 1240 dev_vdbg(&pl08x->adev->dev, 1241 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1242 i, llis_va, llis_va[PL080_LLI_SRC], 1243 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1244 llis_va[PL080_LLI_CCTL]); 1245 llis_va += pl08x->lli_words; 1246 } 1247 } 1248 } 1249 #else 1250 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1251 const u32 *llis_va, int num_llis) {} 1252 #endif 1253 1254 /* 1255 * This fills in the table of LLIs for the transfer descriptor 1256 * Note that we assume we never have to change the burst sizes 1257 * Return 0 for error 1258 */ 1259 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 1260 struct pl08x_txd *txd) 1261 { 1262 struct pl08x_bus_data *mbus, *sbus; 1263 struct pl08x_lli_build_data bd; 1264 int num_llis = 0; 1265 u32 cctl, early_bytes = 0; 1266 size_t max_bytes_per_lli, total_bytes; 1267 u32 *llis_va, *last_lli; 1268 struct pl08x_sg *dsg; 1269 1270 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); 1271 if (!txd->llis_va) { 1272 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); 1273 return 0; 1274 } 1275 1276 bd.txd = txd; 1277 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; 1278 cctl = txd->cctl; 1279 1280 /* Find maximum width of the source bus */ 1281 bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true); 1282 1283 /* Find maximum width of the destination bus */ 1284 bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false); 1285 1286 list_for_each_entry(dsg, &txd->dsg_list, node) { 1287 total_bytes = 0; 1288 cctl = txd->cctl; 1289 1290 bd.srcbus.addr = dsg->src_addr; 1291 bd.dstbus.addr = dsg->dst_addr; 1292 bd.remainder = dsg->len; 1293 bd.srcbus.buswidth = bd.srcbus.maxwidth; 1294 bd.dstbus.buswidth = bd.dstbus.maxwidth; 1295 1296 pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl); 1297 1298 dev_vdbg(&pl08x->adev->dev, 1299 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", 1300 (u64)bd.srcbus.addr, 1301 cctl & PL080_CONTROL_SRC_INCR ? "+" : "", 1302 bd.srcbus.buswidth, 1303 (u64)bd.dstbus.addr, 1304 cctl & PL080_CONTROL_DST_INCR ? "+" : "", 1305 bd.dstbus.buswidth, 1306 bd.remainder); 1307 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", 1308 mbus == &bd.srcbus ? "src" : "dst", 1309 sbus == &bd.srcbus ? "src" : "dst"); 1310 1311 /* 1312 * Zero length is only allowed if all these requirements are 1313 * met: 1314 * - flow controller is peripheral. 1315 * - src.addr is aligned to src.width 1316 * - dst.addr is aligned to dst.width 1317 * 1318 * sg_len == 1 should be true, as there can be two cases here: 1319 * 1320 * - Memory addresses are contiguous and are not scattered. 1321 * Here, Only one sg will be passed by user driver, with 1322 * memory address and zero length. We pass this to controller 1323 * and after the transfer it will receive the last burst 1324 * request from peripheral and so transfer finishes. 1325 * 1326 * - Memory addresses are scattered and are not contiguous. 1327 * Here, Obviously as DMA controller doesn't know when a lli's 1328 * transfer gets over, it can't load next lli. So in this 1329 * case, there has to be an assumption that only one lli is 1330 * supported. Thus, we can't have scattered addresses. 1331 */ 1332 if (!bd.remainder) { 1333 u32 fc; 1334 1335 /* FTDMAC020 only does memory-to-memory */ 1336 if (pl08x->vd->ftdmac020) 1337 fc = PL080_FLOW_MEM2MEM; 1338 else 1339 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> 1340 PL080_CONFIG_FLOW_CONTROL_SHIFT; 1341 if (!((fc >= PL080_FLOW_SRC2DST_DST) && 1342 (fc <= PL080_FLOW_SRC2DST_SRC))) { 1343 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", 1344 __func__); 1345 return 0; 1346 } 1347 1348 if (!IS_BUS_ALIGNED(&bd.srcbus) || 1349 !IS_BUS_ALIGNED(&bd.dstbus)) { 1350 dev_err(&pl08x->adev->dev, 1351 "%s src & dst address must be aligned to src" 1352 " & dst width if peripheral is flow controller", 1353 __func__); 1354 return 0; 1355 } 1356 1357 cctl = pl08x_lli_control_bits(pl08x, cctl, 1358 bd.srcbus.buswidth, bd.dstbus.buswidth, 1359 0); 1360 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1361 0, cctl, 0); 1362 break; 1363 } 1364 1365 /* 1366 * Send byte by byte for following cases 1367 * - Less than a bus width available 1368 * - until master bus is aligned 1369 */ 1370 if (bd.remainder < mbus->buswidth) 1371 early_bytes = bd.remainder; 1372 else if (!IS_BUS_ALIGNED(mbus)) { 1373 early_bytes = mbus->buswidth - 1374 (mbus->addr & (mbus->buswidth - 1)); 1375 if ((bd.remainder - early_bytes) < mbus->buswidth) 1376 early_bytes = bd.remainder; 1377 } 1378 1379 if (early_bytes) { 1380 dev_vdbg(&pl08x->adev->dev, 1381 "%s byte width LLIs (remain 0x%08zx)\n", 1382 __func__, bd.remainder); 1383 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, 1384 num_llis++, &total_bytes); 1385 } 1386 1387 if (bd.remainder) { 1388 /* 1389 * Master now aligned 1390 * - if slave is not then we must set its width down 1391 */ 1392 if (!IS_BUS_ALIGNED(sbus)) { 1393 dev_dbg(&pl08x->adev->dev, 1394 "%s set down bus width to one byte\n", 1395 __func__); 1396 1397 sbus->buswidth = 1; 1398 } 1399 1400 /* 1401 * Bytes transferred = tsize * src width, not 1402 * MIN(buswidths) 1403 */ 1404 max_bytes_per_lli = bd.srcbus.buswidth * 1405 pl08x->vd->max_transfer_size; 1406 dev_vdbg(&pl08x->adev->dev, 1407 "%s max bytes per lli = %zu\n", 1408 __func__, max_bytes_per_lli); 1409 1410 /* 1411 * Make largest possible LLIs until less than one bus 1412 * width left 1413 */ 1414 while (bd.remainder > (mbus->buswidth - 1)) { 1415 size_t lli_len, tsize, width; 1416 1417 /* 1418 * If enough left try to send max possible, 1419 * otherwise try to send the remainder 1420 */ 1421 lli_len = min(bd.remainder, max_bytes_per_lli); 1422 1423 /* 1424 * Check against maximum bus alignment: 1425 * Calculate actual transfer size in relation to 1426 * bus width an get a maximum remainder of the 1427 * highest bus width - 1 1428 */ 1429 width = max(mbus->buswidth, sbus->buswidth); 1430 lli_len = (lli_len / width) * width; 1431 tsize = lli_len / bd.srcbus.buswidth; 1432 1433 dev_vdbg(&pl08x->adev->dev, 1434 "%s fill lli with single lli chunk of " 1435 "size 0x%08zx (remainder 0x%08zx)\n", 1436 __func__, lli_len, bd.remainder); 1437 1438 cctl = pl08x_lli_control_bits(pl08x, cctl, 1439 bd.srcbus.buswidth, bd.dstbus.buswidth, 1440 tsize); 1441 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1442 lli_len, cctl, tsize); 1443 total_bytes += lli_len; 1444 } 1445 1446 /* 1447 * Send any odd bytes 1448 */ 1449 if (bd.remainder) { 1450 dev_vdbg(&pl08x->adev->dev, 1451 "%s align with boundary, send odd bytes (remain %zu)\n", 1452 __func__, bd.remainder); 1453 prep_byte_width_lli(pl08x, &bd, &cctl, 1454 bd.remainder, num_llis++, &total_bytes); 1455 } 1456 } 1457 1458 if (total_bytes != dsg->len) { 1459 dev_err(&pl08x->adev->dev, 1460 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", 1461 __func__, total_bytes, dsg->len); 1462 return 0; 1463 } 1464 1465 if (num_llis >= MAX_NUM_TSFR_LLIS) { 1466 dev_err(&pl08x->adev->dev, 1467 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", 1468 __func__, MAX_NUM_TSFR_LLIS); 1469 return 0; 1470 } 1471 } 1472 1473 llis_va = txd->llis_va; 1474 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; 1475 1476 if (txd->cyclic) { 1477 /* Link back to the first LLI. */ 1478 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; 1479 } else { 1480 /* The final LLI terminates the LLI. */ 1481 last_lli[PL080_LLI_LLI] = 0; 1482 /* The final LLI element shall also fire an interrupt. */ 1483 if (pl08x->vd->ftdmac020) 1484 last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK; 1485 else 1486 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; 1487 } 1488 1489 pl08x_dump_lli(pl08x, llis_va, num_llis); 1490 1491 return num_llis; 1492 } 1493 1494 static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 1495 struct pl08x_txd *txd) 1496 { 1497 struct pl08x_sg *dsg, *_dsg; 1498 1499 if (txd->llis_va) 1500 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); 1501 1502 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { 1503 list_del(&dsg->node); 1504 kfree(dsg); 1505 } 1506 1507 kfree(txd); 1508 } 1509 1510 static void pl08x_desc_free(struct virt_dma_desc *vd) 1511 { 1512 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1513 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1514 1515 dma_descriptor_unmap(&vd->tx); 1516 if (!txd->done) 1517 pl08x_release_mux(plchan); 1518 1519 pl08x_free_txd(plchan->host, txd); 1520 } 1521 1522 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, 1523 struct pl08x_dma_chan *plchan) 1524 { 1525 LIST_HEAD(head); 1526 1527 vchan_get_all_descriptors(&plchan->vc, &head); 1528 vchan_dma_desc_free_list(&plchan->vc, &head); 1529 } 1530 1531 /* 1532 * The DMA ENGINE API 1533 */ 1534 static void pl08x_free_chan_resources(struct dma_chan *chan) 1535 { 1536 /* Ensure all queued descriptors are freed */ 1537 vchan_free_chan_resources(to_virt_chan(chan)); 1538 } 1539 1540 static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( 1541 struct dma_chan *chan, unsigned long flags) 1542 { 1543 struct dma_async_tx_descriptor *retval = NULL; 1544 1545 return retval; 1546 } 1547 1548 /* 1549 * Code accessing dma_async_is_complete() in a tight loop may give problems. 1550 * If slaves are relying on interrupts to signal completion this function 1551 * must not be called with interrupts disabled. 1552 */ 1553 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, 1554 dma_cookie_t cookie, struct dma_tx_state *txstate) 1555 { 1556 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1557 struct virt_dma_desc *vd; 1558 unsigned long flags; 1559 enum dma_status ret; 1560 size_t bytes = 0; 1561 1562 ret = dma_cookie_status(chan, cookie, txstate); 1563 if (ret == DMA_COMPLETE) 1564 return ret; 1565 1566 /* 1567 * There's no point calculating the residue if there's 1568 * no txstate to store the value. 1569 */ 1570 if (!txstate) { 1571 if (plchan->state == PL08X_CHAN_PAUSED) 1572 ret = DMA_PAUSED; 1573 return ret; 1574 } 1575 1576 spin_lock_irqsave(&plchan->vc.lock, flags); 1577 ret = dma_cookie_status(chan, cookie, txstate); 1578 if (ret != DMA_COMPLETE) { 1579 vd = vchan_find_desc(&plchan->vc, cookie); 1580 if (vd) { 1581 /* On the issued list, so hasn't been processed yet */ 1582 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1583 struct pl08x_sg *dsg; 1584 1585 list_for_each_entry(dsg, &txd->dsg_list, node) 1586 bytes += dsg->len; 1587 } else { 1588 bytes = pl08x_getbytes_chan(plchan); 1589 } 1590 } 1591 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1592 1593 /* 1594 * This cookie not complete yet 1595 * Get number of bytes left in the active transactions and queue 1596 */ 1597 dma_set_residue(txstate, bytes); 1598 1599 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) 1600 ret = DMA_PAUSED; 1601 1602 /* Whether waiting or running, we're in progress */ 1603 return ret; 1604 } 1605 1606 /* PrimeCell DMA extension */ 1607 struct burst_table { 1608 u32 burstwords; 1609 u32 reg; 1610 }; 1611 1612 static const struct burst_table burst_sizes[] = { 1613 { 1614 .burstwords = 256, 1615 .reg = PL080_BSIZE_256, 1616 }, 1617 { 1618 .burstwords = 128, 1619 .reg = PL080_BSIZE_128, 1620 }, 1621 { 1622 .burstwords = 64, 1623 .reg = PL080_BSIZE_64, 1624 }, 1625 { 1626 .burstwords = 32, 1627 .reg = PL080_BSIZE_32, 1628 }, 1629 { 1630 .burstwords = 16, 1631 .reg = PL080_BSIZE_16, 1632 }, 1633 { 1634 .burstwords = 8, 1635 .reg = PL080_BSIZE_8, 1636 }, 1637 { 1638 .burstwords = 4, 1639 .reg = PL080_BSIZE_4, 1640 }, 1641 { 1642 .burstwords = 0, 1643 .reg = PL080_BSIZE_1, 1644 }, 1645 }; 1646 1647 /* 1648 * Given the source and destination available bus masks, select which 1649 * will be routed to each port. We try to have source and destination 1650 * on separate ports, but always respect the allowable settings. 1651 */ 1652 static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst) 1653 { 1654 u32 cctl = 0; 1655 u32 dst_ahb2; 1656 u32 src_ahb2; 1657 1658 /* The FTDMAC020 use different bits to indicate src/dst bus */ 1659 if (ftdmac020) { 1660 dst_ahb2 = FTDMAC020_LLI_DST_SEL; 1661 src_ahb2 = FTDMAC020_LLI_SRC_SEL; 1662 } else { 1663 dst_ahb2 = PL080_CONTROL_DST_AHB2; 1664 src_ahb2 = PL080_CONTROL_SRC_AHB2; 1665 } 1666 1667 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) 1668 cctl |= dst_ahb2; 1669 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) 1670 cctl |= src_ahb2; 1671 1672 return cctl; 1673 } 1674 1675 static u32 pl08x_cctl(u32 cctl) 1676 { 1677 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | 1678 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | 1679 PL080_CONTROL_PROT_MASK); 1680 1681 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1682 return cctl | PL080_CONTROL_PROT_SYS; 1683 } 1684 1685 static u32 pl08x_width(enum dma_slave_buswidth width) 1686 { 1687 switch (width) { 1688 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1689 return PL080_WIDTH_8BIT; 1690 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1691 return PL080_WIDTH_16BIT; 1692 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1693 return PL080_WIDTH_32BIT; 1694 default: 1695 return ~0; 1696 } 1697 } 1698 1699 static u32 pl08x_burst(u32 maxburst) 1700 { 1701 int i; 1702 1703 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) 1704 if (burst_sizes[i].burstwords <= maxburst) 1705 break; 1706 1707 return burst_sizes[i].reg; 1708 } 1709 1710 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, 1711 enum dma_slave_buswidth addr_width, u32 maxburst) 1712 { 1713 u32 width, burst, cctl = 0; 1714 1715 width = pl08x_width(addr_width); 1716 if (width == ~0) 1717 return ~0; 1718 1719 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; 1720 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; 1721 1722 /* 1723 * If this channel will only request single transfers, set this 1724 * down to ONE element. Also select one element if no maxburst 1725 * is specified. 1726 */ 1727 if (plchan->cd->single) 1728 maxburst = 1; 1729 1730 burst = pl08x_burst(maxburst); 1731 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; 1732 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; 1733 1734 return pl08x_cctl(cctl); 1735 } 1736 1737 /* 1738 * Slave transactions callback to the slave device to allow 1739 * synchronization of slave DMA signals with the DMAC enable 1740 */ 1741 static void pl08x_issue_pending(struct dma_chan *chan) 1742 { 1743 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1744 unsigned long flags; 1745 1746 spin_lock_irqsave(&plchan->vc.lock, flags); 1747 if (vchan_issue_pending(&plchan->vc)) { 1748 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) 1749 pl08x_phy_alloc_and_start(plchan); 1750 } 1751 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1752 } 1753 1754 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) 1755 { 1756 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 1757 1758 if (txd) 1759 INIT_LIST_HEAD(&txd->dsg_list); 1760 return txd; 1761 } 1762 1763 static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x) 1764 { 1765 u32 cctl = 0; 1766 1767 /* Conjure cctl */ 1768 switch (pl08x->pd->memcpy_burst_size) { 1769 default: 1770 dev_err(&pl08x->adev->dev, 1771 "illegal burst size for memcpy, set to 1\n"); 1772 /* Fall through */ 1773 case PL08X_BURST_SZ_1: 1774 cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | 1775 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; 1776 break; 1777 case PL08X_BURST_SZ_4: 1778 cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | 1779 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; 1780 break; 1781 case PL08X_BURST_SZ_8: 1782 cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | 1783 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; 1784 break; 1785 case PL08X_BURST_SZ_16: 1786 cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | 1787 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; 1788 break; 1789 case PL08X_BURST_SZ_32: 1790 cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | 1791 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; 1792 break; 1793 case PL08X_BURST_SZ_64: 1794 cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | 1795 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; 1796 break; 1797 case PL08X_BURST_SZ_128: 1798 cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | 1799 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; 1800 break; 1801 case PL08X_BURST_SZ_256: 1802 cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | 1803 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; 1804 break; 1805 } 1806 1807 switch (pl08x->pd->memcpy_bus_width) { 1808 default: 1809 dev_err(&pl08x->adev->dev, 1810 "illegal bus width for memcpy, set to 8 bits\n"); 1811 /* Fall through */ 1812 case PL08X_BUS_WIDTH_8_BITS: 1813 cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | 1814 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 1815 break; 1816 case PL08X_BUS_WIDTH_16_BITS: 1817 cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | 1818 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 1819 break; 1820 case PL08X_BUS_WIDTH_32_BITS: 1821 cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | 1822 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 1823 break; 1824 } 1825 1826 /* Protection flags */ 1827 if (pl08x->pd->memcpy_prot_buff) 1828 cctl |= PL080_CONTROL_PROT_BUFF; 1829 if (pl08x->pd->memcpy_prot_cache) 1830 cctl |= PL080_CONTROL_PROT_CACHE; 1831 1832 /* We are the kernel, so we are in privileged mode */ 1833 cctl |= PL080_CONTROL_PROT_SYS; 1834 1835 /* Both to be incremented or the code will break */ 1836 cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1837 1838 if (pl08x->vd->dualmaster) 1839 cctl |= pl08x_select_bus(false, 1840 pl08x->mem_buses, 1841 pl08x->mem_buses); 1842 1843 return cctl; 1844 } 1845 1846 static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x) 1847 { 1848 u32 cctl = 0; 1849 1850 /* Conjure cctl */ 1851 switch (pl08x->pd->memcpy_bus_width) { 1852 default: 1853 dev_err(&pl08x->adev->dev, 1854 "illegal bus width for memcpy, set to 8 bits\n"); 1855 /* Fall through */ 1856 case PL08X_BUS_WIDTH_8_BITS: 1857 cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1858 PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1859 break; 1860 case PL08X_BUS_WIDTH_16_BITS: 1861 cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1862 PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1863 break; 1864 case PL08X_BUS_WIDTH_32_BITS: 1865 cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1866 PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1867 break; 1868 } 1869 1870 /* 1871 * By default mask the TC IRQ on all LLIs, it will be unmasked on 1872 * the last LLI item by other code. 1873 */ 1874 cctl |= FTDMAC020_LLI_TC_MSK; 1875 1876 /* 1877 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL 1878 * and FTDMAC020_LLI_DSTAD_CTL as zero 1879 */ 1880 if (pl08x->vd->dualmaster) 1881 cctl |= pl08x_select_bus(true, 1882 pl08x->mem_buses, 1883 pl08x->mem_buses); 1884 1885 return cctl; 1886 } 1887 1888 /* 1889 * Initialize a descriptor to be used by memcpy submit 1890 */ 1891 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1892 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1893 size_t len, unsigned long flags) 1894 { 1895 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1896 struct pl08x_driver_data *pl08x = plchan->host; 1897 struct pl08x_txd *txd; 1898 struct pl08x_sg *dsg; 1899 int ret; 1900 1901 txd = pl08x_get_txd(plchan); 1902 if (!txd) { 1903 dev_err(&pl08x->adev->dev, 1904 "%s no memory for descriptor\n", __func__); 1905 return NULL; 1906 } 1907 1908 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1909 if (!dsg) { 1910 pl08x_free_txd(pl08x, txd); 1911 return NULL; 1912 } 1913 list_add_tail(&dsg->node, &txd->dsg_list); 1914 1915 dsg->src_addr = src; 1916 dsg->dst_addr = dest; 1917 dsg->len = len; 1918 if (pl08x->vd->ftdmac020) { 1919 /* Writing CCFG zero ENABLES all interrupts */ 1920 txd->ccfg = 0; 1921 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); 1922 } else { 1923 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1924 PL080_CONFIG_TC_IRQ_MASK | 1925 PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1926 txd->cctl = pl08x_memcpy_cctl(pl08x); 1927 } 1928 1929 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1930 if (!ret) { 1931 pl08x_free_txd(pl08x, txd); 1932 return NULL; 1933 } 1934 1935 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1936 } 1937 1938 static struct pl08x_txd *pl08x_init_txd( 1939 struct dma_chan *chan, 1940 enum dma_transfer_direction direction, 1941 dma_addr_t *slave_addr) 1942 { 1943 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1944 struct pl08x_driver_data *pl08x = plchan->host; 1945 struct pl08x_txd *txd; 1946 enum dma_slave_buswidth addr_width; 1947 int ret, tmp; 1948 u8 src_buses, dst_buses; 1949 u32 maxburst, cctl; 1950 1951 txd = pl08x_get_txd(plchan); 1952 if (!txd) { 1953 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1954 return NULL; 1955 } 1956 1957 /* 1958 * Set up addresses, the PrimeCell configured address 1959 * will take precedence since this may configure the 1960 * channel target address dynamically at runtime. 1961 */ 1962 if (direction == DMA_MEM_TO_DEV) { 1963 cctl = PL080_CONTROL_SRC_INCR; 1964 *slave_addr = plchan->cfg.dst_addr; 1965 addr_width = plchan->cfg.dst_addr_width; 1966 maxburst = plchan->cfg.dst_maxburst; 1967 src_buses = pl08x->mem_buses; 1968 dst_buses = plchan->cd->periph_buses; 1969 } else if (direction == DMA_DEV_TO_MEM) { 1970 cctl = PL080_CONTROL_DST_INCR; 1971 *slave_addr = plchan->cfg.src_addr; 1972 addr_width = plchan->cfg.src_addr_width; 1973 maxburst = plchan->cfg.src_maxburst; 1974 src_buses = plchan->cd->periph_buses; 1975 dst_buses = pl08x->mem_buses; 1976 } else { 1977 pl08x_free_txd(pl08x, txd); 1978 dev_err(&pl08x->adev->dev, 1979 "%s direction unsupported\n", __func__); 1980 return NULL; 1981 } 1982 1983 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); 1984 if (cctl == ~0) { 1985 pl08x_free_txd(pl08x, txd); 1986 dev_err(&pl08x->adev->dev, 1987 "DMA slave configuration botched?\n"); 1988 return NULL; 1989 } 1990 1991 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); 1992 1993 if (plchan->cfg.device_fc) 1994 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : 1995 PL080_FLOW_PER2MEM_PER; 1996 else 1997 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : 1998 PL080_FLOW_PER2MEM; 1999 2000 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 2001 PL080_CONFIG_TC_IRQ_MASK | 2002 tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; 2003 2004 ret = pl08x_request_mux(plchan); 2005 if (ret < 0) { 2006 pl08x_free_txd(pl08x, txd); 2007 dev_dbg(&pl08x->adev->dev, 2008 "unable to mux for transfer on %s due to platform restrictions\n", 2009 plchan->name); 2010 return NULL; 2011 } 2012 2013 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", 2014 plchan->signal, plchan->name); 2015 2016 /* Assign the flow control signal to this channel */ 2017 if (direction == DMA_MEM_TO_DEV) 2018 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; 2019 else 2020 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; 2021 2022 return txd; 2023 } 2024 2025 static int pl08x_tx_add_sg(struct pl08x_txd *txd, 2026 enum dma_transfer_direction direction, 2027 dma_addr_t slave_addr, 2028 dma_addr_t buf_addr, 2029 unsigned int len) 2030 { 2031 struct pl08x_sg *dsg; 2032 2033 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 2034 if (!dsg) 2035 return -ENOMEM; 2036 2037 list_add_tail(&dsg->node, &txd->dsg_list); 2038 2039 dsg->len = len; 2040 if (direction == DMA_MEM_TO_DEV) { 2041 dsg->src_addr = buf_addr; 2042 dsg->dst_addr = slave_addr; 2043 } else { 2044 dsg->src_addr = slave_addr; 2045 dsg->dst_addr = buf_addr; 2046 } 2047 2048 return 0; 2049 } 2050 2051 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 2052 struct dma_chan *chan, struct scatterlist *sgl, 2053 unsigned int sg_len, enum dma_transfer_direction direction, 2054 unsigned long flags, void *context) 2055 { 2056 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2057 struct pl08x_driver_data *pl08x = plchan->host; 2058 struct pl08x_txd *txd; 2059 struct scatterlist *sg; 2060 int ret, tmp; 2061 dma_addr_t slave_addr; 2062 2063 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 2064 __func__, sg_dma_len(sgl), plchan->name); 2065 2066 txd = pl08x_init_txd(chan, direction, &slave_addr); 2067 if (!txd) 2068 return NULL; 2069 2070 for_each_sg(sgl, sg, sg_len, tmp) { 2071 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2072 sg_dma_address(sg), 2073 sg_dma_len(sg)); 2074 if (ret) { 2075 pl08x_release_mux(plchan); 2076 pl08x_free_txd(pl08x, txd); 2077 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", 2078 __func__); 2079 return NULL; 2080 } 2081 } 2082 2083 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2084 if (!ret) { 2085 pl08x_release_mux(plchan); 2086 pl08x_free_txd(pl08x, txd); 2087 return NULL; 2088 } 2089 2090 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2091 } 2092 2093 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( 2094 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2095 size_t period_len, enum dma_transfer_direction direction, 2096 unsigned long flags) 2097 { 2098 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2099 struct pl08x_driver_data *pl08x = plchan->host; 2100 struct pl08x_txd *txd; 2101 int ret, tmp; 2102 dma_addr_t slave_addr; 2103 2104 dev_dbg(&pl08x->adev->dev, 2105 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", 2106 __func__, period_len, buf_len, 2107 direction == DMA_MEM_TO_DEV ? "to" : "from", 2108 plchan->name); 2109 2110 txd = pl08x_init_txd(chan, direction, &slave_addr); 2111 if (!txd) 2112 return NULL; 2113 2114 txd->cyclic = true; 2115 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; 2116 for (tmp = 0; tmp < buf_len; tmp += period_len) { 2117 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2118 buf_addr + tmp, period_len); 2119 if (ret) { 2120 pl08x_release_mux(plchan); 2121 pl08x_free_txd(pl08x, txd); 2122 return NULL; 2123 } 2124 } 2125 2126 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2127 if (!ret) { 2128 pl08x_release_mux(plchan); 2129 pl08x_free_txd(pl08x, txd); 2130 return NULL; 2131 } 2132 2133 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2134 } 2135 2136 static int pl08x_config(struct dma_chan *chan, 2137 struct dma_slave_config *config) 2138 { 2139 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2140 struct pl08x_driver_data *pl08x = plchan->host; 2141 2142 if (!plchan->slave) 2143 return -EINVAL; 2144 2145 /* Reject definitely invalid configurations */ 2146 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 2147 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 2148 return -EINVAL; 2149 2150 if (config->device_fc && pl08x->vd->pl080s) { 2151 dev_err(&pl08x->adev->dev, 2152 "%s: PL080S does not support peripheral flow control\n", 2153 __func__); 2154 return -EINVAL; 2155 } 2156 2157 plchan->cfg = *config; 2158 2159 return 0; 2160 } 2161 2162 static int pl08x_terminate_all(struct dma_chan *chan) 2163 { 2164 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2165 struct pl08x_driver_data *pl08x = plchan->host; 2166 unsigned long flags; 2167 2168 spin_lock_irqsave(&plchan->vc.lock, flags); 2169 if (!plchan->phychan && !plchan->at) { 2170 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2171 return 0; 2172 } 2173 2174 plchan->state = PL08X_CHAN_IDLE; 2175 2176 if (plchan->phychan) { 2177 /* 2178 * Mark physical channel as free and free any slave 2179 * signal 2180 */ 2181 pl08x_phy_free(plchan); 2182 } 2183 /* Dequeue jobs and free LLIs */ 2184 if (plchan->at) { 2185 vchan_terminate_vdesc(&plchan->at->vd); 2186 plchan->at = NULL; 2187 } 2188 /* Dequeue jobs not yet fired as well */ 2189 pl08x_free_txd_list(pl08x, plchan); 2190 2191 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2192 2193 return 0; 2194 } 2195 2196 static void pl08x_synchronize(struct dma_chan *chan) 2197 { 2198 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2199 2200 vchan_synchronize(&plchan->vc); 2201 } 2202 2203 static int pl08x_pause(struct dma_chan *chan) 2204 { 2205 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2206 unsigned long flags; 2207 2208 /* 2209 * Anything succeeds on channels with no physical allocation and 2210 * no queued transfers. 2211 */ 2212 spin_lock_irqsave(&plchan->vc.lock, flags); 2213 if (!plchan->phychan && !plchan->at) { 2214 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2215 return 0; 2216 } 2217 2218 pl08x_pause_phy_chan(plchan->phychan); 2219 plchan->state = PL08X_CHAN_PAUSED; 2220 2221 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2222 2223 return 0; 2224 } 2225 2226 static int pl08x_resume(struct dma_chan *chan) 2227 { 2228 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2229 unsigned long flags; 2230 2231 /* 2232 * Anything succeeds on channels with no physical allocation and 2233 * no queued transfers. 2234 */ 2235 spin_lock_irqsave(&plchan->vc.lock, flags); 2236 if (!plchan->phychan && !plchan->at) { 2237 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2238 return 0; 2239 } 2240 2241 pl08x_resume_phy_chan(plchan->phychan); 2242 plchan->state = PL08X_CHAN_RUNNING; 2243 2244 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2245 2246 return 0; 2247 } 2248 2249 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) 2250 { 2251 struct pl08x_dma_chan *plchan; 2252 char *name = chan_id; 2253 2254 /* Reject channels for devices not bound to this driver */ 2255 if (chan->device->dev->driver != &pl08x_amba_driver.drv) 2256 return false; 2257 2258 plchan = to_pl08x_chan(chan); 2259 2260 /* Check that the channel is not taken! */ 2261 if (!strcmp(plchan->name, name)) 2262 return true; 2263 2264 return false; 2265 } 2266 EXPORT_SYMBOL_GPL(pl08x_filter_id); 2267 2268 static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id) 2269 { 2270 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2271 2272 return plchan->cd == chan_id; 2273 } 2274 2275 /* 2276 * Just check that the device is there and active 2277 * TODO: turn this bit on/off depending on the number of physical channels 2278 * actually used, if it is zero... well shut it off. That will save some 2279 * power. Cut the clock at the same time. 2280 */ 2281 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) 2282 { 2283 /* The Nomadik variant does not have the config register */ 2284 if (pl08x->vd->nomadik) 2285 return; 2286 /* The FTDMAC020 variant does this in another register */ 2287 if (pl08x->vd->ftdmac020) { 2288 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); 2289 return; 2290 } 2291 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); 2292 } 2293 2294 static irqreturn_t pl08x_irq(int irq, void *dev) 2295 { 2296 struct pl08x_driver_data *pl08x = dev; 2297 u32 mask = 0, err, tc, i; 2298 2299 /* check & clear - ERR & TC interrupts */ 2300 err = readl(pl08x->base + PL080_ERR_STATUS); 2301 if (err) { 2302 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", 2303 __func__, err); 2304 writel(err, pl08x->base + PL080_ERR_CLEAR); 2305 } 2306 tc = readl(pl08x->base + PL080_TC_STATUS); 2307 if (tc) 2308 writel(tc, pl08x->base + PL080_TC_CLEAR); 2309 2310 if (!err && !tc) 2311 return IRQ_NONE; 2312 2313 for (i = 0; i < pl08x->vd->channels; i++) { 2314 if ((BIT(i) & err) || (BIT(i) & tc)) { 2315 /* Locate physical channel */ 2316 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; 2317 struct pl08x_dma_chan *plchan = phychan->serving; 2318 struct pl08x_txd *tx; 2319 2320 if (!plchan) { 2321 dev_err(&pl08x->adev->dev, 2322 "%s Error TC interrupt on unused channel: 0x%08x\n", 2323 __func__, i); 2324 continue; 2325 } 2326 2327 spin_lock(&plchan->vc.lock); 2328 tx = plchan->at; 2329 if (tx && tx->cyclic) { 2330 vchan_cyclic_callback(&tx->vd); 2331 } else if (tx) { 2332 plchan->at = NULL; 2333 /* 2334 * This descriptor is done, release its mux 2335 * reservation. 2336 */ 2337 pl08x_release_mux(plchan); 2338 tx->done = true; 2339 vchan_cookie_complete(&tx->vd); 2340 2341 /* 2342 * And start the next descriptor (if any), 2343 * otherwise free this channel. 2344 */ 2345 if (vchan_next_desc(&plchan->vc)) 2346 pl08x_start_next_txd(plchan); 2347 else 2348 pl08x_phy_free(plchan); 2349 } 2350 spin_unlock(&plchan->vc.lock); 2351 2352 mask |= BIT(i); 2353 } 2354 } 2355 2356 return mask ? IRQ_HANDLED : IRQ_NONE; 2357 } 2358 2359 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) 2360 { 2361 chan->slave = true; 2362 chan->name = chan->cd->bus_id; 2363 chan->cfg.src_addr = chan->cd->addr; 2364 chan->cfg.dst_addr = chan->cd->addr; 2365 } 2366 2367 /* 2368 * Initialise the DMAC memcpy/slave channels. 2369 * Make a local wrapper to hold required data 2370 */ 2371 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, 2372 struct dma_device *dmadev, unsigned int channels, bool slave) 2373 { 2374 struct pl08x_dma_chan *chan; 2375 int i; 2376 2377 INIT_LIST_HEAD(&dmadev->channels); 2378 2379 /* 2380 * Register as many many memcpy as we have physical channels, 2381 * we won't always be able to use all but the code will have 2382 * to cope with that situation. 2383 */ 2384 for (i = 0; i < channels; i++) { 2385 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 2386 if (!chan) 2387 return -ENOMEM; 2388 2389 chan->host = pl08x; 2390 chan->state = PL08X_CHAN_IDLE; 2391 chan->signal = -1; 2392 2393 if (slave) { 2394 chan->cd = &pl08x->pd->slave_channels[i]; 2395 /* 2396 * Some implementations have muxed signals, whereas some 2397 * use a mux in front of the signals and need dynamic 2398 * assignment of signals. 2399 */ 2400 chan->signal = i; 2401 pl08x_dma_slave_init(chan); 2402 } else { 2403 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); 2404 if (!chan->cd) { 2405 kfree(chan); 2406 return -ENOMEM; 2407 } 2408 chan->cd->bus_id = "memcpy"; 2409 chan->cd->periph_buses = pl08x->pd->mem_buses; 2410 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); 2411 if (!chan->name) { 2412 kfree(chan->cd); 2413 kfree(chan); 2414 return -ENOMEM; 2415 } 2416 } 2417 dev_dbg(&pl08x->adev->dev, 2418 "initialize virtual channel \"%s\"\n", 2419 chan->name); 2420 2421 chan->vc.desc_free = pl08x_desc_free; 2422 vchan_init(&chan->vc, dmadev); 2423 } 2424 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", 2425 i, slave ? "slave" : "memcpy"); 2426 return i; 2427 } 2428 2429 static void pl08x_free_virtual_channels(struct dma_device *dmadev) 2430 { 2431 struct pl08x_dma_chan *chan = NULL; 2432 struct pl08x_dma_chan *next; 2433 2434 list_for_each_entry_safe(chan, 2435 next, &dmadev->channels, vc.chan.device_node) { 2436 list_del(&chan->vc.chan.device_node); 2437 kfree(chan); 2438 } 2439 } 2440 2441 #ifdef CONFIG_DEBUG_FS 2442 static const char *pl08x_state_str(enum pl08x_dma_chan_state state) 2443 { 2444 switch (state) { 2445 case PL08X_CHAN_IDLE: 2446 return "idle"; 2447 case PL08X_CHAN_RUNNING: 2448 return "running"; 2449 case PL08X_CHAN_PAUSED: 2450 return "paused"; 2451 case PL08X_CHAN_WAITING: 2452 return "waiting"; 2453 default: 2454 break; 2455 } 2456 return "UNKNOWN STATE"; 2457 } 2458 2459 static int pl08x_debugfs_show(struct seq_file *s, void *data) 2460 { 2461 struct pl08x_driver_data *pl08x = s->private; 2462 struct pl08x_dma_chan *chan; 2463 struct pl08x_phy_chan *ch; 2464 unsigned long flags; 2465 int i; 2466 2467 seq_printf(s, "PL08x physical channels:\n"); 2468 seq_printf(s, "CHANNEL:\tUSER:\n"); 2469 seq_printf(s, "--------\t-----\n"); 2470 for (i = 0; i < pl08x->vd->channels; i++) { 2471 struct pl08x_dma_chan *virt_chan; 2472 2473 ch = &pl08x->phy_chans[i]; 2474 2475 spin_lock_irqsave(&ch->lock, flags); 2476 virt_chan = ch->serving; 2477 2478 seq_printf(s, "%d\t\t%s%s\n", 2479 ch->id, 2480 virt_chan ? virt_chan->name : "(none)", 2481 ch->locked ? " LOCKED" : ""); 2482 2483 spin_unlock_irqrestore(&ch->lock, flags); 2484 } 2485 2486 seq_printf(s, "\nPL08x virtual memcpy channels:\n"); 2487 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2488 seq_printf(s, "--------\t------\n"); 2489 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { 2490 seq_printf(s, "%s\t\t%s\n", chan->name, 2491 pl08x_state_str(chan->state)); 2492 } 2493 2494 if (pl08x->has_slave) { 2495 seq_printf(s, "\nPL08x virtual slave channels:\n"); 2496 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2497 seq_printf(s, "--------\t------\n"); 2498 list_for_each_entry(chan, &pl08x->slave.channels, 2499 vc.chan.device_node) { 2500 seq_printf(s, "%s\t\t%s\n", chan->name, 2501 pl08x_state_str(chan->state)); 2502 } 2503 } 2504 2505 return 0; 2506 } 2507 2508 static int pl08x_debugfs_open(struct inode *inode, struct file *file) 2509 { 2510 return single_open(file, pl08x_debugfs_show, inode->i_private); 2511 } 2512 2513 static const struct file_operations pl08x_debugfs_operations = { 2514 .open = pl08x_debugfs_open, 2515 .read = seq_read, 2516 .llseek = seq_lseek, 2517 .release = single_release, 2518 }; 2519 2520 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2521 { 2522 /* Expose a simple debugfs interface to view all clocks */ 2523 (void) debugfs_create_file(dev_name(&pl08x->adev->dev), 2524 S_IFREG | S_IRUGO, NULL, pl08x, 2525 &pl08x_debugfs_operations); 2526 } 2527 2528 #else 2529 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2530 { 2531 } 2532 #endif 2533 2534 #ifdef CONFIG_OF 2535 static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, 2536 u32 id) 2537 { 2538 struct pl08x_dma_chan *chan; 2539 2540 /* Trying to get a slave channel from something with no slave support */ 2541 if (!pl08x->has_slave) 2542 return NULL; 2543 2544 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2545 if (chan->signal == id) 2546 return &chan->vc.chan; 2547 } 2548 2549 return NULL; 2550 } 2551 2552 static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, 2553 struct of_dma *ofdma) 2554 { 2555 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; 2556 struct dma_chan *dma_chan; 2557 struct pl08x_dma_chan *plchan; 2558 2559 if (!pl08x) 2560 return NULL; 2561 2562 if (dma_spec->args_count != 2) { 2563 dev_err(&pl08x->adev->dev, 2564 "DMA channel translation requires two cells\n"); 2565 return NULL; 2566 } 2567 2568 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); 2569 if (!dma_chan) { 2570 dev_err(&pl08x->adev->dev, 2571 "DMA slave channel not found\n"); 2572 return NULL; 2573 } 2574 2575 plchan = to_pl08x_chan(dma_chan); 2576 dev_dbg(&pl08x->adev->dev, 2577 "translated channel for signal %d\n", 2578 dma_spec->args[0]); 2579 2580 /* Augment channel data for applicable AHB buses */ 2581 plchan->cd->periph_buses = dma_spec->args[1]; 2582 return dma_get_slave_channel(dma_chan); 2583 } 2584 2585 static int pl08x_of_probe(struct amba_device *adev, 2586 struct pl08x_driver_data *pl08x, 2587 struct device_node *np) 2588 { 2589 struct pl08x_platform_data *pd; 2590 struct pl08x_channel_data *chanp = NULL; 2591 u32 val; 2592 int ret; 2593 int i; 2594 2595 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); 2596 if (!pd) 2597 return -ENOMEM; 2598 2599 /* Eligible bus masters for fetching LLIs */ 2600 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) 2601 pd->lli_buses |= PL08X_AHB1; 2602 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) 2603 pd->lli_buses |= PL08X_AHB2; 2604 if (!pd->lli_buses) { 2605 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); 2606 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; 2607 } 2608 2609 /* Eligible bus masters for memory access */ 2610 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) 2611 pd->mem_buses |= PL08X_AHB1; 2612 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) 2613 pd->mem_buses |= PL08X_AHB2; 2614 if (!pd->mem_buses) { 2615 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); 2616 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; 2617 } 2618 2619 /* Parse the memcpy channel properties */ 2620 ret = of_property_read_u32(np, "memcpy-burst-size", &val); 2621 if (ret) { 2622 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); 2623 val = 1; 2624 } 2625 switch (val) { 2626 default: 2627 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); 2628 /* Fall through */ 2629 case 1: 2630 pd->memcpy_burst_size = PL08X_BURST_SZ_1; 2631 break; 2632 case 4: 2633 pd->memcpy_burst_size = PL08X_BURST_SZ_4; 2634 break; 2635 case 8: 2636 pd->memcpy_burst_size = PL08X_BURST_SZ_8; 2637 break; 2638 case 16: 2639 pd->memcpy_burst_size = PL08X_BURST_SZ_16; 2640 break; 2641 case 32: 2642 pd->memcpy_burst_size = PL08X_BURST_SZ_32; 2643 break; 2644 case 64: 2645 pd->memcpy_burst_size = PL08X_BURST_SZ_64; 2646 break; 2647 case 128: 2648 pd->memcpy_burst_size = PL08X_BURST_SZ_128; 2649 break; 2650 case 256: 2651 pd->memcpy_burst_size = PL08X_BURST_SZ_256; 2652 break; 2653 } 2654 2655 ret = of_property_read_u32(np, "memcpy-bus-width", &val); 2656 if (ret) { 2657 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); 2658 val = 8; 2659 } 2660 switch (val) { 2661 default: 2662 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); 2663 /* Fall through */ 2664 case 8: 2665 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; 2666 break; 2667 case 16: 2668 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; 2669 break; 2670 case 32: 2671 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; 2672 break; 2673 } 2674 2675 /* 2676 * Allocate channel data for all possible slave channels (one 2677 * for each possible signal), channels will then be allocated 2678 * for a device and have it's AHB interfaces set up at 2679 * translation time. 2680 */ 2681 if (pl08x->vd->signals) { 2682 chanp = devm_kcalloc(&adev->dev, 2683 pl08x->vd->signals, 2684 sizeof(struct pl08x_channel_data), 2685 GFP_KERNEL); 2686 if (!chanp) 2687 return -ENOMEM; 2688 2689 pd->slave_channels = chanp; 2690 for (i = 0; i < pl08x->vd->signals; i++) { 2691 /* 2692 * chanp->periph_buses will be assigned at translation 2693 */ 2694 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); 2695 chanp++; 2696 } 2697 pd->num_slave_channels = pl08x->vd->signals; 2698 } 2699 2700 pl08x->pd = pd; 2701 2702 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, 2703 pl08x); 2704 } 2705 #else 2706 static inline int pl08x_of_probe(struct amba_device *adev, 2707 struct pl08x_driver_data *pl08x, 2708 struct device_node *np) 2709 { 2710 return -EINVAL; 2711 } 2712 #endif 2713 2714 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2715 { 2716 struct pl08x_driver_data *pl08x; 2717 struct vendor_data *vd = id->data; 2718 struct device_node *np = adev->dev.of_node; 2719 u32 tsfr_size; 2720 int ret = 0; 2721 int i; 2722 2723 ret = amba_request_regions(adev, NULL); 2724 if (ret) 2725 return ret; 2726 2727 /* Ensure that we can do DMA */ 2728 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2729 if (ret) 2730 goto out_no_pl08x; 2731 2732 /* Create the driver state holder */ 2733 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); 2734 if (!pl08x) { 2735 ret = -ENOMEM; 2736 goto out_no_pl08x; 2737 } 2738 2739 /* Assign useful pointers to the driver state */ 2740 pl08x->adev = adev; 2741 pl08x->vd = vd; 2742 2743 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); 2744 if (!pl08x->base) { 2745 ret = -ENOMEM; 2746 goto out_no_ioremap; 2747 } 2748 2749 if (vd->ftdmac020) { 2750 u32 val; 2751 2752 val = readl(pl08x->base + FTDMAC020_REVISION); 2753 dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", 2754 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff); 2755 val = readl(pl08x->base + FTDMAC020_FEATURE); 2756 dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, " 2757 "%s built-in bridge, %s, %s linked lists\n", 2758 (val >> 12) & 0x0f, 2759 (val & BIT(10)) ? "no" : "has", 2760 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0", 2761 (val & BIT(8)) ? "supports" : "does not support"); 2762 2763 /* Vendor data from feature register */ 2764 if (!(val & BIT(8))) 2765 dev_warn(&pl08x->adev->dev, 2766 "linked lists not supported, required\n"); 2767 vd->channels = (val >> 12) & 0x0f; 2768 vd->dualmaster = !!(val & BIT(9)); 2769 } 2770 2771 /* Initialize memcpy engine */ 2772 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); 2773 pl08x->memcpy.dev = &adev->dev; 2774 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; 2775 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; 2776 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; 2777 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; 2778 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; 2779 pl08x->memcpy.device_config = pl08x_config; 2780 pl08x->memcpy.device_pause = pl08x_pause; 2781 pl08x->memcpy.device_resume = pl08x_resume; 2782 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; 2783 pl08x->memcpy.device_synchronize = pl08x_synchronize; 2784 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2785 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2786 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); 2787 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2788 if (vd->ftdmac020) 2789 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; 2790 2791 2792 /* 2793 * Initialize slave engine, if the block has no signals, that means 2794 * we have no slave support. 2795 */ 2796 if (vd->signals) { 2797 pl08x->has_slave = true; 2798 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); 2799 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); 2800 pl08x->slave.dev = &adev->dev; 2801 pl08x->slave.device_free_chan_resources = 2802 pl08x_free_chan_resources; 2803 pl08x->slave.device_prep_dma_interrupt = 2804 pl08x_prep_dma_interrupt; 2805 pl08x->slave.device_tx_status = pl08x_dma_tx_status; 2806 pl08x->slave.device_issue_pending = pl08x_issue_pending; 2807 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; 2808 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; 2809 pl08x->slave.device_config = pl08x_config; 2810 pl08x->slave.device_pause = pl08x_pause; 2811 pl08x->slave.device_resume = pl08x_resume; 2812 pl08x->slave.device_terminate_all = pl08x_terminate_all; 2813 pl08x->slave.device_synchronize = pl08x_synchronize; 2814 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2815 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2816 pl08x->slave.directions = 2817 BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2818 pl08x->slave.residue_granularity = 2819 DMA_RESIDUE_GRANULARITY_SEGMENT; 2820 } 2821 2822 /* Get the platform data */ 2823 pl08x->pd = dev_get_platdata(&adev->dev); 2824 if (!pl08x->pd) { 2825 if (np) { 2826 ret = pl08x_of_probe(adev, pl08x, np); 2827 if (ret) 2828 goto out_no_platdata; 2829 } else { 2830 dev_err(&adev->dev, "no platform data supplied\n"); 2831 ret = -EINVAL; 2832 goto out_no_platdata; 2833 } 2834 } else { 2835 pl08x->slave.filter.map = pl08x->pd->slave_map; 2836 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; 2837 pl08x->slave.filter.fn = pl08x_filter_fn; 2838 } 2839 2840 /* By default, AHB1 only. If dualmaster, from platform */ 2841 pl08x->lli_buses = PL08X_AHB1; 2842 pl08x->mem_buses = PL08X_AHB1; 2843 if (pl08x->vd->dualmaster) { 2844 pl08x->lli_buses = pl08x->pd->lli_buses; 2845 pl08x->mem_buses = pl08x->pd->mem_buses; 2846 } 2847 2848 if (vd->pl080s) 2849 pl08x->lli_words = PL080S_LLI_WORDS; 2850 else 2851 pl08x->lli_words = PL080_LLI_WORDS; 2852 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); 2853 2854 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 2855 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 2856 tsfr_size, PL08X_ALIGN, 0); 2857 if (!pl08x->pool) { 2858 ret = -ENOMEM; 2859 goto out_no_lli_pool; 2860 } 2861 2862 /* Turn on the PL08x */ 2863 pl08x_ensure_on(pl08x); 2864 2865 /* Clear any pending interrupts */ 2866 if (vd->ftdmac020) 2867 /* This variant has error IRQs in bits 16-19 */ 2868 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); 2869 else 2870 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); 2871 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 2872 2873 /* Attach the interrupt handler */ 2874 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); 2875 if (ret) { 2876 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 2877 __func__, adev->irq[0]); 2878 goto out_no_irq; 2879 } 2880 2881 /* Initialize physical channels */ 2882 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), 2883 GFP_KERNEL); 2884 if (!pl08x->phy_chans) { 2885 ret = -ENOMEM; 2886 goto out_no_phychans; 2887 } 2888 2889 for (i = 0; i < vd->channels; i++) { 2890 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; 2891 2892 ch->id = i; 2893 ch->base = pl08x->base + PL080_Cx_BASE(i); 2894 if (vd->ftdmac020) { 2895 /* FTDMA020 has a special channel busy register */ 2896 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; 2897 ch->reg_config = ch->base + FTDMAC020_CH_CFG; 2898 ch->reg_control = ch->base + FTDMAC020_CH_CSR; 2899 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; 2900 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; 2901 ch->reg_lli = ch->base + FTDMAC020_CH_LLP; 2902 ch->ftdmac020 = true; 2903 } else { 2904 ch->reg_config = ch->base + vd->config_offset; 2905 ch->reg_control = ch->base + PL080_CH_CONTROL; 2906 ch->reg_src = ch->base + PL080_CH_SRC_ADDR; 2907 ch->reg_dst = ch->base + PL080_CH_DST_ADDR; 2908 ch->reg_lli = ch->base + PL080_CH_LLI; 2909 } 2910 if (vd->pl080s) 2911 ch->pl080s = true; 2912 2913 spin_lock_init(&ch->lock); 2914 2915 /* 2916 * Nomadik variants can have channels that are locked 2917 * down for the secure world only. Lock up these channels 2918 * by perpetually serving a dummy virtual channel. 2919 */ 2920 if (vd->nomadik) { 2921 u32 val; 2922 2923 val = readl(ch->reg_config); 2924 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { 2925 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); 2926 ch->locked = true; 2927 } 2928 } 2929 2930 dev_dbg(&adev->dev, "physical channel %d is %s\n", 2931 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); 2932 } 2933 2934 /* Register as many memcpy channels as there are physical channels */ 2935 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, 2936 pl08x->vd->channels, false); 2937 if (ret <= 0) { 2938 dev_warn(&pl08x->adev->dev, 2939 "%s failed to enumerate memcpy channels - %d\n", 2940 __func__, ret); 2941 goto out_no_memcpy; 2942 } 2943 2944 /* Register slave channels */ 2945 if (pl08x->has_slave) { 2946 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, 2947 pl08x->pd->num_slave_channels, true); 2948 if (ret < 0) { 2949 dev_warn(&pl08x->adev->dev, 2950 "%s failed to enumerate slave channels - %d\n", 2951 __func__, ret); 2952 goto out_no_slave; 2953 } 2954 } 2955 2956 ret = dma_async_device_register(&pl08x->memcpy); 2957 if (ret) { 2958 dev_warn(&pl08x->adev->dev, 2959 "%s failed to register memcpy as an async device - %d\n", 2960 __func__, ret); 2961 goto out_no_memcpy_reg; 2962 } 2963 2964 if (pl08x->has_slave) { 2965 ret = dma_async_device_register(&pl08x->slave); 2966 if (ret) { 2967 dev_warn(&pl08x->adev->dev, 2968 "%s failed to register slave as an async device - %d\n", 2969 __func__, ret); 2970 goto out_no_slave_reg; 2971 } 2972 } 2973 2974 amba_set_drvdata(adev, pl08x); 2975 init_pl08x_debugfs(pl08x); 2976 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", 2977 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), 2978 (unsigned long long)adev->res.start, adev->irq[0]); 2979 2980 return 0; 2981 2982 out_no_slave_reg: 2983 dma_async_device_unregister(&pl08x->memcpy); 2984 out_no_memcpy_reg: 2985 if (pl08x->has_slave) 2986 pl08x_free_virtual_channels(&pl08x->slave); 2987 out_no_slave: 2988 pl08x_free_virtual_channels(&pl08x->memcpy); 2989 out_no_memcpy: 2990 kfree(pl08x->phy_chans); 2991 out_no_phychans: 2992 free_irq(adev->irq[0], pl08x); 2993 out_no_irq: 2994 dma_pool_destroy(pl08x->pool); 2995 out_no_lli_pool: 2996 out_no_platdata: 2997 iounmap(pl08x->base); 2998 out_no_ioremap: 2999 kfree(pl08x); 3000 out_no_pl08x: 3001 amba_release_regions(adev); 3002 return ret; 3003 } 3004 3005 /* PL080 has 8 channels and the PL080 have just 2 */ 3006 static struct vendor_data vendor_pl080 = { 3007 .config_offset = PL080_CH_CONFIG, 3008 .channels = 8, 3009 .signals = 16, 3010 .dualmaster = true, 3011 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3012 }; 3013 3014 static struct vendor_data vendor_nomadik = { 3015 .config_offset = PL080_CH_CONFIG, 3016 .channels = 8, 3017 .signals = 32, 3018 .dualmaster = true, 3019 .nomadik = true, 3020 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3021 }; 3022 3023 static struct vendor_data vendor_pl080s = { 3024 .config_offset = PL080S_CH_CONFIG, 3025 .channels = 8, 3026 .signals = 32, 3027 .pl080s = true, 3028 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, 3029 }; 3030 3031 static struct vendor_data vendor_pl081 = { 3032 .config_offset = PL080_CH_CONFIG, 3033 .channels = 2, 3034 .signals = 16, 3035 .dualmaster = false, 3036 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3037 }; 3038 3039 static struct vendor_data vendor_ftdmac020 = { 3040 .config_offset = PL080_CH_CONFIG, 3041 .ftdmac020 = true, 3042 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3043 }; 3044 3045 static const struct amba_id pl08x_ids[] = { 3046 /* Samsung PL080S variant */ 3047 { 3048 .id = 0x0a141080, 3049 .mask = 0xffffffff, 3050 .data = &vendor_pl080s, 3051 }, 3052 /* PL080 */ 3053 { 3054 .id = 0x00041080, 3055 .mask = 0x000fffff, 3056 .data = &vendor_pl080, 3057 }, 3058 /* PL081 */ 3059 { 3060 .id = 0x00041081, 3061 .mask = 0x000fffff, 3062 .data = &vendor_pl081, 3063 }, 3064 /* Nomadik 8815 PL080 variant */ 3065 { 3066 .id = 0x00280080, 3067 .mask = 0x00ffffff, 3068 .data = &vendor_nomadik, 3069 }, 3070 /* Faraday Technology FTDMAC020 */ 3071 { 3072 .id = 0x0003b080, 3073 .mask = 0x000fffff, 3074 .data = &vendor_ftdmac020, 3075 }, 3076 { 0, 0 }, 3077 }; 3078 3079 MODULE_DEVICE_TABLE(amba, pl08x_ids); 3080 3081 static struct amba_driver pl08x_amba_driver = { 3082 .drv.name = DRIVER_NAME, 3083 .id_table = pl08x_ids, 3084 .probe = pl08x_probe, 3085 }; 3086 3087 static int __init pl08x_init(void) 3088 { 3089 int retval; 3090 retval = amba_driver_register(&pl08x_amba_driver); 3091 if (retval) 3092 printk(KERN_WARNING DRIVER_NAME 3093 "failed to register as an AMBA device (%d)\n", 3094 retval); 3095 return retval; 3096 } 3097 subsys_initcall(pl08x_init); 3098