1 /* 2 * DMA driver for Xilinx Video DMA Engine 3 * 4 * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved. 5 * 6 * Based on the Freescale DMA driver. 7 * 8 * Description: 9 * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP 10 * core that provides high-bandwidth direct memory access between memory 11 * and AXI4-Stream type video target peripherals. The core provides efficient 12 * two dimensional DMA operations with independent asynchronous read (S2MM) 13 * and write (MM2S) channel operation. It can be configured to have either 14 * one channel or two channels. If configured as two channels, one is to 15 * transmit to the video device (MM2S) and another is to receive from the 16 * video device (S2MM). Initialization, status, interrupt and management 17 * registers are accessed through an AXI4-Lite slave interface. 18 * 19 * The AXI Direct Memory Access (AXI DMA) core is a soft Xilinx IP core that 20 * provides high-bandwidth one dimensional direct memory access between memory 21 * and AXI4-Stream target peripherals. It supports one receive and one 22 * transmit channel, both of them optional at synthesis time. 23 * 24 * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory 25 * Access (DMA) between a memory-mapped source address and a memory-mapped 26 * destination address. 27 * 28 * This program is free software: you can redistribute it and/or modify 29 * it under the terms of the GNU General Public License as published by 30 * the Free Software Foundation, either version 2 of the License, or 31 * (at your option) any later version. 32 */ 33 34 #include <linux/bitops.h> 35 #include <linux/dmapool.h> 36 #include <linux/dma/xilinx_dma.h> 37 #include <linux/init.h> 38 #include <linux/interrupt.h> 39 #include <linux/io.h> 40 #include <linux/iopoll.h> 41 #include <linux/module.h> 42 #include <linux/of_address.h> 43 #include <linux/of_dma.h> 44 #include <linux/of_platform.h> 45 #include <linux/of_irq.h> 46 #include <linux/slab.h> 47 #include <linux/clk.h> 48 #include <linux/io-64-nonatomic-lo-hi.h> 49 50 #include "../dmaengine.h" 51 52 /* Register/Descriptor Offsets */ 53 #define XILINX_DMA_MM2S_CTRL_OFFSET 0x0000 54 #define XILINX_DMA_S2MM_CTRL_OFFSET 0x0030 55 #define XILINX_VDMA_MM2S_DESC_OFFSET 0x0050 56 #define XILINX_VDMA_S2MM_DESC_OFFSET 0x00a0 57 58 /* Control Registers */ 59 #define XILINX_DMA_REG_DMACR 0x0000 60 #define XILINX_DMA_DMACR_DELAY_MAX 0xff 61 #define XILINX_DMA_DMACR_DELAY_SHIFT 24 62 #define XILINX_DMA_DMACR_FRAME_COUNT_MAX 0xff 63 #define XILINX_DMA_DMACR_FRAME_COUNT_SHIFT 16 64 #define XILINX_DMA_DMACR_ERR_IRQ BIT(14) 65 #define XILINX_DMA_DMACR_DLY_CNT_IRQ BIT(13) 66 #define XILINX_DMA_DMACR_FRM_CNT_IRQ BIT(12) 67 #define XILINX_DMA_DMACR_MASTER_SHIFT 8 68 #define XILINX_DMA_DMACR_FSYNCSRC_SHIFT 5 69 #define XILINX_DMA_DMACR_FRAMECNT_EN BIT(4) 70 #define XILINX_DMA_DMACR_GENLOCK_EN BIT(3) 71 #define XILINX_DMA_DMACR_RESET BIT(2) 72 #define XILINX_DMA_DMACR_CIRC_EN BIT(1) 73 #define XILINX_DMA_DMACR_RUNSTOP BIT(0) 74 #define XILINX_DMA_DMACR_FSYNCSRC_MASK GENMASK(6, 5) 75 76 #define XILINX_DMA_REG_DMASR 0x0004 77 #define XILINX_DMA_DMASR_EOL_LATE_ERR BIT(15) 78 #define XILINX_DMA_DMASR_ERR_IRQ BIT(14) 79 #define XILINX_DMA_DMASR_DLY_CNT_IRQ BIT(13) 80 #define XILINX_DMA_DMASR_FRM_CNT_IRQ BIT(12) 81 #define XILINX_DMA_DMASR_SOF_LATE_ERR BIT(11) 82 #define XILINX_DMA_DMASR_SG_DEC_ERR BIT(10) 83 #define XILINX_DMA_DMASR_SG_SLV_ERR BIT(9) 84 #define XILINX_DMA_DMASR_EOF_EARLY_ERR BIT(8) 85 #define XILINX_DMA_DMASR_SOF_EARLY_ERR BIT(7) 86 #define XILINX_DMA_DMASR_DMA_DEC_ERR BIT(6) 87 #define XILINX_DMA_DMASR_DMA_SLAVE_ERR BIT(5) 88 #define XILINX_DMA_DMASR_DMA_INT_ERR BIT(4) 89 #define XILINX_DMA_DMASR_IDLE BIT(1) 90 #define XILINX_DMA_DMASR_HALTED BIT(0) 91 #define XILINX_DMA_DMASR_DELAY_MASK GENMASK(31, 24) 92 #define XILINX_DMA_DMASR_FRAME_COUNT_MASK GENMASK(23, 16) 93 94 #define XILINX_DMA_REG_CURDESC 0x0008 95 #define XILINX_DMA_REG_TAILDESC 0x0010 96 #define XILINX_DMA_REG_REG_INDEX 0x0014 97 #define XILINX_DMA_REG_FRMSTORE 0x0018 98 #define XILINX_DMA_REG_THRESHOLD 0x001c 99 #define XILINX_DMA_REG_FRMPTR_STS 0x0024 100 #define XILINX_DMA_REG_PARK_PTR 0x0028 101 #define XILINX_DMA_PARK_PTR_WR_REF_SHIFT 8 102 #define XILINX_DMA_PARK_PTR_RD_REF_SHIFT 0 103 #define XILINX_DMA_REG_VDMA_VERSION 0x002c 104 105 /* Register Direct Mode Registers */ 106 #define XILINX_DMA_REG_VSIZE 0x0000 107 #define XILINX_DMA_REG_HSIZE 0x0004 108 109 #define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 110 #define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 111 #define XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT 0 112 113 #define XILINX_VDMA_REG_START_ADDRESS(n) (0x000c + 4 * (n)) 114 #define XILINX_VDMA_REG_START_ADDRESS_64(n) (0x000c + 8 * (n)) 115 116 /* HW specific definitions */ 117 #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20 118 119 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ 120 (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ 121 XILINX_DMA_DMASR_DLY_CNT_IRQ | \ 122 XILINX_DMA_DMASR_ERR_IRQ) 123 124 #define XILINX_DMA_DMASR_ALL_ERR_MASK \ 125 (XILINX_DMA_DMASR_EOL_LATE_ERR | \ 126 XILINX_DMA_DMASR_SOF_LATE_ERR | \ 127 XILINX_DMA_DMASR_SG_DEC_ERR | \ 128 XILINX_DMA_DMASR_SG_SLV_ERR | \ 129 XILINX_DMA_DMASR_EOF_EARLY_ERR | \ 130 XILINX_DMA_DMASR_SOF_EARLY_ERR | \ 131 XILINX_DMA_DMASR_DMA_DEC_ERR | \ 132 XILINX_DMA_DMASR_DMA_SLAVE_ERR | \ 133 XILINX_DMA_DMASR_DMA_INT_ERR) 134 135 /* 136 * Recoverable errors are DMA Internal error, SOF Early, EOF Early 137 * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC 138 * is enabled in the h/w system. 139 */ 140 #define XILINX_DMA_DMASR_ERR_RECOVER_MASK \ 141 (XILINX_DMA_DMASR_SOF_LATE_ERR | \ 142 XILINX_DMA_DMASR_EOF_EARLY_ERR | \ 143 XILINX_DMA_DMASR_SOF_EARLY_ERR | \ 144 XILINX_DMA_DMASR_DMA_INT_ERR) 145 146 /* Axi VDMA Flush on Fsync bits */ 147 #define XILINX_DMA_FLUSH_S2MM 3 148 #define XILINX_DMA_FLUSH_MM2S 2 149 #define XILINX_DMA_FLUSH_BOTH 1 150 151 /* Delay loop counter to prevent hardware failure */ 152 #define XILINX_DMA_LOOP_COUNT 1000000 153 154 /* AXI DMA Specific Registers/Offsets */ 155 #define XILINX_DMA_REG_SRCDSTADDR 0x18 156 #define XILINX_DMA_REG_BTT 0x28 157 158 /* AXI DMA Specific Masks/Bit fields */ 159 #define XILINX_DMA_MAX_TRANS_LEN GENMASK(22, 0) 160 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) 161 #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) 162 #define XILINX_DMA_CR_COALESCE_SHIFT 16 163 #define XILINX_DMA_BD_SOP BIT(27) 164 #define XILINX_DMA_BD_EOP BIT(26) 165 #define XILINX_DMA_COALESCE_MAX 255 166 #define XILINX_DMA_NUM_APP_WORDS 5 167 168 /* Multi-Channel DMA Descriptor offsets*/ 169 #define XILINX_DMA_MCRX_CDESC(x) (0x40 + (x-1) * 0x20) 170 #define XILINX_DMA_MCRX_TDESC(x) (0x48 + (x-1) * 0x20) 171 172 /* Multi-Channel DMA Masks/Shifts */ 173 #define XILINX_DMA_BD_HSIZE_MASK GENMASK(15, 0) 174 #define XILINX_DMA_BD_STRIDE_MASK GENMASK(15, 0) 175 #define XILINX_DMA_BD_VSIZE_MASK GENMASK(31, 19) 176 #define XILINX_DMA_BD_TDEST_MASK GENMASK(4, 0) 177 #define XILINX_DMA_BD_STRIDE_SHIFT 0 178 #define XILINX_DMA_BD_VSIZE_SHIFT 19 179 180 /* AXI CDMA Specific Registers/Offsets */ 181 #define XILINX_CDMA_REG_SRCADDR 0x18 182 #define XILINX_CDMA_REG_DSTADDR 0x20 183 184 /* AXI CDMA Specific Masks */ 185 #define XILINX_CDMA_CR_SGMODE BIT(3) 186 187 /** 188 * struct xilinx_vdma_desc_hw - Hardware Descriptor 189 * @next_desc: Next Descriptor Pointer @0x00 190 * @pad1: Reserved @0x04 191 * @buf_addr: Buffer address @0x08 192 * @buf_addr_msb: MSB of Buffer address @0x0C 193 * @vsize: Vertical Size @0x10 194 * @hsize: Horizontal Size @0x14 195 * @stride: Number of bytes between the first 196 * pixels of each horizontal line @0x18 197 */ 198 struct xilinx_vdma_desc_hw { 199 u32 next_desc; 200 u32 pad1; 201 u32 buf_addr; 202 u32 buf_addr_msb; 203 u32 vsize; 204 u32 hsize; 205 u32 stride; 206 } __aligned(64); 207 208 /** 209 * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA 210 * @next_desc: Next Descriptor Pointer @0x00 211 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04 212 * @buf_addr: Buffer address @0x08 213 * @buf_addr_msb: MSB of Buffer address @0x0C 214 * @pad1: Reserved @0x10 215 * @pad2: Reserved @0x14 216 * @control: Control field @0x18 217 * @status: Status field @0x1C 218 * @app: APP Fields @0x20 - 0x30 219 */ 220 struct xilinx_axidma_desc_hw { 221 u32 next_desc; 222 u32 next_desc_msb; 223 u32 buf_addr; 224 u32 buf_addr_msb; 225 u32 mcdma_control; 226 u32 vsize_stride; 227 u32 control; 228 u32 status; 229 u32 app[XILINX_DMA_NUM_APP_WORDS]; 230 } __aligned(64); 231 232 /** 233 * struct xilinx_cdma_desc_hw - Hardware Descriptor 234 * @next_desc: Next Descriptor Pointer @0x00 235 * @next_descmsb: Next Descriptor Pointer MSB @0x04 236 * @src_addr: Source address @0x08 237 * @src_addrmsb: Source address MSB @0x0C 238 * @dest_addr: Destination address @0x10 239 * @dest_addrmsb: Destination address MSB @0x14 240 * @control: Control field @0x18 241 * @status: Status field @0x1C 242 */ 243 struct xilinx_cdma_desc_hw { 244 u32 next_desc; 245 u32 next_desc_msb; 246 u32 src_addr; 247 u32 src_addr_msb; 248 u32 dest_addr; 249 u32 dest_addr_msb; 250 u32 control; 251 u32 status; 252 } __aligned(64); 253 254 /** 255 * struct xilinx_vdma_tx_segment - Descriptor segment 256 * @hw: Hardware descriptor 257 * @node: Node in the descriptor segments list 258 * @phys: Physical address of segment 259 */ 260 struct xilinx_vdma_tx_segment { 261 struct xilinx_vdma_desc_hw hw; 262 struct list_head node; 263 dma_addr_t phys; 264 } __aligned(64); 265 266 /** 267 * struct xilinx_axidma_tx_segment - Descriptor segment 268 * @hw: Hardware descriptor 269 * @node: Node in the descriptor segments list 270 * @phys: Physical address of segment 271 */ 272 struct xilinx_axidma_tx_segment { 273 struct xilinx_axidma_desc_hw hw; 274 struct list_head node; 275 dma_addr_t phys; 276 } __aligned(64); 277 278 /** 279 * struct xilinx_cdma_tx_segment - Descriptor segment 280 * @hw: Hardware descriptor 281 * @node: Node in the descriptor segments list 282 * @phys: Physical address of segment 283 */ 284 struct xilinx_cdma_tx_segment { 285 struct xilinx_cdma_desc_hw hw; 286 struct list_head node; 287 dma_addr_t phys; 288 } __aligned(64); 289 290 /** 291 * struct xilinx_dma_tx_descriptor - Per Transaction structure 292 * @async_tx: Async transaction descriptor 293 * @segments: TX segments list 294 * @node: Node in the channel descriptors list 295 * @cyclic: Check for cyclic transfers. 296 */ 297 struct xilinx_dma_tx_descriptor { 298 struct dma_async_tx_descriptor async_tx; 299 struct list_head segments; 300 struct list_head node; 301 bool cyclic; 302 }; 303 304 /** 305 * struct xilinx_dma_chan - Driver specific DMA channel structure 306 * @xdev: Driver specific device structure 307 * @ctrl_offset: Control registers offset 308 * @desc_offset: TX descriptor registers offset 309 * @lock: Descriptor operation lock 310 * @pending_list: Descriptors waiting 311 * @active_list: Descriptors ready to submit 312 * @done_list: Complete descriptors 313 * @common: DMA common channel 314 * @desc_pool: Descriptors pool 315 * @dev: The dma device 316 * @irq: Channel IRQ 317 * @id: Channel ID 318 * @direction: Transfer direction 319 * @num_frms: Number of frames 320 * @has_sg: Support scatter transfers 321 * @cyclic: Check for cyclic transfers. 322 * @genlock: Support genlock mode 323 * @err: Channel has errors 324 * @tasklet: Cleanup work after irq 325 * @config: Device configuration info 326 * @flush_on_fsync: Flush on Frame sync 327 * @desc_pendingcount: Descriptor pending count 328 * @ext_addr: Indicates 64 bit addressing is supported by dma channel 329 * @desc_submitcount: Descriptor h/w submitted count 330 * @residue: Residue for AXI DMA 331 * @seg_v: Statically allocated segments base 332 * @cyclic_seg_v: Statically allocated segment base for cyclic transfers 333 * @start_transfer: Differentiate b/w DMA IP's transfer 334 */ 335 struct xilinx_dma_chan { 336 struct xilinx_dma_device *xdev; 337 u32 ctrl_offset; 338 u32 desc_offset; 339 spinlock_t lock; 340 struct list_head pending_list; 341 struct list_head active_list; 342 struct list_head done_list; 343 struct dma_chan common; 344 struct dma_pool *desc_pool; 345 struct device *dev; 346 int irq; 347 int id; 348 enum dma_transfer_direction direction; 349 int num_frms; 350 bool has_sg; 351 bool cyclic; 352 bool genlock; 353 bool err; 354 struct tasklet_struct tasklet; 355 struct xilinx_vdma_config config; 356 bool flush_on_fsync; 357 u32 desc_pendingcount; 358 bool ext_addr; 359 u32 desc_submitcount; 360 u32 residue; 361 struct xilinx_axidma_tx_segment *seg_v; 362 struct xilinx_axidma_tx_segment *cyclic_seg_v; 363 void (*start_transfer)(struct xilinx_dma_chan *chan); 364 u16 tdest; 365 }; 366 367 struct xilinx_dma_config { 368 enum xdma_ip_type dmatype; 369 int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk, 370 struct clk **tx_clk, struct clk **txs_clk, 371 struct clk **rx_clk, struct clk **rxs_clk); 372 }; 373 374 /** 375 * struct xilinx_dma_device - DMA device structure 376 * @regs: I/O mapped base address 377 * @dev: Device Structure 378 * @common: DMA device structure 379 * @chan: Driver specific DMA channel 380 * @has_sg: Specifies whether Scatter-Gather is present or not 381 * @mcdma: Specifies whether Multi-Channel is present or not 382 * @flush_on_fsync: Flush on frame sync 383 * @ext_addr: Indicates 64 bit addressing is supported by dma device 384 * @pdev: Platform device structure pointer 385 * @dma_config: DMA config structure 386 * @axi_clk: DMA Axi4-lite interace clock 387 * @tx_clk: DMA mm2s clock 388 * @txs_clk: DMA mm2s stream clock 389 * @rx_clk: DMA s2mm clock 390 * @rxs_clk: DMA s2mm stream clock 391 * @nr_channels: Number of channels DMA device supports 392 * @chan_id: DMA channel identifier 393 */ 394 struct xilinx_dma_device { 395 void __iomem *regs; 396 struct device *dev; 397 struct dma_device common; 398 struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; 399 bool has_sg; 400 bool mcdma; 401 u32 flush_on_fsync; 402 bool ext_addr; 403 struct platform_device *pdev; 404 const struct xilinx_dma_config *dma_config; 405 struct clk *axi_clk; 406 struct clk *tx_clk; 407 struct clk *txs_clk; 408 struct clk *rx_clk; 409 struct clk *rxs_clk; 410 u32 nr_channels; 411 u32 chan_id; 412 }; 413 414 /* Macros */ 415 #define to_xilinx_chan(chan) \ 416 container_of(chan, struct xilinx_dma_chan, common) 417 #define to_dma_tx_descriptor(tx) \ 418 container_of(tx, struct xilinx_dma_tx_descriptor, async_tx) 419 #define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \ 420 readl_poll_timeout(chan->xdev->regs + chan->ctrl_offset + reg, val, \ 421 cond, delay_us, timeout_us) 422 423 /* IO accessors */ 424 static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg) 425 { 426 return ioread32(chan->xdev->regs + reg); 427 } 428 429 static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value) 430 { 431 iowrite32(value, chan->xdev->regs + reg); 432 } 433 434 static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg, 435 u32 value) 436 { 437 dma_write(chan, chan->desc_offset + reg, value); 438 } 439 440 static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg) 441 { 442 return dma_read(chan, chan->ctrl_offset + reg); 443 } 444 445 static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg, 446 u32 value) 447 { 448 dma_write(chan, chan->ctrl_offset + reg, value); 449 } 450 451 static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg, 452 u32 clr) 453 { 454 dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr); 455 } 456 457 static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg, 458 u32 set) 459 { 460 dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set); 461 } 462 463 /** 464 * vdma_desc_write_64 - 64-bit descriptor write 465 * @chan: Driver specific VDMA channel 466 * @reg: Register to write 467 * @value_lsb: lower address of the descriptor. 468 * @value_msb: upper address of the descriptor. 469 * 470 * Since vdma driver is trying to write to a register offset which is not a 471 * multiple of 64 bits(ex : 0x5c), we are writing as two separate 32 bits 472 * instead of a single 64 bit register write. 473 */ 474 static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg, 475 u32 value_lsb, u32 value_msb) 476 { 477 /* Write the lsb 32 bits*/ 478 writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg); 479 480 /* Write the msb 32 bits */ 481 writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4); 482 } 483 484 static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value) 485 { 486 lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg); 487 } 488 489 static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg, 490 dma_addr_t addr) 491 { 492 if (chan->ext_addr) 493 dma_writeq(chan, reg, addr); 494 else 495 dma_ctrl_write(chan, reg, addr); 496 } 497 498 static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan, 499 struct xilinx_axidma_desc_hw *hw, 500 dma_addr_t buf_addr, size_t sg_used, 501 size_t period_len) 502 { 503 if (chan->ext_addr) { 504 hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len); 505 hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used + 506 period_len); 507 } else { 508 hw->buf_addr = buf_addr + sg_used + period_len; 509 } 510 } 511 512 /* ----------------------------------------------------------------------------- 513 * Descriptors and segments alloc and free 514 */ 515 516 /** 517 * xilinx_vdma_alloc_tx_segment - Allocate transaction segment 518 * @chan: Driver specific DMA channel 519 * 520 * Return: The allocated segment on success and NULL on failure. 521 */ 522 static struct xilinx_vdma_tx_segment * 523 xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan) 524 { 525 struct xilinx_vdma_tx_segment *segment; 526 dma_addr_t phys; 527 528 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); 529 if (!segment) 530 return NULL; 531 532 segment->phys = phys; 533 534 return segment; 535 } 536 537 /** 538 * xilinx_cdma_alloc_tx_segment - Allocate transaction segment 539 * @chan: Driver specific DMA channel 540 * 541 * Return: The allocated segment on success and NULL on failure. 542 */ 543 static struct xilinx_cdma_tx_segment * 544 xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan) 545 { 546 struct xilinx_cdma_tx_segment *segment; 547 dma_addr_t phys; 548 549 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); 550 if (!segment) 551 return NULL; 552 553 segment->phys = phys; 554 555 return segment; 556 } 557 558 /** 559 * xilinx_axidma_alloc_tx_segment - Allocate transaction segment 560 * @chan: Driver specific DMA channel 561 * 562 * Return: The allocated segment on success and NULL on failure. 563 */ 564 static struct xilinx_axidma_tx_segment * 565 xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan) 566 { 567 struct xilinx_axidma_tx_segment *segment; 568 dma_addr_t phys; 569 570 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); 571 if (!segment) 572 return NULL; 573 574 segment->phys = phys; 575 576 return segment; 577 } 578 579 /** 580 * xilinx_dma_free_tx_segment - Free transaction segment 581 * @chan: Driver specific DMA channel 582 * @segment: DMA transaction segment 583 */ 584 static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan, 585 struct xilinx_axidma_tx_segment *segment) 586 { 587 dma_pool_free(chan->desc_pool, segment, segment->phys); 588 } 589 590 /** 591 * xilinx_cdma_free_tx_segment - Free transaction segment 592 * @chan: Driver specific DMA channel 593 * @segment: DMA transaction segment 594 */ 595 static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan, 596 struct xilinx_cdma_tx_segment *segment) 597 { 598 dma_pool_free(chan->desc_pool, segment, segment->phys); 599 } 600 601 /** 602 * xilinx_vdma_free_tx_segment - Free transaction segment 603 * @chan: Driver specific DMA channel 604 * @segment: DMA transaction segment 605 */ 606 static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan, 607 struct xilinx_vdma_tx_segment *segment) 608 { 609 dma_pool_free(chan->desc_pool, segment, segment->phys); 610 } 611 612 /** 613 * xilinx_dma_tx_descriptor - Allocate transaction descriptor 614 * @chan: Driver specific DMA channel 615 * 616 * Return: The allocated descriptor on success and NULL on failure. 617 */ 618 static struct xilinx_dma_tx_descriptor * 619 xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) 620 { 621 struct xilinx_dma_tx_descriptor *desc; 622 623 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 624 if (!desc) 625 return NULL; 626 627 INIT_LIST_HEAD(&desc->segments); 628 629 return desc; 630 } 631 632 /** 633 * xilinx_dma_free_tx_descriptor - Free transaction descriptor 634 * @chan: Driver specific DMA channel 635 * @desc: DMA transaction descriptor 636 */ 637 static void 638 xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan, 639 struct xilinx_dma_tx_descriptor *desc) 640 { 641 struct xilinx_vdma_tx_segment *segment, *next; 642 struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next; 643 struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next; 644 645 if (!desc) 646 return; 647 648 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 649 list_for_each_entry_safe(segment, next, &desc->segments, node) { 650 list_del(&segment->node); 651 xilinx_vdma_free_tx_segment(chan, segment); 652 } 653 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 654 list_for_each_entry_safe(cdma_segment, cdma_next, 655 &desc->segments, node) { 656 list_del(&cdma_segment->node); 657 xilinx_cdma_free_tx_segment(chan, cdma_segment); 658 } 659 } else { 660 list_for_each_entry_safe(axidma_segment, axidma_next, 661 &desc->segments, node) { 662 list_del(&axidma_segment->node); 663 xilinx_dma_free_tx_segment(chan, axidma_segment); 664 } 665 } 666 667 kfree(desc); 668 } 669 670 /* Required functions */ 671 672 /** 673 * xilinx_dma_free_desc_list - Free descriptors list 674 * @chan: Driver specific DMA channel 675 * @list: List to parse and delete the descriptor 676 */ 677 static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan, 678 struct list_head *list) 679 { 680 struct xilinx_dma_tx_descriptor *desc, *next; 681 682 list_for_each_entry_safe(desc, next, list, node) { 683 list_del(&desc->node); 684 xilinx_dma_free_tx_descriptor(chan, desc); 685 } 686 } 687 688 /** 689 * xilinx_dma_free_descriptors - Free channel descriptors 690 * @chan: Driver specific DMA channel 691 */ 692 static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan) 693 { 694 unsigned long flags; 695 696 spin_lock_irqsave(&chan->lock, flags); 697 698 xilinx_dma_free_desc_list(chan, &chan->pending_list); 699 xilinx_dma_free_desc_list(chan, &chan->done_list); 700 xilinx_dma_free_desc_list(chan, &chan->active_list); 701 702 spin_unlock_irqrestore(&chan->lock, flags); 703 } 704 705 /** 706 * xilinx_dma_free_chan_resources - Free channel resources 707 * @dchan: DMA channel 708 */ 709 static void xilinx_dma_free_chan_resources(struct dma_chan *dchan) 710 { 711 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 712 713 dev_dbg(chan->dev, "Free all channel resources.\n"); 714 715 xilinx_dma_free_descriptors(chan); 716 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 717 xilinx_dma_free_tx_segment(chan, chan->cyclic_seg_v); 718 xilinx_dma_free_tx_segment(chan, chan->seg_v); 719 } 720 dma_pool_destroy(chan->desc_pool); 721 chan->desc_pool = NULL; 722 } 723 724 /** 725 * xilinx_dma_chan_handle_cyclic - Cyclic dma callback 726 * @chan: Driver specific dma channel 727 * @desc: dma transaction descriptor 728 * @flags: flags for spin lock 729 */ 730 static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan, 731 struct xilinx_dma_tx_descriptor *desc, 732 unsigned long *flags) 733 { 734 dma_async_tx_callback callback; 735 void *callback_param; 736 737 callback = desc->async_tx.callback; 738 callback_param = desc->async_tx.callback_param; 739 if (callback) { 740 spin_unlock_irqrestore(&chan->lock, *flags); 741 callback(callback_param); 742 spin_lock_irqsave(&chan->lock, *flags); 743 } 744 } 745 746 /** 747 * xilinx_dma_chan_desc_cleanup - Clean channel descriptors 748 * @chan: Driver specific DMA channel 749 */ 750 static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) 751 { 752 struct xilinx_dma_tx_descriptor *desc, *next; 753 unsigned long flags; 754 755 spin_lock_irqsave(&chan->lock, flags); 756 757 list_for_each_entry_safe(desc, next, &chan->done_list, node) { 758 dma_async_tx_callback callback; 759 void *callback_param; 760 761 if (desc->cyclic) { 762 xilinx_dma_chan_handle_cyclic(chan, desc, &flags); 763 break; 764 } 765 766 /* Remove from the list of running transactions */ 767 list_del(&desc->node); 768 769 /* Run the link descriptor callback function */ 770 callback = desc->async_tx.callback; 771 callback_param = desc->async_tx.callback_param; 772 if (callback) { 773 spin_unlock_irqrestore(&chan->lock, flags); 774 callback(callback_param); 775 spin_lock_irqsave(&chan->lock, flags); 776 } 777 778 /* Run any dependencies, then free the descriptor */ 779 dma_run_dependencies(&desc->async_tx); 780 xilinx_dma_free_tx_descriptor(chan, desc); 781 } 782 783 spin_unlock_irqrestore(&chan->lock, flags); 784 } 785 786 /** 787 * xilinx_dma_do_tasklet - Schedule completion tasklet 788 * @data: Pointer to the Xilinx DMA channel structure 789 */ 790 static void xilinx_dma_do_tasklet(unsigned long data) 791 { 792 struct xilinx_dma_chan *chan = (struct xilinx_dma_chan *)data; 793 794 xilinx_dma_chan_desc_cleanup(chan); 795 } 796 797 /** 798 * xilinx_dma_alloc_chan_resources - Allocate channel resources 799 * @dchan: DMA channel 800 * 801 * Return: '0' on success and failure value on error 802 */ 803 static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) 804 { 805 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 806 807 /* Has this channel already been allocated? */ 808 if (chan->desc_pool) 809 return 0; 810 811 /* 812 * We need the descriptor to be aligned to 64bytes 813 * for meeting Xilinx VDMA specification requirement. 814 */ 815 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 816 chan->desc_pool = dma_pool_create("xilinx_dma_desc_pool", 817 chan->dev, 818 sizeof(struct xilinx_axidma_tx_segment), 819 __alignof__(struct xilinx_axidma_tx_segment), 820 0); 821 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 822 chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool", 823 chan->dev, 824 sizeof(struct xilinx_cdma_tx_segment), 825 __alignof__(struct xilinx_cdma_tx_segment), 826 0); 827 } else { 828 chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool", 829 chan->dev, 830 sizeof(struct xilinx_vdma_tx_segment), 831 __alignof__(struct xilinx_vdma_tx_segment), 832 0); 833 } 834 835 if (!chan->desc_pool) { 836 dev_err(chan->dev, 837 "unable to allocate channel %d descriptor pool\n", 838 chan->id); 839 return -ENOMEM; 840 } 841 842 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 843 /* 844 * For AXI DMA case after submitting a pending_list, keep 845 * an extra segment allocated so that the "next descriptor" 846 * pointer on the tail descriptor always points to a 847 * valid descriptor, even when paused after reaching taildesc. 848 * This way, it is possible to issue additional 849 * transfers without halting and restarting the channel. 850 */ 851 chan->seg_v = xilinx_axidma_alloc_tx_segment(chan); 852 853 /* 854 * For cyclic DMA mode we need to program the tail Descriptor 855 * register with a value which is not a part of the BD chain 856 * so allocating a desc segment during channel allocation for 857 * programming tail descriptor. 858 */ 859 chan->cyclic_seg_v = xilinx_axidma_alloc_tx_segment(chan); 860 } 861 862 dma_cookie_init(dchan); 863 864 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 865 /* For AXI DMA resetting once channel will reset the 866 * other channel as well so enable the interrupts here. 867 */ 868 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 869 XILINX_DMA_DMAXR_ALL_IRQ_MASK); 870 } 871 872 if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) 873 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 874 XILINX_CDMA_CR_SGMODE); 875 876 return 0; 877 } 878 879 /** 880 * xilinx_dma_tx_status - Get DMA transaction status 881 * @dchan: DMA channel 882 * @cookie: Transaction identifier 883 * @txstate: Transaction state 884 * 885 * Return: DMA transaction status 886 */ 887 static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan, 888 dma_cookie_t cookie, 889 struct dma_tx_state *txstate) 890 { 891 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 892 struct xilinx_dma_tx_descriptor *desc; 893 struct xilinx_axidma_tx_segment *segment; 894 struct xilinx_axidma_desc_hw *hw; 895 enum dma_status ret; 896 unsigned long flags; 897 u32 residue = 0; 898 899 ret = dma_cookie_status(dchan, cookie, txstate); 900 if (ret == DMA_COMPLETE || !txstate) 901 return ret; 902 903 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 904 spin_lock_irqsave(&chan->lock, flags); 905 906 desc = list_last_entry(&chan->active_list, 907 struct xilinx_dma_tx_descriptor, node); 908 if (chan->has_sg) { 909 list_for_each_entry(segment, &desc->segments, node) { 910 hw = &segment->hw; 911 residue += (hw->control - hw->status) & 912 XILINX_DMA_MAX_TRANS_LEN; 913 } 914 } 915 spin_unlock_irqrestore(&chan->lock, flags); 916 917 chan->residue = residue; 918 dma_set_residue(txstate, chan->residue); 919 } 920 921 return ret; 922 } 923 924 /** 925 * xilinx_dma_is_running - Check if DMA channel is running 926 * @chan: Driver specific DMA channel 927 * 928 * Return: '1' if running, '0' if not. 929 */ 930 static bool xilinx_dma_is_running(struct xilinx_dma_chan *chan) 931 { 932 return !(dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & 933 XILINX_DMA_DMASR_HALTED) && 934 (dma_ctrl_read(chan, XILINX_DMA_REG_DMACR) & 935 XILINX_DMA_DMACR_RUNSTOP); 936 } 937 938 /** 939 * xilinx_dma_is_idle - Check if DMA channel is idle 940 * @chan: Driver specific DMA channel 941 * 942 * Return: '1' if idle, '0' if not. 943 */ 944 static bool xilinx_dma_is_idle(struct xilinx_dma_chan *chan) 945 { 946 return dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & 947 XILINX_DMA_DMASR_IDLE; 948 } 949 950 /** 951 * xilinx_dma_halt - Halt DMA channel 952 * @chan: Driver specific DMA channel 953 */ 954 static void xilinx_dma_halt(struct xilinx_dma_chan *chan) 955 { 956 int err; 957 u32 val; 958 959 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); 960 961 /* Wait for the hardware to halt */ 962 err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, 963 (val & XILINX_DMA_DMASR_HALTED), 0, 964 XILINX_DMA_LOOP_COUNT); 965 966 if (err) { 967 dev_err(chan->dev, "Cannot stop channel %p: %x\n", 968 chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 969 chan->err = true; 970 } 971 } 972 973 /** 974 * xilinx_dma_start - Start DMA channel 975 * @chan: Driver specific DMA channel 976 */ 977 static void xilinx_dma_start(struct xilinx_dma_chan *chan) 978 { 979 int err; 980 u32 val; 981 982 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); 983 984 /* Wait for the hardware to start */ 985 err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, 986 !(val & XILINX_DMA_DMASR_HALTED), 0, 987 XILINX_DMA_LOOP_COUNT); 988 989 if (err) { 990 dev_err(chan->dev, "Cannot start channel %p: %x\n", 991 chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 992 993 chan->err = true; 994 } 995 } 996 997 /** 998 * xilinx_vdma_start_transfer - Starts VDMA transfer 999 * @chan: Driver specific channel struct pointer 1000 */ 1001 static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) 1002 { 1003 struct xilinx_vdma_config *config = &chan->config; 1004 struct xilinx_dma_tx_descriptor *desc, *tail_desc; 1005 u32 reg; 1006 struct xilinx_vdma_tx_segment *tail_segment; 1007 1008 /* This function was invoked with lock held */ 1009 if (chan->err) 1010 return; 1011 1012 if (list_empty(&chan->pending_list)) 1013 return; 1014 1015 desc = list_first_entry(&chan->pending_list, 1016 struct xilinx_dma_tx_descriptor, node); 1017 tail_desc = list_last_entry(&chan->pending_list, 1018 struct xilinx_dma_tx_descriptor, node); 1019 1020 tail_segment = list_last_entry(&tail_desc->segments, 1021 struct xilinx_vdma_tx_segment, node); 1022 1023 /* If it is SG mode and hardware is busy, cannot submit */ 1024 if (chan->has_sg && xilinx_dma_is_running(chan) && 1025 !xilinx_dma_is_idle(chan)) { 1026 dev_dbg(chan->dev, "DMA controller still busy\n"); 1027 return; 1028 } 1029 1030 /* 1031 * If hardware is idle, then all descriptors on the running lists are 1032 * done, start new transfers 1033 */ 1034 if (chan->has_sg) 1035 dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, 1036 desc->async_tx.phys); 1037 1038 /* Configure the hardware using info in the config structure */ 1039 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 1040 1041 if (config->frm_cnt_en) 1042 reg |= XILINX_DMA_DMACR_FRAMECNT_EN; 1043 else 1044 reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN; 1045 1046 /* Configure channel to allow number frame buffers */ 1047 dma_ctrl_write(chan, XILINX_DMA_REG_FRMSTORE, 1048 chan->desc_pendingcount); 1049 1050 /* 1051 * With SG, start with circular mode, so that BDs can be fetched. 1052 * In direct register mode, if not parking, enable circular mode 1053 */ 1054 if (chan->has_sg || !config->park) 1055 reg |= XILINX_DMA_DMACR_CIRC_EN; 1056 1057 if (config->park) 1058 reg &= ~XILINX_DMA_DMACR_CIRC_EN; 1059 1060 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 1061 1062 if (config->park && (config->park_frm >= 0) && 1063 (config->park_frm < chan->num_frms)) { 1064 if (chan->direction == DMA_MEM_TO_DEV) 1065 dma_write(chan, XILINX_DMA_REG_PARK_PTR, 1066 config->park_frm << 1067 XILINX_DMA_PARK_PTR_RD_REF_SHIFT); 1068 else 1069 dma_write(chan, XILINX_DMA_REG_PARK_PTR, 1070 config->park_frm << 1071 XILINX_DMA_PARK_PTR_WR_REF_SHIFT); 1072 } 1073 1074 /* Start the hardware */ 1075 xilinx_dma_start(chan); 1076 1077 if (chan->err) 1078 return; 1079 1080 /* Start the transfer */ 1081 if (chan->has_sg) { 1082 dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, 1083 tail_segment->phys); 1084 } else { 1085 struct xilinx_vdma_tx_segment *segment, *last = NULL; 1086 int i = 0; 1087 1088 if (chan->desc_submitcount < chan->num_frms) 1089 i = chan->desc_submitcount; 1090 1091 list_for_each_entry(segment, &desc->segments, node) { 1092 if (chan->ext_addr) 1093 vdma_desc_write_64(chan, 1094 XILINX_VDMA_REG_START_ADDRESS_64(i++), 1095 segment->hw.buf_addr, 1096 segment->hw.buf_addr_msb); 1097 else 1098 vdma_desc_write(chan, 1099 XILINX_VDMA_REG_START_ADDRESS(i++), 1100 segment->hw.buf_addr); 1101 1102 last = segment; 1103 } 1104 1105 if (!last) 1106 return; 1107 1108 /* HW expects these parameters to be same for one transaction */ 1109 vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); 1110 vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE, 1111 last->hw.stride); 1112 vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); 1113 } 1114 1115 if (!chan->has_sg) { 1116 list_del(&desc->node); 1117 list_add_tail(&desc->node, &chan->active_list); 1118 chan->desc_submitcount++; 1119 chan->desc_pendingcount--; 1120 if (chan->desc_submitcount == chan->num_frms) 1121 chan->desc_submitcount = 0; 1122 } else { 1123 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1124 chan->desc_pendingcount = 0; 1125 } 1126 } 1127 1128 /** 1129 * xilinx_cdma_start_transfer - Starts cdma transfer 1130 * @chan: Driver specific channel struct pointer 1131 */ 1132 static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan) 1133 { 1134 struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; 1135 struct xilinx_cdma_tx_segment *tail_segment; 1136 u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR); 1137 1138 if (chan->err) 1139 return; 1140 1141 if (list_empty(&chan->pending_list)) 1142 return; 1143 1144 head_desc = list_first_entry(&chan->pending_list, 1145 struct xilinx_dma_tx_descriptor, node); 1146 tail_desc = list_last_entry(&chan->pending_list, 1147 struct xilinx_dma_tx_descriptor, node); 1148 tail_segment = list_last_entry(&tail_desc->segments, 1149 struct xilinx_cdma_tx_segment, node); 1150 1151 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { 1152 ctrl_reg &= ~XILINX_DMA_CR_COALESCE_MAX; 1153 ctrl_reg |= chan->desc_pendingcount << 1154 XILINX_DMA_CR_COALESCE_SHIFT; 1155 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg); 1156 } 1157 1158 if (chan->has_sg) { 1159 xilinx_write(chan, XILINX_DMA_REG_CURDESC, 1160 head_desc->async_tx.phys); 1161 1162 /* Update tail ptr register which will start the transfer */ 1163 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1164 tail_segment->phys); 1165 } else { 1166 /* In simple mode */ 1167 struct xilinx_cdma_tx_segment *segment; 1168 struct xilinx_cdma_desc_hw *hw; 1169 1170 segment = list_first_entry(&head_desc->segments, 1171 struct xilinx_cdma_tx_segment, 1172 node); 1173 1174 hw = &segment->hw; 1175 1176 xilinx_write(chan, XILINX_CDMA_REG_SRCADDR, hw->src_addr); 1177 xilinx_write(chan, XILINX_CDMA_REG_DSTADDR, hw->dest_addr); 1178 1179 /* Start the transfer */ 1180 dma_ctrl_write(chan, XILINX_DMA_REG_BTT, 1181 hw->control & XILINX_DMA_MAX_TRANS_LEN); 1182 } 1183 1184 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1185 chan->desc_pendingcount = 0; 1186 } 1187 1188 /** 1189 * xilinx_dma_start_transfer - Starts DMA transfer 1190 * @chan: Driver specific channel struct pointer 1191 */ 1192 static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) 1193 { 1194 struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; 1195 struct xilinx_axidma_tx_segment *tail_segment, *old_head, *new_head; 1196 u32 reg; 1197 1198 if (chan->err) 1199 return; 1200 1201 if (list_empty(&chan->pending_list)) 1202 return; 1203 1204 /* If it is SG mode and hardware is busy, cannot submit */ 1205 if (chan->has_sg && xilinx_dma_is_running(chan) && 1206 !xilinx_dma_is_idle(chan)) { 1207 dev_dbg(chan->dev, "DMA controller still busy\n"); 1208 return; 1209 } 1210 1211 head_desc = list_first_entry(&chan->pending_list, 1212 struct xilinx_dma_tx_descriptor, node); 1213 tail_desc = list_last_entry(&chan->pending_list, 1214 struct xilinx_dma_tx_descriptor, node); 1215 tail_segment = list_last_entry(&tail_desc->segments, 1216 struct xilinx_axidma_tx_segment, node); 1217 1218 if (chan->has_sg && !chan->xdev->mcdma) { 1219 old_head = list_first_entry(&head_desc->segments, 1220 struct xilinx_axidma_tx_segment, node); 1221 new_head = chan->seg_v; 1222 /* Copy Buffer Descriptor fields. */ 1223 new_head->hw = old_head->hw; 1224 1225 /* Swap and save new reserve */ 1226 list_replace_init(&old_head->node, &new_head->node); 1227 chan->seg_v = old_head; 1228 1229 tail_segment->hw.next_desc = chan->seg_v->phys; 1230 head_desc->async_tx.phys = new_head->phys; 1231 } 1232 1233 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 1234 1235 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { 1236 reg &= ~XILINX_DMA_CR_COALESCE_MAX; 1237 reg |= chan->desc_pendingcount << 1238 XILINX_DMA_CR_COALESCE_SHIFT; 1239 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 1240 } 1241 1242 if (chan->has_sg && !chan->xdev->mcdma) 1243 xilinx_write(chan, XILINX_DMA_REG_CURDESC, 1244 head_desc->async_tx.phys); 1245 1246 if (chan->has_sg && chan->xdev->mcdma) { 1247 if (chan->direction == DMA_MEM_TO_DEV) { 1248 dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, 1249 head_desc->async_tx.phys); 1250 } else { 1251 if (!chan->tdest) { 1252 dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, 1253 head_desc->async_tx.phys); 1254 } else { 1255 dma_ctrl_write(chan, 1256 XILINX_DMA_MCRX_CDESC(chan->tdest), 1257 head_desc->async_tx.phys); 1258 } 1259 } 1260 } 1261 1262 xilinx_dma_start(chan); 1263 1264 if (chan->err) 1265 return; 1266 1267 /* Start the transfer */ 1268 if (chan->has_sg && !chan->xdev->mcdma) { 1269 if (chan->cyclic) 1270 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1271 chan->cyclic_seg_v->phys); 1272 else 1273 xilinx_write(chan, XILINX_DMA_REG_TAILDESC, 1274 tail_segment->phys); 1275 } else if (chan->has_sg && chan->xdev->mcdma) { 1276 if (chan->direction == DMA_MEM_TO_DEV) { 1277 dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, 1278 tail_segment->phys); 1279 } else { 1280 if (!chan->tdest) { 1281 dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, 1282 tail_segment->phys); 1283 } else { 1284 dma_ctrl_write(chan, 1285 XILINX_DMA_MCRX_TDESC(chan->tdest), 1286 tail_segment->phys); 1287 } 1288 } 1289 } else { 1290 struct xilinx_axidma_tx_segment *segment; 1291 struct xilinx_axidma_desc_hw *hw; 1292 1293 segment = list_first_entry(&head_desc->segments, 1294 struct xilinx_axidma_tx_segment, 1295 node); 1296 hw = &segment->hw; 1297 1298 xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR, hw->buf_addr); 1299 1300 /* Start the transfer */ 1301 dma_ctrl_write(chan, XILINX_DMA_REG_BTT, 1302 hw->control & XILINX_DMA_MAX_TRANS_LEN); 1303 } 1304 1305 list_splice_tail_init(&chan->pending_list, &chan->active_list); 1306 chan->desc_pendingcount = 0; 1307 } 1308 1309 /** 1310 * xilinx_dma_issue_pending - Issue pending transactions 1311 * @dchan: DMA channel 1312 */ 1313 static void xilinx_dma_issue_pending(struct dma_chan *dchan) 1314 { 1315 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1316 unsigned long flags; 1317 1318 spin_lock_irqsave(&chan->lock, flags); 1319 chan->start_transfer(chan); 1320 spin_unlock_irqrestore(&chan->lock, flags); 1321 } 1322 1323 /** 1324 * xilinx_dma_complete_descriptor - Mark the active descriptor as complete 1325 * @chan : xilinx DMA channel 1326 * 1327 * CONTEXT: hardirq 1328 */ 1329 static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) 1330 { 1331 struct xilinx_dma_tx_descriptor *desc, *next; 1332 1333 /* This function was invoked with lock held */ 1334 if (list_empty(&chan->active_list)) 1335 return; 1336 1337 list_for_each_entry_safe(desc, next, &chan->active_list, node) { 1338 list_del(&desc->node); 1339 if (!desc->cyclic) 1340 dma_cookie_complete(&desc->async_tx); 1341 list_add_tail(&desc->node, &chan->done_list); 1342 } 1343 } 1344 1345 /** 1346 * xilinx_dma_reset - Reset DMA channel 1347 * @chan: Driver specific DMA channel 1348 * 1349 * Return: '0' on success and failure value on error 1350 */ 1351 static int xilinx_dma_reset(struct xilinx_dma_chan *chan) 1352 { 1353 int err; 1354 u32 tmp; 1355 1356 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET); 1357 1358 /* Wait for the hardware to finish reset */ 1359 err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, 1360 !(tmp & XILINX_DMA_DMACR_RESET), 0, 1361 XILINX_DMA_LOOP_COUNT); 1362 1363 if (err) { 1364 dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", 1365 dma_ctrl_read(chan, XILINX_DMA_REG_DMACR), 1366 dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 1367 return -ETIMEDOUT; 1368 } 1369 1370 chan->err = false; 1371 1372 return err; 1373 } 1374 1375 /** 1376 * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts 1377 * @chan: Driver specific DMA channel 1378 * 1379 * Return: '0' on success and failure value on error 1380 */ 1381 static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan) 1382 { 1383 int err; 1384 1385 /* Reset VDMA */ 1386 err = xilinx_dma_reset(chan); 1387 if (err) 1388 return err; 1389 1390 /* Enable interrupts */ 1391 dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, 1392 XILINX_DMA_DMAXR_ALL_IRQ_MASK); 1393 1394 return 0; 1395 } 1396 1397 /** 1398 * xilinx_dma_irq_handler - DMA Interrupt handler 1399 * @irq: IRQ number 1400 * @data: Pointer to the Xilinx DMA channel structure 1401 * 1402 * Return: IRQ_HANDLED/IRQ_NONE 1403 */ 1404 static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) 1405 { 1406 struct xilinx_dma_chan *chan = data; 1407 u32 status; 1408 1409 /* Read the status and ack the interrupts. */ 1410 status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR); 1411 if (!(status & XILINX_DMA_DMAXR_ALL_IRQ_MASK)) 1412 return IRQ_NONE; 1413 1414 dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, 1415 status & XILINX_DMA_DMAXR_ALL_IRQ_MASK); 1416 1417 if (status & XILINX_DMA_DMASR_ERR_IRQ) { 1418 /* 1419 * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the 1420 * error is recoverable, ignore it. Otherwise flag the error. 1421 * 1422 * Only recoverable errors can be cleared in the DMASR register, 1423 * make sure not to write to other error bits to 1. 1424 */ 1425 u32 errors = status & XILINX_DMA_DMASR_ALL_ERR_MASK; 1426 1427 dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, 1428 errors & XILINX_DMA_DMASR_ERR_RECOVER_MASK); 1429 1430 if (!chan->flush_on_fsync || 1431 (errors & ~XILINX_DMA_DMASR_ERR_RECOVER_MASK)) { 1432 dev_err(chan->dev, 1433 "Channel %p has errors %x, cdr %x tdr %x\n", 1434 chan, errors, 1435 dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC), 1436 dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC)); 1437 chan->err = true; 1438 } 1439 } 1440 1441 if (status & XILINX_DMA_DMASR_DLY_CNT_IRQ) { 1442 /* 1443 * Device takes too long to do the transfer when user requires 1444 * responsiveness. 1445 */ 1446 dev_dbg(chan->dev, "Inter-packet latency too long\n"); 1447 } 1448 1449 if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) { 1450 spin_lock(&chan->lock); 1451 xilinx_dma_complete_descriptor(chan); 1452 chan->start_transfer(chan); 1453 spin_unlock(&chan->lock); 1454 } 1455 1456 tasklet_schedule(&chan->tasklet); 1457 return IRQ_HANDLED; 1458 } 1459 1460 /** 1461 * append_desc_queue - Queuing descriptor 1462 * @chan: Driver specific dma channel 1463 * @desc: dma transaction descriptor 1464 */ 1465 static void append_desc_queue(struct xilinx_dma_chan *chan, 1466 struct xilinx_dma_tx_descriptor *desc) 1467 { 1468 struct xilinx_vdma_tx_segment *tail_segment; 1469 struct xilinx_dma_tx_descriptor *tail_desc; 1470 struct xilinx_axidma_tx_segment *axidma_tail_segment; 1471 struct xilinx_cdma_tx_segment *cdma_tail_segment; 1472 1473 if (list_empty(&chan->pending_list)) 1474 goto append; 1475 1476 /* 1477 * Add the hardware descriptor to the chain of hardware descriptors 1478 * that already exists in memory. 1479 */ 1480 tail_desc = list_last_entry(&chan->pending_list, 1481 struct xilinx_dma_tx_descriptor, node); 1482 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 1483 tail_segment = list_last_entry(&tail_desc->segments, 1484 struct xilinx_vdma_tx_segment, 1485 node); 1486 tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 1487 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 1488 cdma_tail_segment = list_last_entry(&tail_desc->segments, 1489 struct xilinx_cdma_tx_segment, 1490 node); 1491 cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 1492 } else { 1493 axidma_tail_segment = list_last_entry(&tail_desc->segments, 1494 struct xilinx_axidma_tx_segment, 1495 node); 1496 axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; 1497 } 1498 1499 /* 1500 * Add the software descriptor and all children to the list 1501 * of pending transactions 1502 */ 1503 append: 1504 list_add_tail(&desc->node, &chan->pending_list); 1505 chan->desc_pendingcount++; 1506 1507 if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) 1508 && unlikely(chan->desc_pendingcount > chan->num_frms)) { 1509 dev_dbg(chan->dev, "desc pendingcount is too high\n"); 1510 chan->desc_pendingcount = chan->num_frms; 1511 } 1512 } 1513 1514 /** 1515 * xilinx_dma_tx_submit - Submit DMA transaction 1516 * @tx: Async transaction descriptor 1517 * 1518 * Return: cookie value on success and failure value on error 1519 */ 1520 static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx) 1521 { 1522 struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); 1523 struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan); 1524 dma_cookie_t cookie; 1525 unsigned long flags; 1526 int err; 1527 1528 if (chan->cyclic) { 1529 xilinx_dma_free_tx_descriptor(chan, desc); 1530 return -EBUSY; 1531 } 1532 1533 if (chan->err) { 1534 /* 1535 * If reset fails, need to hard reset the system. 1536 * Channel is no longer functional 1537 */ 1538 err = xilinx_dma_chan_reset(chan); 1539 if (err < 0) 1540 return err; 1541 } 1542 1543 spin_lock_irqsave(&chan->lock, flags); 1544 1545 cookie = dma_cookie_assign(tx); 1546 1547 /* Put this transaction onto the tail of the pending queue */ 1548 append_desc_queue(chan, desc); 1549 1550 if (desc->cyclic) 1551 chan->cyclic = true; 1552 1553 spin_unlock_irqrestore(&chan->lock, flags); 1554 1555 return cookie; 1556 } 1557 1558 /** 1559 * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a 1560 * DMA_SLAVE transaction 1561 * @dchan: DMA channel 1562 * @xt: Interleaved template pointer 1563 * @flags: transfer ack flags 1564 * 1565 * Return: Async transaction descriptor on success and NULL on failure 1566 */ 1567 static struct dma_async_tx_descriptor * 1568 xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, 1569 struct dma_interleaved_template *xt, 1570 unsigned long flags) 1571 { 1572 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1573 struct xilinx_dma_tx_descriptor *desc; 1574 struct xilinx_vdma_tx_segment *segment, *prev = NULL; 1575 struct xilinx_vdma_desc_hw *hw; 1576 1577 if (!is_slave_direction(xt->dir)) 1578 return NULL; 1579 1580 if (!xt->numf || !xt->sgl[0].size) 1581 return NULL; 1582 1583 if (xt->frame_size != 1) 1584 return NULL; 1585 1586 /* Allocate a transaction descriptor. */ 1587 desc = xilinx_dma_alloc_tx_descriptor(chan); 1588 if (!desc) 1589 return NULL; 1590 1591 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 1592 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 1593 async_tx_ack(&desc->async_tx); 1594 1595 /* Allocate the link descriptor from DMA pool */ 1596 segment = xilinx_vdma_alloc_tx_segment(chan); 1597 if (!segment) 1598 goto error; 1599 1600 /* Fill in the hardware descriptor */ 1601 hw = &segment->hw; 1602 hw->vsize = xt->numf; 1603 hw->hsize = xt->sgl[0].size; 1604 hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) << 1605 XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT; 1606 hw->stride |= chan->config.frm_dly << 1607 XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT; 1608 1609 if (xt->dir != DMA_MEM_TO_DEV) { 1610 if (chan->ext_addr) { 1611 hw->buf_addr = lower_32_bits(xt->dst_start); 1612 hw->buf_addr_msb = upper_32_bits(xt->dst_start); 1613 } else { 1614 hw->buf_addr = xt->dst_start; 1615 } 1616 } else { 1617 if (chan->ext_addr) { 1618 hw->buf_addr = lower_32_bits(xt->src_start); 1619 hw->buf_addr_msb = upper_32_bits(xt->src_start); 1620 } else { 1621 hw->buf_addr = xt->src_start; 1622 } 1623 } 1624 1625 /* Insert the segment into the descriptor segments list. */ 1626 list_add_tail(&segment->node, &desc->segments); 1627 1628 prev = segment; 1629 1630 /* Link the last hardware descriptor with the first. */ 1631 segment = list_first_entry(&desc->segments, 1632 struct xilinx_vdma_tx_segment, node); 1633 desc->async_tx.phys = segment->phys; 1634 1635 return &desc->async_tx; 1636 1637 error: 1638 xilinx_dma_free_tx_descriptor(chan, desc); 1639 return NULL; 1640 } 1641 1642 /** 1643 * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction 1644 * @dchan: DMA channel 1645 * @dma_dst: destination address 1646 * @dma_src: source address 1647 * @len: transfer length 1648 * @flags: transfer ack flags 1649 * 1650 * Return: Async transaction descriptor on success and NULL on failure 1651 */ 1652 static struct dma_async_tx_descriptor * 1653 xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, 1654 dma_addr_t dma_src, size_t len, unsigned long flags) 1655 { 1656 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1657 struct xilinx_dma_tx_descriptor *desc; 1658 struct xilinx_cdma_tx_segment *segment, *prev; 1659 struct xilinx_cdma_desc_hw *hw; 1660 1661 if (!len || len > XILINX_DMA_MAX_TRANS_LEN) 1662 return NULL; 1663 1664 desc = xilinx_dma_alloc_tx_descriptor(chan); 1665 if (!desc) 1666 return NULL; 1667 1668 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 1669 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 1670 1671 /* Allocate the link descriptor from DMA pool */ 1672 segment = xilinx_cdma_alloc_tx_segment(chan); 1673 if (!segment) 1674 goto error; 1675 1676 hw = &segment->hw; 1677 hw->control = len; 1678 hw->src_addr = dma_src; 1679 hw->dest_addr = dma_dst; 1680 if (chan->ext_addr) { 1681 hw->src_addr_msb = upper_32_bits(dma_src); 1682 hw->dest_addr_msb = upper_32_bits(dma_dst); 1683 } 1684 1685 /* Fill the previous next descriptor with current */ 1686 prev = list_last_entry(&desc->segments, 1687 struct xilinx_cdma_tx_segment, node); 1688 prev->hw.next_desc = segment->phys; 1689 1690 /* Insert the segment into the descriptor segments list. */ 1691 list_add_tail(&segment->node, &desc->segments); 1692 1693 prev = segment; 1694 1695 /* Link the last hardware descriptor with the first. */ 1696 segment = list_first_entry(&desc->segments, 1697 struct xilinx_cdma_tx_segment, node); 1698 desc->async_tx.phys = segment->phys; 1699 prev->hw.next_desc = segment->phys; 1700 1701 return &desc->async_tx; 1702 1703 error: 1704 xilinx_dma_free_tx_descriptor(chan, desc); 1705 return NULL; 1706 } 1707 1708 /** 1709 * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 1710 * @dchan: DMA channel 1711 * @sgl: scatterlist to transfer to/from 1712 * @sg_len: number of entries in @scatterlist 1713 * @direction: DMA direction 1714 * @flags: transfer ack flags 1715 * @context: APP words of the descriptor 1716 * 1717 * Return: Async transaction descriptor on success and NULL on failure 1718 */ 1719 static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( 1720 struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, 1721 enum dma_transfer_direction direction, unsigned long flags, 1722 void *context) 1723 { 1724 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1725 struct xilinx_dma_tx_descriptor *desc; 1726 struct xilinx_axidma_tx_segment *segment = NULL, *prev = NULL; 1727 u32 *app_w = (u32 *)context; 1728 struct scatterlist *sg; 1729 size_t copy; 1730 size_t sg_used; 1731 unsigned int i; 1732 1733 if (!is_slave_direction(direction)) 1734 return NULL; 1735 1736 /* Allocate a transaction descriptor. */ 1737 desc = xilinx_dma_alloc_tx_descriptor(chan); 1738 if (!desc) 1739 return NULL; 1740 1741 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 1742 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 1743 1744 /* Build transactions using information in the scatter gather list */ 1745 for_each_sg(sgl, sg, sg_len, i) { 1746 sg_used = 0; 1747 1748 /* Loop until the entire scatterlist entry is used */ 1749 while (sg_used < sg_dma_len(sg)) { 1750 struct xilinx_axidma_desc_hw *hw; 1751 1752 /* Get a free segment */ 1753 segment = xilinx_axidma_alloc_tx_segment(chan); 1754 if (!segment) 1755 goto error; 1756 1757 /* 1758 * Calculate the maximum number of bytes to transfer, 1759 * making sure it is less than the hw limit 1760 */ 1761 copy = min_t(size_t, sg_dma_len(sg) - sg_used, 1762 XILINX_DMA_MAX_TRANS_LEN); 1763 hw = &segment->hw; 1764 1765 /* Fill in the descriptor */ 1766 xilinx_axidma_buf(chan, hw, sg_dma_address(sg), 1767 sg_used, 0); 1768 1769 hw->control = copy; 1770 1771 if (chan->direction == DMA_MEM_TO_DEV) { 1772 if (app_w) 1773 memcpy(hw->app, app_w, sizeof(u32) * 1774 XILINX_DMA_NUM_APP_WORDS); 1775 } 1776 1777 if (prev) 1778 prev->hw.next_desc = segment->phys; 1779 1780 prev = segment; 1781 sg_used += copy; 1782 1783 /* 1784 * Insert the segment into the descriptor segments 1785 * list. 1786 */ 1787 list_add_tail(&segment->node, &desc->segments); 1788 } 1789 } 1790 1791 segment = list_first_entry(&desc->segments, 1792 struct xilinx_axidma_tx_segment, node); 1793 desc->async_tx.phys = segment->phys; 1794 prev->hw.next_desc = segment->phys; 1795 1796 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 1797 if (chan->direction == DMA_MEM_TO_DEV) { 1798 segment->hw.control |= XILINX_DMA_BD_SOP; 1799 segment = list_last_entry(&desc->segments, 1800 struct xilinx_axidma_tx_segment, 1801 node); 1802 segment->hw.control |= XILINX_DMA_BD_EOP; 1803 } 1804 1805 return &desc->async_tx; 1806 1807 error: 1808 xilinx_dma_free_tx_descriptor(chan, desc); 1809 return NULL; 1810 } 1811 1812 /** 1813 * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction 1814 * @chan: DMA channel 1815 * @sgl: scatterlist to transfer to/from 1816 * @sg_len: number of entries in @scatterlist 1817 * @direction: DMA direction 1818 * @flags: transfer ack flags 1819 */ 1820 static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic( 1821 struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len, 1822 size_t period_len, enum dma_transfer_direction direction, 1823 unsigned long flags) 1824 { 1825 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1826 struct xilinx_dma_tx_descriptor *desc; 1827 struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL; 1828 size_t copy, sg_used; 1829 unsigned int num_periods; 1830 int i; 1831 u32 reg; 1832 1833 if (!period_len) 1834 return NULL; 1835 1836 num_periods = buf_len / period_len; 1837 1838 if (!num_periods) 1839 return NULL; 1840 1841 if (!is_slave_direction(direction)) 1842 return NULL; 1843 1844 /* Allocate a transaction descriptor. */ 1845 desc = xilinx_dma_alloc_tx_descriptor(chan); 1846 if (!desc) 1847 return NULL; 1848 1849 chan->direction = direction; 1850 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 1851 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 1852 1853 for (i = 0; i < num_periods; ++i) { 1854 sg_used = 0; 1855 1856 while (sg_used < period_len) { 1857 struct xilinx_axidma_desc_hw *hw; 1858 1859 /* Get a free segment */ 1860 segment = xilinx_axidma_alloc_tx_segment(chan); 1861 if (!segment) 1862 goto error; 1863 1864 /* 1865 * Calculate the maximum number of bytes to transfer, 1866 * making sure it is less than the hw limit 1867 */ 1868 copy = min_t(size_t, period_len - sg_used, 1869 XILINX_DMA_MAX_TRANS_LEN); 1870 hw = &segment->hw; 1871 xilinx_axidma_buf(chan, hw, buf_addr, sg_used, 1872 period_len * i); 1873 hw->control = copy; 1874 1875 if (prev) 1876 prev->hw.next_desc = segment->phys; 1877 1878 prev = segment; 1879 sg_used += copy; 1880 1881 /* 1882 * Insert the segment into the descriptor segments 1883 * list. 1884 */ 1885 list_add_tail(&segment->node, &desc->segments); 1886 } 1887 } 1888 1889 head_segment = list_first_entry(&desc->segments, 1890 struct xilinx_axidma_tx_segment, node); 1891 desc->async_tx.phys = head_segment->phys; 1892 1893 desc->cyclic = true; 1894 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 1895 reg |= XILINX_DMA_CR_CYCLIC_BD_EN_MASK; 1896 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 1897 1898 segment = list_last_entry(&desc->segments, 1899 struct xilinx_axidma_tx_segment, 1900 node); 1901 segment->hw.next_desc = (u32) head_segment->phys; 1902 1903 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 1904 if (direction == DMA_MEM_TO_DEV) { 1905 head_segment->hw.control |= XILINX_DMA_BD_SOP; 1906 segment->hw.control |= XILINX_DMA_BD_EOP; 1907 } 1908 1909 return &desc->async_tx; 1910 1911 error: 1912 xilinx_dma_free_tx_descriptor(chan, desc); 1913 return NULL; 1914 } 1915 1916 /** 1917 * xilinx_dma_prep_interleaved - prepare a descriptor for a 1918 * DMA_SLAVE transaction 1919 * @dchan: DMA channel 1920 * @xt: Interleaved template pointer 1921 * @flags: transfer ack flags 1922 * 1923 * Return: Async transaction descriptor on success and NULL on failure 1924 */ 1925 static struct dma_async_tx_descriptor * 1926 xilinx_dma_prep_interleaved(struct dma_chan *dchan, 1927 struct dma_interleaved_template *xt, 1928 unsigned long flags) 1929 { 1930 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 1931 struct xilinx_dma_tx_descriptor *desc; 1932 struct xilinx_axidma_tx_segment *segment; 1933 struct xilinx_axidma_desc_hw *hw; 1934 1935 if (!is_slave_direction(xt->dir)) 1936 return NULL; 1937 1938 if (!xt->numf || !xt->sgl[0].size) 1939 return NULL; 1940 1941 if (xt->frame_size != 1) 1942 return NULL; 1943 1944 /* Allocate a transaction descriptor. */ 1945 desc = xilinx_dma_alloc_tx_descriptor(chan); 1946 if (!desc) 1947 return NULL; 1948 1949 chan->direction = xt->dir; 1950 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); 1951 desc->async_tx.tx_submit = xilinx_dma_tx_submit; 1952 1953 /* Get a free segment */ 1954 segment = xilinx_axidma_alloc_tx_segment(chan); 1955 if (!segment) 1956 goto error; 1957 1958 hw = &segment->hw; 1959 1960 /* Fill in the descriptor */ 1961 if (xt->dir != DMA_MEM_TO_DEV) 1962 hw->buf_addr = xt->dst_start; 1963 else 1964 hw->buf_addr = xt->src_start; 1965 1966 hw->mcdma_control = chan->tdest & XILINX_DMA_BD_TDEST_MASK; 1967 hw->vsize_stride = (xt->numf << XILINX_DMA_BD_VSIZE_SHIFT) & 1968 XILINX_DMA_BD_VSIZE_MASK; 1969 hw->vsize_stride |= (xt->sgl[0].icg + xt->sgl[0].size) & 1970 XILINX_DMA_BD_STRIDE_MASK; 1971 hw->control = xt->sgl[0].size & XILINX_DMA_BD_HSIZE_MASK; 1972 1973 /* 1974 * Insert the segment into the descriptor segments 1975 * list. 1976 */ 1977 list_add_tail(&segment->node, &desc->segments); 1978 1979 1980 segment = list_first_entry(&desc->segments, 1981 struct xilinx_axidma_tx_segment, node); 1982 desc->async_tx.phys = segment->phys; 1983 1984 /* For the last DMA_MEM_TO_DEV transfer, set EOP */ 1985 if (xt->dir == DMA_MEM_TO_DEV) { 1986 segment->hw.control |= XILINX_DMA_BD_SOP; 1987 segment = list_last_entry(&desc->segments, 1988 struct xilinx_axidma_tx_segment, 1989 node); 1990 segment->hw.control |= XILINX_DMA_BD_EOP; 1991 } 1992 1993 return &desc->async_tx; 1994 1995 error: 1996 xilinx_dma_free_tx_descriptor(chan, desc); 1997 return NULL; 1998 } 1999 2000 /** 2001 * xilinx_dma_terminate_all - Halt the channel and free descriptors 2002 * @chan: Driver specific DMA Channel pointer 2003 */ 2004 static int xilinx_dma_terminate_all(struct dma_chan *dchan) 2005 { 2006 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2007 u32 reg; 2008 2009 if (chan->cyclic) 2010 xilinx_dma_chan_reset(chan); 2011 2012 /* Halt the DMA engine */ 2013 xilinx_dma_halt(chan); 2014 2015 /* Remove and free all of the descriptors in the lists */ 2016 xilinx_dma_free_descriptors(chan); 2017 2018 if (chan->cyclic) { 2019 reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 2020 reg &= ~XILINX_DMA_CR_CYCLIC_BD_EN_MASK; 2021 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); 2022 chan->cyclic = false; 2023 } 2024 2025 return 0; 2026 } 2027 2028 /** 2029 * xilinx_dma_channel_set_config - Configure VDMA channel 2030 * Run-time configuration for Axi VDMA, supports: 2031 * . halt the channel 2032 * . configure interrupt coalescing and inter-packet delay threshold 2033 * . start/stop parking 2034 * . enable genlock 2035 * 2036 * @dchan: DMA channel 2037 * @cfg: VDMA device configuration pointer 2038 * 2039 * Return: '0' on success and failure value on error 2040 */ 2041 int xilinx_vdma_channel_set_config(struct dma_chan *dchan, 2042 struct xilinx_vdma_config *cfg) 2043 { 2044 struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); 2045 u32 dmacr; 2046 2047 if (cfg->reset) 2048 return xilinx_dma_chan_reset(chan); 2049 2050 dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); 2051 2052 chan->config.frm_dly = cfg->frm_dly; 2053 chan->config.park = cfg->park; 2054 2055 /* genlock settings */ 2056 chan->config.gen_lock = cfg->gen_lock; 2057 chan->config.master = cfg->master; 2058 2059 if (cfg->gen_lock && chan->genlock) { 2060 dmacr |= XILINX_DMA_DMACR_GENLOCK_EN; 2061 dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT; 2062 } 2063 2064 chan->config.frm_cnt_en = cfg->frm_cnt_en; 2065 if (cfg->park) 2066 chan->config.park_frm = cfg->park_frm; 2067 else 2068 chan->config.park_frm = -1; 2069 2070 chan->config.coalesc = cfg->coalesc; 2071 chan->config.delay = cfg->delay; 2072 2073 if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) { 2074 dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT; 2075 chan->config.coalesc = cfg->coalesc; 2076 } 2077 2078 if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) { 2079 dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT; 2080 chan->config.delay = cfg->delay; 2081 } 2082 2083 /* FSync Source selection */ 2084 dmacr &= ~XILINX_DMA_DMACR_FSYNCSRC_MASK; 2085 dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT; 2086 2087 dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr); 2088 2089 return 0; 2090 } 2091 EXPORT_SYMBOL(xilinx_vdma_channel_set_config); 2092 2093 /* ----------------------------------------------------------------------------- 2094 * Probe and remove 2095 */ 2096 2097 /** 2098 * xilinx_dma_chan_remove - Per Channel remove function 2099 * @chan: Driver specific DMA channel 2100 */ 2101 static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan) 2102 { 2103 /* Disable all interrupts */ 2104 dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, 2105 XILINX_DMA_DMAXR_ALL_IRQ_MASK); 2106 2107 if (chan->irq > 0) 2108 free_irq(chan->irq, chan); 2109 2110 tasklet_kill(&chan->tasklet); 2111 2112 list_del(&chan->common.device_node); 2113 } 2114 2115 static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2116 struct clk **tx_clk, struct clk **rx_clk, 2117 struct clk **sg_clk, struct clk **tmp_clk) 2118 { 2119 int err; 2120 2121 *tmp_clk = NULL; 2122 2123 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2124 if (IS_ERR(*axi_clk)) { 2125 err = PTR_ERR(*axi_clk); 2126 dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err); 2127 return err; 2128 } 2129 2130 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); 2131 if (IS_ERR(*tx_clk)) 2132 *tx_clk = NULL; 2133 2134 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); 2135 if (IS_ERR(*rx_clk)) 2136 *rx_clk = NULL; 2137 2138 *sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk"); 2139 if (IS_ERR(*sg_clk)) 2140 *sg_clk = NULL; 2141 2142 err = clk_prepare_enable(*axi_clk); 2143 if (err) { 2144 dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); 2145 return err; 2146 } 2147 2148 err = clk_prepare_enable(*tx_clk); 2149 if (err) { 2150 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 2151 goto err_disable_axiclk; 2152 } 2153 2154 err = clk_prepare_enable(*rx_clk); 2155 if (err) { 2156 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 2157 goto err_disable_txclk; 2158 } 2159 2160 err = clk_prepare_enable(*sg_clk); 2161 if (err) { 2162 dev_err(&pdev->dev, "failed to enable sg_clk (%u)\n", err); 2163 goto err_disable_rxclk; 2164 } 2165 2166 return 0; 2167 2168 err_disable_rxclk: 2169 clk_disable_unprepare(*rx_clk); 2170 err_disable_txclk: 2171 clk_disable_unprepare(*tx_clk); 2172 err_disable_axiclk: 2173 clk_disable_unprepare(*axi_clk); 2174 2175 return err; 2176 } 2177 2178 static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2179 struct clk **dev_clk, struct clk **tmp_clk, 2180 struct clk **tmp1_clk, struct clk **tmp2_clk) 2181 { 2182 int err; 2183 2184 *tmp_clk = NULL; 2185 *tmp1_clk = NULL; 2186 *tmp2_clk = NULL; 2187 2188 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2189 if (IS_ERR(*axi_clk)) { 2190 err = PTR_ERR(*axi_clk); 2191 dev_err(&pdev->dev, "failed to get axi_clk (%u)\n", err); 2192 return err; 2193 } 2194 2195 *dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk"); 2196 if (IS_ERR(*dev_clk)) { 2197 err = PTR_ERR(*dev_clk); 2198 dev_err(&pdev->dev, "failed to get dev_clk (%u)\n", err); 2199 return err; 2200 } 2201 2202 err = clk_prepare_enable(*axi_clk); 2203 if (err) { 2204 dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); 2205 return err; 2206 } 2207 2208 err = clk_prepare_enable(*dev_clk); 2209 if (err) { 2210 dev_err(&pdev->dev, "failed to enable dev_clk (%u)\n", err); 2211 goto err_disable_axiclk; 2212 } 2213 2214 return 0; 2215 2216 err_disable_axiclk: 2217 clk_disable_unprepare(*axi_clk); 2218 2219 return err; 2220 } 2221 2222 static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, 2223 struct clk **tx_clk, struct clk **txs_clk, 2224 struct clk **rx_clk, struct clk **rxs_clk) 2225 { 2226 int err; 2227 2228 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); 2229 if (IS_ERR(*axi_clk)) { 2230 err = PTR_ERR(*axi_clk); 2231 dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err); 2232 return err; 2233 } 2234 2235 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); 2236 if (IS_ERR(*tx_clk)) 2237 *tx_clk = NULL; 2238 2239 *txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk"); 2240 if (IS_ERR(*txs_clk)) 2241 *txs_clk = NULL; 2242 2243 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); 2244 if (IS_ERR(*rx_clk)) 2245 *rx_clk = NULL; 2246 2247 *rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk"); 2248 if (IS_ERR(*rxs_clk)) 2249 *rxs_clk = NULL; 2250 2251 err = clk_prepare_enable(*axi_clk); 2252 if (err) { 2253 dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); 2254 return err; 2255 } 2256 2257 err = clk_prepare_enable(*tx_clk); 2258 if (err) { 2259 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 2260 goto err_disable_axiclk; 2261 } 2262 2263 err = clk_prepare_enable(*txs_clk); 2264 if (err) { 2265 dev_err(&pdev->dev, "failed to enable txs_clk (%u)\n", err); 2266 goto err_disable_txclk; 2267 } 2268 2269 err = clk_prepare_enable(*rx_clk); 2270 if (err) { 2271 dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); 2272 goto err_disable_txsclk; 2273 } 2274 2275 err = clk_prepare_enable(*rxs_clk); 2276 if (err) { 2277 dev_err(&pdev->dev, "failed to enable rxs_clk (%u)\n", err); 2278 goto err_disable_rxclk; 2279 } 2280 2281 return 0; 2282 2283 err_disable_rxclk: 2284 clk_disable_unprepare(*rx_clk); 2285 err_disable_txsclk: 2286 clk_disable_unprepare(*txs_clk); 2287 err_disable_txclk: 2288 clk_disable_unprepare(*tx_clk); 2289 err_disable_axiclk: 2290 clk_disable_unprepare(*axi_clk); 2291 2292 return err; 2293 } 2294 2295 static void xdma_disable_allclks(struct xilinx_dma_device *xdev) 2296 { 2297 clk_disable_unprepare(xdev->rxs_clk); 2298 clk_disable_unprepare(xdev->rx_clk); 2299 clk_disable_unprepare(xdev->txs_clk); 2300 clk_disable_unprepare(xdev->tx_clk); 2301 clk_disable_unprepare(xdev->axi_clk); 2302 } 2303 2304 /** 2305 * xilinx_dma_chan_probe - Per Channel Probing 2306 * It get channel features from the device tree entry and 2307 * initialize special channel handling routines 2308 * 2309 * @xdev: Driver specific device structure 2310 * @node: Device node 2311 * 2312 * Return: '0' on success and failure value on error 2313 */ 2314 static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, 2315 struct device_node *node, int chan_id) 2316 { 2317 struct xilinx_dma_chan *chan; 2318 bool has_dre = false; 2319 u32 value, width; 2320 int err; 2321 2322 /* Allocate and initialize the channel structure */ 2323 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); 2324 if (!chan) 2325 return -ENOMEM; 2326 2327 chan->dev = xdev->dev; 2328 chan->xdev = xdev; 2329 chan->has_sg = xdev->has_sg; 2330 chan->desc_pendingcount = 0x0; 2331 chan->ext_addr = xdev->ext_addr; 2332 2333 spin_lock_init(&chan->lock); 2334 INIT_LIST_HEAD(&chan->pending_list); 2335 INIT_LIST_HEAD(&chan->done_list); 2336 INIT_LIST_HEAD(&chan->active_list); 2337 2338 /* Retrieve the channel properties from the device tree */ 2339 has_dre = of_property_read_bool(node, "xlnx,include-dre"); 2340 2341 chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); 2342 2343 err = of_property_read_u32(node, "xlnx,datawidth", &value); 2344 if (err) { 2345 dev_err(xdev->dev, "missing xlnx,datawidth property\n"); 2346 return err; 2347 } 2348 width = value >> 3; /* Convert bits to bytes */ 2349 2350 /* If data width is greater than 8 bytes, DRE is not in hw */ 2351 if (width > 8) 2352 has_dre = false; 2353 2354 if (!has_dre) 2355 xdev->common.copy_align = fls(width - 1); 2356 2357 if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") || 2358 of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || 2359 of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { 2360 chan->direction = DMA_MEM_TO_DEV; 2361 chan->id = chan_id; 2362 chan->tdest = chan_id; 2363 2364 chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; 2365 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 2366 chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET; 2367 2368 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || 2369 xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S) 2370 chan->flush_on_fsync = true; 2371 } 2372 } else if (of_device_is_compatible(node, 2373 "xlnx,axi-vdma-s2mm-channel") || 2374 of_device_is_compatible(node, 2375 "xlnx,axi-dma-s2mm-channel")) { 2376 chan->direction = DMA_DEV_TO_MEM; 2377 chan->id = chan_id; 2378 chan->tdest = chan_id - xdev->nr_channels; 2379 2380 chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET; 2381 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 2382 chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET; 2383 2384 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || 2385 xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM) 2386 chan->flush_on_fsync = true; 2387 } 2388 } else { 2389 dev_err(xdev->dev, "Invalid channel compatible node\n"); 2390 return -EINVAL; 2391 } 2392 2393 /* Request the interrupt */ 2394 chan->irq = irq_of_parse_and_map(node, 0); 2395 err = request_irq(chan->irq, xilinx_dma_irq_handler, IRQF_SHARED, 2396 "xilinx-dma-controller", chan); 2397 if (err) { 2398 dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq); 2399 return err; 2400 } 2401 2402 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) 2403 chan->start_transfer = xilinx_dma_start_transfer; 2404 else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) 2405 chan->start_transfer = xilinx_cdma_start_transfer; 2406 else 2407 chan->start_transfer = xilinx_vdma_start_transfer; 2408 2409 /* Initialize the tasklet */ 2410 tasklet_init(&chan->tasklet, xilinx_dma_do_tasklet, 2411 (unsigned long)chan); 2412 2413 /* 2414 * Initialize the DMA channel and add it to the DMA engine channels 2415 * list. 2416 */ 2417 chan->common.device = &xdev->common; 2418 2419 list_add_tail(&chan->common.device_node, &xdev->common.channels); 2420 xdev->chan[chan->id] = chan; 2421 2422 /* Reset the channel */ 2423 err = xilinx_dma_chan_reset(chan); 2424 if (err < 0) { 2425 dev_err(xdev->dev, "Reset channel failed\n"); 2426 return err; 2427 } 2428 2429 return 0; 2430 } 2431 2432 /** 2433 * xilinx_dma_child_probe - Per child node probe 2434 * It get number of dma-channels per child node from 2435 * device-tree and initializes all the channels. 2436 * 2437 * @xdev: Driver specific device structure 2438 * @node: Device node 2439 * 2440 * Return: 0 always. 2441 */ 2442 static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, 2443 struct device_node *node) { 2444 int ret, i, nr_channels = 1; 2445 2446 ret = of_property_read_u32(node, "dma-channels", &nr_channels); 2447 if ((ret < 0) && xdev->mcdma) 2448 dev_warn(xdev->dev, "missing dma-channels property\n"); 2449 2450 for (i = 0; i < nr_channels; i++) 2451 xilinx_dma_chan_probe(xdev, node, xdev->chan_id++); 2452 2453 xdev->nr_channels += nr_channels; 2454 2455 return 0; 2456 } 2457 2458 /** 2459 * of_dma_xilinx_xlate - Translation function 2460 * @dma_spec: Pointer to DMA specifier as found in the device tree 2461 * @ofdma: Pointer to DMA controller data 2462 * 2463 * Return: DMA channel pointer on success and NULL on error 2464 */ 2465 static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, 2466 struct of_dma *ofdma) 2467 { 2468 struct xilinx_dma_device *xdev = ofdma->of_dma_data; 2469 int chan_id = dma_spec->args[0]; 2470 2471 if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id]) 2472 return NULL; 2473 2474 return dma_get_slave_channel(&xdev->chan[chan_id]->common); 2475 } 2476 2477 static const struct xilinx_dma_config axidma_config = { 2478 .dmatype = XDMA_TYPE_AXIDMA, 2479 .clk_init = axidma_clk_init, 2480 }; 2481 2482 static const struct xilinx_dma_config axicdma_config = { 2483 .dmatype = XDMA_TYPE_CDMA, 2484 .clk_init = axicdma_clk_init, 2485 }; 2486 2487 static const struct xilinx_dma_config axivdma_config = { 2488 .dmatype = XDMA_TYPE_VDMA, 2489 .clk_init = axivdma_clk_init, 2490 }; 2491 2492 static const struct of_device_id xilinx_dma_of_ids[] = { 2493 { .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config }, 2494 { .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config }, 2495 { .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config }, 2496 {} 2497 }; 2498 MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids); 2499 2500 /** 2501 * xilinx_dma_probe - Driver probe function 2502 * @pdev: Pointer to the platform_device structure 2503 * 2504 * Return: '0' on success and failure value on error 2505 */ 2506 static int xilinx_dma_probe(struct platform_device *pdev) 2507 { 2508 int (*clk_init)(struct platform_device *, struct clk **, struct clk **, 2509 struct clk **, struct clk **, struct clk **) 2510 = axivdma_clk_init; 2511 struct device_node *node = pdev->dev.of_node; 2512 struct xilinx_dma_device *xdev; 2513 struct device_node *child, *np = pdev->dev.of_node; 2514 struct resource *io; 2515 u32 num_frames, addr_width; 2516 int i, err; 2517 2518 /* Allocate and initialize the DMA engine structure */ 2519 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); 2520 if (!xdev) 2521 return -ENOMEM; 2522 2523 xdev->dev = &pdev->dev; 2524 if (np) { 2525 const struct of_device_id *match; 2526 2527 match = of_match_node(xilinx_dma_of_ids, np); 2528 if (match && match->data) { 2529 xdev->dma_config = match->data; 2530 clk_init = xdev->dma_config->clk_init; 2531 } 2532 } 2533 2534 err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk, 2535 &xdev->rx_clk, &xdev->rxs_clk); 2536 if (err) 2537 return err; 2538 2539 /* Request and map I/O memory */ 2540 io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2541 xdev->regs = devm_ioremap_resource(&pdev->dev, io); 2542 if (IS_ERR(xdev->regs)) 2543 return PTR_ERR(xdev->regs); 2544 2545 /* Retrieve the DMA engine properties from the device tree */ 2546 xdev->has_sg = of_property_read_bool(node, "xlnx,include-sg"); 2547 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) 2548 xdev->mcdma = of_property_read_bool(node, "xlnx,mcdma"); 2549 2550 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 2551 err = of_property_read_u32(node, "xlnx,num-fstores", 2552 &num_frames); 2553 if (err < 0) { 2554 dev_err(xdev->dev, 2555 "missing xlnx,num-fstores property\n"); 2556 return err; 2557 } 2558 2559 err = of_property_read_u32(node, "xlnx,flush-fsync", 2560 &xdev->flush_on_fsync); 2561 if (err < 0) 2562 dev_warn(xdev->dev, 2563 "missing xlnx,flush-fsync property\n"); 2564 } 2565 2566 err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width); 2567 if (err < 0) 2568 dev_warn(xdev->dev, "missing xlnx,addrwidth property\n"); 2569 2570 if (addr_width > 32) 2571 xdev->ext_addr = true; 2572 else 2573 xdev->ext_addr = false; 2574 2575 /* Set the dma mask bits */ 2576 dma_set_mask(xdev->dev, DMA_BIT_MASK(addr_width)); 2577 2578 /* Initialize the DMA engine */ 2579 xdev->common.dev = &pdev->dev; 2580 2581 INIT_LIST_HEAD(&xdev->common.channels); 2582 if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) { 2583 dma_cap_set(DMA_SLAVE, xdev->common.cap_mask); 2584 dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask); 2585 } 2586 2587 xdev->common.device_alloc_chan_resources = 2588 xilinx_dma_alloc_chan_resources; 2589 xdev->common.device_free_chan_resources = 2590 xilinx_dma_free_chan_resources; 2591 xdev->common.device_terminate_all = xilinx_dma_terminate_all; 2592 xdev->common.device_tx_status = xilinx_dma_tx_status; 2593 xdev->common.device_issue_pending = xilinx_dma_issue_pending; 2594 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { 2595 dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); 2596 xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; 2597 xdev->common.device_prep_dma_cyclic = 2598 xilinx_dma_prep_dma_cyclic; 2599 xdev->common.device_prep_interleaved_dma = 2600 xilinx_dma_prep_interleaved; 2601 /* Residue calculation is supported by only AXI DMA */ 2602 xdev->common.residue_granularity = 2603 DMA_RESIDUE_GRANULARITY_SEGMENT; 2604 } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { 2605 dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask); 2606 xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy; 2607 } else { 2608 xdev->common.device_prep_interleaved_dma = 2609 xilinx_vdma_dma_prep_interleaved; 2610 } 2611 2612 platform_set_drvdata(pdev, xdev); 2613 2614 /* Initialize the channels */ 2615 for_each_child_of_node(node, child) { 2616 err = xilinx_dma_child_probe(xdev, child); 2617 if (err < 0) 2618 goto disable_clks; 2619 } 2620 2621 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 2622 for (i = 0; i < xdev->nr_channels; i++) 2623 if (xdev->chan[i]) 2624 xdev->chan[i]->num_frms = num_frames; 2625 } 2626 2627 /* Register the DMA engine with the core */ 2628 dma_async_device_register(&xdev->common); 2629 2630 err = of_dma_controller_register(node, of_dma_xilinx_xlate, 2631 xdev); 2632 if (err < 0) { 2633 dev_err(&pdev->dev, "Unable to register DMA to DT\n"); 2634 dma_async_device_unregister(&xdev->common); 2635 goto error; 2636 } 2637 2638 dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n"); 2639 2640 return 0; 2641 2642 disable_clks: 2643 xdma_disable_allclks(xdev); 2644 error: 2645 for (i = 0; i < xdev->nr_channels; i++) 2646 if (xdev->chan[i]) 2647 xilinx_dma_chan_remove(xdev->chan[i]); 2648 2649 return err; 2650 } 2651 2652 /** 2653 * xilinx_dma_remove - Driver remove function 2654 * @pdev: Pointer to the platform_device structure 2655 * 2656 * Return: Always '0' 2657 */ 2658 static int xilinx_dma_remove(struct platform_device *pdev) 2659 { 2660 struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); 2661 int i; 2662 2663 of_dma_controller_free(pdev->dev.of_node); 2664 2665 dma_async_device_unregister(&xdev->common); 2666 2667 for (i = 0; i < xdev->nr_channels; i++) 2668 if (xdev->chan[i]) 2669 xilinx_dma_chan_remove(xdev->chan[i]); 2670 2671 xdma_disable_allclks(xdev); 2672 2673 return 0; 2674 } 2675 2676 static struct platform_driver xilinx_vdma_driver = { 2677 .driver = { 2678 .name = "xilinx-vdma", 2679 .of_match_table = xilinx_dma_of_ids, 2680 }, 2681 .probe = xilinx_dma_probe, 2682 .remove = xilinx_dma_remove, 2683 }; 2684 2685 module_platform_driver(xilinx_vdma_driver); 2686 2687 MODULE_AUTHOR("Xilinx, Inc."); 2688 MODULE_DESCRIPTION("Xilinx VDMA driver"); 2689 MODULE_LICENSE("GPL v2"); 2690