1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) STMicroelectronics SA 2017 5 * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com> 6 * Pierre-Yves Mordret <pierre-yves.mordret@st.com> 7 * 8 * Driver for STM32 MDMA controller 9 * 10 * Inspired by stm32-dma.c and dma-jz4780.c 11 */ 12 13 #include <linux/bitfield.h> 14 #include <linux/clk.h> 15 #include <linux/delay.h> 16 #include <linux/dmaengine.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/dmapool.h> 19 #include <linux/err.h> 20 #include <linux/init.h> 21 #include <linux/iopoll.h> 22 #include <linux/jiffies.h> 23 #include <linux/list.h> 24 #include <linux/log2.h> 25 #include <linux/module.h> 26 #include <linux/of.h> 27 #include <linux/of_device.h> 28 #include <linux/of_dma.h> 29 #include <linux/platform_device.h> 30 #include <linux/pm_runtime.h> 31 #include <linux/reset.h> 32 #include <linux/slab.h> 33 34 #include "virt-dma.h" 35 36 #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */ 37 38 /* MDMA Channel x interrupt/status register */ 39 #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */ 40 #define STM32_MDMA_CISR_CRQA BIT(16) 41 #define STM32_MDMA_CISR_TCIF BIT(4) 42 #define STM32_MDMA_CISR_BTIF BIT(3) 43 #define STM32_MDMA_CISR_BRTIF BIT(2) 44 #define STM32_MDMA_CISR_CTCIF BIT(1) 45 #define STM32_MDMA_CISR_TEIF BIT(0) 46 47 /* MDMA Channel x interrupt flag clear register */ 48 #define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x)) 49 #define STM32_MDMA_CIFCR_CLTCIF BIT(4) 50 #define STM32_MDMA_CIFCR_CBTIF BIT(3) 51 #define STM32_MDMA_CIFCR_CBRTIF BIT(2) 52 #define STM32_MDMA_CIFCR_CCTCIF BIT(1) 53 #define STM32_MDMA_CIFCR_CTEIF BIT(0) 54 #define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \ 55 | STM32_MDMA_CIFCR_CBTIF \ 56 | STM32_MDMA_CIFCR_CBRTIF \ 57 | STM32_MDMA_CIFCR_CCTCIF \ 58 | STM32_MDMA_CIFCR_CTEIF) 59 60 /* MDMA Channel x error status register */ 61 #define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x)) 62 #define STM32_MDMA_CESR_BSE BIT(11) 63 #define STM32_MDMA_CESR_ASR BIT(10) 64 #define STM32_MDMA_CESR_TEMD BIT(9) 65 #define STM32_MDMA_CESR_TELD BIT(8) 66 #define STM32_MDMA_CESR_TED BIT(7) 67 #define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0) 68 69 /* MDMA Channel x control register */ 70 #define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x)) 71 #define STM32_MDMA_CCR_SWRQ BIT(16) 72 #define STM32_MDMA_CCR_WEX BIT(14) 73 #define STM32_MDMA_CCR_HEX BIT(13) 74 #define STM32_MDMA_CCR_BEX BIT(12) 75 #define STM32_MDMA_CCR_SM BIT(8) 76 #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6) 77 #define STM32_MDMA_CCR_PL(n) FIELD_PREP(STM32_MDMA_CCR_PL_MASK, (n)) 78 #define STM32_MDMA_CCR_TCIE BIT(5) 79 #define STM32_MDMA_CCR_BTIE BIT(4) 80 #define STM32_MDMA_CCR_BRTIE BIT(3) 81 #define STM32_MDMA_CCR_CTCIE BIT(2) 82 #define STM32_MDMA_CCR_TEIE BIT(1) 83 #define STM32_MDMA_CCR_EN BIT(0) 84 #define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \ 85 | STM32_MDMA_CCR_BTIE \ 86 | STM32_MDMA_CCR_BRTIE \ 87 | STM32_MDMA_CCR_CTCIE \ 88 | STM32_MDMA_CCR_TEIE) 89 90 /* MDMA Channel x transfer configuration register */ 91 #define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x)) 92 #define STM32_MDMA_CTCR_BWM BIT(31) 93 #define STM32_MDMA_CTCR_SWRM BIT(30) 94 #define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28) 95 #define STM32_MDMA_CTCR_TRGM(n) FIELD_PREP(STM32_MDMA_CTCR_TRGM_MSK, (n)) 96 #define STM32_MDMA_CTCR_TRGM_GET(n) FIELD_GET(STM32_MDMA_CTCR_TRGM_MSK, (n)) 97 #define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26) 98 #define STM32_MDMA_CTCR_PAM(n) FIELD_PREP(STM32_MDMA_CTCR_PAM_MASK, (n)) 99 #define STM32_MDMA_CTCR_PKE BIT(25) 100 #define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18) 101 #define STM32_MDMA_CTCR_TLEN(n) FIELD_PREP(STM32_MDMA_CTCR_TLEN_MSK, (n)) 102 #define STM32_MDMA_CTCR_TLEN_GET(n) FIELD_GET(STM32_MDMA_CTCR_TLEN_MSK, (n)) 103 #define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18) 104 #define STM32_MDMA_CTCR_LEN2(n) FIELD_PREP(STM32_MDMA_CTCR_LEN2_MSK, (n)) 105 #define STM32_MDMA_CTCR_LEN2_GET(n) FIELD_GET(STM32_MDMA_CTCR_LEN2_MSK, (n)) 106 #define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15) 107 #define STM32_MDMA_CTCR_DBURST(n) FIELD_PREP(STM32_MDMA_CTCR_DBURST_MASK, (n)) 108 #define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12) 109 #define STM32_MDMA_CTCR_SBURST(n) FIELD_PREP(STM32_MDMA_CTCR_SBURST_MASK, (n)) 110 #define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10) 111 #define STM32_MDMA_CTCR_DINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_DINCOS_MASK, (n)) 112 #define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8) 113 #define STM32_MDMA_CTCR_SINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_SINCOS_MASK, (n)) 114 #define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6) 115 #define STM32_MDMA_CTCR_DSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_DSIZE_MASK, (n)) 116 #define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4) 117 #define STM32_MDMA_CTCR_SSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_SSIZE_MASK, (n)) 118 #define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2) 119 #define STM32_MDMA_CTCR_DINC(n) FIELD_PREP(STM32_MDMA_CTCR_DINC_MASK, (n)) 120 #define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0) 121 #define STM32_MDMA_CTCR_SINC(n) FIELD_PREP(STM32_MDMA_CTCR_SINC_MASK, (n)) 122 #define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \ 123 | STM32_MDMA_CTCR_DINC_MASK \ 124 | STM32_MDMA_CTCR_SINCOS_MASK \ 125 | STM32_MDMA_CTCR_DINCOS_MASK \ 126 | STM32_MDMA_CTCR_LEN2_MSK \ 127 | STM32_MDMA_CTCR_TRGM_MSK) 128 129 /* MDMA Channel x block number of data register */ 130 #define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x)) 131 #define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20) 132 #define STM32_MDMA_CBNDTR_BRC(n) FIELD_PREP(STM32_MDMA_CBNDTR_BRC_MK, (n)) 133 #define STM32_MDMA_CBNDTR_BRC_GET(n) FIELD_GET(STM32_MDMA_CBNDTR_BRC_MK, (n)) 134 135 #define STM32_MDMA_CBNDTR_BRDUM BIT(19) 136 #define STM32_MDMA_CBNDTR_BRSUM BIT(18) 137 #define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0) 138 #define STM32_MDMA_CBNDTR_BNDT(n) FIELD_PREP(STM32_MDMA_CBNDTR_BNDT_MASK, (n)) 139 140 /* MDMA Channel x source address register */ 141 #define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x)) 142 143 /* MDMA Channel x destination address register */ 144 #define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x)) 145 146 /* MDMA Channel x block repeat address update register */ 147 #define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x)) 148 #define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16) 149 #define STM32_MDMA_CBRUR_DUV(n) FIELD_PREP(STM32_MDMA_CBRUR_DUV_MASK, (n)) 150 #define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0) 151 #define STM32_MDMA_CBRUR_SUV(n) FIELD_PREP(STM32_MDMA_CBRUR_SUV_MASK, (n)) 152 153 /* MDMA Channel x link address register */ 154 #define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x)) 155 156 /* MDMA Channel x trigger and bus selection register */ 157 #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x)) 158 #define STM32_MDMA_CTBR_DBUS BIT(17) 159 #define STM32_MDMA_CTBR_SBUS BIT(16) 160 #define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0) 161 #define STM32_MDMA_CTBR_TSEL(n) FIELD_PREP(STM32_MDMA_CTBR_TSEL_MASK, (n)) 162 163 /* MDMA Channel x mask address register */ 164 #define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x)) 165 166 /* MDMA Channel x mask data register */ 167 #define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x)) 168 169 #define STM32_MDMA_MAX_BUF_LEN 128 170 #define STM32_MDMA_MAX_BLOCK_LEN 65536 171 #define STM32_MDMA_MAX_CHANNELS 32 172 #define STM32_MDMA_MAX_REQUESTS 256 173 #define STM32_MDMA_MAX_BURST 128 174 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x3 175 176 enum stm32_mdma_trigger_mode { 177 STM32_MDMA_BUFFER, 178 STM32_MDMA_BLOCK, 179 STM32_MDMA_BLOCK_REP, 180 STM32_MDMA_LINKED_LIST, 181 }; 182 183 enum stm32_mdma_width { 184 STM32_MDMA_BYTE, 185 STM32_MDMA_HALF_WORD, 186 STM32_MDMA_WORD, 187 STM32_MDMA_DOUBLE_WORD, 188 }; 189 190 enum stm32_mdma_inc_mode { 191 STM32_MDMA_FIXED = 0, 192 STM32_MDMA_INC = 2, 193 STM32_MDMA_DEC = 3, 194 }; 195 196 struct stm32_mdma_chan_config { 197 u32 request; 198 u32 priority_level; 199 u32 transfer_config; 200 u32 mask_addr; 201 u32 mask_data; 202 }; 203 204 struct stm32_mdma_hwdesc { 205 u32 ctcr; 206 u32 cbndtr; 207 u32 csar; 208 u32 cdar; 209 u32 cbrur; 210 u32 clar; 211 u32 ctbr; 212 u32 dummy; 213 u32 cmar; 214 u32 cmdr; 215 } __aligned(64); 216 217 struct stm32_mdma_desc_node { 218 struct stm32_mdma_hwdesc *hwdesc; 219 dma_addr_t hwdesc_phys; 220 }; 221 222 struct stm32_mdma_desc { 223 struct virt_dma_desc vdesc; 224 u32 ccr; 225 bool cyclic; 226 u32 count; 227 struct stm32_mdma_desc_node node[]; 228 }; 229 230 struct stm32_mdma_chan { 231 struct virt_dma_chan vchan; 232 struct dma_pool *desc_pool; 233 u32 id; 234 struct stm32_mdma_desc *desc; 235 u32 curr_hwdesc; 236 struct dma_slave_config dma_config; 237 struct stm32_mdma_chan_config chan_config; 238 bool busy; 239 u32 mem_burst; 240 u32 mem_width; 241 }; 242 243 struct stm32_mdma_device { 244 struct dma_device ddev; 245 void __iomem *base; 246 struct clk *clk; 247 int irq; 248 u32 nr_channels; 249 u32 nr_requests; 250 u32 nr_ahb_addr_masks; 251 u32 chan_reserved; 252 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS]; 253 u32 ahb_addr_masks[]; 254 }; 255 256 static struct stm32_mdma_device *stm32_mdma_get_dev( 257 struct stm32_mdma_chan *chan) 258 { 259 return container_of(chan->vchan.chan.device, struct stm32_mdma_device, 260 ddev); 261 } 262 263 static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c) 264 { 265 return container_of(c, struct stm32_mdma_chan, vchan.chan); 266 } 267 268 static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc) 269 { 270 return container_of(vdesc, struct stm32_mdma_desc, vdesc); 271 } 272 273 static struct device *chan2dev(struct stm32_mdma_chan *chan) 274 { 275 return &chan->vchan.chan.dev->device; 276 } 277 278 static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev) 279 { 280 return mdma_dev->ddev.dev; 281 } 282 283 static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg) 284 { 285 return readl_relaxed(dmadev->base + reg); 286 } 287 288 static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val) 289 { 290 writel_relaxed(val, dmadev->base + reg); 291 } 292 293 static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg, 294 u32 mask) 295 { 296 void __iomem *addr = dmadev->base + reg; 297 298 writel_relaxed(readl_relaxed(addr) | mask, addr); 299 } 300 301 static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg, 302 u32 mask) 303 { 304 void __iomem *addr = dmadev->base + reg; 305 306 writel_relaxed(readl_relaxed(addr) & ~mask, addr); 307 } 308 309 static struct stm32_mdma_desc *stm32_mdma_alloc_desc( 310 struct stm32_mdma_chan *chan, u32 count) 311 { 312 struct stm32_mdma_desc *desc; 313 int i; 314 315 desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT); 316 if (!desc) 317 return NULL; 318 319 for (i = 0; i < count; i++) { 320 desc->node[i].hwdesc = 321 dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, 322 &desc->node[i].hwdesc_phys); 323 if (!desc->node[i].hwdesc) 324 goto err; 325 } 326 327 desc->count = count; 328 329 return desc; 330 331 err: 332 dev_err(chan2dev(chan), "Failed to allocate descriptor\n"); 333 while (--i >= 0) 334 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc, 335 desc->node[i].hwdesc_phys); 336 kfree(desc); 337 return NULL; 338 } 339 340 static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc) 341 { 342 struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc); 343 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan); 344 int i; 345 346 for (i = 0; i < desc->count; i++) 347 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc, 348 desc->node[i].hwdesc_phys); 349 kfree(desc); 350 } 351 352 static int stm32_mdma_get_width(struct stm32_mdma_chan *chan, 353 enum dma_slave_buswidth width) 354 { 355 switch (width) { 356 case DMA_SLAVE_BUSWIDTH_1_BYTE: 357 case DMA_SLAVE_BUSWIDTH_2_BYTES: 358 case DMA_SLAVE_BUSWIDTH_4_BYTES: 359 case DMA_SLAVE_BUSWIDTH_8_BYTES: 360 return ffs(width) - 1; 361 default: 362 dev_err(chan2dev(chan), "Dma bus width %i not supported\n", 363 width); 364 return -EINVAL; 365 } 366 } 367 368 static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr, 369 u32 buf_len, u32 tlen) 370 { 371 enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES; 372 373 for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES; 374 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE; 375 max_width >>= 1) { 376 /* 377 * Address and buffer length both have to be aligned on 378 * bus width 379 */ 380 if ((((buf_len | addr) & (max_width - 1)) == 0) && 381 tlen >= max_width) 382 break; 383 } 384 385 return max_width; 386 } 387 388 static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst, 389 enum dma_slave_buswidth width) 390 { 391 u32 best_burst; 392 393 best_burst = min((u32)1 << __ffs(tlen | buf_len), 394 max_burst * width) / width; 395 396 return (best_burst > 0) ? best_burst : 1; 397 } 398 399 static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan) 400 { 401 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 402 u32 ccr, cisr, id, reg; 403 int ret; 404 405 id = chan->id; 406 reg = STM32_MDMA_CCR(id); 407 408 /* Disable interrupts */ 409 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK); 410 411 ccr = stm32_mdma_read(dmadev, reg); 412 if (ccr & STM32_MDMA_CCR_EN) { 413 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN); 414 415 /* Ensure that any ongoing transfer has been completed */ 416 ret = readl_relaxed_poll_timeout_atomic( 417 dmadev->base + STM32_MDMA_CISR(id), cisr, 418 (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000); 419 if (ret) { 420 dev_err(chan2dev(chan), "%s: timeout!\n", __func__); 421 return -EBUSY; 422 } 423 } 424 425 return 0; 426 } 427 428 static void stm32_mdma_stop(struct stm32_mdma_chan *chan) 429 { 430 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 431 u32 status; 432 int ret; 433 434 /* Disable DMA */ 435 ret = stm32_mdma_disable_chan(chan); 436 if (ret < 0) 437 return; 438 439 /* Clear interrupt status if it is there */ 440 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); 441 if (status) { 442 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n", 443 __func__, status); 444 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status); 445 } 446 447 chan->busy = false; 448 } 449 450 static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr, 451 u32 ctbr_mask, u32 src_addr) 452 { 453 u32 mask; 454 int i; 455 456 /* Check if memory device is on AHB or AXI */ 457 *ctbr &= ~ctbr_mask; 458 mask = src_addr & 0xF0000000; 459 for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) { 460 if (mask == dmadev->ahb_addr_masks[i]) { 461 *ctbr |= ctbr_mask; 462 break; 463 } 464 } 465 } 466 467 static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, 468 enum dma_transfer_direction direction, 469 u32 *mdma_ccr, u32 *mdma_ctcr, 470 u32 *mdma_ctbr, dma_addr_t addr, 471 u32 buf_len) 472 { 473 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 474 struct stm32_mdma_chan_config *chan_config = &chan->chan_config; 475 enum dma_slave_buswidth src_addr_width, dst_addr_width; 476 phys_addr_t src_addr, dst_addr; 477 int src_bus_width, dst_bus_width; 478 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst; 479 u32 ccr, ctcr, ctbr, tlen; 480 481 src_addr_width = chan->dma_config.src_addr_width; 482 dst_addr_width = chan->dma_config.dst_addr_width; 483 src_maxburst = chan->dma_config.src_maxburst; 484 dst_maxburst = chan->dma_config.dst_maxburst; 485 486 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); 487 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); 488 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); 489 490 /* Enable HW request mode */ 491 ctcr &= ~STM32_MDMA_CTCR_SWRM; 492 493 /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */ 494 ctcr &= ~STM32_MDMA_CTCR_CFG_MASK; 495 ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK; 496 497 /* 498 * For buffer transfer length (TLEN) we have to set 499 * the number of bytes - 1 in CTCR register 500 */ 501 tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr); 502 ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK; 503 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1)); 504 505 /* Disable Pack Enable */ 506 ctcr &= ~STM32_MDMA_CTCR_PKE; 507 508 /* Check burst size constraints */ 509 if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST || 510 dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) { 511 dev_err(chan2dev(chan), 512 "burst size * bus width higher than %d bytes\n", 513 STM32_MDMA_MAX_BURST); 514 return -EINVAL; 515 } 516 517 if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) || 518 (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) { 519 dev_err(chan2dev(chan), "burst size must be a power of 2\n"); 520 return -EINVAL; 521 } 522 523 /* 524 * Configure channel control: 525 * - Clear SW request as in this case this is a HW one 526 * - Clear WEX, HEX and BEX bits 527 * - Set priority level 528 */ 529 ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX | 530 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK); 531 ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level); 532 533 /* Configure Trigger selection */ 534 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK; 535 ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request); 536 537 switch (direction) { 538 case DMA_MEM_TO_DEV: 539 dst_addr = chan->dma_config.dst_addr; 540 541 /* Set device data size */ 542 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); 543 if (dst_bus_width < 0) 544 return dst_bus_width; 545 ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK; 546 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width); 547 548 /* Set device burst value */ 549 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, 550 dst_maxburst, 551 dst_addr_width); 552 chan->mem_burst = dst_best_burst; 553 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK; 554 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst))); 555 556 /* Set memory data size */ 557 src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen); 558 chan->mem_width = src_addr_width; 559 src_bus_width = stm32_mdma_get_width(chan, src_addr_width); 560 if (src_bus_width < 0) 561 return src_bus_width; 562 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK | 563 STM32_MDMA_CTCR_SINCOS_MASK; 564 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) | 565 STM32_MDMA_CTCR_SINCOS(src_bus_width); 566 567 /* Set memory burst value */ 568 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width; 569 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, 570 src_maxburst, 571 src_addr_width); 572 chan->mem_burst = src_best_burst; 573 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK; 574 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst))); 575 576 /* Select bus */ 577 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, 578 dst_addr); 579 580 if (dst_bus_width != src_bus_width) 581 ctcr |= STM32_MDMA_CTCR_PKE; 582 583 /* Set destination address */ 584 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr); 585 break; 586 587 case DMA_DEV_TO_MEM: 588 src_addr = chan->dma_config.src_addr; 589 590 /* Set device data size */ 591 src_bus_width = stm32_mdma_get_width(chan, src_addr_width); 592 if (src_bus_width < 0) 593 return src_bus_width; 594 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK; 595 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width); 596 597 /* Set device burst value */ 598 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, 599 src_maxburst, 600 src_addr_width); 601 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK; 602 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst))); 603 604 /* Set memory data size */ 605 dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen); 606 chan->mem_width = dst_addr_width; 607 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); 608 if (dst_bus_width < 0) 609 return dst_bus_width; 610 ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK | 611 STM32_MDMA_CTCR_DINCOS_MASK); 612 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) | 613 STM32_MDMA_CTCR_DINCOS(dst_bus_width); 614 615 /* Set memory burst value */ 616 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width; 617 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, 618 dst_maxburst, 619 dst_addr_width); 620 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK; 621 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst))); 622 623 /* Select bus */ 624 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, 625 src_addr); 626 627 if (dst_bus_width != src_bus_width) 628 ctcr |= STM32_MDMA_CTCR_PKE; 629 630 /* Set source address */ 631 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr); 632 break; 633 634 default: 635 dev_err(chan2dev(chan), "Dma direction is not supported\n"); 636 return -EINVAL; 637 } 638 639 *mdma_ccr = ccr; 640 *mdma_ctcr = ctcr; 641 *mdma_ctbr = ctbr; 642 643 return 0; 644 } 645 646 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan, 647 struct stm32_mdma_desc_node *node) 648 { 649 dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys); 650 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr); 651 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr); 652 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar); 653 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar); 654 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur); 655 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar); 656 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr); 657 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar); 658 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr); 659 } 660 661 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan, 662 struct stm32_mdma_desc *desc, 663 enum dma_transfer_direction dir, u32 count, 664 dma_addr_t src_addr, dma_addr_t dst_addr, 665 u32 len, u32 ctcr, u32 ctbr, bool is_last, 666 bool is_first, bool is_cyclic) 667 { 668 struct stm32_mdma_chan_config *config = &chan->chan_config; 669 struct stm32_mdma_hwdesc *hwdesc; 670 u32 next = count + 1; 671 672 hwdesc = desc->node[count].hwdesc; 673 hwdesc->ctcr = ctcr; 674 hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | 675 STM32_MDMA_CBNDTR_BRDUM | 676 STM32_MDMA_CBNDTR_BRSUM | 677 STM32_MDMA_CBNDTR_BNDT_MASK); 678 hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len); 679 hwdesc->csar = src_addr; 680 hwdesc->cdar = dst_addr; 681 hwdesc->cbrur = 0; 682 hwdesc->ctbr = ctbr; 683 hwdesc->cmar = config->mask_addr; 684 hwdesc->cmdr = config->mask_data; 685 686 if (is_last) { 687 if (is_cyclic) 688 hwdesc->clar = desc->node[0].hwdesc_phys; 689 else 690 hwdesc->clar = 0; 691 } else { 692 hwdesc->clar = desc->node[next].hwdesc_phys; 693 } 694 695 stm32_mdma_dump_hwdesc(chan, &desc->node[count]); 696 } 697 698 static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, 699 struct stm32_mdma_desc *desc, 700 struct scatterlist *sgl, u32 sg_len, 701 enum dma_transfer_direction direction) 702 { 703 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 704 struct dma_slave_config *dma_config = &chan->dma_config; 705 struct scatterlist *sg; 706 dma_addr_t src_addr, dst_addr; 707 u32 ccr, ctcr, ctbr; 708 int i, ret = 0; 709 710 for_each_sg(sgl, sg, sg_len, i) { 711 if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) { 712 dev_err(chan2dev(chan), "Invalid block len\n"); 713 return -EINVAL; 714 } 715 716 if (direction == DMA_MEM_TO_DEV) { 717 src_addr = sg_dma_address(sg); 718 dst_addr = dma_config->dst_addr; 719 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, 720 &ctcr, &ctbr, src_addr, 721 sg_dma_len(sg)); 722 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, 723 src_addr); 724 } else { 725 src_addr = dma_config->src_addr; 726 dst_addr = sg_dma_address(sg); 727 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, 728 &ctcr, &ctbr, dst_addr, 729 sg_dma_len(sg)); 730 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, 731 dst_addr); 732 } 733 734 if (ret < 0) 735 return ret; 736 737 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr, 738 dst_addr, sg_dma_len(sg), ctcr, ctbr, 739 i == sg_len - 1, i == 0, false); 740 } 741 742 /* Enable interrupts */ 743 ccr &= ~STM32_MDMA_CCR_IRQ_MASK; 744 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE; 745 if (sg_len > 1) 746 ccr |= STM32_MDMA_CCR_BTIE; 747 desc->ccr = ccr; 748 749 return 0; 750 } 751 752 static struct dma_async_tx_descriptor * 753 stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, 754 u32 sg_len, enum dma_transfer_direction direction, 755 unsigned long flags, void *context) 756 { 757 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 758 struct stm32_mdma_desc *desc; 759 int i, ret; 760 761 /* 762 * Once DMA is in setup cyclic mode the channel we cannot assign this 763 * channel anymore. The DMA channel needs to be aborted or terminated 764 * for allowing another request. 765 */ 766 if (chan->desc && chan->desc->cyclic) { 767 dev_err(chan2dev(chan), 768 "Request not allowed when dma in cyclic mode\n"); 769 return NULL; 770 } 771 772 desc = stm32_mdma_alloc_desc(chan, sg_len); 773 if (!desc) 774 return NULL; 775 776 ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction); 777 if (ret < 0) 778 goto xfer_setup_err; 779 780 desc->cyclic = false; 781 782 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 783 784 xfer_setup_err: 785 for (i = 0; i < desc->count; i++) 786 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc, 787 desc->node[i].hwdesc_phys); 788 kfree(desc); 789 return NULL; 790 } 791 792 static struct dma_async_tx_descriptor * 793 stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, 794 size_t buf_len, size_t period_len, 795 enum dma_transfer_direction direction, 796 unsigned long flags) 797 { 798 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 799 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 800 struct dma_slave_config *dma_config = &chan->dma_config; 801 struct stm32_mdma_desc *desc; 802 dma_addr_t src_addr, dst_addr; 803 u32 ccr, ctcr, ctbr, count; 804 int i, ret; 805 806 /* 807 * Once DMA is in setup cyclic mode the channel we cannot assign this 808 * channel anymore. The DMA channel needs to be aborted or terminated 809 * for allowing another request. 810 */ 811 if (chan->desc && chan->desc->cyclic) { 812 dev_err(chan2dev(chan), 813 "Request not allowed when dma in cyclic mode\n"); 814 return NULL; 815 } 816 817 if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) { 818 dev_err(chan2dev(chan), "Invalid buffer/period len\n"); 819 return NULL; 820 } 821 822 if (buf_len % period_len) { 823 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n"); 824 return NULL; 825 } 826 827 count = buf_len / period_len; 828 829 desc = stm32_mdma_alloc_desc(chan, count); 830 if (!desc) 831 return NULL; 832 833 /* Select bus */ 834 if (direction == DMA_MEM_TO_DEV) { 835 src_addr = buf_addr; 836 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, 837 &ctbr, src_addr, period_len); 838 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, 839 src_addr); 840 } else { 841 dst_addr = buf_addr; 842 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, 843 &ctbr, dst_addr, period_len); 844 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, 845 dst_addr); 846 } 847 848 if (ret < 0) 849 goto xfer_setup_err; 850 851 /* Enable interrupts */ 852 ccr &= ~STM32_MDMA_CCR_IRQ_MASK; 853 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE; 854 desc->ccr = ccr; 855 856 /* Configure hwdesc list */ 857 for (i = 0; i < count; i++) { 858 if (direction == DMA_MEM_TO_DEV) { 859 src_addr = buf_addr + i * period_len; 860 dst_addr = dma_config->dst_addr; 861 } else { 862 src_addr = dma_config->src_addr; 863 dst_addr = buf_addr + i * period_len; 864 } 865 866 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr, 867 dst_addr, period_len, ctcr, ctbr, 868 i == count - 1, i == 0, true); 869 } 870 871 desc->cyclic = true; 872 873 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 874 875 xfer_setup_err: 876 for (i = 0; i < desc->count; i++) 877 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc, 878 desc->node[i].hwdesc_phys); 879 kfree(desc); 880 return NULL; 881 } 882 883 static struct dma_async_tx_descriptor * 884 stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src, 885 size_t len, unsigned long flags) 886 { 887 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 888 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 889 enum dma_slave_buswidth max_width; 890 struct stm32_mdma_desc *desc; 891 struct stm32_mdma_hwdesc *hwdesc; 892 u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst; 893 u32 best_burst, tlen; 894 size_t xfer_count, offset; 895 int src_bus_width, dst_bus_width; 896 int i; 897 898 /* 899 * Once DMA is in setup cyclic mode the channel we cannot assign this 900 * channel anymore. The DMA channel needs to be aborted or terminated 901 * to allow another request 902 */ 903 if (chan->desc && chan->desc->cyclic) { 904 dev_err(chan2dev(chan), 905 "Request not allowed when dma in cyclic mode\n"); 906 return NULL; 907 } 908 909 count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN); 910 desc = stm32_mdma_alloc_desc(chan, count); 911 if (!desc) 912 return NULL; 913 914 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); 915 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); 916 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); 917 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); 918 919 /* Enable sw req, some interrupts and clear other bits */ 920 ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX | 921 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK | 922 STM32_MDMA_CCR_IRQ_MASK); 923 ccr |= STM32_MDMA_CCR_TEIE; 924 925 /* Enable SW request mode, dest/src inc and clear other bits */ 926 ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK | 927 STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE | 928 STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK | 929 STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK | 930 STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK | 931 STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK | 932 STM32_MDMA_CTCR_SINC_MASK); 933 ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) | 934 STM32_MDMA_CTCR_DINC(STM32_MDMA_INC); 935 936 /* Reset HW request */ 937 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK; 938 939 /* Select bus */ 940 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, src); 941 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, dest); 942 943 /* Clear CBNDTR registers */ 944 cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM | 945 STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK); 946 947 if (len <= STM32_MDMA_MAX_BLOCK_LEN) { 948 cbndtr |= STM32_MDMA_CBNDTR_BNDT(len); 949 if (len <= STM32_MDMA_MAX_BUF_LEN) { 950 /* Setup a buffer transfer */ 951 ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE; 952 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER); 953 } else { 954 /* Setup a block transfer */ 955 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE; 956 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK); 957 } 958 959 tlen = STM32_MDMA_MAX_BUF_LEN; 960 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1)); 961 962 /* Set source best burst size */ 963 max_width = stm32_mdma_get_max_width(src, len, tlen); 964 src_bus_width = stm32_mdma_get_width(chan, max_width); 965 966 max_burst = tlen / max_width; 967 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst, 968 max_width); 969 mdma_burst = ilog2(best_burst); 970 971 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) | 972 STM32_MDMA_CTCR_SSIZE(src_bus_width) | 973 STM32_MDMA_CTCR_SINCOS(src_bus_width); 974 975 /* Set destination best burst size */ 976 max_width = stm32_mdma_get_max_width(dest, len, tlen); 977 dst_bus_width = stm32_mdma_get_width(chan, max_width); 978 979 max_burst = tlen / max_width; 980 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst, 981 max_width); 982 mdma_burst = ilog2(best_burst); 983 984 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) | 985 STM32_MDMA_CTCR_DSIZE(dst_bus_width) | 986 STM32_MDMA_CTCR_DINCOS(dst_bus_width); 987 988 if (dst_bus_width != src_bus_width) 989 ctcr |= STM32_MDMA_CTCR_PKE; 990 991 /* Prepare hardware descriptor */ 992 hwdesc = desc->node[0].hwdesc; 993 hwdesc->ctcr = ctcr; 994 hwdesc->cbndtr = cbndtr; 995 hwdesc->csar = src; 996 hwdesc->cdar = dest; 997 hwdesc->cbrur = 0; 998 hwdesc->clar = 0; 999 hwdesc->ctbr = ctbr; 1000 hwdesc->cmar = 0; 1001 hwdesc->cmdr = 0; 1002 1003 stm32_mdma_dump_hwdesc(chan, &desc->node[0]); 1004 } else { 1005 /* Setup a LLI transfer */ 1006 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) | 1007 STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1)); 1008 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE; 1009 tlen = STM32_MDMA_MAX_BUF_LEN; 1010 1011 for (i = 0, offset = 0; offset < len; 1012 i++, offset += xfer_count) { 1013 xfer_count = min_t(size_t, len - offset, 1014 STM32_MDMA_MAX_BLOCK_LEN); 1015 1016 /* Set source best burst size */ 1017 max_width = stm32_mdma_get_max_width(src, len, tlen); 1018 src_bus_width = stm32_mdma_get_width(chan, max_width); 1019 1020 max_burst = tlen / max_width; 1021 best_burst = stm32_mdma_get_best_burst(len, tlen, 1022 max_burst, 1023 max_width); 1024 mdma_burst = ilog2(best_burst); 1025 1026 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) | 1027 STM32_MDMA_CTCR_SSIZE(src_bus_width) | 1028 STM32_MDMA_CTCR_SINCOS(src_bus_width); 1029 1030 /* Set destination best burst size */ 1031 max_width = stm32_mdma_get_max_width(dest, len, tlen); 1032 dst_bus_width = stm32_mdma_get_width(chan, max_width); 1033 1034 max_burst = tlen / max_width; 1035 best_burst = stm32_mdma_get_best_burst(len, tlen, 1036 max_burst, 1037 max_width); 1038 mdma_burst = ilog2(best_burst); 1039 1040 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) | 1041 STM32_MDMA_CTCR_DSIZE(dst_bus_width) | 1042 STM32_MDMA_CTCR_DINCOS(dst_bus_width); 1043 1044 if (dst_bus_width != src_bus_width) 1045 ctcr |= STM32_MDMA_CTCR_PKE; 1046 1047 /* Prepare hardware descriptor */ 1048 stm32_mdma_setup_hwdesc(chan, desc, DMA_MEM_TO_MEM, i, 1049 src + offset, dest + offset, 1050 xfer_count, ctcr, ctbr, 1051 i == count - 1, i == 0, false); 1052 } 1053 } 1054 1055 desc->ccr = ccr; 1056 1057 desc->cyclic = false; 1058 1059 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 1060 } 1061 1062 static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan) 1063 { 1064 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1065 1066 dev_dbg(chan2dev(chan), "CCR: 0x%08x\n", 1067 stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id))); 1068 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", 1069 stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id))); 1070 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", 1071 stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id))); 1072 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", 1073 stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id))); 1074 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", 1075 stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id))); 1076 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", 1077 stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id))); 1078 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", 1079 stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id))); 1080 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", 1081 stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id))); 1082 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", 1083 stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id))); 1084 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n", 1085 stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id))); 1086 } 1087 1088 static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) 1089 { 1090 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1091 struct virt_dma_desc *vdesc; 1092 struct stm32_mdma_hwdesc *hwdesc; 1093 u32 id = chan->id; 1094 u32 status, reg; 1095 1096 vdesc = vchan_next_desc(&chan->vchan); 1097 if (!vdesc) { 1098 chan->desc = NULL; 1099 return; 1100 } 1101 1102 list_del(&vdesc->node); 1103 1104 chan->desc = to_stm32_mdma_desc(vdesc); 1105 hwdesc = chan->desc->node[0].hwdesc; 1106 chan->curr_hwdesc = 0; 1107 1108 stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr); 1109 stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), hwdesc->ctcr); 1110 stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), hwdesc->cbndtr); 1111 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), hwdesc->csar); 1112 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), hwdesc->cdar); 1113 stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), hwdesc->cbrur); 1114 stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), hwdesc->clar); 1115 stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), hwdesc->ctbr); 1116 stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), hwdesc->cmar); 1117 stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), hwdesc->cmdr); 1118 1119 /* Clear interrupt status if it is there */ 1120 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id)); 1121 if (status) 1122 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), status); 1123 1124 stm32_mdma_dump_reg(chan); 1125 1126 /* Start DMA */ 1127 stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN); 1128 1129 /* Set SW request in case of MEM2MEM transfer */ 1130 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) { 1131 reg = STM32_MDMA_CCR(id); 1132 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ); 1133 } 1134 1135 chan->busy = true; 1136 1137 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); 1138 } 1139 1140 static void stm32_mdma_issue_pending(struct dma_chan *c) 1141 { 1142 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1143 unsigned long flags; 1144 1145 spin_lock_irqsave(&chan->vchan.lock, flags); 1146 1147 if (!vchan_issue_pending(&chan->vchan)) 1148 goto end; 1149 1150 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); 1151 1152 if (!chan->desc && !chan->busy) 1153 stm32_mdma_start_transfer(chan); 1154 1155 end: 1156 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1157 } 1158 1159 static int stm32_mdma_pause(struct dma_chan *c) 1160 { 1161 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1162 unsigned long flags; 1163 int ret; 1164 1165 spin_lock_irqsave(&chan->vchan.lock, flags); 1166 ret = stm32_mdma_disable_chan(chan); 1167 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1168 1169 if (!ret) 1170 dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan); 1171 1172 return ret; 1173 } 1174 1175 static int stm32_mdma_resume(struct dma_chan *c) 1176 { 1177 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1178 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1179 struct stm32_mdma_hwdesc *hwdesc; 1180 unsigned long flags; 1181 u32 status, reg; 1182 1183 hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc; 1184 1185 spin_lock_irqsave(&chan->vchan.lock, flags); 1186 1187 /* Re-configure control register */ 1188 stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), chan->desc->ccr); 1189 1190 /* Clear interrupt status if it is there */ 1191 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); 1192 if (status) 1193 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status); 1194 1195 stm32_mdma_dump_reg(chan); 1196 1197 /* Re-start DMA */ 1198 reg = STM32_MDMA_CCR(chan->id); 1199 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN); 1200 1201 /* Set SW request in case of MEM2MEM transfer */ 1202 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) 1203 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ); 1204 1205 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1206 1207 dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan); 1208 1209 return 0; 1210 } 1211 1212 static int stm32_mdma_terminate_all(struct dma_chan *c) 1213 { 1214 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1215 unsigned long flags; 1216 LIST_HEAD(head); 1217 1218 spin_lock_irqsave(&chan->vchan.lock, flags); 1219 if (chan->desc) { 1220 vchan_terminate_vdesc(&chan->desc->vdesc); 1221 if (chan->busy) 1222 stm32_mdma_stop(chan); 1223 chan->desc = NULL; 1224 } 1225 vchan_get_all_descriptors(&chan->vchan, &head); 1226 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1227 1228 vchan_dma_desc_free_list(&chan->vchan, &head); 1229 1230 return 0; 1231 } 1232 1233 static void stm32_mdma_synchronize(struct dma_chan *c) 1234 { 1235 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1236 1237 vchan_synchronize(&chan->vchan); 1238 } 1239 1240 static int stm32_mdma_slave_config(struct dma_chan *c, 1241 struct dma_slave_config *config) 1242 { 1243 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1244 1245 memcpy(&chan->dma_config, config, sizeof(*config)); 1246 1247 return 0; 1248 } 1249 1250 static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan, 1251 struct stm32_mdma_desc *desc, 1252 u32 curr_hwdesc) 1253 { 1254 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1255 struct stm32_mdma_hwdesc *hwdesc; 1256 u32 cbndtr, residue, modulo, burst_size; 1257 int i; 1258 1259 residue = 0; 1260 for (i = curr_hwdesc + 1; i < desc->count; i++) { 1261 hwdesc = desc->node[i].hwdesc; 1262 residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr); 1263 } 1264 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); 1265 residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK; 1266 1267 if (!chan->mem_burst) 1268 return residue; 1269 1270 burst_size = chan->mem_burst * chan->mem_width; 1271 modulo = residue % burst_size; 1272 if (modulo) 1273 residue = residue - modulo + burst_size; 1274 1275 return residue; 1276 } 1277 1278 static enum dma_status stm32_mdma_tx_status(struct dma_chan *c, 1279 dma_cookie_t cookie, 1280 struct dma_tx_state *state) 1281 { 1282 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1283 struct virt_dma_desc *vdesc; 1284 enum dma_status status; 1285 unsigned long flags; 1286 u32 residue = 0; 1287 1288 status = dma_cookie_status(c, cookie, state); 1289 if ((status == DMA_COMPLETE) || (!state)) 1290 return status; 1291 1292 spin_lock_irqsave(&chan->vchan.lock, flags); 1293 1294 vdesc = vchan_find_desc(&chan->vchan, cookie); 1295 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) 1296 residue = stm32_mdma_desc_residue(chan, chan->desc, 1297 chan->curr_hwdesc); 1298 else if (vdesc) 1299 residue = stm32_mdma_desc_residue(chan, 1300 to_stm32_mdma_desc(vdesc), 0); 1301 dma_set_residue(state, residue); 1302 1303 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1304 1305 return status; 1306 } 1307 1308 static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) 1309 { 1310 vchan_cookie_complete(&chan->desc->vdesc); 1311 chan->desc = NULL; 1312 chan->busy = false; 1313 1314 /* Start the next transfer if this driver has a next desc */ 1315 stm32_mdma_start_transfer(chan); 1316 } 1317 1318 static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) 1319 { 1320 struct stm32_mdma_device *dmadev = devid; 1321 struct stm32_mdma_chan *chan; 1322 u32 reg, id, ccr, ien, status; 1323 1324 /* Find out which channel generates the interrupt */ 1325 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0); 1326 if (!status) { 1327 dev_dbg(mdma2dev(dmadev), "spurious it\n"); 1328 return IRQ_NONE; 1329 } 1330 id = __ffs(status); 1331 chan = &dmadev->chan[id]; 1332 1333 /* Handle interrupt for the channel */ 1334 spin_lock(&chan->vchan.lock); 1335 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id)); 1336 /* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */ 1337 status &= ~STM32_MDMA_CISR_CRQA; 1338 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); 1339 ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1; 1340 1341 if (!(status & ien)) { 1342 spin_unlock(&chan->vchan.lock); 1343 if (chan->busy) 1344 dev_warn(chan2dev(chan), 1345 "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); 1346 else 1347 dev_dbg(chan2dev(chan), 1348 "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); 1349 return IRQ_NONE; 1350 } 1351 1352 reg = STM32_MDMA_CIFCR(id); 1353 1354 if (status & STM32_MDMA_CISR_TEIF) { 1355 dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", 1356 readl_relaxed(dmadev->base + STM32_MDMA_CESR(id))); 1357 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF); 1358 status &= ~STM32_MDMA_CISR_TEIF; 1359 } 1360 1361 if (status & STM32_MDMA_CISR_CTCIF) { 1362 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF); 1363 status &= ~STM32_MDMA_CISR_CTCIF; 1364 stm32_mdma_xfer_end(chan); 1365 } 1366 1367 if (status & STM32_MDMA_CISR_BRTIF) { 1368 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF); 1369 status &= ~STM32_MDMA_CISR_BRTIF; 1370 } 1371 1372 if (status & STM32_MDMA_CISR_BTIF) { 1373 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF); 1374 status &= ~STM32_MDMA_CISR_BTIF; 1375 chan->curr_hwdesc++; 1376 if (chan->desc && chan->desc->cyclic) { 1377 if (chan->curr_hwdesc == chan->desc->count) 1378 chan->curr_hwdesc = 0; 1379 vchan_cyclic_callback(&chan->desc->vdesc); 1380 } 1381 } 1382 1383 if (status & STM32_MDMA_CISR_TCIF) { 1384 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF); 1385 status &= ~STM32_MDMA_CISR_TCIF; 1386 } 1387 1388 if (status) { 1389 stm32_mdma_set_bits(dmadev, reg, status); 1390 dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); 1391 if (!(ccr & STM32_MDMA_CCR_EN)) 1392 dev_err(chan2dev(chan), "chan disabled by HW\n"); 1393 } 1394 1395 spin_unlock(&chan->vchan.lock); 1396 1397 return IRQ_HANDLED; 1398 } 1399 1400 static int stm32_mdma_alloc_chan_resources(struct dma_chan *c) 1401 { 1402 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1403 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1404 int ret; 1405 1406 chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device), 1407 c->device->dev, 1408 sizeof(struct stm32_mdma_hwdesc), 1409 __alignof__(struct stm32_mdma_hwdesc), 1410 0); 1411 if (!chan->desc_pool) { 1412 dev_err(chan2dev(chan), "failed to allocate descriptor pool\n"); 1413 return -ENOMEM; 1414 } 1415 1416 ret = pm_runtime_resume_and_get(dmadev->ddev.dev); 1417 if (ret < 0) 1418 return ret; 1419 1420 ret = stm32_mdma_disable_chan(chan); 1421 if (ret < 0) 1422 pm_runtime_put(dmadev->ddev.dev); 1423 1424 return ret; 1425 } 1426 1427 static void stm32_mdma_free_chan_resources(struct dma_chan *c) 1428 { 1429 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1430 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1431 unsigned long flags; 1432 1433 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id); 1434 1435 if (chan->busy) { 1436 spin_lock_irqsave(&chan->vchan.lock, flags); 1437 stm32_mdma_stop(chan); 1438 chan->desc = NULL; 1439 spin_unlock_irqrestore(&chan->vchan.lock, flags); 1440 } 1441 1442 pm_runtime_put(dmadev->ddev.dev); 1443 vchan_free_chan_resources(to_virt_chan(c)); 1444 dmam_pool_destroy(chan->desc_pool); 1445 chan->desc_pool = NULL; 1446 } 1447 1448 static bool stm32_mdma_filter_fn(struct dma_chan *c, void *fn_param) 1449 { 1450 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); 1451 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); 1452 1453 /* Check if chan is marked Secure */ 1454 if (dmadev->chan_reserved & BIT(chan->id)) 1455 return false; 1456 1457 return true; 1458 } 1459 1460 static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, 1461 struct of_dma *ofdma) 1462 { 1463 struct stm32_mdma_device *dmadev = ofdma->of_dma_data; 1464 dma_cap_mask_t mask = dmadev->ddev.cap_mask; 1465 struct stm32_mdma_chan *chan; 1466 struct dma_chan *c; 1467 struct stm32_mdma_chan_config config; 1468 1469 if (dma_spec->args_count < 5) { 1470 dev_err(mdma2dev(dmadev), "Bad number of args\n"); 1471 return NULL; 1472 } 1473 1474 config.request = dma_spec->args[0]; 1475 config.priority_level = dma_spec->args[1]; 1476 config.transfer_config = dma_spec->args[2]; 1477 config.mask_addr = dma_spec->args[3]; 1478 config.mask_data = dma_spec->args[4]; 1479 1480 if (config.request >= dmadev->nr_requests) { 1481 dev_err(mdma2dev(dmadev), "Bad request line\n"); 1482 return NULL; 1483 } 1484 1485 if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) { 1486 dev_err(mdma2dev(dmadev), "Priority level not supported\n"); 1487 return NULL; 1488 } 1489 1490 c = __dma_request_channel(&mask, stm32_mdma_filter_fn, &config, ofdma->of_node); 1491 if (!c) { 1492 dev_err(mdma2dev(dmadev), "No more channels available\n"); 1493 return NULL; 1494 } 1495 1496 chan = to_stm32_mdma_chan(c); 1497 chan->chan_config = config; 1498 1499 return c; 1500 } 1501 1502 static const struct of_device_id stm32_mdma_of_match[] = { 1503 { .compatible = "st,stm32h7-mdma", }, 1504 { /* sentinel */ }, 1505 }; 1506 MODULE_DEVICE_TABLE(of, stm32_mdma_of_match); 1507 1508 static int stm32_mdma_probe(struct platform_device *pdev) 1509 { 1510 struct stm32_mdma_chan *chan; 1511 struct stm32_mdma_device *dmadev; 1512 struct dma_device *dd; 1513 struct device_node *of_node; 1514 struct resource *res; 1515 struct reset_control *rst; 1516 u32 nr_channels, nr_requests; 1517 int i, count, ret; 1518 1519 of_node = pdev->dev.of_node; 1520 if (!of_node) 1521 return -ENODEV; 1522 1523 ret = device_property_read_u32(&pdev->dev, "dma-channels", 1524 &nr_channels); 1525 if (ret) { 1526 nr_channels = STM32_MDMA_MAX_CHANNELS; 1527 dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n", 1528 nr_channels); 1529 } 1530 1531 ret = device_property_read_u32(&pdev->dev, "dma-requests", 1532 &nr_requests); 1533 if (ret) { 1534 nr_requests = STM32_MDMA_MAX_REQUESTS; 1535 dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n", 1536 nr_requests); 1537 } 1538 1539 count = device_property_count_u32(&pdev->dev, "st,ahb-addr-masks"); 1540 if (count < 0) 1541 count = 0; 1542 1543 dmadev = devm_kzalloc(&pdev->dev, 1544 struct_size(dmadev, ahb_addr_masks, count), 1545 GFP_KERNEL); 1546 if (!dmadev) 1547 return -ENOMEM; 1548 1549 dmadev->nr_channels = nr_channels; 1550 dmadev->nr_requests = nr_requests; 1551 device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks", 1552 dmadev->ahb_addr_masks, 1553 count); 1554 dmadev->nr_ahb_addr_masks = count; 1555 1556 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1557 dmadev->base = devm_ioremap_resource(&pdev->dev, res); 1558 if (IS_ERR(dmadev->base)) 1559 return PTR_ERR(dmadev->base); 1560 1561 dmadev->clk = devm_clk_get(&pdev->dev, NULL); 1562 if (IS_ERR(dmadev->clk)) 1563 return dev_err_probe(&pdev->dev, PTR_ERR(dmadev->clk), 1564 "Missing clock controller\n"); 1565 1566 ret = clk_prepare_enable(dmadev->clk); 1567 if (ret < 0) { 1568 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); 1569 return ret; 1570 } 1571 1572 rst = devm_reset_control_get(&pdev->dev, NULL); 1573 if (IS_ERR(rst)) { 1574 ret = PTR_ERR(rst); 1575 if (ret == -EPROBE_DEFER) 1576 goto err_clk; 1577 } else { 1578 reset_control_assert(rst); 1579 udelay(2); 1580 reset_control_deassert(rst); 1581 } 1582 1583 dd = &dmadev->ddev; 1584 dma_cap_set(DMA_SLAVE, dd->cap_mask); 1585 dma_cap_set(DMA_PRIVATE, dd->cap_mask); 1586 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 1587 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 1588 dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources; 1589 dd->device_free_chan_resources = stm32_mdma_free_chan_resources; 1590 dd->device_tx_status = stm32_mdma_tx_status; 1591 dd->device_issue_pending = stm32_mdma_issue_pending; 1592 dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg; 1593 dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic; 1594 dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy; 1595 dd->device_config = stm32_mdma_slave_config; 1596 dd->device_pause = stm32_mdma_pause; 1597 dd->device_resume = stm32_mdma_resume; 1598 dd->device_terminate_all = stm32_mdma_terminate_all; 1599 dd->device_synchronize = stm32_mdma_synchronize; 1600 dd->descriptor_reuse = true; 1601 1602 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1603 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1604 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1605 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1606 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1607 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1608 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | 1609 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); 1610 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 1611 BIT(DMA_MEM_TO_MEM); 1612 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1613 dd->max_burst = STM32_MDMA_MAX_BURST; 1614 dd->dev = &pdev->dev; 1615 INIT_LIST_HEAD(&dd->channels); 1616 1617 for (i = 0; i < dmadev->nr_channels; i++) { 1618 chan = &dmadev->chan[i]; 1619 chan->id = i; 1620 1621 if (stm32_mdma_read(dmadev, STM32_MDMA_CCR(i)) & STM32_MDMA_CCR_SM) 1622 dmadev->chan_reserved |= BIT(i); 1623 1624 chan->vchan.desc_free = stm32_mdma_desc_free; 1625 vchan_init(&chan->vchan, dd); 1626 } 1627 1628 dmadev->irq = platform_get_irq(pdev, 0); 1629 if (dmadev->irq < 0) { 1630 ret = dmadev->irq; 1631 goto err_clk; 1632 } 1633 1634 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler, 1635 0, dev_name(&pdev->dev), dmadev); 1636 if (ret) { 1637 dev_err(&pdev->dev, "failed to request IRQ\n"); 1638 goto err_clk; 1639 } 1640 1641 ret = dmaenginem_async_device_register(dd); 1642 if (ret) 1643 goto err_clk; 1644 1645 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev); 1646 if (ret < 0) { 1647 dev_err(&pdev->dev, 1648 "STM32 MDMA DMA OF registration failed %d\n", ret); 1649 goto err_clk; 1650 } 1651 1652 platform_set_drvdata(pdev, dmadev); 1653 pm_runtime_set_active(&pdev->dev); 1654 pm_runtime_enable(&pdev->dev); 1655 pm_runtime_get_noresume(&pdev->dev); 1656 pm_runtime_put(&pdev->dev); 1657 1658 dev_info(&pdev->dev, "STM32 MDMA driver registered\n"); 1659 1660 return 0; 1661 1662 err_clk: 1663 clk_disable_unprepare(dmadev->clk); 1664 1665 return ret; 1666 } 1667 1668 #ifdef CONFIG_PM 1669 static int stm32_mdma_runtime_suspend(struct device *dev) 1670 { 1671 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); 1672 1673 clk_disable_unprepare(dmadev->clk); 1674 1675 return 0; 1676 } 1677 1678 static int stm32_mdma_runtime_resume(struct device *dev) 1679 { 1680 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); 1681 int ret; 1682 1683 ret = clk_prepare_enable(dmadev->clk); 1684 if (ret) { 1685 dev_err(dev, "failed to prepare_enable clock\n"); 1686 return ret; 1687 } 1688 1689 return 0; 1690 } 1691 #endif 1692 1693 #ifdef CONFIG_PM_SLEEP 1694 static int stm32_mdma_pm_suspend(struct device *dev) 1695 { 1696 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); 1697 u32 ccr, id; 1698 int ret; 1699 1700 ret = pm_runtime_resume_and_get(dev); 1701 if (ret < 0) 1702 return ret; 1703 1704 for (id = 0; id < dmadev->nr_channels; id++) { 1705 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); 1706 if (ccr & STM32_MDMA_CCR_EN) { 1707 dev_warn(dev, "Suspend is prevented by Chan %i\n", id); 1708 return -EBUSY; 1709 } 1710 } 1711 1712 pm_runtime_put_sync(dev); 1713 1714 pm_runtime_force_suspend(dev); 1715 1716 return 0; 1717 } 1718 1719 static int stm32_mdma_pm_resume(struct device *dev) 1720 { 1721 return pm_runtime_force_resume(dev); 1722 } 1723 #endif 1724 1725 static const struct dev_pm_ops stm32_mdma_pm_ops = { 1726 SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume) 1727 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend, 1728 stm32_mdma_runtime_resume, NULL) 1729 }; 1730 1731 static struct platform_driver stm32_mdma_driver = { 1732 .probe = stm32_mdma_probe, 1733 .driver = { 1734 .name = "stm32-mdma", 1735 .of_match_table = stm32_mdma_of_match, 1736 .pm = &stm32_mdma_pm_ops, 1737 }, 1738 }; 1739 1740 static int __init stm32_mdma_init(void) 1741 { 1742 return platform_driver_register(&stm32_mdma_driver); 1743 } 1744 1745 subsys_initcall(stm32_mdma_init); 1746 1747 MODULE_DESCRIPTION("Driver for STM32 MDMA controller"); 1748 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>"); 1749 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>"); 1750 MODULE_LICENSE("GPL v2"); 1751