1 /* 2 * Driver for STM32 DMA controller 3 * 4 * Inspired by dma-jz4740.c and tegra20-apb-dma.c 5 * 6 * Copyright (C) M'boumba Cedric Madianga 2015 7 * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com> 8 * 9 * License terms: GNU General Public License (GPL), version 2 10 */ 11 12 #include <linux/clk.h> 13 #include <linux/delay.h> 14 #include <linux/dmaengine.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/err.h> 17 #include <linux/init.h> 18 #include <linux/jiffies.h> 19 #include <linux/list.h> 20 #include <linux/module.h> 21 #include <linux/of.h> 22 #include <linux/of_device.h> 23 #include <linux/of_dma.h> 24 #include <linux/platform_device.h> 25 #include <linux/reset.h> 26 #include <linux/sched.h> 27 #include <linux/slab.h> 28 29 #include "virt-dma.h" 30 31 #define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */ 32 #define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */ 33 #define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */ 34 #define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */ 35 #define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */ 36 #define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */ 37 #define STM32_DMA_DMEI BIT(2) /* Direct Mode Error Interrupt */ 38 #define STM32_DMA_FEI BIT(0) /* FIFO Error Interrupt */ 39 40 /* DMA Stream x Configuration Register */ 41 #define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */ 42 #define STM32_DMA_SCR_REQ(n) ((n & 0x7) << 25) 43 #define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23) 44 #define STM32_DMA_SCR_MBURST(n) ((n & 0x3) << 23) 45 #define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21) 46 #define STM32_DMA_SCR_PBURST(n) ((n & 0x3) << 21) 47 #define STM32_DMA_SCR_PL_MASK GENMASK(17, 16) 48 #define STM32_DMA_SCR_PL(n) ((n & 0x3) << 16) 49 #define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13) 50 #define STM32_DMA_SCR_MSIZE(n) ((n & 0x3) << 13) 51 #define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11) 52 #define STM32_DMA_SCR_PSIZE(n) ((n & 0x3) << 11) 53 #define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11) 54 #define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6) 55 #define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6) 56 #define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */ 57 #define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */ 58 #define STM32_DMA_SCR_PINCOS BIT(15) /* Peripheral inc offset size */ 59 #define STM32_DMA_SCR_MINC BIT(10) /* Memory increment mode */ 60 #define STM32_DMA_SCR_PINC BIT(9) /* Peripheral increment mode */ 61 #define STM32_DMA_SCR_CIRC BIT(8) /* Circular mode */ 62 #define STM32_DMA_SCR_PFCTRL BIT(5) /* Peripheral Flow Controller */ 63 #define STM32_DMA_SCR_TCIE BIT(4) /* Transfer Cplete Int Enable*/ 64 #define STM32_DMA_SCR_TEIE BIT(2) /* Transfer Error Int Enable */ 65 #define STM32_DMA_SCR_DMEIE BIT(1) /* Direct Mode Err Int Enable */ 66 #define STM32_DMA_SCR_EN BIT(0) /* Stream Enable */ 67 #define STM32_DMA_SCR_CFG_MASK (STM32_DMA_SCR_PINC \ 68 | STM32_DMA_SCR_MINC \ 69 | STM32_DMA_SCR_PINCOS \ 70 | STM32_DMA_SCR_PL_MASK) 71 #define STM32_DMA_SCR_IRQ_MASK (STM32_DMA_SCR_TCIE \ 72 | STM32_DMA_SCR_TEIE \ 73 | STM32_DMA_SCR_DMEIE) 74 75 /* DMA Stream x number of data register */ 76 #define STM32_DMA_SNDTR(x) (0x0014 + 0x18 * (x)) 77 78 /* DMA stream peripheral address register */ 79 #define STM32_DMA_SPAR(x) (0x0018 + 0x18 * (x)) 80 81 /* DMA stream x memory 0 address register */ 82 #define STM32_DMA_SM0AR(x) (0x001c + 0x18 * (x)) 83 84 /* DMA stream x memory 1 address register */ 85 #define STM32_DMA_SM1AR(x) (0x0020 + 0x18 * (x)) 86 87 /* DMA stream x FIFO control register */ 88 #define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x)) 89 #define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0) 90 #define STM32_DMA_SFCR_FTH(n) (n & STM32_DMA_SFCR_FTH_MASK) 91 #define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */ 92 #define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */ 93 #define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \ 94 | STM32_DMA_SFCR_DMDIS) 95 96 /* DMA direction */ 97 #define STM32_DMA_DEV_TO_MEM 0x00 98 #define STM32_DMA_MEM_TO_DEV 0x01 99 #define STM32_DMA_MEM_TO_MEM 0x02 100 101 /* DMA priority level */ 102 #define STM32_DMA_PRIORITY_LOW 0x00 103 #define STM32_DMA_PRIORITY_MEDIUM 0x01 104 #define STM32_DMA_PRIORITY_HIGH 0x02 105 #define STM32_DMA_PRIORITY_VERY_HIGH 0x03 106 107 /* DMA FIFO threshold selection */ 108 #define STM32_DMA_FIFO_THRESHOLD_1QUARTERFULL 0x00 109 #define STM32_DMA_FIFO_THRESHOLD_HALFFULL 0x01 110 #define STM32_DMA_FIFO_THRESHOLD_3QUARTERSFULL 0x02 111 #define STM32_DMA_FIFO_THRESHOLD_FULL 0x03 112 113 #define STM32_DMA_MAX_DATA_ITEMS 0xffff 114 #define STM32_DMA_MAX_CHANNELS 0x08 115 #define STM32_DMA_MAX_REQUEST_ID 0x08 116 #define STM32_DMA_MAX_DATA_PARAM 0x03 117 118 enum stm32_dma_width { 119 STM32_DMA_BYTE, 120 STM32_DMA_HALF_WORD, 121 STM32_DMA_WORD, 122 }; 123 124 enum stm32_dma_burst_size { 125 STM32_DMA_BURST_SINGLE, 126 STM32_DMA_BURST_INCR4, 127 STM32_DMA_BURST_INCR8, 128 STM32_DMA_BURST_INCR16, 129 }; 130 131 struct stm32_dma_cfg { 132 u32 channel_id; 133 u32 request_line; 134 u32 stream_config; 135 u32 threshold; 136 }; 137 138 struct stm32_dma_chan_reg { 139 u32 dma_lisr; 140 u32 dma_hisr; 141 u32 dma_lifcr; 142 u32 dma_hifcr; 143 u32 dma_scr; 144 u32 dma_sndtr; 145 u32 dma_spar; 146 u32 dma_sm0ar; 147 u32 dma_sm1ar; 148 u32 dma_sfcr; 149 }; 150 151 struct stm32_dma_sg_req { 152 u32 len; 153 struct stm32_dma_chan_reg chan_reg; 154 }; 155 156 struct stm32_dma_desc { 157 struct virt_dma_desc vdesc; 158 bool cyclic; 159 u32 num_sgs; 160 struct stm32_dma_sg_req sg_req[]; 161 }; 162 163 struct stm32_dma_chan { 164 struct virt_dma_chan vchan; 165 bool config_init; 166 bool busy; 167 u32 id; 168 u32 irq; 169 struct stm32_dma_desc *desc; 170 u32 next_sg; 171 struct dma_slave_config dma_sconfig; 172 struct stm32_dma_chan_reg chan_reg; 173 }; 174 175 struct stm32_dma_device { 176 struct dma_device ddev; 177 void __iomem *base; 178 struct clk *clk; 179 struct reset_control *rst; 180 bool mem2mem; 181 struct stm32_dma_chan chan[STM32_DMA_MAX_CHANNELS]; 182 }; 183 184 static struct stm32_dma_device *stm32_dma_get_dev(struct stm32_dma_chan *chan) 185 { 186 return container_of(chan->vchan.chan.device, struct stm32_dma_device, 187 ddev); 188 } 189 190 static struct stm32_dma_chan *to_stm32_dma_chan(struct dma_chan *c) 191 { 192 return container_of(c, struct stm32_dma_chan, vchan.chan); 193 } 194 195 static struct stm32_dma_desc *to_stm32_dma_desc(struct virt_dma_desc *vdesc) 196 { 197 return container_of(vdesc, struct stm32_dma_desc, vdesc); 198 } 199 200 static struct device *chan2dev(struct stm32_dma_chan *chan) 201 { 202 return &chan->vchan.chan.dev->device; 203 } 204 205 static u32 stm32_dma_read(struct stm32_dma_device *dmadev, u32 reg) 206 { 207 return readl_relaxed(dmadev->base + reg); 208 } 209 210 static void stm32_dma_write(struct stm32_dma_device *dmadev, u32 reg, u32 val) 211 { 212 writel_relaxed(val, dmadev->base + reg); 213 } 214 215 static struct stm32_dma_desc *stm32_dma_alloc_desc(u32 num_sgs) 216 { 217 return kzalloc(sizeof(struct stm32_dma_desc) + 218 sizeof(struct stm32_dma_sg_req) * num_sgs, GFP_NOWAIT); 219 } 220 221 static int stm32_dma_get_width(struct stm32_dma_chan *chan, 222 enum dma_slave_buswidth width) 223 { 224 switch (width) { 225 case DMA_SLAVE_BUSWIDTH_1_BYTE: 226 return STM32_DMA_BYTE; 227 case DMA_SLAVE_BUSWIDTH_2_BYTES: 228 return STM32_DMA_HALF_WORD; 229 case DMA_SLAVE_BUSWIDTH_4_BYTES: 230 return STM32_DMA_WORD; 231 default: 232 dev_err(chan2dev(chan), "Dma bus width not supported\n"); 233 return -EINVAL; 234 } 235 } 236 237 static int stm32_dma_get_burst(struct stm32_dma_chan *chan, u32 maxburst) 238 { 239 switch (maxburst) { 240 case 0: 241 case 1: 242 return STM32_DMA_BURST_SINGLE; 243 case 4: 244 return STM32_DMA_BURST_INCR4; 245 case 8: 246 return STM32_DMA_BURST_INCR8; 247 case 16: 248 return STM32_DMA_BURST_INCR16; 249 default: 250 dev_err(chan2dev(chan), "Dma burst size not supported\n"); 251 return -EINVAL; 252 } 253 } 254 255 static void stm32_dma_set_fifo_config(struct stm32_dma_chan *chan, 256 u32 src_maxburst, u32 dst_maxburst) 257 { 258 chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_MASK; 259 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_DMEIE; 260 261 if ((!src_maxburst) && (!dst_maxburst)) { 262 /* Using direct mode */ 263 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DMEIE; 264 } else { 265 /* Using FIFO mode */ 266 chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK; 267 } 268 } 269 270 static int stm32_dma_slave_config(struct dma_chan *c, 271 struct dma_slave_config *config) 272 { 273 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 274 275 memcpy(&chan->dma_sconfig, config, sizeof(*config)); 276 277 chan->config_init = true; 278 279 return 0; 280 } 281 282 static u32 stm32_dma_irq_status(struct stm32_dma_chan *chan) 283 { 284 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 285 u32 flags, dma_isr; 286 287 /* 288 * Read "flags" from DMA_xISR register corresponding to the selected 289 * DMA channel at the correct bit offset inside that register. 290 * 291 * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. 292 * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. 293 */ 294 295 if (chan->id & 4) 296 dma_isr = stm32_dma_read(dmadev, STM32_DMA_HISR); 297 else 298 dma_isr = stm32_dma_read(dmadev, STM32_DMA_LISR); 299 300 flags = dma_isr >> (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); 301 302 return flags; 303 } 304 305 static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags) 306 { 307 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 308 u32 dma_ifcr; 309 310 /* 311 * Write "flags" to the DMA_xIFCR register corresponding to the selected 312 * DMA channel at the correct bit offset inside that register. 313 * 314 * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. 315 * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. 316 */ 317 dma_ifcr = flags << (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); 318 319 if (chan->id & 4) 320 stm32_dma_write(dmadev, STM32_DMA_HIFCR, dma_ifcr); 321 else 322 stm32_dma_write(dmadev, STM32_DMA_LIFCR, dma_ifcr); 323 } 324 325 static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) 326 { 327 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 328 unsigned long timeout = jiffies + msecs_to_jiffies(5000); 329 u32 dma_scr, id; 330 331 id = chan->id; 332 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 333 334 if (dma_scr & STM32_DMA_SCR_EN) { 335 dma_scr &= ~STM32_DMA_SCR_EN; 336 stm32_dma_write(dmadev, STM32_DMA_SCR(id), dma_scr); 337 338 do { 339 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 340 dma_scr &= STM32_DMA_SCR_EN; 341 if (!dma_scr) 342 break; 343 344 if (time_after_eq(jiffies, timeout)) { 345 dev_err(chan2dev(chan), "%s: timeout!\n", 346 __func__); 347 return -EBUSY; 348 } 349 cond_resched(); 350 } while (1); 351 } 352 353 return 0; 354 } 355 356 static void stm32_dma_stop(struct stm32_dma_chan *chan) 357 { 358 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 359 u32 dma_scr, dma_sfcr, status; 360 int ret; 361 362 /* Disable interrupts */ 363 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); 364 dma_scr &= ~STM32_DMA_SCR_IRQ_MASK; 365 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); 366 dma_sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); 367 dma_sfcr &= ~STM32_DMA_SFCR_FEIE; 368 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), dma_sfcr); 369 370 /* Disable DMA */ 371 ret = stm32_dma_disable_chan(chan); 372 if (ret < 0) 373 return; 374 375 /* Clear interrupt status if it is there */ 376 status = stm32_dma_irq_status(chan); 377 if (status) { 378 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n", 379 __func__, status); 380 stm32_dma_irq_clear(chan, status); 381 } 382 383 chan->busy = false; 384 } 385 386 static int stm32_dma_terminate_all(struct dma_chan *c) 387 { 388 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 389 unsigned long flags; 390 LIST_HEAD(head); 391 392 spin_lock_irqsave(&chan->vchan.lock, flags); 393 394 if (chan->busy) { 395 stm32_dma_stop(chan); 396 chan->desc = NULL; 397 } 398 399 vchan_get_all_descriptors(&chan->vchan, &head); 400 spin_unlock_irqrestore(&chan->vchan.lock, flags); 401 vchan_dma_desc_free_list(&chan->vchan, &head); 402 403 return 0; 404 } 405 406 static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) 407 { 408 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 409 u32 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); 410 u32 ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); 411 u32 spar = stm32_dma_read(dmadev, STM32_DMA_SPAR(chan->id)); 412 u32 sm0ar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(chan->id)); 413 u32 sm1ar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(chan->id)); 414 u32 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id)); 415 416 dev_dbg(chan2dev(chan), "SCR: 0x%08x\n", scr); 417 dev_dbg(chan2dev(chan), "NDTR: 0x%08x\n", ndtr); 418 dev_dbg(chan2dev(chan), "SPAR: 0x%08x\n", spar); 419 dev_dbg(chan2dev(chan), "SM0AR: 0x%08x\n", sm0ar); 420 dev_dbg(chan2dev(chan), "SM1AR: 0x%08x\n", sm1ar); 421 dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr); 422 } 423 424 static int stm32_dma_start_transfer(struct stm32_dma_chan *chan) 425 { 426 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 427 struct virt_dma_desc *vdesc; 428 struct stm32_dma_sg_req *sg_req; 429 struct stm32_dma_chan_reg *reg; 430 u32 status; 431 int ret; 432 433 ret = stm32_dma_disable_chan(chan); 434 if (ret < 0) 435 return ret; 436 437 if (!chan->desc) { 438 vdesc = vchan_next_desc(&chan->vchan); 439 if (!vdesc) 440 return -EPERM; 441 442 chan->desc = to_stm32_dma_desc(vdesc); 443 chan->next_sg = 0; 444 } 445 446 if (chan->next_sg == chan->desc->num_sgs) 447 chan->next_sg = 0; 448 449 sg_req = &chan->desc->sg_req[chan->next_sg]; 450 reg = &sg_req->chan_reg; 451 452 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); 453 stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); 454 stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar); 455 stm32_dma_write(dmadev, STM32_DMA_SFCR(chan->id), reg->dma_sfcr); 456 stm32_dma_write(dmadev, STM32_DMA_SM1AR(chan->id), reg->dma_sm1ar); 457 stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), reg->dma_sndtr); 458 459 chan->next_sg++; 460 461 /* Clear interrupt status if it is there */ 462 status = stm32_dma_irq_status(chan); 463 if (status) 464 stm32_dma_irq_clear(chan, status); 465 466 stm32_dma_dump_reg(chan); 467 468 /* Start DMA */ 469 reg->dma_scr |= STM32_DMA_SCR_EN; 470 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); 471 472 chan->busy = true; 473 474 return 0; 475 } 476 477 static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) 478 { 479 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 480 struct stm32_dma_sg_req *sg_req; 481 u32 dma_scr, dma_sm0ar, dma_sm1ar, id; 482 483 id = chan->id; 484 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 485 486 if (dma_scr & STM32_DMA_SCR_DBM) { 487 if (chan->next_sg == chan->desc->num_sgs) 488 chan->next_sg = 0; 489 490 sg_req = &chan->desc->sg_req[chan->next_sg]; 491 492 if (dma_scr & STM32_DMA_SCR_CT) { 493 dma_sm0ar = sg_req->chan_reg.dma_sm0ar; 494 stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar); 495 dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n", 496 stm32_dma_read(dmadev, STM32_DMA_SM0AR(id))); 497 } else { 498 dma_sm1ar = sg_req->chan_reg.dma_sm1ar; 499 stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar); 500 dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n", 501 stm32_dma_read(dmadev, STM32_DMA_SM1AR(id))); 502 } 503 504 chan->next_sg++; 505 } 506 } 507 508 static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan) 509 { 510 if (chan->desc) { 511 if (chan->desc->cyclic) { 512 vchan_cyclic_callback(&chan->desc->vdesc); 513 stm32_dma_configure_next_sg(chan); 514 } else { 515 chan->busy = false; 516 if (chan->next_sg == chan->desc->num_sgs) { 517 list_del(&chan->desc->vdesc.node); 518 vchan_cookie_complete(&chan->desc->vdesc); 519 chan->desc = NULL; 520 } 521 stm32_dma_start_transfer(chan); 522 } 523 } 524 } 525 526 static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) 527 { 528 struct stm32_dma_chan *chan = devid; 529 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 530 u32 status, scr; 531 532 spin_lock(&chan->vchan.lock); 533 534 status = stm32_dma_irq_status(chan); 535 scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); 536 537 if ((status & STM32_DMA_TCI) && (scr & STM32_DMA_SCR_TCIE)) { 538 stm32_dma_irq_clear(chan, STM32_DMA_TCI); 539 stm32_dma_handle_chan_done(chan); 540 541 } else { 542 stm32_dma_irq_clear(chan, status); 543 dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status); 544 } 545 546 spin_unlock(&chan->vchan.lock); 547 548 return IRQ_HANDLED; 549 } 550 551 static void stm32_dma_issue_pending(struct dma_chan *c) 552 { 553 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 554 unsigned long flags; 555 int ret; 556 557 spin_lock_irqsave(&chan->vchan.lock, flags); 558 if (!chan->busy) { 559 if (vchan_issue_pending(&chan->vchan) && !chan->desc) { 560 ret = stm32_dma_start_transfer(chan); 561 if ((!ret) && (chan->desc->cyclic)) 562 stm32_dma_configure_next_sg(chan); 563 } 564 } 565 spin_unlock_irqrestore(&chan->vchan.lock, flags); 566 } 567 568 static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, 569 enum dma_transfer_direction direction, 570 enum dma_slave_buswidth *buswidth) 571 { 572 enum dma_slave_buswidth src_addr_width, dst_addr_width; 573 int src_bus_width, dst_bus_width; 574 int src_burst_size, dst_burst_size; 575 u32 src_maxburst, dst_maxburst; 576 u32 dma_scr = 0; 577 578 src_addr_width = chan->dma_sconfig.src_addr_width; 579 dst_addr_width = chan->dma_sconfig.dst_addr_width; 580 src_maxburst = chan->dma_sconfig.src_maxburst; 581 dst_maxburst = chan->dma_sconfig.dst_maxburst; 582 583 switch (direction) { 584 case DMA_MEM_TO_DEV: 585 dst_bus_width = stm32_dma_get_width(chan, dst_addr_width); 586 if (dst_bus_width < 0) 587 return dst_bus_width; 588 589 dst_burst_size = stm32_dma_get_burst(chan, dst_maxburst); 590 if (dst_burst_size < 0) 591 return dst_burst_size; 592 593 if (!src_addr_width) 594 src_addr_width = dst_addr_width; 595 596 src_bus_width = stm32_dma_get_width(chan, src_addr_width); 597 if (src_bus_width < 0) 598 return src_bus_width; 599 600 src_burst_size = stm32_dma_get_burst(chan, src_maxburst); 601 if (src_burst_size < 0) 602 return src_burst_size; 603 604 dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_DEV) | 605 STM32_DMA_SCR_PSIZE(dst_bus_width) | 606 STM32_DMA_SCR_MSIZE(src_bus_width) | 607 STM32_DMA_SCR_PBURST(dst_burst_size) | 608 STM32_DMA_SCR_MBURST(src_burst_size); 609 610 chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr; 611 *buswidth = dst_addr_width; 612 break; 613 614 case DMA_DEV_TO_MEM: 615 src_bus_width = stm32_dma_get_width(chan, src_addr_width); 616 if (src_bus_width < 0) 617 return src_bus_width; 618 619 src_burst_size = stm32_dma_get_burst(chan, src_maxburst); 620 if (src_burst_size < 0) 621 return src_burst_size; 622 623 if (!dst_addr_width) 624 dst_addr_width = src_addr_width; 625 626 dst_bus_width = stm32_dma_get_width(chan, dst_addr_width); 627 if (dst_bus_width < 0) 628 return dst_bus_width; 629 630 dst_burst_size = stm32_dma_get_burst(chan, dst_maxburst); 631 if (dst_burst_size < 0) 632 return dst_burst_size; 633 634 dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_DEV_TO_MEM) | 635 STM32_DMA_SCR_PSIZE(src_bus_width) | 636 STM32_DMA_SCR_MSIZE(dst_bus_width) | 637 STM32_DMA_SCR_PBURST(src_burst_size) | 638 STM32_DMA_SCR_MBURST(dst_burst_size); 639 640 chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr; 641 *buswidth = chan->dma_sconfig.src_addr_width; 642 break; 643 644 default: 645 dev_err(chan2dev(chan), "Dma direction is not supported\n"); 646 return -EINVAL; 647 } 648 649 stm32_dma_set_fifo_config(chan, src_maxburst, dst_maxburst); 650 651 chan->chan_reg.dma_scr &= ~(STM32_DMA_SCR_DIR_MASK | 652 STM32_DMA_SCR_PSIZE_MASK | STM32_DMA_SCR_MSIZE_MASK | 653 STM32_DMA_SCR_PBURST_MASK | STM32_DMA_SCR_MBURST_MASK); 654 chan->chan_reg.dma_scr |= dma_scr; 655 656 return 0; 657 } 658 659 static void stm32_dma_clear_reg(struct stm32_dma_chan_reg *regs) 660 { 661 memset(regs, 0, sizeof(struct stm32_dma_chan_reg)); 662 } 663 664 static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( 665 struct dma_chan *c, struct scatterlist *sgl, 666 u32 sg_len, enum dma_transfer_direction direction, 667 unsigned long flags, void *context) 668 { 669 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 670 struct stm32_dma_desc *desc; 671 struct scatterlist *sg; 672 enum dma_slave_buswidth buswidth; 673 u32 nb_data_items; 674 int i, ret; 675 676 if (!chan->config_init) { 677 dev_err(chan2dev(chan), "dma channel is not configured\n"); 678 return NULL; 679 } 680 681 if (sg_len < 1) { 682 dev_err(chan2dev(chan), "Invalid segment length %d\n", sg_len); 683 return NULL; 684 } 685 686 desc = stm32_dma_alloc_desc(sg_len); 687 if (!desc) 688 return NULL; 689 690 ret = stm32_dma_set_xfer_param(chan, direction, &buswidth); 691 if (ret < 0) 692 goto err; 693 694 /* Set peripheral flow controller */ 695 if (chan->dma_sconfig.device_fc) 696 chan->chan_reg.dma_scr |= STM32_DMA_SCR_PFCTRL; 697 else 698 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; 699 700 for_each_sg(sgl, sg, sg_len, i) { 701 desc->sg_req[i].len = sg_dma_len(sg); 702 703 nb_data_items = desc->sg_req[i].len / buswidth; 704 if (nb_data_items > STM32_DMA_MAX_DATA_ITEMS) { 705 dev_err(chan2dev(chan), "nb items not supported\n"); 706 goto err; 707 } 708 709 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); 710 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr; 711 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr; 712 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; 713 desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg); 714 desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg); 715 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; 716 } 717 718 desc->num_sgs = sg_len; 719 desc->cyclic = false; 720 721 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 722 723 err: 724 kfree(desc); 725 return NULL; 726 } 727 728 static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( 729 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, 730 size_t period_len, enum dma_transfer_direction direction, 731 unsigned long flags) 732 { 733 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 734 struct stm32_dma_desc *desc; 735 enum dma_slave_buswidth buswidth; 736 u32 num_periods, nb_data_items; 737 int i, ret; 738 739 if (!buf_len || !period_len) { 740 dev_err(chan2dev(chan), "Invalid buffer/period len\n"); 741 return NULL; 742 } 743 744 if (!chan->config_init) { 745 dev_err(chan2dev(chan), "dma channel is not configured\n"); 746 return NULL; 747 } 748 749 if (buf_len % period_len) { 750 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n"); 751 return NULL; 752 } 753 754 /* 755 * We allow to take more number of requests till DMA is 756 * not started. The driver will loop over all requests. 757 * Once DMA is started then new requests can be queued only after 758 * terminating the DMA. 759 */ 760 if (chan->busy) { 761 dev_err(chan2dev(chan), "Request not allowed when dma busy\n"); 762 return NULL; 763 } 764 765 ret = stm32_dma_set_xfer_param(chan, direction, &buswidth); 766 if (ret < 0) 767 return NULL; 768 769 nb_data_items = period_len / buswidth; 770 if (nb_data_items > STM32_DMA_MAX_DATA_ITEMS) { 771 dev_err(chan2dev(chan), "number of items not supported\n"); 772 return NULL; 773 } 774 775 /* Enable Circular mode or double buffer mode */ 776 if (buf_len == period_len) 777 chan->chan_reg.dma_scr |= STM32_DMA_SCR_CIRC; 778 else 779 chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; 780 781 /* Clear periph ctrl if client set it */ 782 chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; 783 784 num_periods = buf_len / period_len; 785 786 desc = stm32_dma_alloc_desc(num_periods); 787 if (!desc) 788 return NULL; 789 790 for (i = 0; i < num_periods; i++) { 791 desc->sg_req[i].len = period_len; 792 793 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); 794 desc->sg_req[i].chan_reg.dma_scr = chan->chan_reg.dma_scr; 795 desc->sg_req[i].chan_reg.dma_sfcr = chan->chan_reg.dma_sfcr; 796 desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; 797 desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr; 798 desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr; 799 desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; 800 buf_addr += period_len; 801 } 802 803 desc->num_sgs = num_periods; 804 desc->cyclic = true; 805 806 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 807 } 808 809 static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( 810 struct dma_chan *c, dma_addr_t dest, 811 dma_addr_t src, size_t len, unsigned long flags) 812 { 813 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 814 u32 num_sgs; 815 struct stm32_dma_desc *desc; 816 size_t xfer_count, offset; 817 int i; 818 819 num_sgs = DIV_ROUND_UP(len, STM32_DMA_MAX_DATA_ITEMS); 820 desc = stm32_dma_alloc_desc(num_sgs); 821 if (!desc) 822 return NULL; 823 824 for (offset = 0, i = 0; offset < len; offset += xfer_count, i++) { 825 xfer_count = min_t(size_t, len - offset, 826 STM32_DMA_MAX_DATA_ITEMS); 827 828 desc->sg_req[i].len = xfer_count; 829 830 stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); 831 desc->sg_req[i].chan_reg.dma_scr = 832 STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_MEM) | 833 STM32_DMA_SCR_MINC | 834 STM32_DMA_SCR_PINC | 835 STM32_DMA_SCR_TCIE | 836 STM32_DMA_SCR_TEIE; 837 desc->sg_req[i].chan_reg.dma_sfcr = STM32_DMA_SFCR_DMDIS | 838 STM32_DMA_SFCR_FTH(STM32_DMA_FIFO_THRESHOLD_FULL) | 839 STM32_DMA_SFCR_FEIE; 840 desc->sg_req[i].chan_reg.dma_spar = src + offset; 841 desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset; 842 desc->sg_req[i].chan_reg.dma_sndtr = xfer_count; 843 } 844 845 desc->num_sgs = num_sgs; 846 desc->cyclic = false; 847 848 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 849 } 850 851 static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, 852 struct stm32_dma_desc *desc, 853 u32 next_sg) 854 { 855 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 856 u32 dma_scr, width, residue, count; 857 int i; 858 859 residue = 0; 860 861 for (i = next_sg; i < desc->num_sgs; i++) 862 residue += desc->sg_req[i].len; 863 864 if (next_sg != 0) { 865 dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); 866 width = STM32_DMA_SCR_PSIZE_GET(dma_scr); 867 count = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); 868 869 residue += count << width; 870 } 871 872 return residue; 873 } 874 875 static enum dma_status stm32_dma_tx_status(struct dma_chan *c, 876 dma_cookie_t cookie, 877 struct dma_tx_state *state) 878 { 879 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 880 struct virt_dma_desc *vdesc; 881 enum dma_status status; 882 unsigned long flags; 883 u32 residue = 0; 884 885 status = dma_cookie_status(c, cookie, state); 886 if ((status == DMA_COMPLETE) || (!state)) 887 return status; 888 889 spin_lock_irqsave(&chan->vchan.lock, flags); 890 vdesc = vchan_find_desc(&chan->vchan, cookie); 891 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) 892 residue = stm32_dma_desc_residue(chan, chan->desc, 893 chan->next_sg); 894 else if (vdesc) 895 residue = stm32_dma_desc_residue(chan, 896 to_stm32_dma_desc(vdesc), 0); 897 dma_set_residue(state, residue); 898 899 spin_unlock_irqrestore(&chan->vchan.lock, flags); 900 901 return status; 902 } 903 904 static int stm32_dma_alloc_chan_resources(struct dma_chan *c) 905 { 906 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 907 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 908 int ret; 909 910 chan->config_init = false; 911 ret = clk_prepare_enable(dmadev->clk); 912 if (ret < 0) { 913 dev_err(chan2dev(chan), "clk_prepare_enable failed: %d\n", ret); 914 return ret; 915 } 916 917 ret = stm32_dma_disable_chan(chan); 918 if (ret < 0) 919 clk_disable_unprepare(dmadev->clk); 920 921 return ret; 922 } 923 924 static void stm32_dma_free_chan_resources(struct dma_chan *c) 925 { 926 struct stm32_dma_chan *chan = to_stm32_dma_chan(c); 927 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 928 unsigned long flags; 929 930 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id); 931 932 if (chan->busy) { 933 spin_lock_irqsave(&chan->vchan.lock, flags); 934 stm32_dma_stop(chan); 935 chan->desc = NULL; 936 spin_unlock_irqrestore(&chan->vchan.lock, flags); 937 } 938 939 clk_disable_unprepare(dmadev->clk); 940 941 vchan_free_chan_resources(to_virt_chan(c)); 942 } 943 944 static void stm32_dma_desc_free(struct virt_dma_desc *vdesc) 945 { 946 kfree(container_of(vdesc, struct stm32_dma_desc, vdesc)); 947 } 948 949 static void stm32_dma_set_config(struct stm32_dma_chan *chan, 950 struct stm32_dma_cfg *cfg) 951 { 952 stm32_dma_clear_reg(&chan->chan_reg); 953 954 chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK; 955 chan->chan_reg.dma_scr |= STM32_DMA_SCR_REQ(cfg->request_line); 956 957 /* Enable Interrupts */ 958 chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE; 959 960 chan->chan_reg.dma_sfcr = cfg->threshold & STM32_DMA_SFCR_FTH_MASK; 961 } 962 963 static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec, 964 struct of_dma *ofdma) 965 { 966 struct stm32_dma_device *dmadev = ofdma->of_dma_data; 967 struct stm32_dma_cfg cfg; 968 struct stm32_dma_chan *chan; 969 struct dma_chan *c; 970 971 if (dma_spec->args_count < 4) 972 return NULL; 973 974 cfg.channel_id = dma_spec->args[0]; 975 cfg.request_line = dma_spec->args[1]; 976 cfg.stream_config = dma_spec->args[2]; 977 cfg.threshold = dma_spec->args[3]; 978 979 if ((cfg.channel_id >= STM32_DMA_MAX_CHANNELS) || (cfg.request_line >= 980 STM32_DMA_MAX_REQUEST_ID)) 981 return NULL; 982 983 chan = &dmadev->chan[cfg.channel_id]; 984 985 c = dma_get_slave_channel(&chan->vchan.chan); 986 if (c) 987 stm32_dma_set_config(chan, &cfg); 988 989 return c; 990 } 991 992 static const struct of_device_id stm32_dma_of_match[] = { 993 { .compatible = "st,stm32-dma", }, 994 { /* sentinel */ }, 995 }; 996 MODULE_DEVICE_TABLE(of, stm32_dma_of_match); 997 998 static int stm32_dma_probe(struct platform_device *pdev) 999 { 1000 struct stm32_dma_chan *chan; 1001 struct stm32_dma_device *dmadev; 1002 struct dma_device *dd; 1003 const struct of_device_id *match; 1004 struct resource *res; 1005 int i, ret; 1006 1007 match = of_match_device(stm32_dma_of_match, &pdev->dev); 1008 if (!match) { 1009 dev_err(&pdev->dev, "Error: No device match found\n"); 1010 return -ENODEV; 1011 } 1012 1013 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); 1014 if (!dmadev) 1015 return -ENOMEM; 1016 1017 dd = &dmadev->ddev; 1018 1019 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1020 dmadev->base = devm_ioremap_resource(&pdev->dev, res); 1021 if (IS_ERR(dmadev->base)) 1022 return PTR_ERR(dmadev->base); 1023 1024 dmadev->clk = devm_clk_get(&pdev->dev, NULL); 1025 if (IS_ERR(dmadev->clk)) { 1026 dev_err(&pdev->dev, "Error: Missing controller clock\n"); 1027 return PTR_ERR(dmadev->clk); 1028 } 1029 1030 dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, 1031 "st,mem2mem"); 1032 1033 dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); 1034 if (!IS_ERR(dmadev->rst)) { 1035 reset_control_assert(dmadev->rst); 1036 udelay(2); 1037 reset_control_deassert(dmadev->rst); 1038 } 1039 1040 dma_cap_set(DMA_SLAVE, dd->cap_mask); 1041 dma_cap_set(DMA_PRIVATE, dd->cap_mask); 1042 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 1043 dd->device_alloc_chan_resources = stm32_dma_alloc_chan_resources; 1044 dd->device_free_chan_resources = stm32_dma_free_chan_resources; 1045 dd->device_tx_status = stm32_dma_tx_status; 1046 dd->device_issue_pending = stm32_dma_issue_pending; 1047 dd->device_prep_slave_sg = stm32_dma_prep_slave_sg; 1048 dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic; 1049 dd->device_config = stm32_dma_slave_config; 1050 dd->device_terminate_all = stm32_dma_terminate_all; 1051 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1052 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1053 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1054 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1055 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1056 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1057 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1058 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1059 dd->dev = &pdev->dev; 1060 INIT_LIST_HEAD(&dd->channels); 1061 1062 if (dmadev->mem2mem) { 1063 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 1064 dd->device_prep_dma_memcpy = stm32_dma_prep_dma_memcpy; 1065 dd->directions |= BIT(DMA_MEM_TO_MEM); 1066 } 1067 1068 for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) { 1069 chan = &dmadev->chan[i]; 1070 chan->id = i; 1071 chan->vchan.desc_free = stm32_dma_desc_free; 1072 vchan_init(&chan->vchan, dd); 1073 } 1074 1075 ret = dma_async_device_register(dd); 1076 if (ret) 1077 return ret; 1078 1079 for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) { 1080 chan = &dmadev->chan[i]; 1081 res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 1082 if (!res) { 1083 ret = -EINVAL; 1084 dev_err(&pdev->dev, "No irq resource for chan %d\n", i); 1085 goto err_unregister; 1086 } 1087 chan->irq = res->start; 1088 ret = devm_request_irq(&pdev->dev, chan->irq, 1089 stm32_dma_chan_irq, 0, 1090 dev_name(chan2dev(chan)), chan); 1091 if (ret) { 1092 dev_err(&pdev->dev, 1093 "request_irq failed with err %d channel %d\n", 1094 ret, i); 1095 goto err_unregister; 1096 } 1097 } 1098 1099 ret = of_dma_controller_register(pdev->dev.of_node, 1100 stm32_dma_of_xlate, dmadev); 1101 if (ret < 0) { 1102 dev_err(&pdev->dev, 1103 "STM32 DMA DMA OF registration failed %d\n", ret); 1104 goto err_unregister; 1105 } 1106 1107 platform_set_drvdata(pdev, dmadev); 1108 1109 dev_info(&pdev->dev, "STM32 DMA driver registered\n"); 1110 1111 return 0; 1112 1113 err_unregister: 1114 dma_async_device_unregister(dd); 1115 1116 return ret; 1117 } 1118 1119 static struct platform_driver stm32_dma_driver = { 1120 .driver = { 1121 .name = "stm32-dma", 1122 .of_match_table = stm32_dma_of_match, 1123 }, 1124 }; 1125 1126 static int __init stm32_dma_init(void) 1127 { 1128 return platform_driver_probe(&stm32_dma_driver, stm32_dma_probe); 1129 } 1130 subsys_initcall(stm32_dma_init); 1131