1 // SPDX-License-Identifier: GPL-2.0 2 // 3 // Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved. 4 // 5 // Refer to drivers/dma/imx-sdma.c 6 7 #include <linux/init.h> 8 #include <linux/types.h> 9 #include <linux/mm.h> 10 #include <linux/interrupt.h> 11 #include <linux/clk.h> 12 #include <linux/wait.h> 13 #include <linux/sched.h> 14 #include <linux/semaphore.h> 15 #include <linux/device.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/slab.h> 18 #include <linux/platform_device.h> 19 #include <linux/dmaengine.h> 20 #include <linux/delay.h> 21 #include <linux/module.h> 22 #include <linux/stmp_device.h> 23 #include <linux/of.h> 24 #include <linux/of_device.h> 25 #include <linux/of_dma.h> 26 #include <linux/list.h> 27 #include <linux/dma/mxs-dma.h> 28 29 #include <asm/irq.h> 30 31 #include "dmaengine.h" 32 33 /* 34 * NOTE: The term "PIO" throughout the mxs-dma implementation means 35 * PIO mode of mxs apbh-dma and apbx-dma. With this working mode, 36 * dma can program the controller registers of peripheral devices. 37 */ 38 39 #define dma_is_apbh(mxs_dma) ((mxs_dma)->type == MXS_DMA_APBH) 40 #define apbh_is_old(mxs_dma) ((mxs_dma)->dev_id == IMX23_DMA) 41 42 #define HW_APBHX_CTRL0 0x000 43 #define BM_APBH_CTRL0_APB_BURST8_EN (1 << 29) 44 #define BM_APBH_CTRL0_APB_BURST_EN (1 << 28) 45 #define BP_APBH_CTRL0_RESET_CHANNEL 16 46 #define HW_APBHX_CTRL1 0x010 47 #define HW_APBHX_CTRL2 0x020 48 #define HW_APBHX_CHANNEL_CTRL 0x030 49 #define BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL 16 50 /* 51 * The offset of NXTCMDAR register is different per both dma type and version, 52 * while stride for each channel is all the same 0x70. 53 */ 54 #define HW_APBHX_CHn_NXTCMDAR(d, n) \ 55 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x050 : 0x110) + (n) * 0x70) 56 #define HW_APBHX_CHn_SEMA(d, n) \ 57 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x080 : 0x140) + (n) * 0x70) 58 #define HW_APBHX_CHn_BAR(d, n) \ 59 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x070 : 0x130) + (n) * 0x70) 60 #define HW_APBX_CHn_DEBUG1(d, n) (0x150 + (n) * 0x70) 61 62 /* 63 * ccw bits definitions 64 * 65 * COMMAND: 0..1 (2) 66 * CHAIN: 2 (1) 67 * IRQ: 3 (1) 68 * NAND_LOCK: 4 (1) - not implemented 69 * NAND_WAIT4READY: 5 (1) - not implemented 70 * DEC_SEM: 6 (1) 71 * WAIT4END: 7 (1) 72 * HALT_ON_TERMINATE: 8 (1) 73 * TERMINATE_FLUSH: 9 (1) 74 * RESERVED: 10..11 (2) 75 * PIO_NUM: 12..15 (4) 76 */ 77 #define BP_CCW_COMMAND 0 78 #define BM_CCW_COMMAND (3 << 0) 79 #define CCW_CHAIN (1 << 2) 80 #define CCW_IRQ (1 << 3) 81 #define CCW_WAIT4RDY (1 << 5) 82 #define CCW_DEC_SEM (1 << 6) 83 #define CCW_WAIT4END (1 << 7) 84 #define CCW_HALT_ON_TERM (1 << 8) 85 #define CCW_TERM_FLUSH (1 << 9) 86 #define BP_CCW_PIO_NUM 12 87 #define BM_CCW_PIO_NUM (0xf << 12) 88 89 #define BF_CCW(value, field) (((value) << BP_CCW_##field) & BM_CCW_##field) 90 91 #define MXS_DMA_CMD_NO_XFER 0 92 #define MXS_DMA_CMD_WRITE 1 93 #define MXS_DMA_CMD_READ 2 94 #define MXS_DMA_CMD_DMA_SENSE 3 /* not implemented */ 95 96 struct mxs_dma_ccw { 97 u32 next; 98 u16 bits; 99 u16 xfer_bytes; 100 #define MAX_XFER_BYTES 0xff00 101 u32 bufaddr; 102 #define MXS_PIO_WORDS 16 103 u32 pio_words[MXS_PIO_WORDS]; 104 }; 105 106 #define CCW_BLOCK_SIZE (4 * PAGE_SIZE) 107 #define NUM_CCW (int)(CCW_BLOCK_SIZE / sizeof(struct mxs_dma_ccw)) 108 109 struct mxs_dma_chan { 110 struct mxs_dma_engine *mxs_dma; 111 struct dma_chan chan; 112 struct dma_async_tx_descriptor desc; 113 struct tasklet_struct tasklet; 114 unsigned int chan_irq; 115 struct mxs_dma_ccw *ccw; 116 dma_addr_t ccw_phys; 117 int desc_count; 118 enum dma_status status; 119 unsigned int flags; 120 bool reset; 121 #define MXS_DMA_SG_LOOP (1 << 0) 122 #define MXS_DMA_USE_SEMAPHORE (1 << 1) 123 }; 124 125 #define MXS_DMA_CHANNELS 16 126 #define MXS_DMA_CHANNELS_MASK 0xffff 127 128 enum mxs_dma_devtype { 129 MXS_DMA_APBH, 130 MXS_DMA_APBX, 131 }; 132 133 enum mxs_dma_id { 134 IMX23_DMA, 135 IMX28_DMA, 136 }; 137 138 struct mxs_dma_engine { 139 enum mxs_dma_id dev_id; 140 enum mxs_dma_devtype type; 141 void __iomem *base; 142 struct clk *clk; 143 struct dma_device dma_device; 144 struct mxs_dma_chan mxs_chans[MXS_DMA_CHANNELS]; 145 struct platform_device *pdev; 146 unsigned int nr_channels; 147 }; 148 149 struct mxs_dma_type { 150 enum mxs_dma_id id; 151 enum mxs_dma_devtype type; 152 }; 153 154 static struct mxs_dma_type mxs_dma_types[] = { 155 { 156 .id = IMX23_DMA, 157 .type = MXS_DMA_APBH, 158 }, { 159 .id = IMX23_DMA, 160 .type = MXS_DMA_APBX, 161 }, { 162 .id = IMX28_DMA, 163 .type = MXS_DMA_APBH, 164 }, { 165 .id = IMX28_DMA, 166 .type = MXS_DMA_APBX, 167 } 168 }; 169 170 static const struct platform_device_id mxs_dma_ids[] = { 171 { 172 .name = "imx23-dma-apbh", 173 .driver_data = (kernel_ulong_t) &mxs_dma_types[0], 174 }, { 175 .name = "imx23-dma-apbx", 176 .driver_data = (kernel_ulong_t) &mxs_dma_types[1], 177 }, { 178 .name = "imx28-dma-apbh", 179 .driver_data = (kernel_ulong_t) &mxs_dma_types[2], 180 }, { 181 .name = "imx28-dma-apbx", 182 .driver_data = (kernel_ulong_t) &mxs_dma_types[3], 183 }, { 184 /* end of list */ 185 } 186 }; 187 188 static const struct of_device_id mxs_dma_dt_ids[] = { 189 { .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], }, 190 { .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], }, 191 { .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], }, 192 { .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], }, 193 { /* sentinel */ } 194 }; 195 MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids); 196 197 static struct mxs_dma_chan *to_mxs_dma_chan(struct dma_chan *chan) 198 { 199 return container_of(chan, struct mxs_dma_chan, chan); 200 } 201 202 static void mxs_dma_reset_chan(struct dma_chan *chan) 203 { 204 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 205 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 206 int chan_id = mxs_chan->chan.chan_id; 207 208 /* 209 * mxs dma channel resets can cause a channel stall. To recover from a 210 * channel stall, we have to reset the whole DMA engine. To avoid this, 211 * we use cyclic DMA with semaphores, that are enhanced in 212 * mxs_dma_int_handler. To reset the channel, we can simply stop writing 213 * into the semaphore counter. 214 */ 215 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && 216 mxs_chan->flags & MXS_DMA_SG_LOOP) { 217 mxs_chan->reset = true; 218 } else if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) { 219 writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL), 220 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 221 } else { 222 unsigned long elapsed = 0; 223 const unsigned long max_wait = 50000; /* 50ms */ 224 void __iomem *reg_dbg1 = mxs_dma->base + 225 HW_APBX_CHn_DEBUG1(mxs_dma, chan_id); 226 227 /* 228 * On i.MX28 APBX, the DMA channel can stop working if we reset 229 * the channel while it is in READ_FLUSH (0x08) state. 230 * We wait here until we leave the state. Then we trigger the 231 * reset. Waiting a maximum of 50ms, the kernel shouldn't crash 232 * because of this. 233 */ 234 while ((readl(reg_dbg1) & 0xf) == 0x8 && elapsed < max_wait) { 235 udelay(100); 236 elapsed += 100; 237 } 238 239 if (elapsed >= max_wait) 240 dev_err(&mxs_chan->mxs_dma->pdev->dev, 241 "Failed waiting for the DMA channel %d to leave state READ_FLUSH, trying to reset channel in READ_FLUSH state now\n", 242 chan_id); 243 244 writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL), 245 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET); 246 } 247 248 mxs_chan->status = DMA_COMPLETE; 249 } 250 251 static void mxs_dma_enable_chan(struct dma_chan *chan) 252 { 253 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 254 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 255 int chan_id = mxs_chan->chan.chan_id; 256 257 /* set cmd_addr up */ 258 writel(mxs_chan->ccw_phys, 259 mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(mxs_dma, chan_id)); 260 261 /* write 1 to SEMA to kick off the channel */ 262 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && 263 mxs_chan->flags & MXS_DMA_SG_LOOP) { 264 /* A cyclic DMA consists of at least 2 segments, so initialize 265 * the semaphore with 2 so we have enough time to add 1 to the 266 * semaphore if we need to */ 267 writel(2, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); 268 } else { 269 writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); 270 } 271 mxs_chan->reset = false; 272 } 273 274 static void mxs_dma_disable_chan(struct dma_chan *chan) 275 { 276 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 277 278 mxs_chan->status = DMA_COMPLETE; 279 } 280 281 static int mxs_dma_pause_chan(struct dma_chan *chan) 282 { 283 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 284 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 285 int chan_id = mxs_chan->chan.chan_id; 286 287 /* freeze the channel */ 288 if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) 289 writel(1 << chan_id, 290 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 291 else 292 writel(1 << chan_id, 293 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET); 294 295 mxs_chan->status = DMA_PAUSED; 296 return 0; 297 } 298 299 static int mxs_dma_resume_chan(struct dma_chan *chan) 300 { 301 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 302 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 303 int chan_id = mxs_chan->chan.chan_id; 304 305 /* unfreeze the channel */ 306 if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) 307 writel(1 << chan_id, 308 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_CLR); 309 else 310 writel(1 << chan_id, 311 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_CLR); 312 313 mxs_chan->status = DMA_IN_PROGRESS; 314 return 0; 315 } 316 317 static dma_cookie_t mxs_dma_tx_submit(struct dma_async_tx_descriptor *tx) 318 { 319 return dma_cookie_assign(tx); 320 } 321 322 static void mxs_dma_tasklet(struct tasklet_struct *t) 323 { 324 struct mxs_dma_chan *mxs_chan = from_tasklet(mxs_chan, t, tasklet); 325 326 dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL); 327 } 328 329 static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq) 330 { 331 int i; 332 333 for (i = 0; i != mxs_dma->nr_channels; ++i) 334 if (mxs_dma->mxs_chans[i].chan_irq == irq) 335 return i; 336 337 return -EINVAL; 338 } 339 340 static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id) 341 { 342 struct mxs_dma_engine *mxs_dma = dev_id; 343 struct mxs_dma_chan *mxs_chan; 344 u32 completed; 345 u32 err; 346 int chan = mxs_dma_irq_to_chan(mxs_dma, irq); 347 348 if (chan < 0) 349 return IRQ_NONE; 350 351 /* completion status */ 352 completed = readl(mxs_dma->base + HW_APBHX_CTRL1); 353 completed = (completed >> chan) & 0x1; 354 355 /* Clear interrupt */ 356 writel((1 << chan), 357 mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR); 358 359 /* error status */ 360 err = readl(mxs_dma->base + HW_APBHX_CTRL2); 361 err &= (1 << (MXS_DMA_CHANNELS + chan)) | (1 << chan); 362 363 /* 364 * error status bit is in the upper 16 bits, error irq bit in the lower 365 * 16 bits. We transform it into a simpler error code: 366 * err: 0x00 = no error, 0x01 = TERMINATION, 0x02 = BUS_ERROR 367 */ 368 err = (err >> (MXS_DMA_CHANNELS + chan)) + (err >> chan); 369 370 /* Clear error irq */ 371 writel((1 << chan), 372 mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR); 373 374 /* 375 * When both completion and error of termination bits set at the 376 * same time, we do not take it as an error. IOW, it only becomes 377 * an error we need to handle here in case of either it's a bus 378 * error or a termination error with no completion. 0x01 is termination 379 * error, so we can subtract err & completed to get the real error case. 380 */ 381 err -= err & completed; 382 383 mxs_chan = &mxs_dma->mxs_chans[chan]; 384 385 if (err) { 386 dev_dbg(mxs_dma->dma_device.dev, 387 "%s: error in channel %d\n", __func__, 388 chan); 389 mxs_chan->status = DMA_ERROR; 390 mxs_dma_reset_chan(&mxs_chan->chan); 391 } else if (mxs_chan->status != DMA_COMPLETE) { 392 if (mxs_chan->flags & MXS_DMA_SG_LOOP) { 393 mxs_chan->status = DMA_IN_PROGRESS; 394 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE) 395 writel(1, mxs_dma->base + 396 HW_APBHX_CHn_SEMA(mxs_dma, chan)); 397 } else { 398 mxs_chan->status = DMA_COMPLETE; 399 } 400 } 401 402 if (mxs_chan->status == DMA_COMPLETE) { 403 if (mxs_chan->reset) 404 return IRQ_HANDLED; 405 dma_cookie_complete(&mxs_chan->desc); 406 } 407 408 /* schedule tasklet on this channel */ 409 tasklet_schedule(&mxs_chan->tasklet); 410 411 return IRQ_HANDLED; 412 } 413 414 static int mxs_dma_alloc_chan_resources(struct dma_chan *chan) 415 { 416 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 417 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 418 int ret; 419 420 mxs_chan->ccw = dma_alloc_coherent(mxs_dma->dma_device.dev, 421 CCW_BLOCK_SIZE, 422 &mxs_chan->ccw_phys, GFP_KERNEL); 423 if (!mxs_chan->ccw) { 424 ret = -ENOMEM; 425 goto err_alloc; 426 } 427 428 ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler, 429 0, "mxs-dma", mxs_dma); 430 if (ret) 431 goto err_irq; 432 433 ret = clk_prepare_enable(mxs_dma->clk); 434 if (ret) 435 goto err_clk; 436 437 mxs_dma_reset_chan(chan); 438 439 dma_async_tx_descriptor_init(&mxs_chan->desc, chan); 440 mxs_chan->desc.tx_submit = mxs_dma_tx_submit; 441 442 /* the descriptor is ready */ 443 async_tx_ack(&mxs_chan->desc); 444 445 return 0; 446 447 err_clk: 448 free_irq(mxs_chan->chan_irq, mxs_dma); 449 err_irq: 450 dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE, 451 mxs_chan->ccw, mxs_chan->ccw_phys); 452 err_alloc: 453 return ret; 454 } 455 456 static void mxs_dma_free_chan_resources(struct dma_chan *chan) 457 { 458 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 459 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 460 461 mxs_dma_disable_chan(chan); 462 463 free_irq(mxs_chan->chan_irq, mxs_dma); 464 465 dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE, 466 mxs_chan->ccw, mxs_chan->ccw_phys); 467 468 clk_disable_unprepare(mxs_dma->clk); 469 } 470 471 /* 472 * How to use the flags for ->device_prep_slave_sg() : 473 * [1] If there is only one DMA command in the DMA chain, the code should be: 474 * ...... 475 * ->device_prep_slave_sg(DMA_CTRL_ACK); 476 * ...... 477 * [2] If there are two DMA commands in the DMA chain, the code should be 478 * ...... 479 * ->device_prep_slave_sg(0); 480 * ...... 481 * ->device_prep_slave_sg(DMA_CTRL_ACK); 482 * ...... 483 * [3] If there are more than two DMA commands in the DMA chain, the code 484 * should be: 485 * ...... 486 * ->device_prep_slave_sg(0); // First 487 * ...... 488 * ->device_prep_slave_sg(DMA_CTRL_ACK]); 489 * ...... 490 * ->device_prep_slave_sg(DMA_CTRL_ACK); // Last 491 * ...... 492 */ 493 static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg( 494 struct dma_chan *chan, struct scatterlist *sgl, 495 unsigned int sg_len, enum dma_transfer_direction direction, 496 unsigned long flags, void *context) 497 { 498 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 499 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 500 struct mxs_dma_ccw *ccw; 501 struct scatterlist *sg; 502 u32 i, j; 503 u32 *pio; 504 int idx = 0; 505 506 if (mxs_chan->status == DMA_IN_PROGRESS) 507 idx = mxs_chan->desc_count; 508 509 if (sg_len + idx > NUM_CCW) { 510 dev_err(mxs_dma->dma_device.dev, 511 "maximum number of sg exceeded: %d > %d\n", 512 sg_len, NUM_CCW); 513 goto err_out; 514 } 515 516 mxs_chan->status = DMA_IN_PROGRESS; 517 mxs_chan->flags = 0; 518 519 /* 520 * If the sg is prepared with append flag set, the sg 521 * will be appended to the last prepared sg. 522 */ 523 if (idx) { 524 BUG_ON(idx < 1); 525 ccw = &mxs_chan->ccw[idx - 1]; 526 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx; 527 ccw->bits |= CCW_CHAIN; 528 ccw->bits &= ~CCW_IRQ; 529 ccw->bits &= ~CCW_DEC_SEM; 530 } else { 531 idx = 0; 532 } 533 534 if (direction == DMA_TRANS_NONE) { 535 ccw = &mxs_chan->ccw[idx++]; 536 pio = (u32 *) sgl; 537 538 for (j = 0; j < sg_len;) 539 ccw->pio_words[j++] = *pio++; 540 541 ccw->bits = 0; 542 ccw->bits |= CCW_IRQ; 543 ccw->bits |= CCW_DEC_SEM; 544 if (flags & MXS_DMA_CTRL_WAIT4END) 545 ccw->bits |= CCW_WAIT4END; 546 ccw->bits |= CCW_HALT_ON_TERM; 547 ccw->bits |= CCW_TERM_FLUSH; 548 ccw->bits |= BF_CCW(sg_len, PIO_NUM); 549 ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND); 550 if (flags & MXS_DMA_CTRL_WAIT4RDY) 551 ccw->bits |= CCW_WAIT4RDY; 552 } else { 553 for_each_sg(sgl, sg, sg_len, i) { 554 if (sg_dma_len(sg) > MAX_XFER_BYTES) { 555 dev_err(mxs_dma->dma_device.dev, "maximum bytes for sg entry exceeded: %d > %d\n", 556 sg_dma_len(sg), MAX_XFER_BYTES); 557 goto err_out; 558 } 559 560 ccw = &mxs_chan->ccw[idx++]; 561 562 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx; 563 ccw->bufaddr = sg->dma_address; 564 ccw->xfer_bytes = sg_dma_len(sg); 565 566 ccw->bits = 0; 567 ccw->bits |= CCW_CHAIN; 568 ccw->bits |= CCW_HALT_ON_TERM; 569 ccw->bits |= CCW_TERM_FLUSH; 570 ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ? 571 MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, 572 COMMAND); 573 574 if (i + 1 == sg_len) { 575 ccw->bits &= ~CCW_CHAIN; 576 ccw->bits |= CCW_IRQ; 577 ccw->bits |= CCW_DEC_SEM; 578 if (flags & MXS_DMA_CTRL_WAIT4END) 579 ccw->bits |= CCW_WAIT4END; 580 } 581 } 582 } 583 mxs_chan->desc_count = idx; 584 585 return &mxs_chan->desc; 586 587 err_out: 588 mxs_chan->status = DMA_ERROR; 589 return NULL; 590 } 591 592 static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic( 593 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, 594 size_t period_len, enum dma_transfer_direction direction, 595 unsigned long flags) 596 { 597 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 598 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 599 u32 num_periods = buf_len / period_len; 600 u32 i = 0, buf = 0; 601 602 if (mxs_chan->status == DMA_IN_PROGRESS) 603 return NULL; 604 605 mxs_chan->status = DMA_IN_PROGRESS; 606 mxs_chan->flags |= MXS_DMA_SG_LOOP; 607 mxs_chan->flags |= MXS_DMA_USE_SEMAPHORE; 608 609 if (num_periods > NUM_CCW) { 610 dev_err(mxs_dma->dma_device.dev, 611 "maximum number of sg exceeded: %d > %d\n", 612 num_periods, NUM_CCW); 613 goto err_out; 614 } 615 616 if (period_len > MAX_XFER_BYTES) { 617 dev_err(mxs_dma->dma_device.dev, 618 "maximum period size exceeded: %zu > %d\n", 619 period_len, MAX_XFER_BYTES); 620 goto err_out; 621 } 622 623 while (buf < buf_len) { 624 struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i]; 625 626 if (i + 1 == num_periods) 627 ccw->next = mxs_chan->ccw_phys; 628 else 629 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1); 630 631 ccw->bufaddr = dma_addr; 632 ccw->xfer_bytes = period_len; 633 634 ccw->bits = 0; 635 ccw->bits |= CCW_CHAIN; 636 ccw->bits |= CCW_IRQ; 637 ccw->bits |= CCW_HALT_ON_TERM; 638 ccw->bits |= CCW_TERM_FLUSH; 639 ccw->bits |= CCW_DEC_SEM; 640 ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ? 641 MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND); 642 643 dma_addr += period_len; 644 buf += period_len; 645 646 i++; 647 } 648 mxs_chan->desc_count = i; 649 650 return &mxs_chan->desc; 651 652 err_out: 653 mxs_chan->status = DMA_ERROR; 654 return NULL; 655 } 656 657 static int mxs_dma_terminate_all(struct dma_chan *chan) 658 { 659 mxs_dma_reset_chan(chan); 660 mxs_dma_disable_chan(chan); 661 662 return 0; 663 } 664 665 static enum dma_status mxs_dma_tx_status(struct dma_chan *chan, 666 dma_cookie_t cookie, struct dma_tx_state *txstate) 667 { 668 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 669 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 670 u32 residue = 0; 671 672 if (mxs_chan->status == DMA_IN_PROGRESS && 673 mxs_chan->flags & MXS_DMA_SG_LOOP) { 674 struct mxs_dma_ccw *last_ccw; 675 u32 bar; 676 677 last_ccw = &mxs_chan->ccw[mxs_chan->desc_count - 1]; 678 residue = last_ccw->xfer_bytes + last_ccw->bufaddr; 679 680 bar = readl(mxs_dma->base + 681 HW_APBHX_CHn_BAR(mxs_dma, chan->chan_id)); 682 residue -= bar; 683 } 684 685 dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, 686 residue); 687 688 return mxs_chan->status; 689 } 690 691 static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma) 692 { 693 int ret; 694 695 ret = clk_prepare_enable(mxs_dma->clk); 696 if (ret) 697 return ret; 698 699 ret = stmp_reset_block(mxs_dma->base); 700 if (ret) 701 goto err_out; 702 703 /* enable apbh burst */ 704 if (dma_is_apbh(mxs_dma)) { 705 writel(BM_APBH_CTRL0_APB_BURST_EN, 706 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 707 writel(BM_APBH_CTRL0_APB_BURST8_EN, 708 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 709 } 710 711 /* enable irq for all the channels */ 712 writel(MXS_DMA_CHANNELS_MASK << MXS_DMA_CHANNELS, 713 mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_SET); 714 715 err_out: 716 clk_disable_unprepare(mxs_dma->clk); 717 return ret; 718 } 719 720 struct mxs_dma_filter_param { 721 unsigned int chan_id; 722 }; 723 724 static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param) 725 { 726 struct mxs_dma_filter_param *param = fn_param; 727 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 728 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 729 int chan_irq; 730 731 if (chan->chan_id != param->chan_id) 732 return false; 733 734 chan_irq = platform_get_irq(mxs_dma->pdev, param->chan_id); 735 if (chan_irq < 0) 736 return false; 737 738 mxs_chan->chan_irq = chan_irq; 739 740 return true; 741 } 742 743 static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, 744 struct of_dma *ofdma) 745 { 746 struct mxs_dma_engine *mxs_dma = ofdma->of_dma_data; 747 dma_cap_mask_t mask = mxs_dma->dma_device.cap_mask; 748 struct mxs_dma_filter_param param; 749 750 if (dma_spec->args_count != 1) 751 return NULL; 752 753 param.chan_id = dma_spec->args[0]; 754 755 if (param.chan_id >= mxs_dma->nr_channels) 756 return NULL; 757 758 return __dma_request_channel(&mask, mxs_dma_filter_fn, ¶m, 759 ofdma->of_node); 760 } 761 762 static int __init mxs_dma_probe(struct platform_device *pdev) 763 { 764 struct device_node *np = pdev->dev.of_node; 765 const struct platform_device_id *id_entry; 766 const struct of_device_id *of_id; 767 const struct mxs_dma_type *dma_type; 768 struct mxs_dma_engine *mxs_dma; 769 struct resource *iores; 770 int ret, i; 771 772 mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL); 773 if (!mxs_dma) 774 return -ENOMEM; 775 776 ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels); 777 if (ret) { 778 dev_err(&pdev->dev, "failed to read dma-channels\n"); 779 return ret; 780 } 781 782 of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev); 783 if (of_id) 784 id_entry = of_id->data; 785 else 786 id_entry = platform_get_device_id(pdev); 787 788 dma_type = (struct mxs_dma_type *)id_entry->driver_data; 789 mxs_dma->type = dma_type->type; 790 mxs_dma->dev_id = dma_type->id; 791 792 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); 793 mxs_dma->base = devm_ioremap_resource(&pdev->dev, iores); 794 if (IS_ERR(mxs_dma->base)) 795 return PTR_ERR(mxs_dma->base); 796 797 mxs_dma->clk = devm_clk_get(&pdev->dev, NULL); 798 if (IS_ERR(mxs_dma->clk)) 799 return PTR_ERR(mxs_dma->clk); 800 801 dma_cap_set(DMA_SLAVE, mxs_dma->dma_device.cap_mask); 802 dma_cap_set(DMA_CYCLIC, mxs_dma->dma_device.cap_mask); 803 804 INIT_LIST_HEAD(&mxs_dma->dma_device.channels); 805 806 /* Initialize channel parameters */ 807 for (i = 0; i < MXS_DMA_CHANNELS; i++) { 808 struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i]; 809 810 mxs_chan->mxs_dma = mxs_dma; 811 mxs_chan->chan.device = &mxs_dma->dma_device; 812 dma_cookie_init(&mxs_chan->chan); 813 814 tasklet_setup(&mxs_chan->tasklet, mxs_dma_tasklet); 815 816 817 /* Add the channel to mxs_chan list */ 818 list_add_tail(&mxs_chan->chan.device_node, 819 &mxs_dma->dma_device.channels); 820 } 821 822 ret = mxs_dma_init(mxs_dma); 823 if (ret) 824 return ret; 825 826 mxs_dma->pdev = pdev; 827 mxs_dma->dma_device.dev = &pdev->dev; 828 829 /* mxs_dma gets 65535 bytes maximum sg size */ 830 dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES); 831 832 mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources; 833 mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources; 834 mxs_dma->dma_device.device_tx_status = mxs_dma_tx_status; 835 mxs_dma->dma_device.device_prep_slave_sg = mxs_dma_prep_slave_sg; 836 mxs_dma->dma_device.device_prep_dma_cyclic = mxs_dma_prep_dma_cyclic; 837 mxs_dma->dma_device.device_pause = mxs_dma_pause_chan; 838 mxs_dma->dma_device.device_resume = mxs_dma_resume_chan; 839 mxs_dma->dma_device.device_terminate_all = mxs_dma_terminate_all; 840 mxs_dma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 841 mxs_dma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 842 mxs_dma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 843 mxs_dma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 844 mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan; 845 846 ret = dmaenginem_async_device_register(&mxs_dma->dma_device); 847 if (ret) { 848 dev_err(mxs_dma->dma_device.dev, "unable to register\n"); 849 return ret; 850 } 851 852 ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma); 853 if (ret) { 854 dev_err(mxs_dma->dma_device.dev, 855 "failed to register controller\n"); 856 } 857 858 dev_info(mxs_dma->dma_device.dev, "initialized\n"); 859 860 return 0; 861 } 862 863 static struct platform_driver mxs_dma_driver = { 864 .driver = { 865 .name = "mxs-dma", 866 .of_match_table = mxs_dma_dt_ids, 867 }, 868 .id_table = mxs_dma_ids, 869 }; 870 871 static int __init mxs_dma_module_init(void) 872 { 873 return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe); 874 } 875 subsys_initcall(mxs_dma_module_init); 876