1 /* 2 * Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved. 3 * 4 * Refer to drivers/dma/imx-sdma.c 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/init.h> 12 #include <linux/types.h> 13 #include <linux/mm.h> 14 #include <linux/interrupt.h> 15 #include <linux/clk.h> 16 #include <linux/wait.h> 17 #include <linux/sched.h> 18 #include <linux/semaphore.h> 19 #include <linux/device.h> 20 #include <linux/dma-mapping.h> 21 #include <linux/slab.h> 22 #include <linux/platform_device.h> 23 #include <linux/dmaengine.h> 24 #include <linux/delay.h> 25 #include <linux/module.h> 26 #include <linux/stmp_device.h> 27 #include <linux/of.h> 28 #include <linux/of_device.h> 29 #include <linux/of_dma.h> 30 #include <linux/list.h> 31 32 #include <asm/irq.h> 33 34 #include "dmaengine.h" 35 36 /* 37 * NOTE: The term "PIO" throughout the mxs-dma implementation means 38 * PIO mode of mxs apbh-dma and apbx-dma. With this working mode, 39 * dma can program the controller registers of peripheral devices. 40 */ 41 42 #define dma_is_apbh(mxs_dma) ((mxs_dma)->type == MXS_DMA_APBH) 43 #define apbh_is_old(mxs_dma) ((mxs_dma)->dev_id == IMX23_DMA) 44 45 #define HW_APBHX_CTRL0 0x000 46 #define BM_APBH_CTRL0_APB_BURST8_EN (1 << 29) 47 #define BM_APBH_CTRL0_APB_BURST_EN (1 << 28) 48 #define BP_APBH_CTRL0_RESET_CHANNEL 16 49 #define HW_APBHX_CTRL1 0x010 50 #define HW_APBHX_CTRL2 0x020 51 #define HW_APBHX_CHANNEL_CTRL 0x030 52 #define BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL 16 53 /* 54 * The offset of NXTCMDAR register is different per both dma type and version, 55 * while stride for each channel is all the same 0x70. 56 */ 57 #define HW_APBHX_CHn_NXTCMDAR(d, n) \ 58 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x050 : 0x110) + (n) * 0x70) 59 #define HW_APBHX_CHn_SEMA(d, n) \ 60 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x080 : 0x140) + (n) * 0x70) 61 #define HW_APBHX_CHn_BAR(d, n) \ 62 (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x070 : 0x130) + (n) * 0x70) 63 #define HW_APBX_CHn_DEBUG1(d, n) (0x150 + (n) * 0x70) 64 65 /* 66 * ccw bits definitions 67 * 68 * COMMAND: 0..1 (2) 69 * CHAIN: 2 (1) 70 * IRQ: 3 (1) 71 * NAND_LOCK: 4 (1) - not implemented 72 * NAND_WAIT4READY: 5 (1) - not implemented 73 * DEC_SEM: 6 (1) 74 * WAIT4END: 7 (1) 75 * HALT_ON_TERMINATE: 8 (1) 76 * TERMINATE_FLUSH: 9 (1) 77 * RESERVED: 10..11 (2) 78 * PIO_NUM: 12..15 (4) 79 */ 80 #define BP_CCW_COMMAND 0 81 #define BM_CCW_COMMAND (3 << 0) 82 #define CCW_CHAIN (1 << 2) 83 #define CCW_IRQ (1 << 3) 84 #define CCW_DEC_SEM (1 << 6) 85 #define CCW_WAIT4END (1 << 7) 86 #define CCW_HALT_ON_TERM (1 << 8) 87 #define CCW_TERM_FLUSH (1 << 9) 88 #define BP_CCW_PIO_NUM 12 89 #define BM_CCW_PIO_NUM (0xf << 12) 90 91 #define BF_CCW(value, field) (((value) << BP_CCW_##field) & BM_CCW_##field) 92 93 #define MXS_DMA_CMD_NO_XFER 0 94 #define MXS_DMA_CMD_WRITE 1 95 #define MXS_DMA_CMD_READ 2 96 #define MXS_DMA_CMD_DMA_SENSE 3 /* not implemented */ 97 98 struct mxs_dma_ccw { 99 u32 next; 100 u16 bits; 101 u16 xfer_bytes; 102 #define MAX_XFER_BYTES 0xff00 103 u32 bufaddr; 104 #define MXS_PIO_WORDS 16 105 u32 pio_words[MXS_PIO_WORDS]; 106 }; 107 108 #define CCW_BLOCK_SIZE (4 * PAGE_SIZE) 109 #define NUM_CCW (int)(CCW_BLOCK_SIZE / sizeof(struct mxs_dma_ccw)) 110 111 struct mxs_dma_chan { 112 struct mxs_dma_engine *mxs_dma; 113 struct dma_chan chan; 114 struct dma_async_tx_descriptor desc; 115 struct tasklet_struct tasklet; 116 unsigned int chan_irq; 117 struct mxs_dma_ccw *ccw; 118 dma_addr_t ccw_phys; 119 int desc_count; 120 enum dma_status status; 121 unsigned int flags; 122 bool reset; 123 #define MXS_DMA_SG_LOOP (1 << 0) 124 #define MXS_DMA_USE_SEMAPHORE (1 << 1) 125 }; 126 127 #define MXS_DMA_CHANNELS 16 128 #define MXS_DMA_CHANNELS_MASK 0xffff 129 130 enum mxs_dma_devtype { 131 MXS_DMA_APBH, 132 MXS_DMA_APBX, 133 }; 134 135 enum mxs_dma_id { 136 IMX23_DMA, 137 IMX28_DMA, 138 }; 139 140 struct mxs_dma_engine { 141 enum mxs_dma_id dev_id; 142 enum mxs_dma_devtype type; 143 void __iomem *base; 144 struct clk *clk; 145 struct dma_device dma_device; 146 struct device_dma_parameters dma_parms; 147 struct mxs_dma_chan mxs_chans[MXS_DMA_CHANNELS]; 148 struct platform_device *pdev; 149 unsigned int nr_channels; 150 }; 151 152 struct mxs_dma_type { 153 enum mxs_dma_id id; 154 enum mxs_dma_devtype type; 155 }; 156 157 static struct mxs_dma_type mxs_dma_types[] = { 158 { 159 .id = IMX23_DMA, 160 .type = MXS_DMA_APBH, 161 }, { 162 .id = IMX23_DMA, 163 .type = MXS_DMA_APBX, 164 }, { 165 .id = IMX28_DMA, 166 .type = MXS_DMA_APBH, 167 }, { 168 .id = IMX28_DMA, 169 .type = MXS_DMA_APBX, 170 } 171 }; 172 173 static const struct platform_device_id mxs_dma_ids[] = { 174 { 175 .name = "imx23-dma-apbh", 176 .driver_data = (kernel_ulong_t) &mxs_dma_types[0], 177 }, { 178 .name = "imx23-dma-apbx", 179 .driver_data = (kernel_ulong_t) &mxs_dma_types[1], 180 }, { 181 .name = "imx28-dma-apbh", 182 .driver_data = (kernel_ulong_t) &mxs_dma_types[2], 183 }, { 184 .name = "imx28-dma-apbx", 185 .driver_data = (kernel_ulong_t) &mxs_dma_types[3], 186 }, { 187 /* end of list */ 188 } 189 }; 190 191 static const struct of_device_id mxs_dma_dt_ids[] = { 192 { .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], }, 193 { .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], }, 194 { .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], }, 195 { .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], }, 196 { /* sentinel */ } 197 }; 198 MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids); 199 200 static struct mxs_dma_chan *to_mxs_dma_chan(struct dma_chan *chan) 201 { 202 return container_of(chan, struct mxs_dma_chan, chan); 203 } 204 205 static void mxs_dma_reset_chan(struct dma_chan *chan) 206 { 207 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 208 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 209 int chan_id = mxs_chan->chan.chan_id; 210 211 /* 212 * mxs dma channel resets can cause a channel stall. To recover from a 213 * channel stall, we have to reset the whole DMA engine. To avoid this, 214 * we use cyclic DMA with semaphores, that are enhanced in 215 * mxs_dma_int_handler. To reset the channel, we can simply stop writing 216 * into the semaphore counter. 217 */ 218 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && 219 mxs_chan->flags & MXS_DMA_SG_LOOP) { 220 mxs_chan->reset = true; 221 } else if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) { 222 writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL), 223 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 224 } else { 225 unsigned long elapsed = 0; 226 const unsigned long max_wait = 50000; /* 50ms */ 227 void __iomem *reg_dbg1 = mxs_dma->base + 228 HW_APBX_CHn_DEBUG1(mxs_dma, chan_id); 229 230 /* 231 * On i.MX28 APBX, the DMA channel can stop working if we reset 232 * the channel while it is in READ_FLUSH (0x08) state. 233 * We wait here until we leave the state. Then we trigger the 234 * reset. Waiting a maximum of 50ms, the kernel shouldn't crash 235 * because of this. 236 */ 237 while ((readl(reg_dbg1) & 0xf) == 0x8 && elapsed < max_wait) { 238 udelay(100); 239 elapsed += 100; 240 } 241 242 if (elapsed >= max_wait) 243 dev_err(&mxs_chan->mxs_dma->pdev->dev, 244 "Failed waiting for the DMA channel %d to leave state READ_FLUSH, trying to reset channel in READ_FLUSH state now\n", 245 chan_id); 246 247 writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL), 248 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET); 249 } 250 251 mxs_chan->status = DMA_COMPLETE; 252 } 253 254 static void mxs_dma_enable_chan(struct dma_chan *chan) 255 { 256 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 257 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 258 int chan_id = mxs_chan->chan.chan_id; 259 260 /* set cmd_addr up */ 261 writel(mxs_chan->ccw_phys, 262 mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(mxs_dma, chan_id)); 263 264 /* write 1 to SEMA to kick off the channel */ 265 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && 266 mxs_chan->flags & MXS_DMA_SG_LOOP) { 267 /* A cyclic DMA consists of at least 2 segments, so initialize 268 * the semaphore with 2 so we have enough time to add 1 to the 269 * semaphore if we need to */ 270 writel(2, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); 271 } else { 272 writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); 273 } 274 mxs_chan->reset = false; 275 } 276 277 static void mxs_dma_disable_chan(struct dma_chan *chan) 278 { 279 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 280 281 mxs_chan->status = DMA_COMPLETE; 282 } 283 284 static int mxs_dma_pause_chan(struct dma_chan *chan) 285 { 286 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 287 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 288 int chan_id = mxs_chan->chan.chan_id; 289 290 /* freeze the channel */ 291 if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) 292 writel(1 << chan_id, 293 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 294 else 295 writel(1 << chan_id, 296 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET); 297 298 mxs_chan->status = DMA_PAUSED; 299 return 0; 300 } 301 302 static int mxs_dma_resume_chan(struct dma_chan *chan) 303 { 304 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 305 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 306 int chan_id = mxs_chan->chan.chan_id; 307 308 /* unfreeze the channel */ 309 if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) 310 writel(1 << chan_id, 311 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_CLR); 312 else 313 writel(1 << chan_id, 314 mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_CLR); 315 316 mxs_chan->status = DMA_IN_PROGRESS; 317 return 0; 318 } 319 320 static dma_cookie_t mxs_dma_tx_submit(struct dma_async_tx_descriptor *tx) 321 { 322 return dma_cookie_assign(tx); 323 } 324 325 static void mxs_dma_tasklet(unsigned long data) 326 { 327 struct mxs_dma_chan *mxs_chan = (struct mxs_dma_chan *) data; 328 329 dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL); 330 } 331 332 static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq) 333 { 334 int i; 335 336 for (i = 0; i != mxs_dma->nr_channels; ++i) 337 if (mxs_dma->mxs_chans[i].chan_irq == irq) 338 return i; 339 340 return -EINVAL; 341 } 342 343 static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id) 344 { 345 struct mxs_dma_engine *mxs_dma = dev_id; 346 struct mxs_dma_chan *mxs_chan; 347 u32 completed; 348 u32 err; 349 int chan = mxs_dma_irq_to_chan(mxs_dma, irq); 350 351 if (chan < 0) 352 return IRQ_NONE; 353 354 /* completion status */ 355 completed = readl(mxs_dma->base + HW_APBHX_CTRL1); 356 completed = (completed >> chan) & 0x1; 357 358 /* Clear interrupt */ 359 writel((1 << chan), 360 mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR); 361 362 /* error status */ 363 err = readl(mxs_dma->base + HW_APBHX_CTRL2); 364 err &= (1 << (MXS_DMA_CHANNELS + chan)) | (1 << chan); 365 366 /* 367 * error status bit is in the upper 16 bits, error irq bit in the lower 368 * 16 bits. We transform it into a simpler error code: 369 * err: 0x00 = no error, 0x01 = TERMINATION, 0x02 = BUS_ERROR 370 */ 371 err = (err >> (MXS_DMA_CHANNELS + chan)) + (err >> chan); 372 373 /* Clear error irq */ 374 writel((1 << chan), 375 mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR); 376 377 /* 378 * When both completion and error of termination bits set at the 379 * same time, we do not take it as an error. IOW, it only becomes 380 * an error we need to handle here in case of either it's a bus 381 * error or a termination error with no completion. 0x01 is termination 382 * error, so we can subtract err & completed to get the real error case. 383 */ 384 err -= err & completed; 385 386 mxs_chan = &mxs_dma->mxs_chans[chan]; 387 388 if (err) { 389 dev_dbg(mxs_dma->dma_device.dev, 390 "%s: error in channel %d\n", __func__, 391 chan); 392 mxs_chan->status = DMA_ERROR; 393 mxs_dma_reset_chan(&mxs_chan->chan); 394 } else if (mxs_chan->status != DMA_COMPLETE) { 395 if (mxs_chan->flags & MXS_DMA_SG_LOOP) { 396 mxs_chan->status = DMA_IN_PROGRESS; 397 if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE) 398 writel(1, mxs_dma->base + 399 HW_APBHX_CHn_SEMA(mxs_dma, chan)); 400 } else { 401 mxs_chan->status = DMA_COMPLETE; 402 } 403 } 404 405 if (mxs_chan->status == DMA_COMPLETE) { 406 if (mxs_chan->reset) 407 return IRQ_HANDLED; 408 dma_cookie_complete(&mxs_chan->desc); 409 } 410 411 /* schedule tasklet on this channel */ 412 tasklet_schedule(&mxs_chan->tasklet); 413 414 return IRQ_HANDLED; 415 } 416 417 static int mxs_dma_alloc_chan_resources(struct dma_chan *chan) 418 { 419 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 420 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 421 int ret; 422 423 mxs_chan->ccw = dma_zalloc_coherent(mxs_dma->dma_device.dev, 424 CCW_BLOCK_SIZE, 425 &mxs_chan->ccw_phys, GFP_KERNEL); 426 if (!mxs_chan->ccw) { 427 ret = -ENOMEM; 428 goto err_alloc; 429 } 430 431 ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler, 432 0, "mxs-dma", mxs_dma); 433 if (ret) 434 goto err_irq; 435 436 ret = clk_prepare_enable(mxs_dma->clk); 437 if (ret) 438 goto err_clk; 439 440 mxs_dma_reset_chan(chan); 441 442 dma_async_tx_descriptor_init(&mxs_chan->desc, chan); 443 mxs_chan->desc.tx_submit = mxs_dma_tx_submit; 444 445 /* the descriptor is ready */ 446 async_tx_ack(&mxs_chan->desc); 447 448 return 0; 449 450 err_clk: 451 free_irq(mxs_chan->chan_irq, mxs_dma); 452 err_irq: 453 dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE, 454 mxs_chan->ccw, mxs_chan->ccw_phys); 455 err_alloc: 456 return ret; 457 } 458 459 static void mxs_dma_free_chan_resources(struct dma_chan *chan) 460 { 461 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 462 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 463 464 mxs_dma_disable_chan(chan); 465 466 free_irq(mxs_chan->chan_irq, mxs_dma); 467 468 dma_free_coherent(mxs_dma->dma_device.dev, CCW_BLOCK_SIZE, 469 mxs_chan->ccw, mxs_chan->ccw_phys); 470 471 clk_disable_unprepare(mxs_dma->clk); 472 } 473 474 /* 475 * How to use the flags for ->device_prep_slave_sg() : 476 * [1] If there is only one DMA command in the DMA chain, the code should be: 477 * ...... 478 * ->device_prep_slave_sg(DMA_CTRL_ACK); 479 * ...... 480 * [2] If there are two DMA commands in the DMA chain, the code should be 481 * ...... 482 * ->device_prep_slave_sg(0); 483 * ...... 484 * ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 485 * ...... 486 * [3] If there are more than two DMA commands in the DMA chain, the code 487 * should be: 488 * ...... 489 * ->device_prep_slave_sg(0); // First 490 * ...... 491 * ->device_prep_slave_sg(DMA_PREP_INTERRUPT [| DMA_CTRL_ACK]); 492 * ...... 493 * ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK); // Last 494 * ...... 495 */ 496 static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg( 497 struct dma_chan *chan, struct scatterlist *sgl, 498 unsigned int sg_len, enum dma_transfer_direction direction, 499 unsigned long flags, void *context) 500 { 501 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 502 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 503 struct mxs_dma_ccw *ccw; 504 struct scatterlist *sg; 505 u32 i, j; 506 u32 *pio; 507 bool append = flags & DMA_PREP_INTERRUPT; 508 int idx = append ? mxs_chan->desc_count : 0; 509 510 if (mxs_chan->status == DMA_IN_PROGRESS && !append) 511 return NULL; 512 513 if (sg_len + (append ? idx : 0) > NUM_CCW) { 514 dev_err(mxs_dma->dma_device.dev, 515 "maximum number of sg exceeded: %d > %d\n", 516 sg_len, NUM_CCW); 517 goto err_out; 518 } 519 520 mxs_chan->status = DMA_IN_PROGRESS; 521 mxs_chan->flags = 0; 522 523 /* 524 * If the sg is prepared with append flag set, the sg 525 * will be appended to the last prepared sg. 526 */ 527 if (append) { 528 BUG_ON(idx < 1); 529 ccw = &mxs_chan->ccw[idx - 1]; 530 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx; 531 ccw->bits |= CCW_CHAIN; 532 ccw->bits &= ~CCW_IRQ; 533 ccw->bits &= ~CCW_DEC_SEM; 534 } else { 535 idx = 0; 536 } 537 538 if (direction == DMA_TRANS_NONE) { 539 ccw = &mxs_chan->ccw[idx++]; 540 pio = (u32 *) sgl; 541 542 for (j = 0; j < sg_len;) 543 ccw->pio_words[j++] = *pio++; 544 545 ccw->bits = 0; 546 ccw->bits |= CCW_IRQ; 547 ccw->bits |= CCW_DEC_SEM; 548 if (flags & DMA_CTRL_ACK) 549 ccw->bits |= CCW_WAIT4END; 550 ccw->bits |= CCW_HALT_ON_TERM; 551 ccw->bits |= CCW_TERM_FLUSH; 552 ccw->bits |= BF_CCW(sg_len, PIO_NUM); 553 ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND); 554 } else { 555 for_each_sg(sgl, sg, sg_len, i) { 556 if (sg_dma_len(sg) > MAX_XFER_BYTES) { 557 dev_err(mxs_dma->dma_device.dev, "maximum bytes for sg entry exceeded: %d > %d\n", 558 sg_dma_len(sg), MAX_XFER_BYTES); 559 goto err_out; 560 } 561 562 ccw = &mxs_chan->ccw[idx++]; 563 564 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx; 565 ccw->bufaddr = sg->dma_address; 566 ccw->xfer_bytes = sg_dma_len(sg); 567 568 ccw->bits = 0; 569 ccw->bits |= CCW_CHAIN; 570 ccw->bits |= CCW_HALT_ON_TERM; 571 ccw->bits |= CCW_TERM_FLUSH; 572 ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ? 573 MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, 574 COMMAND); 575 576 if (i + 1 == sg_len) { 577 ccw->bits &= ~CCW_CHAIN; 578 ccw->bits |= CCW_IRQ; 579 ccw->bits |= CCW_DEC_SEM; 580 if (flags & DMA_CTRL_ACK) 581 ccw->bits |= CCW_WAIT4END; 582 } 583 } 584 } 585 mxs_chan->desc_count = idx; 586 587 return &mxs_chan->desc; 588 589 err_out: 590 mxs_chan->status = DMA_ERROR; 591 return NULL; 592 } 593 594 static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic( 595 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, 596 size_t period_len, enum dma_transfer_direction direction, 597 unsigned long flags) 598 { 599 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 600 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 601 u32 num_periods = buf_len / period_len; 602 u32 i = 0, buf = 0; 603 604 if (mxs_chan->status == DMA_IN_PROGRESS) 605 return NULL; 606 607 mxs_chan->status = DMA_IN_PROGRESS; 608 mxs_chan->flags |= MXS_DMA_SG_LOOP; 609 mxs_chan->flags |= MXS_DMA_USE_SEMAPHORE; 610 611 if (num_periods > NUM_CCW) { 612 dev_err(mxs_dma->dma_device.dev, 613 "maximum number of sg exceeded: %d > %d\n", 614 num_periods, NUM_CCW); 615 goto err_out; 616 } 617 618 if (period_len > MAX_XFER_BYTES) { 619 dev_err(mxs_dma->dma_device.dev, 620 "maximum period size exceeded: %zu > %d\n", 621 period_len, MAX_XFER_BYTES); 622 goto err_out; 623 } 624 625 while (buf < buf_len) { 626 struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i]; 627 628 if (i + 1 == num_periods) 629 ccw->next = mxs_chan->ccw_phys; 630 else 631 ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1); 632 633 ccw->bufaddr = dma_addr; 634 ccw->xfer_bytes = period_len; 635 636 ccw->bits = 0; 637 ccw->bits |= CCW_CHAIN; 638 ccw->bits |= CCW_IRQ; 639 ccw->bits |= CCW_HALT_ON_TERM; 640 ccw->bits |= CCW_TERM_FLUSH; 641 ccw->bits |= CCW_DEC_SEM; 642 ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ? 643 MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND); 644 645 dma_addr += period_len; 646 buf += period_len; 647 648 i++; 649 } 650 mxs_chan->desc_count = i; 651 652 return &mxs_chan->desc; 653 654 err_out: 655 mxs_chan->status = DMA_ERROR; 656 return NULL; 657 } 658 659 static int mxs_dma_terminate_all(struct dma_chan *chan) 660 { 661 mxs_dma_reset_chan(chan); 662 mxs_dma_disable_chan(chan); 663 664 return 0; 665 } 666 667 static enum dma_status mxs_dma_tx_status(struct dma_chan *chan, 668 dma_cookie_t cookie, struct dma_tx_state *txstate) 669 { 670 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 671 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 672 u32 residue = 0; 673 674 if (mxs_chan->status == DMA_IN_PROGRESS && 675 mxs_chan->flags & MXS_DMA_SG_LOOP) { 676 struct mxs_dma_ccw *last_ccw; 677 u32 bar; 678 679 last_ccw = &mxs_chan->ccw[mxs_chan->desc_count - 1]; 680 residue = last_ccw->xfer_bytes + last_ccw->bufaddr; 681 682 bar = readl(mxs_dma->base + 683 HW_APBHX_CHn_BAR(mxs_dma, chan->chan_id)); 684 residue -= bar; 685 } 686 687 dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, 688 residue); 689 690 return mxs_chan->status; 691 } 692 693 static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma) 694 { 695 int ret; 696 697 ret = clk_prepare_enable(mxs_dma->clk); 698 if (ret) 699 return ret; 700 701 ret = stmp_reset_block(mxs_dma->base); 702 if (ret) 703 goto err_out; 704 705 /* enable apbh burst */ 706 if (dma_is_apbh(mxs_dma)) { 707 writel(BM_APBH_CTRL0_APB_BURST_EN, 708 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 709 writel(BM_APBH_CTRL0_APB_BURST8_EN, 710 mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); 711 } 712 713 /* enable irq for all the channels */ 714 writel(MXS_DMA_CHANNELS_MASK << MXS_DMA_CHANNELS, 715 mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_SET); 716 717 err_out: 718 clk_disable_unprepare(mxs_dma->clk); 719 return ret; 720 } 721 722 struct mxs_dma_filter_param { 723 struct device_node *of_node; 724 unsigned int chan_id; 725 }; 726 727 static bool mxs_dma_filter_fn(struct dma_chan *chan, void *fn_param) 728 { 729 struct mxs_dma_filter_param *param = fn_param; 730 struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); 731 struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; 732 int chan_irq; 733 734 if (mxs_dma->dma_device.dev->of_node != param->of_node) 735 return false; 736 737 if (chan->chan_id != param->chan_id) 738 return false; 739 740 chan_irq = platform_get_irq(mxs_dma->pdev, param->chan_id); 741 if (chan_irq < 0) 742 return false; 743 744 mxs_chan->chan_irq = chan_irq; 745 746 return true; 747 } 748 749 static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, 750 struct of_dma *ofdma) 751 { 752 struct mxs_dma_engine *mxs_dma = ofdma->of_dma_data; 753 dma_cap_mask_t mask = mxs_dma->dma_device.cap_mask; 754 struct mxs_dma_filter_param param; 755 756 if (dma_spec->args_count != 1) 757 return NULL; 758 759 param.of_node = ofdma->of_node; 760 param.chan_id = dma_spec->args[0]; 761 762 if (param.chan_id >= mxs_dma->nr_channels) 763 return NULL; 764 765 return dma_request_channel(mask, mxs_dma_filter_fn, ¶m); 766 } 767 768 static int __init mxs_dma_probe(struct platform_device *pdev) 769 { 770 struct device_node *np = pdev->dev.of_node; 771 const struct platform_device_id *id_entry; 772 const struct of_device_id *of_id; 773 const struct mxs_dma_type *dma_type; 774 struct mxs_dma_engine *mxs_dma; 775 struct resource *iores; 776 int ret, i; 777 778 mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL); 779 if (!mxs_dma) 780 return -ENOMEM; 781 782 ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels); 783 if (ret) { 784 dev_err(&pdev->dev, "failed to read dma-channels\n"); 785 return ret; 786 } 787 788 of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev); 789 if (of_id) 790 id_entry = of_id->data; 791 else 792 id_entry = platform_get_device_id(pdev); 793 794 dma_type = (struct mxs_dma_type *)id_entry->driver_data; 795 mxs_dma->type = dma_type->type; 796 mxs_dma->dev_id = dma_type->id; 797 798 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); 799 mxs_dma->base = devm_ioremap_resource(&pdev->dev, iores); 800 if (IS_ERR(mxs_dma->base)) 801 return PTR_ERR(mxs_dma->base); 802 803 mxs_dma->clk = devm_clk_get(&pdev->dev, NULL); 804 if (IS_ERR(mxs_dma->clk)) 805 return PTR_ERR(mxs_dma->clk); 806 807 dma_cap_set(DMA_SLAVE, mxs_dma->dma_device.cap_mask); 808 dma_cap_set(DMA_CYCLIC, mxs_dma->dma_device.cap_mask); 809 810 INIT_LIST_HEAD(&mxs_dma->dma_device.channels); 811 812 /* Initialize channel parameters */ 813 for (i = 0; i < MXS_DMA_CHANNELS; i++) { 814 struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i]; 815 816 mxs_chan->mxs_dma = mxs_dma; 817 mxs_chan->chan.device = &mxs_dma->dma_device; 818 dma_cookie_init(&mxs_chan->chan); 819 820 tasklet_init(&mxs_chan->tasklet, mxs_dma_tasklet, 821 (unsigned long) mxs_chan); 822 823 824 /* Add the channel to mxs_chan list */ 825 list_add_tail(&mxs_chan->chan.device_node, 826 &mxs_dma->dma_device.channels); 827 } 828 829 ret = mxs_dma_init(mxs_dma); 830 if (ret) 831 return ret; 832 833 mxs_dma->pdev = pdev; 834 mxs_dma->dma_device.dev = &pdev->dev; 835 836 /* mxs_dma gets 65535 bytes maximum sg size */ 837 mxs_dma->dma_device.dev->dma_parms = &mxs_dma->dma_parms; 838 dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES); 839 840 mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources; 841 mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources; 842 mxs_dma->dma_device.device_tx_status = mxs_dma_tx_status; 843 mxs_dma->dma_device.device_prep_slave_sg = mxs_dma_prep_slave_sg; 844 mxs_dma->dma_device.device_prep_dma_cyclic = mxs_dma_prep_dma_cyclic; 845 mxs_dma->dma_device.device_pause = mxs_dma_pause_chan; 846 mxs_dma->dma_device.device_resume = mxs_dma_resume_chan; 847 mxs_dma->dma_device.device_terminate_all = mxs_dma_terminate_all; 848 mxs_dma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 849 mxs_dma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 850 mxs_dma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 851 mxs_dma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 852 mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan; 853 854 ret = dma_async_device_register(&mxs_dma->dma_device); 855 if (ret) { 856 dev_err(mxs_dma->dma_device.dev, "unable to register\n"); 857 return ret; 858 } 859 860 ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma); 861 if (ret) { 862 dev_err(mxs_dma->dma_device.dev, 863 "failed to register controller\n"); 864 dma_async_device_unregister(&mxs_dma->dma_device); 865 } 866 867 dev_info(mxs_dma->dma_device.dev, "initialized\n"); 868 869 return 0; 870 } 871 872 static struct platform_driver mxs_dma_driver = { 873 .driver = { 874 .name = "mxs-dma", 875 .of_match_table = mxs_dma_dt_ids, 876 }, 877 .id_table = mxs_dma_ids, 878 }; 879 880 static int __init mxs_dma_module_init(void) 881 { 882 return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe); 883 } 884 subsys_initcall(mxs_dma_module_init); 885