1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * OMAP DMAengine support 4 */ 5 #include <linux/cpu_pm.h> 6 #include <linux/delay.h> 7 #include <linux/dmaengine.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dmapool.h> 10 #include <linux/err.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/list.h> 14 #include <linux/module.h> 15 #include <linux/omap-dma.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/of_dma.h> 20 #include <linux/of_device.h> 21 22 #include "../virt-dma.h" 23 24 #define OMAP_SDMA_REQUESTS 127 25 #define OMAP_SDMA_CHANNELS 32 26 27 struct omap_dma_config { 28 int lch_end; 29 unsigned int rw_priority:1; 30 unsigned int needs_busy_check:1; 31 unsigned int may_lose_context:1; 32 unsigned int needs_lch_clear:1; 33 }; 34 35 struct omap_dma_context { 36 u32 irqenable_l0; 37 u32 irqenable_l1; 38 u32 ocp_sysconfig; 39 u32 gcr; 40 }; 41 42 struct omap_dmadev { 43 struct dma_device ddev; 44 spinlock_t lock; 45 void __iomem *base; 46 const struct omap_dma_reg *reg_map; 47 struct omap_system_dma_plat_info *plat; 48 const struct omap_dma_config *cfg; 49 struct notifier_block nb; 50 struct omap_dma_context context; 51 int lch_count; 52 DECLARE_BITMAP(lch_bitmap, OMAP_SDMA_CHANNELS); 53 struct mutex lch_lock; /* for assigning logical channels */ 54 bool legacy; 55 bool ll123_supported; 56 struct dma_pool *desc_pool; 57 unsigned dma_requests; 58 spinlock_t irq_lock; 59 uint32_t irq_enable_mask; 60 struct omap_chan **lch_map; 61 }; 62 63 struct omap_chan { 64 struct virt_dma_chan vc; 65 void __iomem *channel_base; 66 const struct omap_dma_reg *reg_map; 67 uint32_t ccr; 68 69 struct dma_slave_config cfg; 70 unsigned dma_sig; 71 bool cyclic; 72 bool paused; 73 bool running; 74 75 int dma_ch; 76 struct omap_desc *desc; 77 unsigned sgidx; 78 }; 79 80 #define DESC_NXT_SV_REFRESH (0x1 << 24) 81 #define DESC_NXT_SV_REUSE (0x2 << 24) 82 #define DESC_NXT_DV_REFRESH (0x1 << 26) 83 #define DESC_NXT_DV_REUSE (0x2 << 26) 84 #define DESC_NTYPE_TYPE2 (0x2 << 29) 85 86 /* Type 2 descriptor with Source or Destination address update */ 87 struct omap_type2_desc { 88 uint32_t next_desc; 89 uint32_t en; 90 uint32_t addr; /* src or dst */ 91 uint16_t fn; 92 uint16_t cicr; 93 int16_t cdei; 94 int16_t csei; 95 int32_t cdfi; 96 int32_t csfi; 97 } __packed; 98 99 struct omap_sg { 100 dma_addr_t addr; 101 uint32_t en; /* number of elements (24-bit) */ 102 uint32_t fn; /* number of frames (16-bit) */ 103 int32_t fi; /* for double indexing */ 104 int16_t ei; /* for double indexing */ 105 106 /* Linked list */ 107 struct omap_type2_desc *t2_desc; 108 dma_addr_t t2_desc_paddr; 109 }; 110 111 struct omap_desc { 112 struct virt_dma_desc vd; 113 bool using_ll; 114 enum dma_transfer_direction dir; 115 dma_addr_t dev_addr; 116 bool polled; 117 118 int32_t fi; /* for OMAP_DMA_SYNC_PACKET / double indexing */ 119 int16_t ei; /* for double indexing */ 120 uint8_t es; /* CSDP_DATA_TYPE_xxx */ 121 uint32_t ccr; /* CCR value */ 122 uint16_t clnk_ctrl; /* CLNK_CTRL value */ 123 uint16_t cicr; /* CICR value */ 124 uint32_t csdp; /* CSDP value */ 125 126 unsigned sglen; 127 struct omap_sg sg[]; 128 }; 129 130 enum { 131 CAPS_0_SUPPORT_LL123 = BIT(20), /* Linked List type1/2/3 */ 132 CAPS_0_SUPPORT_LL4 = BIT(21), /* Linked List type4 */ 133 134 CCR_FS = BIT(5), 135 CCR_READ_PRIORITY = BIT(6), 136 CCR_ENABLE = BIT(7), 137 CCR_AUTO_INIT = BIT(8), /* OMAP1 only */ 138 CCR_REPEAT = BIT(9), /* OMAP1 only */ 139 CCR_OMAP31_DISABLE = BIT(10), /* OMAP1 only */ 140 CCR_SUSPEND_SENSITIVE = BIT(8), /* OMAP2+ only */ 141 CCR_RD_ACTIVE = BIT(9), /* OMAP2+ only */ 142 CCR_WR_ACTIVE = BIT(10), /* OMAP2+ only */ 143 CCR_SRC_AMODE_CONSTANT = 0 << 12, 144 CCR_SRC_AMODE_POSTINC = 1 << 12, 145 CCR_SRC_AMODE_SGLIDX = 2 << 12, 146 CCR_SRC_AMODE_DBLIDX = 3 << 12, 147 CCR_DST_AMODE_CONSTANT = 0 << 14, 148 CCR_DST_AMODE_POSTINC = 1 << 14, 149 CCR_DST_AMODE_SGLIDX = 2 << 14, 150 CCR_DST_AMODE_DBLIDX = 3 << 14, 151 CCR_CONSTANT_FILL = BIT(16), 152 CCR_TRANSPARENT_COPY = BIT(17), 153 CCR_BS = BIT(18), 154 CCR_SUPERVISOR = BIT(22), 155 CCR_PREFETCH = BIT(23), 156 CCR_TRIGGER_SRC = BIT(24), 157 CCR_BUFFERING_DISABLE = BIT(25), 158 CCR_WRITE_PRIORITY = BIT(26), 159 CCR_SYNC_ELEMENT = 0, 160 CCR_SYNC_FRAME = CCR_FS, 161 CCR_SYNC_BLOCK = CCR_BS, 162 CCR_SYNC_PACKET = CCR_BS | CCR_FS, 163 164 CSDP_DATA_TYPE_8 = 0, 165 CSDP_DATA_TYPE_16 = 1, 166 CSDP_DATA_TYPE_32 = 2, 167 CSDP_SRC_PORT_EMIFF = 0 << 2, /* OMAP1 only */ 168 CSDP_SRC_PORT_EMIFS = 1 << 2, /* OMAP1 only */ 169 CSDP_SRC_PORT_OCP_T1 = 2 << 2, /* OMAP1 only */ 170 CSDP_SRC_PORT_TIPB = 3 << 2, /* OMAP1 only */ 171 CSDP_SRC_PORT_OCP_T2 = 4 << 2, /* OMAP1 only */ 172 CSDP_SRC_PORT_MPUI = 5 << 2, /* OMAP1 only */ 173 CSDP_SRC_PACKED = BIT(6), 174 CSDP_SRC_BURST_1 = 0 << 7, 175 CSDP_SRC_BURST_16 = 1 << 7, 176 CSDP_SRC_BURST_32 = 2 << 7, 177 CSDP_SRC_BURST_64 = 3 << 7, 178 CSDP_DST_PORT_EMIFF = 0 << 9, /* OMAP1 only */ 179 CSDP_DST_PORT_EMIFS = 1 << 9, /* OMAP1 only */ 180 CSDP_DST_PORT_OCP_T1 = 2 << 9, /* OMAP1 only */ 181 CSDP_DST_PORT_TIPB = 3 << 9, /* OMAP1 only */ 182 CSDP_DST_PORT_OCP_T2 = 4 << 9, /* OMAP1 only */ 183 CSDP_DST_PORT_MPUI = 5 << 9, /* OMAP1 only */ 184 CSDP_DST_PACKED = BIT(13), 185 CSDP_DST_BURST_1 = 0 << 14, 186 CSDP_DST_BURST_16 = 1 << 14, 187 CSDP_DST_BURST_32 = 2 << 14, 188 CSDP_DST_BURST_64 = 3 << 14, 189 CSDP_WRITE_NON_POSTED = 0 << 16, 190 CSDP_WRITE_POSTED = 1 << 16, 191 CSDP_WRITE_LAST_NON_POSTED = 2 << 16, 192 193 CICR_TOUT_IE = BIT(0), /* OMAP1 only */ 194 CICR_DROP_IE = BIT(1), 195 CICR_HALF_IE = BIT(2), 196 CICR_FRAME_IE = BIT(3), 197 CICR_LAST_IE = BIT(4), 198 CICR_BLOCK_IE = BIT(5), 199 CICR_PKT_IE = BIT(7), /* OMAP2+ only */ 200 CICR_TRANS_ERR_IE = BIT(8), /* OMAP2+ only */ 201 CICR_SUPERVISOR_ERR_IE = BIT(10), /* OMAP2+ only */ 202 CICR_MISALIGNED_ERR_IE = BIT(11), /* OMAP2+ only */ 203 CICR_DRAIN_IE = BIT(12), /* OMAP2+ only */ 204 CICR_SUPER_BLOCK_IE = BIT(14), /* OMAP2+ only */ 205 206 CLNK_CTRL_ENABLE_LNK = BIT(15), 207 208 CDP_DST_VALID_INC = 0 << 0, 209 CDP_DST_VALID_RELOAD = 1 << 0, 210 CDP_DST_VALID_REUSE = 2 << 0, 211 CDP_SRC_VALID_INC = 0 << 2, 212 CDP_SRC_VALID_RELOAD = 1 << 2, 213 CDP_SRC_VALID_REUSE = 2 << 2, 214 CDP_NTYPE_TYPE1 = 1 << 4, 215 CDP_NTYPE_TYPE2 = 2 << 4, 216 CDP_NTYPE_TYPE3 = 3 << 4, 217 CDP_TMODE_NORMAL = 0 << 8, 218 CDP_TMODE_LLIST = 1 << 8, 219 CDP_FAST = BIT(10), 220 }; 221 222 static const unsigned es_bytes[] = { 223 [CSDP_DATA_TYPE_8] = 1, 224 [CSDP_DATA_TYPE_16] = 2, 225 [CSDP_DATA_TYPE_32] = 4, 226 }; 227 228 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param); 229 static struct of_dma_filter_info omap_dma_info = { 230 .filter_fn = omap_dma_filter_fn, 231 }; 232 233 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d) 234 { 235 return container_of(d, struct omap_dmadev, ddev); 236 } 237 238 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c) 239 { 240 return container_of(c, struct omap_chan, vc.chan); 241 } 242 243 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t) 244 { 245 return container_of(t, struct omap_desc, vd.tx); 246 } 247 248 static void omap_dma_desc_free(struct virt_dma_desc *vd) 249 { 250 struct omap_desc *d = to_omap_dma_desc(&vd->tx); 251 252 if (d->using_ll) { 253 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device); 254 int i; 255 256 for (i = 0; i < d->sglen; i++) { 257 if (d->sg[i].t2_desc) 258 dma_pool_free(od->desc_pool, d->sg[i].t2_desc, 259 d->sg[i].t2_desc_paddr); 260 } 261 } 262 263 kfree(d); 264 } 265 266 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx, 267 enum dma_transfer_direction dir, bool last) 268 { 269 struct omap_sg *sg = &d->sg[idx]; 270 struct omap_type2_desc *t2_desc = sg->t2_desc; 271 272 if (idx) 273 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr; 274 if (last) 275 t2_desc->next_desc = 0xfffffffc; 276 277 t2_desc->en = sg->en; 278 t2_desc->addr = sg->addr; 279 t2_desc->fn = sg->fn & 0xffff; 280 t2_desc->cicr = d->cicr; 281 if (!last) 282 t2_desc->cicr &= ~CICR_BLOCK_IE; 283 284 switch (dir) { 285 case DMA_DEV_TO_MEM: 286 t2_desc->cdei = sg->ei; 287 t2_desc->csei = d->ei; 288 t2_desc->cdfi = sg->fi; 289 t2_desc->csfi = d->fi; 290 291 t2_desc->en |= DESC_NXT_DV_REFRESH; 292 t2_desc->en |= DESC_NXT_SV_REUSE; 293 break; 294 case DMA_MEM_TO_DEV: 295 t2_desc->cdei = d->ei; 296 t2_desc->csei = sg->ei; 297 t2_desc->cdfi = d->fi; 298 t2_desc->csfi = sg->fi; 299 300 t2_desc->en |= DESC_NXT_SV_REFRESH; 301 t2_desc->en |= DESC_NXT_DV_REUSE; 302 break; 303 default: 304 return; 305 } 306 307 t2_desc->en |= DESC_NTYPE_TYPE2; 308 } 309 310 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr) 311 { 312 switch (type) { 313 case OMAP_DMA_REG_16BIT: 314 writew_relaxed(val, addr); 315 break; 316 case OMAP_DMA_REG_2X16BIT: 317 writew_relaxed(val, addr); 318 writew_relaxed(val >> 16, addr + 2); 319 break; 320 case OMAP_DMA_REG_32BIT: 321 writel_relaxed(val, addr); 322 break; 323 default: 324 WARN_ON(1); 325 } 326 } 327 328 static unsigned omap_dma_read(unsigned type, void __iomem *addr) 329 { 330 unsigned val; 331 332 switch (type) { 333 case OMAP_DMA_REG_16BIT: 334 val = readw_relaxed(addr); 335 break; 336 case OMAP_DMA_REG_2X16BIT: 337 val = readw_relaxed(addr); 338 val |= readw_relaxed(addr + 2) << 16; 339 break; 340 case OMAP_DMA_REG_32BIT: 341 val = readl_relaxed(addr); 342 break; 343 default: 344 WARN_ON(1); 345 val = 0; 346 } 347 348 return val; 349 } 350 351 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) 352 { 353 const struct omap_dma_reg *r = od->reg_map + reg; 354 355 WARN_ON(r->stride); 356 357 omap_dma_write(val, r->type, od->base + r->offset); 358 } 359 360 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) 361 { 362 const struct omap_dma_reg *r = od->reg_map + reg; 363 364 WARN_ON(r->stride); 365 366 return omap_dma_read(r->type, od->base + r->offset); 367 } 368 369 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) 370 { 371 const struct omap_dma_reg *r = c->reg_map + reg; 372 373 omap_dma_write(val, r->type, c->channel_base + r->offset); 374 } 375 376 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) 377 { 378 const struct omap_dma_reg *r = c->reg_map + reg; 379 380 return omap_dma_read(r->type, c->channel_base + r->offset); 381 } 382 383 static void omap_dma_clear_csr(struct omap_chan *c) 384 { 385 if (dma_omap1()) 386 omap_dma_chan_read(c, CSR); 387 else 388 omap_dma_chan_write(c, CSR, ~0); 389 } 390 391 static unsigned omap_dma_get_csr(struct omap_chan *c) 392 { 393 unsigned val = omap_dma_chan_read(c, CSR); 394 395 if (!dma_omap1()) 396 omap_dma_chan_write(c, CSR, val); 397 398 return val; 399 } 400 401 static void omap_dma_clear_lch(struct omap_dmadev *od, int lch) 402 { 403 struct omap_chan *c; 404 int i; 405 406 c = od->lch_map[lch]; 407 if (!c) 408 return; 409 410 for (i = CSDP; i <= od->cfg->lch_end; i++) 411 omap_dma_chan_write(c, i, 0); 412 } 413 414 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, 415 unsigned lch) 416 { 417 c->channel_base = od->base + od->plat->channel_stride * lch; 418 419 od->lch_map[lch] = c; 420 } 421 422 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) 423 { 424 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 425 uint16_t cicr = d->cicr; 426 427 if (__dma_omap15xx(od->plat->dma_attr)) 428 omap_dma_chan_write(c, CPC, 0); 429 else 430 omap_dma_chan_write(c, CDAC, 0); 431 432 omap_dma_clear_csr(c); 433 434 if (d->using_ll) { 435 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST; 436 437 if (d->dir == DMA_DEV_TO_MEM) 438 cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE); 439 else 440 cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD); 441 omap_dma_chan_write(c, CDP, cdp); 442 443 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr); 444 omap_dma_chan_write(c, CCDN, 0); 445 omap_dma_chan_write(c, CCFN, 0xffff); 446 omap_dma_chan_write(c, CCEN, 0xffffff); 447 448 cicr &= ~CICR_BLOCK_IE; 449 } else if (od->ll123_supported) { 450 omap_dma_chan_write(c, CDP, 0); 451 } 452 453 /* Enable interrupts */ 454 omap_dma_chan_write(c, CICR, cicr); 455 456 /* Enable channel */ 457 omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); 458 459 c->running = true; 460 } 461 462 static void omap_dma_drain_chan(struct omap_chan *c) 463 { 464 int i; 465 u32 val; 466 467 /* Wait for sDMA FIFO to drain */ 468 for (i = 0; ; i++) { 469 val = omap_dma_chan_read(c, CCR); 470 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) 471 break; 472 473 if (i > 100) 474 break; 475 476 udelay(5); 477 } 478 479 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) 480 dev_err(c->vc.chan.device->dev, 481 "DMA drain did not complete on lch %d\n", 482 c->dma_ch); 483 } 484 485 static int omap_dma_stop(struct omap_chan *c) 486 { 487 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 488 uint32_t val; 489 490 /* disable irq */ 491 omap_dma_chan_write(c, CICR, 0); 492 493 omap_dma_clear_csr(c); 494 495 val = omap_dma_chan_read(c, CCR); 496 if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { 497 uint32_t sysconfig; 498 499 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); 500 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; 501 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); 502 omap_dma_glbl_write(od, OCP_SYSCONFIG, val); 503 504 val = omap_dma_chan_read(c, CCR); 505 val &= ~CCR_ENABLE; 506 omap_dma_chan_write(c, CCR, val); 507 508 if (!(c->ccr & CCR_BUFFERING_DISABLE)) 509 omap_dma_drain_chan(c); 510 511 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); 512 } else { 513 if (!(val & CCR_ENABLE)) 514 return -EINVAL; 515 516 val &= ~CCR_ENABLE; 517 omap_dma_chan_write(c, CCR, val); 518 519 if (!(c->ccr & CCR_BUFFERING_DISABLE)) 520 omap_dma_drain_chan(c); 521 } 522 523 mb(); 524 525 if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { 526 val = omap_dma_chan_read(c, CLNK_CTRL); 527 528 if (dma_omap1()) 529 val |= 1 << 14; /* set the STOP_LNK bit */ 530 else 531 val &= ~CLNK_CTRL_ENABLE_LNK; 532 533 omap_dma_chan_write(c, CLNK_CTRL, val); 534 } 535 c->running = false; 536 return 0; 537 } 538 539 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d) 540 { 541 struct omap_sg *sg = d->sg + c->sgidx; 542 unsigned cxsa, cxei, cxfi; 543 544 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { 545 cxsa = CDSA; 546 cxei = CDEI; 547 cxfi = CDFI; 548 } else { 549 cxsa = CSSA; 550 cxei = CSEI; 551 cxfi = CSFI; 552 } 553 554 omap_dma_chan_write(c, cxsa, sg->addr); 555 omap_dma_chan_write(c, cxei, sg->ei); 556 omap_dma_chan_write(c, cxfi, sg->fi); 557 omap_dma_chan_write(c, CEN, sg->en); 558 omap_dma_chan_write(c, CFN, sg->fn); 559 560 omap_dma_start(c, d); 561 c->sgidx++; 562 } 563 564 static void omap_dma_start_desc(struct omap_chan *c) 565 { 566 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 567 struct omap_desc *d; 568 unsigned cxsa, cxei, cxfi; 569 570 if (!vd) { 571 c->desc = NULL; 572 return; 573 } 574 575 list_del(&vd->node); 576 577 c->desc = d = to_omap_dma_desc(&vd->tx); 578 c->sgidx = 0; 579 580 /* 581 * This provides the necessary barrier to ensure data held in 582 * DMA coherent memory is visible to the DMA engine prior to 583 * the transfer starting. 584 */ 585 mb(); 586 587 omap_dma_chan_write(c, CCR, d->ccr); 588 if (dma_omap1()) 589 omap_dma_chan_write(c, CCR2, d->ccr >> 16); 590 591 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { 592 cxsa = CSSA; 593 cxei = CSEI; 594 cxfi = CSFI; 595 } else { 596 cxsa = CDSA; 597 cxei = CDEI; 598 cxfi = CDFI; 599 } 600 601 omap_dma_chan_write(c, cxsa, d->dev_addr); 602 omap_dma_chan_write(c, cxei, d->ei); 603 omap_dma_chan_write(c, cxfi, d->fi); 604 omap_dma_chan_write(c, CSDP, d->csdp); 605 omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); 606 607 omap_dma_start_sg(c, d); 608 } 609 610 static void omap_dma_callback(int ch, u16 status, void *data) 611 { 612 struct omap_chan *c = data; 613 struct omap_desc *d; 614 unsigned long flags; 615 616 spin_lock_irqsave(&c->vc.lock, flags); 617 d = c->desc; 618 if (d) { 619 if (c->cyclic) { 620 vchan_cyclic_callback(&d->vd); 621 } else if (d->using_ll || c->sgidx == d->sglen) { 622 omap_dma_start_desc(c); 623 vchan_cookie_complete(&d->vd); 624 } else { 625 omap_dma_start_sg(c, d); 626 } 627 } 628 spin_unlock_irqrestore(&c->vc.lock, flags); 629 } 630 631 static irqreturn_t omap_dma_irq(int irq, void *devid) 632 { 633 struct omap_dmadev *od = devid; 634 unsigned status, channel; 635 636 spin_lock(&od->irq_lock); 637 638 status = omap_dma_glbl_read(od, IRQSTATUS_L1); 639 status &= od->irq_enable_mask; 640 if (status == 0) { 641 spin_unlock(&od->irq_lock); 642 return IRQ_NONE; 643 } 644 645 while ((channel = ffs(status)) != 0) { 646 unsigned mask, csr; 647 struct omap_chan *c; 648 649 channel -= 1; 650 mask = BIT(channel); 651 status &= ~mask; 652 653 c = od->lch_map[channel]; 654 if (c == NULL) { 655 /* This should never happen */ 656 dev_err(od->ddev.dev, "invalid channel %u\n", channel); 657 continue; 658 } 659 660 csr = omap_dma_get_csr(c); 661 omap_dma_glbl_write(od, IRQSTATUS_L1, mask); 662 663 omap_dma_callback(channel, csr, c); 664 } 665 666 spin_unlock(&od->irq_lock); 667 668 return IRQ_HANDLED; 669 } 670 671 static int omap_dma_get_lch(struct omap_dmadev *od, int *lch) 672 { 673 int channel; 674 675 mutex_lock(&od->lch_lock); 676 channel = find_first_zero_bit(od->lch_bitmap, od->lch_count); 677 if (channel >= od->lch_count) 678 goto out_busy; 679 set_bit(channel, od->lch_bitmap); 680 mutex_unlock(&od->lch_lock); 681 682 omap_dma_clear_lch(od, channel); 683 *lch = channel; 684 685 return 0; 686 687 out_busy: 688 mutex_unlock(&od->lch_lock); 689 *lch = -EINVAL; 690 691 return -EBUSY; 692 } 693 694 static void omap_dma_put_lch(struct omap_dmadev *od, int lch) 695 { 696 omap_dma_clear_lch(od, lch); 697 mutex_lock(&od->lch_lock); 698 clear_bit(lch, od->lch_bitmap); 699 mutex_unlock(&od->lch_lock); 700 } 701 702 static int omap_dma_alloc_chan_resources(struct dma_chan *chan) 703 { 704 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 705 struct omap_chan *c = to_omap_dma_chan(chan); 706 struct device *dev = od->ddev.dev; 707 int ret; 708 709 if (od->legacy) { 710 ret = omap_request_dma(c->dma_sig, "DMA engine", 711 omap_dma_callback, c, &c->dma_ch); 712 } else { 713 ret = omap_dma_get_lch(od, &c->dma_ch); 714 } 715 716 dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig); 717 718 if (ret >= 0) { 719 omap_dma_assign(od, c, c->dma_ch); 720 721 if (!od->legacy) { 722 unsigned val; 723 724 spin_lock_irq(&od->irq_lock); 725 val = BIT(c->dma_ch); 726 omap_dma_glbl_write(od, IRQSTATUS_L1, val); 727 od->irq_enable_mask |= val; 728 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); 729 730 val = omap_dma_glbl_read(od, IRQENABLE_L0); 731 val &= ~BIT(c->dma_ch); 732 omap_dma_glbl_write(od, IRQENABLE_L0, val); 733 spin_unlock_irq(&od->irq_lock); 734 } 735 } 736 737 if (dma_omap1()) { 738 if (__dma_omap16xx(od->plat->dma_attr)) { 739 c->ccr = CCR_OMAP31_DISABLE; 740 /* Duplicate what plat-omap/dma.c does */ 741 c->ccr |= c->dma_ch + 1; 742 } else { 743 c->ccr = c->dma_sig & 0x1f; 744 } 745 } else { 746 c->ccr = c->dma_sig & 0x1f; 747 c->ccr |= (c->dma_sig & ~0x1f) << 14; 748 } 749 if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) 750 c->ccr |= CCR_BUFFERING_DISABLE; 751 752 return ret; 753 } 754 755 static void omap_dma_free_chan_resources(struct dma_chan *chan) 756 { 757 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 758 struct omap_chan *c = to_omap_dma_chan(chan); 759 760 if (!od->legacy) { 761 spin_lock_irq(&od->irq_lock); 762 od->irq_enable_mask &= ~BIT(c->dma_ch); 763 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); 764 spin_unlock_irq(&od->irq_lock); 765 } 766 767 c->channel_base = NULL; 768 od->lch_map[c->dma_ch] = NULL; 769 vchan_free_chan_resources(&c->vc); 770 771 if (od->legacy) 772 omap_free_dma(c->dma_ch); 773 else 774 omap_dma_put_lch(od, c->dma_ch); 775 776 dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch, 777 c->dma_sig); 778 c->dma_sig = 0; 779 } 780 781 static size_t omap_dma_sg_size(struct omap_sg *sg) 782 { 783 return sg->en * sg->fn; 784 } 785 786 static size_t omap_dma_desc_size(struct omap_desc *d) 787 { 788 unsigned i; 789 size_t size; 790 791 for (size = i = 0; i < d->sglen; i++) 792 size += omap_dma_sg_size(&d->sg[i]); 793 794 return size * es_bytes[d->es]; 795 } 796 797 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) 798 { 799 unsigned i; 800 size_t size, es_size = es_bytes[d->es]; 801 802 for (size = i = 0; i < d->sglen; i++) { 803 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size; 804 805 if (size) 806 size += this_size; 807 else if (addr >= d->sg[i].addr && 808 addr < d->sg[i].addr + this_size) 809 size += d->sg[i].addr + this_size - addr; 810 } 811 return size; 812 } 813 814 /* 815 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is 816 * read before the DMA controller finished disabling the channel. 817 */ 818 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg) 819 { 820 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 821 uint32_t val; 822 823 val = omap_dma_chan_read(c, reg); 824 if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) 825 val = omap_dma_chan_read(c, reg); 826 827 return val; 828 } 829 830 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) 831 { 832 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 833 dma_addr_t addr, cdac; 834 835 if (__dma_omap15xx(od->plat->dma_attr)) { 836 addr = omap_dma_chan_read(c, CPC); 837 } else { 838 addr = omap_dma_chan_read_3_3(c, CSAC); 839 cdac = omap_dma_chan_read_3_3(c, CDAC); 840 841 /* 842 * CDAC == 0 indicates that the DMA transfer on the channel has 843 * not been started (no data has been transferred so far). 844 * Return the programmed source start address in this case. 845 */ 846 if (cdac == 0) 847 addr = omap_dma_chan_read(c, CSSA); 848 } 849 850 if (dma_omap1()) 851 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000; 852 853 return addr; 854 } 855 856 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) 857 { 858 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 859 dma_addr_t addr; 860 861 if (__dma_omap15xx(od->plat->dma_attr)) { 862 addr = omap_dma_chan_read(c, CPC); 863 } else { 864 addr = omap_dma_chan_read_3_3(c, CDAC); 865 866 /* 867 * CDAC == 0 indicates that the DMA transfer on the channel 868 * has not been started (no data has been transferred so 869 * far). Return the programmed destination start address in 870 * this case. 871 */ 872 if (addr == 0) 873 addr = omap_dma_chan_read(c, CDSA); 874 } 875 876 if (dma_omap1()) 877 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000; 878 879 return addr; 880 } 881 882 static enum dma_status omap_dma_tx_status(struct dma_chan *chan, 883 dma_cookie_t cookie, struct dma_tx_state *txstate) 884 { 885 struct omap_chan *c = to_omap_dma_chan(chan); 886 enum dma_status ret; 887 unsigned long flags; 888 struct omap_desc *d = NULL; 889 890 ret = dma_cookie_status(chan, cookie, txstate); 891 if (ret == DMA_COMPLETE) 892 return ret; 893 894 spin_lock_irqsave(&c->vc.lock, flags); 895 if (c->desc && c->desc->vd.tx.cookie == cookie) 896 d = c->desc; 897 898 if (!txstate) 899 goto out; 900 901 if (d) { 902 dma_addr_t pos; 903 904 if (d->dir == DMA_MEM_TO_DEV) 905 pos = omap_dma_get_src_pos(c); 906 else if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) 907 pos = omap_dma_get_dst_pos(c); 908 else 909 pos = 0; 910 911 txstate->residue = omap_dma_desc_size_pos(d, pos); 912 } else { 913 struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie); 914 915 if (vd) 916 txstate->residue = omap_dma_desc_size( 917 to_omap_dma_desc(&vd->tx)); 918 else 919 txstate->residue = 0; 920 } 921 922 out: 923 if (ret == DMA_IN_PROGRESS && c->paused) { 924 ret = DMA_PAUSED; 925 } else if (d && d->polled && c->running) { 926 uint32_t ccr = omap_dma_chan_read(c, CCR); 927 /* 928 * The channel is no longer active, set the return value 929 * accordingly and mark it as completed 930 */ 931 if (!(ccr & CCR_ENABLE)) { 932 ret = DMA_COMPLETE; 933 omap_dma_start_desc(c); 934 vchan_cookie_complete(&d->vd); 935 } 936 } 937 938 spin_unlock_irqrestore(&c->vc.lock, flags); 939 940 return ret; 941 } 942 943 static void omap_dma_issue_pending(struct dma_chan *chan) 944 { 945 struct omap_chan *c = to_omap_dma_chan(chan); 946 unsigned long flags; 947 948 spin_lock_irqsave(&c->vc.lock, flags); 949 if (vchan_issue_pending(&c->vc) && !c->desc) 950 omap_dma_start_desc(c); 951 spin_unlock_irqrestore(&c->vc.lock, flags); 952 } 953 954 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( 955 struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen, 956 enum dma_transfer_direction dir, unsigned long tx_flags, void *context) 957 { 958 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 959 struct omap_chan *c = to_omap_dma_chan(chan); 960 enum dma_slave_buswidth dev_width; 961 struct scatterlist *sgent; 962 struct omap_desc *d; 963 dma_addr_t dev_addr; 964 unsigned i, es, en, frame_bytes; 965 bool ll_failed = false; 966 u32 burst; 967 u32 port_window, port_window_bytes; 968 969 if (dir == DMA_DEV_TO_MEM) { 970 dev_addr = c->cfg.src_addr; 971 dev_width = c->cfg.src_addr_width; 972 burst = c->cfg.src_maxburst; 973 port_window = c->cfg.src_port_window_size; 974 } else if (dir == DMA_MEM_TO_DEV) { 975 dev_addr = c->cfg.dst_addr; 976 dev_width = c->cfg.dst_addr_width; 977 burst = c->cfg.dst_maxburst; 978 port_window = c->cfg.dst_port_window_size; 979 } else { 980 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 981 return NULL; 982 } 983 984 /* Bus width translates to the element size (ES) */ 985 switch (dev_width) { 986 case DMA_SLAVE_BUSWIDTH_1_BYTE: 987 es = CSDP_DATA_TYPE_8; 988 break; 989 case DMA_SLAVE_BUSWIDTH_2_BYTES: 990 es = CSDP_DATA_TYPE_16; 991 break; 992 case DMA_SLAVE_BUSWIDTH_4_BYTES: 993 es = CSDP_DATA_TYPE_32; 994 break; 995 default: /* not reached */ 996 return NULL; 997 } 998 999 /* Now allocate and setup the descriptor. */ 1000 d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC); 1001 if (!d) 1002 return NULL; 1003 1004 d->dir = dir; 1005 d->dev_addr = dev_addr; 1006 d->es = es; 1007 1008 /* When the port_window is used, one frame must cover the window */ 1009 if (port_window) { 1010 burst = port_window; 1011 port_window_bytes = port_window * es_bytes[es]; 1012 1013 d->ei = 1; 1014 /* 1015 * One frame covers the port_window and by configure 1016 * the source frame index to be -1 * (port_window - 1) 1017 * we instruct the sDMA that after a frame is processed 1018 * it should move back to the start of the window. 1019 */ 1020 d->fi = -(port_window_bytes - 1); 1021 } 1022 1023 d->ccr = c->ccr | CCR_SYNC_FRAME; 1024 if (dir == DMA_DEV_TO_MEM) { 1025 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED; 1026 1027 d->ccr |= CCR_DST_AMODE_POSTINC; 1028 if (port_window) { 1029 d->ccr |= CCR_SRC_AMODE_DBLIDX; 1030 1031 if (port_window_bytes >= 64) 1032 d->csdp |= CSDP_SRC_BURST_64; 1033 else if (port_window_bytes >= 32) 1034 d->csdp |= CSDP_SRC_BURST_32; 1035 else if (port_window_bytes >= 16) 1036 d->csdp |= CSDP_SRC_BURST_16; 1037 1038 } else { 1039 d->ccr |= CCR_SRC_AMODE_CONSTANT; 1040 } 1041 } else { 1042 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED; 1043 1044 d->ccr |= CCR_SRC_AMODE_POSTINC; 1045 if (port_window) { 1046 d->ccr |= CCR_DST_AMODE_DBLIDX; 1047 1048 if (port_window_bytes >= 64) 1049 d->csdp |= CSDP_DST_BURST_64; 1050 else if (port_window_bytes >= 32) 1051 d->csdp |= CSDP_DST_BURST_32; 1052 else if (port_window_bytes >= 16) 1053 d->csdp |= CSDP_DST_BURST_16; 1054 } else { 1055 d->ccr |= CCR_DST_AMODE_CONSTANT; 1056 } 1057 } 1058 1059 d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; 1060 d->csdp |= es; 1061 1062 if (dma_omap1()) { 1063 d->cicr |= CICR_TOUT_IE; 1064 1065 if (dir == DMA_DEV_TO_MEM) 1066 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; 1067 else 1068 d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; 1069 } else { 1070 if (dir == DMA_DEV_TO_MEM) 1071 d->ccr |= CCR_TRIGGER_SRC; 1072 1073 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1074 1075 if (port_window) 1076 d->csdp |= CSDP_WRITE_LAST_NON_POSTED; 1077 } 1078 if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) 1079 d->clnk_ctrl = c->dma_ch; 1080 1081 /* 1082 * Build our scatterlist entries: each contains the address, 1083 * the number of elements (EN) in each frame, and the number of 1084 * frames (FN). Number of bytes for this entry = ES * EN * FN. 1085 * 1086 * Burst size translates to number of elements with frame sync. 1087 * Note: DMA engine defines burst to be the number of dev-width 1088 * transfers. 1089 */ 1090 en = burst; 1091 frame_bytes = es_bytes[es] * en; 1092 1093 if (sglen >= 2) 1094 d->using_ll = od->ll123_supported; 1095 1096 for_each_sg(sgl, sgent, sglen, i) { 1097 struct omap_sg *osg = &d->sg[i]; 1098 1099 osg->addr = sg_dma_address(sgent); 1100 osg->en = en; 1101 osg->fn = sg_dma_len(sgent) / frame_bytes; 1102 1103 if (d->using_ll) { 1104 osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC, 1105 &osg->t2_desc_paddr); 1106 if (!osg->t2_desc) { 1107 dev_err(chan->device->dev, 1108 "t2_desc[%d] allocation failed\n", i); 1109 ll_failed = true; 1110 d->using_ll = false; 1111 continue; 1112 } 1113 1114 omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1)); 1115 } 1116 } 1117 1118 d->sglen = sglen; 1119 1120 /* Release the dma_pool entries if one allocation failed */ 1121 if (ll_failed) { 1122 for (i = 0; i < d->sglen; i++) { 1123 struct omap_sg *osg = &d->sg[i]; 1124 1125 if (osg->t2_desc) { 1126 dma_pool_free(od->desc_pool, osg->t2_desc, 1127 osg->t2_desc_paddr); 1128 osg->t2_desc = NULL; 1129 } 1130 } 1131 } 1132 1133 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); 1134 } 1135 1136 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( 1137 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 1138 size_t period_len, enum dma_transfer_direction dir, unsigned long flags) 1139 { 1140 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1141 struct omap_chan *c = to_omap_dma_chan(chan); 1142 enum dma_slave_buswidth dev_width; 1143 struct omap_desc *d; 1144 dma_addr_t dev_addr; 1145 unsigned es; 1146 u32 burst; 1147 1148 if (dir == DMA_DEV_TO_MEM) { 1149 dev_addr = c->cfg.src_addr; 1150 dev_width = c->cfg.src_addr_width; 1151 burst = c->cfg.src_maxburst; 1152 } else if (dir == DMA_MEM_TO_DEV) { 1153 dev_addr = c->cfg.dst_addr; 1154 dev_width = c->cfg.dst_addr_width; 1155 burst = c->cfg.dst_maxburst; 1156 } else { 1157 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 1158 return NULL; 1159 } 1160 1161 /* Bus width translates to the element size (ES) */ 1162 switch (dev_width) { 1163 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1164 es = CSDP_DATA_TYPE_8; 1165 break; 1166 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1167 es = CSDP_DATA_TYPE_16; 1168 break; 1169 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1170 es = CSDP_DATA_TYPE_32; 1171 break; 1172 default: /* not reached */ 1173 return NULL; 1174 } 1175 1176 /* Now allocate and setup the descriptor. */ 1177 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1178 if (!d) 1179 return NULL; 1180 1181 d->dir = dir; 1182 d->dev_addr = dev_addr; 1183 d->fi = burst; 1184 d->es = es; 1185 d->sg[0].addr = buf_addr; 1186 d->sg[0].en = period_len / es_bytes[es]; 1187 d->sg[0].fn = buf_len / period_len; 1188 d->sglen = 1; 1189 1190 d->ccr = c->ccr; 1191 if (dir == DMA_DEV_TO_MEM) 1192 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; 1193 else 1194 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; 1195 1196 d->cicr = CICR_DROP_IE; 1197 if (flags & DMA_PREP_INTERRUPT) 1198 d->cicr |= CICR_FRAME_IE; 1199 1200 d->csdp = es; 1201 1202 if (dma_omap1()) { 1203 d->cicr |= CICR_TOUT_IE; 1204 1205 if (dir == DMA_DEV_TO_MEM) 1206 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; 1207 else 1208 d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; 1209 } else { 1210 if (burst) 1211 d->ccr |= CCR_SYNC_PACKET; 1212 else 1213 d->ccr |= CCR_SYNC_ELEMENT; 1214 1215 if (dir == DMA_DEV_TO_MEM) { 1216 d->ccr |= CCR_TRIGGER_SRC; 1217 d->csdp |= CSDP_DST_PACKED; 1218 } else { 1219 d->csdp |= CSDP_SRC_PACKED; 1220 } 1221 1222 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1223 1224 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1225 } 1226 1227 if (__dma_omap15xx(od->plat->dma_attr)) 1228 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; 1229 else 1230 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; 1231 1232 c->cyclic = true; 1233 1234 return vchan_tx_prep(&c->vc, &d->vd, flags); 1235 } 1236 1237 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy( 1238 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1239 size_t len, unsigned long tx_flags) 1240 { 1241 struct omap_chan *c = to_omap_dma_chan(chan); 1242 struct omap_desc *d; 1243 uint8_t data_type; 1244 1245 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1246 if (!d) 1247 return NULL; 1248 1249 data_type = __ffs((src | dest | len)); 1250 if (data_type > CSDP_DATA_TYPE_32) 1251 data_type = CSDP_DATA_TYPE_32; 1252 1253 d->dir = DMA_MEM_TO_MEM; 1254 d->dev_addr = src; 1255 d->fi = 0; 1256 d->es = data_type; 1257 d->sg[0].en = len / BIT(data_type); 1258 d->sg[0].fn = 1; 1259 d->sg[0].addr = dest; 1260 d->sglen = 1; 1261 d->ccr = c->ccr; 1262 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; 1263 1264 if (tx_flags & DMA_PREP_INTERRUPT) 1265 d->cicr |= CICR_FRAME_IE; 1266 else 1267 d->polled = true; 1268 1269 d->csdp = data_type; 1270 1271 if (dma_omap1()) { 1272 d->cicr |= CICR_TOUT_IE; 1273 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; 1274 } else { 1275 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; 1276 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1277 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1278 } 1279 1280 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); 1281 } 1282 1283 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( 1284 struct dma_chan *chan, struct dma_interleaved_template *xt, 1285 unsigned long flags) 1286 { 1287 struct omap_chan *c = to_omap_dma_chan(chan); 1288 struct omap_desc *d; 1289 struct omap_sg *sg; 1290 uint8_t data_type; 1291 size_t src_icg, dst_icg; 1292 1293 /* Slave mode is not supported */ 1294 if (is_slave_direction(xt->dir)) 1295 return NULL; 1296 1297 if (xt->frame_size != 1 || xt->numf == 0) 1298 return NULL; 1299 1300 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1301 if (!d) 1302 return NULL; 1303 1304 data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size)); 1305 if (data_type > CSDP_DATA_TYPE_32) 1306 data_type = CSDP_DATA_TYPE_32; 1307 1308 sg = &d->sg[0]; 1309 d->dir = DMA_MEM_TO_MEM; 1310 d->dev_addr = xt->src_start; 1311 d->es = data_type; 1312 sg->en = xt->sgl[0].size / BIT(data_type); 1313 sg->fn = xt->numf; 1314 sg->addr = xt->dst_start; 1315 d->sglen = 1; 1316 d->ccr = c->ccr; 1317 1318 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 1319 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 1320 if (src_icg) { 1321 d->ccr |= CCR_SRC_AMODE_DBLIDX; 1322 d->ei = 1; 1323 d->fi = src_icg + 1; 1324 } else if (xt->src_inc) { 1325 d->ccr |= CCR_SRC_AMODE_POSTINC; 1326 d->fi = 0; 1327 } else { 1328 dev_err(chan->device->dev, 1329 "%s: SRC constant addressing is not supported\n", 1330 __func__); 1331 kfree(d); 1332 return NULL; 1333 } 1334 1335 if (dst_icg) { 1336 d->ccr |= CCR_DST_AMODE_DBLIDX; 1337 sg->ei = 1; 1338 sg->fi = dst_icg + 1; 1339 } else if (xt->dst_inc) { 1340 d->ccr |= CCR_DST_AMODE_POSTINC; 1341 sg->fi = 0; 1342 } else { 1343 dev_err(chan->device->dev, 1344 "%s: DST constant addressing is not supported\n", 1345 __func__); 1346 kfree(d); 1347 return NULL; 1348 } 1349 1350 d->cicr = CICR_DROP_IE | CICR_FRAME_IE; 1351 1352 d->csdp = data_type; 1353 1354 if (dma_omap1()) { 1355 d->cicr |= CICR_TOUT_IE; 1356 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; 1357 } else { 1358 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; 1359 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1360 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1361 } 1362 1363 return vchan_tx_prep(&c->vc, &d->vd, flags); 1364 } 1365 1366 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) 1367 { 1368 struct omap_chan *c = to_omap_dma_chan(chan); 1369 1370 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 1371 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 1372 return -EINVAL; 1373 1374 if (cfg->src_maxburst > chan->device->max_burst || 1375 cfg->dst_maxburst > chan->device->max_burst) 1376 return -EINVAL; 1377 1378 memcpy(&c->cfg, cfg, sizeof(c->cfg)); 1379 1380 return 0; 1381 } 1382 1383 static int omap_dma_terminate_all(struct dma_chan *chan) 1384 { 1385 struct omap_chan *c = to_omap_dma_chan(chan); 1386 unsigned long flags; 1387 LIST_HEAD(head); 1388 1389 spin_lock_irqsave(&c->vc.lock, flags); 1390 1391 /* 1392 * Stop DMA activity: we assume the callback will not be called 1393 * after omap_dma_stop() returns (even if it does, it will see 1394 * c->desc is NULL and exit.) 1395 */ 1396 if (c->desc) { 1397 vchan_terminate_vdesc(&c->desc->vd); 1398 c->desc = NULL; 1399 /* Avoid stopping the dma twice */ 1400 if (!c->paused) 1401 omap_dma_stop(c); 1402 } 1403 1404 c->cyclic = false; 1405 c->paused = false; 1406 1407 vchan_get_all_descriptors(&c->vc, &head); 1408 spin_unlock_irqrestore(&c->vc.lock, flags); 1409 vchan_dma_desc_free_list(&c->vc, &head); 1410 1411 return 0; 1412 } 1413 1414 static void omap_dma_synchronize(struct dma_chan *chan) 1415 { 1416 struct omap_chan *c = to_omap_dma_chan(chan); 1417 1418 vchan_synchronize(&c->vc); 1419 } 1420 1421 static int omap_dma_pause(struct dma_chan *chan) 1422 { 1423 struct omap_chan *c = to_omap_dma_chan(chan); 1424 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1425 unsigned long flags; 1426 int ret = -EINVAL; 1427 bool can_pause = false; 1428 1429 spin_lock_irqsave(&od->irq_lock, flags); 1430 1431 if (!c->desc) 1432 goto out; 1433 1434 if (c->cyclic) 1435 can_pause = true; 1436 1437 /* 1438 * We do not allow DMA_MEM_TO_DEV transfers to be paused. 1439 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer: 1440 * "When a channel is disabled during a transfer, the channel undergoes 1441 * an abort, unless it is hardware-source-synchronized …". 1442 * A source-synchronised channel is one where the fetching of data is 1443 * under control of the device. In other words, a device-to-memory 1444 * transfer. So, a destination-synchronised channel (which would be a 1445 * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE 1446 * bit is cleared. 1447 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel 1448 * aborts immediately after completion of current read/write 1449 * transactions and then the FIFO is cleaned up." The term "cleaned up" 1450 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE 1451 * are both clear _before_ disabling the channel, otherwise data loss 1452 * will occur. 1453 * The problem is that if the channel is active, then device activity 1454 * can result in DMA activity starting between reading those as both 1455 * clear and the write to DMA_CCR to clear the enable bit hitting the 1456 * hardware. If the DMA hardware can't drain the data in its FIFO to the 1457 * destination, then data loss "might" occur (say if we write to an UART 1458 * and the UART is not accepting any further data). 1459 */ 1460 else if (c->desc->dir == DMA_DEV_TO_MEM) 1461 can_pause = true; 1462 1463 if (can_pause && !c->paused) { 1464 ret = omap_dma_stop(c); 1465 if (!ret) 1466 c->paused = true; 1467 } 1468 out: 1469 spin_unlock_irqrestore(&od->irq_lock, flags); 1470 1471 return ret; 1472 } 1473 1474 static int omap_dma_resume(struct dma_chan *chan) 1475 { 1476 struct omap_chan *c = to_omap_dma_chan(chan); 1477 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1478 unsigned long flags; 1479 int ret = -EINVAL; 1480 1481 spin_lock_irqsave(&od->irq_lock, flags); 1482 1483 if (c->paused && c->desc) { 1484 mb(); 1485 1486 /* Restore channel link register */ 1487 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl); 1488 1489 omap_dma_start(c, c->desc); 1490 c->paused = false; 1491 ret = 0; 1492 } 1493 spin_unlock_irqrestore(&od->irq_lock, flags); 1494 1495 return ret; 1496 } 1497 1498 static int omap_dma_chan_init(struct omap_dmadev *od) 1499 { 1500 struct omap_chan *c; 1501 1502 c = kzalloc(sizeof(*c), GFP_KERNEL); 1503 if (!c) 1504 return -ENOMEM; 1505 1506 c->reg_map = od->reg_map; 1507 c->vc.desc_free = omap_dma_desc_free; 1508 vchan_init(&c->vc, &od->ddev); 1509 1510 return 0; 1511 } 1512 1513 static void omap_dma_free(struct omap_dmadev *od) 1514 { 1515 while (!list_empty(&od->ddev.channels)) { 1516 struct omap_chan *c = list_first_entry(&od->ddev.channels, 1517 struct omap_chan, vc.chan.device_node); 1518 1519 list_del(&c->vc.chan.device_node); 1520 tasklet_kill(&c->vc.task); 1521 kfree(c); 1522 } 1523 } 1524 1525 /* Currently only used for omap2. For omap1, also a check for lcd_dma is needed */ 1526 static int omap_dma_busy_notifier(struct notifier_block *nb, 1527 unsigned long cmd, void *v) 1528 { 1529 struct omap_dmadev *od; 1530 struct omap_chan *c; 1531 int lch = -1; 1532 1533 od = container_of(nb, struct omap_dmadev, nb); 1534 1535 switch (cmd) { 1536 case CPU_CLUSTER_PM_ENTER: 1537 while (1) { 1538 lch = find_next_bit(od->lch_bitmap, od->lch_count, 1539 lch + 1); 1540 if (lch >= od->lch_count) 1541 break; 1542 c = od->lch_map[lch]; 1543 if (!c) 1544 continue; 1545 if (omap_dma_chan_read(c, CCR) & CCR_ENABLE) 1546 return NOTIFY_BAD; 1547 } 1548 break; 1549 case CPU_CLUSTER_PM_ENTER_FAILED: 1550 case CPU_CLUSTER_PM_EXIT: 1551 break; 1552 } 1553 1554 return NOTIFY_OK; 1555 } 1556 1557 /* 1558 * We are using IRQENABLE_L1, and legacy DMA code was using IRQENABLE_L0. 1559 * As the DSP may be using IRQENABLE_L2 and L3, let's not touch those for 1560 * now. Context save seems to be only currently needed on omap3. 1561 */ 1562 static void omap_dma_context_save(struct omap_dmadev *od) 1563 { 1564 od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0); 1565 od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1); 1566 od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); 1567 od->context.gcr = omap_dma_glbl_read(od, GCR); 1568 } 1569 1570 static void omap_dma_context_restore(struct omap_dmadev *od) 1571 { 1572 int i; 1573 1574 omap_dma_glbl_write(od, GCR, od->context.gcr); 1575 omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig); 1576 omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0); 1577 omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1); 1578 1579 /* Clear IRQSTATUS_L0 as legacy DMA code is no longer doing it */ 1580 if (od->plat->errata & DMA_ROMCODE_BUG) 1581 omap_dma_glbl_write(od, IRQSTATUS_L0, 0); 1582 1583 /* Clear dma channels */ 1584 for (i = 0; i < od->lch_count; i++) 1585 omap_dma_clear_lch(od, i); 1586 } 1587 1588 /* Currently only used for omap3 */ 1589 static int omap_dma_context_notifier(struct notifier_block *nb, 1590 unsigned long cmd, void *v) 1591 { 1592 struct omap_dmadev *od; 1593 1594 od = container_of(nb, struct omap_dmadev, nb); 1595 1596 switch (cmd) { 1597 case CPU_CLUSTER_PM_ENTER: 1598 omap_dma_context_save(od); 1599 break; 1600 case CPU_CLUSTER_PM_ENTER_FAILED: 1601 case CPU_CLUSTER_PM_EXIT: 1602 omap_dma_context_restore(od); 1603 break; 1604 } 1605 1606 return NOTIFY_OK; 1607 } 1608 1609 static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate, 1610 int max_fifo_depth, int tparams) 1611 { 1612 u32 val; 1613 1614 /* Set only for omap2430 and later */ 1615 if (!od->cfg->rw_priority) 1616 return; 1617 1618 if (max_fifo_depth == 0) 1619 max_fifo_depth = 1; 1620 if (arb_rate == 0) 1621 arb_rate = 1; 1622 1623 val = 0xff & max_fifo_depth; 1624 val |= (0x3 & tparams) << 12; 1625 val |= (arb_rate & 0xff) << 16; 1626 1627 omap_dma_glbl_write(od, GCR, val); 1628 } 1629 1630 #define OMAP_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 1631 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 1632 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 1633 1634 /* 1635 * No flags currently set for default configuration as omap1 is still 1636 * using platform data. 1637 */ 1638 static const struct omap_dma_config default_cfg; 1639 1640 static int omap_dma_probe(struct platform_device *pdev) 1641 { 1642 const struct omap_dma_config *conf; 1643 struct omap_dmadev *od; 1644 struct resource *res; 1645 int rc, i, irq; 1646 u32 val; 1647 1648 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); 1649 if (!od) 1650 return -ENOMEM; 1651 1652 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1653 od->base = devm_ioremap_resource(&pdev->dev, res); 1654 if (IS_ERR(od->base)) 1655 return PTR_ERR(od->base); 1656 1657 conf = of_device_get_match_data(&pdev->dev); 1658 if (conf) { 1659 od->cfg = conf; 1660 od->plat = dev_get_platdata(&pdev->dev); 1661 if (!od->plat) { 1662 dev_err(&pdev->dev, "omap_system_dma_plat_info is missing"); 1663 return -ENODEV; 1664 } 1665 } else { 1666 od->cfg = &default_cfg; 1667 1668 od->plat = omap_get_plat_info(); 1669 if (!od->plat) 1670 return -EPROBE_DEFER; 1671 } 1672 1673 od->reg_map = od->plat->reg_map; 1674 1675 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 1676 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); 1677 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); 1678 dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask); 1679 od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; 1680 od->ddev.device_free_chan_resources = omap_dma_free_chan_resources; 1681 od->ddev.device_tx_status = omap_dma_tx_status; 1682 od->ddev.device_issue_pending = omap_dma_issue_pending; 1683 od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg; 1684 od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic; 1685 od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy; 1686 od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved; 1687 od->ddev.device_config = omap_dma_slave_config; 1688 od->ddev.device_pause = omap_dma_pause; 1689 od->ddev.device_resume = omap_dma_resume; 1690 od->ddev.device_terminate_all = omap_dma_terminate_all; 1691 od->ddev.device_synchronize = omap_dma_synchronize; 1692 od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS; 1693 od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS; 1694 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1695 if (__dma_omap15xx(od->plat->dma_attr)) 1696 od->ddev.residue_granularity = 1697 DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1698 else 1699 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1700 od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */ 1701 od->ddev.dev = &pdev->dev; 1702 INIT_LIST_HEAD(&od->ddev.channels); 1703 mutex_init(&od->lch_lock); 1704 spin_lock_init(&od->lock); 1705 spin_lock_init(&od->irq_lock); 1706 1707 /* Number of DMA requests */ 1708 od->dma_requests = OMAP_SDMA_REQUESTS; 1709 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 1710 "dma-requests", 1711 &od->dma_requests)) { 1712 dev_info(&pdev->dev, 1713 "Missing dma-requests property, using %u.\n", 1714 OMAP_SDMA_REQUESTS); 1715 } 1716 1717 /* Number of available logical channels */ 1718 if (!pdev->dev.of_node) { 1719 od->lch_count = od->plat->dma_attr->lch_count; 1720 if (unlikely(!od->lch_count)) 1721 od->lch_count = OMAP_SDMA_CHANNELS; 1722 } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels", 1723 &od->lch_count)) { 1724 dev_info(&pdev->dev, 1725 "Missing dma-channels property, using %u.\n", 1726 OMAP_SDMA_CHANNELS); 1727 od->lch_count = OMAP_SDMA_CHANNELS; 1728 } 1729 1730 /* Mask of allowed logical channels */ 1731 if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node, 1732 "dma-channel-mask", 1733 &val)) { 1734 /* Tag channels not in mask as reserved */ 1735 val = ~val; 1736 bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count); 1737 } 1738 if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED) 1739 bitmap_set(od->lch_bitmap, 0, 2); 1740 1741 od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count, 1742 sizeof(*od->lch_map), 1743 GFP_KERNEL); 1744 if (!od->lch_map) 1745 return -ENOMEM; 1746 1747 for (i = 0; i < od->dma_requests; i++) { 1748 rc = omap_dma_chan_init(od); 1749 if (rc) { 1750 omap_dma_free(od); 1751 return rc; 1752 } 1753 } 1754 1755 irq = platform_get_irq(pdev, 1); 1756 if (irq <= 0) { 1757 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); 1758 od->legacy = true; 1759 } else { 1760 /* Disable all interrupts */ 1761 od->irq_enable_mask = 0; 1762 omap_dma_glbl_write(od, IRQENABLE_L1, 0); 1763 1764 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, 1765 IRQF_SHARED, "omap-dma-engine", od); 1766 if (rc) { 1767 omap_dma_free(od); 1768 return rc; 1769 } 1770 } 1771 1772 if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123) 1773 od->ll123_supported = true; 1774 1775 od->ddev.filter.map = od->plat->slave_map; 1776 od->ddev.filter.mapcnt = od->plat->slavecnt; 1777 od->ddev.filter.fn = omap_dma_filter_fn; 1778 1779 if (od->ll123_supported) { 1780 od->desc_pool = dma_pool_create(dev_name(&pdev->dev), 1781 &pdev->dev, 1782 sizeof(struct omap_type2_desc), 1783 4, 0); 1784 if (!od->desc_pool) { 1785 dev_err(&pdev->dev, 1786 "unable to allocate descriptor pool\n"); 1787 od->ll123_supported = false; 1788 } 1789 } 1790 1791 rc = dma_async_device_register(&od->ddev); 1792 if (rc) { 1793 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", 1794 rc); 1795 omap_dma_free(od); 1796 return rc; 1797 } 1798 1799 platform_set_drvdata(pdev, od); 1800 1801 if (pdev->dev.of_node) { 1802 omap_dma_info.dma_cap = od->ddev.cap_mask; 1803 1804 /* Device-tree DMA controller registration */ 1805 rc = of_dma_controller_register(pdev->dev.of_node, 1806 of_dma_simple_xlate, &omap_dma_info); 1807 if (rc) { 1808 pr_warn("OMAP-DMA: failed to register DMA controller\n"); 1809 dma_async_device_unregister(&od->ddev); 1810 omap_dma_free(od); 1811 } 1812 } 1813 1814 omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0); 1815 1816 if (od->cfg->needs_busy_check) { 1817 od->nb.notifier_call = omap_dma_busy_notifier; 1818 cpu_pm_register_notifier(&od->nb); 1819 } else if (od->cfg->may_lose_context) { 1820 od->nb.notifier_call = omap_dma_context_notifier; 1821 cpu_pm_register_notifier(&od->nb); 1822 } 1823 1824 dev_info(&pdev->dev, "OMAP DMA engine driver%s\n", 1825 od->ll123_supported ? " (LinkedList1/2/3 supported)" : ""); 1826 1827 return rc; 1828 } 1829 1830 static int omap_dma_remove(struct platform_device *pdev) 1831 { 1832 struct omap_dmadev *od = platform_get_drvdata(pdev); 1833 int irq; 1834 1835 if (od->cfg->may_lose_context) 1836 cpu_pm_unregister_notifier(&od->nb); 1837 1838 if (pdev->dev.of_node) 1839 of_dma_controller_free(pdev->dev.of_node); 1840 1841 irq = platform_get_irq(pdev, 1); 1842 devm_free_irq(&pdev->dev, irq, od); 1843 1844 dma_async_device_unregister(&od->ddev); 1845 1846 if (!od->legacy) { 1847 /* Disable all interrupts */ 1848 omap_dma_glbl_write(od, IRQENABLE_L0, 0); 1849 } 1850 1851 if (od->ll123_supported) 1852 dma_pool_destroy(od->desc_pool); 1853 1854 omap_dma_free(od); 1855 1856 return 0; 1857 } 1858 1859 static const struct omap_dma_config omap2420_data = { 1860 .lch_end = CCFN, 1861 .rw_priority = true, 1862 .needs_lch_clear = true, 1863 .needs_busy_check = true, 1864 }; 1865 1866 static const struct omap_dma_config omap2430_data = { 1867 .lch_end = CCFN, 1868 .rw_priority = true, 1869 .needs_lch_clear = true, 1870 }; 1871 1872 static const struct omap_dma_config omap3430_data = { 1873 .lch_end = CCFN, 1874 .rw_priority = true, 1875 .needs_lch_clear = true, 1876 .may_lose_context = true, 1877 }; 1878 1879 static const struct omap_dma_config omap3630_data = { 1880 .lch_end = CCDN, 1881 .rw_priority = true, 1882 .needs_lch_clear = true, 1883 .may_lose_context = true, 1884 }; 1885 1886 static const struct omap_dma_config omap4_data = { 1887 .lch_end = CCDN, 1888 .rw_priority = true, 1889 .needs_lch_clear = true, 1890 }; 1891 1892 static const struct of_device_id omap_dma_match[] = { 1893 { .compatible = "ti,omap2420-sdma", .data = &omap2420_data, }, 1894 { .compatible = "ti,omap2430-sdma", .data = &omap2430_data, }, 1895 { .compatible = "ti,omap3430-sdma", .data = &omap3430_data, }, 1896 { .compatible = "ti,omap3630-sdma", .data = &omap3630_data, }, 1897 { .compatible = "ti,omap4430-sdma", .data = &omap4_data, }, 1898 {}, 1899 }; 1900 MODULE_DEVICE_TABLE(of, omap_dma_match); 1901 1902 static struct platform_driver omap_dma_driver = { 1903 .probe = omap_dma_probe, 1904 .remove = omap_dma_remove, 1905 .driver = { 1906 .name = "omap-dma-engine", 1907 .of_match_table = of_match_ptr(omap_dma_match), 1908 }, 1909 }; 1910 1911 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param) 1912 { 1913 if (chan->device->dev->driver == &omap_dma_driver.driver) { 1914 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1915 struct omap_chan *c = to_omap_dma_chan(chan); 1916 unsigned req = *(unsigned *)param; 1917 1918 if (req <= od->dma_requests) { 1919 c->dma_sig = req; 1920 return true; 1921 } 1922 } 1923 return false; 1924 } 1925 1926 static int omap_dma_init(void) 1927 { 1928 return platform_driver_register(&omap_dma_driver); 1929 } 1930 subsys_initcall(omap_dma_init); 1931 1932 static void __exit omap_dma_exit(void) 1933 { 1934 platform_driver_unregister(&omap_dma_driver); 1935 } 1936 module_exit(omap_dma_exit); 1937 1938 MODULE_AUTHOR("Russell King"); 1939 MODULE_LICENSE("GPL"); 1940