Lines Matching +full:omap +full:- +full:sdma
1 // SPDX-License-Identifier: GPL-2.0-only
3 * OMAP DMAengine support
8 #include <linux/dma-mapping.h>
15 #include <linux/omap-dma.h>
22 #include "../virt-dma.h"
101 uint32_t en; /* number of elements (24-bit) */
102 uint32_t fn; /* number of frames (16-bit) */
250 struct omap_desc *d = to_omap_dma_desc(&vd->tx); in omap_dma_desc_free()
252 if (d->using_ll) { in omap_dma_desc_free()
253 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device); in omap_dma_desc_free()
256 for (i = 0; i < d->sglen; i++) { in omap_dma_desc_free()
257 if (d->sg[i].t2_desc) in omap_dma_desc_free()
258 dma_pool_free(od->desc_pool, d->sg[i].t2_desc, in omap_dma_desc_free()
259 d->sg[i].t2_desc_paddr); in omap_dma_desc_free()
269 struct omap_sg *sg = &d->sg[idx]; in omap_dma_fill_type2_desc()
270 struct omap_type2_desc *t2_desc = sg->t2_desc; in omap_dma_fill_type2_desc()
273 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr; in omap_dma_fill_type2_desc()
275 t2_desc->next_desc = 0xfffffffc; in omap_dma_fill_type2_desc()
277 t2_desc->en = sg->en; in omap_dma_fill_type2_desc()
278 t2_desc->addr = sg->addr; in omap_dma_fill_type2_desc()
279 t2_desc->fn = sg->fn & 0xffff; in omap_dma_fill_type2_desc()
280 t2_desc->cicr = d->cicr; in omap_dma_fill_type2_desc()
282 t2_desc->cicr &= ~CICR_BLOCK_IE; in omap_dma_fill_type2_desc()
286 t2_desc->cdei = sg->ei; in omap_dma_fill_type2_desc()
287 t2_desc->csei = d->ei; in omap_dma_fill_type2_desc()
288 t2_desc->cdfi = sg->fi; in omap_dma_fill_type2_desc()
289 t2_desc->csfi = d->fi; in omap_dma_fill_type2_desc()
291 t2_desc->en |= DESC_NXT_DV_REFRESH; in omap_dma_fill_type2_desc()
292 t2_desc->en |= DESC_NXT_SV_REUSE; in omap_dma_fill_type2_desc()
295 t2_desc->cdei = d->ei; in omap_dma_fill_type2_desc()
296 t2_desc->csei = sg->ei; in omap_dma_fill_type2_desc()
297 t2_desc->cdfi = d->fi; in omap_dma_fill_type2_desc()
298 t2_desc->csfi = sg->fi; in omap_dma_fill_type2_desc()
300 t2_desc->en |= DESC_NXT_SV_REFRESH; in omap_dma_fill_type2_desc()
301 t2_desc->en |= DESC_NXT_DV_REUSE; in omap_dma_fill_type2_desc()
307 t2_desc->en |= DESC_NTYPE_TYPE2; in omap_dma_fill_type2_desc()
353 const struct omap_dma_reg *r = od->reg_map + reg; in omap_dma_glbl_write()
355 WARN_ON(r->stride); in omap_dma_glbl_write()
357 omap_dma_write(val, r->type, od->base + r->offset); in omap_dma_glbl_write()
362 const struct omap_dma_reg *r = od->reg_map + reg; in omap_dma_glbl_read()
364 WARN_ON(r->stride); in omap_dma_glbl_read()
366 return omap_dma_read(r->type, od->base + r->offset); in omap_dma_glbl_read()
371 const struct omap_dma_reg *r = c->reg_map + reg; in omap_dma_chan_write()
373 omap_dma_write(val, r->type, c->channel_base + r->offset); in omap_dma_chan_write()
378 const struct omap_dma_reg *r = c->reg_map + reg; in omap_dma_chan_read()
380 return omap_dma_read(r->type, c->channel_base + r->offset); in omap_dma_chan_read()
406 c = od->lch_map[lch]; in omap_dma_clear_lch()
410 for (i = CSDP; i <= od->cfg->lch_end; i++) in omap_dma_clear_lch()
417 c->channel_base = od->base + od->plat->channel_stride * lch; in omap_dma_assign()
419 od->lch_map[lch] = c; in omap_dma_assign()
424 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); in omap_dma_start()
425 uint16_t cicr = d->cicr; in omap_dma_start()
427 if (__dma_omap15xx(od->plat->dma_attr)) in omap_dma_start()
434 if (d->using_ll) { in omap_dma_start()
437 if (d->dir == DMA_DEV_TO_MEM) in omap_dma_start()
443 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr); in omap_dma_start()
449 } else if (od->ll123_supported) { in omap_dma_start()
457 omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); in omap_dma_start()
459 c->running = true; in omap_dma_start()
467 /* Wait for sDMA FIFO to drain */ in omap_dma_drain_chan()
480 dev_err(c->vc.chan.device->dev, in omap_dma_drain_chan()
482 c->dma_ch); in omap_dma_drain_chan()
487 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); in omap_dma_stop()
496 if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { in omap_dma_stop()
508 if (!(c->ccr & CCR_BUFFERING_DISABLE)) in omap_dma_stop()
514 return -EINVAL; in omap_dma_stop()
519 if (!(c->ccr & CCR_BUFFERING_DISABLE)) in omap_dma_stop()
525 if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { in omap_dma_stop()
535 c->running = false; in omap_dma_stop()
541 struct omap_sg *sg = d->sg + c->sgidx; in omap_dma_start_sg()
544 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { in omap_dma_start_sg()
554 omap_dma_chan_write(c, cxsa, sg->addr); in omap_dma_start_sg()
555 omap_dma_chan_write(c, cxei, sg->ei); in omap_dma_start_sg()
556 omap_dma_chan_write(c, cxfi, sg->fi); in omap_dma_start_sg()
557 omap_dma_chan_write(c, CEN, sg->en); in omap_dma_start_sg()
558 omap_dma_chan_write(c, CFN, sg->fn); in omap_dma_start_sg()
561 c->sgidx++; in omap_dma_start_sg()
566 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); in omap_dma_start_desc()
571 c->desc = NULL; in omap_dma_start_desc()
575 list_del(&vd->node); in omap_dma_start_desc()
577 c->desc = d = to_omap_dma_desc(&vd->tx); in omap_dma_start_desc()
578 c->sgidx = 0; in omap_dma_start_desc()
587 omap_dma_chan_write(c, CCR, d->ccr); in omap_dma_start_desc()
589 omap_dma_chan_write(c, CCR2, d->ccr >> 16); in omap_dma_start_desc()
591 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { in omap_dma_start_desc()
601 omap_dma_chan_write(c, cxsa, d->dev_addr); in omap_dma_start_desc()
602 omap_dma_chan_write(c, cxei, d->ei); in omap_dma_start_desc()
603 omap_dma_chan_write(c, cxfi, d->fi); in omap_dma_start_desc()
604 omap_dma_chan_write(c, CSDP, d->csdp); in omap_dma_start_desc()
605 omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); in omap_dma_start_desc()
616 spin_lock_irqsave(&c->vc.lock, flags); in omap_dma_callback()
617 d = c->desc; in omap_dma_callback()
619 if (c->cyclic) { in omap_dma_callback()
620 vchan_cyclic_callback(&d->vd); in omap_dma_callback()
621 } else if (d->using_ll || c->sgidx == d->sglen) { in omap_dma_callback()
623 vchan_cookie_complete(&d->vd); in omap_dma_callback()
628 spin_unlock_irqrestore(&c->vc.lock, flags); in omap_dma_callback()
636 spin_lock(&od->irq_lock); in omap_dma_irq()
639 status &= od->irq_enable_mask; in omap_dma_irq()
641 spin_unlock(&od->irq_lock); in omap_dma_irq()
649 channel -= 1; in omap_dma_irq()
653 c = od->lch_map[channel]; in omap_dma_irq()
656 dev_err(od->ddev.dev, "invalid channel %u\n", channel); in omap_dma_irq()
666 spin_unlock(&od->irq_lock); in omap_dma_irq()
675 mutex_lock(&od->lch_lock); in omap_dma_get_lch()
676 channel = find_first_zero_bit(od->lch_bitmap, od->lch_count); in omap_dma_get_lch()
677 if (channel >= od->lch_count) in omap_dma_get_lch()
679 set_bit(channel, od->lch_bitmap); in omap_dma_get_lch()
680 mutex_unlock(&od->lch_lock); in omap_dma_get_lch()
688 mutex_unlock(&od->lch_lock); in omap_dma_get_lch()
689 *lch = -EINVAL; in omap_dma_get_lch()
691 return -EBUSY; in omap_dma_get_lch()
697 mutex_lock(&od->lch_lock); in omap_dma_put_lch()
698 clear_bit(lch, od->lch_bitmap); in omap_dma_put_lch()
699 mutex_unlock(&od->lch_lock); in omap_dma_put_lch()
704 return IS_ENABLED(CONFIG_ARCH_OMAP1) && od->legacy; in omap_dma_legacy()
709 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_alloc_chan_resources()
711 struct device *dev = od->ddev.dev; in omap_dma_alloc_chan_resources()
715 ret = omap_request_dma(c->dma_sig, "DMA engine", in omap_dma_alloc_chan_resources()
716 omap_dma_callback, c, &c->dma_ch); in omap_dma_alloc_chan_resources()
718 ret = omap_dma_get_lch(od, &c->dma_ch); in omap_dma_alloc_chan_resources()
721 dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig); in omap_dma_alloc_chan_resources()
724 omap_dma_assign(od, c, c->dma_ch); in omap_dma_alloc_chan_resources()
729 spin_lock_irq(&od->irq_lock); in omap_dma_alloc_chan_resources()
730 val = BIT(c->dma_ch); in omap_dma_alloc_chan_resources()
732 od->irq_enable_mask |= val; in omap_dma_alloc_chan_resources()
733 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); in omap_dma_alloc_chan_resources()
736 val &= ~BIT(c->dma_ch); in omap_dma_alloc_chan_resources()
738 spin_unlock_irq(&od->irq_lock); in omap_dma_alloc_chan_resources()
743 if (__dma_omap16xx(od->plat->dma_attr)) { in omap_dma_alloc_chan_resources()
744 c->ccr = CCR_OMAP31_DISABLE; in omap_dma_alloc_chan_resources()
745 /* Duplicate what plat-omap/dma.c does */ in omap_dma_alloc_chan_resources()
746 c->ccr |= c->dma_ch + 1; in omap_dma_alloc_chan_resources()
748 c->ccr = c->dma_sig & 0x1f; in omap_dma_alloc_chan_resources()
751 c->ccr = c->dma_sig & 0x1f; in omap_dma_alloc_chan_resources()
752 c->ccr |= (c->dma_sig & ~0x1f) << 14; in omap_dma_alloc_chan_resources()
754 if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) in omap_dma_alloc_chan_resources()
755 c->ccr |= CCR_BUFFERING_DISABLE; in omap_dma_alloc_chan_resources()
762 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_free_chan_resources()
766 spin_lock_irq(&od->irq_lock); in omap_dma_free_chan_resources()
767 od->irq_enable_mask &= ~BIT(c->dma_ch); in omap_dma_free_chan_resources()
768 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); in omap_dma_free_chan_resources()
769 spin_unlock_irq(&od->irq_lock); in omap_dma_free_chan_resources()
772 c->channel_base = NULL; in omap_dma_free_chan_resources()
773 od->lch_map[c->dma_ch] = NULL; in omap_dma_free_chan_resources()
774 vchan_free_chan_resources(&c->vc); in omap_dma_free_chan_resources()
777 omap_free_dma(c->dma_ch); in omap_dma_free_chan_resources()
779 omap_dma_put_lch(od, c->dma_ch); in omap_dma_free_chan_resources()
781 dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch, in omap_dma_free_chan_resources()
782 c->dma_sig); in omap_dma_free_chan_resources()
783 c->dma_sig = 0; in omap_dma_free_chan_resources()
788 return sg->en * sg->fn; in omap_dma_sg_size()
796 for (size = i = 0; i < d->sglen; i++) in omap_dma_desc_size()
797 size += omap_dma_sg_size(&d->sg[i]); in omap_dma_desc_size()
799 return size * es_bytes[d->es]; in omap_dma_desc_size()
805 size_t size, es_size = es_bytes[d->es]; in omap_dma_desc_size_pos()
807 for (size = i = 0; i < d->sglen; i++) { in omap_dma_desc_size_pos()
808 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size; in omap_dma_desc_size_pos()
812 else if (addr >= d->sg[i].addr && in omap_dma_desc_size_pos()
813 addr < d->sg[i].addr + this_size) in omap_dma_desc_size_pos()
814 size += d->sg[i].addr + this_size - addr; in omap_dma_desc_size_pos()
820 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
825 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); in omap_dma_chan_read_3_3()
829 if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) in omap_dma_chan_read_3_3()
837 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); in omap_dma_get_src_pos()
840 if (__dma_omap15xx(od->plat->dma_attr)) { in omap_dma_get_src_pos()
863 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); in omap_dma_get_dst_pos()
866 if (__dma_omap15xx(od->plat->dma_attr)) { in omap_dma_get_dst_pos()
899 spin_lock_irqsave(&c->vc.lock, flags); in omap_dma_tx_status()
900 if (c->desc && c->desc->vd.tx.cookie == cookie) in omap_dma_tx_status()
901 d = c->desc; in omap_dma_tx_status()
909 if (d->dir == DMA_MEM_TO_DEV) in omap_dma_tx_status()
911 else if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) in omap_dma_tx_status()
916 txstate->residue = omap_dma_desc_size_pos(d, pos); in omap_dma_tx_status()
918 struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie); in omap_dma_tx_status()
921 txstate->residue = omap_dma_desc_size( in omap_dma_tx_status()
922 to_omap_dma_desc(&vd->tx)); in omap_dma_tx_status()
924 txstate->residue = 0; in omap_dma_tx_status()
928 if (ret == DMA_IN_PROGRESS && c->paused) { in omap_dma_tx_status()
930 } else if (d && d->polled && c->running) { in omap_dma_tx_status()
939 vchan_cookie_complete(&d->vd); in omap_dma_tx_status()
943 spin_unlock_irqrestore(&c->vc.lock, flags); in omap_dma_tx_status()
953 spin_lock_irqsave(&c->vc.lock, flags); in omap_dma_issue_pending()
954 if (vchan_issue_pending(&c->vc) && !c->desc) in omap_dma_issue_pending()
956 spin_unlock_irqrestore(&c->vc.lock, flags); in omap_dma_issue_pending()
963 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_prep_slave_sg()
975 dev_addr = c->cfg.src_addr; in omap_dma_prep_slave_sg()
976 dev_width = c->cfg.src_addr_width; in omap_dma_prep_slave_sg()
977 burst = c->cfg.src_maxburst; in omap_dma_prep_slave_sg()
978 port_window = c->cfg.src_port_window_size; in omap_dma_prep_slave_sg()
980 dev_addr = c->cfg.dst_addr; in omap_dma_prep_slave_sg()
981 dev_width = c->cfg.dst_addr_width; in omap_dma_prep_slave_sg()
982 burst = c->cfg.dst_maxburst; in omap_dma_prep_slave_sg()
983 port_window = c->cfg.dst_port_window_size; in omap_dma_prep_slave_sg()
985 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); in omap_dma_prep_slave_sg()
1009 d->dir = dir; in omap_dma_prep_slave_sg()
1010 d->dev_addr = dev_addr; in omap_dma_prep_slave_sg()
1011 d->es = es; in omap_dma_prep_slave_sg()
1018 d->ei = 1; in omap_dma_prep_slave_sg()
1021 * the source frame index to be -1 * (port_window - 1) in omap_dma_prep_slave_sg()
1022 * we instruct the sDMA that after a frame is processed in omap_dma_prep_slave_sg()
1025 d->fi = -(port_window_bytes - 1); in omap_dma_prep_slave_sg()
1028 d->ccr = c->ccr | CCR_SYNC_FRAME; in omap_dma_prep_slave_sg()
1030 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED; in omap_dma_prep_slave_sg()
1032 d->ccr |= CCR_DST_AMODE_POSTINC; in omap_dma_prep_slave_sg()
1034 d->ccr |= CCR_SRC_AMODE_DBLIDX; in omap_dma_prep_slave_sg()
1037 d->csdp |= CSDP_SRC_BURST_64; in omap_dma_prep_slave_sg()
1039 d->csdp |= CSDP_SRC_BURST_32; in omap_dma_prep_slave_sg()
1041 d->csdp |= CSDP_SRC_BURST_16; in omap_dma_prep_slave_sg()
1044 d->ccr |= CCR_SRC_AMODE_CONSTANT; in omap_dma_prep_slave_sg()
1047 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED; in omap_dma_prep_slave_sg()
1049 d->ccr |= CCR_SRC_AMODE_POSTINC; in omap_dma_prep_slave_sg()
1051 d->ccr |= CCR_DST_AMODE_DBLIDX; in omap_dma_prep_slave_sg()
1054 d->csdp |= CSDP_DST_BURST_64; in omap_dma_prep_slave_sg()
1056 d->csdp |= CSDP_DST_BURST_32; in omap_dma_prep_slave_sg()
1058 d->csdp |= CSDP_DST_BURST_16; in omap_dma_prep_slave_sg()
1060 d->ccr |= CCR_DST_AMODE_CONSTANT; in omap_dma_prep_slave_sg()
1064 d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; in omap_dma_prep_slave_sg()
1065 d->csdp |= es; in omap_dma_prep_slave_sg()
1068 d->cicr |= CICR_TOUT_IE; in omap_dma_prep_slave_sg()
1071 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; in omap_dma_prep_slave_sg()
1073 d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; in omap_dma_prep_slave_sg()
1076 d->ccr |= CCR_TRIGGER_SRC; in omap_dma_prep_slave_sg()
1078 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; in omap_dma_prep_slave_sg()
1081 d->csdp |= CSDP_WRITE_LAST_NON_POSTED; in omap_dma_prep_slave_sg()
1083 if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) in omap_dma_prep_slave_sg()
1084 d->clnk_ctrl = c->dma_ch; in omap_dma_prep_slave_sg()
1092 * Note: DMA engine defines burst to be the number of dev-width in omap_dma_prep_slave_sg()
1099 d->using_ll = od->ll123_supported; in omap_dma_prep_slave_sg()
1102 struct omap_sg *osg = &d->sg[i]; in omap_dma_prep_slave_sg()
1104 osg->addr = sg_dma_address(sgent); in omap_dma_prep_slave_sg()
1105 osg->en = en; in omap_dma_prep_slave_sg()
1106 osg->fn = sg_dma_len(sgent) / frame_bytes; in omap_dma_prep_slave_sg()
1108 if (d->using_ll) { in omap_dma_prep_slave_sg()
1109 osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC, in omap_dma_prep_slave_sg()
1110 &osg->t2_desc_paddr); in omap_dma_prep_slave_sg()
1111 if (!osg->t2_desc) { in omap_dma_prep_slave_sg()
1112 dev_err(chan->device->dev, in omap_dma_prep_slave_sg()
1115 d->using_ll = false; in omap_dma_prep_slave_sg()
1119 omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1)); in omap_dma_prep_slave_sg()
1123 d->sglen = sglen; in omap_dma_prep_slave_sg()
1127 for (i = 0; i < d->sglen; i++) { in omap_dma_prep_slave_sg()
1128 struct omap_sg *osg = &d->sg[i]; in omap_dma_prep_slave_sg()
1130 if (osg->t2_desc) { in omap_dma_prep_slave_sg()
1131 dma_pool_free(od->desc_pool, osg->t2_desc, in omap_dma_prep_slave_sg()
1132 osg->t2_desc_paddr); in omap_dma_prep_slave_sg()
1133 osg->t2_desc = NULL; in omap_dma_prep_slave_sg()
1138 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); in omap_dma_prep_slave_sg()
1145 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_prep_dma_cyclic()
1154 dev_addr = c->cfg.src_addr; in omap_dma_prep_dma_cyclic()
1155 dev_width = c->cfg.src_addr_width; in omap_dma_prep_dma_cyclic()
1156 burst = c->cfg.src_maxburst; in omap_dma_prep_dma_cyclic()
1158 dev_addr = c->cfg.dst_addr; in omap_dma_prep_dma_cyclic()
1159 dev_width = c->cfg.dst_addr_width; in omap_dma_prep_dma_cyclic()
1160 burst = c->cfg.dst_maxburst; in omap_dma_prep_dma_cyclic()
1162 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); in omap_dma_prep_dma_cyclic()
1182 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); in omap_dma_prep_dma_cyclic()
1186 d->dir = dir; in omap_dma_prep_dma_cyclic()
1187 d->dev_addr = dev_addr; in omap_dma_prep_dma_cyclic()
1188 d->fi = burst; in omap_dma_prep_dma_cyclic()
1189 d->es = es; in omap_dma_prep_dma_cyclic()
1190 d->sg[0].addr = buf_addr; in omap_dma_prep_dma_cyclic()
1191 d->sg[0].en = period_len / es_bytes[es]; in omap_dma_prep_dma_cyclic()
1192 d->sg[0].fn = buf_len / period_len; in omap_dma_prep_dma_cyclic()
1193 d->sglen = 1; in omap_dma_prep_dma_cyclic()
1195 d->ccr = c->ccr; in omap_dma_prep_dma_cyclic()
1197 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; in omap_dma_prep_dma_cyclic()
1199 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; in omap_dma_prep_dma_cyclic()
1201 d->cicr = CICR_DROP_IE; in omap_dma_prep_dma_cyclic()
1203 d->cicr |= CICR_FRAME_IE; in omap_dma_prep_dma_cyclic()
1205 d->csdp = es; in omap_dma_prep_dma_cyclic()
1208 d->cicr |= CICR_TOUT_IE; in omap_dma_prep_dma_cyclic()
1211 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; in omap_dma_prep_dma_cyclic()
1213 d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; in omap_dma_prep_dma_cyclic()
1216 d->ccr |= CCR_SYNC_PACKET; in omap_dma_prep_dma_cyclic()
1218 d->ccr |= CCR_SYNC_ELEMENT; in omap_dma_prep_dma_cyclic()
1221 d->ccr |= CCR_TRIGGER_SRC; in omap_dma_prep_dma_cyclic()
1222 d->csdp |= CSDP_DST_PACKED; in omap_dma_prep_dma_cyclic()
1224 d->csdp |= CSDP_SRC_PACKED; in omap_dma_prep_dma_cyclic()
1227 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; in omap_dma_prep_dma_cyclic()
1229 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; in omap_dma_prep_dma_cyclic()
1232 if (__dma_omap15xx(od->plat->dma_attr)) in omap_dma_prep_dma_cyclic()
1233 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; in omap_dma_prep_dma_cyclic()
1235 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; in omap_dma_prep_dma_cyclic()
1237 c->cyclic = true; in omap_dma_prep_dma_cyclic()
1239 return vchan_tx_prep(&c->vc, &d->vd, flags); in omap_dma_prep_dma_cyclic()
1250 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); in omap_dma_prep_dma_memcpy()
1258 d->dir = DMA_MEM_TO_MEM; in omap_dma_prep_dma_memcpy()
1259 d->dev_addr = src; in omap_dma_prep_dma_memcpy()
1260 d->fi = 0; in omap_dma_prep_dma_memcpy()
1261 d->es = data_type; in omap_dma_prep_dma_memcpy()
1262 d->sg[0].en = len / BIT(data_type); in omap_dma_prep_dma_memcpy()
1263 d->sg[0].fn = 1; in omap_dma_prep_dma_memcpy()
1264 d->sg[0].addr = dest; in omap_dma_prep_dma_memcpy()
1265 d->sglen = 1; in omap_dma_prep_dma_memcpy()
1266 d->ccr = c->ccr; in omap_dma_prep_dma_memcpy()
1267 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; in omap_dma_prep_dma_memcpy()
1270 d->cicr |= CICR_FRAME_IE; in omap_dma_prep_dma_memcpy()
1272 d->polled = true; in omap_dma_prep_dma_memcpy()
1274 d->csdp = data_type; in omap_dma_prep_dma_memcpy()
1277 d->cicr |= CICR_TOUT_IE; in omap_dma_prep_dma_memcpy()
1278 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; in omap_dma_prep_dma_memcpy()
1280 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; in omap_dma_prep_dma_memcpy()
1281 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; in omap_dma_prep_dma_memcpy()
1282 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; in omap_dma_prep_dma_memcpy()
1285 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); in omap_dma_prep_dma_memcpy()
1299 if (is_slave_direction(xt->dir)) in omap_dma_prep_dma_interleaved()
1302 if (xt->frame_size != 1 || xt->numf == 0) in omap_dma_prep_dma_interleaved()
1305 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); in omap_dma_prep_dma_interleaved()
1309 data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size)); in omap_dma_prep_dma_interleaved()
1313 sg = &d->sg[0]; in omap_dma_prep_dma_interleaved()
1314 d->dir = DMA_MEM_TO_MEM; in omap_dma_prep_dma_interleaved()
1315 d->dev_addr = xt->src_start; in omap_dma_prep_dma_interleaved()
1316 d->es = data_type; in omap_dma_prep_dma_interleaved()
1317 sg->en = xt->sgl[0].size / BIT(data_type); in omap_dma_prep_dma_interleaved()
1318 sg->fn = xt->numf; in omap_dma_prep_dma_interleaved()
1319 sg->addr = xt->dst_start; in omap_dma_prep_dma_interleaved()
1320 d->sglen = 1; in omap_dma_prep_dma_interleaved()
1321 d->ccr = c->ccr; in omap_dma_prep_dma_interleaved()
1323 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); in omap_dma_prep_dma_interleaved()
1324 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); in omap_dma_prep_dma_interleaved()
1326 d->ccr |= CCR_SRC_AMODE_DBLIDX; in omap_dma_prep_dma_interleaved()
1327 d->ei = 1; in omap_dma_prep_dma_interleaved()
1328 d->fi = src_icg + 1; in omap_dma_prep_dma_interleaved()
1329 } else if (xt->src_inc) { in omap_dma_prep_dma_interleaved()
1330 d->ccr |= CCR_SRC_AMODE_POSTINC; in omap_dma_prep_dma_interleaved()
1331 d->fi = 0; in omap_dma_prep_dma_interleaved()
1333 dev_err(chan->device->dev, in omap_dma_prep_dma_interleaved()
1341 d->ccr |= CCR_DST_AMODE_DBLIDX; in omap_dma_prep_dma_interleaved()
1342 sg->ei = 1; in omap_dma_prep_dma_interleaved()
1343 sg->fi = dst_icg + 1; in omap_dma_prep_dma_interleaved()
1344 } else if (xt->dst_inc) { in omap_dma_prep_dma_interleaved()
1345 d->ccr |= CCR_DST_AMODE_POSTINC; in omap_dma_prep_dma_interleaved()
1346 sg->fi = 0; in omap_dma_prep_dma_interleaved()
1348 dev_err(chan->device->dev, in omap_dma_prep_dma_interleaved()
1355 d->cicr = CICR_DROP_IE | CICR_FRAME_IE; in omap_dma_prep_dma_interleaved()
1357 d->csdp = data_type; in omap_dma_prep_dma_interleaved()
1360 d->cicr |= CICR_TOUT_IE; in omap_dma_prep_dma_interleaved()
1361 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; in omap_dma_prep_dma_interleaved()
1363 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; in omap_dma_prep_dma_interleaved()
1364 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; in omap_dma_prep_dma_interleaved()
1365 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; in omap_dma_prep_dma_interleaved()
1368 return vchan_tx_prep(&c->vc, &d->vd, flags); in omap_dma_prep_dma_interleaved()
1375 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || in omap_dma_slave_config()
1376 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) in omap_dma_slave_config()
1377 return -EINVAL; in omap_dma_slave_config()
1379 if (cfg->src_maxburst > chan->device->max_burst || in omap_dma_slave_config()
1380 cfg->dst_maxburst > chan->device->max_burst) in omap_dma_slave_config()
1381 return -EINVAL; in omap_dma_slave_config()
1383 memcpy(&c->cfg, cfg, sizeof(c->cfg)); in omap_dma_slave_config()
1394 spin_lock_irqsave(&c->vc.lock, flags); in omap_dma_terminate_all()
1399 * c->desc is NULL and exit.) in omap_dma_terminate_all()
1401 if (c->desc) { in omap_dma_terminate_all()
1402 vchan_terminate_vdesc(&c->desc->vd); in omap_dma_terminate_all()
1403 c->desc = NULL; in omap_dma_terminate_all()
1405 if (!c->paused) in omap_dma_terminate_all()
1409 c->cyclic = false; in omap_dma_terminate_all()
1410 c->paused = false; in omap_dma_terminate_all()
1412 vchan_get_all_descriptors(&c->vc, &head); in omap_dma_terminate_all()
1413 spin_unlock_irqrestore(&c->vc.lock, flags); in omap_dma_terminate_all()
1414 vchan_dma_desc_free_list(&c->vc, &head); in omap_dma_terminate_all()
1423 vchan_synchronize(&c->vc); in omap_dma_synchronize()
1429 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_pause()
1431 int ret = -EINVAL; in omap_dma_pause()
1434 spin_lock_irqsave(&od->irq_lock, flags); in omap_dma_pause()
1436 if (!c->desc) in omap_dma_pause()
1439 if (c->cyclic) in omap_dma_pause()
1446 * an abort, unless it is hardware-source-synchronized …". in omap_dma_pause()
1447 * A source-synchronised channel is one where the fetching of data is in omap_dma_pause()
1448 * under control of the device. In other words, a device-to-memory in omap_dma_pause()
1449 * transfer. So, a destination-synchronised channel (which would be a in omap_dma_pause()
1450 * memory-to-device transfer) undergoes an abort if the CCR_ENABLE in omap_dma_pause()
1465 else if (c->desc->dir == DMA_DEV_TO_MEM) in omap_dma_pause()
1468 if (can_pause && !c->paused) { in omap_dma_pause()
1471 c->paused = true; in omap_dma_pause()
1474 spin_unlock_irqrestore(&od->irq_lock, flags); in omap_dma_pause()
1482 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_resume()
1484 int ret = -EINVAL; in omap_dma_resume()
1486 spin_lock_irqsave(&od->irq_lock, flags); in omap_dma_resume()
1488 if (c->paused && c->desc) { in omap_dma_resume()
1492 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl); in omap_dma_resume()
1494 omap_dma_start(c, c->desc); in omap_dma_resume()
1495 c->paused = false; in omap_dma_resume()
1498 spin_unlock_irqrestore(&od->irq_lock, flags); in omap_dma_resume()
1509 return -ENOMEM; in omap_dma_chan_init()
1511 c->reg_map = od->reg_map; in omap_dma_chan_init()
1512 c->vc.desc_free = omap_dma_desc_free; in omap_dma_chan_init()
1513 vchan_init(&c->vc, &od->ddev); in omap_dma_chan_init()
1520 while (!list_empty(&od->ddev.channels)) { in omap_dma_free()
1521 struct omap_chan *c = list_first_entry(&od->ddev.channels, in omap_dma_free()
1524 list_del(&c->vc.chan.device_node); in omap_dma_free()
1525 tasklet_kill(&c->vc.task); in omap_dma_free()
1534 int lch = -1; in omap_dma_busy()
1537 lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1); in omap_dma_busy()
1538 if (lch >= od->lch_count) in omap_dma_busy()
1540 c = od->lch_map[lch]; in omap_dma_busy()
1578 od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0); in omap_dma_context_save()
1579 od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1); in omap_dma_context_save()
1580 od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); in omap_dma_context_save()
1581 od->context.gcr = omap_dma_glbl_read(od, GCR); in omap_dma_context_save()
1588 omap_dma_glbl_write(od, GCR, od->context.gcr); in omap_dma_context_restore()
1589 omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig); in omap_dma_context_restore()
1590 omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0); in omap_dma_context_restore()
1591 omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1); in omap_dma_context_restore()
1594 if (od->plat->errata & DMA_ROMCODE_BUG) in omap_dma_context_restore()
1598 for (i = 0; i < od->lch_count; i++) in omap_dma_context_restore()
1632 if (!od->cfg->rw_priority) in omap_dma_init_gcr()
1664 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); in omap_dma_probe()
1666 return -ENOMEM; in omap_dma_probe()
1668 od->base = devm_platform_ioremap_resource(pdev, 0); in omap_dma_probe()
1669 if (IS_ERR(od->base)) in omap_dma_probe()
1670 return PTR_ERR(od->base); in omap_dma_probe()
1672 conf = of_device_get_match_data(&pdev->dev); in omap_dma_probe()
1674 od->cfg = conf; in omap_dma_probe()
1675 od->plat = dev_get_platdata(&pdev->dev); in omap_dma_probe()
1676 if (!od->plat) { in omap_dma_probe()
1677 dev_err(&pdev->dev, "omap_system_dma_plat_info is missing"); in omap_dma_probe()
1678 return -ENODEV; in omap_dma_probe()
1681 od->cfg = &default_cfg; in omap_dma_probe()
1683 od->plat = omap_get_plat_info(); in omap_dma_probe()
1684 if (!od->plat) in omap_dma_probe()
1685 return -EPROBE_DEFER; in omap_dma_probe()
1687 return -ENODEV; in omap_dma_probe()
1690 od->reg_map = od->plat->reg_map; in omap_dma_probe()
1692 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); in omap_dma_probe()
1693 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); in omap_dma_probe()
1694 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); in omap_dma_probe()
1695 dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask); in omap_dma_probe()
1696 od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; in omap_dma_probe()
1697 od->ddev.device_free_chan_resources = omap_dma_free_chan_resources; in omap_dma_probe()
1698 od->ddev.device_tx_status = omap_dma_tx_status; in omap_dma_probe()
1699 od->ddev.device_issue_pending = omap_dma_issue_pending; in omap_dma_probe()
1700 od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg; in omap_dma_probe()
1701 od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic; in omap_dma_probe()
1702 od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy; in omap_dma_probe()
1703 od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved; in omap_dma_probe()
1704 od->ddev.device_config = omap_dma_slave_config; in omap_dma_probe()
1705 od->ddev.device_pause = omap_dma_pause; in omap_dma_probe()
1706 od->ddev.device_resume = omap_dma_resume; in omap_dma_probe()
1707 od->ddev.device_terminate_all = omap_dma_terminate_all; in omap_dma_probe()
1708 od->ddev.device_synchronize = omap_dma_synchronize; in omap_dma_probe()
1709 od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS; in omap_dma_probe()
1710 od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS; in omap_dma_probe()
1711 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in omap_dma_probe()
1712 if (__dma_omap15xx(od->plat->dma_attr)) in omap_dma_probe()
1713 od->ddev.residue_granularity = in omap_dma_probe()
1716 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in omap_dma_probe()
1717 od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */ in omap_dma_probe()
1718 od->ddev.dev = &pdev->dev; in omap_dma_probe()
1719 INIT_LIST_HEAD(&od->ddev.channels); in omap_dma_probe()
1720 mutex_init(&od->lch_lock); in omap_dma_probe()
1721 spin_lock_init(&od->lock); in omap_dma_probe()
1722 spin_lock_init(&od->irq_lock); in omap_dma_probe()
1725 od->dma_requests = OMAP_SDMA_REQUESTS; in omap_dma_probe()
1726 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, in omap_dma_probe()
1727 "dma-requests", in omap_dma_probe()
1728 &od->dma_requests)) { in omap_dma_probe()
1729 dev_info(&pdev->dev, in omap_dma_probe()
1730 "Missing dma-requests property, using %u.\n", in omap_dma_probe()
1735 if (!pdev->dev.of_node) { in omap_dma_probe()
1736 od->lch_count = od->plat->dma_attr->lch_count; in omap_dma_probe()
1737 if (unlikely(!od->lch_count)) in omap_dma_probe()
1738 od->lch_count = OMAP_SDMA_CHANNELS; in omap_dma_probe()
1739 } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels", in omap_dma_probe()
1740 &od->lch_count)) { in omap_dma_probe()
1741 dev_info(&pdev->dev, in omap_dma_probe()
1742 "Missing dma-channels property, using %u.\n", in omap_dma_probe()
1744 od->lch_count = OMAP_SDMA_CHANNELS; in omap_dma_probe()
1748 if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node, in omap_dma_probe()
1749 "dma-channel-mask", in omap_dma_probe()
1753 bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count); in omap_dma_probe()
1755 if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED) in omap_dma_probe()
1756 bitmap_set(od->lch_bitmap, 0, 2); in omap_dma_probe()
1758 od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count, in omap_dma_probe()
1759 sizeof(*od->lch_map), in omap_dma_probe()
1761 if (!od->lch_map) in omap_dma_probe()
1762 return -ENOMEM; in omap_dma_probe()
1764 for (i = 0; i < od->dma_requests; i++) { in omap_dma_probe()
1774 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); in omap_dma_probe()
1775 od->legacy = true; in omap_dma_probe()
1778 od->irq_enable_mask = 0; in omap_dma_probe()
1781 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, in omap_dma_probe()
1782 IRQF_SHARED, "omap-dma-engine", od); in omap_dma_probe()
1790 od->ll123_supported = true; in omap_dma_probe()
1792 od->ddev.filter.map = od->plat->slave_map; in omap_dma_probe()
1793 od->ddev.filter.mapcnt = od->plat->slavecnt; in omap_dma_probe()
1794 od->ddev.filter.fn = omap_dma_filter_fn; in omap_dma_probe()
1796 if (od->ll123_supported) { in omap_dma_probe()
1797 od->desc_pool = dma_pool_create(dev_name(&pdev->dev), in omap_dma_probe()
1798 &pdev->dev, in omap_dma_probe()
1801 if (!od->desc_pool) { in omap_dma_probe()
1802 dev_err(&pdev->dev, in omap_dma_probe()
1804 od->ll123_supported = false; in omap_dma_probe()
1808 rc = dma_async_device_register(&od->ddev); in omap_dma_probe()
1810 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", in omap_dma_probe()
1818 if (pdev->dev.of_node) { in omap_dma_probe()
1819 omap_dma_info.dma_cap = od->ddev.cap_mask; in omap_dma_probe()
1821 /* Device-tree DMA controller registration */ in omap_dma_probe()
1822 rc = of_dma_controller_register(pdev->dev.of_node, in omap_dma_probe()
1825 pr_warn("OMAP-DMA: failed to register DMA controller\n"); in omap_dma_probe()
1826 dma_async_device_unregister(&od->ddev); in omap_dma_probe()
1833 if (od->cfg->needs_busy_check) { in omap_dma_probe()
1834 od->nb.notifier_call = omap_dma_busy_notifier; in omap_dma_probe()
1835 cpu_pm_register_notifier(&od->nb); in omap_dma_probe()
1836 } else if (od->cfg->may_lose_context) { in omap_dma_probe()
1837 od->nb.notifier_call = omap_dma_context_notifier; in omap_dma_probe()
1838 cpu_pm_register_notifier(&od->nb); in omap_dma_probe()
1841 dev_info(&pdev->dev, "OMAP DMA engine driver%s\n", in omap_dma_probe()
1842 od->ll123_supported ? " (LinkedList1/2/3 supported)" : ""); in omap_dma_probe()
1852 if (od->cfg->may_lose_context) in omap_dma_remove()
1853 cpu_pm_unregister_notifier(&od->nb); in omap_dma_remove()
1855 if (pdev->dev.of_node) in omap_dma_remove()
1856 of_dma_controller_free(pdev->dev.of_node); in omap_dma_remove()
1859 devm_free_irq(&pdev->dev, irq, od); in omap_dma_remove()
1861 dma_async_device_unregister(&od->ddev); in omap_dma_remove()
1868 if (od->ll123_supported) in omap_dma_remove()
1869 dma_pool_destroy(od->desc_pool); in omap_dma_remove()
1910 { .compatible = "ti,omap2420-sdma", .data = &omap2420_data, },
1911 { .compatible = "ti,omap2430-sdma", .data = &omap2430_data, },
1912 { .compatible = "ti,omap3430-sdma", .data = &omap3430_data, },
1913 { .compatible = "ti,omap3630-sdma", .data = &omap3630_data, },
1914 { .compatible = "ti,omap4430-sdma", .data = &omap4_data, },
1923 .name = "omap-dma-engine",
1930 if (chan->device->dev->driver == &omap_dma_driver.driver) { in omap_dma_filter_fn()
1931 struct omap_dmadev *od = to_omap_dma_dev(chan->device); in omap_dma_filter_fn()
1935 if (req <= od->dma_requests) { in omap_dma_filter_fn()
1936 c->dma_sig = req; in omap_dma_filter_fn()