219 static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)  in chan2dev()  argument
221 	return &edmac->chan.dev->device;  in chan2dev()
240 static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,  in ep93xx_dma_set_active()  argument
243 	BUG_ON(!list_empty(&edmac->active));  in ep93xx_dma_set_active()
245 	list_add_tail(&desc->node, &edmac->active);  in ep93xx_dma_set_active()
261 		list_move_tail(&d->node, &edmac->active);  in ep93xx_dma_set_active()
267 ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_get_active()  argument
269 	return list_first_entry_or_null(&edmac->active,  in ep93xx_dma_get_active()
285 static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_advance_active()  argument
289 	list_rotate_left(&edmac->active);  in ep93xx_dma_advance_active()
291 	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_advance_active()
294 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_advance_active()
309 static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)  in m2p_set_control()  argument
311 	writel(control, edmac->regs + M2P_CONTROL);  in m2p_set_control()
316 	readl(edmac->regs + M2P_CONTROL);  in m2p_set_control()
319 static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)  in m2p_hw_setup()  argument
321 	struct ep93xx_dma_data *data = edmac->chan.private;  in m2p_hw_setup()
324 	writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);  in m2p_hw_setup()
328 	m2p_set_control(edmac, control);  in m2p_hw_setup()
330 	edmac->buffer = 0;  in m2p_hw_setup()
335 static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)  in m2p_channel_state()  argument
337 	return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;  in m2p_channel_state()
340 static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac)  in m2p_hw_synchronize()  argument
345 	spin_lock_irqsave(&edmac->lock, flags);  in m2p_hw_synchronize()
346 	control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_synchronize()
348 	m2p_set_control(edmac, control);  in m2p_hw_synchronize()
349 	spin_unlock_irqrestore(&edmac->lock, flags);  in m2p_hw_synchronize()
351 	while (m2p_channel_state(edmac) >= M2P_STATE_ON)  in m2p_hw_synchronize()
355 static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)  in m2p_hw_shutdown()  argument
357 	m2p_set_control(edmac, 0);  in m2p_hw_shutdown()
359 	while (m2p_channel_state(edmac) != M2P_STATE_IDLE)  in m2p_hw_shutdown()
360 		dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n");  in m2p_hw_shutdown()
363 static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)  in m2p_fill_desc()  argument
368 	desc = ep93xx_dma_get_active(edmac);  in m2p_fill_desc()
370 		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");  in m2p_fill_desc()
374 	if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)  in m2p_fill_desc()
379 	if (edmac->buffer == 0) {  in m2p_fill_desc()
380 		writel(desc->size, edmac->regs + M2P_MAXCNT0);  in m2p_fill_desc()
381 		writel(bus_addr, edmac->regs + M2P_BASE0);  in m2p_fill_desc()
383 		writel(desc->size, edmac->regs + M2P_MAXCNT1);  in m2p_fill_desc()
384 		writel(bus_addr, edmac->regs + M2P_BASE1);  in m2p_fill_desc()
387 	edmac->buffer ^= 1;  in m2p_fill_desc()
390 static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)  in m2p_hw_submit()  argument
392 	u32 control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_submit()
394 	m2p_fill_desc(edmac);  in m2p_hw_submit()
397 	if (ep93xx_dma_advance_active(edmac)) {  in m2p_hw_submit()
398 		m2p_fill_desc(edmac);  in m2p_hw_submit()
402 	m2p_set_control(edmac, control);  in m2p_hw_submit()
405 static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)  in m2p_hw_interrupt()  argument
407 	u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);  in m2p_hw_interrupt()
411 		struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);  in m2p_hw_interrupt()
414 		writel(1, edmac->regs + M2P_INTERRUPT);  in m2p_hw_interrupt()
424 		dev_err(chan2dev(edmac),  in m2p_hw_interrupt()
442 	if (ep93xx_dma_advance_active(edmac)) {  in m2p_hw_interrupt()
443 		m2p_fill_desc(edmac);  in m2p_hw_interrupt()
448 	control = readl(edmac->regs + M2P_CONTROL);  in m2p_hw_interrupt()
450 	m2p_set_control(edmac, control);  in m2p_hw_interrupt()
459 static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)  in m2m_hw_setup()  argument
461 	const struct ep93xx_dma_data *data = edmac->chan.private;  in m2m_hw_setup()
466 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_setup()
516 	writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_setup()
520 static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)  in m2m_hw_shutdown()  argument
523 	writel(0, edmac->regs + M2M_CONTROL);  in m2m_hw_shutdown()
526 static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)  in m2m_fill_desc()  argument
530 	desc = ep93xx_dma_get_active(edmac);  in m2m_fill_desc()
532 		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");  in m2m_fill_desc()
536 	if (edmac->buffer == 0) {  in m2m_fill_desc()
537 		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);  in m2m_fill_desc()
538 		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);  in m2m_fill_desc()
539 		writel(desc->size, edmac->regs + M2M_BCR0);  in m2m_fill_desc()
541 		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);  in m2m_fill_desc()
542 		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);  in m2m_fill_desc()
543 		writel(desc->size, edmac->regs + M2M_BCR1);  in m2m_fill_desc()
546 	edmac->buffer ^= 1;  in m2m_fill_desc()
549 static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)  in m2m_hw_submit()  argument
551 	struct ep93xx_dma_data *data = edmac->chan.private;  in m2m_hw_submit()
552 	u32 control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
560 	control |= edmac->runtime_ctrl;  in m2m_hw_submit()
562 	m2m_fill_desc(edmac);  in m2m_hw_submit()
565 	if (ep93xx_dma_advance_active(edmac)) {  in m2m_hw_submit()
566 		m2m_fill_desc(edmac);  in m2m_hw_submit()
575 	writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
583 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_submit()
597 static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)  in m2m_hw_interrupt()  argument
599 	u32 status = readl(edmac->regs + M2M_STATUS);  in m2m_hw_interrupt()
608 	if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))  in m2m_hw_interrupt()
613 		writel(0, edmac->regs + M2M_INTERRUPT);  in m2m_hw_interrupt()
620 	desc = ep93xx_dma_get_active(edmac);  in m2m_hw_interrupt()
637 		if (ep93xx_dma_advance_active(edmac)) {  in m2m_hw_interrupt()
638 			m2m_fill_desc(edmac);  in m2m_hw_interrupt()
639 			if (done && !edmac->chan.private) {  in m2m_hw_interrupt()
641 				control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
643 				writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
659 		control = readl(edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
662 		writel(control, edmac->regs + M2M_CONTROL);  in m2m_hw_interrupt()
677 ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_desc_get()  argument
683 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_desc_get()
684 	list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {  in ep93xx_dma_desc_get()
701 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_desc_get()
705 static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,  in ep93xx_dma_desc_put()  argument
711 		spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_desc_put()
712 		list_splice_init(&desc->tx_list, &edmac->free_list);  in ep93xx_dma_desc_put()
713 		list_add(&desc->node, &edmac->free_list);  in ep93xx_dma_desc_put()
714 		spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_desc_put()
726 static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)  in ep93xx_dma_advance_work()  argument
731 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_advance_work()
732 	if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {  in ep93xx_dma_advance_work()
733 		spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_advance_work()
738 	new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);  in ep93xx_dma_advance_work()
741 	ep93xx_dma_set_active(edmac, new);  in ep93xx_dma_advance_work()
744 	edmac->edma->hw_submit(edmac);  in ep93xx_dma_advance_work()
745 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_advance_work()
750 	struct ep93xx_dma_chan *edmac = from_tasklet(edmac, t, tasklet);  in ep93xx_dma_tasklet()  local
756 	spin_lock_irq(&edmac->lock);  in ep93xx_dma_tasklet()
762 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_tasklet()
766 			if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_tasklet()
768 			list_splice_init(&edmac->active, &list);  in ep93xx_dma_tasklet()
772 	spin_unlock_irq(&edmac->lock);  in ep93xx_dma_tasklet()
775 	ep93xx_dma_advance_work(edmac);  in ep93xx_dma_tasklet()
780 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_tasklet()
788 	struct ep93xx_dma_chan *edmac = dev_id;  in ep93xx_dma_interrupt()  local
792 	spin_lock(&edmac->lock);  in ep93xx_dma_interrupt()
794 	desc = ep93xx_dma_get_active(edmac);  in ep93xx_dma_interrupt()
796 		dev_warn(chan2dev(edmac),  in ep93xx_dma_interrupt()
798 		spin_unlock(&edmac->lock);  in ep93xx_dma_interrupt()
802 	switch (edmac->edma->hw_interrupt(edmac)) {  in ep93xx_dma_interrupt()
805 		tasklet_schedule(&edmac->tasklet);  in ep93xx_dma_interrupt()
809 		if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))  in ep93xx_dma_interrupt()
810 			tasklet_schedule(&edmac->tasklet);  in ep93xx_dma_interrupt()
814 		dev_warn(chan2dev(edmac), "unknown interrupt!\n");  in ep93xx_dma_interrupt()
819 	spin_unlock(&edmac->lock);  in ep93xx_dma_interrupt()
833 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);  in ep93xx_dma_tx_submit()  local
838 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_tx_submit()
848 	if (list_empty(&edmac->active)) {  in ep93xx_dma_tx_submit()
849 		ep93xx_dma_set_active(edmac, desc);  in ep93xx_dma_tx_submit()
850 		edmac->edma->hw_submit(edmac);  in ep93xx_dma_tx_submit()
852 		list_add_tail(&desc->node, &edmac->queue);  in ep93xx_dma_tx_submit()
855 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_tx_submit()
869 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_alloc_chan_resources()  local
875 	if (!edmac->edma->m2m) {  in ep93xx_dma_alloc_chan_resources()
900 	ret = clk_prepare_enable(edmac->clk);  in ep93xx_dma_alloc_chan_resources()
904 	ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);  in ep93xx_dma_alloc_chan_resources()
908 	spin_lock_irq(&edmac->lock);  in ep93xx_dma_alloc_chan_resources()
909 	dma_cookie_init(&edmac->chan);  in ep93xx_dma_alloc_chan_resources()
910 	ret = edmac->edma->hw_setup(edmac);  in ep93xx_dma_alloc_chan_resources()
911 	spin_unlock_irq(&edmac->lock);  in ep93xx_dma_alloc_chan_resources()
921 			dev_warn(chan2dev(edmac), "not enough descriptors\n");  in ep93xx_dma_alloc_chan_resources()
931 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_alloc_chan_resources()
937 	free_irq(edmac->irq, edmac);  in ep93xx_dma_alloc_chan_resources()
939 	clk_disable_unprepare(edmac->clk);  in ep93xx_dma_alloc_chan_resources()
953 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_free_chan_resources()  local
958 	BUG_ON(!list_empty(&edmac->active));  in ep93xx_dma_free_chan_resources()
959 	BUG_ON(!list_empty(&edmac->queue));  in ep93xx_dma_free_chan_resources()
961 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_free_chan_resources()
962 	edmac->edma->hw_shutdown(edmac);  in ep93xx_dma_free_chan_resources()
963 	edmac->runtime_addr = 0;  in ep93xx_dma_free_chan_resources()
964 	edmac->runtime_ctrl = 0;  in ep93xx_dma_free_chan_resources()
965 	edmac->buffer = 0;  in ep93xx_dma_free_chan_resources()
966 	list_splice_init(&edmac->free_list, &list);  in ep93xx_dma_free_chan_resources()
967 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_free_chan_resources()
972 	clk_disable_unprepare(edmac->clk);  in ep93xx_dma_free_chan_resources()
973 	free_irq(edmac->irq, edmac);  in ep93xx_dma_free_chan_resources()
990 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_dma_memcpy()  local
996 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_dma_memcpy()
998 			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");  in ep93xx_dma_prep_dma_memcpy()
1019 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_dma_memcpy()
1039 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_slave_sg()  local
1044 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {  in ep93xx_dma_prep_slave_sg()
1045 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_slave_sg()
1050 	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {  in ep93xx_dma_prep_slave_sg()
1051 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_slave_sg()
1056 	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);  in ep93xx_dma_prep_slave_sg()
1063 			dev_warn(chan2dev(edmac), "too big transfer size %zu\n",  in ep93xx_dma_prep_slave_sg()
1068 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_slave_sg()
1070 			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");  in ep93xx_dma_prep_slave_sg()
1076 			desc->dst_addr = edmac->runtime_addr;  in ep93xx_dma_prep_slave_sg()
1078 			desc->src_addr = edmac->runtime_addr;  in ep93xx_dma_prep_slave_sg()
1095 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_slave_sg()
1121 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_prep_dma_cyclic()  local
1125 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {  in ep93xx_dma_prep_dma_cyclic()
1126 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_dma_cyclic()
1131 	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {  in ep93xx_dma_prep_dma_cyclic()
1132 		dev_warn(chan2dev(edmac),  in ep93xx_dma_prep_dma_cyclic()
1138 		dev_warn(chan2dev(edmac), "too big period length %zu\n",  in ep93xx_dma_prep_dma_cyclic()
1143 	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);  in ep93xx_dma_prep_dma_cyclic()
1148 		desc = ep93xx_dma_desc_get(edmac);  in ep93xx_dma_prep_dma_cyclic()
1150 			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");  in ep93xx_dma_prep_dma_cyclic()
1156 			desc->dst_addr = edmac->runtime_addr;  in ep93xx_dma_prep_dma_cyclic()
1158 			desc->src_addr = edmac->runtime_addr;  in ep93xx_dma_prep_dma_cyclic()
1175 	ep93xx_dma_desc_put(edmac, first);  in ep93xx_dma_prep_dma_cyclic()
1193 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_synchronize()  local
1195 	if (edmac->edma->hw_synchronize)  in ep93xx_dma_synchronize()
1196 		edmac->edma->hw_synchronize(edmac);  in ep93xx_dma_synchronize()
1208 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_terminate_all()  local
1213 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_terminate_all()
1215 	edmac->edma->hw_shutdown(edmac);  in ep93xx_dma_terminate_all()
1216 	clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);  in ep93xx_dma_terminate_all()
1217 	list_splice_init(&edmac->active, &list);  in ep93xx_dma_terminate_all()
1218 	list_splice_init(&edmac->queue, &list);  in ep93xx_dma_terminate_all()
1223 	edmac->edma->hw_setup(edmac);  in ep93xx_dma_terminate_all()
1224 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_terminate_all()
1227 		ep93xx_dma_desc_put(edmac, desc);  in ep93xx_dma_terminate_all()
1235 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_slave_config()  local
1237 	memcpy(&edmac->slave_config, config, sizeof(*config));  in ep93xx_dma_slave_config()
1246 	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);  in ep93xx_dma_slave_config_write()  local
1251 	if (!edmac->edma->m2m)  in ep93xx_dma_slave_config_write()
1283 	spin_lock_irqsave(&edmac->lock, flags);  in ep93xx_dma_slave_config_write()
1284 	edmac->runtime_addr = addr;  in ep93xx_dma_slave_config_write()
1285 	edmac->runtime_ctrl = ctrl;  in ep93xx_dma_slave_config_write()
1286 	spin_unlock_irqrestore(&edmac->lock, flags);  in ep93xx_dma_slave_config_write()
1336 		struct ep93xx_dma_chan *edmac = &edma->channels[i];  in ep93xx_dma_probe()  local
1338 		edmac->chan.device = dma_dev;  in ep93xx_dma_probe()
1339 		edmac->regs = cdata->base;  in ep93xx_dma_probe()
1340 		edmac->irq = cdata->irq;  in ep93xx_dma_probe()
1341 		edmac->edma = edma;  in ep93xx_dma_probe()
1343 		edmac->clk = clk_get(NULL, cdata->name);  in ep93xx_dma_probe()
1344 		if (IS_ERR(edmac->clk)) {  in ep93xx_dma_probe()
1350 		spin_lock_init(&edmac->lock);  in ep93xx_dma_probe()
1351 		INIT_LIST_HEAD(&edmac->active);  in ep93xx_dma_probe()
1352 		INIT_LIST_HEAD(&edmac->queue);  in ep93xx_dma_probe()
1353 		INIT_LIST_HEAD(&edmac->free_list);  in ep93xx_dma_probe()
1354 		tasklet_setup(&edmac->tasklet, ep93xx_dma_tasklet);  in ep93xx_dma_probe()
1356 		list_add_tail(&edmac->chan.device_node,  in ep93xx_dma_probe()
1398 			struct ep93xx_dma_chan *edmac = &edma->channels[i];  in ep93xx_dma_probe()  local
1399 			if (!IS_ERR_OR_NULL(edmac->clk))  in ep93xx_dma_probe()
1400 				clk_put(edmac->clk);  in ep93xx_dma_probe()