30 static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
331 tsi721_desc_fill_init(struct tsi721_tx_desc *desc,  in tsi721_desc_fill_init()  argument
342 				      (desc->rtype << 19) | desc->destid);  in tsi721_desc_fill_init()
343 	bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |  in tsi721_desc_fill_init()
345 	rio_addr = (desc->rio_addr >> 2) |  in tsi721_desc_fill_init()
346 				((u64)(desc->rio_addr_u & 0x3) << 62);  in tsi721_desc_fill_init()
373 			      struct tsi721_tx_desc *desc)  in tsi721_dma_tx_err()  argument
375 	struct dma_async_tx_descriptor *txd = &desc->txd;  in tsi721_dma_tx_err()
379 	list_move(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tx_err()
409 static int tsi721_submit_sg(struct tsi721_tx_desc *desc)  in tsi721_submit_sg()  argument
411 	struct dma_chan *dchan = desc->txd.chan;  in tsi721_submit_sg()
435 	rio_addr = desc->rio_addr;  in tsi721_submit_sg()
453 	for_each_sg(desc->sg, sg, desc->sg_len, i) {  in tsi721_submit_sg()
456 			bdma_chan->id, i, desc->sg_len,  in tsi721_submit_sg()
482 		desc->rio_addr = rio_addr;  in tsi721_submit_sg()
488 			desc->sg = sg;  in tsi721_submit_sg()
489 			desc->sg_len -= i;  in tsi721_submit_sg()
494 		err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);  in tsi721_submit_sg()
501 			  bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr);  in tsi721_submit_sg()
519 			desc->sg_len = 0;  in tsi721_submit_sg()
533 				struct tsi721_tx_desc *desc)  in tsi721_advance_work()  argument
546 	if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) {  in tsi721_advance_work()
547 		desc = list_first_entry(&bdma_chan->queue,  in tsi721_advance_work()
549 		list_del_init((&desc->desc_node));  in tsi721_advance_work()
550 		bdma_chan->active_tx = desc;  in tsi721_advance_work()
553 	if (desc) {  in tsi721_advance_work()
554 		err = tsi721_submit_sg(desc);  in tsi721_advance_work()
558 			tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_advance_work()
582 		struct tsi721_tx_desc *desc;  in tsi721_dma_tasklet()  local
584 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
588 			bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr);  in tsi721_dma_tasklet()
639 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
640 		desc->status = DMA_ERROR;  in tsi721_dma_tasklet()
641 		dma_cookie_complete(&desc->txd);  in tsi721_dma_tasklet()
642 		list_add(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tasklet()
656 		struct tsi721_tx_desc *desc;  in tsi721_dma_tasklet()  local
660 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
662 		if (desc->sg_len == 0) {  in tsi721_dma_tasklet()
666 			desc->status = DMA_COMPLETE;  in tsi721_dma_tasklet()
667 			dma_cookie_complete(&desc->txd);  in tsi721_dma_tasklet()
668 			if (desc->txd.flags & DMA_PREP_INTERRUPT) {  in tsi721_dma_tasklet()
669 				callback = desc->txd.callback;  in tsi721_dma_tasklet()
670 				param = desc->txd.callback_param;  in tsi721_dma_tasklet()
672 			list_add(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tasklet()
693 	struct tsi721_tx_desc *desc = to_tsi721_desc(txd);  in tsi721_tx_submit()  local
698 	if (!list_empty(&desc->desc_node)) {  in tsi721_tx_submit()
713 	desc->status = DMA_IN_PROGRESS;  in tsi721_tx_submit()
714 	list_add_tail(&desc->desc_node, &bdma_chan->queue);  in tsi721_tx_submit()
724 	struct tsi721_tx_desc *desc;  in tsi721_alloc_chan_resources()  local
740 	desc = kcalloc(dma_txqueue_sz, sizeof(struct tsi721_tx_desc),  in tsi721_alloc_chan_resources()
742 	if (!desc) {  in tsi721_alloc_chan_resources()
747 	bdma_chan->tx_desc = desc;  in tsi721_alloc_chan_resources()
750 		dma_async_tx_descriptor_init(&desc[i].txd, dchan);  in tsi721_alloc_chan_resources()
751 		desc[i].txd.tx_submit = tsi721_tx_submit;  in tsi721_alloc_chan_resources()
752 		desc[i].txd.flags = DMA_CTRL_ACK;  in tsi721_alloc_chan_resources()
753 		list_add(&desc[i].desc_node, &bdma_chan->free_list);  in tsi721_alloc_chan_resources()
830 	struct tsi721_tx_desc *desc;  in tsi721_prep_rio_sg()  local
869 		desc = list_first_entry(&bdma_chan->free_list,  in tsi721_prep_rio_sg()
871 		list_del_init(&desc->desc_node);  in tsi721_prep_rio_sg()
872 		desc->destid = rext->destid;  in tsi721_prep_rio_sg()
873 		desc->rio_addr = rext->rio_addr;  in tsi721_prep_rio_sg()
874 		desc->rio_addr_u = 0;  in tsi721_prep_rio_sg()
875 		desc->rtype = rtype;  in tsi721_prep_rio_sg()
876 		desc->sg_len	= sg_len;  in tsi721_prep_rio_sg()
877 		desc->sg	= sgl;  in tsi721_prep_rio_sg()
878 		txd		= &desc->txd;  in tsi721_prep_rio_sg()
896 	struct tsi721_tx_desc *desc, *_d;  in tsi721_terminate_all()  local
924 	list_for_each_entry_safe(desc, _d, &list, desc_node)  in tsi721_terminate_all()
925 		tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_terminate_all()