56 	struct dw_desc		*desc = txd_to_dw_desc(tx);  in dwc_tx_submit()  local
70 	list_add_tail(&desc->desc_node, &dwc->queue);  in dwc_tx_submit()
73 		 __func__, desc->txd.cookie);  in dwc_tx_submit()
81 	struct dw_desc *desc;  in dwc_desc_get()  local
84 	desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);  in dwc_desc_get()
85 	if (!desc)  in dwc_desc_get()
89 	INIT_LIST_HEAD(&desc->tx_list);  in dwc_desc_get()
90 	dma_async_tx_descriptor_init(&desc->txd, &dwc->chan);  in dwc_desc_get()
91 	desc->txd.tx_submit = dwc_tx_submit;  in dwc_desc_get()
92 	desc->txd.flags = DMA_CTRL_ACK;  in dwc_desc_get()
93 	desc->txd.phys = phys;  in dwc_desc_get()
94 	return desc;  in dwc_desc_get()
97 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)  in dwc_desc_put()  argument
102 	if (unlikely(!desc))  in dwc_desc_put()
105 	list_for_each_entry_safe(child, _next, &desc->tx_list, desc_node) {  in dwc_desc_put()
111 	dma_pool_free(dw->desc_pool, desc, desc->txd.phys);  in dwc_desc_put()
150 				       struct dw_desc *desc)  in dwc_do_single_block()  argument
159 	ctllo = lli_read(desc, ctllo) | DWC_CTLL_INT_EN;  in dwc_do_single_block()
161 	channel_writel(dwc, SAR, lli_read(desc, sar));  in dwc_do_single_block()
162 	channel_writel(dwc, DAR, lli_read(desc, dar));  in dwc_do_single_block()
164 	channel_writel(dwc, CTL_HI, lli_read(desc, ctlhi));  in dwc_do_single_block()
219 	struct dw_desc *desc;  in dwc_dostart_first_queued()  local
225 	desc = dwc_first_active(dwc);  in dwc_dostart_first_queued()
226 	dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);  in dwc_dostart_first_queued()
227 	dwc_dostart(dwc, desc);  in dwc_dostart_first_queued()
233 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,  in dwc_descriptor_complete()  argument
236 	struct dma_async_tx_descriptor	*txd = &desc->txd;  in dwc_descriptor_complete()
251 	list_for_each_entry(child, &desc->tx_list, desc_node)  in dwc_descriptor_complete()
253 	async_tx_ack(&desc->txd);  in dwc_descriptor_complete()
254 	dwc_desc_put(dwc, desc);  in dwc_descriptor_complete()
262 	struct dw_desc *desc, *_desc;  in dwc_complete_all()  local
284 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in dwc_complete_all()
285 		dwc_descriptor_complete(dwc, desc, true);  in dwc_complete_all()
301 	struct dw_desc *desc, *_desc;  in dwc_scan_descriptors()  local
321 			desc = dwc_first_active(dwc);  in dwc_scan_descriptors()
323 			head = &desc->tx_list;  in dwc_scan_descriptors()
327 					desc->residue -= desc->len;  in dwc_scan_descriptors()
329 					desc->residue -= to_dw_desc(active->prev)->len;  in dwc_scan_descriptors()
363 	list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {  in dwc_scan_descriptors()
365 		desc->residue = desc->total_len;  in dwc_scan_descriptors()
368 		if (desc->txd.phys == DWC_LLP_LOC(llp)) {  in dwc_scan_descriptors()
374 		if (lli_read(desc, llp) == llp) {  in dwc_scan_descriptors()
376 			desc->residue -= dwc_get_sent(dwc);  in dwc_scan_descriptors()
381 		desc->residue -= desc->len;  in dwc_scan_descriptors()
382 		list_for_each_entry(child, &desc->tx_list, desc_node) {  in dwc_scan_descriptors()
385 				desc->residue -= dwc_get_sent(dwc);  in dwc_scan_descriptors()
389 			desc->residue -= child->len;  in dwc_scan_descriptors()
397 		dwc_descriptor_complete(dwc, desc, true);  in dwc_scan_descriptors()
411 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)  in dwc_dump_lli()  argument
414 		 lli_read(desc, sar),  in dwc_dump_lli()
415 		 lli_read(desc, dar),  in dwc_dump_lli()
416 		 lli_read(desc, llp),  in dwc_dump_lli()
417 		 lli_read(desc, ctlhi),  in dwc_dump_lli()
418 		 lli_read(desc, ctllo));  in dwc_dump_lli()
543 	struct dw_desc		*desc;  in dwc_prep_dma_memcpy()  local
577 		desc = dwc_desc_get(dwc);  in dwc_prep_dma_memcpy()
578 		if (!desc)  in dwc_prep_dma_memcpy()
583 		lli_write(desc, sar, src + offset);  in dwc_prep_dma_memcpy()
584 		lli_write(desc, dar, dest + offset);  in dwc_prep_dma_memcpy()
585 		lli_write(desc, ctllo, ctllo);  in dwc_prep_dma_memcpy()
586 		lli_write(desc, ctlhi, ctlhi);  in dwc_prep_dma_memcpy()
587 		desc->len = xfer_count;  in dwc_prep_dma_memcpy()
590 			first = desc;  in dwc_prep_dma_memcpy()
592 			lli_write(prev, llp, desc->txd.phys | lms);  in dwc_prep_dma_memcpy()
593 			list_add_tail(&desc->desc_node, &first->tx_list);  in dwc_prep_dma_memcpy()
595 		prev = desc;  in dwc_prep_dma_memcpy()
655 			struct dw_desc	*desc;  in dwc_prep_slave_sg()  local
665 			desc = dwc_desc_get(dwc);  in dwc_prep_slave_sg()
666 			if (!desc)  in dwc_prep_slave_sg()
671 			lli_write(desc, sar, mem);  in dwc_prep_slave_sg()
672 			lli_write(desc, dar, reg);  in dwc_prep_slave_sg()
673 			lli_write(desc, ctlhi, ctlhi);  in dwc_prep_slave_sg()
674 			lli_write(desc, ctllo, ctllo | DWC_CTLL_SRC_WIDTH(mem_width));  in dwc_prep_slave_sg()
675 			desc->len = dlen;  in dwc_prep_slave_sg()
678 				first = desc;  in dwc_prep_slave_sg()
680 				lli_write(prev, llp, desc->txd.phys | lms);  in dwc_prep_slave_sg()
681 				list_add_tail(&desc->desc_node, &first->tx_list);  in dwc_prep_slave_sg()
683 			prev = desc;  in dwc_prep_slave_sg()
705 			struct dw_desc	*desc;  in dwc_prep_slave_sg()  local
713 			desc = dwc_desc_get(dwc);  in dwc_prep_slave_sg()
714 			if (!desc)  in dwc_prep_slave_sg()
719 			lli_write(desc, sar, reg);  in dwc_prep_slave_sg()
720 			lli_write(desc, dar, mem);  in dwc_prep_slave_sg()
721 			lli_write(desc, ctlhi, ctlhi);  in dwc_prep_slave_sg()
723 			lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width));  in dwc_prep_slave_sg()
724 			desc->len = dlen;  in dwc_prep_slave_sg()
727 				first = desc;  in dwc_prep_slave_sg()
729 				lli_write(prev, llp, desc->txd.phys | lms);  in dwc_prep_slave_sg()
730 				list_add_tail(&desc->desc_node, &first->tx_list);  in dwc_prep_slave_sg()
732 			prev = desc;  in dwc_prep_slave_sg()
929 	struct dw_desc		*desc, *_desc;  in dwc_terminate_all()  local
950 	list_for_each_entry_safe(desc, _desc, &list, desc_node)  in dwc_terminate_all()
951 		dwc_descriptor_complete(dwc, desc, false);  in dwc_terminate_all()
958 	struct dw_desc *desc;  in dwc_find_desc()  local
960 	list_for_each_entry(desc, &dwc->active_list, desc_node)  in dwc_find_desc()
961 		if (desc->txd.cookie == c)  in dwc_find_desc()
962 			return desc;  in dwc_find_desc()
970 	struct dw_desc *desc;  in dwc_get_residue_and_status()  local
976 	desc = dwc_find_desc(dwc, cookie);  in dwc_get_residue_and_status()
977 	if (desc) {  in dwc_get_residue_and_status()
978 		if (desc == dwc_first_active(dwc)) {  in dwc_get_residue_and_status()
979 			residue = desc->residue;  in dwc_get_residue_and_status()
985 			residue = desc->total_len;  in dwc_get_residue_and_status()