286 	struct at_desc		*desc;  member
472 static void set_lli_eol(struct at_desc *desc, unsigned int i)  in set_lli_eol()  argument
474 	u32 ctrlb = desc->sg[i].lli->ctrlb;  in set_lli_eol()
479 	desc->sg[i].lli->ctrlb = ctrlb;  in set_lli_eol()
480 	desc->sg[i].lli->dscr = 0;  in set_lli_eol()
538 static void atdma_lli_chain(struct at_desc *desc, unsigned int i)  in atdma_lli_chain()  argument
540 	struct atdma_sg *atdma_sg = &desc->sg[i];  in atdma_lli_chain()
543 		desc->sg[i - 1].lli->dscr = atdma_sg->lli_phys;  in atdma_lli_chain()
553 	struct at_desc *desc;  in atc_dostart()  local
556 		atchan->desc = NULL;  in atc_dostart()
563 	atchan->desc = desc = to_atdma_desc(&vd->tx);  in atc_dostart()
569 	channel_writel(atchan, DSCR, desc->sg[0].lli_phys);  in atc_dostart()
571 		       FIELD_PREP(ATC_SPIP_HOLE, desc->src_hole) |  in atc_dostart()
572 		       FIELD_PREP(ATC_SPIP_BOUNDARY, desc->boundary));  in atc_dostart()
574 		       FIELD_PREP(ATC_DPIP_HOLE, desc->dst_hole) |  in atc_dostart()
575 		       FIELD_PREP(ATC_DPIP_BOUNDARY, desc->boundary));  in atc_dostart()
587 	struct at_desc *desc = to_atdma_desc(&vd->tx);  in atdma_desc_free()  local
590 	for (i = 0; i < desc->sglen; i++) {  in atdma_desc_free()
591 		if (desc->sg[i].lli)  in atdma_desc_free()
592 			dma_pool_free(atdma->lli_pool, desc->sg[i].lli,  in atdma_desc_free()
593 				      desc->sg[i].lli_phys);  in atdma_desc_free()
597 	if (desc->memset_buffer) {  in atdma_desc_free()
598 		dma_pool_free(atdma->memset_pool, desc->memset_vaddr,  in atdma_desc_free()
599 			      desc->memset_paddr);  in atdma_desc_free()
600 		desc->memset_buffer = false;  in atdma_desc_free()
603 	kfree(desc);  in atdma_desc_free()
670 				struct at_desc *desc, u32 *residue)  in atc_get_llis_residue()  argument
675 	len = desc->total_len;  in atc_get_llis_residue()
707 	if (desc->sg[0].lli->dscr == dscr) {  in atc_get_llis_residue()
711 	len -= desc->sg[0].len;  in atc_get_llis_residue()
713 	for (i = 1; i < desc->sglen; i++) {  in atc_get_llis_residue()
714 		if (desc->sg[i].lli && desc->sg[i].lli->dscr == dscr)  in atc_get_llis_residue()
716 		len -= desc->sg[i].len;  in atc_get_llis_residue()
741 	struct at_desc *desc = NULL;  in atc_get_residue()  local
746 		desc = to_atdma_desc(&vd->tx);  in atc_get_residue()
747 	else if (atchan->desc && atchan->desc->vd.tx.cookie == cookie)  in atc_get_residue()
748 		desc = atchan->desc;  in atc_get_residue()
750 	if (!desc)  in atc_get_residue()
753 	if (desc->sg[0].lli->dscr)  in atc_get_residue()
755 		return atc_get_llis_residue(atchan, desc, residue);  in atc_get_residue()
758 	len = desc->total_len;  in atc_get_residue()
771 	struct at_desc *desc = atchan->desc;  in atc_handle_error()  local
785 		 desc->vd.tx.cookie);  in atc_handle_error()
786 	for (i = 0; i < desc->sglen; i++)  in atc_handle_error()
787 		atc_dump_lli(atchan, desc->sg[i].lli);  in atc_handle_error()
793 	struct at_desc *desc;  in atdma_handle_chan_done()  local
796 	desc = atchan->desc;  in atdma_handle_chan_done()
798 	if (desc) {  in atdma_handle_chan_done()
805 			vchan_cyclic_callback(&desc->vd);  in atdma_handle_chan_done()
807 			vchan_cookie_complete(&desc->vd);  in atdma_handle_chan_done()
808 			atchan->desc = NULL;  in atdma_handle_chan_done()
865 	struct at_desc		*desc;  in atc_prep_dma_interleaved()  local
922 	desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC);  in atc_prep_dma_interleaved()
923 	if (!desc)  in atc_prep_dma_interleaved()
925 	desc->sglen = 1;  in atc_prep_dma_interleaved()
927 	atdma_sg = desc->sg;  in atc_prep_dma_interleaved()
931 		kfree(desc);  in atc_prep_dma_interleaved()
941 	desc->boundary = first->size >> dwidth;  in atc_prep_dma_interleaved()
942 	desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1;  in atc_prep_dma_interleaved()
943 	desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1;  in atc_prep_dma_interleaved()
946 	desc->total_len = len;  in atc_prep_dma_interleaved()
948 	set_lli_eol(desc, 0);  in atc_prep_dma_interleaved()
949 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_dma_interleaved()
966 	struct at_desc		*desc = NULL;  in atc_prep_dma_memcpy()  local
985 	desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC);  in atc_prep_dma_memcpy()
986 	if (!desc)  in atc_prep_dma_memcpy()
988 	desc->sglen = sg_len;  in atc_prep_dma_memcpy()
1006 		struct atdma_sg *atdma_sg = &desc->sg[i];  in atc_prep_dma_memcpy()
1023 		desc->sg[i].len = xfer_count << src_width;  in atc_prep_dma_memcpy()
1025 		atdma_lli_chain(desc, i);  in atc_prep_dma_memcpy()
1028 	desc->total_len = len;  in atc_prep_dma_memcpy()
1031 	set_lli_eol(desc, i - 1);  in atc_prep_dma_memcpy()
1033 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_dma_memcpy()
1036 	atdma_desc_free(&desc->vd);  in atc_prep_dma_memcpy()
1089 	struct at_desc		*desc;  in atc_prep_dma_memset()  local
1124 	desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC);  in atc_prep_dma_memset()
1125 	if (!desc)  in atc_prep_dma_memset()
1127 	desc->sglen = 1;  in atc_prep_dma_memset()
1129 	ret = atdma_create_memset_lli(chan, desc->sg, paddr, dest, len);  in atc_prep_dma_memset()
1133 	desc->memset_paddr = paddr;  in atc_prep_dma_memset()
1134 	desc->memset_vaddr = vaddr;  in atc_prep_dma_memset()
1135 	desc->memset_buffer = true;  in atc_prep_dma_memset()
1137 	desc->total_len = len;  in atc_prep_dma_memset()
1140 	set_lli_eol(desc, 0);  in atc_prep_dma_memset()
1142 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_dma_memset()
1145 	kfree(desc);  in atc_prep_dma_memset()
1159 	struct at_desc		*desc;  in atc_prep_dma_memset_sg()  local
1184 	desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC);  in atc_prep_dma_memset_sg()
1185 	if (!desc)  in atc_prep_dma_memset_sg()
1187 	desc->sglen = sg_len;  in atc_prep_dma_memset_sg()
1202 		ret = atdma_create_memset_lli(chan, &desc->sg[i], paddr, dest,  in atc_prep_dma_memset_sg()
1207 		atdma_lli_chain(desc, i);  in atc_prep_dma_memset_sg()
1211 	desc->memset_paddr = paddr;  in atc_prep_dma_memset_sg()
1212 	desc->memset_vaddr = vaddr;  in atc_prep_dma_memset_sg()
1213 	desc->memset_buffer = true;  in atc_prep_dma_memset_sg()
1215 	desc->total_len = total_len;  in atc_prep_dma_memset_sg()
1218 	set_lli_eol(desc, i - 1);  in atc_prep_dma_memset_sg()
1220 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_dma_memset_sg()
1223 	atdma_desc_free(&desc->vd);  in atc_prep_dma_memset_sg()
1247 	struct at_desc		*desc;  in atc_prep_slave_sg()  local
1267 	desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC);  in atc_prep_slave_sg()
1268 	if (!desc)  in atc_prep_slave_sg()
1270 	desc->sglen = sg_len;  in atc_prep_slave_sg()
1288 			struct atdma_sg *atdma_sg = &desc->sg[i];  in atc_prep_slave_sg()
1321 			desc->sg[i].len = len;  in atc_prep_slave_sg()
1322 			atdma_lli_chain(desc, i);  in atc_prep_slave_sg()
1337 			struct atdma_sg *atdma_sg = &desc->sg[i];  in atc_prep_slave_sg()
1367 			desc->sg[i].len = len;  in atc_prep_slave_sg()
1370 			atdma_lli_chain(desc, i);  in atc_prep_slave_sg()
1378 	set_lli_eol(desc, i - 1);  in atc_prep_slave_sg()
1380 	desc->total_len = total_len;  in atc_prep_slave_sg()
1382 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_slave_sg()
1387 	atdma_desc_free(&desc->vd);  in atc_prep_slave_sg()
1416 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,  in atc_dma_cyclic_fill_desc()  argument
1424 	struct atdma_sg		*atdma_sg = &desc->sg[i];  in atc_dma_cyclic_fill_desc()
1468 	desc->sg[i].len = period_len;  in atc_dma_cyclic_fill_desc()
1490 	struct at_desc		*desc;  in atc_prep_dma_cyclic()  local
1524 	desc = kzalloc(struct_size(desc, sg, periods), GFP_ATOMIC);  in atc_prep_dma_cyclic()
1525 	if (!desc)  in atc_prep_dma_cyclic()
1527 	desc->sglen = periods;  in atc_prep_dma_cyclic()
1531 		if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,  in atc_prep_dma_cyclic()
1534 		atdma_lli_chain(desc, i);  in atc_prep_dma_cyclic()
1536 	desc->total_len = buf_len;  in atc_prep_dma_cyclic()
1538 	desc->sg[i - 1].lli->dscr = desc->sg[0].lli_phys;  in atc_prep_dma_cyclic()
1540 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);  in atc_prep_dma_cyclic()
1543 	atdma_desc_free(&desc->vd);  in atc_prep_dma_cyclic()
1635 	if (atchan->desc) {  in atc_terminate_all()
1636 		vchan_terminate_vdesc(&atchan->desc->vd);  in atc_terminate_all()
1637 		atchan->desc = NULL;  in atc_terminate_all()
1702 	if (vchan_issue_pending(&atchan->vc) && !atchan->desc) {  in atc_issue_pending()