137 	struct nbpf_desc *desc;  member
169 	struct nbpf_desc desc[NBPF_DESCS_PER_PAGE];  member
387 static int nbpf_start(struct nbpf_desc *desc)  in nbpf_start()  argument
389 	struct nbpf_channel *chan = desc->chan;  in nbpf_start()
390 	struct nbpf_link_desc *ldesc = list_first_entry(&desc->sg, struct nbpf_link_desc, node);  in nbpf_start()
510 	struct nbpf_desc *desc = ldesc->desc;  in nbpf_prep_one()  local
511 	struct nbpf_channel *chan = desc->chan;  in nbpf_prep_one()
615 		struct nbpf_desc *desc = list_first_entry(&chan->active,  in nbpf_issue_pending()  local
617 		if (!nbpf_start(desc))  in nbpf_issue_pending()
618 			chan->running = desc;  in nbpf_issue_pending()
643 			struct nbpf_desc *desc;  in nbpf_tx_status()  local
646 			list_for_each_entry(desc, &chan->active, node)  in nbpf_tx_status()
647 				if (desc->async_tx.cookie == cookie) {  in nbpf_tx_status()
653 				list_for_each_entry(desc, &chan->queued, node)  in nbpf_tx_status()
654 					if (desc->async_tx.cookie == cookie) {  in nbpf_tx_status()
660 			state->residue = found ? desc->length : 0;  in nbpf_tx_status()
674 	struct nbpf_desc *desc = container_of(tx, struct nbpf_desc, async_tx);  in nbpf_tx_submit()  local
675 	struct nbpf_channel *chan = desc->chan;  in nbpf_tx_submit()
681 	list_add_tail(&desc->node, &chan->queued);  in nbpf_tx_submit()
695 	struct nbpf_desc *desc;  in nbpf_desc_page_alloc()  local
719 	for (i = 0, desc = dpage->desc;  in nbpf_desc_page_alloc()
720 	     i < ARRAY_SIZE(dpage->desc);  in nbpf_desc_page_alloc()
721 	     i++, desc++) {  in nbpf_desc_page_alloc()
722 		dma_async_tx_descriptor_init(&desc->async_tx, dchan);  in nbpf_desc_page_alloc()
723 		desc->async_tx.tx_submit = nbpf_tx_submit;  in nbpf_desc_page_alloc()
724 		desc->chan = chan;  in nbpf_desc_page_alloc()
725 		INIT_LIST_HEAD(&desc->sg);  in nbpf_desc_page_alloc()
726 		list_add_tail(&desc->node, &head);  in nbpf_desc_page_alloc()
739 	return ARRAY_SIZE(dpage->desc);  in nbpf_desc_page_alloc()
742 static void nbpf_desc_put(struct nbpf_desc *desc)  in nbpf_desc_put()  argument
744 	struct nbpf_channel *chan = desc->chan;  in nbpf_desc_put()
749 	list_for_each_entry_safe(ldesc, tmp, &desc->sg, node)  in nbpf_desc_put()
752 	list_add(&desc->node, &chan->free);  in nbpf_desc_put()
758 	struct nbpf_desc *desc, *tmp;  in nbpf_scan_acked()  local
763 	list_for_each_entry_safe(desc, tmp, &chan->done, node)  in nbpf_scan_acked()
764 		if (async_tx_test_ack(&desc->async_tx) && desc->user_wait) {  in nbpf_scan_acked()
765 			list_move(&desc->node, &head);  in nbpf_scan_acked()
766 			desc->user_wait = false;  in nbpf_scan_acked()
770 	list_for_each_entry_safe(desc, tmp, &head, node) {  in nbpf_scan_acked()
771 		list_del(&desc->node);  in nbpf_scan_acked()
772 		nbpf_desc_put(desc);  in nbpf_scan_acked()
784 	struct nbpf_desc *desc = NULL;  in nbpf_desc_get()  local
803 		desc = list_first_entry(&chan->free, struct nbpf_desc, node);  in nbpf_desc_get()
804 		list_del(&desc->node);  in nbpf_desc_get()
812 					nbpf_desc_put(desc);  in nbpf_desc_get()
821 			ldesc->desc = desc;  in nbpf_desc_get()
826 			list_move_tail(&ldesc->node, &desc->sg);  in nbpf_desc_get()
830 	} while (!desc);  in nbpf_desc_get()
836 	return desc;  in nbpf_desc_get()
841 	struct nbpf_desc *desc, *tmp;  in nbpf_chan_idle()  local
855 	list_for_each_entry_safe(desc, tmp, &head, node) {  in nbpf_chan_idle()
857 			__func__, desc, desc->async_tx.cookie);  in nbpf_chan_idle()
858 		list_del(&desc->node);  in nbpf_chan_idle()
859 		nbpf_desc_put(desc);  in nbpf_chan_idle()
926 	struct nbpf_desc *desc;  in nbpf_prep_sg()  local
951 	desc = nbpf_desc_get(chan, len);  in nbpf_prep_sg()
952 	if (!desc)  in nbpf_prep_sg()
955 	desc->async_tx.flags = flags;  in nbpf_prep_sg()
956 	desc->async_tx.cookie = -EBUSY;  in nbpf_prep_sg()
957 	desc->user_wait = false;  in nbpf_prep_sg()
963 	list_for_each_entry(ldesc, &desc->sg, node) {  in nbpf_prep_sg()
970 			nbpf_desc_put(desc);  in nbpf_prep_sg()
982 	desc->length = data_len;  in nbpf_prep_sg()
985 	return &desc->async_tx;  in nbpf_prep_sg()
1118 	struct nbpf_desc *desc, *tmp;  in nbpf_chan_tasklet()  local
1126 		list_for_each_entry_safe(desc, tmp, &chan->done, node) {  in nbpf_chan_tasklet()
1127 			if (!desc->user_wait) {  in nbpf_chan_tasklet()
1131 			} else if (async_tx_test_ack(&desc->async_tx)) {  in nbpf_chan_tasklet()
1136 				list_del(&desc->node);  in nbpf_chan_tasklet()
1138 				nbpf_desc_put(desc);  in nbpf_chan_tasklet()
1153 		dma_cookie_complete(&desc->async_tx);  in nbpf_chan_tasklet()
1159 		if (async_tx_test_ack(&desc->async_tx)) {  in nbpf_chan_tasklet()
1160 			list_del(&desc->node);  in nbpf_chan_tasklet()
1163 			desc->user_wait = true;  in nbpf_chan_tasklet()
1167 		dmaengine_desc_get_callback(&desc->async_tx, &cb);  in nbpf_chan_tasklet()
1175 			nbpf_desc_put(desc);  in nbpf_chan_tasklet()
1183 	struct nbpf_desc *desc;  in nbpf_chan_irq()  local
1195 	desc = chan->running;  in nbpf_chan_irq()
1196 	if (WARN_ON(!desc)) {  in nbpf_chan_irq()
1204 	list_move_tail(&desc->node, &chan->done);  in nbpf_chan_irq()
1208 		desc = list_first_entry(&chan->active,  in nbpf_chan_irq()
1210 		if (!nbpf_start(desc))  in nbpf_chan_irq()
1211 			chan->running = desc;  in nbpf_chan_irq()