76 	struct ccp_dma_desc *desc, *dtmp;  in ccp_free_desc_resources()  local
78 	list_for_each_entry_safe(desc, dtmp, list, entry) {  in ccp_free_desc_resources()
79 		ccp_free_cmd_resources(ccp, &desc->active);  in ccp_free_desc_resources()
80 		ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_free_desc_resources()
82 		list_del(&desc->entry);  in ccp_free_desc_resources()
83 		kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_free_desc_resources()
108 	struct ccp_dma_desc *desc, *dtmp;  in ccp_cleanup_desc_resources()  local
110 	list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {  in ccp_cleanup_desc_resources()
111 		if (!async_tx_test_ack(&desc->tx_desc))  in ccp_cleanup_desc_resources()
114 		dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);  in ccp_cleanup_desc_resources()
116 		ccp_free_cmd_resources(ccp, &desc->active);  in ccp_cleanup_desc_resources()
117 		ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_cleanup_desc_resources()
119 		list_del(&desc->entry);  in ccp_cleanup_desc_resources()
120 		kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_cleanup_desc_resources()
139 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)  in ccp_issue_next_cmd()  argument
144 	cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);  in ccp_issue_next_cmd()
145 	list_move(&cmd->entry, &desc->active);  in ccp_issue_next_cmd()
147 	dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,  in ccp_issue_next_cmd()
148 		desc->tx_desc.cookie, cmd);  in ccp_issue_next_cmd()
154 	dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,  in ccp_issue_next_cmd()
155 		ret, desc->tx_desc.cookie, cmd);  in ccp_issue_next_cmd()
160 static void ccp_free_active_cmd(struct ccp_dma_desc *desc)  in ccp_free_active_cmd()  argument
164 	cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,  in ccp_free_active_cmd()
169 	dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",  in ccp_free_active_cmd()
170 		__func__, desc->tx_desc.cookie, cmd);  in ccp_free_active_cmd()
173 	kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);  in ccp_free_active_cmd()
177 						struct ccp_dma_desc *desc)  in __ccp_next_dma_desc()  argument
180 	if (desc)  in __ccp_next_dma_desc()
181 		list_move(&desc->entry, &chan->complete);  in __ccp_next_dma_desc()
184 	desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,  in __ccp_next_dma_desc()
187 	return desc;  in __ccp_next_dma_desc()
191 						   struct ccp_dma_desc *desc)  in ccp_handle_active_desc()  argument
198 		if (desc) {  in ccp_handle_active_desc()
200 			ccp_free_active_cmd(desc);  in ccp_handle_active_desc()
202 			if (!list_empty(&desc->pending)) {  in ccp_handle_active_desc()
204 				if (desc->status != DMA_ERROR)  in ccp_handle_active_desc()
205 					return desc;  in ccp_handle_active_desc()
208 				ccp_free_cmd_resources(desc->ccp,  in ccp_handle_active_desc()
209 						       &desc->pending);  in ccp_handle_active_desc()
212 			tx_desc = &desc->tx_desc;  in ccp_handle_active_desc()
219 		if (desc) {  in ccp_handle_active_desc()
220 			if (desc->status != DMA_ERROR)  in ccp_handle_active_desc()
221 				desc->status = DMA_COMPLETE;  in ccp_handle_active_desc()
223 			dev_dbg(desc->ccp->dev,  in ccp_handle_active_desc()
225 				desc->tx_desc.cookie, desc->status);  in ccp_handle_active_desc()
231 		desc = __ccp_next_dma_desc(chan, desc);  in ccp_handle_active_desc()
240 	} while (desc);  in ccp_handle_active_desc()
247 	struct ccp_dma_desc *desc;  in __ccp_pending_to_active()  local
252 	desc = list_empty(&chan->active)  in __ccp_pending_to_active()
258 	return desc;  in __ccp_pending_to_active()
263 	struct ccp_dma_desc *desc = data;  in ccp_cmd_callback()  local
270 	chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,  in ccp_cmd_callback()
274 		__func__, desc->tx_desc.cookie, err);  in ccp_cmd_callback()
277 		desc->status = DMA_ERROR;  in ccp_cmd_callback()
281 		desc = ccp_handle_active_desc(chan, desc);  in ccp_cmd_callback()
284 		if (!desc || (chan->status == DMA_PAUSED))  in ccp_cmd_callback()
287 		ret = ccp_issue_next_cmd(desc);  in ccp_cmd_callback()
291 		desc->status = DMA_ERROR;  in ccp_cmd_callback()
299 	struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,  in ccp_tx_submit()  local
310 	list_move_tail(&desc->entry, &chan->pending);  in ccp_tx_submit()
334 	struct ccp_dma_desc *desc;  in ccp_alloc_dma_desc()  local
336 	desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);  in ccp_alloc_dma_desc()
337 	if (!desc)  in ccp_alloc_dma_desc()
340 	dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);  in ccp_alloc_dma_desc()
341 	desc->tx_desc.flags = flags;  in ccp_alloc_dma_desc()
342 	desc->tx_desc.tx_submit = ccp_tx_submit;  in ccp_alloc_dma_desc()
343 	desc->ccp = chan->ccp;  in ccp_alloc_dma_desc()
344 	INIT_LIST_HEAD(&desc->entry);  in ccp_alloc_dma_desc()
345 	INIT_LIST_HEAD(&desc->pending);  in ccp_alloc_dma_desc()
346 	INIT_LIST_HEAD(&desc->active);  in ccp_alloc_dma_desc()
347 	desc->status = DMA_IN_PROGRESS;  in ccp_alloc_dma_desc()
349 	return desc;  in ccp_alloc_dma_desc()
362 	struct ccp_dma_desc *desc;  in ccp_create_desc()  local
378 	desc = ccp_alloc_dma_desc(chan, flags);  in ccp_create_desc()
379 	if (!desc)  in ccp_create_desc()
438 		ccp_cmd->data = desc;  in ccp_create_desc()
440 		list_add_tail(&cmd->entry, &desc->pending);  in ccp_create_desc()
456 	desc->len = total_len;  in ccp_create_desc()
458 	if (list_empty(&desc->pending))  in ccp_create_desc()
461 	dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);  in ccp_create_desc()
465 	list_add_tail(&desc->entry, &chan->created);  in ccp_create_desc()
469 	return desc;  in ccp_create_desc()
472 	ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_create_desc()
473 	kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_create_desc()
484 	struct ccp_dma_desc *desc;  in ccp_prep_dma_memcpy()  local
499 	desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);  in ccp_prep_dma_memcpy()
500 	if (!desc)  in ccp_prep_dma_memcpy()
503 	return &desc->tx_desc;  in ccp_prep_dma_memcpy()
511 	struct ccp_dma_desc *desc;  in ccp_prep_dma_interrupt()  local
513 	desc = ccp_alloc_dma_desc(chan, flags);  in ccp_prep_dma_interrupt()
514 	if (!desc)  in ccp_prep_dma_interrupt()
517 	return &desc->tx_desc;  in ccp_prep_dma_interrupt()
524 	struct ccp_dma_desc *desc;  in ccp_issue_pending()  local
531 	desc = __ccp_pending_to_active(chan);  in ccp_issue_pending()
536 	if (desc)  in ccp_issue_pending()
537 		ccp_cmd_callback(desc, 0);  in ccp_issue_pending()
546 	struct ccp_dma_desc *desc;  in ccp_tx_status()  local
560 		list_for_each_entry(desc, &chan->complete, entry) {  in ccp_tx_status()
561 			if (desc->tx_desc.cookie != cookie)  in ccp_tx_status()
564 			ret = desc->status;  in ccp_tx_status()
593 	struct ccp_dma_desc *desc;  in ccp_resume()  local
598 	desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,  in ccp_resume()
607 	if (desc)  in ccp_resume()
608 		ccp_cmd_callback(desc, 0);  in ccp_resume()