90 	struct fsl_re_chan *re_chan;  in fsl_re_tx_submit()  local
95 	re_chan = container_of(tx->chan, struct fsl_re_chan, chan);  in fsl_re_tx_submit()
97 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
99 	list_add_tail(&desc->node, &re_chan->submit_q);  in fsl_re_tx_submit()
100 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_tx_submit()
108 	struct fsl_re_chan *re_chan;  in fsl_re_issue_pending()  local
113 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_issue_pending()
115 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
117 		in_be32(&re_chan->jrregs->inbring_slot_avail));  in fsl_re_issue_pending()
119 	list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) {  in fsl_re_issue_pending()
123 		list_move_tail(&desc->node, &re_chan->active_q);  in fsl_re_issue_pending()
125 		memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count],  in fsl_re_issue_pending()
128 		re_chan->inb_count = (re_chan->inb_count + 1) &  in fsl_re_issue_pending()
130 		out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1));  in fsl_re_issue_pending()
133 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_issue_pending()
143 static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan)  in fsl_re_cleanup_descs()  argument
148 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
149 	list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) {  in fsl_re_cleanup_descs()
151 			list_move_tail(&desc->node, &re_chan->free_q);  in fsl_re_cleanup_descs()
153 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_cleanup_descs()
155 	fsl_re_issue_pending(&re_chan->chan);  in fsl_re_cleanup_descs()
160 	struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask);  in fsl_re_dequeue()  local
167 	fsl_re_cleanup_descs(re_chan);  in fsl_re_dequeue()
169 	spin_lock_irqsave(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
170 	count =	FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full));  in fsl_re_dequeue()
173 		hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count];  in fsl_re_dequeue()
174 		list_for_each_entry_safe(desc, _desc, &re_chan->active_q,  in fsl_re_dequeue()
186 			list_move_tail(&desc->node, &re_chan->ack_q);  in fsl_re_dequeue()
188 			dev_err(re_chan->dev,  in fsl_re_dequeue()
192 		oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK;  in fsl_re_dequeue()
193 		re_chan->oub_count = oub_count;  in fsl_re_dequeue()
195 		out_be32(&re_chan->jrregs->oubring_job_rmvd,  in fsl_re_dequeue()
198 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);  in fsl_re_dequeue()
204 	struct fsl_re_chan *re_chan;  in fsl_re_isr()  local
207 	re_chan = dev_get_drvdata((struct device *)data);  in fsl_re_isr()
209 	irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status);  in fsl_re_isr()
219 		status = in_be32(&re_chan->jrregs->jr_status);  in fsl_re_isr()
220 		dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n",  in fsl_re_isr()
225 	out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR);  in fsl_re_isr()
227 	tasklet_schedule(&re_chan->irqtask);  in fsl_re_isr()
250 static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,  in fsl_re_init_desc()  argument
254 	desc->re_chan = re_chan;  in fsl_re_init_desc()
256 	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);  in fsl_re_init_desc()
271 static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan,  in fsl_re_chan_alloc_desc()  argument
279 	fsl_re_cleanup_descs(re_chan);  in fsl_re_chan_alloc_desc()
281 	spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
282 	if (!list_empty(&re_chan->free_q)) {  in fsl_re_chan_alloc_desc()
284 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_chan_alloc_desc()
290 	spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
297 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,  in fsl_re_chan_alloc_desc()
304 		desc = fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_chan_alloc_desc()
307 		spin_lock_irqsave(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
308 		re_chan->alloc_count++;  in fsl_re_chan_alloc_desc()
309 		spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag);  in fsl_re_chan_alloc_desc()
320 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_genq()  local
329 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_genq()
331 		dev_err(re_chan->dev, "genq tx length %zu, max length %d\n",  in fsl_re_prep_dma_genq()
336 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_genq()
407 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_pq()  local
416 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_pq()
418 		dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n",  in fsl_re_prep_dma_pq()
460 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_pq()
509 			dev_err(re_chan->dev, "PQ tx continuation error!\n");  in fsl_re_prep_dma_pq()
529 	struct fsl_re_chan *re_chan;  in fsl_re_prep_dma_memcpy()  local
536 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_prep_dma_memcpy()
539 		dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n",  in fsl_re_prep_dma_memcpy()
544 	desc = fsl_re_chan_alloc_desc(re_chan, flags);  in fsl_re_prep_dma_memcpy()
574 	struct fsl_re_chan *re_chan;  in fsl_re_alloc_chan_resources()  local
580 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_alloc_chan_resources()
586 		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,  in fsl_re_alloc_chan_resources()
594 		fsl_re_init_desc(re_chan, desc, cf, paddr);  in fsl_re_alloc_chan_resources()
596 		list_add_tail(&desc->node, &re_chan->free_q);  in fsl_re_alloc_chan_resources()
597 		re_chan->alloc_count++;  in fsl_re_alloc_chan_resources()
599 	return re_chan->alloc_count;  in fsl_re_alloc_chan_resources()
604 	struct fsl_re_chan *re_chan;  in fsl_re_free_chan_resources()  local
607 	re_chan = container_of(chan, struct fsl_re_chan, chan);  in fsl_re_free_chan_resources()
608 	while (re_chan->alloc_count--) {  in fsl_re_free_chan_resources()
609 		desc = list_first_entry(&re_chan->free_q,  in fsl_re_free_chan_resources()
614 		dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,  in fsl_re_free_chan_resources()
619 	if (!list_empty(&re_chan->free_q))  in fsl_re_free_chan_resources()
620 		dev_err(re_chan->dev, "chan resource cannot be cleaned!\n");  in fsl_re_free_chan_resources()