113 		struct ppc440spe_adma_desc_slot *desc,
117 		struct ppc440spe_adma_desc_slot *desc,
120 		struct ppc440spe_adma_desc_slot *desc,
231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_interrupt()  argument
238 		p = desc->hw_desc;  in ppc440spe_desc_init_interrupt()
239 		memset(desc->hw_desc, 0, sizeof(struct xor_cb));  in ppc440spe_desc_init_interrupt()
245 		memset(desc->hw_desc, 0, sizeof(struct dma_cdb));  in ppc440spe_desc_init_interrupt()
247 		set_bit(PPC440SPE_DESC_INT, &desc->flags);  in ppc440spe_desc_init_interrupt()
260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)  in ppc440spe_desc_init_null_xor()  argument
262 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));  in ppc440spe_desc_init_null_xor()
263 	desc->hw_next = NULL;  in ppc440spe_desc_init_null_xor()
264 	desc->src_cnt = 0;  in ppc440spe_desc_init_null_xor()
265 	desc->dst_cnt = 1;  in ppc440spe_desc_init_null_xor()
271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_xor()  argument
274 	struct xor_cb *hw_desc = desc->hw_desc;  in ppc440spe_desc_init_xor()
276 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));  in ppc440spe_desc_init_xor()
277 	desc->hw_next = NULL;  in ppc440spe_desc_init_xor()
278 	desc->src_cnt = src_cnt;  in ppc440spe_desc_init_xor()
279 	desc->dst_cnt = 1;  in ppc440spe_desc_init_xor()
291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_dma2pq()  argument
294 	struct xor_cb *hw_desc = desc->hw_desc;  in ppc440spe_desc_init_dma2pq()
296 	memset(desc->hw_desc, 0, sizeof(struct xor_cb));  in ppc440spe_desc_init_dma2pq()
297 	desc->hw_next = NULL;  in ppc440spe_desc_init_dma2pq()
298 	desc->src_cnt = src_cnt;  in ppc440spe_desc_init_dma2pq()
299 	desc->dst_cnt = dst_cnt;  in ppc440spe_desc_init_dma2pq()
300 	memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));  in ppc440spe_desc_init_dma2pq()
301 	desc->descs_per_op = 0;  in ppc440spe_desc_init_dma2pq()
317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_dma01pq()  argument
326 	set_bits(op, &desc->flags);  in ppc440spe_desc_init_dma01pq()
327 	desc->src_cnt = src_cnt;  in ppc440spe_desc_init_dma01pq()
328 	desc->dst_cnt = dst_cnt;  in ppc440spe_desc_init_dma01pq()
333 	dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?  in ppc440spe_desc_init_dma01pq()
336 	list_for_each_entry(iter, &desc->group_list, chain_node) {  in ppc440spe_desc_init_dma01pq()
341 				&desc->group_list))) {  in ppc440spe_desc_init_dma01pq()
361 	if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {  in ppc440spe_desc_init_dma01pq()
367 		iter = list_first_entry(&desc->group_list,  in ppc440spe_desc_init_dma01pq()
371 		if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {  in ppc440spe_desc_init_dma01pq()
379 		if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {  in ppc440spe_desc_init_dma01pq()
387 		list_for_each_entry_from(iter, &desc->group_list, chain_node) {  in ppc440spe_desc_init_dma01pq()
398 		iter = list_first_entry(&desc->group_list,  in ppc440spe_desc_init_dma01pq()
404 		if (desc->dst_cnt == DMA_DEST_MAX_NUM) {  in ppc440spe_desc_init_dma01pq()
413 		if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {  in ppc440spe_desc_init_dma01pq()
417 			list_for_each_entry_from(iter, &desc->group_list,  in ppc440spe_desc_init_dma01pq()
431 				struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_dma01pqzero_sum()  argument
445 	iter = list_first_entry(&desc->group_list,  in ppc440spe_desc_init_dma01pqzero_sum()
455 	list_for_each_entry_from(iter, &desc->group_list, chain_node) {  in ppc440spe_desc_init_dma01pqzero_sum()
476 					 &desc->group_list))) {  in ppc440spe_desc_init_dma01pqzero_sum()
494 	desc->src_cnt = src_cnt;  in ppc440spe_desc_init_dma01pqzero_sum()
495 	desc->dst_cnt = dst_cnt;  in ppc440spe_desc_init_dma01pqzero_sum()
501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_init_memcpy()  argument
504 	struct dma_cdb *hw_desc = desc->hw_desc;  in ppc440spe_desc_init_memcpy()
506 	memset(desc->hw_desc, 0, sizeof(struct dma_cdb));  in ppc440spe_desc_init_memcpy()
507 	desc->hw_next = NULL;  in ppc440spe_desc_init_memcpy()
508 	desc->src_cnt = 1;  in ppc440spe_desc_init_memcpy()
509 	desc->dst_cnt = 1;  in ppc440spe_desc_init_memcpy()
512 		set_bit(PPC440SPE_DESC_INT, &desc->flags);  in ppc440spe_desc_init_memcpy()
514 		clear_bit(PPC440SPE_DESC_INT, &desc->flags);  in ppc440spe_desc_init_memcpy()
522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_src_addr()  argument
542 		dma_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_src_addr()
547 		xor_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_src_addr()
557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_src_mult()  argument
567 		dma_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_src_mult()
601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_dest_addr()  argument
622 		dma_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_dest_addr()
631 		xor_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_dest_addr()
642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_byte_count()  argument
652 		dma_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_byte_count()
656 		xor_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_byte_count()
677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_dcheck()  argument
685 		dma_hw_desc = desc->hw_desc;  in ppc440spe_desc_set_dcheck()
769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_get_link()  argument
772 	if (!desc->hw_next)  in ppc440spe_desc_get_link()
775 	return desc->hw_next->phys;  in ppc440spe_desc_get_link()
782 	struct ppc440spe_adma_desc_slot *desc, int num_slots)  in ppc440spe_desc_is_aligned()  argument
784 	return (desc->idx & (num_slots - 1)) ? 0 : 1;  in ppc440spe_desc_is_aligned()
1075 		struct ppc440spe_adma_desc_slot *desc)  in ppc440spe_dma_put_desc()  argument
1080 	pcdb = desc->phys;  in ppc440spe_dma_put_desc()
1081 	if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))  in ppc440spe_dma_put_desc()
1084 	chan_last_sub[chan->device->id] = desc;  in ppc440spe_dma_put_desc()
1086 	ADMA_LL_DBG(print_cb(chan, desc->hw_desc));  in ppc440spe_dma_put_desc()
1462 		struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_run_tx_complete_actions()  argument
1466 	BUG_ON(desc->async_tx.cookie < 0);  in ppc440spe_adma_run_tx_complete_actions()
1467 	if (desc->async_tx.cookie > 0) {  in ppc440spe_adma_run_tx_complete_actions()
1468 		cookie = desc->async_tx.cookie;  in ppc440spe_adma_run_tx_complete_actions()
1469 		desc->async_tx.cookie = 0;  in ppc440spe_adma_run_tx_complete_actions()
1471 		dma_descriptor_unmap(&desc->async_tx);  in ppc440spe_adma_run_tx_complete_actions()
1475 		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);  in ppc440spe_adma_run_tx_complete_actions()
1479 	dma_run_dependencies(&desc->async_tx);  in ppc440spe_adma_run_tx_complete_actions()
1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_clean_slot()  argument
1493 	if (!async_tx_test_ack(&desc->async_tx))  in ppc440spe_adma_clean_slot()
1499 	if (list_is_last(&desc->chain_node, &chan->chain) ||  in ppc440spe_adma_clean_slot()
1500 	    desc->phys == ppc440spe_chan_get_current_descriptor(chan))  in ppc440spe_adma_clean_slot()
1513 		struct dma_cdb *cdb = desc->hw_desc;  in ppc440spe_adma_clean_slot()
1519 		desc->phys, desc->idx, desc->slots_per_op);  in ppc440spe_adma_clean_slot()
1521 	list_del(&desc->chain_node);  in ppc440spe_adma_clean_slot()
1522 	ppc440spe_adma_free_slots(desc, chan);  in ppc440spe_adma_clean_slot()
1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_rxor_set_region()  argument
1847 	struct xor_cb *xcb = desc->hw_desc;  in ppc440spe_rxor_set_region()
1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_rxor_set_src()  argument
1858 	struct xor_cb *xcb = desc->hw_desc;  in ppc440spe_rxor_set_src()
1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_rxor_set_mult()  argument
1870 	struct xor_cb *xcb = desc->hw_desc;  in ppc440spe_rxor_set_mult()
2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2067 		struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_init_dma2rxor_slot()  argument
2074 		ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,  in ppc440spe_adma_init_dma2rxor_slot()
2075 						 desc->src_cnt, (u32)src[i]);  in ppc440spe_adma_init_dma2rxor_slot()
3084 			struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_desc_set_xor_src_cnt()  argument
3087 	struct xor_cb *hw_desc = desc->hw_desc;  in ppc440spe_desc_set_xor_src_cnt()
3222 		struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_dma2rxor_inc_addr()  argument
3227 		ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);  in ppc440spe_adma_dma2rxor_inc_addr()
3229 		ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);  in ppc440spe_adma_dma2rxor_inc_addr()
3244 	struct ppc440spe_adma_desc_slot *desc = hdesc;  in ppc440spe_adma_dma2rxor_prep_src()  local
3248 		desc = list_entry(hdesc->chain_node.next,  in ppc440spe_adma_dma2rxor_prep_src()
3260 				ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3264 					desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3270 			set_bit(cursor->addr_count, &desc->reverse_flags[0]);  in ppc440spe_adma_dma2rxor_prep_src()
3272 				ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3276 					desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3286 				desc->reverse_flags)  in ppc440spe_adma_dma2rxor_prep_src()
3293 			ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3297 				desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3301 			ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3306 					desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3311 			ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3316 					desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3321 			ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3326 					desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3332 			ppc440spe_rxor_set_region(desc,  in ppc440spe_adma_dma2rxor_prep_src()
3336 				desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3345 				desc, cursor, index, src_cnt);  in ppc440spe_adma_dma2rxor_prep_src()
3358 		struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_dma2rxor_set_src()  argument
3361 	struct xor_cb *xcb = desc->hw_desc;  in ppc440spe_adma_dma2rxor_set_src()
3369 			desc = list_entry(desc->chain_node.next,  in ppc440spe_adma_dma2rxor_set_src()
3371 			xcb = desc->hw_desc;  in ppc440spe_adma_dma2rxor_set_src()
3383 	if (test_bit(k-1, desc->reverse_flags)) {  in ppc440spe_adma_dma2rxor_set_src()
3386 			ppc440spe_rxor_set_src(desc, k - 1, addr);  in ppc440spe_adma_dma2rxor_set_src()
3390 			ppc440spe_rxor_set_src(desc, k - 1, addr);  in ppc440spe_adma_dma2rxor_set_src()
3399 		struct ppc440spe_adma_desc_slot *desc,  in ppc440spe_adma_dma2rxor_set_mult()  argument
3402 	struct xor_cb *xcb = desc->hw_desc;  in ppc440spe_adma_dma2rxor_set_mult()
3410 			desc = list_entry(desc->chain_node.next,  in ppc440spe_adma_dma2rxor_set_mult()
3413 			xcb = desc->hw_desc;  in ppc440spe_adma_dma2rxor_set_mult()
3424 	if (test_bit(k-1, desc->reverse_flags)) {  in ppc440spe_adma_dma2rxor_set_mult()
3426 		ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);  in ppc440spe_adma_dma2rxor_set_mult()
3429 		ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);  in ppc440spe_adma_dma2rxor_set_mult()