222 static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe,  in ath12k_ce_rx_buf_enqueue_pipe()  argument
225 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_buf_enqueue_pipe()
226 	struct ath12k_ce_ring *ring = pipe->dest_ring;  in ath12k_ce_rx_buf_enqueue_pipe()
260 	pipe->rx_buf_needed--;  in ath12k_ce_rx_buf_enqueue_pipe()
271 static int ath12k_ce_rx_post_pipe(struct ath12k_ce_pipe *pipe)  in ath12k_ce_rx_post_pipe()  argument
273 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_post_pipe()
278 	if (!(pipe->dest_ring || pipe->status_ring))  in ath12k_ce_rx_post_pipe()
282 	while (pipe->rx_buf_needed) {  in ath12k_ce_rx_post_pipe()
283 		skb = dev_alloc_skb(pipe->buf_sz);  in ath12k_ce_rx_post_pipe()
303 		ret = ath12k_ce_rx_buf_enqueue_pipe(pipe, skb, paddr);  in ath12k_ce_rx_post_pipe()
319 static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe,  in ath12k_ce_completed_recv_next()  argument
322 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_recv_next()
331 	sw_index = pipe->dest_ring->sw_index;  in ath12k_ce_completed_recv_next()
332 	nentries_mask = pipe->dest_ring->nentries_mask;  in ath12k_ce_completed_recv_next()
334 	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];  in ath12k_ce_completed_recv_next()
351 	*skb = pipe->dest_ring->skb[sw_index];  in ath12k_ce_completed_recv_next()
352 	pipe->dest_ring->skb[sw_index] = NULL;  in ath12k_ce_completed_recv_next()
355 	pipe->dest_ring->sw_index = sw_index;  in ath12k_ce_completed_recv_next()
357 	pipe->rx_buf_needed++;  in ath12k_ce_completed_recv_next()
368 static void ath12k_ce_recv_process_cb(struct ath12k_ce_pipe *pipe)  in ath12k_ce_recv_process_cb()  argument
370 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_recv_process_cb()
377 	while (ath12k_ce_completed_recv_next(pipe, &skb, &nbytes) == 0) {  in ath12k_ce_recv_process_cb()
395 			   pipe->pipe_num, skb->len);  in ath12k_ce_recv_process_cb()
396 		pipe->recv_cb(ab, skb);  in ath12k_ce_recv_process_cb()
399 	ret = ath12k_ce_rx_post_pipe(pipe);  in ath12k_ce_recv_process_cb()
402 			    pipe->pipe_num, ret);  in ath12k_ce_recv_process_cb()
408 static struct sk_buff *ath12k_ce_completed_send_next(struct ath12k_ce_pipe *pipe)  in ath12k_ce_completed_send_next()  argument
410 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_send_next()
419 	sw_index = pipe->src_ring->sw_index;  in ath12k_ce_completed_send_next()
420 	nentries_mask = pipe->src_ring->nentries_mask;  in ath12k_ce_completed_send_next()
422 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_completed_send_next()
434 	skb = pipe->src_ring->skb[sw_index];  in ath12k_ce_completed_send_next()
436 	pipe->src_ring->skb[sw_index] = NULL;  in ath12k_ce_completed_send_next()
439 	pipe->src_ring->sw_index = sw_index;  in ath12k_ce_completed_send_next()
449 static void ath12k_ce_send_done_cb(struct ath12k_ce_pipe *pipe)  in ath12k_ce_send_done_cb()  argument
451 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_send_done_cb()
454 	while (!IS_ERR(skb = ath12k_ce_completed_send_next(pipe))) {  in ath12k_ce_send_done_cb()
581 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_alloc_pipe()  local
587 	pipe->attr_flags = attr->flags;  in ath12k_ce_alloc_pipe()
590 		pipe->send_cb = ath12k_ce_send_done_cb;  in ath12k_ce_alloc_pipe()
596 		pipe->src_ring = ring;  in ath12k_ce_alloc_pipe()
600 		pipe->recv_cb = attr->recv_cb;  in ath12k_ce_alloc_pipe()
606 		pipe->dest_ring = ring;  in ath12k_ce_alloc_pipe()
612 		pipe->status_ring = ring;  in ath12k_ce_alloc_pipe()
620 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_per_engine_service()  local
622 	if (pipe->send_cb)  in ath12k_ce_per_engine_service()
623 		pipe->send_cb(pipe);  in ath12k_ce_per_engine_service()
625 	if (pipe->recv_cb)  in ath12k_ce_per_engine_service()
626 		ath12k_ce_recv_process_cb(pipe);  in ath12k_ce_per_engine_service()
631 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_poll_send_completed()  local
633 	if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && pipe->send_cb)  in ath12k_ce_poll_send_completed()
634 		pipe->send_cb(pipe);  in ath12k_ce_poll_send_completed()
640 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_send()  local
653 	if (pipe->attr_flags & CE_ATTR_DIS_INTR) {  in ath12k_ce_send()
655 		write_index = pipe->src_ring->write_index;  in ath12k_ce_send()
657 		sw_index = pipe->src_ring->sw_index;  in ath12k_ce_send()
662 			num_used = pipe->src_ring->nentries - sw_index +  in ath12k_ce_send()
668 			ath12k_ce_poll_send_completed(ab, pipe->pipe_num);  in ath12k_ce_send()
676 	write_index = pipe->src_ring->write_index;  in ath12k_ce_send()
677 	nentries_mask = pipe->src_ring->nentries_mask;  in ath12k_ce_send()
679 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_send()
698 	if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA)  in ath12k_ce_send()
704 	pipe->src_ring->skb[write_index] = skb;  in ath12k_ce_send()
705 	pipe->src_ring->write_index = CE_RING_IDX_INCR(nentries_mask,  in ath12k_ce_send()
718 static void ath12k_ce_rx_pipe_cleanup(struct ath12k_ce_pipe *pipe)  in ath12k_ce_rx_pipe_cleanup()  argument
720 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_pipe_cleanup()
721 	struct ath12k_ce_ring *ring = pipe->dest_ring;  in ath12k_ce_rx_pipe_cleanup()
725 	if (!(ring && pipe->buf_sz))  in ath12k_ce_rx_pipe_cleanup()
742 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_cleanup_pipes()  local
746 		pipe = &ab->ce.ce_pipe[pipe_num];  in ath12k_ce_cleanup_pipes()
747 		ath12k_ce_rx_pipe_cleanup(pipe);  in ath12k_ce_cleanup_pipes()
758 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_rx_post_buf()  local
763 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_rx_post_buf()
764 		ret = ath12k_ce_rx_post_pipe(pipe);  in ath12k_ce_rx_post_buf()
826 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_init_pipes()  local
834 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_init_pipes()
836 		if (pipe->src_ring) {  in ath12k_ce_init_pipes()
837 			ret = ath12k_ce_init_ring(ab, pipe->src_ring, i,  in ath12k_ce_init_pipes()
846 			pipe->src_ring->write_index = 0;  in ath12k_ce_init_pipes()
847 			pipe->src_ring->sw_index = 0;  in ath12k_ce_init_pipes()
850 		if (pipe->dest_ring) {  in ath12k_ce_init_pipes()
851 			ret = ath12k_ce_init_ring(ab, pipe->dest_ring, i,  in ath12k_ce_init_pipes()
860 			pipe->rx_buf_needed = pipe->dest_ring->nentries ?  in ath12k_ce_init_pipes()
861 					      pipe->dest_ring->nentries - 2 : 0;  in ath12k_ce_init_pipes()
863 			pipe->dest_ring->write_index = 0;  in ath12k_ce_init_pipes()
864 			pipe->dest_ring->sw_index = 0;  in ath12k_ce_init_pipes()
867 		if (pipe->status_ring) {  in ath12k_ce_init_pipes()
868 			ret = ath12k_ce_init_ring(ab, pipe->status_ring, i,  in ath12k_ce_init_pipes()
877 			pipe->status_ring->write_index = 0;  in ath12k_ce_init_pipes()
878 			pipe->status_ring->sw_index = 0;  in ath12k_ce_init_pipes()
887 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_free_pipes()  local
892 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_free_pipes()
894 		if (pipe->src_ring) {  in ath12k_ce_free_pipes()
897 					  pipe->src_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
899 					  pipe->src_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
900 					  pipe->src_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
901 			kfree(pipe->src_ring);  in ath12k_ce_free_pipes()
902 			pipe->src_ring = NULL;  in ath12k_ce_free_pipes()
905 		if (pipe->dest_ring) {  in ath12k_ce_free_pipes()
908 					  pipe->dest_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
910 					  pipe->dest_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
911 					  pipe->dest_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
912 			kfree(pipe->dest_ring);  in ath12k_ce_free_pipes()
913 			pipe->dest_ring = NULL;  in ath12k_ce_free_pipes()
916 		if (pipe->status_ring) {  in ath12k_ce_free_pipes()
920 					  pipe->status_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
922 					  pipe->status_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
923 					  pipe->status_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
924 			kfree(pipe->status_ring);  in ath12k_ce_free_pipes()
925 			pipe->status_ring = NULL;  in ath12k_ce_free_pipes()
932 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_alloc_pipes()  local
941 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_alloc_pipes()
942 		pipe->pipe_num = i;  in ath12k_ce_alloc_pipes()
943 		pipe->ab = ab;  in ath12k_ce_alloc_pipes()
944 		pipe->buf_sz = attr->src_sz_max;  in ath12k_ce_alloc_pipes()