189 	 * for split ring, it just contains last used index
200 		/* Available for split ring */
201 		struct vring_virtqueue_split split;  member
440  * Split ring specific functions - *_split().
463 	struct vring_desc_extra *extra = vq->split.desc_extra;  in vring_unmap_one_split()
524 	struct vring_desc_extra *extra = vring->split.desc_extra;  in virtqueue_add_desc_split()
580 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);  in virtqueue_add_split()
591 		desc = vq->split.vring.desc;  in virtqueue_add_split()
647 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=  in virtqueue_add_split()
662 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
674 		vq->free_head = vq->split.desc_extra[head].next;  in virtqueue_add_split()
679 	vq->split.desc_state[head].data = data;  in virtqueue_add_split()
681 		vq->split.desc_state[head].indir_desc = desc;  in virtqueue_add_split()
683 		vq->split.desc_state[head].indir_desc = ctx;  in virtqueue_add_split()
687 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);  in virtqueue_add_split()
688 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
693 	vq->split.avail_idx_shadow++;  in virtqueue_add_split()
694 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
695 						vq->split.avail_idx_shadow);  in virtqueue_add_split()
745 	old = vq->split.avail_idx_shadow - vq->num_added;  in virtqueue_kick_prepare_split()
746 	new = vq->split.avail_idx_shadow;  in virtqueue_kick_prepare_split()
754 					vring_avail_event(&vq->split.vring)),  in virtqueue_kick_prepare_split()
757 		needs_kick = !(vq->split.vring.used->flags &  in virtqueue_kick_prepare_split()
772 	vq->split.desc_state[head].data = NULL;  in detach_buf_split()
777 	while (vq->split.vring.desc[i].flags & nextflag) {  in detach_buf_split()
779 		i = vq->split.desc_extra[i].next;  in detach_buf_split()
784 	vq->split.desc_extra[i].next = vq->free_head;  in detach_buf_split()
792 				vq->split.desc_state[head].indir_desc;  in detach_buf_split()
799 		len = vq->split.desc_extra[head].len;  in detach_buf_split()
801 		BUG_ON(!(vq->split.desc_extra[head].flags &  in detach_buf_split()
811 		vq->split.desc_state[head].indir_desc = NULL;  in detach_buf_split()
813 		*ctx = vq->split.desc_state[head].indir_desc;  in detach_buf_split()
820 			vq->split.vring.used->idx);  in more_used_split()
848 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));  in virtqueue_get_buf_ctx_split()
850 			vq->split.vring.used->ring[last_used].id);  in virtqueue_get_buf_ctx_split()
852 			vq->split.vring.used->ring[last_used].len);  in virtqueue_get_buf_ctx_split()
854 	if (unlikely(i >= vq->split.vring.num)) {  in virtqueue_get_buf_ctx_split()
858 	if (unlikely(!vq->split.desc_state[i].data)) {  in virtqueue_get_buf_ctx_split()
864 	ret = vq->split.desc_state[i].data;  in virtqueue_get_buf_ctx_split()
870 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))  in virtqueue_get_buf_ctx_split()
872 				&vring_used_event(&vq->split.vring),  in virtqueue_get_buf_ctx_split()
885 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {  in virtqueue_disable_cb_split()
886 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_disable_cb_split()
897 			vring_used_event(&vq->split.vring) = 0x0;  in virtqueue_disable_cb_split()
899 			vq->split.vring.avail->flags =  in virtqueue_disable_cb_split()
901 						vq->split.avail_flags_shadow);  in virtqueue_disable_cb_split()
917 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_prepare_split()
918 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_prepare_split()
920 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_prepare_split()
922 						vq->split.avail_flags_shadow);  in virtqueue_enable_cb_prepare_split()
924 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
935 			vq->split.vring.used->idx);  in virtqueue_poll_split()
950 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {  in virtqueue_enable_cb_delayed_split()
951 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_enable_cb_delayed_split()
953 			vq->split.vring.avail->flags =  in virtqueue_enable_cb_delayed_split()
955 						vq->split.avail_flags_shadow);  in virtqueue_enable_cb_delayed_split()
958 	bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;  in virtqueue_enable_cb_delayed_split()
961 			&vring_used_event(&vq->split.vring),  in virtqueue_enable_cb_delayed_split()
964 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
982 	for (i = 0; i < vq->split.vring.num; i++) {  in virtqueue_detach_unused_buf_split()
983 		if (!vq->split.desc_state[i].data)  in virtqueue_detach_unused_buf_split()
986 		buf = vq->split.desc_state[i].data;  in virtqueue_detach_unused_buf_split()
988 		vq->split.avail_idx_shadow--;  in virtqueue_detach_unused_buf_split()
989 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
990 				vq->split.avail_idx_shadow);  in virtqueue_detach_unused_buf_split()
995 	BUG_ON(vq->vq.num_free != vq->split.vring.num);  in virtqueue_detach_unused_buf_split()
1024 	num = vq->split.vring.num;  in virtqueue_reinit_split()
1026 	vq->split.vring.avail->flags = 0;  in virtqueue_reinit_split()
1027 	vq->split.vring.avail->idx = 0;  in virtqueue_reinit_split()
1030 	vq->split.vring.avail->ring[num] = 0;  in virtqueue_reinit_split()
1032 	vq->split.vring.used->flags = 0;  in virtqueue_reinit_split()
1033 	vq->split.vring.used->idx = 0;  in virtqueue_reinit_split()
1036 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;  in virtqueue_reinit_split()
1040 	virtqueue_vring_init_split(&vq->split, vq);  in virtqueue_reinit_split()
1046 	vq->split = *vring_split;  in virtqueue_vring_attach_split()
1183 				      vq->split.vring_align,  in virtqueue_resize_split()
1184 				      vq->split.may_reduce_num,  in virtqueue_resize_split()
2335  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2355  * virtqueue_notify - second half of split virtqueue_kick call.
2601 /* Only available for split ring */
2750 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2794 	num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_set_dma_premapped()
2848 /* Only available for split ring */
2900 					 vq->split.queue_size_in_bytes,  in vring_free()
2901 					 vq->split.vring.desc,  in vring_free()
2902 					 vq->split.queue_dma_addr,  in vring_free()
2907 		kfree(vq->split.desc_state);  in vring_free()
2908 		kfree(vq->split.desc_extra);  in vring_free()
2937 		next = vq->split.avail_idx_shadow;  in vring_notification_data()
2984 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
3070 	return vq->split.queue_dma_addr;  in virtqueue_get_desc_addr()
3083 	return vq->split.queue_dma_addr +  in virtqueue_get_avail_addr()
3084 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);  in virtqueue_get_avail_addr()
3097 	return vq->split.queue_dma_addr +  in virtqueue_get_used_addr()
3098 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);  in virtqueue_get_used_addr()
3102 /* Only available for split ring */
3105 	return &to_vvq(vq)->split.vring;  in virtqueue_get_vring()