91 			    struct vhost_virtqueue *vq)  in vhost_transport_do_send_pkt()  argument
98 	mutex_lock(&vq->mutex);  in vhost_transport_do_send_pkt()
100 	if (!vhost_vq_get_backend(vq))  in vhost_transport_do_send_pkt()
103 	if (!vq_meta_prefetch(vq))  in vhost_transport_do_send_pkt()
107 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
122 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
126 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_transport_do_send_pkt()
133 		if (head == vq->num) {  in vhost_transport_do_send_pkt()
138 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
139 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
147 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);  in vhost_transport_do_send_pkt()
151 		iov_len = iov_length(&vq->iov[out], in);  in vhost_transport_do_send_pkt()
154 			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);  in vhost_transport_do_send_pkt()
158 		iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[out], in, iov_len);  in vhost_transport_do_send_pkt()
196 			vq_err(vq, "Faulted on copying pkt hdr\n");  in vhost_transport_do_send_pkt()
203 			vq_err(vq, "Faulted on copying pkt buf\n");  in vhost_transport_do_send_pkt()
212 		vhost_add_used(vq, head, sizeof(*hdr) + payload_len);  in vhost_transport_do_send_pkt()
245 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_transport_do_send_pkt()
247 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
250 	mutex_unlock(&vq->mutex);  in vhost_transport_do_send_pkt()
258 	struct vhost_virtqueue *vq;  in vhost_transport_send_pkt_work()  local
262 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
264 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
326 vhost_vsock_alloc_skb(struct vhost_virtqueue *vq,  in vhost_vsock_alloc_skb()  argument
337 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);  in vhost_vsock_alloc_skb()
341 	len = iov_length(vq->iov, out);  in vhost_vsock_alloc_skb()
348 	iov_iter_init(&iov_iter, ITER_SOURCE, vq->iov, out, len);  in vhost_vsock_alloc_skb()
353 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",  in vhost_vsock_alloc_skb()
376 		vq_err(vq, "Expected %zu byte payload, got %zu bytes\n",  in vhost_vsock_alloc_skb()
388 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()  local
394 	return val < vq->num;  in vhost_vsock_more_replies()
467 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_tx_kick()  local
469 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()
476 	mutex_lock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
478 	if (!vhost_vq_get_backend(vq))  in vhost_vsock_handle_tx_kick()
481 	if (!vq_meta_prefetch(vq))  in vhost_vsock_handle_tx_kick()
484 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
496 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_vsock_handle_tx_kick()
501 		if (head == vq->num) {  in vhost_vsock_handle_tx_kick()
502 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
503 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
509 		skb = vhost_vsock_alloc_skb(vq, out, in);  in vhost_vsock_handle_tx_kick()
511 			vq_err(vq, "Faulted on pkt\n");  in vhost_vsock_handle_tx_kick()
530 		vhost_add_used(vq, head, 0);  in vhost_vsock_handle_tx_kick()
532 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_vsock_handle_tx_kick()
536 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
539 	mutex_unlock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
544 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_rx_kick()  local
546 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()
549 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
554 	struct vhost_virtqueue *vq;  in vhost_vsock_start()  local
565 		vq = &vsock->vqs[i];  in vhost_vsock_start()
567 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
569 		if (!vhost_vq_access_ok(vq)) {  in vhost_vsock_start()
574 		if (!vhost_vq_get_backend(vq)) {  in vhost_vsock_start()
575 			vhost_vq_set_backend(vq, vsock);  in vhost_vsock_start()
576 			ret = vhost_vq_init_access(vq);  in vhost_vsock_start()
581 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
593 	vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
594 	mutex_unlock(&vq->mutex);  in vhost_vsock_start()
597 		vq = &vsock->vqs[i];  in vhost_vsock_start()
599 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
600 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
601 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
622 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()  local
624 		mutex_lock(&vq->mutex);  in vhost_vsock_stop()
625 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_stop()
626 		mutex_unlock(&vq->mutex);  in vhost_vsock_stop()
786 	struct vhost_virtqueue *vq;  in vhost_vsock_set_features()  local
806 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
807 		mutex_lock(&vq->mutex);  in vhost_vsock_set_features()
808 		vq->acked_features = features;  in vhost_vsock_set_features()
809 		mutex_unlock(&vq->mutex);  in vhost_vsock_set_features()