1 /*
2 * vhost shadow virtqueue
3 *
4 * SPDX-FileCopyrightText: Red Hat, Inc. 2021
5 * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
6 *
7 * SPDX-License-Identifier: GPL-2.0-or-later
8 */
9
10 #include "qemu/osdep.h"
11 #include "hw/virtio/vhost-shadow-virtqueue.h"
12
13 #include "qemu/error-report.h"
14 #include "qapi/error.h"
15 #include "qemu/main-loop.h"
16 #include "qemu/log.h"
17 #include "qemu/memalign.h"
18 #include "linux-headers/linux/vhost.h"
19
20 /**
21 * Validate the transport device features that both guests can use with the SVQ
22 * and SVQs can use with the device.
23 *
24 * @dev_features: The features
25 * @errp: Error pointer
26 */
vhost_svq_valid_features(uint64_t features,Error ** errp)27 bool vhost_svq_valid_features(uint64_t features, Error **errp)
28 {
29 bool ok = true;
30 uint64_t svq_features = features;
31
32 for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
33 ++b) {
34 switch (b) {
35 case VIRTIO_F_ANY_LAYOUT:
36 case VIRTIO_RING_F_EVENT_IDX:
37 continue;
38
39 case VIRTIO_F_ACCESS_PLATFORM:
40 /* SVQ trust in the host's IOMMU to translate addresses */
41 case VIRTIO_F_VERSION_1:
42 /* SVQ trust that the guest vring is little endian */
43 if (!(svq_features & BIT_ULL(b))) {
44 svq_features |= BIT_ULL(b);
45 ok = false;
46 }
47 continue;
48
49 default:
50 if (svq_features & BIT_ULL(b)) {
51 svq_features &= ~BIT_ULL(b);
52 ok = false;
53 }
54 }
55 }
56
57 if (!ok) {
58 error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
59 ", ok: 0x%"PRIx64, features, svq_features);
60 }
61 return ok;
62 }
63
64 /**
65 * Number of descriptors that the SVQ can make available from the guest.
66 *
67 * @svq: The svq
68 */
vhost_svq_available_slots(const VhostShadowVirtqueue * svq)69 uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
70 {
71 return svq->num_free;
72 }
73
74 /**
75 * Translate addresses between the qemu's virtual address and the SVQ IOVA
76 *
77 * @svq: Shadow VirtQueue
78 * @vaddr: Translated IOVA addresses
79 * @iovec: Source qemu's VA addresses
80 * @num: Length of iovec and minimum length of vaddr
81 */
vhost_svq_translate_addr(const VhostShadowVirtqueue * svq,hwaddr * addrs,const struct iovec * iovec,size_t num)82 static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
83 hwaddr *addrs, const struct iovec *iovec,
84 size_t num)
85 {
86 if (num == 0) {
87 return true;
88 }
89
90 for (size_t i = 0; i < num; ++i) {
91 DMAMap needle = {
92 .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
93 .size = iovec[i].iov_len,
94 };
95 Int128 needle_last, map_last;
96 size_t off;
97
98 const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
99 /*
100 * Map cannot be NULL since iova map contains all guest space and
101 * qemu already has a physical address mapped
102 */
103 if (unlikely(!map)) {
104 qemu_log_mask(LOG_GUEST_ERROR,
105 "Invalid address 0x%"HWADDR_PRIx" given by guest",
106 needle.translated_addr);
107 return false;
108 }
109
110 off = needle.translated_addr - map->translated_addr;
111 addrs[i] = map->iova + off;
112
113 needle_last = int128_add(int128_make64(needle.translated_addr),
114 int128_makes64(iovec[i].iov_len - 1));
115 map_last = int128_make64(map->translated_addr + map->size);
116 if (unlikely(int128_gt(needle_last, map_last))) {
117 qemu_log_mask(LOG_GUEST_ERROR,
118 "Guest buffer expands over iova range");
119 return false;
120 }
121 }
122
123 return true;
124 }
125
126 /**
127 * Write descriptors to SVQ vring
128 *
129 * @svq: The shadow virtqueue
130 * @sg: Cache for hwaddr
131 * @iovec: The iovec from the guest
132 * @num: iovec length
133 * @more_descs: True if more descriptors come in the chain
134 * @write: True if they are writeable descriptors
135 *
136 * Return true if success, false otherwise and print error.
137 */
vhost_svq_vring_write_descs(VhostShadowVirtqueue * svq,hwaddr * sg,const struct iovec * iovec,size_t num,bool more_descs,bool write)138 static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
139 const struct iovec *iovec, size_t num,
140 bool more_descs, bool write)
141 {
142 uint16_t i = svq->free_head, last = svq->free_head;
143 unsigned n;
144 uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
145 vring_desc_t *descs = svq->vring.desc;
146 bool ok;
147
148 if (num == 0) {
149 return true;
150 }
151
152 ok = vhost_svq_translate_addr(svq, sg, iovec, num);
153 if (unlikely(!ok)) {
154 return false;
155 }
156
157 for (n = 0; n < num; n++) {
158 if (more_descs || (n + 1 < num)) {
159 descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
160 descs[i].next = cpu_to_le16(svq->desc_next[i]);
161 } else {
162 descs[i].flags = flags;
163 }
164 descs[i].addr = cpu_to_le64(sg[n]);
165 descs[i].len = cpu_to_le32(iovec[n].iov_len);
166
167 last = i;
168 i = cpu_to_le16(svq->desc_next[i]);
169 }
170
171 svq->free_head = le16_to_cpu(svq->desc_next[last]);
172 return true;
173 }
174
vhost_svq_add_split(VhostShadowVirtqueue * svq,const struct iovec * out_sg,size_t out_num,const struct iovec * in_sg,size_t in_num,unsigned * head)175 static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
176 const struct iovec *out_sg, size_t out_num,
177 const struct iovec *in_sg, size_t in_num,
178 unsigned *head)
179 {
180 unsigned avail_idx;
181 vring_avail_t *avail = svq->vring.avail;
182 bool ok;
183 g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num));
184
185 *head = svq->free_head;
186
187 /* We need some descriptors here */
188 if (unlikely(!out_num && !in_num)) {
189 qemu_log_mask(LOG_GUEST_ERROR,
190 "Guest provided element with no descriptors");
191 return false;
192 }
193
194 ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, in_num > 0,
195 false);
196 if (unlikely(!ok)) {
197 return false;
198 }
199
200 ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, false, true);
201 if (unlikely(!ok)) {
202 return false;
203 }
204
205 /*
206 * Put the entry in the available array (but don't update avail->idx until
207 * they do sync).
208 */
209 avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
210 avail->ring[avail_idx] = cpu_to_le16(*head);
211 svq->shadow_avail_idx++;
212
213 /* Update the avail index after write the descriptor */
214 smp_wmb();
215 avail->idx = cpu_to_le16(svq->shadow_avail_idx);
216
217 return true;
218 }
219
vhost_svq_kick(VhostShadowVirtqueue * svq)220 static void vhost_svq_kick(VhostShadowVirtqueue *svq)
221 {
222 bool needs_kick;
223
224 /*
225 * We need to expose the available array entries before checking the used
226 * flags
227 */
228 smp_mb();
229
230 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
231 uint16_t avail_event = *(uint16_t *)(&svq->vring.used->ring[svq->vring.num]);
232 needs_kick = vring_need_event(avail_event, svq->shadow_avail_idx, svq->shadow_avail_idx - 1);
233 } else {
234 needs_kick = !(svq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
235 }
236
237 if (!needs_kick) {
238 return;
239 }
240
241 event_notifier_set(&svq->hdev_kick);
242 }
243
244 /**
245 * Add an element to a SVQ.
246 *
247 * Return -EINVAL if element is invalid, -ENOSPC if dev queue is full
248 */
vhost_svq_add(VhostShadowVirtqueue * svq,const struct iovec * out_sg,size_t out_num,const struct iovec * in_sg,size_t in_num,VirtQueueElement * elem)249 int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,
250 size_t out_num, const struct iovec *in_sg, size_t in_num,
251 VirtQueueElement *elem)
252 {
253 unsigned qemu_head;
254 unsigned ndescs = in_num + out_num;
255 bool ok;
256
257 if (unlikely(ndescs > vhost_svq_available_slots(svq))) {
258 return -ENOSPC;
259 }
260
261 ok = vhost_svq_add_split(svq, out_sg, out_num, in_sg, in_num, &qemu_head);
262 if (unlikely(!ok)) {
263 return -EINVAL;
264 }
265
266 svq->num_free -= ndescs;
267 svq->desc_state[qemu_head].elem = elem;
268 svq->desc_state[qemu_head].ndescs = ndescs;
269 vhost_svq_kick(svq);
270 return 0;
271 }
272
273 /* Convenience wrapper to add a guest's element to SVQ */
vhost_svq_add_element(VhostShadowVirtqueue * svq,VirtQueueElement * elem)274 static int vhost_svq_add_element(VhostShadowVirtqueue *svq,
275 VirtQueueElement *elem)
276 {
277 return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->in_sg,
278 elem->in_num, elem);
279 }
280
281 /**
282 * Forward available buffers.
283 *
284 * @svq: Shadow VirtQueue
285 *
286 * Note that this function does not guarantee that all guest's available
287 * buffers are available to the device in SVQ avail ring. The guest may have
288 * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
289 * qemu vaddr.
290 *
291 * If that happens, guest's kick notifications will be disabled until the
292 * device uses some buffers.
293 */
vhost_handle_guest_kick(VhostShadowVirtqueue * svq)294 static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
295 {
296 /* Clear event notifier */
297 event_notifier_test_and_clear(&svq->svq_kick);
298
299 /* Forward to the device as many available buffers as possible */
300 do {
301 virtio_queue_set_notification(svq->vq, false);
302
303 while (true) {
304 g_autofree VirtQueueElement *elem = NULL;
305 int r;
306
307 if (svq->next_guest_avail_elem) {
308 elem = g_steal_pointer(&svq->next_guest_avail_elem);
309 } else {
310 elem = virtqueue_pop(svq->vq, sizeof(*elem));
311 }
312
313 if (!elem) {
314 break;
315 }
316
317 if (svq->ops) {
318 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque);
319 } else {
320 r = vhost_svq_add_element(svq, elem);
321 }
322 if (unlikely(r != 0)) {
323 if (r == -ENOSPC) {
324 /*
325 * This condition is possible since a contiguous buffer in
326 * GPA does not imply a contiguous buffer in qemu's VA
327 * scatter-gather segments. If that happens, the buffer
328 * exposed to the device needs to be a chain of descriptors
329 * at this moment.
330 *
331 * SVQ cannot hold more available buffers if we are here:
332 * queue the current guest descriptor and ignore kicks
333 * until some elements are used.
334 */
335 svq->next_guest_avail_elem = g_steal_pointer(&elem);
336 }
337
338 /* VQ is full or broken, just return and ignore kicks */
339 return;
340 }
341 /* elem belongs to SVQ or external caller now */
342 elem = NULL;
343 }
344
345 virtio_queue_set_notification(svq->vq, true);
346 } while (!virtio_queue_empty(svq->vq));
347 }
348
349 /**
350 * Handle guest's kick.
351 *
352 * @n: guest kick event notifier, the one that guest set to notify svq.
353 */
vhost_handle_guest_kick_notifier(EventNotifier * n)354 static void vhost_handle_guest_kick_notifier(EventNotifier *n)
355 {
356 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
357 event_notifier_test_and_clear(n);
358 vhost_handle_guest_kick(svq);
359 }
360
vhost_svq_more_used(VhostShadowVirtqueue * svq)361 static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
362 {
363 uint16_t *used_idx = &svq->vring.used->idx;
364 if (svq->last_used_idx != svq->shadow_used_idx) {
365 return true;
366 }
367
368 svq->shadow_used_idx = cpu_to_le16(*(volatile uint16_t *)used_idx);
369
370 return svq->last_used_idx != svq->shadow_used_idx;
371 }
372
373 /**
374 * Enable vhost device calls after disable them.
375 *
376 * @svq: The svq
377 *
378 * It returns false if there are pending used buffers from the vhost device,
379 * avoiding the possible races between SVQ checking for more work and enabling
380 * callbacks. True if SVQ used vring has no more pending buffers.
381 */
vhost_svq_enable_notification(VhostShadowVirtqueue * svq)382 static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
383 {
384 if (virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
385 uint16_t *used_event = (uint16_t *)&svq->vring.avail->ring[svq->vring.num];
386 *used_event = svq->shadow_used_idx;
387 } else {
388 svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
389 }
390
391 /* Make sure the event is enabled before the read of used_idx */
392 smp_mb();
393 return !vhost_svq_more_used(svq);
394 }
395
vhost_svq_disable_notification(VhostShadowVirtqueue * svq)396 static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
397 {
398 /*
399 * No need to disable notification in the event idx case, since used event
400 * index is already an index too far away.
401 */
402 if (!virtio_vdev_has_feature(svq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
403 svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
404 }
405 }
406
vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue * svq,uint16_t num,uint16_t i)407 static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
408 uint16_t num, uint16_t i)
409 {
410 for (uint16_t j = 0; j < (num - 1); ++j) {
411 i = le16_to_cpu(svq->desc_next[i]);
412 }
413
414 return i;
415 }
416
417 G_GNUC_WARN_UNUSED_RESULT
vhost_svq_get_buf(VhostShadowVirtqueue * svq,uint32_t * len)418 static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
419 uint32_t *len)
420 {
421 const vring_used_t *used = svq->vring.used;
422 vring_used_elem_t used_elem;
423 uint16_t last_used, last_used_chain, num;
424
425 if (!vhost_svq_more_used(svq)) {
426 return NULL;
427 }
428
429 /* Only get used array entries after they have been exposed by dev */
430 smp_rmb();
431 last_used = svq->last_used_idx & (svq->vring.num - 1);
432 used_elem.id = le32_to_cpu(used->ring[last_used].id);
433 used_elem.len = le32_to_cpu(used->ring[last_used].len);
434
435 svq->last_used_idx++;
436 if (unlikely(used_elem.id >= svq->vring.num)) {
437 qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
438 svq->vdev->name, used_elem.id);
439 return NULL;
440 }
441
442 if (unlikely(!svq->desc_state[used_elem.id].ndescs)) {
443 qemu_log_mask(LOG_GUEST_ERROR,
444 "Device %s says index %u is used, but it was not available",
445 svq->vdev->name, used_elem.id);
446 return NULL;
447 }
448
449 num = svq->desc_state[used_elem.id].ndescs;
450 svq->desc_state[used_elem.id].ndescs = 0;
451 last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);
452 svq->desc_next[last_used_chain] = svq->free_head;
453 svq->free_head = used_elem.id;
454 svq->num_free += num;
455
456 *len = used_elem.len;
457 return g_steal_pointer(&svq->desc_state[used_elem.id].elem);
458 }
459
460 /**
461 * Push an element to SVQ, returning it to the guest.
462 */
vhost_svq_push_elem(VhostShadowVirtqueue * svq,const VirtQueueElement * elem,uint32_t len)463 void vhost_svq_push_elem(VhostShadowVirtqueue *svq,
464 const VirtQueueElement *elem, uint32_t len)
465 {
466 virtqueue_push(svq->vq, elem, len);
467 if (svq->next_guest_avail_elem) {
468 /*
469 * Avail ring was full when vhost_svq_flush was called, so it's a
470 * good moment to make more descriptors available if possible.
471 */
472 vhost_handle_guest_kick(svq);
473 }
474 }
475
vhost_svq_flush(VhostShadowVirtqueue * svq,bool check_for_avail_queue)476 static void vhost_svq_flush(VhostShadowVirtqueue *svq,
477 bool check_for_avail_queue)
478 {
479 VirtQueue *vq = svq->vq;
480
481 /* Forward as many used buffers as possible. */
482 do {
483 unsigned i = 0;
484
485 vhost_svq_disable_notification(svq);
486 while (true) {
487 uint32_t len;
488 g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
489 if (!elem) {
490 break;
491 }
492
493 if (unlikely(i >= svq->vring.num)) {
494 qemu_log_mask(LOG_GUEST_ERROR,
495 "More than %u used buffers obtained in a %u size SVQ",
496 i, svq->vring.num);
497 virtqueue_fill(vq, elem, len, i);
498 virtqueue_flush(vq, i);
499 return;
500 }
501 virtqueue_fill(vq, elem, len, i++);
502 }
503
504 virtqueue_flush(vq, i);
505 event_notifier_set(&svq->svq_call);
506
507 if (check_for_avail_queue && svq->next_guest_avail_elem) {
508 /*
509 * Avail ring was full when vhost_svq_flush was called, so it's a
510 * good moment to make more descriptors available if possible.
511 */
512 vhost_handle_guest_kick(svq);
513 }
514 } while (!vhost_svq_enable_notification(svq));
515 }
516
517 /**
518 * Poll the SVQ to wait for the device to use the specified number
519 * of elements and return the total length written by the device.
520 *
521 * This function race with main event loop SVQ polling, so extra
522 * synchronization is needed.
523 *
524 * @svq: The svq
525 * @num: The number of elements that need to be used
526 */
vhost_svq_poll(VhostShadowVirtqueue * svq,size_t num)527 size_t vhost_svq_poll(VhostShadowVirtqueue *svq, size_t num)
528 {
529 size_t len = 0;
530
531 while (num--) {
532 g_autofree VirtQueueElement *elem = NULL;
533 int64_t start_us = g_get_monotonic_time();
534 uint32_t r = 0;
535
536 do {
537 if (vhost_svq_more_used(svq)) {
538 break;
539 }
540
541 if (unlikely(g_get_monotonic_time() - start_us > 10e6)) {
542 return len;
543 }
544 } while (true);
545
546 elem = vhost_svq_get_buf(svq, &r);
547 len += r;
548 }
549
550 return len;
551 }
552
553 /**
554 * Forward used buffers.
555 *
556 * @n: hdev call event notifier, the one that device set to notify svq.
557 *
558 * Note that we are not making any buffers available in the loop, there is no
559 * way that it runs more than virtqueue size times.
560 */
vhost_svq_handle_call(EventNotifier * n)561 static void vhost_svq_handle_call(EventNotifier *n)
562 {
563 VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
564 hdev_call);
565 event_notifier_test_and_clear(n);
566 vhost_svq_flush(svq, true);
567 }
568
569 /**
570 * Set the call notifier for the SVQ to call the guest
571 *
572 * @svq: Shadow virtqueue
573 * @call_fd: call notifier
574 *
575 * Called on BQL context.
576 */
vhost_svq_set_svq_call_fd(VhostShadowVirtqueue * svq,int call_fd)577 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
578 {
579 if (call_fd == VHOST_FILE_UNBIND) {
580 /*
581 * Fail event_notifier_set if called handling device call.
582 *
583 * SVQ still needs device notifications, since it needs to keep
584 * forwarding used buffers even with the unbind.
585 */
586 memset(&svq->svq_call, 0, sizeof(svq->svq_call));
587 } else {
588 event_notifier_init_fd(&svq->svq_call, call_fd);
589 }
590 }
591
592 /**
593 * Get the shadow vq vring address.
594 * @svq: Shadow virtqueue
595 * @addr: Destination to store address
596 */
vhost_svq_get_vring_addr(const VhostShadowVirtqueue * svq,struct vhost_vring_addr * addr)597 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
598 struct vhost_vring_addr *addr)
599 {
600 addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
601 addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
602 addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
603 }
604
vhost_svq_driver_area_size(const VhostShadowVirtqueue * svq)605 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
606 {
607 size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
608 size_t avail_size = offsetof(vring_avail_t, ring[svq->vring.num]) +
609 sizeof(uint16_t);
610
611 return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size());
612 }
613
vhost_svq_device_area_size(const VhostShadowVirtqueue * svq)614 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
615 {
616 size_t used_size = offsetof(vring_used_t, ring[svq->vring.num]) +
617 sizeof(uint16_t);
618 return ROUND_UP(used_size, qemu_real_host_page_size());
619 }
620
621 /**
622 * Set a new file descriptor for the guest to kick the SVQ and notify for avail
623 *
624 * @svq: The svq
625 * @svq_kick_fd: The svq kick fd
626 *
627 * Note that the SVQ will never close the old file descriptor.
628 */
vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue * svq,int svq_kick_fd)629 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
630 {
631 EventNotifier *svq_kick = &svq->svq_kick;
632 bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
633 bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
634
635 if (poll_stop) {
636 event_notifier_set_handler(svq_kick, NULL);
637 }
638
639 event_notifier_init_fd(svq_kick, svq_kick_fd);
640 /*
641 * event_notifier_set_handler already checks for guest's notifications if
642 * they arrive at the new file descriptor in the switch, so there is no
643 * need to explicitly check for them.
644 */
645 if (poll_start) {
646 event_notifier_set(svq_kick);
647 event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
648 }
649 }
650
651 /**
652 * Start the shadow virtqueue operation.
653 *
654 * @svq: Shadow Virtqueue
655 * @vdev: VirtIO device
656 * @vq: Virtqueue to shadow
657 * @iova_tree: Tree to perform descriptors translations
658 */
vhost_svq_start(VhostShadowVirtqueue * svq,VirtIODevice * vdev,VirtQueue * vq,VhostIOVATree * iova_tree)659 void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
660 VirtQueue *vq, VhostIOVATree *iova_tree)
661 {
662 size_t desc_size;
663
664 event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
665 svq->next_guest_avail_elem = NULL;
666 svq->shadow_avail_idx = 0;
667 svq->shadow_used_idx = 0;
668 svq->last_used_idx = 0;
669 svq->vdev = vdev;
670 svq->vq = vq;
671 svq->iova_tree = iova_tree;
672
673 svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
674 svq->num_free = svq->vring.num;
675 svq->vring.desc = mmap(NULL, vhost_svq_driver_area_size(svq),
676 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
677 -1, 0);
678 desc_size = sizeof(vring_desc_t) * svq->vring.num;
679 svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
680 svq->vring.used = mmap(NULL, vhost_svq_device_area_size(svq),
681 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS,
682 -1, 0);
683 svq->desc_state = g_new0(SVQDescState, svq->vring.num);
684 svq->desc_next = g_new0(uint16_t, svq->vring.num);
685 for (unsigned i = 0; i < svq->vring.num - 1; i++) {
686 svq->desc_next[i] = cpu_to_le16(i + 1);
687 }
688 }
689
690 /**
691 * Stop the shadow virtqueue operation.
692 * @svq: Shadow Virtqueue
693 */
vhost_svq_stop(VhostShadowVirtqueue * svq)694 void vhost_svq_stop(VhostShadowVirtqueue *svq)
695 {
696 vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND);
697 g_autofree VirtQueueElement *next_avail_elem = NULL;
698
699 if (!svq->vq) {
700 return;
701 }
702
703 /* Send all pending used descriptors to guest */
704 vhost_svq_flush(svq, false);
705
706 for (unsigned i = 0; i < svq->vring.num; ++i) {
707 g_autofree VirtQueueElement *elem = NULL;
708 elem = g_steal_pointer(&svq->desc_state[i].elem);
709 if (elem) {
710 /*
711 * TODO: This is ok for networking, but other kinds of devices
712 * might have problems with just unpop these.
713 */
714 virtqueue_unpop(svq->vq, elem, 0);
715 }
716 }
717
718 next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
719 if (next_avail_elem) {
720 virtqueue_unpop(svq->vq, next_avail_elem, 0);
721 }
722 svq->vq = NULL;
723 g_free(svq->desc_next);
724 g_free(svq->desc_state);
725 munmap(svq->vring.desc, vhost_svq_driver_area_size(svq));
726 munmap(svq->vring.used, vhost_svq_device_area_size(svq));
727 event_notifier_set_handler(&svq->hdev_call, NULL);
728 }
729
730 /**
731 * Creates vhost shadow virtqueue, and instructs the vhost device to use the
732 * shadow methods and file descriptors.
733 *
734 * @ops: SVQ owner callbacks
735 * @ops_opaque: ops opaque pointer
736 */
vhost_svq_new(const VhostShadowVirtqueueOps * ops,void * ops_opaque)737 VhostShadowVirtqueue *vhost_svq_new(const VhostShadowVirtqueueOps *ops,
738 void *ops_opaque)
739 {
740 VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
741
742 event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
743 svq->ops = ops;
744 svq->ops_opaque = ops_opaque;
745 return svq;
746 }
747
748 /**
749 * Free the resources of the shadow virtqueue.
750 *
751 * @pvq: gpointer to SVQ so it can be used by autofree functions.
752 */
vhost_svq_free(gpointer pvq)753 void vhost_svq_free(gpointer pvq)
754 {
755 VhostShadowVirtqueue *vq = pvq;
756 vhost_svq_stop(vq);
757 g_free(vq);
758 }
759