1 /*
2  * vhost shadow virtqueue
3  *
4  * SPDX-FileCopyrightText: Red Hat, Inc. 2021
5  * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
6  *
7  * SPDX-License-Identifier: GPL-2.0-or-later
8  */
9 
10 #include "qemu/osdep.h"
11 #include "hw/virtio/vhost-shadow-virtqueue.h"
12 
13 #include "qemu/error-report.h"
14 #include "qapi/error.h"
15 #include "qemu/main-loop.h"
16 #include "qemu/log.h"
17 #include "qemu/memalign.h"
18 #include "linux-headers/linux/vhost.h"
19 
20 /**
21  * Validate the transport device features that both guests can use with the SVQ
22  * and SVQs can use with the device.
23  *
24  * @dev_features: The features
25  * @errp: Error pointer
26  */
27 bool vhost_svq_valid_features(uint64_t features, Error **errp)
28 {
29     bool ok = true;
30     uint64_t svq_features = features;
31 
32     for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
33          ++b) {
34         switch (b) {
35         case VIRTIO_F_ANY_LAYOUT:
36             continue;
37 
38         case VIRTIO_F_ACCESS_PLATFORM:
39             /* SVQ trust in the host's IOMMU to translate addresses */
40         case VIRTIO_F_VERSION_1:
41             /* SVQ trust that the guest vring is little endian */
42             if (!(svq_features & BIT_ULL(b))) {
43                 svq_features |= BIT_ULL(b);
44                 ok = false;
45             }
46             continue;
47 
48         default:
49             if (svq_features & BIT_ULL(b)) {
50                 svq_features &= ~BIT_ULL(b);
51                 ok = false;
52             }
53         }
54     }
55 
56     if (!ok) {
57         error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
58                          ", ok: 0x%"PRIx64, features, svq_features);
59     }
60     return ok;
61 }
62 
63 /**
64  * Number of descriptors that the SVQ can make available from the guest.
65  *
66  * @svq: The svq
67  */
68 static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
69 {
70     return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
71 }
72 
73 /**
74  * Translate addresses between the qemu's virtual address and the SVQ IOVA
75  *
76  * @svq: Shadow VirtQueue
77  * @vaddr: Translated IOVA addresses
78  * @iovec: Source qemu's VA addresses
79  * @num: Length of iovec and minimum length of vaddr
80  */
81 static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
82                                      hwaddr *addrs, const struct iovec *iovec,
83                                      size_t num)
84 {
85     if (num == 0) {
86         return true;
87     }
88 
89     for (size_t i = 0; i < num; ++i) {
90         DMAMap needle = {
91             .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
92             .size = iovec[i].iov_len,
93         };
94         Int128 needle_last, map_last;
95         size_t off;
96 
97         const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
98         /*
99          * Map cannot be NULL since iova map contains all guest space and
100          * qemu already has a physical address mapped
101          */
102         if (unlikely(!map)) {
103             qemu_log_mask(LOG_GUEST_ERROR,
104                           "Invalid address 0x%"HWADDR_PRIx" given by guest",
105                           needle.translated_addr);
106             return false;
107         }
108 
109         off = needle.translated_addr - map->translated_addr;
110         addrs[i] = map->iova + off;
111 
112         needle_last = int128_add(int128_make64(needle.translated_addr),
113                                  int128_make64(iovec[i].iov_len));
114         map_last = int128_make64(map->translated_addr + map->size);
115         if (unlikely(int128_gt(needle_last, map_last))) {
116             qemu_log_mask(LOG_GUEST_ERROR,
117                           "Guest buffer expands over iova range");
118             return false;
119         }
120     }
121 
122     return true;
123 }
124 
125 /**
126  * Write descriptors to SVQ vring
127  *
128  * @svq: The shadow virtqueue
129  * @sg: Cache for hwaddr
130  * @iovec: The iovec from the guest
131  * @num: iovec length
132  * @more_descs: True if more descriptors come in the chain
133  * @write: True if they are writeable descriptors
134  *
135  * Return true if success, false otherwise and print error.
136  */
137 static bool vhost_svq_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
138                                         const struct iovec *iovec, size_t num,
139                                         bool more_descs, bool write)
140 {
141     uint16_t i = svq->free_head, last = svq->free_head;
142     unsigned n;
143     uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
144     vring_desc_t *descs = svq->vring.desc;
145     bool ok;
146 
147     if (num == 0) {
148         return true;
149     }
150 
151     ok = vhost_svq_translate_addr(svq, sg, iovec, num);
152     if (unlikely(!ok)) {
153         return false;
154     }
155 
156     for (n = 0; n < num; n++) {
157         if (more_descs || (n + 1 < num)) {
158             descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
159             descs[i].next = cpu_to_le16(svq->desc_next[i]);
160         } else {
161             descs[i].flags = flags;
162         }
163         descs[i].addr = cpu_to_le64(sg[n]);
164         descs[i].len = cpu_to_le32(iovec[n].iov_len);
165 
166         last = i;
167         i = cpu_to_le16(svq->desc_next[i]);
168     }
169 
170     svq->free_head = le16_to_cpu(svq->desc_next[last]);
171     return true;
172 }
173 
174 static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
175                                 const struct iovec *out_sg, size_t out_num,
176                                 const struct iovec *in_sg, size_t in_num,
177                                 unsigned *head)
178 {
179     unsigned avail_idx;
180     vring_avail_t *avail = svq->vring.avail;
181     bool ok;
182     g_autofree hwaddr *sgs = g_new(hwaddr, MAX(out_num, in_num));
183 
184     *head = svq->free_head;
185 
186     /* We need some descriptors here */
187     if (unlikely(!out_num && !in_num)) {
188         qemu_log_mask(LOG_GUEST_ERROR,
189                       "Guest provided element with no descriptors");
190         return false;
191     }
192 
193     ok = vhost_svq_vring_write_descs(svq, sgs, out_sg, out_num, in_num > 0,
194                                      false);
195     if (unlikely(!ok)) {
196         return false;
197     }
198 
199     ok = vhost_svq_vring_write_descs(svq, sgs, in_sg, in_num, false, true);
200     if (unlikely(!ok)) {
201         return false;
202     }
203 
204     /*
205      * Put the entry in the available array (but don't update avail->idx until
206      * they do sync).
207      */
208     avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
209     avail->ring[avail_idx] = cpu_to_le16(*head);
210     svq->shadow_avail_idx++;
211 
212     /* Update the avail index after write the descriptor */
213     smp_wmb();
214     avail->idx = cpu_to_le16(svq->shadow_avail_idx);
215 
216     return true;
217 }
218 
219 static void vhost_svq_kick(VhostShadowVirtqueue *svq)
220 {
221     /*
222      * We need to expose the available array entries before checking the used
223      * flags
224      */
225     smp_mb();
226     if (svq->vring.used->flags & VRING_USED_F_NO_NOTIFY) {
227         return;
228     }
229 
230     event_notifier_set(&svq->hdev_kick);
231 }
232 
233 /**
234  * Add an element to a SVQ.
235  *
236  * Return -EINVAL if element is invalid, -ENOSPC if dev queue is full
237  */
238 int vhost_svq_add(VhostShadowVirtqueue *svq, const struct iovec *out_sg,
239                   size_t out_num, const struct iovec *in_sg, size_t in_num,
240                   VirtQueueElement *elem)
241 {
242     unsigned qemu_head;
243     unsigned ndescs = in_num + out_num;
244     bool ok;
245 
246     if (unlikely(ndescs > vhost_svq_available_slots(svq))) {
247         return -ENOSPC;
248     }
249 
250     ok = vhost_svq_add_split(svq, out_sg, out_num, in_sg, in_num, &qemu_head);
251     if (unlikely(!ok)) {
252         return -EINVAL;
253     }
254 
255     svq->desc_state[qemu_head].elem = elem;
256     svq->desc_state[qemu_head].ndescs = ndescs;
257     vhost_svq_kick(svq);
258     return 0;
259 }
260 
261 /* Convenience wrapper to add a guest's element to SVQ */
262 static int vhost_svq_add_element(VhostShadowVirtqueue *svq,
263                                  VirtQueueElement *elem)
264 {
265     return vhost_svq_add(svq, elem->out_sg, elem->out_num, elem->in_sg,
266                          elem->in_num, elem);
267 }
268 
269 /**
270  * Forward available buffers.
271  *
272  * @svq: Shadow VirtQueue
273  *
274  * Note that this function does not guarantee that all guest's available
275  * buffers are available to the device in SVQ avail ring. The guest may have
276  * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
277  * qemu vaddr.
278  *
279  * If that happens, guest's kick notifications will be disabled until the
280  * device uses some buffers.
281  */
282 static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
283 {
284     /* Clear event notifier */
285     event_notifier_test_and_clear(&svq->svq_kick);
286 
287     /* Forward to the device as many available buffers as possible */
288     do {
289         virtio_queue_set_notification(svq->vq, false);
290 
291         while (true) {
292             g_autofree VirtQueueElement *elem;
293             int r;
294 
295             if (svq->next_guest_avail_elem) {
296                 elem = g_steal_pointer(&svq->next_guest_avail_elem);
297             } else {
298                 elem = virtqueue_pop(svq->vq, sizeof(*elem));
299             }
300 
301             if (!elem) {
302                 break;
303             }
304 
305             if (svq->ops) {
306                 r = svq->ops->avail_handler(svq, elem, svq->ops_opaque);
307             } else {
308                 r = vhost_svq_add_element(svq, elem);
309             }
310             if (unlikely(r != 0)) {
311                 if (r == -ENOSPC) {
312                     /*
313                      * This condition is possible since a contiguous buffer in
314                      * GPA does not imply a contiguous buffer in qemu's VA
315                      * scatter-gather segments. If that happens, the buffer
316                      * exposed to the device needs to be a chain of descriptors
317                      * at this moment.
318                      *
319                      * SVQ cannot hold more available buffers if we are here:
320                      * queue the current guest descriptor and ignore kicks
321                      * until some elements are used.
322                      */
323                     svq->next_guest_avail_elem = g_steal_pointer(&elem);
324                 }
325 
326                 /* VQ is full or broken, just return and ignore kicks */
327                 return;
328             }
329             /* elem belongs to SVQ or external caller now */
330             elem = NULL;
331         }
332 
333         virtio_queue_set_notification(svq->vq, true);
334     } while (!virtio_queue_empty(svq->vq));
335 }
336 
337 /**
338  * Handle guest's kick.
339  *
340  * @n: guest kick event notifier, the one that guest set to notify svq.
341  */
342 static void vhost_handle_guest_kick_notifier(EventNotifier *n)
343 {
344     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
345     event_notifier_test_and_clear(n);
346     vhost_handle_guest_kick(svq);
347 }
348 
349 static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
350 {
351     uint16_t *used_idx = &svq->vring.used->idx;
352     if (svq->last_used_idx != svq->shadow_used_idx) {
353         return true;
354     }
355 
356     svq->shadow_used_idx = cpu_to_le16(*(volatile uint16_t *)used_idx);
357 
358     return svq->last_used_idx != svq->shadow_used_idx;
359 }
360 
361 /**
362  * Enable vhost device calls after disable them.
363  *
364  * @svq: The svq
365  *
366  * It returns false if there are pending used buffers from the vhost device,
367  * avoiding the possible races between SVQ checking for more work and enabling
368  * callbacks. True if SVQ used vring has no more pending buffers.
369  */
370 static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
371 {
372     svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
373     /* Make sure the flag is written before the read of used_idx */
374     smp_mb();
375     return !vhost_svq_more_used(svq);
376 }
377 
378 static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
379 {
380     svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
381 }
382 
383 static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
384                                              uint16_t num, uint16_t i)
385 {
386     for (uint16_t j = 0; j < (num - 1); ++j) {
387         i = le16_to_cpu(svq->desc_next[i]);
388     }
389 
390     return i;
391 }
392 
393 static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
394                                            uint32_t *len)
395 {
396     const vring_used_t *used = svq->vring.used;
397     vring_used_elem_t used_elem;
398     uint16_t last_used, last_used_chain, num;
399 
400     if (!vhost_svq_more_used(svq)) {
401         return NULL;
402     }
403 
404     /* Only get used array entries after they have been exposed by dev */
405     smp_rmb();
406     last_used = svq->last_used_idx & (svq->vring.num - 1);
407     used_elem.id = le32_to_cpu(used->ring[last_used].id);
408     used_elem.len = le32_to_cpu(used->ring[last_used].len);
409 
410     svq->last_used_idx++;
411     if (unlikely(used_elem.id >= svq->vring.num)) {
412         qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
413                       svq->vdev->name, used_elem.id);
414         return NULL;
415     }
416 
417     if (unlikely(!svq->desc_state[used_elem.id].ndescs)) {
418         qemu_log_mask(LOG_GUEST_ERROR,
419             "Device %s says index %u is used, but it was not available",
420             svq->vdev->name, used_elem.id);
421         return NULL;
422     }
423 
424     num = svq->desc_state[used_elem.id].ndescs;
425     svq->desc_state[used_elem.id].ndescs = 0;
426     last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);
427     svq->desc_next[last_used_chain] = svq->free_head;
428     svq->free_head = used_elem.id;
429 
430     *len = used_elem.len;
431     return g_steal_pointer(&svq->desc_state[used_elem.id].elem);
432 }
433 
434 /**
435  * Push an element to SVQ, returning it to the guest.
436  */
437 void vhost_svq_push_elem(VhostShadowVirtqueue *svq,
438                          const VirtQueueElement *elem, uint32_t len)
439 {
440     virtqueue_push(svq->vq, elem, len);
441     if (svq->next_guest_avail_elem) {
442         /*
443          * Avail ring was full when vhost_svq_flush was called, so it's a
444          * good moment to make more descriptors available if possible.
445          */
446         vhost_handle_guest_kick(svq);
447     }
448 }
449 
450 static void vhost_svq_flush(VhostShadowVirtqueue *svq,
451                             bool check_for_avail_queue)
452 {
453     VirtQueue *vq = svq->vq;
454 
455     /* Forward as many used buffers as possible. */
456     do {
457         unsigned i = 0;
458 
459         vhost_svq_disable_notification(svq);
460         while (true) {
461             uint32_t len;
462             g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
463             if (!elem) {
464                 break;
465             }
466 
467             if (unlikely(i >= svq->vring.num)) {
468                 qemu_log_mask(LOG_GUEST_ERROR,
469                          "More than %u used buffers obtained in a %u size SVQ",
470                          i, svq->vring.num);
471                 virtqueue_fill(vq, elem, len, i);
472                 virtqueue_flush(vq, i);
473                 return;
474             }
475             virtqueue_fill(vq, elem, len, i++);
476         }
477 
478         virtqueue_flush(vq, i);
479         event_notifier_set(&svq->svq_call);
480 
481         if (check_for_avail_queue && svq->next_guest_avail_elem) {
482             /*
483              * Avail ring was full when vhost_svq_flush was called, so it's a
484              * good moment to make more descriptors available if possible.
485              */
486             vhost_handle_guest_kick(svq);
487         }
488     } while (!vhost_svq_enable_notification(svq));
489 }
490 
491 /**
492  * Poll the SVQ for one device used buffer.
493  *
494  * This function race with main event loop SVQ polling, so extra
495  * synchronization is needed.
496  *
497  * Return the length written by the device.
498  */
499 size_t vhost_svq_poll(VhostShadowVirtqueue *svq)
500 {
501     int64_t start_us = g_get_monotonic_time();
502     uint32_t len;
503 
504     do {
505         if (vhost_svq_more_used(svq)) {
506             break;
507         }
508 
509         if (unlikely(g_get_monotonic_time() - start_us > 10e6)) {
510             return 0;
511         }
512     } while (true);
513 
514     vhost_svq_get_buf(svq, &len);
515     return len;
516 }
517 
518 /**
519  * Forward used buffers.
520  *
521  * @n: hdev call event notifier, the one that device set to notify svq.
522  *
523  * Note that we are not making any buffers available in the loop, there is no
524  * way that it runs more than virtqueue size times.
525  */
526 static void vhost_svq_handle_call(EventNotifier *n)
527 {
528     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
529                                              hdev_call);
530     event_notifier_test_and_clear(n);
531     vhost_svq_flush(svq, true);
532 }
533 
534 /**
535  * Set the call notifier for the SVQ to call the guest
536  *
537  * @svq: Shadow virtqueue
538  * @call_fd: call notifier
539  *
540  * Called on BQL context.
541  */
542 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
543 {
544     if (call_fd == VHOST_FILE_UNBIND) {
545         /*
546          * Fail event_notifier_set if called handling device call.
547          *
548          * SVQ still needs device notifications, since it needs to keep
549          * forwarding used buffers even with the unbind.
550          */
551         memset(&svq->svq_call, 0, sizeof(svq->svq_call));
552     } else {
553         event_notifier_init_fd(&svq->svq_call, call_fd);
554     }
555 }
556 
557 /**
558  * Get the shadow vq vring address.
559  * @svq: Shadow virtqueue
560  * @addr: Destination to store address
561  */
562 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
563                               struct vhost_vring_addr *addr)
564 {
565     addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
566     addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
567     addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
568 }
569 
570 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
571 {
572     size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
573     size_t avail_size = offsetof(vring_avail_t, ring) +
574                                              sizeof(uint16_t) * svq->vring.num;
575 
576     return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size());
577 }
578 
579 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
580 {
581     size_t used_size = offsetof(vring_used_t, ring) +
582                                     sizeof(vring_used_elem_t) * svq->vring.num;
583     return ROUND_UP(used_size, qemu_real_host_page_size());
584 }
585 
586 /**
587  * Set a new file descriptor for the guest to kick the SVQ and notify for avail
588  *
589  * @svq: The svq
590  * @svq_kick_fd: The svq kick fd
591  *
592  * Note that the SVQ will never close the old file descriptor.
593  */
594 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
595 {
596     EventNotifier *svq_kick = &svq->svq_kick;
597     bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
598     bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
599 
600     if (poll_stop) {
601         event_notifier_set_handler(svq_kick, NULL);
602     }
603 
604     event_notifier_init_fd(svq_kick, svq_kick_fd);
605     /*
606      * event_notifier_set_handler already checks for guest's notifications if
607      * they arrive at the new file descriptor in the switch, so there is no
608      * need to explicitly check for them.
609      */
610     if (poll_start) {
611         event_notifier_set(svq_kick);
612         event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
613     }
614 }
615 
616 /**
617  * Start the shadow virtqueue operation.
618  *
619  * @svq: Shadow Virtqueue
620  * @vdev: VirtIO device
621  * @vq: Virtqueue to shadow
622  */
623 void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
624                      VirtQueue *vq)
625 {
626     size_t desc_size, driver_size, device_size;
627 
628     svq->next_guest_avail_elem = NULL;
629     svq->shadow_avail_idx = 0;
630     svq->shadow_used_idx = 0;
631     svq->last_used_idx = 0;
632     svq->vdev = vdev;
633     svq->vq = vq;
634 
635     svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
636     driver_size = vhost_svq_driver_area_size(svq);
637     device_size = vhost_svq_device_area_size(svq);
638     svq->vring.desc = qemu_memalign(qemu_real_host_page_size(), driver_size);
639     desc_size = sizeof(vring_desc_t) * svq->vring.num;
640     svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
641     memset(svq->vring.desc, 0, driver_size);
642     svq->vring.used = qemu_memalign(qemu_real_host_page_size(), device_size);
643     memset(svq->vring.used, 0, device_size);
644     svq->desc_state = g_new0(SVQDescState, svq->vring.num);
645     svq->desc_next = g_new0(uint16_t, svq->vring.num);
646     for (unsigned i = 0; i < svq->vring.num - 1; i++) {
647         svq->desc_next[i] = cpu_to_le16(i + 1);
648     }
649 }
650 
651 /**
652  * Stop the shadow virtqueue operation.
653  * @svq: Shadow Virtqueue
654  */
655 void vhost_svq_stop(VhostShadowVirtqueue *svq)
656 {
657     vhost_svq_set_svq_kick_fd(svq, VHOST_FILE_UNBIND);
658     g_autofree VirtQueueElement *next_avail_elem = NULL;
659 
660     if (!svq->vq) {
661         return;
662     }
663 
664     /* Send all pending used descriptors to guest */
665     vhost_svq_flush(svq, false);
666 
667     for (unsigned i = 0; i < svq->vring.num; ++i) {
668         g_autofree VirtQueueElement *elem = NULL;
669         elem = g_steal_pointer(&svq->desc_state[i].elem);
670         if (elem) {
671             virtqueue_detach_element(svq->vq, elem, 0);
672         }
673     }
674 
675     next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
676     if (next_avail_elem) {
677         virtqueue_detach_element(svq->vq, next_avail_elem, 0);
678     }
679     svq->vq = NULL;
680     g_free(svq->desc_next);
681     g_free(svq->desc_state);
682     qemu_vfree(svq->vring.desc);
683     qemu_vfree(svq->vring.used);
684 }
685 
686 /**
687  * Creates vhost shadow virtqueue, and instructs the vhost device to use the
688  * shadow methods and file descriptors.
689  *
690  * @iova_tree: Tree to perform descriptors translations
691  * @ops: SVQ owner callbacks
692  * @ops_opaque: ops opaque pointer
693  *
694  * Returns the new virtqueue or NULL.
695  *
696  * In case of error, reason is reported through error_report.
697  */
698 VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree,
699                                     const VhostShadowVirtqueueOps *ops,
700                                     void *ops_opaque)
701 {
702     g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
703     int r;
704 
705     r = event_notifier_init(&svq->hdev_kick, 0);
706     if (r != 0) {
707         error_report("Couldn't create kick event notifier: %s (%d)",
708                      g_strerror(errno), errno);
709         goto err_init_hdev_kick;
710     }
711 
712     r = event_notifier_init(&svq->hdev_call, 0);
713     if (r != 0) {
714         error_report("Couldn't create call event notifier: %s (%d)",
715                      g_strerror(errno), errno);
716         goto err_init_hdev_call;
717     }
718 
719     event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
720     event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
721     svq->iova_tree = iova_tree;
722     svq->ops = ops;
723     svq->ops_opaque = ops_opaque;
724     return g_steal_pointer(&svq);
725 
726 err_init_hdev_call:
727     event_notifier_cleanup(&svq->hdev_kick);
728 
729 err_init_hdev_kick:
730     return NULL;
731 }
732 
733 /**
734  * Free the resources of the shadow virtqueue.
735  *
736  * @pvq: gpointer to SVQ so it can be used by autofree functions.
737  */
738 void vhost_svq_free(gpointer pvq)
739 {
740     VhostShadowVirtqueue *vq = pvq;
741     vhost_svq_stop(vq);
742     event_notifier_cleanup(&vq->hdev_kick);
743     event_notifier_set_handler(&vq->hdev_call, NULL);
744     event_notifier_cleanup(&vq->hdev_call);
745     g_free(vq);
746 }
747