1 /*
2  * vhost shadow virtqueue
3  *
4  * SPDX-FileCopyrightText: Red Hat, Inc. 2021
5  * SPDX-FileContributor: Author: Eugenio Pérez <eperezma@redhat.com>
6  *
7  * SPDX-License-Identifier: GPL-2.0-or-later
8  */
9 
10 #include "qemu/osdep.h"
11 #include "hw/virtio/vhost-shadow-virtqueue.h"
12 
13 #include "qemu/error-report.h"
14 #include "qapi/error.h"
15 #include "qemu/main-loop.h"
16 #include "qemu/log.h"
17 #include "qemu/memalign.h"
18 #include "linux-headers/linux/vhost.h"
19 
20 /**
21  * Validate the transport device features that both guests can use with the SVQ
22  * and SVQs can use with the device.
23  *
24  * @dev_features: The features
25  * @errp: Error pointer
26  */
27 bool vhost_svq_valid_features(uint64_t features, Error **errp)
28 {
29     bool ok = true;
30     uint64_t svq_features = features;
31 
32     for (uint64_t b = VIRTIO_TRANSPORT_F_START; b <= VIRTIO_TRANSPORT_F_END;
33          ++b) {
34         switch (b) {
35         case VIRTIO_F_ANY_LAYOUT:
36             continue;
37 
38         case VIRTIO_F_ACCESS_PLATFORM:
39             /* SVQ trust in the host's IOMMU to translate addresses */
40         case VIRTIO_F_VERSION_1:
41             /* SVQ trust that the guest vring is little endian */
42             if (!(svq_features & BIT_ULL(b))) {
43                 svq_features |= BIT_ULL(b);
44                 ok = false;
45             }
46             continue;
47 
48         default:
49             if (svq_features & BIT_ULL(b)) {
50                 svq_features &= ~BIT_ULL(b);
51                 ok = false;
52             }
53         }
54     }
55 
56     if (!ok) {
57         error_setg(errp, "SVQ Invalid device feature flags, offer: 0x%"PRIx64
58                          ", ok: 0x%"PRIx64, features, svq_features);
59     }
60     return ok;
61 }
62 
63 /**
64  * Number of descriptors that the SVQ can make available from the guest.
65  *
66  * @svq: The svq
67  */
68 static uint16_t vhost_svq_available_slots(const VhostShadowVirtqueue *svq)
69 {
70     return svq->vring.num - (svq->shadow_avail_idx - svq->shadow_used_idx);
71 }
72 
73 /**
74  * Translate addresses between the qemu's virtual address and the SVQ IOVA
75  *
76  * @svq: Shadow VirtQueue
77  * @vaddr: Translated IOVA addresses
78  * @iovec: Source qemu's VA addresses
79  * @num: Length of iovec and minimum length of vaddr
80  */
81 static bool vhost_svq_translate_addr(const VhostShadowVirtqueue *svq,
82                                      hwaddr *addrs, const struct iovec *iovec,
83                                      size_t num)
84 {
85     if (num == 0) {
86         return true;
87     }
88 
89     for (size_t i = 0; i < num; ++i) {
90         DMAMap needle = {
91             .translated_addr = (hwaddr)(uintptr_t)iovec[i].iov_base,
92             .size = iovec[i].iov_len,
93         };
94         Int128 needle_last, map_last;
95         size_t off;
96 
97         const DMAMap *map = vhost_iova_tree_find_iova(svq->iova_tree, &needle);
98         /*
99          * Map cannot be NULL since iova map contains all guest space and
100          * qemu already has a physical address mapped
101          */
102         if (unlikely(!map)) {
103             qemu_log_mask(LOG_GUEST_ERROR,
104                           "Invalid address 0x%"HWADDR_PRIx" given by guest",
105                           needle.translated_addr);
106             return false;
107         }
108 
109         off = needle.translated_addr - map->translated_addr;
110         addrs[i] = map->iova + off;
111 
112         needle_last = int128_add(int128_make64(needle.translated_addr),
113                                  int128_make64(iovec[i].iov_len));
114         map_last = int128_make64(map->translated_addr + map->size);
115         if (unlikely(int128_gt(needle_last, map_last))) {
116             qemu_log_mask(LOG_GUEST_ERROR,
117                           "Guest buffer expands over iova range");
118             return false;
119         }
120     }
121 
122     return true;
123 }
124 
125 static void vhost_vring_write_descs(VhostShadowVirtqueue *svq, hwaddr *sg,
126                                     const struct iovec *iovec, size_t num,
127                                     bool more_descs, bool write)
128 {
129     uint16_t i = svq->free_head, last = svq->free_head;
130     unsigned n;
131     uint16_t flags = write ? cpu_to_le16(VRING_DESC_F_WRITE) : 0;
132     vring_desc_t *descs = svq->vring.desc;
133 
134     if (num == 0) {
135         return;
136     }
137 
138     for (n = 0; n < num; n++) {
139         if (more_descs || (n + 1 < num)) {
140             descs[i].flags = flags | cpu_to_le16(VRING_DESC_F_NEXT);
141             descs[i].next = cpu_to_le16(svq->desc_next[i]);
142         } else {
143             descs[i].flags = flags;
144         }
145         descs[i].addr = cpu_to_le64(sg[n]);
146         descs[i].len = cpu_to_le32(iovec[n].iov_len);
147 
148         last = i;
149         i = cpu_to_le16(svq->desc_next[i]);
150     }
151 
152     svq->free_head = le16_to_cpu(svq->desc_next[last]);
153 }
154 
155 static bool vhost_svq_add_split(VhostShadowVirtqueue *svq,
156                                 VirtQueueElement *elem, unsigned *head)
157 {
158     unsigned avail_idx;
159     vring_avail_t *avail = svq->vring.avail;
160     bool ok;
161     g_autofree hwaddr *sgs = g_new(hwaddr, MAX(elem->out_num, elem->in_num));
162 
163     *head = svq->free_head;
164 
165     /* We need some descriptors here */
166     if (unlikely(!elem->out_num && !elem->in_num)) {
167         qemu_log_mask(LOG_GUEST_ERROR,
168                       "Guest provided element with no descriptors");
169         return false;
170     }
171 
172     ok = vhost_svq_translate_addr(svq, sgs, elem->out_sg, elem->out_num);
173     if (unlikely(!ok)) {
174         return false;
175     }
176     vhost_vring_write_descs(svq, sgs, elem->out_sg, elem->out_num,
177                             elem->in_num > 0, false);
178 
179 
180     ok = vhost_svq_translate_addr(svq, sgs, elem->in_sg, elem->in_num);
181     if (unlikely(!ok)) {
182         return false;
183     }
184 
185     vhost_vring_write_descs(svq, sgs, elem->in_sg, elem->in_num, false, true);
186 
187     /*
188      * Put the entry in the available array (but don't update avail->idx until
189      * they do sync).
190      */
191     avail_idx = svq->shadow_avail_idx & (svq->vring.num - 1);
192     avail->ring[avail_idx] = cpu_to_le16(*head);
193     svq->shadow_avail_idx++;
194 
195     /* Update the avail index after write the descriptor */
196     smp_wmb();
197     avail->idx = cpu_to_le16(svq->shadow_avail_idx);
198 
199     return true;
200 }
201 
202 /**
203  * Add an element to a SVQ.
204  *
205  * The caller must check that there is enough slots for the new element. It
206  * takes ownership of the element: In case of failure, it is free and the SVQ
207  * is considered broken.
208  */
209 static bool vhost_svq_add(VhostShadowVirtqueue *svq, VirtQueueElement *elem)
210 {
211     unsigned qemu_head;
212     bool ok = vhost_svq_add_split(svq, elem, &qemu_head);
213     if (unlikely(!ok)) {
214         g_free(elem);
215         return false;
216     }
217 
218     svq->ring_id_maps[qemu_head] = elem;
219     return true;
220 }
221 
222 static void vhost_svq_kick(VhostShadowVirtqueue *svq)
223 {
224     /*
225      * We need to expose the available array entries before checking the used
226      * flags
227      */
228     smp_mb();
229     if (svq->vring.used->flags & VRING_USED_F_NO_NOTIFY) {
230         return;
231     }
232 
233     event_notifier_set(&svq->hdev_kick);
234 }
235 
236 /**
237  * Forward available buffers.
238  *
239  * @svq: Shadow VirtQueue
240  *
241  * Note that this function does not guarantee that all guest's available
242  * buffers are available to the device in SVQ avail ring. The guest may have
243  * exposed a GPA / GIOVA contiguous buffer, but it may not be contiguous in
244  * qemu vaddr.
245  *
246  * If that happens, guest's kick notifications will be disabled until the
247  * device uses some buffers.
248  */
249 static void vhost_handle_guest_kick(VhostShadowVirtqueue *svq)
250 {
251     /* Clear event notifier */
252     event_notifier_test_and_clear(&svq->svq_kick);
253 
254     /* Forward to the device as many available buffers as possible */
255     do {
256         virtio_queue_set_notification(svq->vq, false);
257 
258         while (true) {
259             VirtQueueElement *elem;
260             bool ok;
261 
262             if (svq->next_guest_avail_elem) {
263                 elem = g_steal_pointer(&svq->next_guest_avail_elem);
264             } else {
265                 elem = virtqueue_pop(svq->vq, sizeof(*elem));
266             }
267 
268             if (!elem) {
269                 break;
270             }
271 
272             if (elem->out_num + elem->in_num > vhost_svq_available_slots(svq)) {
273                 /*
274                  * This condition is possible since a contiguous buffer in GPA
275                  * does not imply a contiguous buffer in qemu's VA
276                  * scatter-gather segments. If that happens, the buffer exposed
277                  * to the device needs to be a chain of descriptors at this
278                  * moment.
279                  *
280                  * SVQ cannot hold more available buffers if we are here:
281                  * queue the current guest descriptor and ignore further kicks
282                  * until some elements are used.
283                  */
284                 svq->next_guest_avail_elem = elem;
285                 return;
286             }
287 
288             ok = vhost_svq_add(svq, elem);
289             if (unlikely(!ok)) {
290                 /* VQ is broken, just return and ignore any other kicks */
291                 return;
292             }
293             vhost_svq_kick(svq);
294         }
295 
296         virtio_queue_set_notification(svq->vq, true);
297     } while (!virtio_queue_empty(svq->vq));
298 }
299 
300 /**
301  * Handle guest's kick.
302  *
303  * @n: guest kick event notifier, the one that guest set to notify svq.
304  */
305 static void vhost_handle_guest_kick_notifier(EventNotifier *n)
306 {
307     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue, svq_kick);
308     event_notifier_test_and_clear(n);
309     vhost_handle_guest_kick(svq);
310 }
311 
312 static bool vhost_svq_more_used(VhostShadowVirtqueue *svq)
313 {
314     if (svq->last_used_idx != svq->shadow_used_idx) {
315         return true;
316     }
317 
318     svq->shadow_used_idx = cpu_to_le16(svq->vring.used->idx);
319 
320     return svq->last_used_idx != svq->shadow_used_idx;
321 }
322 
323 /**
324  * Enable vhost device calls after disable them.
325  *
326  * @svq: The svq
327  *
328  * It returns false if there are pending used buffers from the vhost device,
329  * avoiding the possible races between SVQ checking for more work and enabling
330  * callbacks. True if SVQ used vring has no more pending buffers.
331  */
332 static bool vhost_svq_enable_notification(VhostShadowVirtqueue *svq)
333 {
334     svq->vring.avail->flags &= ~cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
335     /* Make sure the flag is written before the read of used_idx */
336     smp_mb();
337     return !vhost_svq_more_used(svq);
338 }
339 
340 static void vhost_svq_disable_notification(VhostShadowVirtqueue *svq)
341 {
342     svq->vring.avail->flags |= cpu_to_le16(VRING_AVAIL_F_NO_INTERRUPT);
343 }
344 
345 static uint16_t vhost_svq_last_desc_of_chain(const VhostShadowVirtqueue *svq,
346                                              uint16_t num, uint16_t i)
347 {
348     for (uint16_t j = 0; j < (num - 1); ++j) {
349         i = le16_to_cpu(svq->desc_next[i]);
350     }
351 
352     return i;
353 }
354 
355 static VirtQueueElement *vhost_svq_get_buf(VhostShadowVirtqueue *svq,
356                                            uint32_t *len)
357 {
358     const vring_used_t *used = svq->vring.used;
359     vring_used_elem_t used_elem;
360     uint16_t last_used, last_used_chain, num;
361 
362     if (!vhost_svq_more_used(svq)) {
363         return NULL;
364     }
365 
366     /* Only get used array entries after they have been exposed by dev */
367     smp_rmb();
368     last_used = svq->last_used_idx & (svq->vring.num - 1);
369     used_elem.id = le32_to_cpu(used->ring[last_used].id);
370     used_elem.len = le32_to_cpu(used->ring[last_used].len);
371 
372     svq->last_used_idx++;
373     if (unlikely(used_elem.id >= svq->vring.num)) {
374         qemu_log_mask(LOG_GUEST_ERROR, "Device %s says index %u is used",
375                       svq->vdev->name, used_elem.id);
376         return NULL;
377     }
378 
379     if (unlikely(!svq->ring_id_maps[used_elem.id])) {
380         qemu_log_mask(LOG_GUEST_ERROR,
381             "Device %s says index %u is used, but it was not available",
382             svq->vdev->name, used_elem.id);
383         return NULL;
384     }
385 
386     num = svq->ring_id_maps[used_elem.id]->in_num +
387           svq->ring_id_maps[used_elem.id]->out_num;
388     last_used_chain = vhost_svq_last_desc_of_chain(svq, num, used_elem.id);
389     svq->desc_next[last_used_chain] = svq->free_head;
390     svq->free_head = used_elem.id;
391 
392     *len = used_elem.len;
393     return g_steal_pointer(&svq->ring_id_maps[used_elem.id]);
394 }
395 
396 static void vhost_svq_flush(VhostShadowVirtqueue *svq,
397                             bool check_for_avail_queue)
398 {
399     VirtQueue *vq = svq->vq;
400 
401     /* Forward as many used buffers as possible. */
402     do {
403         unsigned i = 0;
404 
405         vhost_svq_disable_notification(svq);
406         while (true) {
407             uint32_t len;
408             g_autofree VirtQueueElement *elem = vhost_svq_get_buf(svq, &len);
409             if (!elem) {
410                 break;
411             }
412 
413             if (unlikely(i >= svq->vring.num)) {
414                 qemu_log_mask(LOG_GUEST_ERROR,
415                          "More than %u used buffers obtained in a %u size SVQ",
416                          i, svq->vring.num);
417                 virtqueue_fill(vq, elem, len, i);
418                 virtqueue_flush(vq, i);
419                 return;
420             }
421             virtqueue_fill(vq, elem, len, i++);
422         }
423 
424         virtqueue_flush(vq, i);
425         event_notifier_set(&svq->svq_call);
426 
427         if (check_for_avail_queue && svq->next_guest_avail_elem) {
428             /*
429              * Avail ring was full when vhost_svq_flush was called, so it's a
430              * good moment to make more descriptors available if possible.
431              */
432             vhost_handle_guest_kick(svq);
433         }
434     } while (!vhost_svq_enable_notification(svq));
435 }
436 
437 /**
438  * Forward used buffers.
439  *
440  * @n: hdev call event notifier, the one that device set to notify svq.
441  *
442  * Note that we are not making any buffers available in the loop, there is no
443  * way that it runs more than virtqueue size times.
444  */
445 static void vhost_svq_handle_call(EventNotifier *n)
446 {
447     VhostShadowVirtqueue *svq = container_of(n, VhostShadowVirtqueue,
448                                              hdev_call);
449     event_notifier_test_and_clear(n);
450     vhost_svq_flush(svq, true);
451 }
452 
453 /**
454  * Set the call notifier for the SVQ to call the guest
455  *
456  * @svq: Shadow virtqueue
457  * @call_fd: call notifier
458  *
459  * Called on BQL context.
460  */
461 void vhost_svq_set_svq_call_fd(VhostShadowVirtqueue *svq, int call_fd)
462 {
463     if (call_fd == VHOST_FILE_UNBIND) {
464         /*
465          * Fail event_notifier_set if called handling device call.
466          *
467          * SVQ still needs device notifications, since it needs to keep
468          * forwarding used buffers even with the unbind.
469          */
470         memset(&svq->svq_call, 0, sizeof(svq->svq_call));
471     } else {
472         event_notifier_init_fd(&svq->svq_call, call_fd);
473     }
474 }
475 
476 /**
477  * Get the shadow vq vring address.
478  * @svq: Shadow virtqueue
479  * @addr: Destination to store address
480  */
481 void vhost_svq_get_vring_addr(const VhostShadowVirtqueue *svq,
482                               struct vhost_vring_addr *addr)
483 {
484     addr->desc_user_addr = (uint64_t)(uintptr_t)svq->vring.desc;
485     addr->avail_user_addr = (uint64_t)(uintptr_t)svq->vring.avail;
486     addr->used_user_addr = (uint64_t)(uintptr_t)svq->vring.used;
487 }
488 
489 size_t vhost_svq_driver_area_size(const VhostShadowVirtqueue *svq)
490 {
491     size_t desc_size = sizeof(vring_desc_t) * svq->vring.num;
492     size_t avail_size = offsetof(vring_avail_t, ring) +
493                                              sizeof(uint16_t) * svq->vring.num;
494 
495     return ROUND_UP(desc_size + avail_size, qemu_real_host_page_size());
496 }
497 
498 size_t vhost_svq_device_area_size(const VhostShadowVirtqueue *svq)
499 {
500     size_t used_size = offsetof(vring_used_t, ring) +
501                                     sizeof(vring_used_elem_t) * svq->vring.num;
502     return ROUND_UP(used_size, qemu_real_host_page_size());
503 }
504 
505 /**
506  * Set a new file descriptor for the guest to kick the SVQ and notify for avail
507  *
508  * @svq: The svq
509  * @svq_kick_fd: The svq kick fd
510  *
511  * Note that the SVQ will never close the old file descriptor.
512  */
513 void vhost_svq_set_svq_kick_fd(VhostShadowVirtqueue *svq, int svq_kick_fd)
514 {
515     EventNotifier *svq_kick = &svq->svq_kick;
516     bool poll_stop = VHOST_FILE_UNBIND != event_notifier_get_fd(svq_kick);
517     bool poll_start = svq_kick_fd != VHOST_FILE_UNBIND;
518 
519     if (poll_stop) {
520         event_notifier_set_handler(svq_kick, NULL);
521     }
522 
523     /*
524      * event_notifier_set_handler already checks for guest's notifications if
525      * they arrive at the new file descriptor in the switch, so there is no
526      * need to explicitly check for them.
527      */
528     if (poll_start) {
529         event_notifier_init_fd(svq_kick, svq_kick_fd);
530         event_notifier_set(svq_kick);
531         event_notifier_set_handler(svq_kick, vhost_handle_guest_kick_notifier);
532     }
533 }
534 
535 /**
536  * Start the shadow virtqueue operation.
537  *
538  * @svq: Shadow Virtqueue
539  * @vdev: VirtIO device
540  * @vq: Virtqueue to shadow
541  */
542 void vhost_svq_start(VhostShadowVirtqueue *svq, VirtIODevice *vdev,
543                      VirtQueue *vq)
544 {
545     size_t desc_size, driver_size, device_size;
546 
547     svq->next_guest_avail_elem = NULL;
548     svq->shadow_avail_idx = 0;
549     svq->shadow_used_idx = 0;
550     svq->last_used_idx = 0;
551     svq->vdev = vdev;
552     svq->vq = vq;
553 
554     svq->vring.num = virtio_queue_get_num(vdev, virtio_get_queue_index(vq));
555     driver_size = vhost_svq_driver_area_size(svq);
556     device_size = vhost_svq_device_area_size(svq);
557     svq->vring.desc = qemu_memalign(qemu_real_host_page_size(), driver_size);
558     desc_size = sizeof(vring_desc_t) * svq->vring.num;
559     svq->vring.avail = (void *)((char *)svq->vring.desc + desc_size);
560     memset(svq->vring.desc, 0, driver_size);
561     svq->vring.used = qemu_memalign(qemu_real_host_page_size(), device_size);
562     memset(svq->vring.used, 0, device_size);
563     svq->ring_id_maps = g_new0(VirtQueueElement *, svq->vring.num);
564     svq->desc_next = g_new0(uint16_t, svq->vring.num);
565     for (unsigned i = 0; i < svq->vring.num - 1; i++) {
566         svq->desc_next[i] = cpu_to_le16(i + 1);
567     }
568 }
569 
570 /**
571  * Stop the shadow virtqueue operation.
572  * @svq: Shadow Virtqueue
573  */
574 void vhost_svq_stop(VhostShadowVirtqueue *svq)
575 {
576     event_notifier_set_handler(&svq->svq_kick, NULL);
577     g_autofree VirtQueueElement *next_avail_elem = NULL;
578 
579     if (!svq->vq) {
580         return;
581     }
582 
583     /* Send all pending used descriptors to guest */
584     vhost_svq_flush(svq, false);
585 
586     for (unsigned i = 0; i < svq->vring.num; ++i) {
587         g_autofree VirtQueueElement *elem = NULL;
588         elem = g_steal_pointer(&svq->ring_id_maps[i]);
589         if (elem) {
590             virtqueue_detach_element(svq->vq, elem, 0);
591         }
592     }
593 
594     next_avail_elem = g_steal_pointer(&svq->next_guest_avail_elem);
595     if (next_avail_elem) {
596         virtqueue_detach_element(svq->vq, next_avail_elem, 0);
597     }
598     svq->vq = NULL;
599     g_free(svq->desc_next);
600     g_free(svq->ring_id_maps);
601     qemu_vfree(svq->vring.desc);
602     qemu_vfree(svq->vring.used);
603 }
604 
605 /**
606  * Creates vhost shadow virtqueue, and instructs the vhost device to use the
607  * shadow methods and file descriptors.
608  *
609  * @iova_tree: Tree to perform descriptors translations
610  *
611  * Returns the new virtqueue or NULL.
612  *
613  * In case of error, reason is reported through error_report.
614  */
615 VhostShadowVirtqueue *vhost_svq_new(VhostIOVATree *iova_tree)
616 {
617     g_autofree VhostShadowVirtqueue *svq = g_new0(VhostShadowVirtqueue, 1);
618     int r;
619 
620     r = event_notifier_init(&svq->hdev_kick, 0);
621     if (r != 0) {
622         error_report("Couldn't create kick event notifier: %s (%d)",
623                      g_strerror(errno), errno);
624         goto err_init_hdev_kick;
625     }
626 
627     r = event_notifier_init(&svq->hdev_call, 0);
628     if (r != 0) {
629         error_report("Couldn't create call event notifier: %s (%d)",
630                      g_strerror(errno), errno);
631         goto err_init_hdev_call;
632     }
633 
634     event_notifier_init_fd(&svq->svq_kick, VHOST_FILE_UNBIND);
635     event_notifier_set_handler(&svq->hdev_call, vhost_svq_handle_call);
636     svq->iova_tree = iova_tree;
637     return g_steal_pointer(&svq);
638 
639 err_init_hdev_call:
640     event_notifier_cleanup(&svq->hdev_kick);
641 
642 err_init_hdev_kick:
643     return NULL;
644 }
645 
646 /**
647  * Free the resources of the shadow virtqueue.
648  *
649  * @pvq: gpointer to SVQ so it can be used by autofree functions.
650  */
651 void vhost_svq_free(gpointer pvq)
652 {
653     VhostShadowVirtqueue *vq = pvq;
654     vhost_svq_stop(vq);
655     event_notifier_cleanup(&vq->hdev_kick);
656     event_notifier_set_handler(&vq->hdev_call, NULL);
657     event_notifier_cleanup(&vq->hdev_call);
658     g_free(vq);
659 }
660