xref: /openbmc/qemu/hw/virtio/vhost-vdpa.c (revision 5edb02e8004c2f1d5026a02cd9378046973f47af)
1 /*
2  * vhost-vdpa
3  *
4  *  Copyright(c) 2017-2018 Intel Corporation.
5  *  Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include <linux/vhost.h>
14 #include <linux/vfio.h>
15 #include <sys/eventfd.h>
16 #include <sys/ioctl.h>
17 #include "exec/target_page.h"
18 #include "hw/virtio/vhost.h"
19 #include "hw/virtio/vhost-backend.h"
20 #include "hw/virtio/virtio-net.h"
21 #include "hw/virtio/vhost-shadow-virtqueue.h"
22 #include "hw/virtio/vhost-vdpa.h"
23 #include "exec/address-spaces.h"
24 #include "migration/blocker.h"
25 #include "qemu/cutils.h"
26 #include "qemu/main-loop.h"
27 #include "trace.h"
28 #include "qapi/error.h"
29 
30 /*
31  * Return one past the end of the end of section. Be careful with uint64_t
32  * conversions!
33  */
34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section,
35                                      int page_mask)
36 {
37     Int128 llend = int128_make64(section->offset_within_address_space);
38     llend = int128_add(llend, section->size);
39     llend = int128_and(llend, int128_exts64(page_mask));
40 
41     return llend;
42 }
43 
44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section,
45                                                 uint64_t iova_min,
46                                                 uint64_t iova_max,
47                                                 int page_mask)
48 {
49     Int128 llend;
50 
51     if ((!memory_region_is_ram(section->mr) &&
52          !memory_region_is_iommu(section->mr)) ||
53         memory_region_is_protected(section->mr) ||
54         /* vhost-vDPA doesn't allow MMIO to be mapped  */
55         memory_region_is_ram_device(section->mr)) {
56         return true;
57     }
58 
59     if (section->offset_within_address_space < iova_min) {
60         error_report("RAM section out of device range (min=0x%" PRIx64
61                      ", addr=0x%" HWADDR_PRIx ")",
62                      iova_min, section->offset_within_address_space);
63         return true;
64     }
65     /*
66      * While using vIOMMU, sometimes the section will be larger than iova_max,
67      * but the memory that actually maps is smaller, so move the check to
68      * function vhost_vdpa_iommu_map_notify(). That function will use the actual
69      * size that maps to the kernel
70      */
71 
72     if (!memory_region_is_iommu(section->mr)) {
73         llend = vhost_vdpa_section_end(section, page_mask);
74         if (int128_gt(llend, int128_make64(iova_max))) {
75             error_report("RAM section out of device range (max=0x%" PRIx64
76                          ", end addr=0x%" PRIx64 ")",
77                          iova_max, int128_get64(llend));
78             return true;
79         }
80     }
81 
82     return false;
83 }
84 
85 /*
86  * The caller must set asid = 0 if the device does not support asid.
87  * This is not an ABI break since it is set to 0 by the initializer anyway.
88  */
89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
90                        hwaddr size, void *vaddr, bool readonly)
91 {
92     struct vhost_msg_v2 msg = {};
93     int fd = v->device_fd;
94     int ret = 0;
95 
96     msg.type = v->msg_type;
97     msg.asid = asid;
98     msg.iotlb.iova = iova;
99     msg.iotlb.size = size;
100     msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
101     msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
102     msg.iotlb.type = VHOST_IOTLB_UPDATE;
103 
104     trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova,
105                              msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm,
106                              msg.iotlb.type);
107 
108     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
109         error_report("failed to write, fd=%d, errno=%d (%s)",
110             fd, errno, strerror(errno));
111         return -EIO ;
112     }
113 
114     return ret;
115 }
116 
117 /*
118  * The caller must set asid = 0 if the device does not support asid.
119  * This is not an ABI break since it is set to 0 by the initializer anyway.
120  */
121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
122                          hwaddr size)
123 {
124     struct vhost_msg_v2 msg = {};
125     int fd = v->device_fd;
126     int ret = 0;
127 
128     msg.type = v->msg_type;
129     msg.asid = asid;
130     msg.iotlb.iova = iova;
131     msg.iotlb.size = size;
132     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
133 
134     trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova,
135                                msg.iotlb.size, msg.iotlb.type);
136 
137     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
138         error_report("failed to write, fd=%d, errno=%d (%s)",
139             fd, errno, strerror(errno));
140         return -EIO ;
141     }
142 
143     return ret;
144 }
145 
146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v)
147 {
148     int fd = v->device_fd;
149     struct vhost_msg_v2 msg = {
150         .type = v->msg_type,
151         .iotlb.type = VHOST_IOTLB_BATCH_BEGIN,
152     };
153 
154     trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type);
155     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
156         error_report("failed to write, fd=%d, errno=%d (%s)",
157                      fd, errno, strerror(errno));
158     }
159 }
160 
161 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v)
162 {
163     if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) &&
164         !v->iotlb_batch_begin_sent) {
165         vhost_vdpa_listener_begin_batch(v);
166     }
167 
168     v->iotlb_batch_begin_sent = true;
169 }
170 
171 static void vhost_vdpa_listener_commit(MemoryListener *listener)
172 {
173     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
174     struct vhost_dev *dev = v->dev;
175     struct vhost_msg_v2 msg = {};
176     int fd = v->device_fd;
177 
178     if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
179         return;
180     }
181 
182     if (!v->iotlb_batch_begin_sent) {
183         return;
184     }
185 
186     msg.type = v->msg_type;
187     msg.iotlb.type = VHOST_IOTLB_BATCH_END;
188 
189     trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type);
190     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
191         error_report("failed to write, fd=%d, errno=%d (%s)",
192                      fd, errno, strerror(errno));
193     }
194 
195     v->iotlb_batch_begin_sent = false;
196 }
197 
198 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
199 {
200     struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n);
201 
202     hwaddr iova = iotlb->iova + iommu->iommu_offset;
203     struct vhost_vdpa *v = iommu->dev;
204     void *vaddr;
205     int ret;
206     Int128 llend;
207 
208     if (iotlb->target_as != &address_space_memory) {
209         error_report("Wrong target AS \"%s\", only system memory is allowed",
210                      iotlb->target_as->name ? iotlb->target_as->name : "none");
211         return;
212     }
213     RCU_READ_LOCK_GUARD();
214     /* check if RAM section out of device range */
215     llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova));
216     if (int128_gt(llend, int128_make64(v->iova_range.last))) {
217         error_report("RAM section out of device range (max=0x%" PRIx64
218                      ", end addr=0x%" PRIx64 ")",
219                      v->iova_range.last, int128_get64(llend));
220         return;
221     }
222 
223     if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
224         bool read_only;
225 
226         if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) {
227             return;
228         }
229         ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
230                                  iotlb->addr_mask + 1, vaddr, read_only);
231         if (ret) {
232             error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", "
233                          "0x%" HWADDR_PRIx ", %p) = %d (%m)",
234                          v, iova, iotlb->addr_mask + 1, vaddr, ret);
235         }
236     } else {
237         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
238                                    iotlb->addr_mask + 1);
239         if (ret) {
240             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
241                          "0x%" HWADDR_PRIx ") = %d (%m)",
242                          v, iova, iotlb->addr_mask + 1, ret);
243         }
244     }
245 }
246 
247 static void vhost_vdpa_iommu_region_add(MemoryListener *listener,
248                                         MemoryRegionSection *section)
249 {
250     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
251 
252     struct vdpa_iommu *iommu;
253     Int128 end;
254     int iommu_idx;
255     IOMMUMemoryRegion *iommu_mr;
256     int ret;
257 
258     iommu_mr = IOMMU_MEMORY_REGION(section->mr);
259 
260     iommu = g_malloc0(sizeof(*iommu));
261     end = int128_add(int128_make64(section->offset_within_region),
262                      section->size);
263     end = int128_sub(end, int128_one());
264     iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
265                                                    MEMTXATTRS_UNSPECIFIED);
266     iommu->iommu_mr = iommu_mr;
267     iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify,
268                         IOMMU_NOTIFIER_IOTLB_EVENTS,
269                         section->offset_within_region,
270                         int128_get64(end),
271                         iommu_idx);
272     iommu->iommu_offset = section->offset_within_address_space -
273                           section->offset_within_region;
274     iommu->dev = v;
275 
276     ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
277     if (ret) {
278         g_free(iommu);
279         return;
280     }
281 
282     QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next);
283     memory_region_iommu_replay(iommu->iommu_mr, &iommu->n);
284 
285     return;
286 }
287 
288 static void vhost_vdpa_iommu_region_del(MemoryListener *listener,
289                                         MemoryRegionSection *section)
290 {
291     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
292 
293     struct vdpa_iommu *iommu;
294 
295     QLIST_FOREACH(iommu, &v->iommu_list, iommu_next)
296     {
297         if (MEMORY_REGION(iommu->iommu_mr) == section->mr &&
298             iommu->n.start == section->offset_within_region) {
299             memory_region_unregister_iommu_notifier(section->mr, &iommu->n);
300             QLIST_REMOVE(iommu, iommu_next);
301             g_free(iommu);
302             break;
303         }
304     }
305 }
306 
307 static void vhost_vdpa_listener_region_add(MemoryListener *listener,
308                                            MemoryRegionSection *section)
309 {
310     DMAMap mem_region = {};
311     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
312     hwaddr iova;
313     Int128 llend, llsize;
314     void *vaddr;
315     int ret;
316     int page_size = qemu_target_page_size();
317     int page_mask = -page_size;
318 
319     if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first,
320                                             v->iova_range.last, page_mask)) {
321         return;
322     }
323     if (memory_region_is_iommu(section->mr)) {
324         vhost_vdpa_iommu_region_add(listener, section);
325         return;
326     }
327 
328     if (unlikely((section->offset_within_address_space & ~page_mask) !=
329                  (section->offset_within_region & ~page_mask))) {
330         trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name,
331                        section->offset_within_address_space & ~page_mask,
332                        section->offset_within_region & ~page_mask);
333         return;
334     }
335 
336     iova = ROUND_UP(section->offset_within_address_space, page_size);
337     llend = vhost_vdpa_section_end(section, page_mask);
338     if (int128_ge(int128_make64(iova), llend)) {
339         return;
340     }
341 
342     memory_region_ref(section->mr);
343 
344     /* Here we assume that memory_region_is_ram(section->mr)==true */
345 
346     vaddr = memory_region_get_ram_ptr(section->mr) +
347             section->offset_within_region +
348             (iova - section->offset_within_address_space);
349 
350     trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend),
351                                          vaddr, section->readonly);
352 
353     llsize = int128_sub(llend, int128_make64(iova));
354     if (v->shadow_data) {
355         int r;
356 
357         mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
358         mem_region.size = int128_get64(llsize) - 1,
359         mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
360 
361         r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region);
362         if (unlikely(r != IOVA_OK)) {
363             error_report("Can't allocate a mapping (%d)", r);
364             goto fail;
365         }
366 
367         iova = mem_region.iova;
368     }
369 
370     vhost_vdpa_iotlb_batch_begin_once(v);
371     ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
372                              int128_get64(llsize), vaddr, section->readonly);
373     if (ret) {
374         error_report("vhost vdpa map fail!");
375         goto fail_map;
376     }
377 
378     return;
379 
380 fail_map:
381     if (v->shadow_data) {
382         vhost_iova_tree_remove(v->shared->iova_tree, mem_region);
383     }
384 
385 fail:
386     /*
387      * On the initfn path, store the first error in the container so we
388      * can gracefully fail.  Runtime, there's not much we can do other
389      * than throw a hardware error.
390      */
391     error_report("vhost-vdpa: DMA mapping failed, unable to continue");
392     return;
393 
394 }
395 
396 static void vhost_vdpa_listener_region_del(MemoryListener *listener,
397                                            MemoryRegionSection *section)
398 {
399     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
400     hwaddr iova;
401     Int128 llend, llsize;
402     int ret;
403     int page_size = qemu_target_page_size();
404     int page_mask = -page_size;
405 
406     if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first,
407                                             v->iova_range.last, page_mask)) {
408         return;
409     }
410     if (memory_region_is_iommu(section->mr)) {
411         vhost_vdpa_iommu_region_del(listener, section);
412     }
413 
414     if (unlikely((section->offset_within_address_space & ~page_mask) !=
415                  (section->offset_within_region & ~page_mask))) {
416         trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name,
417                        section->offset_within_address_space & ~page_mask,
418                        section->offset_within_region & ~page_mask);
419         return;
420     }
421 
422     iova = ROUND_UP(section->offset_within_address_space, page_size);
423     llend = vhost_vdpa_section_end(section, page_mask);
424 
425     trace_vhost_vdpa_listener_region_del(v, iova,
426         int128_get64(int128_sub(llend, int128_one())));
427 
428     if (int128_ge(int128_make64(iova), llend)) {
429         return;
430     }
431 
432     llsize = int128_sub(llend, int128_make64(iova));
433 
434     if (v->shadow_data) {
435         const DMAMap *result;
436         const void *vaddr = memory_region_get_ram_ptr(section->mr) +
437             section->offset_within_region +
438             (iova - section->offset_within_address_space);
439         DMAMap mem_region = {
440             .translated_addr = (hwaddr)(uintptr_t)vaddr,
441             .size = int128_get64(llsize) - 1,
442         };
443 
444         result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region);
445         if (!result) {
446             /* The memory listener map wasn't mapped */
447             return;
448         }
449         iova = result->iova;
450         vhost_iova_tree_remove(v->shared->iova_tree, *result);
451     }
452     vhost_vdpa_iotlb_batch_begin_once(v);
453     /*
454      * The unmap ioctl doesn't accept a full 64-bit. need to check it
455      */
456     if (int128_eq(llsize, int128_2_64())) {
457         llsize = int128_rshift(llsize, 1);
458         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
459                                    int128_get64(llsize));
460 
461         if (ret) {
462             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
463                          "0x%" HWADDR_PRIx ") = %d (%m)",
464                          v, iova, int128_get64(llsize), ret);
465         }
466         iova += int128_get64(llsize);
467     }
468     ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
469                                int128_get64(llsize));
470 
471     if (ret) {
472         error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
473                      "0x%" HWADDR_PRIx ") = %d (%m)",
474                      v, iova, int128_get64(llsize), ret);
475     }
476 
477     memory_region_unref(section->mr);
478 }
479 /*
480  * IOTLB API is used by vhost-vdpa which requires incremental updating
481  * of the mapping. So we can not use generic vhost memory listener which
482  * depends on the addnop().
483  */
484 static const MemoryListener vhost_vdpa_memory_listener = {
485     .name = "vhost-vdpa",
486     .commit = vhost_vdpa_listener_commit,
487     .region_add = vhost_vdpa_listener_region_add,
488     .region_del = vhost_vdpa_listener_region_del,
489 };
490 
491 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
492                              void *arg)
493 {
494     struct vhost_vdpa *v = dev->opaque;
495     int fd = v->device_fd;
496     int ret;
497 
498     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
499 
500     ret = ioctl(fd, request, arg);
501     return ret < 0 ? -errno : ret;
502 }
503 
504 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
505 {
506     uint8_t s;
507     int ret;
508 
509     trace_vhost_vdpa_add_status(dev, status);
510     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
511     if (ret < 0) {
512         return ret;
513     }
514     if ((s & status) == status) {
515         /* Don't set bits already set */
516         return 0;
517     }
518 
519     s |= status;
520 
521     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
522     if (ret < 0) {
523         return ret;
524     }
525 
526     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
527     if (ret < 0) {
528         return ret;
529     }
530 
531     if (!(s & status)) {
532         return -EIO;
533     }
534 
535     return 0;
536 }
537 
538 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range)
539 {
540     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
541 
542     return ret < 0 ? -errno : 0;
543 }
544 
545 /*
546  * The use of this function is for requests that only need to be
547  * applied once. Typically such request occurs at the beginning
548  * of operation, and before setting up queues. It should not be
549  * used for request that performs operation until all queues are
550  * set, which would need to check dev->vq_index_end instead.
551  */
552 static bool vhost_vdpa_first_dev(struct vhost_dev *dev)
553 {
554     struct vhost_vdpa *v = dev->opaque;
555 
556     return v->index == 0;
557 }
558 
559 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
560                                        uint64_t *features)
561 {
562     int ret;
563 
564     ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
565     trace_vhost_vdpa_get_features(dev, *features);
566     return ret;
567 }
568 
569 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v)
570 {
571     g_autoptr(GPtrArray) shadow_vqs = NULL;
572 
573     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
574     for (unsigned n = 0; n < hdev->nvqs; ++n) {
575         VhostShadowVirtqueue *svq;
576 
577         svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque);
578         g_ptr_array_add(shadow_vqs, svq);
579     }
580 
581     v->shadow_vqs = g_steal_pointer(&shadow_vqs);
582 }
583 
584 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
585 {
586     struct vhost_vdpa *v;
587     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
588     trace_vhost_vdpa_init(dev, opaque);
589     int ret;
590 
591     v = opaque;
592     v->dev = dev;
593     dev->opaque =  opaque ;
594     v->listener = vhost_vdpa_memory_listener;
595     v->msg_type = VHOST_IOTLB_MSG_V2;
596     vhost_vdpa_init_svq(dev, v);
597 
598     error_propagate(&dev->migration_blocker, v->migration_blocker);
599     if (!vhost_vdpa_first_dev(dev)) {
600         return 0;
601     }
602 
603     /*
604      * If dev->shadow_vqs_enabled at initialization that means the device has
605      * been started with x-svq=on, so don't block migration
606      */
607     if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) {
608         /* We don't have dev->features yet */
609         uint64_t features;
610         ret = vhost_vdpa_get_dev_features(dev, &features);
611         if (unlikely(ret)) {
612             error_setg_errno(errp, -ret, "Could not get device features");
613             return ret;
614         }
615         vhost_svq_valid_features(features, &dev->migration_blocker);
616     }
617 
618     /*
619      * Similar to VFIO, we end up pinning all guest memory and have to
620      * disable discarding of RAM.
621      */
622     ret = ram_block_discard_disable(true);
623     if (ret) {
624         error_report("Cannot set discarding of RAM broken");
625         return ret;
626     }
627 
628     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
629                                VIRTIO_CONFIG_S_DRIVER);
630 
631     return 0;
632 }
633 
634 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
635                                             int queue_index)
636 {
637     size_t page_size = qemu_real_host_page_size();
638     struct vhost_vdpa *v = dev->opaque;
639     VirtIODevice *vdev = dev->vdev;
640     VhostVDPAHostNotifier *n;
641 
642     n = &v->notifier[queue_index];
643 
644     if (n->addr) {
645         virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
646         object_unparent(OBJECT(&n->mr));
647         munmap(n->addr, page_size);
648         n->addr = NULL;
649     }
650 }
651 
652 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
653 {
654     size_t page_size = qemu_real_host_page_size();
655     struct vhost_vdpa *v = dev->opaque;
656     VirtIODevice *vdev = dev->vdev;
657     VhostVDPAHostNotifier *n;
658     int fd = v->device_fd;
659     void *addr;
660     char *name;
661 
662     vhost_vdpa_host_notifier_uninit(dev, queue_index);
663 
664     n = &v->notifier[queue_index];
665 
666     addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
667                 queue_index * page_size);
668     if (addr == MAP_FAILED) {
669         goto err;
670     }
671 
672     name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
673                            v, queue_index);
674     memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
675                                       page_size, addr);
676     g_free(name);
677 
678     if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
679         object_unparent(OBJECT(&n->mr));
680         munmap(addr, page_size);
681         goto err;
682     }
683     n->addr = addr;
684 
685     return 0;
686 
687 err:
688     return -1;
689 }
690 
691 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
692 {
693     int i;
694 
695     /*
696      * Pack all the changes to the memory regions in a single
697      * transaction to avoid a few updating of the address space
698      * topology.
699      */
700     memory_region_transaction_begin();
701 
702     for (i = dev->vq_index; i < dev->vq_index + n; i++) {
703         vhost_vdpa_host_notifier_uninit(dev, i);
704     }
705 
706     memory_region_transaction_commit();
707 }
708 
709 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
710 {
711     struct vhost_vdpa *v = dev->opaque;
712     int i;
713 
714     if (v->shadow_vqs_enabled) {
715         /* FIXME SVQ is not compatible with host notifiers mr */
716         return;
717     }
718 
719     /*
720      * Pack all the changes to the memory regions in a single
721      * transaction to avoid a few updating of the address space
722      * topology.
723      */
724     memory_region_transaction_begin();
725 
726     for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
727         if (vhost_vdpa_host_notifier_init(dev, i)) {
728             vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index);
729             break;
730         }
731     }
732 
733     memory_region_transaction_commit();
734 }
735 
736 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
737 {
738     struct vhost_vdpa *v = dev->opaque;
739     size_t idx;
740 
741     for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
742         vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
743     }
744     g_ptr_array_free(v->shadow_vqs, true);
745 }
746 
747 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
748 {
749     struct vhost_vdpa *v;
750     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
751     v = dev->opaque;
752     trace_vhost_vdpa_cleanup(dev, v);
753     if (vhost_vdpa_first_dev(dev)) {
754         ram_block_discard_disable(false);
755     }
756 
757     vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
758     memory_listener_unregister(&v->listener);
759     vhost_vdpa_svq_cleanup(dev);
760 
761     dev->opaque = NULL;
762 
763     return 0;
764 }
765 
766 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
767 {
768     trace_vhost_vdpa_memslots_limit(dev, INT_MAX);
769     return INT_MAX;
770 }
771 
772 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
773                                     struct vhost_memory *mem)
774 {
775     if (!vhost_vdpa_first_dev(dev)) {
776         return 0;
777     }
778 
779     trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding);
780     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) &&
781         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) {
782         int i;
783         for (i = 0; i < mem->nregions; i++) {
784             trace_vhost_vdpa_dump_regions(dev, i,
785                                           mem->regions[i].guest_phys_addr,
786                                           mem->regions[i].memory_size,
787                                           mem->regions[i].userspace_addr,
788                                           mem->regions[i].flags_padding);
789         }
790     }
791     if (mem->padding) {
792         return -EINVAL;
793     }
794 
795     return 0;
796 }
797 
798 static int vhost_vdpa_set_features(struct vhost_dev *dev,
799                                    uint64_t features)
800 {
801     struct vhost_vdpa *v = dev->opaque;
802     int ret;
803 
804     if (!vhost_vdpa_first_dev(dev)) {
805         return 0;
806     }
807 
808     if (v->shadow_vqs_enabled) {
809         if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
810             /*
811              * QEMU is just trying to enable or disable logging. SVQ handles
812              * this sepparately, so no need to forward this.
813              */
814             v->acked_features = features;
815             return 0;
816         }
817 
818         v->acked_features = features;
819 
820         /* We must not ack _F_LOG if SVQ is enabled */
821         features &= ~BIT_ULL(VHOST_F_LOG_ALL);
822     }
823 
824     trace_vhost_vdpa_set_features(dev, features);
825     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
826     if (ret) {
827         return ret;
828     }
829 
830     return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
831 }
832 
833 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev)
834 {
835     uint64_t features;
836     uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 |
837         0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH |
838         0x1ULL << VHOST_BACKEND_F_IOTLB_ASID |
839         0x1ULL << VHOST_BACKEND_F_SUSPEND;
840     int r;
841 
842     if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
843         return -EFAULT;
844     }
845 
846     features &= f;
847 
848     if (vhost_vdpa_first_dev(dev)) {
849         r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features);
850         if (r) {
851             return -EFAULT;
852         }
853     }
854 
855     dev->backend_cap = features;
856 
857     return 0;
858 }
859 
860 static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
861                                     uint32_t *device_id)
862 {
863     int ret;
864     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
865     trace_vhost_vdpa_get_device_id(dev, *device_id);
866     return ret;
867 }
868 
869 static int vhost_vdpa_reset_device(struct vhost_dev *dev)
870 {
871     struct vhost_vdpa *v = dev->opaque;
872     int ret;
873     uint8_t status = 0;
874 
875     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
876     trace_vhost_vdpa_reset_device(dev);
877     v->suspended = false;
878     return ret;
879 }
880 
881 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
882 {
883     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
884 
885     trace_vhost_vdpa_get_vq_index(dev, idx, idx);
886     return idx;
887 }
888 
889 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx)
890 {
891     struct vhost_dev *dev = v->dev;
892     struct vhost_vring_state state = {
893         .index = idx,
894         .num = 1,
895     };
896     int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
897 
898     trace_vhost_vdpa_set_vring_ready(dev, idx, r);
899     return r;
900 }
901 
902 static int vhost_vdpa_set_config_call(struct vhost_dev *dev,
903                                        int fd)
904 {
905     trace_vhost_vdpa_set_config_call(dev, fd);
906     return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd);
907 }
908 
909 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config,
910                                    uint32_t config_len)
911 {
912     int b, len;
913     char line[QEMU_HEXDUMP_LINE_LEN];
914 
915     for (b = 0; b < config_len; b += 16) {
916         len = config_len - b;
917         qemu_hexdump_line(line, b, config, len, false);
918         trace_vhost_vdpa_dump_config(dev, line);
919     }
920 }
921 
922 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
923                                    uint32_t offset, uint32_t size,
924                                    uint32_t flags)
925 {
926     struct vhost_vdpa_config *config;
927     int ret;
928     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
929 
930     trace_vhost_vdpa_set_config(dev, offset, size, flags);
931     config = g_malloc(size + config_size);
932     config->off = offset;
933     config->len = size;
934     memcpy(config->buf, data, size);
935     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) &&
936         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
937         vhost_vdpa_dump_config(dev, data, size);
938     }
939     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
940     g_free(config);
941     return ret;
942 }
943 
944 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
945                                    uint32_t config_len, Error **errp)
946 {
947     struct vhost_vdpa_config *v_config;
948     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
949     int ret;
950 
951     trace_vhost_vdpa_get_config(dev, config, config_len);
952     v_config = g_malloc(config_len + config_size);
953     v_config->len = config_len;
954     v_config->off = 0;
955     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
956     memcpy(config, v_config->buf, config_len);
957     g_free(v_config);
958     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) &&
959         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
960         vhost_vdpa_dump_config(dev, config, config_len);
961     }
962     return ret;
963  }
964 
965 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
966                                          struct vhost_vring_state *ring)
967 {
968     trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
969     return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
970 }
971 
972 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
973                                          struct vhost_vring_file *file)
974 {
975     trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
976     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
977 }
978 
979 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
980                                          struct vhost_vring_file *file)
981 {
982     trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
983     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
984 }
985 
986 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
987                                          struct vhost_vring_addr *addr)
988 {
989     trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
990                                 addr->desc_user_addr, addr->used_user_addr,
991                                 addr->avail_user_addr,
992                                 addr->log_guest_addr);
993 
994     return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
995 
996 }
997 
998 /**
999  * Set the shadow virtqueue descriptors to the device
1000  *
1001  * @dev: The vhost device model
1002  * @svq: The shadow virtqueue
1003  * @idx: The index of the virtqueue in the vhost device
1004  * @errp: Error
1005  *
1006  * Note that this function does not rewind kick file descriptor if cannot set
1007  * call one.
1008  */
1009 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
1010                                   VhostShadowVirtqueue *svq, unsigned idx,
1011                                   Error **errp)
1012 {
1013     struct vhost_vring_file file = {
1014         .index = dev->vq_index + idx,
1015     };
1016     const EventNotifier *event_notifier = &svq->hdev_kick;
1017     int r;
1018 
1019     r = event_notifier_init(&svq->hdev_kick, 0);
1020     if (r != 0) {
1021         error_setg_errno(errp, -r, "Couldn't create kick event notifier");
1022         goto err_init_hdev_kick;
1023     }
1024 
1025     r = event_notifier_init(&svq->hdev_call, 0);
1026     if (r != 0) {
1027         error_setg_errno(errp, -r, "Couldn't create call event notifier");
1028         goto err_init_hdev_call;
1029     }
1030 
1031     file.fd = event_notifier_get_fd(event_notifier);
1032     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
1033     if (unlikely(r != 0)) {
1034         error_setg_errno(errp, -r, "Can't set device kick fd");
1035         goto err_init_set_dev_fd;
1036     }
1037 
1038     event_notifier = &svq->hdev_call;
1039     file.fd = event_notifier_get_fd(event_notifier);
1040     r = vhost_vdpa_set_vring_dev_call(dev, &file);
1041     if (unlikely(r != 0)) {
1042         error_setg_errno(errp, -r, "Can't set device call fd");
1043         goto err_init_set_dev_fd;
1044     }
1045 
1046     return 0;
1047 
1048 err_init_set_dev_fd:
1049     event_notifier_set_handler(&svq->hdev_call, NULL);
1050 
1051 err_init_hdev_call:
1052     event_notifier_cleanup(&svq->hdev_kick);
1053 
1054 err_init_hdev_kick:
1055     return r;
1056 }
1057 
1058 /**
1059  * Unmap a SVQ area in the device
1060  */
1061 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr)
1062 {
1063     const DMAMap needle = {
1064         .translated_addr = addr,
1065     };
1066     const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree,
1067                                                      &needle);
1068     hwaddr size;
1069     int r;
1070 
1071     if (unlikely(!result)) {
1072         error_report("Unable to find SVQ address to unmap");
1073         return;
1074     }
1075 
1076     size = ROUND_UP(result->size, qemu_real_host_page_size());
1077     r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size);
1078     if (unlikely(r < 0)) {
1079         error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r);
1080         return;
1081     }
1082 
1083     vhost_iova_tree_remove(v->shared->iova_tree, *result);
1084 }
1085 
1086 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
1087                                        const VhostShadowVirtqueue *svq)
1088 {
1089     struct vhost_vdpa *v = dev->opaque;
1090     struct vhost_vring_addr svq_addr;
1091 
1092     vhost_svq_get_vring_addr(svq, &svq_addr);
1093 
1094     vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr);
1095 
1096     vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr);
1097 }
1098 
1099 /**
1100  * Map the SVQ area in the device
1101  *
1102  * @v: Vhost-vdpa device
1103  * @needle: The area to search iova
1104  * @errorp: Error pointer
1105  */
1106 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
1107                                     Error **errp)
1108 {
1109     int r;
1110 
1111     r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle);
1112     if (unlikely(r != IOVA_OK)) {
1113         error_setg(errp, "Cannot allocate iova (%d)", r);
1114         return false;
1115     }
1116 
1117     r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova,
1118                            needle->size + 1,
1119                            (void *)(uintptr_t)needle->translated_addr,
1120                            needle->perm == IOMMU_RO);
1121     if (unlikely(r != 0)) {
1122         error_setg_errno(errp, -r, "Cannot map region to device");
1123         vhost_iova_tree_remove(v->shared->iova_tree, *needle);
1124     }
1125 
1126     return r == 0;
1127 }
1128 
1129 /**
1130  * Map the shadow virtqueue rings in the device
1131  *
1132  * @dev: The vhost device
1133  * @svq: The shadow virtqueue
1134  * @addr: Assigned IOVA addresses
1135  * @errp: Error pointer
1136  */
1137 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
1138                                      const VhostShadowVirtqueue *svq,
1139                                      struct vhost_vring_addr *addr,
1140                                      Error **errp)
1141 {
1142     ERRP_GUARD();
1143     DMAMap device_region, driver_region;
1144     struct vhost_vring_addr svq_addr;
1145     struct vhost_vdpa *v = dev->opaque;
1146     size_t device_size = vhost_svq_device_area_size(svq);
1147     size_t driver_size = vhost_svq_driver_area_size(svq);
1148     size_t avail_offset;
1149     bool ok;
1150 
1151     vhost_svq_get_vring_addr(svq, &svq_addr);
1152 
1153     driver_region = (DMAMap) {
1154         .translated_addr = svq_addr.desc_user_addr,
1155         .size = driver_size - 1,
1156         .perm = IOMMU_RO,
1157     };
1158     ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
1159     if (unlikely(!ok)) {
1160         error_prepend(errp, "Cannot create vq driver region: ");
1161         return false;
1162     }
1163     addr->desc_user_addr = driver_region.iova;
1164     avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
1165     addr->avail_user_addr = driver_region.iova + avail_offset;
1166 
1167     device_region = (DMAMap) {
1168         .translated_addr = svq_addr.used_user_addr,
1169         .size = device_size - 1,
1170         .perm = IOMMU_RW,
1171     };
1172     ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
1173     if (unlikely(!ok)) {
1174         error_prepend(errp, "Cannot create vq device region: ");
1175         vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr);
1176     }
1177     addr->used_user_addr = device_region.iova;
1178 
1179     return ok;
1180 }
1181 
1182 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
1183                                  VhostShadowVirtqueue *svq, unsigned idx,
1184                                  Error **errp)
1185 {
1186     uint16_t vq_index = dev->vq_index + idx;
1187     struct vhost_vring_state s = {
1188         .index = vq_index,
1189     };
1190     int r;
1191 
1192     r = vhost_vdpa_set_dev_vring_base(dev, &s);
1193     if (unlikely(r)) {
1194         error_setg_errno(errp, -r, "Cannot set vring base");
1195         return false;
1196     }
1197 
1198     r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
1199     return r == 0;
1200 }
1201 
1202 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
1203 {
1204     struct vhost_vdpa *v = dev->opaque;
1205     Error *err = NULL;
1206     unsigned i;
1207 
1208     if (!v->shadow_vqs_enabled) {
1209         return true;
1210     }
1211 
1212     for (i = 0; i < v->shadow_vqs->len; ++i) {
1213         VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
1214         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1215         struct vhost_vring_addr addr = {
1216             .index = dev->vq_index + i,
1217         };
1218         int r;
1219         bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
1220         if (unlikely(!ok)) {
1221             goto err;
1222         }
1223 
1224         vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree);
1225         ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
1226         if (unlikely(!ok)) {
1227             goto err_map;
1228         }
1229 
1230         /* Override vring GPA set by vhost subsystem */
1231         r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
1232         if (unlikely(r != 0)) {
1233             error_setg_errno(&err, -r, "Cannot set device address");
1234             goto err_set_addr;
1235         }
1236     }
1237 
1238     return true;
1239 
1240 err_set_addr:
1241     vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
1242 
1243 err_map:
1244     vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
1245 
1246 err:
1247     error_reportf_err(err, "Cannot setup SVQ %u: ", i);
1248     for (unsigned j = 0; j < i; ++j) {
1249         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
1250         vhost_vdpa_svq_unmap_rings(dev, svq);
1251         vhost_svq_stop(svq);
1252     }
1253 
1254     return false;
1255 }
1256 
1257 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev)
1258 {
1259     struct vhost_vdpa *v = dev->opaque;
1260 
1261     if (!v->shadow_vqs_enabled) {
1262         return;
1263     }
1264 
1265     for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
1266         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1267 
1268         vhost_svq_stop(svq);
1269         vhost_vdpa_svq_unmap_rings(dev, svq);
1270 
1271         event_notifier_cleanup(&svq->hdev_kick);
1272         event_notifier_cleanup(&svq->hdev_call);
1273     }
1274 }
1275 
1276 static void vhost_vdpa_suspend(struct vhost_dev *dev)
1277 {
1278     struct vhost_vdpa *v = dev->opaque;
1279     int r;
1280 
1281     if (!vhost_vdpa_first_dev(dev)) {
1282         return;
1283     }
1284 
1285     if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) {
1286         trace_vhost_vdpa_suspend(dev);
1287         r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND);
1288         if (unlikely(r)) {
1289             error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno);
1290         } else {
1291             v->suspended = true;
1292             return;
1293         }
1294     }
1295 
1296     vhost_vdpa_reset_device(dev);
1297 }
1298 
1299 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
1300 {
1301     struct vhost_vdpa *v = dev->opaque;
1302     bool ok;
1303     trace_vhost_vdpa_dev_start(dev, started);
1304 
1305     if (started) {
1306         vhost_vdpa_host_notifiers_init(dev);
1307         ok = vhost_vdpa_svqs_start(dev);
1308         if (unlikely(!ok)) {
1309             return -1;
1310         }
1311     } else {
1312         vhost_vdpa_suspend(dev);
1313         vhost_vdpa_svqs_stop(dev);
1314         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
1315     }
1316 
1317     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1318         return 0;
1319     }
1320 
1321     if (started) {
1322         if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) {
1323             error_report("SVQ can not work while IOMMU enable, please disable"
1324                          "IOMMU and try again");
1325             return -1;
1326         }
1327         memory_listener_register(&v->listener, dev->vdev->dma_as);
1328 
1329         return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1330     }
1331 
1332     return 0;
1333 }
1334 
1335 static void vhost_vdpa_reset_status(struct vhost_dev *dev)
1336 {
1337     struct vhost_vdpa *v = dev->opaque;
1338 
1339     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1340         return;
1341     }
1342 
1343     vhost_vdpa_reset_device(dev);
1344     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
1345                                VIRTIO_CONFIG_S_DRIVER);
1346     memory_listener_unregister(&v->listener);
1347 }
1348 
1349 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
1350                                      struct vhost_log *log)
1351 {
1352     struct vhost_vdpa *v = dev->opaque;
1353     if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) {
1354         return 0;
1355     }
1356 
1357     trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd,
1358                                   log->log);
1359     return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
1360 }
1361 
1362 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
1363                                        struct vhost_vring_addr *addr)
1364 {
1365     struct vhost_vdpa *v = dev->opaque;
1366 
1367     if (v->shadow_vqs_enabled) {
1368         /*
1369          * Device vring addr was set at device start. SVQ base is handled by
1370          * VirtQueue code.
1371          */
1372         return 0;
1373     }
1374 
1375     return vhost_vdpa_set_vring_dev_addr(dev, addr);
1376 }
1377 
1378 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
1379                                       struct vhost_vring_state *ring)
1380 {
1381     trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num);
1382     return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
1383 }
1384 
1385 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
1386                                        struct vhost_vring_state *ring)
1387 {
1388     struct vhost_vdpa *v = dev->opaque;
1389 
1390     if (v->shadow_vqs_enabled) {
1391         /*
1392          * Device vring base was set at device start. SVQ base is handled by
1393          * VirtQueue code.
1394          */
1395         return 0;
1396     }
1397 
1398     return vhost_vdpa_set_dev_vring_base(dev, ring);
1399 }
1400 
1401 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
1402                                        struct vhost_vring_state *ring)
1403 {
1404     struct vhost_vdpa *v = dev->opaque;
1405     int ret;
1406 
1407     if (v->shadow_vqs_enabled) {
1408         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
1409         return 0;
1410     }
1411 
1412     if (!v->suspended) {
1413         /*
1414          * Cannot trust in value returned by device, let vhost recover used
1415          * idx from guest.
1416          */
1417         return -1;
1418     }
1419 
1420     ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
1421     trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
1422     return ret;
1423 }
1424 
1425 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
1426                                        struct vhost_vring_file *file)
1427 {
1428     struct vhost_vdpa *v = dev->opaque;
1429     int vdpa_idx = file->index - dev->vq_index;
1430 
1431     if (v->shadow_vqs_enabled) {
1432         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1433         vhost_svq_set_svq_kick_fd(svq, file->fd);
1434         return 0;
1435     } else {
1436         return vhost_vdpa_set_vring_dev_kick(dev, file);
1437     }
1438 }
1439 
1440 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
1441                                        struct vhost_vring_file *file)
1442 {
1443     struct vhost_vdpa *v = dev->opaque;
1444     int vdpa_idx = file->index - dev->vq_index;
1445     VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1446 
1447     /* Remember last call fd because we can switch to SVQ anytime. */
1448     vhost_svq_set_svq_call_fd(svq, file->fd);
1449     if (v->shadow_vqs_enabled) {
1450         return 0;
1451     }
1452 
1453     return vhost_vdpa_set_vring_dev_call(dev, file);
1454 }
1455 
1456 static int vhost_vdpa_get_features(struct vhost_dev *dev,
1457                                      uint64_t *features)
1458 {
1459     int ret = vhost_vdpa_get_dev_features(dev, features);
1460 
1461     if (ret == 0) {
1462         /* Add SVQ logging capabilities */
1463         *features |= BIT_ULL(VHOST_F_LOG_ALL);
1464     }
1465 
1466     return ret;
1467 }
1468 
1469 static int vhost_vdpa_set_owner(struct vhost_dev *dev)
1470 {
1471     if (!vhost_vdpa_first_dev(dev)) {
1472         return 0;
1473     }
1474 
1475     trace_vhost_vdpa_set_owner(dev);
1476     return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
1477 }
1478 
1479 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
1480                     struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
1481 {
1482     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
1483     addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
1484     addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
1485     addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
1486     trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr,
1487                                  addr->avail_user_addr, addr->used_user_addr);
1488     return 0;
1489 }
1490 
1491 static bool  vhost_vdpa_force_iommu(struct vhost_dev *dev)
1492 {
1493     return true;
1494 }
1495 
1496 const VhostOps vdpa_ops = {
1497         .backend_type = VHOST_BACKEND_TYPE_VDPA,
1498         .vhost_backend_init = vhost_vdpa_init,
1499         .vhost_backend_cleanup = vhost_vdpa_cleanup,
1500         .vhost_set_log_base = vhost_vdpa_set_log_base,
1501         .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
1502         .vhost_set_vring_num = vhost_vdpa_set_vring_num,
1503         .vhost_set_vring_base = vhost_vdpa_set_vring_base,
1504         .vhost_get_vring_base = vhost_vdpa_get_vring_base,
1505         .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
1506         .vhost_set_vring_call = vhost_vdpa_set_vring_call,
1507         .vhost_get_features = vhost_vdpa_get_features,
1508         .vhost_set_backend_cap = vhost_vdpa_set_backend_cap,
1509         .vhost_set_owner = vhost_vdpa_set_owner,
1510         .vhost_set_vring_endian = NULL,
1511         .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
1512         .vhost_set_mem_table = vhost_vdpa_set_mem_table,
1513         .vhost_set_features = vhost_vdpa_set_features,
1514         .vhost_reset_device = vhost_vdpa_reset_device,
1515         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
1516         .vhost_get_config  = vhost_vdpa_get_config,
1517         .vhost_set_config = vhost_vdpa_set_config,
1518         .vhost_requires_shm_log = NULL,
1519         .vhost_migration_done = NULL,
1520         .vhost_net_set_mtu = NULL,
1521         .vhost_set_iotlb_callback = NULL,
1522         .vhost_send_device_iotlb_msg = NULL,
1523         .vhost_dev_start = vhost_vdpa_dev_start,
1524         .vhost_get_device_id = vhost_vdpa_get_device_id,
1525         .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
1526         .vhost_force_iommu = vhost_vdpa_force_iommu,
1527         .vhost_set_config_call = vhost_vdpa_set_config_call,
1528         .vhost_reset_status = vhost_vdpa_reset_status,
1529 };
1530