xref: /openbmc/qemu/hw/virtio/vhost-vdpa.c (revision f12b2498e561ddf9ab45940deb15ef1bb046f699)
1 /*
2  * vhost-vdpa
3  *
4  *  Copyright(c) 2017-2018 Intel Corporation.
5  *  Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include <linux/vhost.h>
14 #include <linux/vfio.h>
15 #include <sys/eventfd.h>
16 #include <sys/ioctl.h>
17 #include "exec/target_page.h"
18 #include "hw/virtio/vhost.h"
19 #include "hw/virtio/vhost-backend.h"
20 #include "hw/virtio/virtio-net.h"
21 #include "hw/virtio/vhost-shadow-virtqueue.h"
22 #include "hw/virtio/vhost-vdpa.h"
23 #include "exec/address-spaces.h"
24 #include "migration/blocker.h"
25 #include "qemu/cutils.h"
26 #include "qemu/main-loop.h"
27 #include "trace.h"
28 #include "qapi/error.h"
29 
30 /*
31  * Return one past the end of the end of section. Be careful with uint64_t
32  * conversions!
33  */
34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section,
35                                      int page_mask)
36 {
37     Int128 llend = int128_make64(section->offset_within_address_space);
38     llend = int128_add(llend, section->size);
39     llend = int128_and(llend, int128_exts64(page_mask));
40 
41     return llend;
42 }
43 
44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section,
45                                                 uint64_t iova_min,
46                                                 uint64_t iova_max,
47                                                 int page_mask)
48 {
49     Int128 llend;
50 
51     if ((!memory_region_is_ram(section->mr) &&
52          !memory_region_is_iommu(section->mr)) ||
53         memory_region_is_protected(section->mr) ||
54         /* vhost-vDPA doesn't allow MMIO to be mapped  */
55         memory_region_is_ram_device(section->mr)) {
56         return true;
57     }
58 
59     if (section->offset_within_address_space < iova_min) {
60         error_report("RAM section out of device range (min=0x%" PRIx64
61                      ", addr=0x%" HWADDR_PRIx ")",
62                      iova_min, section->offset_within_address_space);
63         return true;
64     }
65     /*
66      * While using vIOMMU, sometimes the section will be larger than iova_max,
67      * but the memory that actually maps is smaller, so move the check to
68      * function vhost_vdpa_iommu_map_notify(). That function will use the actual
69      * size that maps to the kernel
70      */
71 
72     if (!memory_region_is_iommu(section->mr)) {
73         llend = vhost_vdpa_section_end(section, page_mask);
74         if (int128_gt(llend, int128_make64(iova_max))) {
75             error_report("RAM section out of device range (max=0x%" PRIx64
76                          ", end addr=0x%" PRIx64 ")",
77                          iova_max, int128_get64(llend));
78             return true;
79         }
80     }
81 
82     return false;
83 }
84 
85 /*
86  * The caller must set asid = 0 if the device does not support asid.
87  * This is not an ABI break since it is set to 0 by the initializer anyway.
88  */
89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
90                        hwaddr size, void *vaddr, bool readonly)
91 {
92     struct vhost_msg_v2 msg = {};
93     int fd = v->shared->device_fd;
94     int ret = 0;
95 
96     msg.type = v->msg_type;
97     msg.asid = asid;
98     msg.iotlb.iova = iova;
99     msg.iotlb.size = size;
100     msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
101     msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
102     msg.iotlb.type = VHOST_IOTLB_UPDATE;
103 
104     trace_vhost_vdpa_dma_map(v->shared, fd, msg.type, msg.asid, msg.iotlb.iova,
105                              msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm,
106                              msg.iotlb.type);
107 
108     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
109         error_report("failed to write, fd=%d, errno=%d (%s)",
110             fd, errno, strerror(errno));
111         return -EIO ;
112     }
113 
114     return ret;
115 }
116 
117 /*
118  * The caller must set asid = 0 if the device does not support asid.
119  * This is not an ABI break since it is set to 0 by the initializer anyway.
120  */
121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
122                          hwaddr size)
123 {
124     struct vhost_msg_v2 msg = {};
125     int fd = v->shared->device_fd;
126     int ret = 0;
127 
128     msg.type = v->msg_type;
129     msg.asid = asid;
130     msg.iotlb.iova = iova;
131     msg.iotlb.size = size;
132     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
133 
134     trace_vhost_vdpa_dma_unmap(v->shared, fd, msg.type, msg.asid,
135                                msg.iotlb.iova, msg.iotlb.size, msg.iotlb.type);
136 
137     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
138         error_report("failed to write, fd=%d, errno=%d (%s)",
139             fd, errno, strerror(errno));
140         return -EIO ;
141     }
142 
143     return ret;
144 }
145 
146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v)
147 {
148     int fd = v->shared->device_fd;
149     struct vhost_msg_v2 msg = {
150         .type = v->msg_type,
151         .iotlb.type = VHOST_IOTLB_BATCH_BEGIN,
152     };
153 
154     trace_vhost_vdpa_listener_begin_batch(v->shared, fd, msg.type,
155                                           msg.iotlb.type);
156     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
157         error_report("failed to write, fd=%d, errno=%d (%s)",
158                      fd, errno, strerror(errno));
159     }
160 }
161 
162 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v)
163 {
164     if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) &&
165         !v->iotlb_batch_begin_sent) {
166         vhost_vdpa_listener_begin_batch(v);
167     }
168 
169     v->iotlb_batch_begin_sent = true;
170 }
171 
172 static void vhost_vdpa_listener_commit(MemoryListener *listener)
173 {
174     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
175     struct vhost_dev *dev = v->dev;
176     struct vhost_msg_v2 msg = {};
177     int fd = v->shared->device_fd;
178 
179     if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
180         return;
181     }
182 
183     if (!v->iotlb_batch_begin_sent) {
184         return;
185     }
186 
187     msg.type = v->msg_type;
188     msg.iotlb.type = VHOST_IOTLB_BATCH_END;
189 
190     trace_vhost_vdpa_listener_commit(v->shared, fd, msg.type, msg.iotlb.type);
191     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
192         error_report("failed to write, fd=%d, errno=%d (%s)",
193                      fd, errno, strerror(errno));
194     }
195 
196     v->iotlb_batch_begin_sent = false;
197 }
198 
199 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
200 {
201     struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n);
202 
203     hwaddr iova = iotlb->iova + iommu->iommu_offset;
204     struct vhost_vdpa *v = iommu->dev;
205     void *vaddr;
206     int ret;
207     Int128 llend;
208 
209     if (iotlb->target_as != &address_space_memory) {
210         error_report("Wrong target AS \"%s\", only system memory is allowed",
211                      iotlb->target_as->name ? iotlb->target_as->name : "none");
212         return;
213     }
214     RCU_READ_LOCK_GUARD();
215     /* check if RAM section out of device range */
216     llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova));
217     if (int128_gt(llend, int128_make64(v->shared->iova_range.last))) {
218         error_report("RAM section out of device range (max=0x%" PRIx64
219                      ", end addr=0x%" PRIx64 ")",
220                      v->shared->iova_range.last, int128_get64(llend));
221         return;
222     }
223 
224     if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
225         bool read_only;
226 
227         if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) {
228             return;
229         }
230         ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
231                                  iotlb->addr_mask + 1, vaddr, read_only);
232         if (ret) {
233             error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", "
234                          "0x%" HWADDR_PRIx ", %p) = %d (%m)",
235                          v, iova, iotlb->addr_mask + 1, vaddr, ret);
236         }
237     } else {
238         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
239                                    iotlb->addr_mask + 1);
240         if (ret) {
241             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
242                          "0x%" HWADDR_PRIx ") = %d (%m)",
243                          v, iova, iotlb->addr_mask + 1, ret);
244         }
245     }
246 }
247 
248 static void vhost_vdpa_iommu_region_add(MemoryListener *listener,
249                                         MemoryRegionSection *section)
250 {
251     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
252 
253     struct vdpa_iommu *iommu;
254     Int128 end;
255     int iommu_idx;
256     IOMMUMemoryRegion *iommu_mr;
257     int ret;
258 
259     iommu_mr = IOMMU_MEMORY_REGION(section->mr);
260 
261     iommu = g_malloc0(sizeof(*iommu));
262     end = int128_add(int128_make64(section->offset_within_region),
263                      section->size);
264     end = int128_sub(end, int128_one());
265     iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
266                                                    MEMTXATTRS_UNSPECIFIED);
267     iommu->iommu_mr = iommu_mr;
268     iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify,
269                         IOMMU_NOTIFIER_IOTLB_EVENTS,
270                         section->offset_within_region,
271                         int128_get64(end),
272                         iommu_idx);
273     iommu->iommu_offset = section->offset_within_address_space -
274                           section->offset_within_region;
275     iommu->dev = v;
276 
277     ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
278     if (ret) {
279         g_free(iommu);
280         return;
281     }
282 
283     QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next);
284     memory_region_iommu_replay(iommu->iommu_mr, &iommu->n);
285 
286     return;
287 }
288 
289 static void vhost_vdpa_iommu_region_del(MemoryListener *listener,
290                                         MemoryRegionSection *section)
291 {
292     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
293 
294     struct vdpa_iommu *iommu;
295 
296     QLIST_FOREACH(iommu, &v->iommu_list, iommu_next)
297     {
298         if (MEMORY_REGION(iommu->iommu_mr) == section->mr &&
299             iommu->n.start == section->offset_within_region) {
300             memory_region_unregister_iommu_notifier(section->mr, &iommu->n);
301             QLIST_REMOVE(iommu, iommu_next);
302             g_free(iommu);
303             break;
304         }
305     }
306 }
307 
308 static void vhost_vdpa_listener_region_add(MemoryListener *listener,
309                                            MemoryRegionSection *section)
310 {
311     DMAMap mem_region = {};
312     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
313     hwaddr iova;
314     Int128 llend, llsize;
315     void *vaddr;
316     int ret;
317     int page_size = qemu_target_page_size();
318     int page_mask = -page_size;
319 
320     if (vhost_vdpa_listener_skipped_section(section,
321                                             v->shared->iova_range.first,
322                                             v->shared->iova_range.last,
323                                             page_mask)) {
324         return;
325     }
326     if (memory_region_is_iommu(section->mr)) {
327         vhost_vdpa_iommu_region_add(listener, section);
328         return;
329     }
330 
331     if (unlikely((section->offset_within_address_space & ~page_mask) !=
332                  (section->offset_within_region & ~page_mask))) {
333         trace_vhost_vdpa_listener_region_add_unaligned(v->shared,
334                        section->mr->name,
335                        section->offset_within_address_space & ~page_mask,
336                        section->offset_within_region & ~page_mask);
337         return;
338     }
339 
340     iova = ROUND_UP(section->offset_within_address_space, page_size);
341     llend = vhost_vdpa_section_end(section, page_mask);
342     if (int128_ge(int128_make64(iova), llend)) {
343         return;
344     }
345 
346     memory_region_ref(section->mr);
347 
348     /* Here we assume that memory_region_is_ram(section->mr)==true */
349 
350     vaddr = memory_region_get_ram_ptr(section->mr) +
351             section->offset_within_region +
352             (iova - section->offset_within_address_space);
353 
354     trace_vhost_vdpa_listener_region_add(v->shared, iova, int128_get64(llend),
355                                          vaddr, section->readonly);
356 
357     llsize = int128_sub(llend, int128_make64(iova));
358     if (v->shared->shadow_data) {
359         int r;
360 
361         mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
362         mem_region.size = int128_get64(llsize) - 1,
363         mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
364 
365         r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region);
366         if (unlikely(r != IOVA_OK)) {
367             error_report("Can't allocate a mapping (%d)", r);
368             goto fail;
369         }
370 
371         iova = mem_region.iova;
372     }
373 
374     vhost_vdpa_iotlb_batch_begin_once(v);
375     ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
376                              int128_get64(llsize), vaddr, section->readonly);
377     if (ret) {
378         error_report("vhost vdpa map fail!");
379         goto fail_map;
380     }
381 
382     return;
383 
384 fail_map:
385     if (v->shared->shadow_data) {
386         vhost_iova_tree_remove(v->shared->iova_tree, mem_region);
387     }
388 
389 fail:
390     /*
391      * On the initfn path, store the first error in the container so we
392      * can gracefully fail.  Runtime, there's not much we can do other
393      * than throw a hardware error.
394      */
395     error_report("vhost-vdpa: DMA mapping failed, unable to continue");
396     return;
397 
398 }
399 
400 static void vhost_vdpa_listener_region_del(MemoryListener *listener,
401                                            MemoryRegionSection *section)
402 {
403     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
404     hwaddr iova;
405     Int128 llend, llsize;
406     int ret;
407     int page_size = qemu_target_page_size();
408     int page_mask = -page_size;
409 
410     if (vhost_vdpa_listener_skipped_section(section,
411                                             v->shared->iova_range.first,
412                                             v->shared->iova_range.last,
413                                             page_mask)) {
414         return;
415     }
416     if (memory_region_is_iommu(section->mr)) {
417         vhost_vdpa_iommu_region_del(listener, section);
418     }
419 
420     if (unlikely((section->offset_within_address_space & ~page_mask) !=
421                  (section->offset_within_region & ~page_mask))) {
422         trace_vhost_vdpa_listener_region_del_unaligned(v->shared,
423                        section->mr->name,
424                        section->offset_within_address_space & ~page_mask,
425                        section->offset_within_region & ~page_mask);
426         return;
427     }
428 
429     iova = ROUND_UP(section->offset_within_address_space, page_size);
430     llend = vhost_vdpa_section_end(section, page_mask);
431 
432     trace_vhost_vdpa_listener_region_del(v->shared, iova,
433         int128_get64(int128_sub(llend, int128_one())));
434 
435     if (int128_ge(int128_make64(iova), llend)) {
436         return;
437     }
438 
439     llsize = int128_sub(llend, int128_make64(iova));
440 
441     if (v->shared->shadow_data) {
442         const DMAMap *result;
443         const void *vaddr = memory_region_get_ram_ptr(section->mr) +
444             section->offset_within_region +
445             (iova - section->offset_within_address_space);
446         DMAMap mem_region = {
447             .translated_addr = (hwaddr)(uintptr_t)vaddr,
448             .size = int128_get64(llsize) - 1,
449         };
450 
451         result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region);
452         if (!result) {
453             /* The memory listener map wasn't mapped */
454             return;
455         }
456         iova = result->iova;
457         vhost_iova_tree_remove(v->shared->iova_tree, *result);
458     }
459     vhost_vdpa_iotlb_batch_begin_once(v);
460     /*
461      * The unmap ioctl doesn't accept a full 64-bit. need to check it
462      */
463     if (int128_eq(llsize, int128_2_64())) {
464         llsize = int128_rshift(llsize, 1);
465         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
466                                    int128_get64(llsize));
467 
468         if (ret) {
469             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
470                          "0x%" HWADDR_PRIx ") = %d (%m)",
471                          v, iova, int128_get64(llsize), ret);
472         }
473         iova += int128_get64(llsize);
474     }
475     ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
476                                int128_get64(llsize));
477 
478     if (ret) {
479         error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
480                      "0x%" HWADDR_PRIx ") = %d (%m)",
481                      v, iova, int128_get64(llsize), ret);
482     }
483 
484     memory_region_unref(section->mr);
485 }
486 /*
487  * IOTLB API is used by vhost-vdpa which requires incremental updating
488  * of the mapping. So we can not use generic vhost memory listener which
489  * depends on the addnop().
490  */
491 static const MemoryListener vhost_vdpa_memory_listener = {
492     .name = "vhost-vdpa",
493     .commit = vhost_vdpa_listener_commit,
494     .region_add = vhost_vdpa_listener_region_add,
495     .region_del = vhost_vdpa_listener_region_del,
496 };
497 
498 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
499                              void *arg)
500 {
501     struct vhost_vdpa *v = dev->opaque;
502     int fd = v->shared->device_fd;
503     int ret;
504 
505     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
506 
507     ret = ioctl(fd, request, arg);
508     return ret < 0 ? -errno : ret;
509 }
510 
511 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
512 {
513     uint8_t s;
514     int ret;
515 
516     trace_vhost_vdpa_add_status(dev, status);
517     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
518     if (ret < 0) {
519         return ret;
520     }
521     if ((s & status) == status) {
522         /* Don't set bits already set */
523         return 0;
524     }
525 
526     s |= status;
527 
528     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
529     if (ret < 0) {
530         return ret;
531     }
532 
533     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
534     if (ret < 0) {
535         return ret;
536     }
537 
538     if (!(s & status)) {
539         return -EIO;
540     }
541 
542     return 0;
543 }
544 
545 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range)
546 {
547     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
548 
549     return ret < 0 ? -errno : 0;
550 }
551 
552 /*
553  * The use of this function is for requests that only need to be
554  * applied once. Typically such request occurs at the beginning
555  * of operation, and before setting up queues. It should not be
556  * used for request that performs operation until all queues are
557  * set, which would need to check dev->vq_index_end instead.
558  */
559 static bool vhost_vdpa_first_dev(struct vhost_dev *dev)
560 {
561     struct vhost_vdpa *v = dev->opaque;
562 
563     return v->index == 0;
564 }
565 
566 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
567                                        uint64_t *features)
568 {
569     int ret;
570 
571     ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
572     trace_vhost_vdpa_get_features(dev, *features);
573     return ret;
574 }
575 
576 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v)
577 {
578     g_autoptr(GPtrArray) shadow_vqs = NULL;
579 
580     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
581     for (unsigned n = 0; n < hdev->nvqs; ++n) {
582         VhostShadowVirtqueue *svq;
583 
584         svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque);
585         g_ptr_array_add(shadow_vqs, svq);
586     }
587 
588     v->shadow_vqs = g_steal_pointer(&shadow_vqs);
589 }
590 
591 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
592 {
593     struct vhost_vdpa *v = opaque;
594     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
595     trace_vhost_vdpa_init(dev, v->shared, opaque);
596     int ret;
597 
598     v->dev = dev;
599     dev->opaque =  opaque ;
600     v->listener = vhost_vdpa_memory_listener;
601     v->msg_type = VHOST_IOTLB_MSG_V2;
602     vhost_vdpa_init_svq(dev, v);
603 
604     error_propagate(&dev->migration_blocker, v->migration_blocker);
605     if (!vhost_vdpa_first_dev(dev)) {
606         return 0;
607     }
608 
609     /*
610      * If dev->shadow_vqs_enabled at initialization that means the device has
611      * been started with x-svq=on, so don't block migration
612      */
613     if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) {
614         /* We don't have dev->features yet */
615         uint64_t features;
616         ret = vhost_vdpa_get_dev_features(dev, &features);
617         if (unlikely(ret)) {
618             error_setg_errno(errp, -ret, "Could not get device features");
619             return ret;
620         }
621         vhost_svq_valid_features(features, &dev->migration_blocker);
622     }
623 
624     /*
625      * Similar to VFIO, we end up pinning all guest memory and have to
626      * disable discarding of RAM.
627      */
628     ret = ram_block_discard_disable(true);
629     if (ret) {
630         error_report("Cannot set discarding of RAM broken");
631         return ret;
632     }
633 
634     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
635                                VIRTIO_CONFIG_S_DRIVER);
636 
637     return 0;
638 }
639 
640 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
641                                             int queue_index)
642 {
643     size_t page_size = qemu_real_host_page_size();
644     struct vhost_vdpa *v = dev->opaque;
645     VirtIODevice *vdev = dev->vdev;
646     VhostVDPAHostNotifier *n;
647 
648     n = &v->notifier[queue_index];
649 
650     if (n->addr) {
651         virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
652         object_unparent(OBJECT(&n->mr));
653         munmap(n->addr, page_size);
654         n->addr = NULL;
655     }
656 }
657 
658 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
659 {
660     size_t page_size = qemu_real_host_page_size();
661     struct vhost_vdpa *v = dev->opaque;
662     VirtIODevice *vdev = dev->vdev;
663     VhostVDPAHostNotifier *n;
664     int fd = v->shared->device_fd;
665     void *addr;
666     char *name;
667 
668     vhost_vdpa_host_notifier_uninit(dev, queue_index);
669 
670     n = &v->notifier[queue_index];
671 
672     addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
673                 queue_index * page_size);
674     if (addr == MAP_FAILED) {
675         goto err;
676     }
677 
678     name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
679                            v, queue_index);
680     memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
681                                       page_size, addr);
682     g_free(name);
683 
684     if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
685         object_unparent(OBJECT(&n->mr));
686         munmap(addr, page_size);
687         goto err;
688     }
689     n->addr = addr;
690 
691     return 0;
692 
693 err:
694     return -1;
695 }
696 
697 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
698 {
699     int i;
700 
701     /*
702      * Pack all the changes to the memory regions in a single
703      * transaction to avoid a few updating of the address space
704      * topology.
705      */
706     memory_region_transaction_begin();
707 
708     for (i = dev->vq_index; i < dev->vq_index + n; i++) {
709         vhost_vdpa_host_notifier_uninit(dev, i);
710     }
711 
712     memory_region_transaction_commit();
713 }
714 
715 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
716 {
717     struct vhost_vdpa *v = dev->opaque;
718     int i;
719 
720     if (v->shadow_vqs_enabled) {
721         /* FIXME SVQ is not compatible with host notifiers mr */
722         return;
723     }
724 
725     /*
726      * Pack all the changes to the memory regions in a single
727      * transaction to avoid a few updating of the address space
728      * topology.
729      */
730     memory_region_transaction_begin();
731 
732     for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
733         if (vhost_vdpa_host_notifier_init(dev, i)) {
734             vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index);
735             break;
736         }
737     }
738 
739     memory_region_transaction_commit();
740 }
741 
742 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
743 {
744     struct vhost_vdpa *v = dev->opaque;
745     size_t idx;
746 
747     for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
748         vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
749     }
750     g_ptr_array_free(v->shadow_vqs, true);
751 }
752 
753 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
754 {
755     struct vhost_vdpa *v;
756     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
757     v = dev->opaque;
758     trace_vhost_vdpa_cleanup(dev, v);
759     if (vhost_vdpa_first_dev(dev)) {
760         ram_block_discard_disable(false);
761     }
762 
763     vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
764     memory_listener_unregister(&v->listener);
765     vhost_vdpa_svq_cleanup(dev);
766 
767     dev->opaque = NULL;
768 
769     return 0;
770 }
771 
772 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
773 {
774     trace_vhost_vdpa_memslots_limit(dev, INT_MAX);
775     return INT_MAX;
776 }
777 
778 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
779                                     struct vhost_memory *mem)
780 {
781     if (!vhost_vdpa_first_dev(dev)) {
782         return 0;
783     }
784 
785     trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding);
786     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) &&
787         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) {
788         int i;
789         for (i = 0; i < mem->nregions; i++) {
790             trace_vhost_vdpa_dump_regions(dev, i,
791                                           mem->regions[i].guest_phys_addr,
792                                           mem->regions[i].memory_size,
793                                           mem->regions[i].userspace_addr,
794                                           mem->regions[i].flags_padding);
795         }
796     }
797     if (mem->padding) {
798         return -EINVAL;
799     }
800 
801     return 0;
802 }
803 
804 static int vhost_vdpa_set_features(struct vhost_dev *dev,
805                                    uint64_t features)
806 {
807     struct vhost_vdpa *v = dev->opaque;
808     int ret;
809 
810     if (!vhost_vdpa_first_dev(dev)) {
811         return 0;
812     }
813 
814     if (v->shadow_vqs_enabled) {
815         if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
816             /*
817              * QEMU is just trying to enable or disable logging. SVQ handles
818              * this sepparately, so no need to forward this.
819              */
820             v->acked_features = features;
821             return 0;
822         }
823 
824         v->acked_features = features;
825 
826         /* We must not ack _F_LOG if SVQ is enabled */
827         features &= ~BIT_ULL(VHOST_F_LOG_ALL);
828     }
829 
830     trace_vhost_vdpa_set_features(dev, features);
831     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
832     if (ret) {
833         return ret;
834     }
835 
836     return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
837 }
838 
839 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev)
840 {
841     uint64_t features;
842     uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 |
843         0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH |
844         0x1ULL << VHOST_BACKEND_F_IOTLB_ASID |
845         0x1ULL << VHOST_BACKEND_F_SUSPEND;
846     int r;
847 
848     if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
849         return -EFAULT;
850     }
851 
852     features &= f;
853 
854     if (vhost_vdpa_first_dev(dev)) {
855         r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features);
856         if (r) {
857             return -EFAULT;
858         }
859     }
860 
861     dev->backend_cap = features;
862 
863     return 0;
864 }
865 
866 static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
867                                     uint32_t *device_id)
868 {
869     int ret;
870     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
871     trace_vhost_vdpa_get_device_id(dev, *device_id);
872     return ret;
873 }
874 
875 static int vhost_vdpa_reset_device(struct vhost_dev *dev)
876 {
877     struct vhost_vdpa *v = dev->opaque;
878     int ret;
879     uint8_t status = 0;
880 
881     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
882     trace_vhost_vdpa_reset_device(dev);
883     v->suspended = false;
884     return ret;
885 }
886 
887 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
888 {
889     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
890 
891     trace_vhost_vdpa_get_vq_index(dev, idx, idx);
892     return idx;
893 }
894 
895 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx)
896 {
897     struct vhost_dev *dev = v->dev;
898     struct vhost_vring_state state = {
899         .index = idx,
900         .num = 1,
901     };
902     int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
903 
904     trace_vhost_vdpa_set_vring_ready(dev, idx, r);
905     return r;
906 }
907 
908 static int vhost_vdpa_set_config_call(struct vhost_dev *dev,
909                                        int fd)
910 {
911     trace_vhost_vdpa_set_config_call(dev, fd);
912     return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd);
913 }
914 
915 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config,
916                                    uint32_t config_len)
917 {
918     int b, len;
919     char line[QEMU_HEXDUMP_LINE_LEN];
920 
921     for (b = 0; b < config_len; b += 16) {
922         len = config_len - b;
923         qemu_hexdump_line(line, b, config, len, false);
924         trace_vhost_vdpa_dump_config(dev, line);
925     }
926 }
927 
928 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
929                                    uint32_t offset, uint32_t size,
930                                    uint32_t flags)
931 {
932     struct vhost_vdpa_config *config;
933     int ret;
934     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
935 
936     trace_vhost_vdpa_set_config(dev, offset, size, flags);
937     config = g_malloc(size + config_size);
938     config->off = offset;
939     config->len = size;
940     memcpy(config->buf, data, size);
941     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) &&
942         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
943         vhost_vdpa_dump_config(dev, data, size);
944     }
945     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
946     g_free(config);
947     return ret;
948 }
949 
950 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
951                                    uint32_t config_len, Error **errp)
952 {
953     struct vhost_vdpa_config *v_config;
954     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
955     int ret;
956 
957     trace_vhost_vdpa_get_config(dev, config, config_len);
958     v_config = g_malloc(config_len + config_size);
959     v_config->len = config_len;
960     v_config->off = 0;
961     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
962     memcpy(config, v_config->buf, config_len);
963     g_free(v_config);
964     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) &&
965         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
966         vhost_vdpa_dump_config(dev, config, config_len);
967     }
968     return ret;
969  }
970 
971 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
972                                          struct vhost_vring_state *ring)
973 {
974     trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
975     return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
976 }
977 
978 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
979                                          struct vhost_vring_file *file)
980 {
981     trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
982     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
983 }
984 
985 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
986                                          struct vhost_vring_file *file)
987 {
988     trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
989     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
990 }
991 
992 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
993                                          struct vhost_vring_addr *addr)
994 {
995     trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
996                                 addr->desc_user_addr, addr->used_user_addr,
997                                 addr->avail_user_addr,
998                                 addr->log_guest_addr);
999 
1000     return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
1001 
1002 }
1003 
1004 /**
1005  * Set the shadow virtqueue descriptors to the device
1006  *
1007  * @dev: The vhost device model
1008  * @svq: The shadow virtqueue
1009  * @idx: The index of the virtqueue in the vhost device
1010  * @errp: Error
1011  *
1012  * Note that this function does not rewind kick file descriptor if cannot set
1013  * call one.
1014  */
1015 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
1016                                   VhostShadowVirtqueue *svq, unsigned idx,
1017                                   Error **errp)
1018 {
1019     struct vhost_vring_file file = {
1020         .index = dev->vq_index + idx,
1021     };
1022     const EventNotifier *event_notifier = &svq->hdev_kick;
1023     int r;
1024 
1025     r = event_notifier_init(&svq->hdev_kick, 0);
1026     if (r != 0) {
1027         error_setg_errno(errp, -r, "Couldn't create kick event notifier");
1028         goto err_init_hdev_kick;
1029     }
1030 
1031     r = event_notifier_init(&svq->hdev_call, 0);
1032     if (r != 0) {
1033         error_setg_errno(errp, -r, "Couldn't create call event notifier");
1034         goto err_init_hdev_call;
1035     }
1036 
1037     file.fd = event_notifier_get_fd(event_notifier);
1038     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
1039     if (unlikely(r != 0)) {
1040         error_setg_errno(errp, -r, "Can't set device kick fd");
1041         goto err_init_set_dev_fd;
1042     }
1043 
1044     event_notifier = &svq->hdev_call;
1045     file.fd = event_notifier_get_fd(event_notifier);
1046     r = vhost_vdpa_set_vring_dev_call(dev, &file);
1047     if (unlikely(r != 0)) {
1048         error_setg_errno(errp, -r, "Can't set device call fd");
1049         goto err_init_set_dev_fd;
1050     }
1051 
1052     return 0;
1053 
1054 err_init_set_dev_fd:
1055     event_notifier_set_handler(&svq->hdev_call, NULL);
1056 
1057 err_init_hdev_call:
1058     event_notifier_cleanup(&svq->hdev_kick);
1059 
1060 err_init_hdev_kick:
1061     return r;
1062 }
1063 
1064 /**
1065  * Unmap a SVQ area in the device
1066  */
1067 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr)
1068 {
1069     const DMAMap needle = {
1070         .translated_addr = addr,
1071     };
1072     const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree,
1073                                                      &needle);
1074     hwaddr size;
1075     int r;
1076 
1077     if (unlikely(!result)) {
1078         error_report("Unable to find SVQ address to unmap");
1079         return;
1080     }
1081 
1082     size = ROUND_UP(result->size, qemu_real_host_page_size());
1083     r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size);
1084     if (unlikely(r < 0)) {
1085         error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r);
1086         return;
1087     }
1088 
1089     vhost_iova_tree_remove(v->shared->iova_tree, *result);
1090 }
1091 
1092 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
1093                                        const VhostShadowVirtqueue *svq)
1094 {
1095     struct vhost_vdpa *v = dev->opaque;
1096     struct vhost_vring_addr svq_addr;
1097 
1098     vhost_svq_get_vring_addr(svq, &svq_addr);
1099 
1100     vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr);
1101 
1102     vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr);
1103 }
1104 
1105 /**
1106  * Map the SVQ area in the device
1107  *
1108  * @v: Vhost-vdpa device
1109  * @needle: The area to search iova
1110  * @errorp: Error pointer
1111  */
1112 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
1113                                     Error **errp)
1114 {
1115     int r;
1116 
1117     r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle);
1118     if (unlikely(r != IOVA_OK)) {
1119         error_setg(errp, "Cannot allocate iova (%d)", r);
1120         return false;
1121     }
1122 
1123     r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova,
1124                            needle->size + 1,
1125                            (void *)(uintptr_t)needle->translated_addr,
1126                            needle->perm == IOMMU_RO);
1127     if (unlikely(r != 0)) {
1128         error_setg_errno(errp, -r, "Cannot map region to device");
1129         vhost_iova_tree_remove(v->shared->iova_tree, *needle);
1130     }
1131 
1132     return r == 0;
1133 }
1134 
1135 /**
1136  * Map the shadow virtqueue rings in the device
1137  *
1138  * @dev: The vhost device
1139  * @svq: The shadow virtqueue
1140  * @addr: Assigned IOVA addresses
1141  * @errp: Error pointer
1142  */
1143 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
1144                                      const VhostShadowVirtqueue *svq,
1145                                      struct vhost_vring_addr *addr,
1146                                      Error **errp)
1147 {
1148     ERRP_GUARD();
1149     DMAMap device_region, driver_region;
1150     struct vhost_vring_addr svq_addr;
1151     struct vhost_vdpa *v = dev->opaque;
1152     size_t device_size = vhost_svq_device_area_size(svq);
1153     size_t driver_size = vhost_svq_driver_area_size(svq);
1154     size_t avail_offset;
1155     bool ok;
1156 
1157     vhost_svq_get_vring_addr(svq, &svq_addr);
1158 
1159     driver_region = (DMAMap) {
1160         .translated_addr = svq_addr.desc_user_addr,
1161         .size = driver_size - 1,
1162         .perm = IOMMU_RO,
1163     };
1164     ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
1165     if (unlikely(!ok)) {
1166         error_prepend(errp, "Cannot create vq driver region: ");
1167         return false;
1168     }
1169     addr->desc_user_addr = driver_region.iova;
1170     avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
1171     addr->avail_user_addr = driver_region.iova + avail_offset;
1172 
1173     device_region = (DMAMap) {
1174         .translated_addr = svq_addr.used_user_addr,
1175         .size = device_size - 1,
1176         .perm = IOMMU_RW,
1177     };
1178     ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
1179     if (unlikely(!ok)) {
1180         error_prepend(errp, "Cannot create vq device region: ");
1181         vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr);
1182     }
1183     addr->used_user_addr = device_region.iova;
1184 
1185     return ok;
1186 }
1187 
1188 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
1189                                  VhostShadowVirtqueue *svq, unsigned idx,
1190                                  Error **errp)
1191 {
1192     uint16_t vq_index = dev->vq_index + idx;
1193     struct vhost_vring_state s = {
1194         .index = vq_index,
1195     };
1196     int r;
1197 
1198     r = vhost_vdpa_set_dev_vring_base(dev, &s);
1199     if (unlikely(r)) {
1200         error_setg_errno(errp, -r, "Cannot set vring base");
1201         return false;
1202     }
1203 
1204     r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
1205     return r == 0;
1206 }
1207 
1208 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
1209 {
1210     struct vhost_vdpa *v = dev->opaque;
1211     Error *err = NULL;
1212     unsigned i;
1213 
1214     if (!v->shadow_vqs_enabled) {
1215         return true;
1216     }
1217 
1218     for (i = 0; i < v->shadow_vqs->len; ++i) {
1219         VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
1220         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1221         struct vhost_vring_addr addr = {
1222             .index = dev->vq_index + i,
1223         };
1224         int r;
1225         bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
1226         if (unlikely(!ok)) {
1227             goto err;
1228         }
1229 
1230         vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree);
1231         ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
1232         if (unlikely(!ok)) {
1233             goto err_map;
1234         }
1235 
1236         /* Override vring GPA set by vhost subsystem */
1237         r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
1238         if (unlikely(r != 0)) {
1239             error_setg_errno(&err, -r, "Cannot set device address");
1240             goto err_set_addr;
1241         }
1242     }
1243 
1244     return true;
1245 
1246 err_set_addr:
1247     vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
1248 
1249 err_map:
1250     vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
1251 
1252 err:
1253     error_reportf_err(err, "Cannot setup SVQ %u: ", i);
1254     for (unsigned j = 0; j < i; ++j) {
1255         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
1256         vhost_vdpa_svq_unmap_rings(dev, svq);
1257         vhost_svq_stop(svq);
1258     }
1259 
1260     return false;
1261 }
1262 
1263 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev)
1264 {
1265     struct vhost_vdpa *v = dev->opaque;
1266 
1267     if (!v->shadow_vqs_enabled) {
1268         return;
1269     }
1270 
1271     for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
1272         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1273 
1274         vhost_svq_stop(svq);
1275         vhost_vdpa_svq_unmap_rings(dev, svq);
1276 
1277         event_notifier_cleanup(&svq->hdev_kick);
1278         event_notifier_cleanup(&svq->hdev_call);
1279     }
1280 }
1281 
1282 static void vhost_vdpa_suspend(struct vhost_dev *dev)
1283 {
1284     struct vhost_vdpa *v = dev->opaque;
1285     int r;
1286 
1287     if (!vhost_vdpa_first_dev(dev)) {
1288         return;
1289     }
1290 
1291     if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) {
1292         trace_vhost_vdpa_suspend(dev);
1293         r = ioctl(v->shared->device_fd, VHOST_VDPA_SUSPEND);
1294         if (unlikely(r)) {
1295             error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno);
1296         } else {
1297             v->suspended = true;
1298             return;
1299         }
1300     }
1301 
1302     vhost_vdpa_reset_device(dev);
1303 }
1304 
1305 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
1306 {
1307     struct vhost_vdpa *v = dev->opaque;
1308     bool ok;
1309     trace_vhost_vdpa_dev_start(dev, started);
1310 
1311     if (started) {
1312         vhost_vdpa_host_notifiers_init(dev);
1313         ok = vhost_vdpa_svqs_start(dev);
1314         if (unlikely(!ok)) {
1315             return -1;
1316         }
1317     } else {
1318         vhost_vdpa_suspend(dev);
1319         vhost_vdpa_svqs_stop(dev);
1320         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
1321     }
1322 
1323     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1324         return 0;
1325     }
1326 
1327     if (started) {
1328         if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) {
1329             error_report("SVQ can not work while IOMMU enable, please disable"
1330                          "IOMMU and try again");
1331             return -1;
1332         }
1333         memory_listener_register(&v->listener, dev->vdev->dma_as);
1334 
1335         return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1336     }
1337 
1338     return 0;
1339 }
1340 
1341 static void vhost_vdpa_reset_status(struct vhost_dev *dev)
1342 {
1343     struct vhost_vdpa *v = dev->opaque;
1344 
1345     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1346         return;
1347     }
1348 
1349     vhost_vdpa_reset_device(dev);
1350     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
1351                                VIRTIO_CONFIG_S_DRIVER);
1352     memory_listener_unregister(&v->listener);
1353 }
1354 
1355 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
1356                                      struct vhost_log *log)
1357 {
1358     struct vhost_vdpa *v = dev->opaque;
1359     if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) {
1360         return 0;
1361     }
1362 
1363     trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd,
1364                                   log->log);
1365     return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
1366 }
1367 
1368 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
1369                                        struct vhost_vring_addr *addr)
1370 {
1371     struct vhost_vdpa *v = dev->opaque;
1372 
1373     if (v->shadow_vqs_enabled) {
1374         /*
1375          * Device vring addr was set at device start. SVQ base is handled by
1376          * VirtQueue code.
1377          */
1378         return 0;
1379     }
1380 
1381     return vhost_vdpa_set_vring_dev_addr(dev, addr);
1382 }
1383 
1384 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
1385                                       struct vhost_vring_state *ring)
1386 {
1387     trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num);
1388     return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
1389 }
1390 
1391 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
1392                                        struct vhost_vring_state *ring)
1393 {
1394     struct vhost_vdpa *v = dev->opaque;
1395 
1396     if (v->shadow_vqs_enabled) {
1397         /*
1398          * Device vring base was set at device start. SVQ base is handled by
1399          * VirtQueue code.
1400          */
1401         return 0;
1402     }
1403 
1404     return vhost_vdpa_set_dev_vring_base(dev, ring);
1405 }
1406 
1407 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
1408                                        struct vhost_vring_state *ring)
1409 {
1410     struct vhost_vdpa *v = dev->opaque;
1411     int ret;
1412 
1413     if (v->shadow_vqs_enabled) {
1414         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
1415         return 0;
1416     }
1417 
1418     if (!v->suspended) {
1419         /*
1420          * Cannot trust in value returned by device, let vhost recover used
1421          * idx from guest.
1422          */
1423         return -1;
1424     }
1425 
1426     ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
1427     trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
1428     return ret;
1429 }
1430 
1431 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
1432                                        struct vhost_vring_file *file)
1433 {
1434     struct vhost_vdpa *v = dev->opaque;
1435     int vdpa_idx = file->index - dev->vq_index;
1436 
1437     if (v->shadow_vqs_enabled) {
1438         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1439         vhost_svq_set_svq_kick_fd(svq, file->fd);
1440         return 0;
1441     } else {
1442         return vhost_vdpa_set_vring_dev_kick(dev, file);
1443     }
1444 }
1445 
1446 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
1447                                        struct vhost_vring_file *file)
1448 {
1449     struct vhost_vdpa *v = dev->opaque;
1450     int vdpa_idx = file->index - dev->vq_index;
1451     VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1452 
1453     /* Remember last call fd because we can switch to SVQ anytime. */
1454     vhost_svq_set_svq_call_fd(svq, file->fd);
1455     if (v->shadow_vqs_enabled) {
1456         return 0;
1457     }
1458 
1459     return vhost_vdpa_set_vring_dev_call(dev, file);
1460 }
1461 
1462 static int vhost_vdpa_get_features(struct vhost_dev *dev,
1463                                      uint64_t *features)
1464 {
1465     int ret = vhost_vdpa_get_dev_features(dev, features);
1466 
1467     if (ret == 0) {
1468         /* Add SVQ logging capabilities */
1469         *features |= BIT_ULL(VHOST_F_LOG_ALL);
1470     }
1471 
1472     return ret;
1473 }
1474 
1475 static int vhost_vdpa_set_owner(struct vhost_dev *dev)
1476 {
1477     if (!vhost_vdpa_first_dev(dev)) {
1478         return 0;
1479     }
1480 
1481     trace_vhost_vdpa_set_owner(dev);
1482     return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
1483 }
1484 
1485 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
1486                     struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
1487 {
1488     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
1489     addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
1490     addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
1491     addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
1492     trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr,
1493                                  addr->avail_user_addr, addr->used_user_addr);
1494     return 0;
1495 }
1496 
1497 static bool  vhost_vdpa_force_iommu(struct vhost_dev *dev)
1498 {
1499     return true;
1500 }
1501 
1502 const VhostOps vdpa_ops = {
1503         .backend_type = VHOST_BACKEND_TYPE_VDPA,
1504         .vhost_backend_init = vhost_vdpa_init,
1505         .vhost_backend_cleanup = vhost_vdpa_cleanup,
1506         .vhost_set_log_base = vhost_vdpa_set_log_base,
1507         .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
1508         .vhost_set_vring_num = vhost_vdpa_set_vring_num,
1509         .vhost_set_vring_base = vhost_vdpa_set_vring_base,
1510         .vhost_get_vring_base = vhost_vdpa_get_vring_base,
1511         .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
1512         .vhost_set_vring_call = vhost_vdpa_set_vring_call,
1513         .vhost_get_features = vhost_vdpa_get_features,
1514         .vhost_set_backend_cap = vhost_vdpa_set_backend_cap,
1515         .vhost_set_owner = vhost_vdpa_set_owner,
1516         .vhost_set_vring_endian = NULL,
1517         .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
1518         .vhost_set_mem_table = vhost_vdpa_set_mem_table,
1519         .vhost_set_features = vhost_vdpa_set_features,
1520         .vhost_reset_device = vhost_vdpa_reset_device,
1521         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
1522         .vhost_get_config  = vhost_vdpa_get_config,
1523         .vhost_set_config = vhost_vdpa_set_config,
1524         .vhost_requires_shm_log = NULL,
1525         .vhost_migration_done = NULL,
1526         .vhost_net_set_mtu = NULL,
1527         .vhost_set_iotlb_callback = NULL,
1528         .vhost_send_device_iotlb_msg = NULL,
1529         .vhost_dev_start = vhost_vdpa_dev_start,
1530         .vhost_get_device_id = vhost_vdpa_get_device_id,
1531         .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
1532         .vhost_force_iommu = vhost_vdpa_force_iommu,
1533         .vhost_set_config_call = vhost_vdpa_set_config_call,
1534         .vhost_reset_status = vhost_vdpa_reset_status,
1535 };
1536