xref: /openbmc/qemu/hw/virtio/vhost-vdpa.c (revision 8c5e9809225c387026476f5eefc2f8ae749a72f7)
1 /*
2  * vhost-vdpa
3  *
4  *  Copyright(c) 2017-2018 Intel Corporation.
5  *  Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include <linux/vhost.h>
14 #include <linux/vfio.h>
15 #include <sys/eventfd.h>
16 #include <sys/ioctl.h>
17 #include "exec/target_page.h"
18 #include "hw/virtio/vhost.h"
19 #include "hw/virtio/vhost-backend.h"
20 #include "hw/virtio/virtio-net.h"
21 #include "hw/virtio/vhost-shadow-virtqueue.h"
22 #include "hw/virtio/vhost-vdpa.h"
23 #include "exec/address-spaces.h"
24 #include "migration/blocker.h"
25 #include "qemu/cutils.h"
26 #include "qemu/main-loop.h"
27 #include "trace.h"
28 #include "qapi/error.h"
29 
30 /*
31  * Return one past the end of the end of section. Be careful with uint64_t
32  * conversions!
33  */
34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section,
35                                      int page_mask)
36 {
37     Int128 llend = int128_make64(section->offset_within_address_space);
38     llend = int128_add(llend, section->size);
39     llend = int128_and(llend, int128_exts64(page_mask));
40 
41     return llend;
42 }
43 
44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section,
45                                                 uint64_t iova_min,
46                                                 uint64_t iova_max,
47                                                 int page_mask)
48 {
49     Int128 llend;
50 
51     if ((!memory_region_is_ram(section->mr) &&
52          !memory_region_is_iommu(section->mr)) ||
53         memory_region_is_protected(section->mr) ||
54         /* vhost-vDPA doesn't allow MMIO to be mapped  */
55         memory_region_is_ram_device(section->mr)) {
56         return true;
57     }
58 
59     if (section->offset_within_address_space < iova_min) {
60         error_report("RAM section out of device range (min=0x%" PRIx64
61                      ", addr=0x%" HWADDR_PRIx ")",
62                      iova_min, section->offset_within_address_space);
63         return true;
64     }
65     /*
66      * While using vIOMMU, sometimes the section will be larger than iova_max,
67      * but the memory that actually maps is smaller, so move the check to
68      * function vhost_vdpa_iommu_map_notify(). That function will use the actual
69      * size that maps to the kernel
70      */
71 
72     if (!memory_region_is_iommu(section->mr)) {
73         llend = vhost_vdpa_section_end(section, page_mask);
74         if (int128_gt(llend, int128_make64(iova_max))) {
75             error_report("RAM section out of device range (max=0x%" PRIx64
76                          ", end addr=0x%" PRIx64 ")",
77                          iova_max, int128_get64(llend));
78             return true;
79         }
80     }
81 
82     return false;
83 }
84 
85 /*
86  * The caller must set asid = 0 if the device does not support asid.
87  * This is not an ABI break since it is set to 0 by the initializer anyway.
88  */
89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
90                        hwaddr size, void *vaddr, bool readonly)
91 {
92     struct vhost_msg_v2 msg = {};
93     int fd = v->device_fd;
94     int ret = 0;
95 
96     msg.type = v->msg_type;
97     msg.asid = asid;
98     msg.iotlb.iova = iova;
99     msg.iotlb.size = size;
100     msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
101     msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
102     msg.iotlb.type = VHOST_IOTLB_UPDATE;
103 
104     trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova,
105                              msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm,
106                              msg.iotlb.type);
107 
108     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
109         error_report("failed to write, fd=%d, errno=%d (%s)",
110             fd, errno, strerror(errno));
111         return -EIO ;
112     }
113 
114     return ret;
115 }
116 
117 /*
118  * The caller must set asid = 0 if the device does not support asid.
119  * This is not an ABI break since it is set to 0 by the initializer anyway.
120  */
121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
122                          hwaddr size)
123 {
124     struct vhost_msg_v2 msg = {};
125     int fd = v->device_fd;
126     int ret = 0;
127 
128     msg.type = v->msg_type;
129     msg.asid = asid;
130     msg.iotlb.iova = iova;
131     msg.iotlb.size = size;
132     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
133 
134     trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova,
135                                msg.iotlb.size, msg.iotlb.type);
136 
137     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
138         error_report("failed to write, fd=%d, errno=%d (%s)",
139             fd, errno, strerror(errno));
140         return -EIO ;
141     }
142 
143     return ret;
144 }
145 
146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v)
147 {
148     int fd = v->device_fd;
149     struct vhost_msg_v2 msg = {
150         .type = v->msg_type,
151         .iotlb.type = VHOST_IOTLB_BATCH_BEGIN,
152     };
153 
154     trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type);
155     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
156         error_report("failed to write, fd=%d, errno=%d (%s)",
157                      fd, errno, strerror(errno));
158     }
159 }
160 
161 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v)
162 {
163     if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) &&
164         !v->iotlb_batch_begin_sent) {
165         vhost_vdpa_listener_begin_batch(v);
166     }
167 
168     v->iotlb_batch_begin_sent = true;
169 }
170 
171 static void vhost_vdpa_listener_commit(MemoryListener *listener)
172 {
173     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
174     struct vhost_dev *dev = v->dev;
175     struct vhost_msg_v2 msg = {};
176     int fd = v->device_fd;
177 
178     if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
179         return;
180     }
181 
182     if (!v->iotlb_batch_begin_sent) {
183         return;
184     }
185 
186     msg.type = v->msg_type;
187     msg.iotlb.type = VHOST_IOTLB_BATCH_END;
188 
189     trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type);
190     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
191         error_report("failed to write, fd=%d, errno=%d (%s)",
192                      fd, errno, strerror(errno));
193     }
194 
195     v->iotlb_batch_begin_sent = false;
196 }
197 
198 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
199 {
200     struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n);
201 
202     hwaddr iova = iotlb->iova + iommu->iommu_offset;
203     struct vhost_vdpa *v = iommu->dev;
204     void *vaddr;
205     int ret;
206     Int128 llend;
207 
208     if (iotlb->target_as != &address_space_memory) {
209         error_report("Wrong target AS \"%s\", only system memory is allowed",
210                      iotlb->target_as->name ? iotlb->target_as->name : "none");
211         return;
212     }
213     RCU_READ_LOCK_GUARD();
214     /* check if RAM section out of device range */
215     llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova));
216     if (int128_gt(llend, int128_make64(v->iova_range.last))) {
217         error_report("RAM section out of device range (max=0x%" PRIx64
218                      ", end addr=0x%" PRIx64 ")",
219                      v->iova_range.last, int128_get64(llend));
220         return;
221     }
222 
223     if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
224         bool read_only;
225 
226         if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) {
227             return;
228         }
229         ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
230                                  iotlb->addr_mask + 1, vaddr, read_only);
231         if (ret) {
232             error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", "
233                          "0x%" HWADDR_PRIx ", %p) = %d (%m)",
234                          v, iova, iotlb->addr_mask + 1, vaddr, ret);
235         }
236     } else {
237         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
238                                    iotlb->addr_mask + 1);
239         if (ret) {
240             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
241                          "0x%" HWADDR_PRIx ") = %d (%m)",
242                          v, iova, iotlb->addr_mask + 1, ret);
243         }
244     }
245 }
246 
247 static void vhost_vdpa_iommu_region_add(MemoryListener *listener,
248                                         MemoryRegionSection *section)
249 {
250     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
251 
252     struct vdpa_iommu *iommu;
253     Int128 end;
254     int iommu_idx;
255     IOMMUMemoryRegion *iommu_mr;
256     int ret;
257 
258     iommu_mr = IOMMU_MEMORY_REGION(section->mr);
259 
260     iommu = g_malloc0(sizeof(*iommu));
261     end = int128_add(int128_make64(section->offset_within_region),
262                      section->size);
263     end = int128_sub(end, int128_one());
264     iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
265                                                    MEMTXATTRS_UNSPECIFIED);
266     iommu->iommu_mr = iommu_mr;
267     iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify,
268                         IOMMU_NOTIFIER_IOTLB_EVENTS,
269                         section->offset_within_region,
270                         int128_get64(end),
271                         iommu_idx);
272     iommu->iommu_offset = section->offset_within_address_space -
273                           section->offset_within_region;
274     iommu->dev = v;
275 
276     ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
277     if (ret) {
278         g_free(iommu);
279         return;
280     }
281 
282     QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next);
283     memory_region_iommu_replay(iommu->iommu_mr, &iommu->n);
284 
285     return;
286 }
287 
288 static void vhost_vdpa_iommu_region_del(MemoryListener *listener,
289                                         MemoryRegionSection *section)
290 {
291     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
292 
293     struct vdpa_iommu *iommu;
294 
295     QLIST_FOREACH(iommu, &v->iommu_list, iommu_next)
296     {
297         if (MEMORY_REGION(iommu->iommu_mr) == section->mr &&
298             iommu->n.start == section->offset_within_region) {
299             memory_region_unregister_iommu_notifier(section->mr, &iommu->n);
300             QLIST_REMOVE(iommu, iommu_next);
301             g_free(iommu);
302             break;
303         }
304     }
305 }
306 
307 static void vhost_vdpa_listener_region_add(MemoryListener *listener,
308                                            MemoryRegionSection *section)
309 {
310     DMAMap mem_region = {};
311     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
312     hwaddr iova;
313     Int128 llend, llsize;
314     void *vaddr;
315     int ret;
316     int page_size = qemu_target_page_size();
317     int page_mask = -page_size;
318 
319     if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first,
320                                             v->iova_range.last, page_mask)) {
321         return;
322     }
323     if (memory_region_is_iommu(section->mr)) {
324         vhost_vdpa_iommu_region_add(listener, section);
325         return;
326     }
327 
328     if (unlikely((section->offset_within_address_space & ~page_mask) !=
329                  (section->offset_within_region & ~page_mask))) {
330         trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name,
331                        section->offset_within_address_space & ~page_mask,
332                        section->offset_within_region & ~page_mask);
333         return;
334     }
335 
336     iova = ROUND_UP(section->offset_within_address_space, page_size);
337     llend = vhost_vdpa_section_end(section, page_mask);
338     if (int128_ge(int128_make64(iova), llend)) {
339         return;
340     }
341 
342     memory_region_ref(section->mr);
343 
344     /* Here we assume that memory_region_is_ram(section->mr)==true */
345 
346     vaddr = memory_region_get_ram_ptr(section->mr) +
347             section->offset_within_region +
348             (iova - section->offset_within_address_space);
349 
350     trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend),
351                                          vaddr, section->readonly);
352 
353     llsize = int128_sub(llend, int128_make64(iova));
354     if (v->shadow_data) {
355         int r;
356 
357         mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
358         mem_region.size = int128_get64(llsize) - 1,
359         mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
360 
361         r = vhost_iova_tree_map_alloc(v->iova_tree, &mem_region);
362         if (unlikely(r != IOVA_OK)) {
363             error_report("Can't allocate a mapping (%d)", r);
364             goto fail;
365         }
366 
367         iova = mem_region.iova;
368     }
369 
370     vhost_vdpa_iotlb_batch_begin_once(v);
371     ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
372                              int128_get64(llsize), vaddr, section->readonly);
373     if (ret) {
374         error_report("vhost vdpa map fail!");
375         goto fail_map;
376     }
377 
378     return;
379 
380 fail_map:
381     if (v->shadow_data) {
382         vhost_iova_tree_remove(v->iova_tree, mem_region);
383     }
384 
385 fail:
386     /*
387      * On the initfn path, store the first error in the container so we
388      * can gracefully fail.  Runtime, there's not much we can do other
389      * than throw a hardware error.
390      */
391     error_report("vhost-vdpa: DMA mapping failed, unable to continue");
392     return;
393 
394 }
395 
396 static void vhost_vdpa_listener_region_del(MemoryListener *listener,
397                                            MemoryRegionSection *section)
398 {
399     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
400     hwaddr iova;
401     Int128 llend, llsize;
402     int ret;
403     int page_size = qemu_target_page_size();
404     int page_mask = -page_size;
405 
406     if (vhost_vdpa_listener_skipped_section(section, v->iova_range.first,
407                                             v->iova_range.last, page_mask)) {
408         return;
409     }
410     if (memory_region_is_iommu(section->mr)) {
411         vhost_vdpa_iommu_region_del(listener, section);
412     }
413 
414     if (unlikely((section->offset_within_address_space & ~page_mask) !=
415                  (section->offset_within_region & ~page_mask))) {
416         trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name,
417                        section->offset_within_address_space & ~page_mask,
418                        section->offset_within_region & ~page_mask);
419         return;
420     }
421 
422     iova = ROUND_UP(section->offset_within_address_space, page_size);
423     llend = vhost_vdpa_section_end(section, page_mask);
424 
425     trace_vhost_vdpa_listener_region_del(v, iova,
426         int128_get64(int128_sub(llend, int128_one())));
427 
428     if (int128_ge(int128_make64(iova), llend)) {
429         return;
430     }
431 
432     llsize = int128_sub(llend, int128_make64(iova));
433 
434     if (v->shadow_data) {
435         const DMAMap *result;
436         const void *vaddr = memory_region_get_ram_ptr(section->mr) +
437             section->offset_within_region +
438             (iova - section->offset_within_address_space);
439         DMAMap mem_region = {
440             .translated_addr = (hwaddr)(uintptr_t)vaddr,
441             .size = int128_get64(llsize) - 1,
442         };
443 
444         result = vhost_iova_tree_find_iova(v->iova_tree, &mem_region);
445         if (!result) {
446             /* The memory listener map wasn't mapped */
447             return;
448         }
449         iova = result->iova;
450         vhost_iova_tree_remove(v->iova_tree, *result);
451     }
452     vhost_vdpa_iotlb_batch_begin_once(v);
453     /*
454      * The unmap ioctl doesn't accept a full 64-bit. need to check it
455      */
456     if (int128_eq(llsize, int128_2_64())) {
457         llsize = int128_rshift(llsize, 1);
458         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
459                                    int128_get64(llsize));
460 
461         if (ret) {
462             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
463                          "0x%" HWADDR_PRIx ") = %d (%m)",
464                          v, iova, int128_get64(llsize), ret);
465         }
466         iova += int128_get64(llsize);
467     }
468     ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
469                                int128_get64(llsize));
470 
471     if (ret) {
472         error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
473                      "0x%" HWADDR_PRIx ") = %d (%m)",
474                      v, iova, int128_get64(llsize), ret);
475     }
476 
477     memory_region_unref(section->mr);
478 }
479 /*
480  * IOTLB API is used by vhost-vdpa which requires incremental updating
481  * of the mapping. So we can not use generic vhost memory listener which
482  * depends on the addnop().
483  */
484 static const MemoryListener vhost_vdpa_memory_listener = {
485     .name = "vhost-vdpa",
486     .commit = vhost_vdpa_listener_commit,
487     .region_add = vhost_vdpa_listener_region_add,
488     .region_del = vhost_vdpa_listener_region_del,
489 };
490 
491 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
492                              void *arg)
493 {
494     struct vhost_vdpa *v = dev->opaque;
495     int fd = v->device_fd;
496     int ret;
497 
498     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
499 
500     ret = ioctl(fd, request, arg);
501     return ret < 0 ? -errno : ret;
502 }
503 
504 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
505 {
506     uint8_t s;
507     int ret;
508 
509     trace_vhost_vdpa_add_status(dev, status);
510     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
511     if (ret < 0) {
512         return ret;
513     }
514     if ((s & status) == status) {
515         /* Don't set bits already set */
516         return 0;
517     }
518 
519     s |= status;
520 
521     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
522     if (ret < 0) {
523         return ret;
524     }
525 
526     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
527     if (ret < 0) {
528         return ret;
529     }
530 
531     if (!(s & status)) {
532         return -EIO;
533     }
534 
535     return 0;
536 }
537 
538 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range)
539 {
540     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
541 
542     return ret < 0 ? -errno : 0;
543 }
544 
545 /*
546  * The use of this function is for requests that only need to be
547  * applied once. Typically such request occurs at the beginning
548  * of operation, and before setting up queues. It should not be
549  * used for request that performs operation until all queues are
550  * set, which would need to check dev->vq_index_end instead.
551  */
552 static bool vhost_vdpa_first_dev(struct vhost_dev *dev)
553 {
554     struct vhost_vdpa *v = dev->opaque;
555 
556     return v->index == 0;
557 }
558 
559 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
560                                        uint64_t *features)
561 {
562     int ret;
563 
564     ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
565     trace_vhost_vdpa_get_features(dev, *features);
566     return ret;
567 }
568 
569 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v)
570 {
571     g_autoptr(GPtrArray) shadow_vqs = NULL;
572 
573     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
574     for (unsigned n = 0; n < hdev->nvqs; ++n) {
575         VhostShadowVirtqueue *svq;
576 
577         svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque);
578         g_ptr_array_add(shadow_vqs, svq);
579     }
580 
581     v->shadow_vqs = g_steal_pointer(&shadow_vqs);
582 }
583 
584 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
585 {
586     struct vhost_vdpa *v;
587     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
588     trace_vhost_vdpa_init(dev, opaque);
589     int ret;
590 
591     v = opaque;
592     v->dev = dev;
593     dev->opaque =  opaque ;
594     v->listener = vhost_vdpa_memory_listener;
595     v->msg_type = VHOST_IOTLB_MSG_V2;
596     vhost_vdpa_init_svq(dev, v);
597 
598     error_propagate(&dev->migration_blocker, v->migration_blocker);
599     if (!vhost_vdpa_first_dev(dev)) {
600         return 0;
601     }
602 
603     /*
604      * If dev->shadow_vqs_enabled at initialization that means the device has
605      * been started with x-svq=on, so don't block migration
606      */
607     if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) {
608         /* We don't have dev->features yet */
609         uint64_t features;
610         ret = vhost_vdpa_get_dev_features(dev, &features);
611         if (unlikely(ret)) {
612             error_setg_errno(errp, -ret, "Could not get device features");
613             return ret;
614         }
615         vhost_svq_valid_features(features, &dev->migration_blocker);
616     }
617 
618     /*
619      * Similar to VFIO, we end up pinning all guest memory and have to
620      * disable discarding of RAM.
621      */
622     ret = ram_block_discard_disable(true);
623     if (ret) {
624         error_report("Cannot set discarding of RAM broken");
625         return ret;
626     }
627 
628     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
629                                VIRTIO_CONFIG_S_DRIVER);
630 
631     return 0;
632 }
633 
634 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
635                                             int queue_index)
636 {
637     size_t page_size = qemu_real_host_page_size();
638     struct vhost_vdpa *v = dev->opaque;
639     VirtIODevice *vdev = dev->vdev;
640     VhostVDPAHostNotifier *n;
641 
642     n = &v->notifier[queue_index];
643 
644     if (n->addr) {
645         virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
646         object_unparent(OBJECT(&n->mr));
647         munmap(n->addr, page_size);
648         n->addr = NULL;
649     }
650 }
651 
652 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
653 {
654     size_t page_size = qemu_real_host_page_size();
655     struct vhost_vdpa *v = dev->opaque;
656     VirtIODevice *vdev = dev->vdev;
657     VhostVDPAHostNotifier *n;
658     int fd = v->device_fd;
659     void *addr;
660     char *name;
661 
662     vhost_vdpa_host_notifier_uninit(dev, queue_index);
663 
664     n = &v->notifier[queue_index];
665 
666     addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
667                 queue_index * page_size);
668     if (addr == MAP_FAILED) {
669         goto err;
670     }
671 
672     name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
673                            v, queue_index);
674     memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
675                                       page_size, addr);
676     g_free(name);
677 
678     if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
679         object_unparent(OBJECT(&n->mr));
680         munmap(addr, page_size);
681         goto err;
682     }
683     n->addr = addr;
684 
685     return 0;
686 
687 err:
688     return -1;
689 }
690 
691 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
692 {
693     int i;
694 
695     /*
696      * Pack all the changes to the memory regions in a single
697      * transaction to avoid a few updating of the address space
698      * topology.
699      */
700     memory_region_transaction_begin();
701 
702     for (i = dev->vq_index; i < dev->vq_index + n; i++) {
703         vhost_vdpa_host_notifier_uninit(dev, i);
704     }
705 
706     memory_region_transaction_commit();
707 }
708 
709 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
710 {
711     struct vhost_vdpa *v = dev->opaque;
712     int i;
713 
714     if (v->shadow_vqs_enabled) {
715         /* FIXME SVQ is not compatible with host notifiers mr */
716         return;
717     }
718 
719     /*
720      * Pack all the changes to the memory regions in a single
721      * transaction to avoid a few updating of the address space
722      * topology.
723      */
724     memory_region_transaction_begin();
725 
726     for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
727         if (vhost_vdpa_host_notifier_init(dev, i)) {
728             vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index);
729             break;
730         }
731     }
732 
733     memory_region_transaction_commit();
734 }
735 
736 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
737 {
738     struct vhost_vdpa *v = dev->opaque;
739     size_t idx;
740 
741     for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
742         vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
743     }
744     g_ptr_array_free(v->shadow_vqs, true);
745 }
746 
747 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
748 {
749     struct vhost_vdpa *v;
750     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
751     v = dev->opaque;
752     trace_vhost_vdpa_cleanup(dev, v);
753     if (vhost_vdpa_first_dev(dev)) {
754         ram_block_discard_disable(false);
755     }
756 
757     vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
758     memory_listener_unregister(&v->listener);
759     vhost_vdpa_svq_cleanup(dev);
760 
761     dev->opaque = NULL;
762 
763     return 0;
764 }
765 
766 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
767 {
768     trace_vhost_vdpa_memslots_limit(dev, INT_MAX);
769     return INT_MAX;
770 }
771 
772 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
773                                     struct vhost_memory *mem)
774 {
775     if (!vhost_vdpa_first_dev(dev)) {
776         return 0;
777     }
778 
779     trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding);
780     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) &&
781         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) {
782         int i;
783         for (i = 0; i < mem->nregions; i++) {
784             trace_vhost_vdpa_dump_regions(dev, i,
785                                           mem->regions[i].guest_phys_addr,
786                                           mem->regions[i].memory_size,
787                                           mem->regions[i].userspace_addr,
788                                           mem->regions[i].flags_padding);
789         }
790     }
791     if (mem->padding) {
792         return -EINVAL;
793     }
794 
795     return 0;
796 }
797 
798 static int vhost_vdpa_set_features(struct vhost_dev *dev,
799                                    uint64_t features)
800 {
801     struct vhost_vdpa *v = dev->opaque;
802     int ret;
803 
804     if (!vhost_vdpa_first_dev(dev)) {
805         return 0;
806     }
807 
808     if (v->shadow_vqs_enabled) {
809         if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
810             /*
811              * QEMU is just trying to enable or disable logging. SVQ handles
812              * this sepparately, so no need to forward this.
813              */
814             v->acked_features = features;
815             return 0;
816         }
817 
818         v->acked_features = features;
819 
820         /* We must not ack _F_LOG if SVQ is enabled */
821         features &= ~BIT_ULL(VHOST_F_LOG_ALL);
822     }
823 
824     trace_vhost_vdpa_set_features(dev, features);
825     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
826     if (ret) {
827         return ret;
828     }
829 
830     return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
831 }
832 
833 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev)
834 {
835     uint64_t features;
836     uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 |
837         0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH |
838         0x1ULL << VHOST_BACKEND_F_IOTLB_ASID |
839         0x1ULL << VHOST_BACKEND_F_SUSPEND;
840     int r;
841 
842     if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
843         return -EFAULT;
844     }
845 
846     features &= f;
847 
848     if (vhost_vdpa_first_dev(dev)) {
849         r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features);
850         if (r) {
851             return -EFAULT;
852         }
853     }
854 
855     dev->backend_cap = features;
856 
857     return 0;
858 }
859 
860 static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
861                                     uint32_t *device_id)
862 {
863     int ret;
864     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
865     trace_vhost_vdpa_get_device_id(dev, *device_id);
866     return ret;
867 }
868 
869 static int vhost_vdpa_reset_device(struct vhost_dev *dev)
870 {
871     struct vhost_vdpa *v = dev->opaque;
872     int ret;
873     uint8_t status = 0;
874 
875     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
876     trace_vhost_vdpa_reset_device(dev);
877     v->suspended = false;
878     return ret;
879 }
880 
881 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
882 {
883     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
884 
885     trace_vhost_vdpa_get_vq_index(dev, idx, idx);
886     return idx;
887 }
888 
889 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx)
890 {
891     struct vhost_dev *dev = v->dev;
892     struct vhost_vring_state state = {
893         .index = idx,
894         .num = 1,
895     };
896     int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
897 
898     trace_vhost_vdpa_set_vring_ready(dev, idx, r);
899     return r;
900 }
901 
902 static int vhost_vdpa_set_config_call(struct vhost_dev *dev,
903                                        int fd)
904 {
905     trace_vhost_vdpa_set_config_call(dev, fd);
906     return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd);
907 }
908 
909 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config,
910                                    uint32_t config_len)
911 {
912     int b, len;
913     char line[QEMU_HEXDUMP_LINE_LEN];
914 
915     for (b = 0; b < config_len; b += 16) {
916         len = config_len - b;
917         qemu_hexdump_line(line, b, config, len, false);
918         trace_vhost_vdpa_dump_config(dev, line);
919     }
920 }
921 
922 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
923                                    uint32_t offset, uint32_t size,
924                                    uint32_t flags)
925 {
926     struct vhost_vdpa_config *config;
927     int ret;
928     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
929 
930     trace_vhost_vdpa_set_config(dev, offset, size, flags);
931     config = g_malloc(size + config_size);
932     config->off = offset;
933     config->len = size;
934     memcpy(config->buf, data, size);
935     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) &&
936         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
937         vhost_vdpa_dump_config(dev, data, size);
938     }
939     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
940     g_free(config);
941     return ret;
942 }
943 
944 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
945                                    uint32_t config_len, Error **errp)
946 {
947     struct vhost_vdpa_config *v_config;
948     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
949     int ret;
950 
951     trace_vhost_vdpa_get_config(dev, config, config_len);
952     v_config = g_malloc(config_len + config_size);
953     v_config->len = config_len;
954     v_config->off = 0;
955     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
956     memcpy(config, v_config->buf, config_len);
957     g_free(v_config);
958     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) &&
959         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
960         vhost_vdpa_dump_config(dev, config, config_len);
961     }
962     return ret;
963  }
964 
965 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
966                                          struct vhost_vring_state *ring)
967 {
968     trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
969     return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
970 }
971 
972 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
973                                          struct vhost_vring_file *file)
974 {
975     trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
976     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
977 }
978 
979 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
980                                          struct vhost_vring_file *file)
981 {
982     trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
983     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
984 }
985 
986 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
987                                          struct vhost_vring_addr *addr)
988 {
989     trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
990                                 addr->desc_user_addr, addr->used_user_addr,
991                                 addr->avail_user_addr,
992                                 addr->log_guest_addr);
993 
994     return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
995 
996 }
997 
998 /**
999  * Set the shadow virtqueue descriptors to the device
1000  *
1001  * @dev: The vhost device model
1002  * @svq: The shadow virtqueue
1003  * @idx: The index of the virtqueue in the vhost device
1004  * @errp: Error
1005  *
1006  * Note that this function does not rewind kick file descriptor if cannot set
1007  * call one.
1008  */
1009 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
1010                                   VhostShadowVirtqueue *svq, unsigned idx,
1011                                   Error **errp)
1012 {
1013     struct vhost_vring_file file = {
1014         .index = dev->vq_index + idx,
1015     };
1016     const EventNotifier *event_notifier = &svq->hdev_kick;
1017     int r;
1018 
1019     r = event_notifier_init(&svq->hdev_kick, 0);
1020     if (r != 0) {
1021         error_setg_errno(errp, -r, "Couldn't create kick event notifier");
1022         goto err_init_hdev_kick;
1023     }
1024 
1025     r = event_notifier_init(&svq->hdev_call, 0);
1026     if (r != 0) {
1027         error_setg_errno(errp, -r, "Couldn't create call event notifier");
1028         goto err_init_hdev_call;
1029     }
1030 
1031     file.fd = event_notifier_get_fd(event_notifier);
1032     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
1033     if (unlikely(r != 0)) {
1034         error_setg_errno(errp, -r, "Can't set device kick fd");
1035         goto err_init_set_dev_fd;
1036     }
1037 
1038     event_notifier = &svq->hdev_call;
1039     file.fd = event_notifier_get_fd(event_notifier);
1040     r = vhost_vdpa_set_vring_dev_call(dev, &file);
1041     if (unlikely(r != 0)) {
1042         error_setg_errno(errp, -r, "Can't set device call fd");
1043         goto err_init_set_dev_fd;
1044     }
1045 
1046     return 0;
1047 
1048 err_init_set_dev_fd:
1049     event_notifier_set_handler(&svq->hdev_call, NULL);
1050 
1051 err_init_hdev_call:
1052     event_notifier_cleanup(&svq->hdev_kick);
1053 
1054 err_init_hdev_kick:
1055     return r;
1056 }
1057 
1058 /**
1059  * Unmap a SVQ area in the device
1060  */
1061 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr)
1062 {
1063     const DMAMap needle = {
1064         .translated_addr = addr,
1065     };
1066     const DMAMap *result = vhost_iova_tree_find_iova(v->iova_tree, &needle);
1067     hwaddr size;
1068     int r;
1069 
1070     if (unlikely(!result)) {
1071         error_report("Unable to find SVQ address to unmap");
1072         return;
1073     }
1074 
1075     size = ROUND_UP(result->size, qemu_real_host_page_size());
1076     r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size);
1077     if (unlikely(r < 0)) {
1078         error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r);
1079         return;
1080     }
1081 
1082     vhost_iova_tree_remove(v->iova_tree, *result);
1083 }
1084 
1085 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
1086                                        const VhostShadowVirtqueue *svq)
1087 {
1088     struct vhost_vdpa *v = dev->opaque;
1089     struct vhost_vring_addr svq_addr;
1090 
1091     vhost_svq_get_vring_addr(svq, &svq_addr);
1092 
1093     vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr);
1094 
1095     vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr);
1096 }
1097 
1098 /**
1099  * Map the SVQ area in the device
1100  *
1101  * @v: Vhost-vdpa device
1102  * @needle: The area to search iova
1103  * @errorp: Error pointer
1104  */
1105 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
1106                                     Error **errp)
1107 {
1108     int r;
1109 
1110     r = vhost_iova_tree_map_alloc(v->iova_tree, needle);
1111     if (unlikely(r != IOVA_OK)) {
1112         error_setg(errp, "Cannot allocate iova (%d)", r);
1113         return false;
1114     }
1115 
1116     r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova,
1117                            needle->size + 1,
1118                            (void *)(uintptr_t)needle->translated_addr,
1119                            needle->perm == IOMMU_RO);
1120     if (unlikely(r != 0)) {
1121         error_setg_errno(errp, -r, "Cannot map region to device");
1122         vhost_iova_tree_remove(v->iova_tree, *needle);
1123     }
1124 
1125     return r == 0;
1126 }
1127 
1128 /**
1129  * Map the shadow virtqueue rings in the device
1130  *
1131  * @dev: The vhost device
1132  * @svq: The shadow virtqueue
1133  * @addr: Assigned IOVA addresses
1134  * @errp: Error pointer
1135  */
1136 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
1137                                      const VhostShadowVirtqueue *svq,
1138                                      struct vhost_vring_addr *addr,
1139                                      Error **errp)
1140 {
1141     ERRP_GUARD();
1142     DMAMap device_region, driver_region;
1143     struct vhost_vring_addr svq_addr;
1144     struct vhost_vdpa *v = dev->opaque;
1145     size_t device_size = vhost_svq_device_area_size(svq);
1146     size_t driver_size = vhost_svq_driver_area_size(svq);
1147     size_t avail_offset;
1148     bool ok;
1149 
1150     vhost_svq_get_vring_addr(svq, &svq_addr);
1151 
1152     driver_region = (DMAMap) {
1153         .translated_addr = svq_addr.desc_user_addr,
1154         .size = driver_size - 1,
1155         .perm = IOMMU_RO,
1156     };
1157     ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
1158     if (unlikely(!ok)) {
1159         error_prepend(errp, "Cannot create vq driver region: ");
1160         return false;
1161     }
1162     addr->desc_user_addr = driver_region.iova;
1163     avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
1164     addr->avail_user_addr = driver_region.iova + avail_offset;
1165 
1166     device_region = (DMAMap) {
1167         .translated_addr = svq_addr.used_user_addr,
1168         .size = device_size - 1,
1169         .perm = IOMMU_RW,
1170     };
1171     ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
1172     if (unlikely(!ok)) {
1173         error_prepend(errp, "Cannot create vq device region: ");
1174         vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr);
1175     }
1176     addr->used_user_addr = device_region.iova;
1177 
1178     return ok;
1179 }
1180 
1181 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
1182                                  VhostShadowVirtqueue *svq, unsigned idx,
1183                                  Error **errp)
1184 {
1185     uint16_t vq_index = dev->vq_index + idx;
1186     struct vhost_vring_state s = {
1187         .index = vq_index,
1188     };
1189     int r;
1190 
1191     r = vhost_vdpa_set_dev_vring_base(dev, &s);
1192     if (unlikely(r)) {
1193         error_setg_errno(errp, -r, "Cannot set vring base");
1194         return false;
1195     }
1196 
1197     r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
1198     return r == 0;
1199 }
1200 
1201 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
1202 {
1203     struct vhost_vdpa *v = dev->opaque;
1204     Error *err = NULL;
1205     unsigned i;
1206 
1207     if (!v->shadow_vqs_enabled) {
1208         return true;
1209     }
1210 
1211     for (i = 0; i < v->shadow_vqs->len; ++i) {
1212         VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
1213         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1214         struct vhost_vring_addr addr = {
1215             .index = dev->vq_index + i,
1216         };
1217         int r;
1218         bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
1219         if (unlikely(!ok)) {
1220             goto err;
1221         }
1222 
1223         vhost_svq_start(svq, dev->vdev, vq, v->iova_tree);
1224         ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
1225         if (unlikely(!ok)) {
1226             goto err_map;
1227         }
1228 
1229         /* Override vring GPA set by vhost subsystem */
1230         r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
1231         if (unlikely(r != 0)) {
1232             error_setg_errno(&err, -r, "Cannot set device address");
1233             goto err_set_addr;
1234         }
1235     }
1236 
1237     return true;
1238 
1239 err_set_addr:
1240     vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
1241 
1242 err_map:
1243     vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
1244 
1245 err:
1246     error_reportf_err(err, "Cannot setup SVQ %u: ", i);
1247     for (unsigned j = 0; j < i; ++j) {
1248         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
1249         vhost_vdpa_svq_unmap_rings(dev, svq);
1250         vhost_svq_stop(svq);
1251     }
1252 
1253     return false;
1254 }
1255 
1256 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev)
1257 {
1258     struct vhost_vdpa *v = dev->opaque;
1259 
1260     if (!v->shadow_vqs_enabled) {
1261         return;
1262     }
1263 
1264     for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
1265         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1266 
1267         vhost_svq_stop(svq);
1268         vhost_vdpa_svq_unmap_rings(dev, svq);
1269 
1270         event_notifier_cleanup(&svq->hdev_kick);
1271         event_notifier_cleanup(&svq->hdev_call);
1272     }
1273 }
1274 
1275 static void vhost_vdpa_suspend(struct vhost_dev *dev)
1276 {
1277     struct vhost_vdpa *v = dev->opaque;
1278     int r;
1279 
1280     if (!vhost_vdpa_first_dev(dev)) {
1281         return;
1282     }
1283 
1284     if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) {
1285         trace_vhost_vdpa_suspend(dev);
1286         r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND);
1287         if (unlikely(r)) {
1288             error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno);
1289         } else {
1290             v->suspended = true;
1291             return;
1292         }
1293     }
1294 
1295     vhost_vdpa_reset_device(dev);
1296 }
1297 
1298 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
1299 {
1300     struct vhost_vdpa *v = dev->opaque;
1301     bool ok;
1302     trace_vhost_vdpa_dev_start(dev, started);
1303 
1304     if (started) {
1305         vhost_vdpa_host_notifiers_init(dev);
1306         ok = vhost_vdpa_svqs_start(dev);
1307         if (unlikely(!ok)) {
1308             return -1;
1309         }
1310     } else {
1311         vhost_vdpa_suspend(dev);
1312         vhost_vdpa_svqs_stop(dev);
1313         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
1314     }
1315 
1316     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1317         return 0;
1318     }
1319 
1320     if (started) {
1321         if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) {
1322             error_report("SVQ can not work while IOMMU enable, please disable"
1323                          "IOMMU and try again");
1324             return -1;
1325         }
1326         memory_listener_register(&v->listener, dev->vdev->dma_as);
1327 
1328         return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1329     }
1330 
1331     return 0;
1332 }
1333 
1334 static void vhost_vdpa_reset_status(struct vhost_dev *dev)
1335 {
1336     struct vhost_vdpa *v = dev->opaque;
1337 
1338     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1339         return;
1340     }
1341 
1342     vhost_vdpa_reset_device(dev);
1343     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
1344                                VIRTIO_CONFIG_S_DRIVER);
1345     memory_listener_unregister(&v->listener);
1346 }
1347 
1348 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
1349                                      struct vhost_log *log)
1350 {
1351     struct vhost_vdpa *v = dev->opaque;
1352     if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) {
1353         return 0;
1354     }
1355 
1356     trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd,
1357                                   log->log);
1358     return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
1359 }
1360 
1361 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
1362                                        struct vhost_vring_addr *addr)
1363 {
1364     struct vhost_vdpa *v = dev->opaque;
1365 
1366     if (v->shadow_vqs_enabled) {
1367         /*
1368          * Device vring addr was set at device start. SVQ base is handled by
1369          * VirtQueue code.
1370          */
1371         return 0;
1372     }
1373 
1374     return vhost_vdpa_set_vring_dev_addr(dev, addr);
1375 }
1376 
1377 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
1378                                       struct vhost_vring_state *ring)
1379 {
1380     trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num);
1381     return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
1382 }
1383 
1384 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
1385                                        struct vhost_vring_state *ring)
1386 {
1387     struct vhost_vdpa *v = dev->opaque;
1388 
1389     if (v->shadow_vqs_enabled) {
1390         /*
1391          * Device vring base was set at device start. SVQ base is handled by
1392          * VirtQueue code.
1393          */
1394         return 0;
1395     }
1396 
1397     return vhost_vdpa_set_dev_vring_base(dev, ring);
1398 }
1399 
1400 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
1401                                        struct vhost_vring_state *ring)
1402 {
1403     struct vhost_vdpa *v = dev->opaque;
1404     int ret;
1405 
1406     if (v->shadow_vqs_enabled) {
1407         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
1408         return 0;
1409     }
1410 
1411     if (!v->suspended) {
1412         /*
1413          * Cannot trust in value returned by device, let vhost recover used
1414          * idx from guest.
1415          */
1416         return -1;
1417     }
1418 
1419     ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
1420     trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
1421     return ret;
1422 }
1423 
1424 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
1425                                        struct vhost_vring_file *file)
1426 {
1427     struct vhost_vdpa *v = dev->opaque;
1428     int vdpa_idx = file->index - dev->vq_index;
1429 
1430     if (v->shadow_vqs_enabled) {
1431         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1432         vhost_svq_set_svq_kick_fd(svq, file->fd);
1433         return 0;
1434     } else {
1435         return vhost_vdpa_set_vring_dev_kick(dev, file);
1436     }
1437 }
1438 
1439 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
1440                                        struct vhost_vring_file *file)
1441 {
1442     struct vhost_vdpa *v = dev->opaque;
1443     int vdpa_idx = file->index - dev->vq_index;
1444     VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1445 
1446     /* Remember last call fd because we can switch to SVQ anytime. */
1447     vhost_svq_set_svq_call_fd(svq, file->fd);
1448     if (v->shadow_vqs_enabled) {
1449         return 0;
1450     }
1451 
1452     return vhost_vdpa_set_vring_dev_call(dev, file);
1453 }
1454 
1455 static int vhost_vdpa_get_features(struct vhost_dev *dev,
1456                                      uint64_t *features)
1457 {
1458     int ret = vhost_vdpa_get_dev_features(dev, features);
1459 
1460     if (ret == 0) {
1461         /* Add SVQ logging capabilities */
1462         *features |= BIT_ULL(VHOST_F_LOG_ALL);
1463     }
1464 
1465     return ret;
1466 }
1467 
1468 static int vhost_vdpa_set_owner(struct vhost_dev *dev)
1469 {
1470     if (!vhost_vdpa_first_dev(dev)) {
1471         return 0;
1472     }
1473 
1474     trace_vhost_vdpa_set_owner(dev);
1475     return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
1476 }
1477 
1478 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
1479                     struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
1480 {
1481     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
1482     addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
1483     addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
1484     addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
1485     trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr,
1486                                  addr->avail_user_addr, addr->used_user_addr);
1487     return 0;
1488 }
1489 
1490 static bool  vhost_vdpa_force_iommu(struct vhost_dev *dev)
1491 {
1492     return true;
1493 }
1494 
1495 const VhostOps vdpa_ops = {
1496         .backend_type = VHOST_BACKEND_TYPE_VDPA,
1497         .vhost_backend_init = vhost_vdpa_init,
1498         .vhost_backend_cleanup = vhost_vdpa_cleanup,
1499         .vhost_set_log_base = vhost_vdpa_set_log_base,
1500         .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
1501         .vhost_set_vring_num = vhost_vdpa_set_vring_num,
1502         .vhost_set_vring_base = vhost_vdpa_set_vring_base,
1503         .vhost_get_vring_base = vhost_vdpa_get_vring_base,
1504         .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
1505         .vhost_set_vring_call = vhost_vdpa_set_vring_call,
1506         .vhost_get_features = vhost_vdpa_get_features,
1507         .vhost_set_backend_cap = vhost_vdpa_set_backend_cap,
1508         .vhost_set_owner = vhost_vdpa_set_owner,
1509         .vhost_set_vring_endian = NULL,
1510         .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
1511         .vhost_set_mem_table = vhost_vdpa_set_mem_table,
1512         .vhost_set_features = vhost_vdpa_set_features,
1513         .vhost_reset_device = vhost_vdpa_reset_device,
1514         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
1515         .vhost_get_config  = vhost_vdpa_get_config,
1516         .vhost_set_config = vhost_vdpa_set_config,
1517         .vhost_requires_shm_log = NULL,
1518         .vhost_migration_done = NULL,
1519         .vhost_net_set_mtu = NULL,
1520         .vhost_set_iotlb_callback = NULL,
1521         .vhost_send_device_iotlb_msg = NULL,
1522         .vhost_dev_start = vhost_vdpa_dev_start,
1523         .vhost_get_device_id = vhost_vdpa_get_device_id,
1524         .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
1525         .vhost_force_iommu = vhost_vdpa_force_iommu,
1526         .vhost_set_config_call = vhost_vdpa_set_config_call,
1527         .vhost_reset_status = vhost_vdpa_reset_status,
1528 };
1529