xref: /openbmc/qemu/hw/virtio/vhost-vdpa.c (revision ae25ff41b72366248aa89bdc8be58aa86f67e4c3)
1 /*
2  * vhost-vdpa
3  *
4  *  Copyright(c) 2017-2018 Intel Corporation.
5  *  Copyright(c) 2020 Red Hat, Inc.
6  *
7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
8  * See the COPYING file in the top-level directory.
9  *
10  */
11 
12 #include "qemu/osdep.h"
13 #include <linux/vhost.h>
14 #include <linux/vfio.h>
15 #include <sys/eventfd.h>
16 #include <sys/ioctl.h>
17 #include "exec/target_page.h"
18 #include "hw/virtio/vhost.h"
19 #include "hw/virtio/vhost-backend.h"
20 #include "hw/virtio/virtio-net.h"
21 #include "hw/virtio/vhost-shadow-virtqueue.h"
22 #include "hw/virtio/vhost-vdpa.h"
23 #include "exec/address-spaces.h"
24 #include "migration/blocker.h"
25 #include "qemu/cutils.h"
26 #include "qemu/main-loop.h"
27 #include "trace.h"
28 #include "qapi/error.h"
29 
30 /*
31  * Return one past the end of the end of section. Be careful with uint64_t
32  * conversions!
33  */
34 static Int128 vhost_vdpa_section_end(const MemoryRegionSection *section,
35                                      int page_mask)
36 {
37     Int128 llend = int128_make64(section->offset_within_address_space);
38     llend = int128_add(llend, section->size);
39     llend = int128_and(llend, int128_exts64(page_mask));
40 
41     return llend;
42 }
43 
44 static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section,
45                                                 uint64_t iova_min,
46                                                 uint64_t iova_max,
47                                                 int page_mask)
48 {
49     Int128 llend;
50 
51     if ((!memory_region_is_ram(section->mr) &&
52          !memory_region_is_iommu(section->mr)) ||
53         memory_region_is_protected(section->mr) ||
54         /* vhost-vDPA doesn't allow MMIO to be mapped  */
55         memory_region_is_ram_device(section->mr)) {
56         return true;
57     }
58 
59     if (section->offset_within_address_space < iova_min) {
60         error_report("RAM section out of device range (min=0x%" PRIx64
61                      ", addr=0x%" HWADDR_PRIx ")",
62                      iova_min, section->offset_within_address_space);
63         return true;
64     }
65     /*
66      * While using vIOMMU, sometimes the section will be larger than iova_max,
67      * but the memory that actually maps is smaller, so move the check to
68      * function vhost_vdpa_iommu_map_notify(). That function will use the actual
69      * size that maps to the kernel
70      */
71 
72     if (!memory_region_is_iommu(section->mr)) {
73         llend = vhost_vdpa_section_end(section, page_mask);
74         if (int128_gt(llend, int128_make64(iova_max))) {
75             error_report("RAM section out of device range (max=0x%" PRIx64
76                          ", end addr=0x%" PRIx64 ")",
77                          iova_max, int128_get64(llend));
78             return true;
79         }
80     }
81 
82     return false;
83 }
84 
85 /*
86  * The caller must set asid = 0 if the device does not support asid.
87  * This is not an ABI break since it is set to 0 by the initializer anyway.
88  */
89 int vhost_vdpa_dma_map(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
90                        hwaddr size, void *vaddr, bool readonly)
91 {
92     struct vhost_msg_v2 msg = {};
93     int fd = v->device_fd;
94     int ret = 0;
95 
96     msg.type = v->msg_type;
97     msg.asid = asid;
98     msg.iotlb.iova = iova;
99     msg.iotlb.size = size;
100     msg.iotlb.uaddr = (uint64_t)(uintptr_t)vaddr;
101     msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
102     msg.iotlb.type = VHOST_IOTLB_UPDATE;
103 
104     trace_vhost_vdpa_dma_map(v, fd, msg.type, msg.asid, msg.iotlb.iova,
105                              msg.iotlb.size, msg.iotlb.uaddr, msg.iotlb.perm,
106                              msg.iotlb.type);
107 
108     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
109         error_report("failed to write, fd=%d, errno=%d (%s)",
110             fd, errno, strerror(errno));
111         return -EIO ;
112     }
113 
114     return ret;
115 }
116 
117 /*
118  * The caller must set asid = 0 if the device does not support asid.
119  * This is not an ABI break since it is set to 0 by the initializer anyway.
120  */
121 int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, uint32_t asid, hwaddr iova,
122                          hwaddr size)
123 {
124     struct vhost_msg_v2 msg = {};
125     int fd = v->device_fd;
126     int ret = 0;
127 
128     msg.type = v->msg_type;
129     msg.asid = asid;
130     msg.iotlb.iova = iova;
131     msg.iotlb.size = size;
132     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
133 
134     trace_vhost_vdpa_dma_unmap(v, fd, msg.type, msg.asid, msg.iotlb.iova,
135                                msg.iotlb.size, msg.iotlb.type);
136 
137     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
138         error_report("failed to write, fd=%d, errno=%d (%s)",
139             fd, errno, strerror(errno));
140         return -EIO ;
141     }
142 
143     return ret;
144 }
145 
146 static void vhost_vdpa_listener_begin_batch(struct vhost_vdpa *v)
147 {
148     int fd = v->device_fd;
149     struct vhost_msg_v2 msg = {
150         .type = v->msg_type,
151         .iotlb.type = VHOST_IOTLB_BATCH_BEGIN,
152     };
153 
154     trace_vhost_vdpa_listener_begin_batch(v, fd, msg.type, msg.iotlb.type);
155     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
156         error_report("failed to write, fd=%d, errno=%d (%s)",
157                      fd, errno, strerror(errno));
158     }
159 }
160 
161 static void vhost_vdpa_iotlb_batch_begin_once(struct vhost_vdpa *v)
162 {
163     if (v->dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH) &&
164         !v->iotlb_batch_begin_sent) {
165         vhost_vdpa_listener_begin_batch(v);
166     }
167 
168     v->iotlb_batch_begin_sent = true;
169 }
170 
171 static void vhost_vdpa_listener_commit(MemoryListener *listener)
172 {
173     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
174     struct vhost_dev *dev = v->dev;
175     struct vhost_msg_v2 msg = {};
176     int fd = v->device_fd;
177 
178     if (!(dev->backend_cap & (0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH))) {
179         return;
180     }
181 
182     if (!v->iotlb_batch_begin_sent) {
183         return;
184     }
185 
186     msg.type = v->msg_type;
187     msg.iotlb.type = VHOST_IOTLB_BATCH_END;
188 
189     trace_vhost_vdpa_listener_commit(v, fd, msg.type, msg.iotlb.type);
190     if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
191         error_report("failed to write, fd=%d, errno=%d (%s)",
192                      fd, errno, strerror(errno));
193     }
194 
195     v->iotlb_batch_begin_sent = false;
196 }
197 
198 static void vhost_vdpa_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
199 {
200     struct vdpa_iommu *iommu = container_of(n, struct vdpa_iommu, n);
201 
202     hwaddr iova = iotlb->iova + iommu->iommu_offset;
203     struct vhost_vdpa *v = iommu->dev;
204     void *vaddr;
205     int ret;
206     Int128 llend;
207 
208     if (iotlb->target_as != &address_space_memory) {
209         error_report("Wrong target AS \"%s\", only system memory is allowed",
210                      iotlb->target_as->name ? iotlb->target_as->name : "none");
211         return;
212     }
213     RCU_READ_LOCK_GUARD();
214     /* check if RAM section out of device range */
215     llend = int128_add(int128_makes64(iotlb->addr_mask), int128_makes64(iova));
216     if (int128_gt(llend, int128_make64(v->shared->iova_range.last))) {
217         error_report("RAM section out of device range (max=0x%" PRIx64
218                      ", end addr=0x%" PRIx64 ")",
219                      v->shared->iova_range.last, int128_get64(llend));
220         return;
221     }
222 
223     if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
224         bool read_only;
225 
226         if (!memory_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, NULL)) {
227             return;
228         }
229         ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
230                                  iotlb->addr_mask + 1, vaddr, read_only);
231         if (ret) {
232             error_report("vhost_vdpa_dma_map(%p, 0x%" HWADDR_PRIx ", "
233                          "0x%" HWADDR_PRIx ", %p) = %d (%m)",
234                          v, iova, iotlb->addr_mask + 1, vaddr, ret);
235         }
236     } else {
237         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
238                                    iotlb->addr_mask + 1);
239         if (ret) {
240             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
241                          "0x%" HWADDR_PRIx ") = %d (%m)",
242                          v, iova, iotlb->addr_mask + 1, ret);
243         }
244     }
245 }
246 
247 static void vhost_vdpa_iommu_region_add(MemoryListener *listener,
248                                         MemoryRegionSection *section)
249 {
250     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
251 
252     struct vdpa_iommu *iommu;
253     Int128 end;
254     int iommu_idx;
255     IOMMUMemoryRegion *iommu_mr;
256     int ret;
257 
258     iommu_mr = IOMMU_MEMORY_REGION(section->mr);
259 
260     iommu = g_malloc0(sizeof(*iommu));
261     end = int128_add(int128_make64(section->offset_within_region),
262                      section->size);
263     end = int128_sub(end, int128_one());
264     iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
265                                                    MEMTXATTRS_UNSPECIFIED);
266     iommu->iommu_mr = iommu_mr;
267     iommu_notifier_init(&iommu->n, vhost_vdpa_iommu_map_notify,
268                         IOMMU_NOTIFIER_IOTLB_EVENTS,
269                         section->offset_within_region,
270                         int128_get64(end),
271                         iommu_idx);
272     iommu->iommu_offset = section->offset_within_address_space -
273                           section->offset_within_region;
274     iommu->dev = v;
275 
276     ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
277     if (ret) {
278         g_free(iommu);
279         return;
280     }
281 
282     QLIST_INSERT_HEAD(&v->iommu_list, iommu, iommu_next);
283     memory_region_iommu_replay(iommu->iommu_mr, &iommu->n);
284 
285     return;
286 }
287 
288 static void vhost_vdpa_iommu_region_del(MemoryListener *listener,
289                                         MemoryRegionSection *section)
290 {
291     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
292 
293     struct vdpa_iommu *iommu;
294 
295     QLIST_FOREACH(iommu, &v->iommu_list, iommu_next)
296     {
297         if (MEMORY_REGION(iommu->iommu_mr) == section->mr &&
298             iommu->n.start == section->offset_within_region) {
299             memory_region_unregister_iommu_notifier(section->mr, &iommu->n);
300             QLIST_REMOVE(iommu, iommu_next);
301             g_free(iommu);
302             break;
303         }
304     }
305 }
306 
307 static void vhost_vdpa_listener_region_add(MemoryListener *listener,
308                                            MemoryRegionSection *section)
309 {
310     DMAMap mem_region = {};
311     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
312     hwaddr iova;
313     Int128 llend, llsize;
314     void *vaddr;
315     int ret;
316     int page_size = qemu_target_page_size();
317     int page_mask = -page_size;
318 
319     if (vhost_vdpa_listener_skipped_section(section,
320                                             v->shared->iova_range.first,
321                                             v->shared->iova_range.last,
322                                             page_mask)) {
323         return;
324     }
325     if (memory_region_is_iommu(section->mr)) {
326         vhost_vdpa_iommu_region_add(listener, section);
327         return;
328     }
329 
330     if (unlikely((section->offset_within_address_space & ~page_mask) !=
331                  (section->offset_within_region & ~page_mask))) {
332         trace_vhost_vdpa_listener_region_add_unaligned(v, section->mr->name,
333                        section->offset_within_address_space & ~page_mask,
334                        section->offset_within_region & ~page_mask);
335         return;
336     }
337 
338     iova = ROUND_UP(section->offset_within_address_space, page_size);
339     llend = vhost_vdpa_section_end(section, page_mask);
340     if (int128_ge(int128_make64(iova), llend)) {
341         return;
342     }
343 
344     memory_region_ref(section->mr);
345 
346     /* Here we assume that memory_region_is_ram(section->mr)==true */
347 
348     vaddr = memory_region_get_ram_ptr(section->mr) +
349             section->offset_within_region +
350             (iova - section->offset_within_address_space);
351 
352     trace_vhost_vdpa_listener_region_add(v, iova, int128_get64(llend),
353                                          vaddr, section->readonly);
354 
355     llsize = int128_sub(llend, int128_make64(iova));
356     if (v->shadow_data) {
357         int r;
358 
359         mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
360         mem_region.size = int128_get64(llsize) - 1,
361         mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
362 
363         r = vhost_iova_tree_map_alloc(v->shared->iova_tree, &mem_region);
364         if (unlikely(r != IOVA_OK)) {
365             error_report("Can't allocate a mapping (%d)", r);
366             goto fail;
367         }
368 
369         iova = mem_region.iova;
370     }
371 
372     vhost_vdpa_iotlb_batch_begin_once(v);
373     ret = vhost_vdpa_dma_map(v, VHOST_VDPA_GUEST_PA_ASID, iova,
374                              int128_get64(llsize), vaddr, section->readonly);
375     if (ret) {
376         error_report("vhost vdpa map fail!");
377         goto fail_map;
378     }
379 
380     return;
381 
382 fail_map:
383     if (v->shadow_data) {
384         vhost_iova_tree_remove(v->shared->iova_tree, mem_region);
385     }
386 
387 fail:
388     /*
389      * On the initfn path, store the first error in the container so we
390      * can gracefully fail.  Runtime, there's not much we can do other
391      * than throw a hardware error.
392      */
393     error_report("vhost-vdpa: DMA mapping failed, unable to continue");
394     return;
395 
396 }
397 
398 static void vhost_vdpa_listener_region_del(MemoryListener *listener,
399                                            MemoryRegionSection *section)
400 {
401     struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
402     hwaddr iova;
403     Int128 llend, llsize;
404     int ret;
405     int page_size = qemu_target_page_size();
406     int page_mask = -page_size;
407 
408     if (vhost_vdpa_listener_skipped_section(section,
409                                             v->shared->iova_range.first,
410                                             v->shared->iova_range.last,
411                                             page_mask)) {
412         return;
413     }
414     if (memory_region_is_iommu(section->mr)) {
415         vhost_vdpa_iommu_region_del(listener, section);
416     }
417 
418     if (unlikely((section->offset_within_address_space & ~page_mask) !=
419                  (section->offset_within_region & ~page_mask))) {
420         trace_vhost_vdpa_listener_region_del_unaligned(v, section->mr->name,
421                        section->offset_within_address_space & ~page_mask,
422                        section->offset_within_region & ~page_mask);
423         return;
424     }
425 
426     iova = ROUND_UP(section->offset_within_address_space, page_size);
427     llend = vhost_vdpa_section_end(section, page_mask);
428 
429     trace_vhost_vdpa_listener_region_del(v, iova,
430         int128_get64(int128_sub(llend, int128_one())));
431 
432     if (int128_ge(int128_make64(iova), llend)) {
433         return;
434     }
435 
436     llsize = int128_sub(llend, int128_make64(iova));
437 
438     if (v->shadow_data) {
439         const DMAMap *result;
440         const void *vaddr = memory_region_get_ram_ptr(section->mr) +
441             section->offset_within_region +
442             (iova - section->offset_within_address_space);
443         DMAMap mem_region = {
444             .translated_addr = (hwaddr)(uintptr_t)vaddr,
445             .size = int128_get64(llsize) - 1,
446         };
447 
448         result = vhost_iova_tree_find_iova(v->shared->iova_tree, &mem_region);
449         if (!result) {
450             /* The memory listener map wasn't mapped */
451             return;
452         }
453         iova = result->iova;
454         vhost_iova_tree_remove(v->shared->iova_tree, *result);
455     }
456     vhost_vdpa_iotlb_batch_begin_once(v);
457     /*
458      * The unmap ioctl doesn't accept a full 64-bit. need to check it
459      */
460     if (int128_eq(llsize, int128_2_64())) {
461         llsize = int128_rshift(llsize, 1);
462         ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
463                                    int128_get64(llsize));
464 
465         if (ret) {
466             error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
467                          "0x%" HWADDR_PRIx ") = %d (%m)",
468                          v, iova, int128_get64(llsize), ret);
469         }
470         iova += int128_get64(llsize);
471     }
472     ret = vhost_vdpa_dma_unmap(v, VHOST_VDPA_GUEST_PA_ASID, iova,
473                                int128_get64(llsize));
474 
475     if (ret) {
476         error_report("vhost_vdpa_dma_unmap(%p, 0x%" HWADDR_PRIx ", "
477                      "0x%" HWADDR_PRIx ") = %d (%m)",
478                      v, iova, int128_get64(llsize), ret);
479     }
480 
481     memory_region_unref(section->mr);
482 }
483 /*
484  * IOTLB API is used by vhost-vdpa which requires incremental updating
485  * of the mapping. So we can not use generic vhost memory listener which
486  * depends on the addnop().
487  */
488 static const MemoryListener vhost_vdpa_memory_listener = {
489     .name = "vhost-vdpa",
490     .commit = vhost_vdpa_listener_commit,
491     .region_add = vhost_vdpa_listener_region_add,
492     .region_del = vhost_vdpa_listener_region_del,
493 };
494 
495 static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
496                              void *arg)
497 {
498     struct vhost_vdpa *v = dev->opaque;
499     int fd = v->device_fd;
500     int ret;
501 
502     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
503 
504     ret = ioctl(fd, request, arg);
505     return ret < 0 ? -errno : ret;
506 }
507 
508 static int vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
509 {
510     uint8_t s;
511     int ret;
512 
513     trace_vhost_vdpa_add_status(dev, status);
514     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
515     if (ret < 0) {
516         return ret;
517     }
518     if ((s & status) == status) {
519         /* Don't set bits already set */
520         return 0;
521     }
522 
523     s |= status;
524 
525     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
526     if (ret < 0) {
527         return ret;
528     }
529 
530     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s);
531     if (ret < 0) {
532         return ret;
533     }
534 
535     if (!(s & status)) {
536         return -EIO;
537     }
538 
539     return 0;
540 }
541 
542 int vhost_vdpa_get_iova_range(int fd, struct vhost_vdpa_iova_range *iova_range)
543 {
544     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
545 
546     return ret < 0 ? -errno : 0;
547 }
548 
549 /*
550  * The use of this function is for requests that only need to be
551  * applied once. Typically such request occurs at the beginning
552  * of operation, and before setting up queues. It should not be
553  * used for request that performs operation until all queues are
554  * set, which would need to check dev->vq_index_end instead.
555  */
556 static bool vhost_vdpa_first_dev(struct vhost_dev *dev)
557 {
558     struct vhost_vdpa *v = dev->opaque;
559 
560     return v->index == 0;
561 }
562 
563 static int vhost_vdpa_get_dev_features(struct vhost_dev *dev,
564                                        uint64_t *features)
565 {
566     int ret;
567 
568     ret = vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
569     trace_vhost_vdpa_get_features(dev, *features);
570     return ret;
571 }
572 
573 static void vhost_vdpa_init_svq(struct vhost_dev *hdev, struct vhost_vdpa *v)
574 {
575     g_autoptr(GPtrArray) shadow_vqs = NULL;
576 
577     shadow_vqs = g_ptr_array_new_full(hdev->nvqs, vhost_svq_free);
578     for (unsigned n = 0; n < hdev->nvqs; ++n) {
579         VhostShadowVirtqueue *svq;
580 
581         svq = vhost_svq_new(v->shadow_vq_ops, v->shadow_vq_ops_opaque);
582         g_ptr_array_add(shadow_vqs, svq);
583     }
584 
585     v->shadow_vqs = g_steal_pointer(&shadow_vqs);
586 }
587 
588 static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque, Error **errp)
589 {
590     struct vhost_vdpa *v;
591     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
592     trace_vhost_vdpa_init(dev, opaque);
593     int ret;
594 
595     v = opaque;
596     v->dev = dev;
597     dev->opaque =  opaque ;
598     v->listener = vhost_vdpa_memory_listener;
599     v->msg_type = VHOST_IOTLB_MSG_V2;
600     vhost_vdpa_init_svq(dev, v);
601 
602     error_propagate(&dev->migration_blocker, v->migration_blocker);
603     if (!vhost_vdpa_first_dev(dev)) {
604         return 0;
605     }
606 
607     /*
608      * If dev->shadow_vqs_enabled at initialization that means the device has
609      * been started with x-svq=on, so don't block migration
610      */
611     if (dev->migration_blocker == NULL && !v->shadow_vqs_enabled) {
612         /* We don't have dev->features yet */
613         uint64_t features;
614         ret = vhost_vdpa_get_dev_features(dev, &features);
615         if (unlikely(ret)) {
616             error_setg_errno(errp, -ret, "Could not get device features");
617             return ret;
618         }
619         vhost_svq_valid_features(features, &dev->migration_blocker);
620     }
621 
622     /*
623      * Similar to VFIO, we end up pinning all guest memory and have to
624      * disable discarding of RAM.
625      */
626     ret = ram_block_discard_disable(true);
627     if (ret) {
628         error_report("Cannot set discarding of RAM broken");
629         return ret;
630     }
631 
632     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
633                                VIRTIO_CONFIG_S_DRIVER);
634 
635     return 0;
636 }
637 
638 static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
639                                             int queue_index)
640 {
641     size_t page_size = qemu_real_host_page_size();
642     struct vhost_vdpa *v = dev->opaque;
643     VirtIODevice *vdev = dev->vdev;
644     VhostVDPAHostNotifier *n;
645 
646     n = &v->notifier[queue_index];
647 
648     if (n->addr) {
649         virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
650         object_unparent(OBJECT(&n->mr));
651         munmap(n->addr, page_size);
652         n->addr = NULL;
653     }
654 }
655 
656 static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
657 {
658     size_t page_size = qemu_real_host_page_size();
659     struct vhost_vdpa *v = dev->opaque;
660     VirtIODevice *vdev = dev->vdev;
661     VhostVDPAHostNotifier *n;
662     int fd = v->device_fd;
663     void *addr;
664     char *name;
665 
666     vhost_vdpa_host_notifier_uninit(dev, queue_index);
667 
668     n = &v->notifier[queue_index];
669 
670     addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
671                 queue_index * page_size);
672     if (addr == MAP_FAILED) {
673         goto err;
674     }
675 
676     name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
677                            v, queue_index);
678     memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
679                                       page_size, addr);
680     g_free(name);
681 
682     if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
683         object_unparent(OBJECT(&n->mr));
684         munmap(addr, page_size);
685         goto err;
686     }
687     n->addr = addr;
688 
689     return 0;
690 
691 err:
692     return -1;
693 }
694 
695 static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
696 {
697     int i;
698 
699     /*
700      * Pack all the changes to the memory regions in a single
701      * transaction to avoid a few updating of the address space
702      * topology.
703      */
704     memory_region_transaction_begin();
705 
706     for (i = dev->vq_index; i < dev->vq_index + n; i++) {
707         vhost_vdpa_host_notifier_uninit(dev, i);
708     }
709 
710     memory_region_transaction_commit();
711 }
712 
713 static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
714 {
715     struct vhost_vdpa *v = dev->opaque;
716     int i;
717 
718     if (v->shadow_vqs_enabled) {
719         /* FIXME SVQ is not compatible with host notifiers mr */
720         return;
721     }
722 
723     /*
724      * Pack all the changes to the memory regions in a single
725      * transaction to avoid a few updating of the address space
726      * topology.
727      */
728     memory_region_transaction_begin();
729 
730     for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) {
731         if (vhost_vdpa_host_notifier_init(dev, i)) {
732             vhost_vdpa_host_notifiers_uninit(dev, i - dev->vq_index);
733             break;
734         }
735     }
736 
737     memory_region_transaction_commit();
738 }
739 
740 static void vhost_vdpa_svq_cleanup(struct vhost_dev *dev)
741 {
742     struct vhost_vdpa *v = dev->opaque;
743     size_t idx;
744 
745     for (idx = 0; idx < v->shadow_vqs->len; ++idx) {
746         vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, idx));
747     }
748     g_ptr_array_free(v->shadow_vqs, true);
749 }
750 
751 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
752 {
753     struct vhost_vdpa *v;
754     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
755     v = dev->opaque;
756     trace_vhost_vdpa_cleanup(dev, v);
757     if (vhost_vdpa_first_dev(dev)) {
758         ram_block_discard_disable(false);
759     }
760 
761     vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
762     memory_listener_unregister(&v->listener);
763     vhost_vdpa_svq_cleanup(dev);
764 
765     dev->opaque = NULL;
766 
767     return 0;
768 }
769 
770 static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
771 {
772     trace_vhost_vdpa_memslots_limit(dev, INT_MAX);
773     return INT_MAX;
774 }
775 
776 static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
777                                     struct vhost_memory *mem)
778 {
779     if (!vhost_vdpa_first_dev(dev)) {
780         return 0;
781     }
782 
783     trace_vhost_vdpa_set_mem_table(dev, mem->nregions, mem->padding);
784     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_MEM_TABLE) &&
785         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_REGIONS)) {
786         int i;
787         for (i = 0; i < mem->nregions; i++) {
788             trace_vhost_vdpa_dump_regions(dev, i,
789                                           mem->regions[i].guest_phys_addr,
790                                           mem->regions[i].memory_size,
791                                           mem->regions[i].userspace_addr,
792                                           mem->regions[i].flags_padding);
793         }
794     }
795     if (mem->padding) {
796         return -EINVAL;
797     }
798 
799     return 0;
800 }
801 
802 static int vhost_vdpa_set_features(struct vhost_dev *dev,
803                                    uint64_t features)
804 {
805     struct vhost_vdpa *v = dev->opaque;
806     int ret;
807 
808     if (!vhost_vdpa_first_dev(dev)) {
809         return 0;
810     }
811 
812     if (v->shadow_vqs_enabled) {
813         if ((v->acked_features ^ features) == BIT_ULL(VHOST_F_LOG_ALL)) {
814             /*
815              * QEMU is just trying to enable or disable logging. SVQ handles
816              * this sepparately, so no need to forward this.
817              */
818             v->acked_features = features;
819             return 0;
820         }
821 
822         v->acked_features = features;
823 
824         /* We must not ack _F_LOG if SVQ is enabled */
825         features &= ~BIT_ULL(VHOST_F_LOG_ALL);
826     }
827 
828     trace_vhost_vdpa_set_features(dev, features);
829     ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
830     if (ret) {
831         return ret;
832     }
833 
834     return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
835 }
836 
837 static int vhost_vdpa_set_backend_cap(struct vhost_dev *dev)
838 {
839     uint64_t features;
840     uint64_t f = 0x1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2 |
841         0x1ULL << VHOST_BACKEND_F_IOTLB_BATCH |
842         0x1ULL << VHOST_BACKEND_F_IOTLB_ASID |
843         0x1ULL << VHOST_BACKEND_F_SUSPEND;
844     int r;
845 
846     if (vhost_vdpa_call(dev, VHOST_GET_BACKEND_FEATURES, &features)) {
847         return -EFAULT;
848     }
849 
850     features &= f;
851 
852     if (vhost_vdpa_first_dev(dev)) {
853         r = vhost_vdpa_call(dev, VHOST_SET_BACKEND_FEATURES, &features);
854         if (r) {
855             return -EFAULT;
856         }
857     }
858 
859     dev->backend_cap = features;
860 
861     return 0;
862 }
863 
864 static int vhost_vdpa_get_device_id(struct vhost_dev *dev,
865                                     uint32_t *device_id)
866 {
867     int ret;
868     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
869     trace_vhost_vdpa_get_device_id(dev, *device_id);
870     return ret;
871 }
872 
873 static int vhost_vdpa_reset_device(struct vhost_dev *dev)
874 {
875     struct vhost_vdpa *v = dev->opaque;
876     int ret;
877     uint8_t status = 0;
878 
879     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
880     trace_vhost_vdpa_reset_device(dev);
881     v->suspended = false;
882     return ret;
883 }
884 
885 static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
886 {
887     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
888 
889     trace_vhost_vdpa_get_vq_index(dev, idx, idx);
890     return idx;
891 }
892 
893 int vhost_vdpa_set_vring_ready(struct vhost_vdpa *v, unsigned idx)
894 {
895     struct vhost_dev *dev = v->dev;
896     struct vhost_vring_state state = {
897         .index = idx,
898         .num = 1,
899     };
900     int r = vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
901 
902     trace_vhost_vdpa_set_vring_ready(dev, idx, r);
903     return r;
904 }
905 
906 static int vhost_vdpa_set_config_call(struct vhost_dev *dev,
907                                        int fd)
908 {
909     trace_vhost_vdpa_set_config_call(dev, fd);
910     return vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG_CALL, &fd);
911 }
912 
913 static void vhost_vdpa_dump_config(struct vhost_dev *dev, const uint8_t *config,
914                                    uint32_t config_len)
915 {
916     int b, len;
917     char line[QEMU_HEXDUMP_LINE_LEN];
918 
919     for (b = 0; b < config_len; b += 16) {
920         len = config_len - b;
921         qemu_hexdump_line(line, b, config, len, false);
922         trace_vhost_vdpa_dump_config(dev, line);
923     }
924 }
925 
926 static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
927                                    uint32_t offset, uint32_t size,
928                                    uint32_t flags)
929 {
930     struct vhost_vdpa_config *config;
931     int ret;
932     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
933 
934     trace_vhost_vdpa_set_config(dev, offset, size, flags);
935     config = g_malloc(size + config_size);
936     config->off = offset;
937     config->len = size;
938     memcpy(config->buf, data, size);
939     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_SET_CONFIG) &&
940         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
941         vhost_vdpa_dump_config(dev, data, size);
942     }
943     ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, config);
944     g_free(config);
945     return ret;
946 }
947 
948 static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
949                                    uint32_t config_len, Error **errp)
950 {
951     struct vhost_vdpa_config *v_config;
952     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
953     int ret;
954 
955     trace_vhost_vdpa_get_config(dev, config, config_len);
956     v_config = g_malloc(config_len + config_size);
957     v_config->len = config_len;
958     v_config->off = 0;
959     ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, v_config);
960     memcpy(config, v_config->buf, config_len);
961     g_free(v_config);
962     if (trace_event_get_state_backends(TRACE_VHOST_VDPA_GET_CONFIG) &&
963         trace_event_get_state_backends(TRACE_VHOST_VDPA_DUMP_CONFIG)) {
964         vhost_vdpa_dump_config(dev, config, config_len);
965     }
966     return ret;
967  }
968 
969 static int vhost_vdpa_set_dev_vring_base(struct vhost_dev *dev,
970                                          struct vhost_vring_state *ring)
971 {
972     trace_vhost_vdpa_set_vring_base(dev, ring->index, ring->num);
973     return vhost_vdpa_call(dev, VHOST_SET_VRING_BASE, ring);
974 }
975 
976 static int vhost_vdpa_set_vring_dev_kick(struct vhost_dev *dev,
977                                          struct vhost_vring_file *file)
978 {
979     trace_vhost_vdpa_set_vring_kick(dev, file->index, file->fd);
980     return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
981 }
982 
983 static int vhost_vdpa_set_vring_dev_call(struct vhost_dev *dev,
984                                          struct vhost_vring_file *file)
985 {
986     trace_vhost_vdpa_set_vring_call(dev, file->index, file->fd);
987     return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
988 }
989 
990 static int vhost_vdpa_set_vring_dev_addr(struct vhost_dev *dev,
991                                          struct vhost_vring_addr *addr)
992 {
993     trace_vhost_vdpa_set_vring_addr(dev, addr->index, addr->flags,
994                                 addr->desc_user_addr, addr->used_user_addr,
995                                 addr->avail_user_addr,
996                                 addr->log_guest_addr);
997 
998     return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
999 
1000 }
1001 
1002 /**
1003  * Set the shadow virtqueue descriptors to the device
1004  *
1005  * @dev: The vhost device model
1006  * @svq: The shadow virtqueue
1007  * @idx: The index of the virtqueue in the vhost device
1008  * @errp: Error
1009  *
1010  * Note that this function does not rewind kick file descriptor if cannot set
1011  * call one.
1012  */
1013 static int vhost_vdpa_svq_set_fds(struct vhost_dev *dev,
1014                                   VhostShadowVirtqueue *svq, unsigned idx,
1015                                   Error **errp)
1016 {
1017     struct vhost_vring_file file = {
1018         .index = dev->vq_index + idx,
1019     };
1020     const EventNotifier *event_notifier = &svq->hdev_kick;
1021     int r;
1022 
1023     r = event_notifier_init(&svq->hdev_kick, 0);
1024     if (r != 0) {
1025         error_setg_errno(errp, -r, "Couldn't create kick event notifier");
1026         goto err_init_hdev_kick;
1027     }
1028 
1029     r = event_notifier_init(&svq->hdev_call, 0);
1030     if (r != 0) {
1031         error_setg_errno(errp, -r, "Couldn't create call event notifier");
1032         goto err_init_hdev_call;
1033     }
1034 
1035     file.fd = event_notifier_get_fd(event_notifier);
1036     r = vhost_vdpa_set_vring_dev_kick(dev, &file);
1037     if (unlikely(r != 0)) {
1038         error_setg_errno(errp, -r, "Can't set device kick fd");
1039         goto err_init_set_dev_fd;
1040     }
1041 
1042     event_notifier = &svq->hdev_call;
1043     file.fd = event_notifier_get_fd(event_notifier);
1044     r = vhost_vdpa_set_vring_dev_call(dev, &file);
1045     if (unlikely(r != 0)) {
1046         error_setg_errno(errp, -r, "Can't set device call fd");
1047         goto err_init_set_dev_fd;
1048     }
1049 
1050     return 0;
1051 
1052 err_init_set_dev_fd:
1053     event_notifier_set_handler(&svq->hdev_call, NULL);
1054 
1055 err_init_hdev_call:
1056     event_notifier_cleanup(&svq->hdev_kick);
1057 
1058 err_init_hdev_kick:
1059     return r;
1060 }
1061 
1062 /**
1063  * Unmap a SVQ area in the device
1064  */
1065 static void vhost_vdpa_svq_unmap_ring(struct vhost_vdpa *v, hwaddr addr)
1066 {
1067     const DMAMap needle = {
1068         .translated_addr = addr,
1069     };
1070     const DMAMap *result = vhost_iova_tree_find_iova(v->shared->iova_tree,
1071                                                      &needle);
1072     hwaddr size;
1073     int r;
1074 
1075     if (unlikely(!result)) {
1076         error_report("Unable to find SVQ address to unmap");
1077         return;
1078     }
1079 
1080     size = ROUND_UP(result->size, qemu_real_host_page_size());
1081     r = vhost_vdpa_dma_unmap(v, v->address_space_id, result->iova, size);
1082     if (unlikely(r < 0)) {
1083         error_report("Unable to unmap SVQ vring: %s (%d)", g_strerror(-r), -r);
1084         return;
1085     }
1086 
1087     vhost_iova_tree_remove(v->shared->iova_tree, *result);
1088 }
1089 
1090 static void vhost_vdpa_svq_unmap_rings(struct vhost_dev *dev,
1091                                        const VhostShadowVirtqueue *svq)
1092 {
1093     struct vhost_vdpa *v = dev->opaque;
1094     struct vhost_vring_addr svq_addr;
1095 
1096     vhost_svq_get_vring_addr(svq, &svq_addr);
1097 
1098     vhost_vdpa_svq_unmap_ring(v, svq_addr.desc_user_addr);
1099 
1100     vhost_vdpa_svq_unmap_ring(v, svq_addr.used_user_addr);
1101 }
1102 
1103 /**
1104  * Map the SVQ area in the device
1105  *
1106  * @v: Vhost-vdpa device
1107  * @needle: The area to search iova
1108  * @errorp: Error pointer
1109  */
1110 static bool vhost_vdpa_svq_map_ring(struct vhost_vdpa *v, DMAMap *needle,
1111                                     Error **errp)
1112 {
1113     int r;
1114 
1115     r = vhost_iova_tree_map_alloc(v->shared->iova_tree, needle);
1116     if (unlikely(r != IOVA_OK)) {
1117         error_setg(errp, "Cannot allocate iova (%d)", r);
1118         return false;
1119     }
1120 
1121     r = vhost_vdpa_dma_map(v, v->address_space_id, needle->iova,
1122                            needle->size + 1,
1123                            (void *)(uintptr_t)needle->translated_addr,
1124                            needle->perm == IOMMU_RO);
1125     if (unlikely(r != 0)) {
1126         error_setg_errno(errp, -r, "Cannot map region to device");
1127         vhost_iova_tree_remove(v->shared->iova_tree, *needle);
1128     }
1129 
1130     return r == 0;
1131 }
1132 
1133 /**
1134  * Map the shadow virtqueue rings in the device
1135  *
1136  * @dev: The vhost device
1137  * @svq: The shadow virtqueue
1138  * @addr: Assigned IOVA addresses
1139  * @errp: Error pointer
1140  */
1141 static bool vhost_vdpa_svq_map_rings(struct vhost_dev *dev,
1142                                      const VhostShadowVirtqueue *svq,
1143                                      struct vhost_vring_addr *addr,
1144                                      Error **errp)
1145 {
1146     ERRP_GUARD();
1147     DMAMap device_region, driver_region;
1148     struct vhost_vring_addr svq_addr;
1149     struct vhost_vdpa *v = dev->opaque;
1150     size_t device_size = vhost_svq_device_area_size(svq);
1151     size_t driver_size = vhost_svq_driver_area_size(svq);
1152     size_t avail_offset;
1153     bool ok;
1154 
1155     vhost_svq_get_vring_addr(svq, &svq_addr);
1156 
1157     driver_region = (DMAMap) {
1158         .translated_addr = svq_addr.desc_user_addr,
1159         .size = driver_size - 1,
1160         .perm = IOMMU_RO,
1161     };
1162     ok = vhost_vdpa_svq_map_ring(v, &driver_region, errp);
1163     if (unlikely(!ok)) {
1164         error_prepend(errp, "Cannot create vq driver region: ");
1165         return false;
1166     }
1167     addr->desc_user_addr = driver_region.iova;
1168     avail_offset = svq_addr.avail_user_addr - svq_addr.desc_user_addr;
1169     addr->avail_user_addr = driver_region.iova + avail_offset;
1170 
1171     device_region = (DMAMap) {
1172         .translated_addr = svq_addr.used_user_addr,
1173         .size = device_size - 1,
1174         .perm = IOMMU_RW,
1175     };
1176     ok = vhost_vdpa_svq_map_ring(v, &device_region, errp);
1177     if (unlikely(!ok)) {
1178         error_prepend(errp, "Cannot create vq device region: ");
1179         vhost_vdpa_svq_unmap_ring(v, driver_region.translated_addr);
1180     }
1181     addr->used_user_addr = device_region.iova;
1182 
1183     return ok;
1184 }
1185 
1186 static bool vhost_vdpa_svq_setup(struct vhost_dev *dev,
1187                                  VhostShadowVirtqueue *svq, unsigned idx,
1188                                  Error **errp)
1189 {
1190     uint16_t vq_index = dev->vq_index + idx;
1191     struct vhost_vring_state s = {
1192         .index = vq_index,
1193     };
1194     int r;
1195 
1196     r = vhost_vdpa_set_dev_vring_base(dev, &s);
1197     if (unlikely(r)) {
1198         error_setg_errno(errp, -r, "Cannot set vring base");
1199         return false;
1200     }
1201 
1202     r = vhost_vdpa_svq_set_fds(dev, svq, idx, errp);
1203     return r == 0;
1204 }
1205 
1206 static bool vhost_vdpa_svqs_start(struct vhost_dev *dev)
1207 {
1208     struct vhost_vdpa *v = dev->opaque;
1209     Error *err = NULL;
1210     unsigned i;
1211 
1212     if (!v->shadow_vqs_enabled) {
1213         return true;
1214     }
1215 
1216     for (i = 0; i < v->shadow_vqs->len; ++i) {
1217         VirtQueue *vq = virtio_get_queue(dev->vdev, dev->vq_index + i);
1218         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1219         struct vhost_vring_addr addr = {
1220             .index = dev->vq_index + i,
1221         };
1222         int r;
1223         bool ok = vhost_vdpa_svq_setup(dev, svq, i, &err);
1224         if (unlikely(!ok)) {
1225             goto err;
1226         }
1227 
1228         vhost_svq_start(svq, dev->vdev, vq, v->shared->iova_tree);
1229         ok = vhost_vdpa_svq_map_rings(dev, svq, &addr, &err);
1230         if (unlikely(!ok)) {
1231             goto err_map;
1232         }
1233 
1234         /* Override vring GPA set by vhost subsystem */
1235         r = vhost_vdpa_set_vring_dev_addr(dev, &addr);
1236         if (unlikely(r != 0)) {
1237             error_setg_errno(&err, -r, "Cannot set device address");
1238             goto err_set_addr;
1239         }
1240     }
1241 
1242     return true;
1243 
1244 err_set_addr:
1245     vhost_vdpa_svq_unmap_rings(dev, g_ptr_array_index(v->shadow_vqs, i));
1246 
1247 err_map:
1248     vhost_svq_stop(g_ptr_array_index(v->shadow_vqs, i));
1249 
1250 err:
1251     error_reportf_err(err, "Cannot setup SVQ %u: ", i);
1252     for (unsigned j = 0; j < i; ++j) {
1253         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, j);
1254         vhost_vdpa_svq_unmap_rings(dev, svq);
1255         vhost_svq_stop(svq);
1256     }
1257 
1258     return false;
1259 }
1260 
1261 static void vhost_vdpa_svqs_stop(struct vhost_dev *dev)
1262 {
1263     struct vhost_vdpa *v = dev->opaque;
1264 
1265     if (!v->shadow_vqs_enabled) {
1266         return;
1267     }
1268 
1269     for (unsigned i = 0; i < v->shadow_vqs->len; ++i) {
1270         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, i);
1271 
1272         vhost_svq_stop(svq);
1273         vhost_vdpa_svq_unmap_rings(dev, svq);
1274 
1275         event_notifier_cleanup(&svq->hdev_kick);
1276         event_notifier_cleanup(&svq->hdev_call);
1277     }
1278 }
1279 
1280 static void vhost_vdpa_suspend(struct vhost_dev *dev)
1281 {
1282     struct vhost_vdpa *v = dev->opaque;
1283     int r;
1284 
1285     if (!vhost_vdpa_first_dev(dev)) {
1286         return;
1287     }
1288 
1289     if (dev->backend_cap & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) {
1290         trace_vhost_vdpa_suspend(dev);
1291         r = ioctl(v->device_fd, VHOST_VDPA_SUSPEND);
1292         if (unlikely(r)) {
1293             error_report("Cannot suspend: %s(%d)", g_strerror(errno), errno);
1294         } else {
1295             v->suspended = true;
1296             return;
1297         }
1298     }
1299 
1300     vhost_vdpa_reset_device(dev);
1301 }
1302 
1303 static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
1304 {
1305     struct vhost_vdpa *v = dev->opaque;
1306     bool ok;
1307     trace_vhost_vdpa_dev_start(dev, started);
1308 
1309     if (started) {
1310         vhost_vdpa_host_notifiers_init(dev);
1311         ok = vhost_vdpa_svqs_start(dev);
1312         if (unlikely(!ok)) {
1313             return -1;
1314         }
1315     } else {
1316         vhost_vdpa_suspend(dev);
1317         vhost_vdpa_svqs_stop(dev);
1318         vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
1319     }
1320 
1321     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1322         return 0;
1323     }
1324 
1325     if (started) {
1326         if (vhost_dev_has_iommu(dev) && (v->shadow_vqs_enabled)) {
1327             error_report("SVQ can not work while IOMMU enable, please disable"
1328                          "IOMMU and try again");
1329             return -1;
1330         }
1331         memory_listener_register(&v->listener, dev->vdev->dma_as);
1332 
1333         return vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
1334     }
1335 
1336     return 0;
1337 }
1338 
1339 static void vhost_vdpa_reset_status(struct vhost_dev *dev)
1340 {
1341     struct vhost_vdpa *v = dev->opaque;
1342 
1343     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
1344         return;
1345     }
1346 
1347     vhost_vdpa_reset_device(dev);
1348     vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
1349                                VIRTIO_CONFIG_S_DRIVER);
1350     memory_listener_unregister(&v->listener);
1351 }
1352 
1353 static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
1354                                      struct vhost_log *log)
1355 {
1356     struct vhost_vdpa *v = dev->opaque;
1357     if (v->shadow_vqs_enabled || !vhost_vdpa_first_dev(dev)) {
1358         return 0;
1359     }
1360 
1361     trace_vhost_vdpa_set_log_base(dev, base, log->size, log->refcnt, log->fd,
1362                                   log->log);
1363     return vhost_vdpa_call(dev, VHOST_SET_LOG_BASE, &base);
1364 }
1365 
1366 static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
1367                                        struct vhost_vring_addr *addr)
1368 {
1369     struct vhost_vdpa *v = dev->opaque;
1370 
1371     if (v->shadow_vqs_enabled) {
1372         /*
1373          * Device vring addr was set at device start. SVQ base is handled by
1374          * VirtQueue code.
1375          */
1376         return 0;
1377     }
1378 
1379     return vhost_vdpa_set_vring_dev_addr(dev, addr);
1380 }
1381 
1382 static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
1383                                       struct vhost_vring_state *ring)
1384 {
1385     trace_vhost_vdpa_set_vring_num(dev, ring->index, ring->num);
1386     return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
1387 }
1388 
1389 static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
1390                                        struct vhost_vring_state *ring)
1391 {
1392     struct vhost_vdpa *v = dev->opaque;
1393 
1394     if (v->shadow_vqs_enabled) {
1395         /*
1396          * Device vring base was set at device start. SVQ base is handled by
1397          * VirtQueue code.
1398          */
1399         return 0;
1400     }
1401 
1402     return vhost_vdpa_set_dev_vring_base(dev, ring);
1403 }
1404 
1405 static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
1406                                        struct vhost_vring_state *ring)
1407 {
1408     struct vhost_vdpa *v = dev->opaque;
1409     int ret;
1410 
1411     if (v->shadow_vqs_enabled) {
1412         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
1413         return 0;
1414     }
1415 
1416     if (!v->suspended) {
1417         /*
1418          * Cannot trust in value returned by device, let vhost recover used
1419          * idx from guest.
1420          */
1421         return -1;
1422     }
1423 
1424     ret = vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
1425     trace_vhost_vdpa_get_vring_base(dev, ring->index, ring->num);
1426     return ret;
1427 }
1428 
1429 static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
1430                                        struct vhost_vring_file *file)
1431 {
1432     struct vhost_vdpa *v = dev->opaque;
1433     int vdpa_idx = file->index - dev->vq_index;
1434 
1435     if (v->shadow_vqs_enabled) {
1436         VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1437         vhost_svq_set_svq_kick_fd(svq, file->fd);
1438         return 0;
1439     } else {
1440         return vhost_vdpa_set_vring_dev_kick(dev, file);
1441     }
1442 }
1443 
1444 static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,
1445                                        struct vhost_vring_file *file)
1446 {
1447     struct vhost_vdpa *v = dev->opaque;
1448     int vdpa_idx = file->index - dev->vq_index;
1449     VhostShadowVirtqueue *svq = g_ptr_array_index(v->shadow_vqs, vdpa_idx);
1450 
1451     /* Remember last call fd because we can switch to SVQ anytime. */
1452     vhost_svq_set_svq_call_fd(svq, file->fd);
1453     if (v->shadow_vqs_enabled) {
1454         return 0;
1455     }
1456 
1457     return vhost_vdpa_set_vring_dev_call(dev, file);
1458 }
1459 
1460 static int vhost_vdpa_get_features(struct vhost_dev *dev,
1461                                      uint64_t *features)
1462 {
1463     int ret = vhost_vdpa_get_dev_features(dev, features);
1464 
1465     if (ret == 0) {
1466         /* Add SVQ logging capabilities */
1467         *features |= BIT_ULL(VHOST_F_LOG_ALL);
1468     }
1469 
1470     return ret;
1471 }
1472 
1473 static int vhost_vdpa_set_owner(struct vhost_dev *dev)
1474 {
1475     if (!vhost_vdpa_first_dev(dev)) {
1476         return 0;
1477     }
1478 
1479     trace_vhost_vdpa_set_owner(dev);
1480     return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
1481 }
1482 
1483 static int vhost_vdpa_vq_get_addr(struct vhost_dev *dev,
1484                     struct vhost_vring_addr *addr, struct vhost_virtqueue *vq)
1485 {
1486     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
1487     addr->desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
1488     addr->avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
1489     addr->used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
1490     trace_vhost_vdpa_vq_get_addr(dev, vq, addr->desc_user_addr,
1491                                  addr->avail_user_addr, addr->used_user_addr);
1492     return 0;
1493 }
1494 
1495 static bool  vhost_vdpa_force_iommu(struct vhost_dev *dev)
1496 {
1497     return true;
1498 }
1499 
1500 const VhostOps vdpa_ops = {
1501         .backend_type = VHOST_BACKEND_TYPE_VDPA,
1502         .vhost_backend_init = vhost_vdpa_init,
1503         .vhost_backend_cleanup = vhost_vdpa_cleanup,
1504         .vhost_set_log_base = vhost_vdpa_set_log_base,
1505         .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
1506         .vhost_set_vring_num = vhost_vdpa_set_vring_num,
1507         .vhost_set_vring_base = vhost_vdpa_set_vring_base,
1508         .vhost_get_vring_base = vhost_vdpa_get_vring_base,
1509         .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
1510         .vhost_set_vring_call = vhost_vdpa_set_vring_call,
1511         .vhost_get_features = vhost_vdpa_get_features,
1512         .vhost_set_backend_cap = vhost_vdpa_set_backend_cap,
1513         .vhost_set_owner = vhost_vdpa_set_owner,
1514         .vhost_set_vring_endian = NULL,
1515         .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
1516         .vhost_set_mem_table = vhost_vdpa_set_mem_table,
1517         .vhost_set_features = vhost_vdpa_set_features,
1518         .vhost_reset_device = vhost_vdpa_reset_device,
1519         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
1520         .vhost_get_config  = vhost_vdpa_get_config,
1521         .vhost_set_config = vhost_vdpa_set_config,
1522         .vhost_requires_shm_log = NULL,
1523         .vhost_migration_done = NULL,
1524         .vhost_net_set_mtu = NULL,
1525         .vhost_set_iotlb_callback = NULL,
1526         .vhost_send_device_iotlb_msg = NULL,
1527         .vhost_dev_start = vhost_vdpa_dev_start,
1528         .vhost_get_device_id = vhost_vdpa_get_device_id,
1529         .vhost_vq_get_addr = vhost_vdpa_vq_get_addr,
1530         .vhost_force_iommu = vhost_vdpa_force_iommu,
1531         .vhost_set_config_call = vhost_vdpa_set_config_call,
1532         .vhost_reset_status = vhost_vdpa_reset_status,
1533 };
1534