xref: /openbmc/qemu/hw/virtio/vhost-user.c (revision 7c211eb0)
1 /*
2  * vhost-user
3  *
4  * Copyright (c) 2013 Virtual Open Systems Sarl.
5  *
6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
7  * See the COPYING file in the top-level directory.
8  *
9  */
10 
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/virtio-dmabuf.h"
14 #include "hw/virtio/vhost.h"
15 #include "hw/virtio/virtio-crypto.h"
16 #include "hw/virtio/vhost-user.h"
17 #include "hw/virtio/vhost-backend.h"
18 #include "hw/virtio/virtio.h"
19 #include "hw/virtio/virtio-net.h"
20 #include "chardev/char-fe.h"
21 #include "io/channel-socket.h"
22 #include "sysemu/kvm.h"
23 #include "qemu/error-report.h"
24 #include "qemu/main-loop.h"
25 #include "qemu/uuid.h"
26 #include "qemu/sockets.h"
27 #include "sysemu/runstate.h"
28 #include "sysemu/cryptodev.h"
29 #include "migration/postcopy-ram.h"
30 #include "trace.h"
31 #include "exec/ramblock.h"
32 
33 #include <sys/ioctl.h>
34 #include <sys/socket.h>
35 #include <sys/un.h>
36 
37 #include "standard-headers/linux/vhost_types.h"
38 
39 #ifdef CONFIG_LINUX
40 #include <linux/userfaultfd.h>
41 #endif
42 
43 #define VHOST_MEMORY_BASELINE_NREGIONS    8
44 #define VHOST_USER_F_PROTOCOL_FEATURES 30
45 #define VHOST_USER_BACKEND_MAX_FDS     8
46 
47 #if defined(TARGET_PPC) || defined(TARGET_PPC64)
48 #include "hw/ppc/spapr.h"
49 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
50 
51 #else
52 #define VHOST_USER_MAX_RAM_SLOTS 512
53 #endif
54 
55 /*
56  * Maximum size of virtio device config space
57  */
58 #define VHOST_USER_MAX_CONFIG_SIZE 256
59 
60 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
61 
62 typedef enum VhostUserRequest {
63     VHOST_USER_NONE = 0,
64     VHOST_USER_GET_FEATURES = 1,
65     VHOST_USER_SET_FEATURES = 2,
66     VHOST_USER_SET_OWNER = 3,
67     VHOST_USER_RESET_OWNER = 4,
68     VHOST_USER_SET_MEM_TABLE = 5,
69     VHOST_USER_SET_LOG_BASE = 6,
70     VHOST_USER_SET_LOG_FD = 7,
71     VHOST_USER_SET_VRING_NUM = 8,
72     VHOST_USER_SET_VRING_ADDR = 9,
73     VHOST_USER_SET_VRING_BASE = 10,
74     VHOST_USER_GET_VRING_BASE = 11,
75     VHOST_USER_SET_VRING_KICK = 12,
76     VHOST_USER_SET_VRING_CALL = 13,
77     VHOST_USER_SET_VRING_ERR = 14,
78     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
79     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
80     VHOST_USER_GET_QUEUE_NUM = 17,
81     VHOST_USER_SET_VRING_ENABLE = 18,
82     VHOST_USER_SEND_RARP = 19,
83     VHOST_USER_NET_SET_MTU = 20,
84     VHOST_USER_SET_BACKEND_REQ_FD = 21,
85     VHOST_USER_IOTLB_MSG = 22,
86     VHOST_USER_SET_VRING_ENDIAN = 23,
87     VHOST_USER_GET_CONFIG = 24,
88     VHOST_USER_SET_CONFIG = 25,
89     VHOST_USER_CREATE_CRYPTO_SESSION = 26,
90     VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
91     VHOST_USER_POSTCOPY_ADVISE  = 28,
92     VHOST_USER_POSTCOPY_LISTEN  = 29,
93     VHOST_USER_POSTCOPY_END     = 30,
94     VHOST_USER_GET_INFLIGHT_FD = 31,
95     VHOST_USER_SET_INFLIGHT_FD = 32,
96     VHOST_USER_GPU_SET_SOCKET = 33,
97     VHOST_USER_RESET_DEVICE = 34,
98     /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
99     VHOST_USER_GET_MAX_MEM_SLOTS = 36,
100     VHOST_USER_ADD_MEM_REG = 37,
101     VHOST_USER_REM_MEM_REG = 38,
102     VHOST_USER_SET_STATUS = 39,
103     VHOST_USER_GET_STATUS = 40,
104     VHOST_USER_GET_SHARED_OBJECT = 41,
105     VHOST_USER_SET_DEVICE_STATE_FD = 42,
106     VHOST_USER_CHECK_DEVICE_STATE = 43,
107     VHOST_USER_MAX
108 } VhostUserRequest;
109 
110 typedef enum VhostUserBackendRequest {
111     VHOST_USER_BACKEND_NONE = 0,
112     VHOST_USER_BACKEND_IOTLB_MSG = 1,
113     VHOST_USER_BACKEND_CONFIG_CHANGE_MSG = 2,
114     VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG = 3,
115     VHOST_USER_BACKEND_SHARED_OBJECT_ADD = 6,
116     VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE = 7,
117     VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP = 8,
118     VHOST_USER_BACKEND_MAX
119 }  VhostUserBackendRequest;
120 
121 typedef struct VhostUserMemoryRegion {
122     uint64_t guest_phys_addr;
123     uint64_t memory_size;
124     uint64_t userspace_addr;
125     uint64_t mmap_offset;
126 } VhostUserMemoryRegion;
127 
128 typedef struct VhostUserMemory {
129     uint32_t nregions;
130     uint32_t padding;
131     VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
132 } VhostUserMemory;
133 
134 typedef struct VhostUserMemRegMsg {
135     uint64_t padding;
136     VhostUserMemoryRegion region;
137 } VhostUserMemRegMsg;
138 
139 typedef struct VhostUserLog {
140     uint64_t mmap_size;
141     uint64_t mmap_offset;
142 } VhostUserLog;
143 
144 typedef struct VhostUserConfig {
145     uint32_t offset;
146     uint32_t size;
147     uint32_t flags;
148     uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
149 } VhostUserConfig;
150 
151 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN    512
152 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN  64
153 #define VHOST_CRYPTO_ASYM_MAX_KEY_LEN  1024
154 
155 typedef struct VhostUserCryptoSession {
156     uint64_t op_code;
157     union {
158         struct {
159             CryptoDevBackendSymSessionInfo session_setup_data;
160             uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
161             uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
162         } sym;
163         struct {
164             CryptoDevBackendAsymSessionInfo session_setup_data;
165             uint8_t key[VHOST_CRYPTO_ASYM_MAX_KEY_LEN];
166         } asym;
167     } u;
168 
169     /* session id for success, -1 on errors */
170     int64_t session_id;
171 } VhostUserCryptoSession;
172 
173 static VhostUserConfig c __attribute__ ((unused));
174 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
175                                    + sizeof(c.size) \
176                                    + sizeof(c.flags))
177 
178 typedef struct VhostUserVringArea {
179     uint64_t u64;
180     uint64_t size;
181     uint64_t offset;
182 } VhostUserVringArea;
183 
184 typedef struct VhostUserInflight {
185     uint64_t mmap_size;
186     uint64_t mmap_offset;
187     uint16_t num_queues;
188     uint16_t queue_size;
189 } VhostUserInflight;
190 
191 typedef struct VhostUserShared {
192     unsigned char uuid[16];
193 } VhostUserShared;
194 
195 typedef struct {
196     VhostUserRequest request;
197 
198 #define VHOST_USER_VERSION_MASK     (0x3)
199 #define VHOST_USER_REPLY_MASK       (0x1 << 2)
200 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
201     uint32_t flags;
202     uint32_t size; /* the following payload size */
203 } QEMU_PACKED VhostUserHeader;
204 
205 /* Request payload of VHOST_USER_SET_DEVICE_STATE_FD */
206 typedef struct VhostUserTransferDeviceState {
207     uint32_t direction;
208     uint32_t phase;
209 } VhostUserTransferDeviceState;
210 
211 typedef union {
212 #define VHOST_USER_VRING_IDX_MASK   (0xff)
213 #define VHOST_USER_VRING_NOFD_MASK  (0x1 << 8)
214         uint64_t u64;
215         struct vhost_vring_state state;
216         struct vhost_vring_addr addr;
217         VhostUserMemory memory;
218         VhostUserMemRegMsg mem_reg;
219         VhostUserLog log;
220         struct vhost_iotlb_msg iotlb;
221         VhostUserConfig config;
222         VhostUserCryptoSession session;
223         VhostUserVringArea area;
224         VhostUserInflight inflight;
225         VhostUserShared object;
226         VhostUserTransferDeviceState transfer_state;
227 } VhostUserPayload;
228 
229 typedef struct VhostUserMsg {
230     VhostUserHeader hdr;
231     VhostUserPayload payload;
232 } QEMU_PACKED VhostUserMsg;
233 
234 static VhostUserMsg m __attribute__ ((unused));
235 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
236 
237 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
238 
239 /* The version of the protocol we support */
240 #define VHOST_USER_VERSION    (0x1)
241 
242 struct vhost_user {
243     struct vhost_dev *dev;
244     /* Shared between vhost devs of the same virtio device */
245     VhostUserState *user;
246     QIOChannel *backend_ioc;
247     GSource *backend_src;
248     NotifierWithReturn postcopy_notifier;
249     struct PostCopyFD  postcopy_fd;
250     uint64_t           postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
251     /* Length of the region_rb and region_rb_offset arrays */
252     size_t             region_rb_len;
253     /* RAMBlock associated with a given region */
254     RAMBlock         **region_rb;
255     /*
256      * The offset from the start of the RAMBlock to the start of the
257      * vhost region.
258      */
259     ram_addr_t        *region_rb_offset;
260 
261     /* True once we've entered postcopy_listen */
262     bool               postcopy_listen;
263 
264     /* Our current regions */
265     int num_shadow_regions;
266     struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
267 };
268 
269 struct scrub_regions {
270     struct vhost_memory_region *region;
271     int reg_idx;
272     int fd_idx;
273 };
274 
vhost_user_read_header(struct vhost_dev * dev,VhostUserMsg * msg)275 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
276 {
277     struct vhost_user *u = dev->opaque;
278     CharBackend *chr = u->user->chr;
279     uint8_t *p = (uint8_t *) msg;
280     int r, size = VHOST_USER_HDR_SIZE;
281 
282     r = qemu_chr_fe_read_all(chr, p, size);
283     if (r != size) {
284         int saved_errno = errno;
285         error_report("Failed to read msg header. Read %d instead of %d."
286                      " Original request %d.", r, size, msg->hdr.request);
287         return r < 0 ? -saved_errno : -EIO;
288     }
289 
290     /* validate received flags */
291     if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
292         error_report("Failed to read msg header."
293                 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
294                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
295         return -EPROTO;
296     }
297 
298     trace_vhost_user_read(msg->hdr.request, msg->hdr.flags);
299 
300     return 0;
301 }
302 
vhost_user_read(struct vhost_dev * dev,VhostUserMsg * msg)303 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
304 {
305     struct vhost_user *u = dev->opaque;
306     CharBackend *chr = u->user->chr;
307     uint8_t *p = (uint8_t *) msg;
308     int r, size;
309 
310     r = vhost_user_read_header(dev, msg);
311     if (r < 0) {
312         return r;
313     }
314 
315     /* validate message size is sane */
316     if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
317         error_report("Failed to read msg header."
318                 " Size %d exceeds the maximum %zu.", msg->hdr.size,
319                 VHOST_USER_PAYLOAD_SIZE);
320         return -EPROTO;
321     }
322 
323     if (msg->hdr.size) {
324         p += VHOST_USER_HDR_SIZE;
325         size = msg->hdr.size;
326         r = qemu_chr_fe_read_all(chr, p, size);
327         if (r != size) {
328             int saved_errno = errno;
329             error_report("Failed to read msg payload."
330                          " Read %d instead of %d.", r, msg->hdr.size);
331             return r < 0 ? -saved_errno : -EIO;
332         }
333     }
334 
335     return 0;
336 }
337 
process_message_reply(struct vhost_dev * dev,const VhostUserMsg * msg)338 static int process_message_reply(struct vhost_dev *dev,
339                                  const VhostUserMsg *msg)
340 {
341     int ret;
342     VhostUserMsg msg_reply;
343 
344     if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
345         return 0;
346     }
347 
348     ret = vhost_user_read(dev, &msg_reply);
349     if (ret < 0) {
350         return ret;
351     }
352 
353     if (msg_reply.hdr.request != msg->hdr.request) {
354         error_report("Received unexpected msg type. "
355                      "Expected %d received %d",
356                      msg->hdr.request, msg_reply.hdr.request);
357         return -EPROTO;
358     }
359 
360     return msg_reply.payload.u64 ? -EIO : 0;
361 }
362 
vhost_user_per_device_request(VhostUserRequest request)363 static bool vhost_user_per_device_request(VhostUserRequest request)
364 {
365     switch (request) {
366     case VHOST_USER_SET_OWNER:
367     case VHOST_USER_RESET_OWNER:
368     case VHOST_USER_SET_MEM_TABLE:
369     case VHOST_USER_GET_QUEUE_NUM:
370     case VHOST_USER_NET_SET_MTU:
371     case VHOST_USER_RESET_DEVICE:
372     case VHOST_USER_ADD_MEM_REG:
373     case VHOST_USER_REM_MEM_REG:
374     case VHOST_USER_SET_LOG_BASE:
375         return true;
376     default:
377         return false;
378     }
379 }
380 
381 /* most non-init callers ignore the error */
vhost_user_write(struct vhost_dev * dev,VhostUserMsg * msg,int * fds,int fd_num)382 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
383                             int *fds, int fd_num)
384 {
385     struct vhost_user *u = dev->opaque;
386     CharBackend *chr = u->user->chr;
387     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
388 
389     /*
390      * Some devices, like virtio-scsi, are implemented as a single vhost_dev,
391      * while others, like virtio-net, contain multiple vhost_devs. For
392      * operations such as configuring device memory mappings or issuing device
393      * resets, which affect the whole device instead of individual VQs,
394      * vhost-user messages should only be sent once.
395      *
396      * Devices with multiple vhost_devs are given an associated dev->vq_index
397      * so per_device requests are only sent if vq_index is 0.
398      */
399     if (vhost_user_per_device_request(msg->hdr.request)
400         && dev->vq_index != 0) {
401         msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
402         return 0;
403     }
404 
405     if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
406         error_report("Failed to set msg fds.");
407         return -EINVAL;
408     }
409 
410     ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
411     if (ret != size) {
412         int saved_errno = errno;
413         error_report("Failed to write msg."
414                      " Wrote %d instead of %d.", ret, size);
415         return ret < 0 ? -saved_errno : -EIO;
416     }
417 
418     trace_vhost_user_write(msg->hdr.request, msg->hdr.flags);
419 
420     return 0;
421 }
422 
vhost_user_gpu_set_socket(struct vhost_dev * dev,int fd)423 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
424 {
425     VhostUserMsg msg = {
426         .hdr.request = VHOST_USER_GPU_SET_SOCKET,
427         .hdr.flags = VHOST_USER_VERSION,
428     };
429 
430     return vhost_user_write(dev, &msg, &fd, 1);
431 }
432 
vhost_user_set_log_base(struct vhost_dev * dev,uint64_t base,struct vhost_log * log)433 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
434                                    struct vhost_log *log)
435 {
436     int fds[VHOST_USER_MAX_RAM_SLOTS];
437     size_t fd_num = 0;
438     bool shmfd = virtio_has_feature(dev->protocol_features,
439                                     VHOST_USER_PROTOCOL_F_LOG_SHMFD);
440     int ret;
441     VhostUserMsg msg = {
442         .hdr.request = VHOST_USER_SET_LOG_BASE,
443         .hdr.flags = VHOST_USER_VERSION,
444         .payload.log.mmap_size = log->size * sizeof(*(log->log)),
445         .payload.log.mmap_offset = 0,
446         .hdr.size = sizeof(msg.payload.log),
447     };
448 
449     /* Send only once with first queue pair */
450     if (dev->vq_index != 0) {
451         return 0;
452     }
453 
454     if (shmfd && log->fd != -1) {
455         fds[fd_num++] = log->fd;
456     }
457 
458     ret = vhost_user_write(dev, &msg, fds, fd_num);
459     if (ret < 0) {
460         return ret;
461     }
462 
463     if (shmfd) {
464         msg.hdr.size = 0;
465         ret = vhost_user_read(dev, &msg);
466         if (ret < 0) {
467             return ret;
468         }
469 
470         if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
471             error_report("Received unexpected msg type. "
472                          "Expected %d received %d",
473                          VHOST_USER_SET_LOG_BASE, msg.hdr.request);
474             return -EPROTO;
475         }
476     }
477 
478     return 0;
479 }
480 
vhost_user_get_mr_data(uint64_t addr,ram_addr_t * offset,int * fd)481 static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
482                                             int *fd)
483 {
484     MemoryRegion *mr;
485 
486     assert((uintptr_t)addr == addr);
487     mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
488     *fd = memory_region_get_fd(mr);
489     *offset += mr->ram_block->fd_offset;
490 
491     return mr;
492 }
493 
vhost_user_fill_msg_region(VhostUserMemoryRegion * dst,struct vhost_memory_region * src,uint64_t mmap_offset)494 static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
495                                        struct vhost_memory_region *src,
496                                        uint64_t mmap_offset)
497 {
498     assert(src != NULL && dst != NULL);
499     dst->userspace_addr = src->userspace_addr;
500     dst->memory_size = src->memory_size;
501     dst->guest_phys_addr = src->guest_phys_addr;
502     dst->mmap_offset = mmap_offset;
503 }
504 
vhost_user_fill_set_mem_table_msg(struct vhost_user * u,struct vhost_dev * dev,VhostUserMsg * msg,int * fds,size_t * fd_num,bool track_ramblocks)505 static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
506                                              struct vhost_dev *dev,
507                                              VhostUserMsg *msg,
508                                              int *fds, size_t *fd_num,
509                                              bool track_ramblocks)
510 {
511     int i, fd;
512     ram_addr_t offset;
513     MemoryRegion *mr;
514     struct vhost_memory_region *reg;
515     VhostUserMemoryRegion region_buffer;
516 
517     msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
518 
519     for (i = 0; i < dev->mem->nregions; ++i) {
520         reg = dev->mem->regions + i;
521 
522         mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
523         if (fd > 0) {
524             if (track_ramblocks) {
525                 assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
526                 trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
527                                                       reg->memory_size,
528                                                       reg->guest_phys_addr,
529                                                       reg->userspace_addr,
530                                                       offset);
531                 u->region_rb_offset[i] = offset;
532                 u->region_rb[i] = mr->ram_block;
533             } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
534                 error_report("Failed preparing vhost-user memory table msg");
535                 return -ENOBUFS;
536             }
537             vhost_user_fill_msg_region(&region_buffer, reg, offset);
538             msg->payload.memory.regions[*fd_num] = region_buffer;
539             fds[(*fd_num)++] = fd;
540         } else if (track_ramblocks) {
541             u->region_rb_offset[i] = 0;
542             u->region_rb[i] = NULL;
543         }
544     }
545 
546     msg->payload.memory.nregions = *fd_num;
547 
548     if (!*fd_num) {
549         error_report("Failed initializing vhost-user memory map, "
550                      "consider using -object memory-backend-file share=on");
551         return -EINVAL;
552     }
553 
554     msg->hdr.size = sizeof(msg->payload.memory.nregions);
555     msg->hdr.size += sizeof(msg->payload.memory.padding);
556     msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
557 
558     return 0;
559 }
560 
reg_equal(struct vhost_memory_region * shadow_reg,struct vhost_memory_region * vdev_reg)561 static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
562                              struct vhost_memory_region *vdev_reg)
563 {
564     return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
565         shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
566         shadow_reg->memory_size == vdev_reg->memory_size;
567 }
568 
scrub_shadow_regions(struct vhost_dev * dev,struct scrub_regions * add_reg,int * nr_add_reg,struct scrub_regions * rem_reg,int * nr_rem_reg,uint64_t * shadow_pcb,bool track_ramblocks)569 static void scrub_shadow_regions(struct vhost_dev *dev,
570                                  struct scrub_regions *add_reg,
571                                  int *nr_add_reg,
572                                  struct scrub_regions *rem_reg,
573                                  int *nr_rem_reg, uint64_t *shadow_pcb,
574                                  bool track_ramblocks)
575 {
576     struct vhost_user *u = dev->opaque;
577     bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
578     struct vhost_memory_region *reg, *shadow_reg;
579     int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
580     ram_addr_t offset;
581     MemoryRegion *mr;
582     bool matching;
583 
584     /*
585      * Find memory regions present in our shadow state which are not in
586      * the device's current memory state.
587      *
588      * Mark regions in both the shadow and device state as "found".
589      */
590     for (i = 0; i < u->num_shadow_regions; i++) {
591         shadow_reg = &u->shadow_regions[i];
592         matching = false;
593 
594         for (j = 0; j < dev->mem->nregions; j++) {
595             reg = &dev->mem->regions[j];
596 
597             mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
598 
599             if (reg_equal(shadow_reg, reg)) {
600                 matching = true;
601                 found[j] = true;
602                 if (track_ramblocks) {
603                     /*
604                      * Reset postcopy client bases, region_rb, and
605                      * region_rb_offset in case regions are removed.
606                      */
607                     if (fd > 0) {
608                         u->region_rb_offset[j] = offset;
609                         u->region_rb[j] = mr->ram_block;
610                         shadow_pcb[j] = u->postcopy_client_bases[i];
611                     } else {
612                         u->region_rb_offset[j] = 0;
613                         u->region_rb[j] = NULL;
614                     }
615                 }
616                 break;
617             }
618         }
619 
620         /*
621          * If the region was not found in the current device memory state
622          * create an entry for it in the removed list.
623          */
624         if (!matching) {
625             rem_reg[rm_idx].region = shadow_reg;
626             rem_reg[rm_idx++].reg_idx = i;
627         }
628     }
629 
630     /*
631      * For regions not marked "found", create entries in the added list.
632      *
633      * Note their indexes in the device memory state and the indexes of their
634      * file descriptors.
635      */
636     for (i = 0; i < dev->mem->nregions; i++) {
637         reg = &dev->mem->regions[i];
638         vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
639         if (fd > 0) {
640             ++fd_num;
641         }
642 
643         /*
644          * If the region was in both the shadow and device state we don't
645          * need to send a VHOST_USER_ADD_MEM_REG message for it.
646          */
647         if (found[i]) {
648             continue;
649         }
650 
651         add_reg[add_idx].region = reg;
652         add_reg[add_idx].reg_idx = i;
653         add_reg[add_idx++].fd_idx = fd_num;
654     }
655     *nr_rem_reg = rm_idx;
656     *nr_add_reg = add_idx;
657 
658     return;
659 }
660 
send_remove_regions(struct vhost_dev * dev,struct scrub_regions * remove_reg,int nr_rem_reg,VhostUserMsg * msg,bool reply_supported)661 static int send_remove_regions(struct vhost_dev *dev,
662                                struct scrub_regions *remove_reg,
663                                int nr_rem_reg, VhostUserMsg *msg,
664                                bool reply_supported)
665 {
666     struct vhost_user *u = dev->opaque;
667     struct vhost_memory_region *shadow_reg;
668     int i, fd, shadow_reg_idx, ret;
669     ram_addr_t offset;
670     VhostUserMemoryRegion region_buffer;
671 
672     /*
673      * The regions in remove_reg appear in the same order they do in the
674      * shadow table. Therefore we can minimize memory copies by iterating
675      * through remove_reg backwards.
676      */
677     for (i = nr_rem_reg - 1; i >= 0; i--) {
678         shadow_reg = remove_reg[i].region;
679         shadow_reg_idx = remove_reg[i].reg_idx;
680 
681         vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
682 
683         if (fd > 0) {
684             msg->hdr.request = VHOST_USER_REM_MEM_REG;
685             vhost_user_fill_msg_region(&region_buffer, shadow_reg, 0);
686             msg->payload.mem_reg.region = region_buffer;
687 
688             ret = vhost_user_write(dev, msg, NULL, 0);
689             if (ret < 0) {
690                 return ret;
691             }
692 
693             if (reply_supported) {
694                 ret = process_message_reply(dev, msg);
695                 if (ret) {
696                     return ret;
697                 }
698             }
699         }
700 
701         /*
702          * At this point we know the backend has unmapped the region. It is now
703          * safe to remove it from the shadow table.
704          */
705         memmove(&u->shadow_regions[shadow_reg_idx],
706                 &u->shadow_regions[shadow_reg_idx + 1],
707                 sizeof(struct vhost_memory_region) *
708                 (u->num_shadow_regions - shadow_reg_idx - 1));
709         u->num_shadow_regions--;
710     }
711 
712     return 0;
713 }
714 
send_add_regions(struct vhost_dev * dev,struct scrub_regions * add_reg,int nr_add_reg,VhostUserMsg * msg,uint64_t * shadow_pcb,bool reply_supported,bool track_ramblocks)715 static int send_add_regions(struct vhost_dev *dev,
716                             struct scrub_regions *add_reg, int nr_add_reg,
717                             VhostUserMsg *msg, uint64_t *shadow_pcb,
718                             bool reply_supported, bool track_ramblocks)
719 {
720     struct vhost_user *u = dev->opaque;
721     int i, fd, ret, reg_idx, reg_fd_idx;
722     struct vhost_memory_region *reg;
723     MemoryRegion *mr;
724     ram_addr_t offset;
725     VhostUserMsg msg_reply;
726     VhostUserMemoryRegion region_buffer;
727 
728     for (i = 0; i < nr_add_reg; i++) {
729         reg = add_reg[i].region;
730         reg_idx = add_reg[i].reg_idx;
731         reg_fd_idx = add_reg[i].fd_idx;
732 
733         mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
734 
735         if (fd > 0) {
736             if (track_ramblocks) {
737                 trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
738                                                       reg->memory_size,
739                                                       reg->guest_phys_addr,
740                                                       reg->userspace_addr,
741                                                       offset);
742                 u->region_rb_offset[reg_idx] = offset;
743                 u->region_rb[reg_idx] = mr->ram_block;
744             }
745             msg->hdr.request = VHOST_USER_ADD_MEM_REG;
746             vhost_user_fill_msg_region(&region_buffer, reg, offset);
747             msg->payload.mem_reg.region = region_buffer;
748 
749             ret = vhost_user_write(dev, msg, &fd, 1);
750             if (ret < 0) {
751                 return ret;
752             }
753 
754             if (track_ramblocks) {
755                 uint64_t reply_gpa;
756 
757                 ret = vhost_user_read(dev, &msg_reply);
758                 if (ret < 0) {
759                     return ret;
760                 }
761 
762                 reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
763 
764                 if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
765                     error_report("%s: Received unexpected msg type."
766                                  "Expected %d received %d", __func__,
767                                  VHOST_USER_ADD_MEM_REG,
768                                  msg_reply.hdr.request);
769                     return -EPROTO;
770                 }
771 
772                 /*
773                  * We're using the same structure, just reusing one of the
774                  * fields, so it should be the same size.
775                  */
776                 if (msg_reply.hdr.size != msg->hdr.size) {
777                     error_report("%s: Unexpected size for postcopy reply "
778                                  "%d vs %d", __func__, msg_reply.hdr.size,
779                                  msg->hdr.size);
780                     return -EPROTO;
781                 }
782 
783                 /* Get the postcopy client base from the backend's reply. */
784                 if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
785                     shadow_pcb[reg_idx] =
786                         msg_reply.payload.mem_reg.region.userspace_addr;
787                     trace_vhost_user_set_mem_table_postcopy(
788                         msg_reply.payload.mem_reg.region.userspace_addr,
789                         msg->payload.mem_reg.region.userspace_addr,
790                         reg_fd_idx, reg_idx);
791                 } else {
792                     error_report("%s: invalid postcopy reply for region. "
793                                  "Got guest physical address %" PRIX64 ", expected "
794                                  "%" PRIX64, __func__, reply_gpa,
795                                  dev->mem->regions[reg_idx].guest_phys_addr);
796                     return -EPROTO;
797                 }
798             } else if (reply_supported) {
799                 ret = process_message_reply(dev, msg);
800                 if (ret) {
801                     return ret;
802                 }
803             }
804         } else if (track_ramblocks) {
805             u->region_rb_offset[reg_idx] = 0;
806             u->region_rb[reg_idx] = NULL;
807         }
808 
809         /*
810          * At this point, we know the backend has mapped in the new
811          * region, if the region has a valid file descriptor.
812          *
813          * The region should now be added to the shadow table.
814          */
815         u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
816             reg->guest_phys_addr;
817         u->shadow_regions[u->num_shadow_regions].userspace_addr =
818             reg->userspace_addr;
819         u->shadow_regions[u->num_shadow_regions].memory_size =
820             reg->memory_size;
821         u->num_shadow_regions++;
822     }
823 
824     return 0;
825 }
826 
vhost_user_add_remove_regions(struct vhost_dev * dev,VhostUserMsg * msg,bool reply_supported,bool track_ramblocks)827 static int vhost_user_add_remove_regions(struct vhost_dev *dev,
828                                          VhostUserMsg *msg,
829                                          bool reply_supported,
830                                          bool track_ramblocks)
831 {
832     struct vhost_user *u = dev->opaque;
833     struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
834     struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
835     uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
836     int nr_add_reg, nr_rem_reg;
837     int ret;
838 
839     msg->hdr.size = sizeof(msg->payload.mem_reg);
840 
841     /* Find the regions which need to be removed or added. */
842     scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
843                          shadow_pcb, track_ramblocks);
844 
845     if (nr_rem_reg) {
846         ret = send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
847                                   reply_supported);
848         if (ret < 0) {
849             goto err;
850         }
851     }
852 
853     if (nr_add_reg) {
854         ret = send_add_regions(dev, add_reg, nr_add_reg, msg, shadow_pcb,
855                                reply_supported, track_ramblocks);
856         if (ret < 0) {
857             goto err;
858         }
859     }
860 
861     if (track_ramblocks) {
862         memcpy(u->postcopy_client_bases, shadow_pcb,
863                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
864         /*
865          * Now we've registered this with the postcopy code, we ack to the
866          * client, because now we're in the position to be able to deal with
867          * any faults it generates.
868          */
869         /* TODO: Use this for failure cases as well with a bad value. */
870         msg->hdr.size = sizeof(msg->payload.u64);
871         msg->payload.u64 = 0; /* OK */
872 
873         ret = vhost_user_write(dev, msg, NULL, 0);
874         if (ret < 0) {
875             return ret;
876         }
877     }
878 
879     return 0;
880 
881 err:
882     if (track_ramblocks) {
883         memcpy(u->postcopy_client_bases, shadow_pcb,
884                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
885     }
886 
887     return ret;
888 }
889 
vhost_user_set_mem_table_postcopy(struct vhost_dev * dev,struct vhost_memory * mem,bool reply_supported,bool config_mem_slots)890 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
891                                              struct vhost_memory *mem,
892                                              bool reply_supported,
893                                              bool config_mem_slots)
894 {
895     struct vhost_user *u = dev->opaque;
896     int fds[VHOST_MEMORY_BASELINE_NREGIONS];
897     size_t fd_num = 0;
898     VhostUserMsg msg_reply;
899     int region_i, msg_i;
900     int ret;
901 
902     VhostUserMsg msg = {
903         .hdr.flags = VHOST_USER_VERSION,
904     };
905 
906     if (u->region_rb_len < dev->mem->nregions) {
907         u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
908         u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
909                                       dev->mem->nregions);
910         memset(&(u->region_rb[u->region_rb_len]), '\0',
911                sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
912         memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
913                sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
914         u->region_rb_len = dev->mem->nregions;
915     }
916 
917     if (config_mem_slots) {
918         ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, true);
919         if (ret < 0) {
920             return ret;
921         }
922     } else {
923         ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
924                                                 true);
925         if (ret < 0) {
926             return ret;
927         }
928 
929         ret = vhost_user_write(dev, &msg, fds, fd_num);
930         if (ret < 0) {
931             return ret;
932         }
933 
934         ret = vhost_user_read(dev, &msg_reply);
935         if (ret < 0) {
936             return ret;
937         }
938 
939         if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
940             error_report("%s: Received unexpected msg type."
941                          "Expected %d received %d", __func__,
942                          VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
943             return -EPROTO;
944         }
945 
946         /*
947          * We're using the same structure, just reusing one of the
948          * fields, so it should be the same size.
949          */
950         if (msg_reply.hdr.size != msg.hdr.size) {
951             error_report("%s: Unexpected size for postcopy reply "
952                          "%d vs %d", __func__, msg_reply.hdr.size,
953                          msg.hdr.size);
954             return -EPROTO;
955         }
956 
957         memset(u->postcopy_client_bases, 0,
958                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
959 
960         /*
961          * They're in the same order as the regions that were sent
962          * but some of the regions were skipped (above) if they
963          * didn't have fd's
964          */
965         for (msg_i = 0, region_i = 0;
966              region_i < dev->mem->nregions;
967              region_i++) {
968             if (msg_i < fd_num &&
969                 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
970                 dev->mem->regions[region_i].guest_phys_addr) {
971                 u->postcopy_client_bases[region_i] =
972                     msg_reply.payload.memory.regions[msg_i].userspace_addr;
973                 trace_vhost_user_set_mem_table_postcopy(
974                     msg_reply.payload.memory.regions[msg_i].userspace_addr,
975                     msg.payload.memory.regions[msg_i].userspace_addr,
976                     msg_i, region_i);
977                 msg_i++;
978             }
979         }
980         if (msg_i != fd_num) {
981             error_report("%s: postcopy reply not fully consumed "
982                          "%d vs %zd",
983                          __func__, msg_i, fd_num);
984             return -EIO;
985         }
986 
987         /*
988          * Now we've registered this with the postcopy code, we ack to the
989          * client, because now we're in the position to be able to deal
990          * with any faults it generates.
991          */
992         /* TODO: Use this for failure cases as well with a bad value. */
993         msg.hdr.size = sizeof(msg.payload.u64);
994         msg.payload.u64 = 0; /* OK */
995         ret = vhost_user_write(dev, &msg, NULL, 0);
996         if (ret < 0) {
997             return ret;
998         }
999     }
1000 
1001     return 0;
1002 }
1003 
vhost_user_set_mem_table(struct vhost_dev * dev,struct vhost_memory * mem)1004 static int vhost_user_set_mem_table(struct vhost_dev *dev,
1005                                     struct vhost_memory *mem)
1006 {
1007     struct vhost_user *u = dev->opaque;
1008     int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1009     size_t fd_num = 0;
1010     bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
1011     bool reply_supported = virtio_has_feature(dev->protocol_features,
1012                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1013     bool config_mem_slots =
1014         virtio_has_feature(dev->protocol_features,
1015                            VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
1016     int ret;
1017 
1018     if (do_postcopy) {
1019         /*
1020          * Postcopy has enough differences that it's best done in it's own
1021          * version
1022          */
1023         return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
1024                                                  config_mem_slots);
1025     }
1026 
1027     VhostUserMsg msg = {
1028         .hdr.flags = VHOST_USER_VERSION,
1029     };
1030 
1031     if (reply_supported) {
1032         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1033     }
1034 
1035     if (config_mem_slots) {
1036         ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, false);
1037         if (ret < 0) {
1038             return ret;
1039         }
1040     } else {
1041         ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1042                                                 false);
1043         if (ret < 0) {
1044             return ret;
1045         }
1046 
1047         ret = vhost_user_write(dev, &msg, fds, fd_num);
1048         if (ret < 0) {
1049             return ret;
1050         }
1051 
1052         if (reply_supported) {
1053             return process_message_reply(dev, &msg);
1054         }
1055     }
1056 
1057     return 0;
1058 }
1059 
vhost_user_set_vring_endian(struct vhost_dev * dev,struct vhost_vring_state * ring)1060 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1061                                        struct vhost_vring_state *ring)
1062 {
1063     bool cross_endian = virtio_has_feature(dev->protocol_features,
1064                                            VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1065     VhostUserMsg msg = {
1066         .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1067         .hdr.flags = VHOST_USER_VERSION,
1068         .payload.state = *ring,
1069         .hdr.size = sizeof(msg.payload.state),
1070     };
1071 
1072     if (!cross_endian) {
1073         error_report("vhost-user trying to send unhandled ioctl");
1074         return -ENOTSUP;
1075     }
1076 
1077     return vhost_user_write(dev, &msg, NULL, 0);
1078 }
1079 
vhost_user_get_u64(struct vhost_dev * dev,int request,uint64_t * u64)1080 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
1081 {
1082     int ret;
1083     VhostUserMsg msg = {
1084         .hdr.request = request,
1085         .hdr.flags = VHOST_USER_VERSION,
1086     };
1087 
1088     if (vhost_user_per_device_request(request) && dev->vq_index != 0) {
1089         return 0;
1090     }
1091 
1092     ret = vhost_user_write(dev, &msg, NULL, 0);
1093     if (ret < 0) {
1094         return ret;
1095     }
1096 
1097     ret = vhost_user_read(dev, &msg);
1098     if (ret < 0) {
1099         return ret;
1100     }
1101 
1102     if (msg.hdr.request != request) {
1103         error_report("Received unexpected msg type. Expected %d received %d",
1104                      request, msg.hdr.request);
1105         return -EPROTO;
1106     }
1107 
1108     if (msg.hdr.size != sizeof(msg.payload.u64)) {
1109         error_report("Received bad msg size.");
1110         return -EPROTO;
1111     }
1112 
1113     *u64 = msg.payload.u64;
1114 
1115     return 0;
1116 }
1117 
vhost_user_get_features(struct vhost_dev * dev,uint64_t * features)1118 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
1119 {
1120     if (vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features) < 0) {
1121         return -EPROTO;
1122     }
1123 
1124     return 0;
1125 }
1126 
1127 /* Note: "msg->hdr.flags" may be modified. */
vhost_user_write_sync(struct vhost_dev * dev,VhostUserMsg * msg,bool wait_for_reply)1128 static int vhost_user_write_sync(struct vhost_dev *dev, VhostUserMsg *msg,
1129                                  bool wait_for_reply)
1130 {
1131     int ret;
1132 
1133     if (wait_for_reply) {
1134         bool reply_supported = virtio_has_feature(dev->protocol_features,
1135                                           VHOST_USER_PROTOCOL_F_REPLY_ACK);
1136         if (reply_supported) {
1137             msg->hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1138         }
1139     }
1140 
1141     ret = vhost_user_write(dev, msg, NULL, 0);
1142     if (ret < 0) {
1143         return ret;
1144     }
1145 
1146     if (wait_for_reply) {
1147         uint64_t dummy;
1148 
1149         if (msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1150             return process_message_reply(dev, msg);
1151         }
1152 
1153        /*
1154         * We need to wait for a reply but the backend does not
1155         * support replies for the command we just sent.
1156         * Send VHOST_USER_GET_FEATURES which makes all backends
1157         * send a reply.
1158         */
1159         return vhost_user_get_features(dev, &dummy);
1160     }
1161 
1162     return 0;
1163 }
1164 
vhost_set_vring(struct vhost_dev * dev,unsigned long int request,struct vhost_vring_state * ring,bool wait_for_reply)1165 static int vhost_set_vring(struct vhost_dev *dev,
1166                            unsigned long int request,
1167                            struct vhost_vring_state *ring,
1168                            bool wait_for_reply)
1169 {
1170     VhostUserMsg msg = {
1171         .hdr.request = request,
1172         .hdr.flags = VHOST_USER_VERSION,
1173         .payload.state = *ring,
1174         .hdr.size = sizeof(msg.payload.state),
1175     };
1176 
1177     return vhost_user_write_sync(dev, &msg, wait_for_reply);
1178 }
1179 
vhost_user_set_vring_num(struct vhost_dev * dev,struct vhost_vring_state * ring)1180 static int vhost_user_set_vring_num(struct vhost_dev *dev,
1181                                     struct vhost_vring_state *ring)
1182 {
1183     return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring, false);
1184 }
1185 
vhost_user_host_notifier_free(VhostUserHostNotifier * n)1186 static void vhost_user_host_notifier_free(VhostUserHostNotifier *n)
1187 {
1188     assert(n && n->unmap_addr);
1189     munmap(n->unmap_addr, qemu_real_host_page_size());
1190     n->unmap_addr = NULL;
1191 }
1192 
1193 /*
1194  * clean-up function for notifier, will finally free the structure
1195  * under rcu.
1196  */
vhost_user_host_notifier_remove(VhostUserHostNotifier * n,VirtIODevice * vdev)1197 static void vhost_user_host_notifier_remove(VhostUserHostNotifier *n,
1198                                             VirtIODevice *vdev)
1199 {
1200     if (n->addr) {
1201         if (vdev) {
1202             virtio_queue_set_host_notifier_mr(vdev, n->idx, &n->mr, false);
1203         }
1204         assert(!n->unmap_addr);
1205         n->unmap_addr = n->addr;
1206         n->addr = NULL;
1207         call_rcu(n, vhost_user_host_notifier_free, rcu);
1208     }
1209 }
1210 
vhost_user_set_vring_base(struct vhost_dev * dev,struct vhost_vring_state * ring)1211 static int vhost_user_set_vring_base(struct vhost_dev *dev,
1212                                      struct vhost_vring_state *ring)
1213 {
1214     return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring, false);
1215 }
1216 
vhost_user_set_vring_enable(struct vhost_dev * dev,int enable)1217 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1218 {
1219     int i;
1220 
1221     if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1222         return -EINVAL;
1223     }
1224 
1225     for (i = 0; i < dev->nvqs; ++i) {
1226         int ret;
1227         struct vhost_vring_state state = {
1228             .index = dev->vq_index + i,
1229             .num   = enable,
1230         };
1231 
1232         /*
1233          * SET_VRING_ENABLE travels from guest to QEMU to vhost-user backend /
1234          * control plane thread via unix domain socket. Virtio requests travel
1235          * from guest to vhost-user backend / data plane thread via eventfd.
1236          * Even if the guest enables the ring first, and pushes its first virtio
1237          * request second (conforming to the virtio spec), the data plane thread
1238          * in the backend may see the virtio request before the control plane
1239          * thread sees the queue enablement. This causes (in fact, requires) the
1240          * data plane thread to discard the virtio request (it arrived on a
1241          * seemingly disabled queue). To prevent this out-of-order delivery,
1242          * don't let the guest proceed to pushing the virtio request until the
1243          * backend control plane acknowledges enabling the queue -- IOW, pass
1244          * wait_for_reply=true below.
1245          */
1246         ret = vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state, true);
1247         if (ret < 0) {
1248             /*
1249              * Restoring the previous state is likely infeasible, as well as
1250              * proceeding regardless the error, so just bail out and hope for
1251              * the device-level recovery.
1252              */
1253             return ret;
1254         }
1255     }
1256 
1257     return 0;
1258 }
1259 
fetch_notifier(VhostUserState * u,int idx)1260 static VhostUserHostNotifier *fetch_notifier(VhostUserState *u,
1261                                              int idx)
1262 {
1263     if (idx >= u->notifiers->len) {
1264         return NULL;
1265     }
1266     return g_ptr_array_index(u->notifiers, idx);
1267 }
1268 
vhost_user_get_vring_base(struct vhost_dev * dev,struct vhost_vring_state * ring)1269 static int vhost_user_get_vring_base(struct vhost_dev *dev,
1270                                      struct vhost_vring_state *ring)
1271 {
1272     int ret;
1273     VhostUserMsg msg = {
1274         .hdr.request = VHOST_USER_GET_VRING_BASE,
1275         .hdr.flags = VHOST_USER_VERSION,
1276         .payload.state = *ring,
1277         .hdr.size = sizeof(msg.payload.state),
1278     };
1279     struct vhost_user *u = dev->opaque;
1280 
1281     VhostUserHostNotifier *n = fetch_notifier(u->user, ring->index);
1282     if (n) {
1283         vhost_user_host_notifier_remove(n, dev->vdev);
1284     }
1285 
1286     ret = vhost_user_write(dev, &msg, NULL, 0);
1287     if (ret < 0) {
1288         return ret;
1289     }
1290 
1291     ret = vhost_user_read(dev, &msg);
1292     if (ret < 0) {
1293         return ret;
1294     }
1295 
1296     if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1297         error_report("Received unexpected msg type. Expected %d received %d",
1298                      VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1299         return -EPROTO;
1300     }
1301 
1302     if (msg.hdr.size != sizeof(msg.payload.state)) {
1303         error_report("Received bad msg size.");
1304         return -EPROTO;
1305     }
1306 
1307     *ring = msg.payload.state;
1308 
1309     return 0;
1310 }
1311 
vhost_set_vring_file(struct vhost_dev * dev,VhostUserRequest request,struct vhost_vring_file * file)1312 static int vhost_set_vring_file(struct vhost_dev *dev,
1313                                 VhostUserRequest request,
1314                                 struct vhost_vring_file *file)
1315 {
1316     int fds[VHOST_USER_MAX_RAM_SLOTS];
1317     size_t fd_num = 0;
1318     VhostUserMsg msg = {
1319         .hdr.request = request,
1320         .hdr.flags = VHOST_USER_VERSION,
1321         .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1322         .hdr.size = sizeof(msg.payload.u64),
1323     };
1324 
1325     if (file->fd > 0) {
1326         fds[fd_num++] = file->fd;
1327     } else {
1328         msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
1329     }
1330 
1331     return vhost_user_write(dev, &msg, fds, fd_num);
1332 }
1333 
vhost_user_set_vring_kick(struct vhost_dev * dev,struct vhost_vring_file * file)1334 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1335                                      struct vhost_vring_file *file)
1336 {
1337     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1338 }
1339 
vhost_user_set_vring_call(struct vhost_dev * dev,struct vhost_vring_file * file)1340 static int vhost_user_set_vring_call(struct vhost_dev *dev,
1341                                      struct vhost_vring_file *file)
1342 {
1343     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1344 }
1345 
vhost_user_set_vring_err(struct vhost_dev * dev,struct vhost_vring_file * file)1346 static int vhost_user_set_vring_err(struct vhost_dev *dev,
1347                                     struct vhost_vring_file *file)
1348 {
1349     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_ERR, file);
1350 }
1351 
vhost_user_set_vring_addr(struct vhost_dev * dev,struct vhost_vring_addr * addr)1352 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1353                                      struct vhost_vring_addr *addr)
1354 {
1355     VhostUserMsg msg = {
1356         .hdr.request = VHOST_USER_SET_VRING_ADDR,
1357         .hdr.flags = VHOST_USER_VERSION,
1358         .payload.addr = *addr,
1359         .hdr.size = sizeof(msg.payload.addr),
1360     };
1361 
1362     /*
1363      * wait for a reply if logging is enabled to make sure
1364      * backend is actually logging changes
1365      */
1366     bool wait_for_reply = addr->flags & (1 << VHOST_VRING_F_LOG);
1367 
1368     return vhost_user_write_sync(dev, &msg, wait_for_reply);
1369 }
1370 
vhost_user_set_u64(struct vhost_dev * dev,int request,uint64_t u64,bool wait_for_reply)1371 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64,
1372                               bool wait_for_reply)
1373 {
1374     VhostUserMsg msg = {
1375         .hdr.request = request,
1376         .hdr.flags = VHOST_USER_VERSION,
1377         .payload.u64 = u64,
1378         .hdr.size = sizeof(msg.payload.u64),
1379     };
1380 
1381     return vhost_user_write_sync(dev, &msg, wait_for_reply);
1382 }
1383 
vhost_user_set_status(struct vhost_dev * dev,uint8_t status)1384 static int vhost_user_set_status(struct vhost_dev *dev, uint8_t status)
1385 {
1386     return vhost_user_set_u64(dev, VHOST_USER_SET_STATUS, status, false);
1387 }
1388 
vhost_user_get_status(struct vhost_dev * dev,uint8_t * status)1389 static int vhost_user_get_status(struct vhost_dev *dev, uint8_t *status)
1390 {
1391     uint64_t value;
1392     int ret;
1393 
1394     ret = vhost_user_get_u64(dev, VHOST_USER_GET_STATUS, &value);
1395     if (ret < 0) {
1396         return ret;
1397     }
1398     *status = value;
1399 
1400     return 0;
1401 }
1402 
vhost_user_add_status(struct vhost_dev * dev,uint8_t status)1403 static int vhost_user_add_status(struct vhost_dev *dev, uint8_t status)
1404 {
1405     uint8_t s;
1406     int ret;
1407 
1408     ret = vhost_user_get_status(dev, &s);
1409     if (ret < 0) {
1410         return ret;
1411     }
1412 
1413     if ((s & status) == status) {
1414         return 0;
1415     }
1416     s |= status;
1417 
1418     return vhost_user_set_status(dev, s);
1419 }
1420 
vhost_user_set_features(struct vhost_dev * dev,uint64_t features)1421 static int vhost_user_set_features(struct vhost_dev *dev,
1422                                    uint64_t features)
1423 {
1424     /*
1425      * wait for a reply if logging is enabled to make sure
1426      * backend is actually logging changes
1427      */
1428     bool log_enabled = features & (0x1ULL << VHOST_F_LOG_ALL);
1429     int ret;
1430 
1431     /*
1432      * We need to include any extra backend only feature bits that
1433      * might be needed by our device. Currently this includes the
1434      * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol
1435      * features.
1436      */
1437     ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES,
1438                               features | dev->backend_features,
1439                               log_enabled);
1440 
1441     if (virtio_has_feature(dev->protocol_features,
1442                            VHOST_USER_PROTOCOL_F_STATUS)) {
1443         if (!ret) {
1444             return vhost_user_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
1445         }
1446     }
1447 
1448     return ret;
1449 }
1450 
vhost_user_set_protocol_features(struct vhost_dev * dev,uint64_t features)1451 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1452                                             uint64_t features)
1453 {
1454     return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features,
1455                               false);
1456 }
1457 
vhost_user_set_owner(struct vhost_dev * dev)1458 static int vhost_user_set_owner(struct vhost_dev *dev)
1459 {
1460     VhostUserMsg msg = {
1461         .hdr.request = VHOST_USER_SET_OWNER,
1462         .hdr.flags = VHOST_USER_VERSION,
1463     };
1464 
1465     return vhost_user_write(dev, &msg, NULL, 0);
1466 }
1467 
vhost_user_get_max_memslots(struct vhost_dev * dev,uint64_t * max_memslots)1468 static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1469                                        uint64_t *max_memslots)
1470 {
1471     uint64_t backend_max_memslots;
1472     int err;
1473 
1474     err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1475                              &backend_max_memslots);
1476     if (err < 0) {
1477         return err;
1478     }
1479 
1480     *max_memslots = backend_max_memslots;
1481 
1482     return 0;
1483 }
1484 
vhost_user_reset_device(struct vhost_dev * dev)1485 static int vhost_user_reset_device(struct vhost_dev *dev)
1486 {
1487     VhostUserMsg msg = {
1488         .hdr.flags = VHOST_USER_VERSION,
1489         .hdr.request = VHOST_USER_RESET_DEVICE,
1490     };
1491 
1492     /*
1493      * Historically, reset was not implemented so only reset devices
1494      * that are expecting it.
1495      */
1496     if (!virtio_has_feature(dev->protocol_features,
1497                             VHOST_USER_PROTOCOL_F_RESET_DEVICE)) {
1498         return -ENOSYS;
1499     }
1500 
1501     return vhost_user_write(dev, &msg, NULL, 0);
1502 }
1503 
vhost_user_backend_handle_config_change(struct vhost_dev * dev)1504 static int vhost_user_backend_handle_config_change(struct vhost_dev *dev)
1505 {
1506     if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1507         return -ENOSYS;
1508     }
1509 
1510     return dev->config_ops->vhost_dev_config_notifier(dev);
1511 }
1512 
1513 /*
1514  * Fetch or create the notifier for a given idx. Newly created
1515  * notifiers are added to the pointer array that tracks them.
1516  */
fetch_or_create_notifier(VhostUserState * u,int idx)1517 static VhostUserHostNotifier *fetch_or_create_notifier(VhostUserState *u,
1518                                                        int idx)
1519 {
1520     VhostUserHostNotifier *n = NULL;
1521     if (idx >= u->notifiers->len) {
1522         g_ptr_array_set_size(u->notifiers, idx + 1);
1523     }
1524 
1525     n = g_ptr_array_index(u->notifiers, idx);
1526     if (!n) {
1527         /*
1528          * In case notification arrive out-of-order,
1529          * make room for current index.
1530          */
1531         g_ptr_array_remove_index(u->notifiers, idx);
1532         n = g_new0(VhostUserHostNotifier, 1);
1533         n->idx = idx;
1534         g_ptr_array_insert(u->notifiers, idx, n);
1535         trace_vhost_user_create_notifier(idx, n);
1536     }
1537 
1538     return n;
1539 }
1540 
vhost_user_backend_handle_vring_host_notifier(struct vhost_dev * dev,VhostUserVringArea * area,int fd)1541 static int vhost_user_backend_handle_vring_host_notifier(struct vhost_dev *dev,
1542                                                        VhostUserVringArea *area,
1543                                                        int fd)
1544 {
1545     int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1546     size_t page_size = qemu_real_host_page_size();
1547     struct vhost_user *u = dev->opaque;
1548     VhostUserState *user = u->user;
1549     VirtIODevice *vdev = dev->vdev;
1550     VhostUserHostNotifier *n;
1551     void *addr;
1552     char *name;
1553 
1554     if (!virtio_has_feature(dev->protocol_features,
1555                             VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
1556         vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
1557         return -EINVAL;
1558     }
1559 
1560     /*
1561      * Fetch notifier and invalidate any old data before setting up
1562      * new mapped address.
1563      */
1564     n = fetch_or_create_notifier(user, queue_idx);
1565     vhost_user_host_notifier_remove(n, vdev);
1566 
1567     if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1568         return 0;
1569     }
1570 
1571     /* Sanity check. */
1572     if (area->size != page_size) {
1573         return -EINVAL;
1574     }
1575 
1576     addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1577                 fd, area->offset);
1578     if (addr == MAP_FAILED) {
1579         return -EFAULT;
1580     }
1581 
1582     name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1583                            user, queue_idx);
1584     if (!n->mr.ram) { /* Don't init again after suspend. */
1585         memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1586                                           page_size, addr);
1587     } else {
1588         n->mr.ram_block->host = addr;
1589     }
1590     g_free(name);
1591 
1592     if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1593         object_unparent(OBJECT(&n->mr));
1594         munmap(addr, page_size);
1595         return -ENXIO;
1596     }
1597 
1598     n->addr = addr;
1599 
1600     return 0;
1601 }
1602 
1603 static int
vhost_user_backend_handle_shared_object_add(struct vhost_dev * dev,VhostUserShared * object)1604 vhost_user_backend_handle_shared_object_add(struct vhost_dev *dev,
1605                                             VhostUserShared *object)
1606 {
1607     QemuUUID uuid;
1608 
1609     memcpy(uuid.data, object->uuid, sizeof(object->uuid));
1610     return virtio_add_vhost_device(&uuid, dev);
1611 }
1612 
1613 static int
vhost_user_backend_handle_shared_object_remove(struct vhost_dev * dev,VhostUserShared * object)1614 vhost_user_backend_handle_shared_object_remove(struct vhost_dev *dev,
1615                                                VhostUserShared *object)
1616 {
1617     QemuUUID uuid;
1618 
1619     memcpy(uuid.data, object->uuid, sizeof(object->uuid));
1620     switch (virtio_object_type(&uuid)) {
1621     case TYPE_VHOST_DEV:
1622     {
1623         struct vhost_dev *owner = virtio_lookup_vhost_device(&uuid);
1624         if (dev != owner) {
1625             /* Not allowed to remove non-owned entries */
1626             return 0;
1627         }
1628         break;
1629     }
1630     default:
1631         /* Not allowed to remove non-owned entries */
1632         return 0;
1633     }
1634 
1635     return virtio_remove_resource(&uuid);
1636 }
1637 
vhost_user_send_resp(QIOChannel * ioc,VhostUserHeader * hdr,VhostUserPayload * payload,Error ** errp)1638 static bool vhost_user_send_resp(QIOChannel *ioc, VhostUserHeader *hdr,
1639                                  VhostUserPayload *payload, Error **errp)
1640 {
1641     struct iovec iov[] = {
1642         { .iov_base = hdr,      .iov_len = VHOST_USER_HDR_SIZE },
1643         { .iov_base = payload,  .iov_len = hdr->size },
1644     };
1645 
1646     hdr->flags &= ~VHOST_USER_NEED_REPLY_MASK;
1647     hdr->flags |= VHOST_USER_REPLY_MASK;
1648 
1649     return !qio_channel_writev_all(ioc, iov, ARRAY_SIZE(iov), errp);
1650 }
1651 
1652 static bool
vhost_user_backend_send_dmabuf_fd(QIOChannel * ioc,VhostUserHeader * hdr,VhostUserPayload * payload,Error ** errp)1653 vhost_user_backend_send_dmabuf_fd(QIOChannel *ioc, VhostUserHeader *hdr,
1654                                   VhostUserPayload *payload, Error **errp)
1655 {
1656     hdr->size = sizeof(payload->u64);
1657     return vhost_user_send_resp(ioc, hdr, payload, errp);
1658 }
1659 
vhost_user_get_shared_object(struct vhost_dev * dev,unsigned char * uuid,int * dmabuf_fd)1660 int vhost_user_get_shared_object(struct vhost_dev *dev, unsigned char *uuid,
1661                                  int *dmabuf_fd)
1662 {
1663     struct vhost_user *u = dev->opaque;
1664     CharBackend *chr = u->user->chr;
1665     int ret;
1666     VhostUserMsg msg = {
1667         .hdr.request = VHOST_USER_GET_SHARED_OBJECT,
1668         .hdr.flags = VHOST_USER_VERSION,
1669     };
1670     memcpy(msg.payload.object.uuid, uuid, sizeof(msg.payload.object.uuid));
1671 
1672     ret = vhost_user_write(dev, &msg, NULL, 0);
1673     if (ret < 0) {
1674         return ret;
1675     }
1676 
1677     ret = vhost_user_read(dev, &msg);
1678     if (ret < 0) {
1679         return ret;
1680     }
1681 
1682     if (msg.hdr.request != VHOST_USER_GET_SHARED_OBJECT) {
1683         error_report("Received unexpected msg type. "
1684                      "Expected %d received %d",
1685                      VHOST_USER_GET_SHARED_OBJECT, msg.hdr.request);
1686         return -EPROTO;
1687     }
1688 
1689     *dmabuf_fd = qemu_chr_fe_get_msgfd(chr);
1690     if (*dmabuf_fd < 0) {
1691         error_report("Failed to get dmabuf fd");
1692         return -EIO;
1693     }
1694 
1695     return 0;
1696 }
1697 
1698 static int
vhost_user_backend_handle_shared_object_lookup(struct vhost_user * u,QIOChannel * ioc,VhostUserHeader * hdr,VhostUserPayload * payload)1699 vhost_user_backend_handle_shared_object_lookup(struct vhost_user *u,
1700                                                QIOChannel *ioc,
1701                                                VhostUserHeader *hdr,
1702                                                VhostUserPayload *payload)
1703 {
1704     QemuUUID uuid;
1705     CharBackend *chr = u->user->chr;
1706     Error *local_err = NULL;
1707     int dmabuf_fd = -1;
1708     int fd_num = 0;
1709 
1710     memcpy(uuid.data, payload->object.uuid, sizeof(payload->object.uuid));
1711 
1712     payload->u64 = 0;
1713     switch (virtio_object_type(&uuid)) {
1714     case TYPE_DMABUF:
1715         dmabuf_fd = virtio_lookup_dmabuf(&uuid);
1716         break;
1717     case TYPE_VHOST_DEV:
1718     {
1719         struct vhost_dev *dev = virtio_lookup_vhost_device(&uuid);
1720         if (dev == NULL) {
1721             payload->u64 = -EINVAL;
1722             break;
1723         }
1724         int ret = vhost_user_get_shared_object(dev, uuid.data, &dmabuf_fd);
1725         if (ret < 0) {
1726             payload->u64 = ret;
1727         }
1728         break;
1729     }
1730     case TYPE_INVALID:
1731         payload->u64 = -EINVAL;
1732         break;
1733     }
1734 
1735     if (dmabuf_fd != -1) {
1736         fd_num++;
1737     }
1738 
1739     if (qemu_chr_fe_set_msgfds(chr, &dmabuf_fd, fd_num) < 0) {
1740         error_report("Failed to set msg fds.");
1741         payload->u64 = -EINVAL;
1742     }
1743 
1744     if (!vhost_user_backend_send_dmabuf_fd(ioc, hdr, payload, &local_err)) {
1745         error_report_err(local_err);
1746         return -EINVAL;
1747     }
1748 
1749     return 0;
1750 }
1751 
close_backend_channel(struct vhost_user * u)1752 static void close_backend_channel(struct vhost_user *u)
1753 {
1754     g_source_destroy(u->backend_src);
1755     g_source_unref(u->backend_src);
1756     u->backend_src = NULL;
1757     object_unref(OBJECT(u->backend_ioc));
1758     u->backend_ioc = NULL;
1759 }
1760 
backend_read(QIOChannel * ioc,GIOCondition condition,gpointer opaque)1761 static gboolean backend_read(QIOChannel *ioc, GIOCondition condition,
1762                            gpointer opaque)
1763 {
1764     struct vhost_dev *dev = opaque;
1765     struct vhost_user *u = dev->opaque;
1766     VhostUserHeader hdr = { 0, };
1767     VhostUserPayload payload = { 0, };
1768     Error *local_err = NULL;
1769     gboolean rc = G_SOURCE_CONTINUE;
1770     int ret = 0;
1771     struct iovec iov;
1772     g_autofree int *fd = NULL;
1773     size_t fdsize = 0;
1774     int i;
1775 
1776     /* Read header */
1777     iov.iov_base = &hdr;
1778     iov.iov_len = VHOST_USER_HDR_SIZE;
1779 
1780     if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
1781         error_report_err(local_err);
1782         goto err;
1783     }
1784 
1785     if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1786         error_report("Failed to read msg header."
1787                 " Size %d exceeds the maximum %zu.", hdr.size,
1788                 VHOST_USER_PAYLOAD_SIZE);
1789         goto err;
1790     }
1791 
1792     /* Read payload */
1793     if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
1794         error_report_err(local_err);
1795         goto err;
1796     }
1797 
1798     switch (hdr.request) {
1799     case VHOST_USER_BACKEND_IOTLB_MSG:
1800         ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1801         break;
1802     case VHOST_USER_BACKEND_CONFIG_CHANGE_MSG:
1803         ret = vhost_user_backend_handle_config_change(dev);
1804         break;
1805     case VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG:
1806         ret = vhost_user_backend_handle_vring_host_notifier(dev, &payload.area,
1807                                                           fd ? fd[0] : -1);
1808         break;
1809     case VHOST_USER_BACKEND_SHARED_OBJECT_ADD:
1810         ret = vhost_user_backend_handle_shared_object_add(dev, &payload.object);
1811         break;
1812     case VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE:
1813         ret = vhost_user_backend_handle_shared_object_remove(dev,
1814                                                              &payload.object);
1815         break;
1816     case VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP:
1817         ret = vhost_user_backend_handle_shared_object_lookup(dev->opaque, ioc,
1818                                                              &hdr, &payload);
1819         break;
1820     default:
1821         error_report("Received unexpected msg type: %d.", hdr.request);
1822         ret = -EINVAL;
1823     }
1824 
1825     /*
1826      * REPLY_ACK feature handling. Other reply types has to be managed
1827      * directly in their request handlers.
1828      */
1829     if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1830         payload.u64 = !!ret;
1831         hdr.size = sizeof(payload.u64);
1832 
1833         if (!vhost_user_send_resp(ioc, &hdr, &payload, &local_err)) {
1834             error_report_err(local_err);
1835             goto err;
1836         }
1837     }
1838 
1839     goto fdcleanup;
1840 
1841 err:
1842     close_backend_channel(u);
1843     rc = G_SOURCE_REMOVE;
1844 
1845 fdcleanup:
1846     if (fd) {
1847         for (i = 0; i < fdsize; i++) {
1848             close(fd[i]);
1849         }
1850     }
1851     return rc;
1852 }
1853 
vhost_setup_backend_channel(struct vhost_dev * dev)1854 static int vhost_setup_backend_channel(struct vhost_dev *dev)
1855 {
1856     VhostUserMsg msg = {
1857         .hdr.request = VHOST_USER_SET_BACKEND_REQ_FD,
1858         .hdr.flags = VHOST_USER_VERSION,
1859     };
1860     struct vhost_user *u = dev->opaque;
1861     int sv[2], ret = 0;
1862     bool reply_supported = virtio_has_feature(dev->protocol_features,
1863                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1864     Error *local_err = NULL;
1865     QIOChannel *ioc;
1866 
1867     if (!virtio_has_feature(dev->protocol_features,
1868                             VHOST_USER_PROTOCOL_F_BACKEND_REQ)) {
1869         return 0;
1870     }
1871 
1872     if (qemu_socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1873         int saved_errno = errno;
1874         error_report("socketpair() failed");
1875         return -saved_errno;
1876     }
1877 
1878     ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err));
1879     if (!ioc) {
1880         error_report_err(local_err);
1881         return -ECONNREFUSED;
1882     }
1883     u->backend_ioc = ioc;
1884     u->backend_src = qio_channel_add_watch_source(u->backend_ioc,
1885                                                 G_IO_IN | G_IO_HUP,
1886                                                 backend_read, dev, NULL, NULL);
1887 
1888     if (reply_supported) {
1889         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1890     }
1891 
1892     ret = vhost_user_write(dev, &msg, &sv[1], 1);
1893     if (ret) {
1894         goto out;
1895     }
1896 
1897     if (reply_supported) {
1898         ret = process_message_reply(dev, &msg);
1899     }
1900 
1901 out:
1902     close(sv[1]);
1903     if (ret) {
1904         close_backend_channel(u);
1905     }
1906 
1907     return ret;
1908 }
1909 
1910 #ifdef CONFIG_LINUX
1911 /*
1912  * Called back from the postcopy fault thread when a fault is received on our
1913  * ufd.
1914  * TODO: This is Linux specific
1915  */
vhost_user_postcopy_fault_handler(struct PostCopyFD * pcfd,void * ufd)1916 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1917                                              void *ufd)
1918 {
1919     struct vhost_dev *dev = pcfd->data;
1920     struct vhost_user *u = dev->opaque;
1921     struct uffd_msg *msg = ufd;
1922     uint64_t faultaddr = msg->arg.pagefault.address;
1923     RAMBlock *rb = NULL;
1924     uint64_t rb_offset;
1925     int i;
1926 
1927     trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1928                                             dev->mem->nregions);
1929     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1930         trace_vhost_user_postcopy_fault_handler_loop(i,
1931                 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1932         if (faultaddr >= u->postcopy_client_bases[i]) {
1933             /* Ofset of the fault address in the vhost region */
1934             uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1935             if (region_offset < dev->mem->regions[i].memory_size) {
1936                 rb_offset = region_offset + u->region_rb_offset[i];
1937                 trace_vhost_user_postcopy_fault_handler_found(i,
1938                         region_offset, rb_offset);
1939                 rb = u->region_rb[i];
1940                 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1941                                                     rb_offset);
1942             }
1943         }
1944     }
1945     error_report("%s: Failed to find region for fault %" PRIx64,
1946                  __func__, faultaddr);
1947     return -1;
1948 }
1949 
vhost_user_postcopy_waker(struct PostCopyFD * pcfd,RAMBlock * rb,uint64_t offset)1950 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1951                                      uint64_t offset)
1952 {
1953     struct vhost_dev *dev = pcfd->data;
1954     struct vhost_user *u = dev->opaque;
1955     int i;
1956 
1957     trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1958 
1959     if (!u) {
1960         return 0;
1961     }
1962     /* Translate the offset into an address in the clients address space */
1963     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1964         if (u->region_rb[i] == rb &&
1965             offset >= u->region_rb_offset[i] &&
1966             offset < (u->region_rb_offset[i] +
1967                       dev->mem->regions[i].memory_size)) {
1968             uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1969                                    u->postcopy_client_bases[i];
1970             trace_vhost_user_postcopy_waker_found(client_addr);
1971             return postcopy_wake_shared(pcfd, client_addr, rb);
1972         }
1973     }
1974 
1975     trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1976     return 0;
1977 }
1978 #endif
1979 
1980 /*
1981  * Called at the start of an inbound postcopy on reception of the
1982  * 'advise' command.
1983  */
vhost_user_postcopy_advise(struct vhost_dev * dev,Error ** errp)1984 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1985 {
1986 #ifdef CONFIG_LINUX
1987     struct vhost_user *u = dev->opaque;
1988     CharBackend *chr = u->user->chr;
1989     int ufd;
1990     int ret;
1991     VhostUserMsg msg = {
1992         .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1993         .hdr.flags = VHOST_USER_VERSION,
1994     };
1995 
1996     ret = vhost_user_write(dev, &msg, NULL, 0);
1997     if (ret < 0) {
1998         error_setg(errp, "Failed to send postcopy_advise to vhost");
1999         return ret;
2000     }
2001 
2002     ret = vhost_user_read(dev, &msg);
2003     if (ret < 0) {
2004         error_setg(errp, "Failed to get postcopy_advise reply from vhost");
2005         return ret;
2006     }
2007 
2008     if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
2009         error_setg(errp, "Unexpected msg type. Expected %d received %d",
2010                      VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
2011         return -EPROTO;
2012     }
2013 
2014     if (msg.hdr.size) {
2015         error_setg(errp, "Received bad msg size.");
2016         return -EPROTO;
2017     }
2018     ufd = qemu_chr_fe_get_msgfd(chr);
2019     if (ufd < 0) {
2020         error_setg(errp, "%s: Failed to get ufd", __func__);
2021         return -EIO;
2022     }
2023     qemu_socket_set_nonblock(ufd);
2024 
2025     /* register ufd with userfault thread */
2026     u->postcopy_fd.fd = ufd;
2027     u->postcopy_fd.data = dev;
2028     u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
2029     u->postcopy_fd.waker = vhost_user_postcopy_waker;
2030     u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
2031     postcopy_register_shared_ufd(&u->postcopy_fd);
2032     return 0;
2033 #else
2034     error_setg(errp, "Postcopy not supported on non-Linux systems");
2035     return -ENOSYS;
2036 #endif
2037 }
2038 
2039 /*
2040  * Called at the switch to postcopy on reception of the 'listen' command.
2041  */
vhost_user_postcopy_listen(struct vhost_dev * dev,Error ** errp)2042 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
2043 {
2044     struct vhost_user *u = dev->opaque;
2045     int ret;
2046     VhostUserMsg msg = {
2047         .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
2048         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2049     };
2050     u->postcopy_listen = true;
2051 
2052     trace_vhost_user_postcopy_listen();
2053 
2054     ret = vhost_user_write(dev, &msg, NULL, 0);
2055     if (ret < 0) {
2056         error_setg(errp, "Failed to send postcopy_listen to vhost");
2057         return ret;
2058     }
2059 
2060     ret = process_message_reply(dev, &msg);
2061     if (ret) {
2062         error_setg(errp, "Failed to receive reply to postcopy_listen");
2063         return ret;
2064     }
2065 
2066     return 0;
2067 }
2068 
2069 /*
2070  * Called at the end of postcopy
2071  */
vhost_user_postcopy_end(struct vhost_dev * dev,Error ** errp)2072 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
2073 {
2074     VhostUserMsg msg = {
2075         .hdr.request = VHOST_USER_POSTCOPY_END,
2076         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2077     };
2078     int ret;
2079     struct vhost_user *u = dev->opaque;
2080 
2081     trace_vhost_user_postcopy_end_entry();
2082 
2083     ret = vhost_user_write(dev, &msg, NULL, 0);
2084     if (ret < 0) {
2085         error_setg(errp, "Failed to send postcopy_end to vhost");
2086         return ret;
2087     }
2088 
2089     ret = process_message_reply(dev, &msg);
2090     if (ret) {
2091         error_setg(errp, "Failed to receive reply to postcopy_end");
2092         return ret;
2093     }
2094     postcopy_unregister_shared_ufd(&u->postcopy_fd);
2095     close(u->postcopy_fd.fd);
2096     u->postcopy_fd.handler = NULL;
2097 
2098     trace_vhost_user_postcopy_end_exit();
2099 
2100     return 0;
2101 }
2102 
vhost_user_postcopy_notifier(NotifierWithReturn * notifier,void * opaque,Error ** errp)2103 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
2104                                         void *opaque, Error **errp)
2105 {
2106     struct PostcopyNotifyData *pnd = opaque;
2107     struct vhost_user *u = container_of(notifier, struct vhost_user,
2108                                          postcopy_notifier);
2109     struct vhost_dev *dev = u->dev;
2110 
2111     switch (pnd->reason) {
2112     case POSTCOPY_NOTIFY_PROBE:
2113         if (!virtio_has_feature(dev->protocol_features,
2114                                 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
2115             /* TODO: Get the device name into this error somehow */
2116             error_setg(errp,
2117                        "vhost-user backend not capable of postcopy");
2118             return -ENOENT;
2119         }
2120         break;
2121 
2122     case POSTCOPY_NOTIFY_INBOUND_ADVISE:
2123         return vhost_user_postcopy_advise(dev, errp);
2124 
2125     case POSTCOPY_NOTIFY_INBOUND_LISTEN:
2126         return vhost_user_postcopy_listen(dev, errp);
2127 
2128     case POSTCOPY_NOTIFY_INBOUND_END:
2129         return vhost_user_postcopy_end(dev, errp);
2130 
2131     default:
2132         /* We ignore notifications we don't know */
2133         break;
2134     }
2135 
2136     return 0;
2137 }
2138 
vhost_user_backend_init(struct vhost_dev * dev,void * opaque,Error ** errp)2139 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque,
2140                                    Error **errp)
2141 {
2142     uint64_t features, ram_slots;
2143     struct vhost_user *u;
2144     VhostUserState *vus = (VhostUserState *) opaque;
2145     int err;
2146 
2147     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2148 
2149     u = g_new0(struct vhost_user, 1);
2150     u->user = vus;
2151     u->dev = dev;
2152     dev->opaque = u;
2153 
2154     err = vhost_user_get_features(dev, &features);
2155     if (err < 0) {
2156         error_setg_errno(errp, -err, "vhost_backend_init failed");
2157         return err;
2158     }
2159 
2160     if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
2161         bool supports_f_config = vus->supports_config ||
2162             (dev->config_ops && dev->config_ops->vhost_dev_config_notifier);
2163         uint64_t protocol_features;
2164 
2165         dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
2166 
2167         err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
2168                                  &protocol_features);
2169         if (err < 0) {
2170             error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2171             return -EPROTO;
2172         }
2173 
2174         /*
2175          * We will use all the protocol features we support - although
2176          * we suppress F_CONFIG if we know QEMUs internal code can not support
2177          * it.
2178          */
2179         protocol_features &= VHOST_USER_PROTOCOL_FEATURE_MASK;
2180 
2181         if (supports_f_config) {
2182             if (!virtio_has_feature(protocol_features,
2183                                     VHOST_USER_PROTOCOL_F_CONFIG)) {
2184                 error_setg(errp, "vhost-user device expecting "
2185                            "VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
2186                            "not support it.");
2187                 return -EPROTO;
2188             }
2189         } else {
2190             if (virtio_has_feature(protocol_features,
2191                                    VHOST_USER_PROTOCOL_F_CONFIG)) {
2192                 warn_report("vhost-user backend supports "
2193                             "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
2194                 protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
2195             }
2196         }
2197 
2198         /* final set of protocol features */
2199         dev->protocol_features = protocol_features;
2200         err = vhost_user_set_protocol_features(dev, dev->protocol_features);
2201         if (err < 0) {
2202             error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2203             return -EPROTO;
2204         }
2205 
2206         /* query the max queues we support if backend supports Multiple Queue */
2207         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
2208             err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
2209                                      &dev->max_queues);
2210             if (err < 0) {
2211                 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2212                 return -EPROTO;
2213             }
2214         } else {
2215             dev->max_queues = 1;
2216         }
2217 
2218         if (dev->num_queues && dev->max_queues < dev->num_queues) {
2219             error_setg(errp, "The maximum number of queues supported by the "
2220                        "backend is %" PRIu64, dev->max_queues);
2221             return -EINVAL;
2222         }
2223 
2224         if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
2225                 !(virtio_has_feature(dev->protocol_features,
2226                     VHOST_USER_PROTOCOL_F_BACKEND_REQ) &&
2227                  virtio_has_feature(dev->protocol_features,
2228                     VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
2229             error_setg(errp, "IOMMU support requires reply-ack and "
2230                        "backend-req protocol features.");
2231             return -EINVAL;
2232         }
2233 
2234         /* get max memory regions if backend supports configurable RAM slots */
2235         if (!virtio_has_feature(dev->protocol_features,
2236                                 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
2237             u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
2238         } else {
2239             err = vhost_user_get_max_memslots(dev, &ram_slots);
2240             if (err < 0) {
2241                 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2242                 return -EPROTO;
2243             }
2244 
2245             if (ram_slots < u->user->memory_slots) {
2246                 error_setg(errp, "The backend specified a max ram slots limit "
2247                            "of %" PRIu64", when the prior validated limit was "
2248                            "%d. This limit should never decrease.", ram_slots,
2249                            u->user->memory_slots);
2250                 return -EINVAL;
2251             }
2252 
2253             u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
2254         }
2255     }
2256 
2257     if (dev->migration_blocker == NULL &&
2258         !virtio_has_feature(dev->protocol_features,
2259                             VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
2260         error_setg(&dev->migration_blocker,
2261                    "Migration disabled: vhost-user backend lacks "
2262                    "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
2263     }
2264 
2265     if (dev->vq_index == 0) {
2266         err = vhost_setup_backend_channel(dev);
2267         if (err < 0) {
2268             error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2269             return -EPROTO;
2270         }
2271     }
2272 
2273     u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
2274     postcopy_add_notifier(&u->postcopy_notifier);
2275 
2276     return 0;
2277 }
2278 
vhost_user_backend_cleanup(struct vhost_dev * dev)2279 static int vhost_user_backend_cleanup(struct vhost_dev *dev)
2280 {
2281     struct vhost_user *u;
2282 
2283     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2284 
2285     u = dev->opaque;
2286     if (u->postcopy_notifier.notify) {
2287         postcopy_remove_notifier(&u->postcopy_notifier);
2288         u->postcopy_notifier.notify = NULL;
2289     }
2290     u->postcopy_listen = false;
2291     if (u->postcopy_fd.handler) {
2292         postcopy_unregister_shared_ufd(&u->postcopy_fd);
2293         close(u->postcopy_fd.fd);
2294         u->postcopy_fd.handler = NULL;
2295     }
2296     if (u->backend_ioc) {
2297         close_backend_channel(u);
2298     }
2299     g_free(u->region_rb);
2300     u->region_rb = NULL;
2301     g_free(u->region_rb_offset);
2302     u->region_rb_offset = NULL;
2303     u->region_rb_len = 0;
2304     g_free(u);
2305     dev->opaque = 0;
2306 
2307     return 0;
2308 }
2309 
vhost_user_get_vq_index(struct vhost_dev * dev,int idx)2310 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
2311 {
2312     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
2313 
2314     return idx;
2315 }
2316 
vhost_user_memslots_limit(struct vhost_dev * dev)2317 static int vhost_user_memslots_limit(struct vhost_dev *dev)
2318 {
2319     struct vhost_user *u = dev->opaque;
2320 
2321     return u->user->memory_slots;
2322 }
2323 
vhost_user_requires_shm_log(struct vhost_dev * dev)2324 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
2325 {
2326     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2327 
2328     return virtio_has_feature(dev->protocol_features,
2329                               VHOST_USER_PROTOCOL_F_LOG_SHMFD);
2330 }
2331 
vhost_user_migration_done(struct vhost_dev * dev,char * mac_addr)2332 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
2333 {
2334     VhostUserMsg msg = { };
2335 
2336     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2337 
2338     /* If guest supports GUEST_ANNOUNCE do nothing */
2339     if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
2340         return 0;
2341     }
2342 
2343     /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2344     if (virtio_has_feature(dev->protocol_features,
2345                            VHOST_USER_PROTOCOL_F_RARP)) {
2346         msg.hdr.request = VHOST_USER_SEND_RARP;
2347         msg.hdr.flags = VHOST_USER_VERSION;
2348         memcpy((char *)&msg.payload.u64, mac_addr, 6);
2349         msg.hdr.size = sizeof(msg.payload.u64);
2350 
2351         return vhost_user_write(dev, &msg, NULL, 0);
2352     }
2353     return -ENOTSUP;
2354 }
2355 
vhost_user_net_set_mtu(struct vhost_dev * dev,uint16_t mtu)2356 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
2357 {
2358     VhostUserMsg msg;
2359     bool reply_supported = virtio_has_feature(dev->protocol_features,
2360                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
2361     int ret;
2362 
2363     if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
2364         return 0;
2365     }
2366 
2367     msg.hdr.request = VHOST_USER_NET_SET_MTU;
2368     msg.payload.u64 = mtu;
2369     msg.hdr.size = sizeof(msg.payload.u64);
2370     msg.hdr.flags = VHOST_USER_VERSION;
2371     if (reply_supported) {
2372         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2373     }
2374 
2375     ret = vhost_user_write(dev, &msg, NULL, 0);
2376     if (ret < 0) {
2377         return ret;
2378     }
2379 
2380     /* If reply_ack supported, backend has to ack specified MTU is valid */
2381     if (reply_supported) {
2382         return process_message_reply(dev, &msg);
2383     }
2384 
2385     return 0;
2386 }
2387 
vhost_user_send_device_iotlb_msg(struct vhost_dev * dev,struct vhost_iotlb_msg * imsg)2388 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2389                                             struct vhost_iotlb_msg *imsg)
2390 {
2391     int ret;
2392     VhostUserMsg msg = {
2393         .hdr.request = VHOST_USER_IOTLB_MSG,
2394         .hdr.size = sizeof(msg.payload.iotlb),
2395         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2396         .payload.iotlb = *imsg,
2397     };
2398 
2399     ret = vhost_user_write(dev, &msg, NULL, 0);
2400     if (ret < 0) {
2401         return ret;
2402     }
2403 
2404     return process_message_reply(dev, &msg);
2405 }
2406 
2407 
vhost_user_set_iotlb_callback(struct vhost_dev * dev,int enabled)2408 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2409 {
2410     /* No-op as the receive channel is not dedicated to IOTLB messages. */
2411 }
2412 
vhost_user_get_config(struct vhost_dev * dev,uint8_t * config,uint32_t config_len,Error ** errp)2413 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
2414                                  uint32_t config_len, Error **errp)
2415 {
2416     int ret;
2417     VhostUserMsg msg = {
2418         .hdr.request = VHOST_USER_GET_CONFIG,
2419         .hdr.flags = VHOST_USER_VERSION,
2420         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2421     };
2422 
2423     if (!virtio_has_feature(dev->protocol_features,
2424                 VHOST_USER_PROTOCOL_F_CONFIG)) {
2425         error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2426         return -EINVAL;
2427     }
2428 
2429     assert(config_len <= VHOST_USER_MAX_CONFIG_SIZE);
2430 
2431     msg.payload.config.offset = 0;
2432     msg.payload.config.size = config_len;
2433     ret = vhost_user_write(dev, &msg, NULL, 0);
2434     if (ret < 0) {
2435         error_setg_errno(errp, -ret, "vhost_get_config failed");
2436         return ret;
2437     }
2438 
2439     ret = vhost_user_read(dev, &msg);
2440     if (ret < 0) {
2441         error_setg_errno(errp, -ret, "vhost_get_config failed");
2442         return ret;
2443     }
2444 
2445     if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2446         error_setg(errp,
2447                    "Received unexpected msg type. Expected %d received %d",
2448                    VHOST_USER_GET_CONFIG, msg.hdr.request);
2449         return -EPROTO;
2450     }
2451 
2452     if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2453         error_setg(errp, "Received bad msg size.");
2454         return -EPROTO;
2455     }
2456 
2457     memcpy(config, msg.payload.config.region, config_len);
2458 
2459     return 0;
2460 }
2461 
vhost_user_set_config(struct vhost_dev * dev,const uint8_t * data,uint32_t offset,uint32_t size,uint32_t flags)2462 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
2463                                  uint32_t offset, uint32_t size, uint32_t flags)
2464 {
2465     int ret;
2466     uint8_t *p;
2467     bool reply_supported = virtio_has_feature(dev->protocol_features,
2468                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
2469 
2470     VhostUserMsg msg = {
2471         .hdr.request = VHOST_USER_SET_CONFIG,
2472         .hdr.flags = VHOST_USER_VERSION,
2473         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2474     };
2475 
2476     if (!virtio_has_feature(dev->protocol_features,
2477                 VHOST_USER_PROTOCOL_F_CONFIG)) {
2478         return -ENOTSUP;
2479     }
2480 
2481     if (reply_supported) {
2482         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2483     }
2484 
2485     if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2486         return -EINVAL;
2487     }
2488 
2489     msg.payload.config.offset = offset,
2490     msg.payload.config.size = size,
2491     msg.payload.config.flags = flags,
2492     p = msg.payload.config.region;
2493     memcpy(p, data, size);
2494 
2495     ret = vhost_user_write(dev, &msg, NULL, 0);
2496     if (ret < 0) {
2497         return ret;
2498     }
2499 
2500     if (reply_supported) {
2501         return process_message_reply(dev, &msg);
2502     }
2503 
2504     return 0;
2505 }
2506 
vhost_user_crypto_create_session(struct vhost_dev * dev,void * session_info,uint64_t * session_id)2507 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2508                                             void *session_info,
2509                                             uint64_t *session_id)
2510 {
2511     int ret;
2512     bool crypto_session = virtio_has_feature(dev->protocol_features,
2513                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2514     CryptoDevBackendSessionInfo *backend_info = session_info;
2515     VhostUserMsg msg = {
2516         .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2517         .hdr.flags = VHOST_USER_VERSION,
2518         .hdr.size = sizeof(msg.payload.session),
2519     };
2520 
2521     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2522 
2523     if (!crypto_session) {
2524         error_report("vhost-user trying to send unhandled ioctl");
2525         return -ENOTSUP;
2526     }
2527 
2528     if (backend_info->op_code == VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION) {
2529         CryptoDevBackendAsymSessionInfo *sess = &backend_info->u.asym_sess_info;
2530         size_t keylen;
2531 
2532         memcpy(&msg.payload.session.u.asym.session_setup_data, sess,
2533                sizeof(CryptoDevBackendAsymSessionInfo));
2534         if (sess->keylen) {
2535             keylen = sizeof(msg.payload.session.u.asym.key);
2536             if (sess->keylen > keylen) {
2537                 error_report("Unsupported asymmetric key size");
2538                 return -ENOTSUP;
2539             }
2540 
2541             memcpy(&msg.payload.session.u.asym.key, sess->key,
2542                    sess->keylen);
2543         }
2544     } else {
2545         CryptoDevBackendSymSessionInfo *sess = &backend_info->u.sym_sess_info;
2546         size_t keylen;
2547 
2548         memcpy(&msg.payload.session.u.sym.session_setup_data, sess,
2549                sizeof(CryptoDevBackendSymSessionInfo));
2550         if (sess->key_len) {
2551             keylen = sizeof(msg.payload.session.u.sym.key);
2552             if (sess->key_len > keylen) {
2553                 error_report("Unsupported cipher key size");
2554                 return -ENOTSUP;
2555             }
2556 
2557             memcpy(&msg.payload.session.u.sym.key, sess->cipher_key,
2558                    sess->key_len);
2559         }
2560 
2561         if (sess->auth_key_len > 0) {
2562             keylen = sizeof(msg.payload.session.u.sym.auth_key);
2563             if (sess->auth_key_len > keylen) {
2564                 error_report("Unsupported auth key size");
2565                 return -ENOTSUP;
2566             }
2567 
2568             memcpy(&msg.payload.session.u.sym.auth_key, sess->auth_key,
2569                    sess->auth_key_len);
2570         }
2571     }
2572 
2573     msg.payload.session.op_code = backend_info->op_code;
2574     msg.payload.session.session_id = backend_info->session_id;
2575     ret = vhost_user_write(dev, &msg, NULL, 0);
2576     if (ret < 0) {
2577         error_report("vhost_user_write() return %d, create session failed",
2578                      ret);
2579         return ret;
2580     }
2581 
2582     ret = vhost_user_read(dev, &msg);
2583     if (ret < 0) {
2584         error_report("vhost_user_read() return %d, create session failed",
2585                      ret);
2586         return ret;
2587     }
2588 
2589     if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2590         error_report("Received unexpected msg type. Expected %d received %d",
2591                      VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2592         return -EPROTO;
2593     }
2594 
2595     if (msg.hdr.size != sizeof(msg.payload.session)) {
2596         error_report("Received bad msg size.");
2597         return -EPROTO;
2598     }
2599 
2600     if (msg.payload.session.session_id < 0) {
2601         error_report("Bad session id: %" PRId64 "",
2602                               msg.payload.session.session_id);
2603         return -EINVAL;
2604     }
2605     *session_id = msg.payload.session.session_id;
2606 
2607     return 0;
2608 }
2609 
2610 static int
vhost_user_crypto_close_session(struct vhost_dev * dev,uint64_t session_id)2611 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2612 {
2613     int ret;
2614     bool crypto_session = virtio_has_feature(dev->protocol_features,
2615                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2616     VhostUserMsg msg = {
2617         .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
2618         .hdr.flags = VHOST_USER_VERSION,
2619         .hdr.size = sizeof(msg.payload.u64),
2620     };
2621     msg.payload.u64 = session_id;
2622 
2623     if (!crypto_session) {
2624         error_report("vhost-user trying to send unhandled ioctl");
2625         return -ENOTSUP;
2626     }
2627 
2628     ret = vhost_user_write(dev, &msg, NULL, 0);
2629     if (ret < 0) {
2630         error_report("vhost_user_write() return %d, close session failed",
2631                      ret);
2632         return ret;
2633     }
2634 
2635     return 0;
2636 }
2637 
vhost_user_no_private_memslots(struct vhost_dev * dev)2638 static bool vhost_user_no_private_memslots(struct vhost_dev *dev)
2639 {
2640     return true;
2641 }
2642 
vhost_user_get_inflight_fd(struct vhost_dev * dev,uint16_t queue_size,struct vhost_inflight * inflight)2643 static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
2644                                       uint16_t queue_size,
2645                                       struct vhost_inflight *inflight)
2646 {
2647     void *addr;
2648     int fd;
2649     int ret;
2650     struct vhost_user *u = dev->opaque;
2651     CharBackend *chr = u->user->chr;
2652     VhostUserMsg msg = {
2653         .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
2654         .hdr.flags = VHOST_USER_VERSION,
2655         .payload.inflight.num_queues = dev->nvqs,
2656         .payload.inflight.queue_size = queue_size,
2657         .hdr.size = sizeof(msg.payload.inflight),
2658     };
2659 
2660     if (!virtio_has_feature(dev->protocol_features,
2661                             VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2662         return 0;
2663     }
2664 
2665     ret = vhost_user_write(dev, &msg, NULL, 0);
2666     if (ret < 0) {
2667         return ret;
2668     }
2669 
2670     ret = vhost_user_read(dev, &msg);
2671     if (ret < 0) {
2672         return ret;
2673     }
2674 
2675     if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
2676         error_report("Received unexpected msg type. "
2677                      "Expected %d received %d",
2678                      VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
2679         return -EPROTO;
2680     }
2681 
2682     if (msg.hdr.size != sizeof(msg.payload.inflight)) {
2683         error_report("Received bad msg size.");
2684         return -EPROTO;
2685     }
2686 
2687     if (!msg.payload.inflight.mmap_size) {
2688         return 0;
2689     }
2690 
2691     fd = qemu_chr_fe_get_msgfd(chr);
2692     if (fd < 0) {
2693         error_report("Failed to get mem fd");
2694         return -EIO;
2695     }
2696 
2697     addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
2698                 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
2699 
2700     if (addr == MAP_FAILED) {
2701         error_report("Failed to mmap mem fd");
2702         close(fd);
2703         return -EFAULT;
2704     }
2705 
2706     inflight->addr = addr;
2707     inflight->fd = fd;
2708     inflight->size = msg.payload.inflight.mmap_size;
2709     inflight->offset = msg.payload.inflight.mmap_offset;
2710     inflight->queue_size = queue_size;
2711 
2712     return 0;
2713 }
2714 
vhost_user_set_inflight_fd(struct vhost_dev * dev,struct vhost_inflight * inflight)2715 static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
2716                                       struct vhost_inflight *inflight)
2717 {
2718     VhostUserMsg msg = {
2719         .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
2720         .hdr.flags = VHOST_USER_VERSION,
2721         .payload.inflight.mmap_size = inflight->size,
2722         .payload.inflight.mmap_offset = inflight->offset,
2723         .payload.inflight.num_queues = dev->nvqs,
2724         .payload.inflight.queue_size = inflight->queue_size,
2725         .hdr.size = sizeof(msg.payload.inflight),
2726     };
2727 
2728     if (!virtio_has_feature(dev->protocol_features,
2729                             VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2730         return 0;
2731     }
2732 
2733     return vhost_user_write(dev, &msg, &inflight->fd, 1);
2734 }
2735 
vhost_user_state_destroy(gpointer data)2736 static void vhost_user_state_destroy(gpointer data)
2737 {
2738     VhostUserHostNotifier *n = (VhostUserHostNotifier *) data;
2739     if (n) {
2740         vhost_user_host_notifier_remove(n, NULL);
2741         object_unparent(OBJECT(&n->mr));
2742         /*
2743          * We can't free until vhost_user_host_notifier_remove has
2744          * done it's thing so schedule the free with RCU.
2745          */
2746         g_free_rcu(n, rcu);
2747     }
2748 }
2749 
vhost_user_init(VhostUserState * user,CharBackend * chr,Error ** errp)2750 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
2751 {
2752     if (user->chr) {
2753         error_setg(errp, "Cannot initialize vhost-user state");
2754         return false;
2755     }
2756     user->chr = chr;
2757     user->memory_slots = 0;
2758     user->notifiers = g_ptr_array_new_full(VIRTIO_QUEUE_MAX / 4,
2759                                            &vhost_user_state_destroy);
2760     return true;
2761 }
2762 
vhost_user_cleanup(VhostUserState * user)2763 void vhost_user_cleanup(VhostUserState *user)
2764 {
2765     if (!user->chr) {
2766         return;
2767     }
2768     memory_region_transaction_begin();
2769     user->notifiers = (GPtrArray *) g_ptr_array_free(user->notifiers, true);
2770     memory_region_transaction_commit();
2771     user->chr = NULL;
2772 }
2773 
2774 
2775 typedef struct {
2776     vu_async_close_fn cb;
2777     DeviceState *dev;
2778     CharBackend *cd;
2779     struct vhost_dev *vhost;
2780 } VhostAsyncCallback;
2781 
vhost_user_async_close_bh(void * opaque)2782 static void vhost_user_async_close_bh(void *opaque)
2783 {
2784     VhostAsyncCallback *data = opaque;
2785 
2786     data->cb(data->dev);
2787 
2788     g_free(data);
2789 }
2790 
2791 /*
2792  * We only schedule the work if the machine is running. If suspended
2793  * we want to keep all the in-flight data as is for migration
2794  * purposes.
2795  */
vhost_user_async_close(DeviceState * d,CharBackend * chardev,struct vhost_dev * vhost,vu_async_close_fn cb)2796 void vhost_user_async_close(DeviceState *d,
2797                             CharBackend *chardev, struct vhost_dev *vhost,
2798                             vu_async_close_fn cb)
2799 {
2800     if (!runstate_check(RUN_STATE_SHUTDOWN)) {
2801         /*
2802          * A close event may happen during a read/write, but vhost
2803          * code assumes the vhost_dev remains setup, so delay the
2804          * stop & clear.
2805          */
2806         AioContext *ctx = qemu_get_current_aio_context();
2807         VhostAsyncCallback *data = g_new0(VhostAsyncCallback, 1);
2808 
2809         /* Save data for the callback */
2810         data->cb = cb;
2811         data->dev = d;
2812         data->cd = chardev;
2813         data->vhost = vhost;
2814 
2815         /* Disable any further notifications on the chardev */
2816         qemu_chr_fe_set_handlers(chardev,
2817                                  NULL, NULL, NULL, NULL, NULL, NULL,
2818                                  false);
2819 
2820         aio_bh_schedule_oneshot(ctx, vhost_user_async_close_bh, data);
2821 
2822         /*
2823          * Move vhost device to the stopped state. The vhost-user device
2824          * will be clean up and disconnected in BH. This can be useful in
2825          * the vhost migration code. If disconnect was caught there is an
2826          * option for the general vhost code to get the dev state without
2827          * knowing its type (in this case vhost-user).
2828          *
2829          * Note if the vhost device is fully cleared by the time we
2830          * execute the bottom half we won't continue with the cleanup.
2831          */
2832         vhost->started = false;
2833     }
2834 }
2835 
vhost_user_dev_start(struct vhost_dev * dev,bool started)2836 static int vhost_user_dev_start(struct vhost_dev *dev, bool started)
2837 {
2838     if (!virtio_has_feature(dev->protocol_features,
2839                             VHOST_USER_PROTOCOL_F_STATUS)) {
2840         return 0;
2841     }
2842 
2843     /* Set device status only for last queue pair */
2844     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
2845         return 0;
2846     }
2847 
2848     if (started) {
2849         return vhost_user_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
2850                                           VIRTIO_CONFIG_S_DRIVER |
2851                                           VIRTIO_CONFIG_S_DRIVER_OK);
2852     } else {
2853         return 0;
2854     }
2855 }
2856 
vhost_user_reset_status(struct vhost_dev * dev)2857 static void vhost_user_reset_status(struct vhost_dev *dev)
2858 {
2859     /* Set device status only for last queue pair */
2860     if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
2861         return;
2862     }
2863 
2864     if (virtio_has_feature(dev->protocol_features,
2865                            VHOST_USER_PROTOCOL_F_STATUS)) {
2866         vhost_user_set_status(dev, 0);
2867     }
2868 }
2869 
vhost_user_supports_device_state(struct vhost_dev * dev)2870 static bool vhost_user_supports_device_state(struct vhost_dev *dev)
2871 {
2872     return virtio_has_feature(dev->protocol_features,
2873                               VHOST_USER_PROTOCOL_F_DEVICE_STATE);
2874 }
2875 
vhost_user_set_device_state_fd(struct vhost_dev * dev,VhostDeviceStateDirection direction,VhostDeviceStatePhase phase,int fd,int * reply_fd,Error ** errp)2876 static int vhost_user_set_device_state_fd(struct vhost_dev *dev,
2877                                           VhostDeviceStateDirection direction,
2878                                           VhostDeviceStatePhase phase,
2879                                           int fd,
2880                                           int *reply_fd,
2881                                           Error **errp)
2882 {
2883     int ret;
2884     struct vhost_user *vu = dev->opaque;
2885     VhostUserMsg msg = {
2886         .hdr = {
2887             .request = VHOST_USER_SET_DEVICE_STATE_FD,
2888             .flags = VHOST_USER_VERSION,
2889             .size = sizeof(msg.payload.transfer_state),
2890         },
2891         .payload.transfer_state = {
2892             .direction = direction,
2893             .phase = phase,
2894         },
2895     };
2896 
2897     *reply_fd = -1;
2898 
2899     if (!vhost_user_supports_device_state(dev)) {
2900         close(fd);
2901         error_setg(errp, "Back-end does not support migration state transfer");
2902         return -ENOTSUP;
2903     }
2904 
2905     ret = vhost_user_write(dev, &msg, &fd, 1);
2906     close(fd);
2907     if (ret < 0) {
2908         error_setg_errno(errp, -ret,
2909                          "Failed to send SET_DEVICE_STATE_FD message");
2910         return ret;
2911     }
2912 
2913     ret = vhost_user_read(dev, &msg);
2914     if (ret < 0) {
2915         error_setg_errno(errp, -ret,
2916                          "Failed to receive SET_DEVICE_STATE_FD reply");
2917         return ret;
2918     }
2919 
2920     if (msg.hdr.request != VHOST_USER_SET_DEVICE_STATE_FD) {
2921         error_setg(errp,
2922                    "Received unexpected message type, expected %d, received %d",
2923                    VHOST_USER_SET_DEVICE_STATE_FD, msg.hdr.request);
2924         return -EPROTO;
2925     }
2926 
2927     if (msg.hdr.size != sizeof(msg.payload.u64)) {
2928         error_setg(errp,
2929                    "Received bad message size, expected %zu, received %" PRIu32,
2930                    sizeof(msg.payload.u64), msg.hdr.size);
2931         return -EPROTO;
2932     }
2933 
2934     if ((msg.payload.u64 & 0xff) != 0) {
2935         error_setg(errp, "Back-end did not accept migration state transfer");
2936         return -EIO;
2937     }
2938 
2939     if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK)) {
2940         *reply_fd = qemu_chr_fe_get_msgfd(vu->user->chr);
2941         if (*reply_fd < 0) {
2942             error_setg(errp,
2943                        "Failed to get back-end-provided transfer pipe FD");
2944             *reply_fd = -1;
2945             return -EIO;
2946         }
2947     }
2948 
2949     return 0;
2950 }
2951 
vhost_user_check_device_state(struct vhost_dev * dev,Error ** errp)2952 static int vhost_user_check_device_state(struct vhost_dev *dev, Error **errp)
2953 {
2954     int ret;
2955     VhostUserMsg msg = {
2956         .hdr = {
2957             .request = VHOST_USER_CHECK_DEVICE_STATE,
2958             .flags = VHOST_USER_VERSION,
2959             .size = 0,
2960         },
2961     };
2962 
2963     if (!vhost_user_supports_device_state(dev)) {
2964         error_setg(errp, "Back-end does not support migration state transfer");
2965         return -ENOTSUP;
2966     }
2967 
2968     ret = vhost_user_write(dev, &msg, NULL, 0);
2969     if (ret < 0) {
2970         error_setg_errno(errp, -ret,
2971                          "Failed to send CHECK_DEVICE_STATE message");
2972         return ret;
2973     }
2974 
2975     ret = vhost_user_read(dev, &msg);
2976     if (ret < 0) {
2977         error_setg_errno(errp, -ret,
2978                          "Failed to receive CHECK_DEVICE_STATE reply");
2979         return ret;
2980     }
2981 
2982     if (msg.hdr.request != VHOST_USER_CHECK_DEVICE_STATE) {
2983         error_setg(errp,
2984                    "Received unexpected message type, expected %d, received %d",
2985                    VHOST_USER_CHECK_DEVICE_STATE, msg.hdr.request);
2986         return -EPROTO;
2987     }
2988 
2989     if (msg.hdr.size != sizeof(msg.payload.u64)) {
2990         error_setg(errp,
2991                    "Received bad message size, expected %zu, received %" PRIu32,
2992                    sizeof(msg.payload.u64), msg.hdr.size);
2993         return -EPROTO;
2994     }
2995 
2996     if (msg.payload.u64 != 0) {
2997         error_setg(errp, "Back-end failed to process its internal state");
2998         return -EIO;
2999     }
3000 
3001     return 0;
3002 }
3003 
3004 const VhostOps user_ops = {
3005         .backend_type = VHOST_BACKEND_TYPE_USER,
3006         .vhost_backend_init = vhost_user_backend_init,
3007         .vhost_backend_cleanup = vhost_user_backend_cleanup,
3008         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
3009         .vhost_backend_no_private_memslots = vhost_user_no_private_memslots,
3010         .vhost_set_log_base = vhost_user_set_log_base,
3011         .vhost_set_mem_table = vhost_user_set_mem_table,
3012         .vhost_set_vring_addr = vhost_user_set_vring_addr,
3013         .vhost_set_vring_endian = vhost_user_set_vring_endian,
3014         .vhost_set_vring_num = vhost_user_set_vring_num,
3015         .vhost_set_vring_base = vhost_user_set_vring_base,
3016         .vhost_get_vring_base = vhost_user_get_vring_base,
3017         .vhost_set_vring_kick = vhost_user_set_vring_kick,
3018         .vhost_set_vring_call = vhost_user_set_vring_call,
3019         .vhost_set_vring_err = vhost_user_set_vring_err,
3020         .vhost_set_features = vhost_user_set_features,
3021         .vhost_get_features = vhost_user_get_features,
3022         .vhost_set_owner = vhost_user_set_owner,
3023         .vhost_reset_device = vhost_user_reset_device,
3024         .vhost_get_vq_index = vhost_user_get_vq_index,
3025         .vhost_set_vring_enable = vhost_user_set_vring_enable,
3026         .vhost_requires_shm_log = vhost_user_requires_shm_log,
3027         .vhost_migration_done = vhost_user_migration_done,
3028         .vhost_net_set_mtu = vhost_user_net_set_mtu,
3029         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
3030         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
3031         .vhost_get_config = vhost_user_get_config,
3032         .vhost_set_config = vhost_user_set_config,
3033         .vhost_crypto_create_session = vhost_user_crypto_create_session,
3034         .vhost_crypto_close_session = vhost_user_crypto_close_session,
3035         .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
3036         .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
3037         .vhost_dev_start = vhost_user_dev_start,
3038         .vhost_reset_status = vhost_user_reset_status,
3039         .vhost_supports_device_state = vhost_user_supports_device_state,
3040         .vhost_set_device_state_fd = vhost_user_set_device_state_fd,
3041         .vhost_check_device_state = vhost_user_check_device_state,
3042 };
3043