1 /* 2 * vhost-user 3 * 4 * Copyright (c) 2013 Virtual Open Systems Sarl. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 * 9 */ 10 11 #include "qemu/osdep.h" 12 #include "qapi/error.h" 13 #include "hw/virtio/vhost.h" 14 #include "hw/virtio/vhost-user.h" 15 #include "hw/virtio/vhost-backend.h" 16 #include "hw/virtio/virtio.h" 17 #include "hw/virtio/virtio-net.h" 18 #include "chardev/char-fe.h" 19 #include "sysemu/kvm.h" 20 #include "qemu/error-report.h" 21 #include "qemu/sockets.h" 22 #include "sysemu/cryptodev.h" 23 #include "migration/migration.h" 24 #include "migration/postcopy-ram.h" 25 #include "trace.h" 26 27 #include <sys/ioctl.h> 28 #include <sys/socket.h> 29 #include <sys/un.h> 30 31 #include "standard-headers/linux/vhost_types.h" 32 33 #ifdef CONFIG_LINUX 34 #include <linux/userfaultfd.h> 35 #endif 36 37 #define VHOST_MEMORY_MAX_NREGIONS 8 38 #define VHOST_USER_F_PROTOCOL_FEATURES 30 39 #define VHOST_USER_SLAVE_MAX_FDS 8 40 41 /* 42 * Maximum size of virtio device config space 43 */ 44 #define VHOST_USER_MAX_CONFIG_SIZE 256 45 46 enum VhostUserProtocolFeature { 47 VHOST_USER_PROTOCOL_F_MQ = 0, 48 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 49 VHOST_USER_PROTOCOL_F_RARP = 2, 50 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 51 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 52 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 53 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 54 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 55 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 56 VHOST_USER_PROTOCOL_F_CONFIG = 9, 57 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10, 58 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11, 59 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12, 60 VHOST_USER_PROTOCOL_F_MAX 61 }; 62 63 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1) 64 65 typedef enum VhostUserRequest { 66 VHOST_USER_NONE = 0, 67 VHOST_USER_GET_FEATURES = 1, 68 VHOST_USER_SET_FEATURES = 2, 69 VHOST_USER_SET_OWNER = 3, 70 VHOST_USER_RESET_OWNER = 4, 71 VHOST_USER_SET_MEM_TABLE = 5, 72 VHOST_USER_SET_LOG_BASE = 6, 73 VHOST_USER_SET_LOG_FD = 7, 74 VHOST_USER_SET_VRING_NUM = 8, 75 VHOST_USER_SET_VRING_ADDR = 9, 76 VHOST_USER_SET_VRING_BASE = 10, 77 VHOST_USER_GET_VRING_BASE = 11, 78 VHOST_USER_SET_VRING_KICK = 12, 79 VHOST_USER_SET_VRING_CALL = 13, 80 VHOST_USER_SET_VRING_ERR = 14, 81 VHOST_USER_GET_PROTOCOL_FEATURES = 15, 82 VHOST_USER_SET_PROTOCOL_FEATURES = 16, 83 VHOST_USER_GET_QUEUE_NUM = 17, 84 VHOST_USER_SET_VRING_ENABLE = 18, 85 VHOST_USER_SEND_RARP = 19, 86 VHOST_USER_NET_SET_MTU = 20, 87 VHOST_USER_SET_SLAVE_REQ_FD = 21, 88 VHOST_USER_IOTLB_MSG = 22, 89 VHOST_USER_SET_VRING_ENDIAN = 23, 90 VHOST_USER_GET_CONFIG = 24, 91 VHOST_USER_SET_CONFIG = 25, 92 VHOST_USER_CREATE_CRYPTO_SESSION = 26, 93 VHOST_USER_CLOSE_CRYPTO_SESSION = 27, 94 VHOST_USER_POSTCOPY_ADVISE = 28, 95 VHOST_USER_POSTCOPY_LISTEN = 29, 96 VHOST_USER_POSTCOPY_END = 30, 97 VHOST_USER_GET_INFLIGHT_FD = 31, 98 VHOST_USER_SET_INFLIGHT_FD = 32, 99 VHOST_USER_GPU_SET_SOCKET = 33, 100 VHOST_USER_MAX 101 } VhostUserRequest; 102 103 typedef enum VhostUserSlaveRequest { 104 VHOST_USER_SLAVE_NONE = 0, 105 VHOST_USER_SLAVE_IOTLB_MSG = 1, 106 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, 107 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3, 108 VHOST_USER_SLAVE_MAX 109 } VhostUserSlaveRequest; 110 111 typedef struct VhostUserMemoryRegion { 112 uint64_t guest_phys_addr; 113 uint64_t memory_size; 114 uint64_t userspace_addr; 115 uint64_t mmap_offset; 116 } VhostUserMemoryRegion; 117 118 typedef struct VhostUserMemory { 119 uint32_t nregions; 120 uint32_t padding; 121 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS]; 122 } VhostUserMemory; 123 124 typedef struct VhostUserLog { 125 uint64_t mmap_size; 126 uint64_t mmap_offset; 127 } VhostUserLog; 128 129 typedef struct VhostUserConfig { 130 uint32_t offset; 131 uint32_t size; 132 uint32_t flags; 133 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE]; 134 } VhostUserConfig; 135 136 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512 137 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64 138 139 typedef struct VhostUserCryptoSession { 140 /* session id for success, -1 on errors */ 141 int64_t session_id; 142 CryptoDevBackendSymSessionInfo session_setup_data; 143 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN]; 144 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN]; 145 } VhostUserCryptoSession; 146 147 static VhostUserConfig c __attribute__ ((unused)); 148 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \ 149 + sizeof(c.size) \ 150 + sizeof(c.flags)) 151 152 typedef struct VhostUserVringArea { 153 uint64_t u64; 154 uint64_t size; 155 uint64_t offset; 156 } VhostUserVringArea; 157 158 typedef struct VhostUserInflight { 159 uint64_t mmap_size; 160 uint64_t mmap_offset; 161 uint16_t num_queues; 162 uint16_t queue_size; 163 } VhostUserInflight; 164 165 typedef struct { 166 VhostUserRequest request; 167 168 #define VHOST_USER_VERSION_MASK (0x3) 169 #define VHOST_USER_REPLY_MASK (0x1<<2) 170 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3) 171 uint32_t flags; 172 uint32_t size; /* the following payload size */ 173 } QEMU_PACKED VhostUserHeader; 174 175 typedef union { 176 #define VHOST_USER_VRING_IDX_MASK (0xff) 177 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8) 178 uint64_t u64; 179 struct vhost_vring_state state; 180 struct vhost_vring_addr addr; 181 VhostUserMemory memory; 182 VhostUserLog log; 183 struct vhost_iotlb_msg iotlb; 184 VhostUserConfig config; 185 VhostUserCryptoSession session; 186 VhostUserVringArea area; 187 VhostUserInflight inflight; 188 } VhostUserPayload; 189 190 typedef struct VhostUserMsg { 191 VhostUserHeader hdr; 192 VhostUserPayload payload; 193 } QEMU_PACKED VhostUserMsg; 194 195 static VhostUserMsg m __attribute__ ((unused)); 196 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader)) 197 198 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload)) 199 200 /* The version of the protocol we support */ 201 #define VHOST_USER_VERSION (0x1) 202 203 struct vhost_user { 204 struct vhost_dev *dev; 205 /* Shared between vhost devs of the same virtio device */ 206 VhostUserState *user; 207 int slave_fd; 208 NotifierWithReturn postcopy_notifier; 209 struct PostCopyFD postcopy_fd; 210 uint64_t postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS]; 211 /* Length of the region_rb and region_rb_offset arrays */ 212 size_t region_rb_len; 213 /* RAMBlock associated with a given region */ 214 RAMBlock **region_rb; 215 /* The offset from the start of the RAMBlock to the start of the 216 * vhost region. 217 */ 218 ram_addr_t *region_rb_offset; 219 220 /* True once we've entered postcopy_listen */ 221 bool postcopy_listen; 222 }; 223 224 static bool ioeventfd_enabled(void) 225 { 226 return !kvm_enabled() || kvm_eventfds_enabled(); 227 } 228 229 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg) 230 { 231 struct vhost_user *u = dev->opaque; 232 CharBackend *chr = u->user->chr; 233 uint8_t *p = (uint8_t *) msg; 234 int r, size = VHOST_USER_HDR_SIZE; 235 236 r = qemu_chr_fe_read_all(chr, p, size); 237 if (r != size) { 238 error_report("Failed to read msg header. Read %d instead of %d." 239 " Original request %d.", r, size, msg->hdr.request); 240 return -1; 241 } 242 243 /* validate received flags */ 244 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) { 245 error_report("Failed to read msg header." 246 " Flags 0x%x instead of 0x%x.", msg->hdr.flags, 247 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION); 248 return -1; 249 } 250 251 return 0; 252 } 253 254 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg) 255 { 256 struct vhost_user *u = dev->opaque; 257 CharBackend *chr = u->user->chr; 258 uint8_t *p = (uint8_t *) msg; 259 int r, size; 260 261 if (vhost_user_read_header(dev, msg) < 0) { 262 return -1; 263 } 264 265 /* validate message size is sane */ 266 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) { 267 error_report("Failed to read msg header." 268 " Size %d exceeds the maximum %zu.", msg->hdr.size, 269 VHOST_USER_PAYLOAD_SIZE); 270 return -1; 271 } 272 273 if (msg->hdr.size) { 274 p += VHOST_USER_HDR_SIZE; 275 size = msg->hdr.size; 276 r = qemu_chr_fe_read_all(chr, p, size); 277 if (r != size) { 278 error_report("Failed to read msg payload." 279 " Read %d instead of %d.", r, msg->hdr.size); 280 return -1; 281 } 282 } 283 284 return 0; 285 } 286 287 static int process_message_reply(struct vhost_dev *dev, 288 const VhostUserMsg *msg) 289 { 290 VhostUserMsg msg_reply; 291 292 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) { 293 return 0; 294 } 295 296 if (vhost_user_read(dev, &msg_reply) < 0) { 297 return -1; 298 } 299 300 if (msg_reply.hdr.request != msg->hdr.request) { 301 error_report("Received unexpected msg type." 302 "Expected %d received %d", 303 msg->hdr.request, msg_reply.hdr.request); 304 return -1; 305 } 306 307 return msg_reply.payload.u64 ? -1 : 0; 308 } 309 310 static bool vhost_user_one_time_request(VhostUserRequest request) 311 { 312 switch (request) { 313 case VHOST_USER_SET_OWNER: 314 case VHOST_USER_RESET_OWNER: 315 case VHOST_USER_SET_MEM_TABLE: 316 case VHOST_USER_GET_QUEUE_NUM: 317 case VHOST_USER_NET_SET_MTU: 318 return true; 319 default: 320 return false; 321 } 322 } 323 324 /* most non-init callers ignore the error */ 325 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, 326 int *fds, int fd_num) 327 { 328 struct vhost_user *u = dev->opaque; 329 CharBackend *chr = u->user->chr; 330 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size; 331 332 /* 333 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE, 334 * we just need send it once in the first time. For later such 335 * request, we just ignore it. 336 */ 337 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) { 338 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 339 return 0; 340 } 341 342 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) { 343 error_report("Failed to set msg fds."); 344 return -1; 345 } 346 347 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size); 348 if (ret != size) { 349 error_report("Failed to write msg." 350 " Wrote %d instead of %d.", ret, size); 351 return -1; 352 } 353 354 return 0; 355 } 356 357 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd) 358 { 359 VhostUserMsg msg = { 360 .hdr.request = VHOST_USER_GPU_SET_SOCKET, 361 .hdr.flags = VHOST_USER_VERSION, 362 }; 363 364 return vhost_user_write(dev, &msg, &fd, 1); 365 } 366 367 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base, 368 struct vhost_log *log) 369 { 370 int fds[VHOST_MEMORY_MAX_NREGIONS]; 371 size_t fd_num = 0; 372 bool shmfd = virtio_has_feature(dev->protocol_features, 373 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 374 VhostUserMsg msg = { 375 .hdr.request = VHOST_USER_SET_LOG_BASE, 376 .hdr.flags = VHOST_USER_VERSION, 377 .payload.log.mmap_size = log->size * sizeof(*(log->log)), 378 .payload.log.mmap_offset = 0, 379 .hdr.size = sizeof(msg.payload.log), 380 }; 381 382 if (shmfd && log->fd != -1) { 383 fds[fd_num++] = log->fd; 384 } 385 386 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 387 return -1; 388 } 389 390 if (shmfd) { 391 msg.hdr.size = 0; 392 if (vhost_user_read(dev, &msg) < 0) { 393 return -1; 394 } 395 396 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) { 397 error_report("Received unexpected msg type. " 398 "Expected %d received %d", 399 VHOST_USER_SET_LOG_BASE, msg.hdr.request); 400 return -1; 401 } 402 } 403 404 return 0; 405 } 406 407 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, 408 struct vhost_memory *mem) 409 { 410 struct vhost_user *u = dev->opaque; 411 int fds[VHOST_MEMORY_MAX_NREGIONS]; 412 int i, fd; 413 size_t fd_num = 0; 414 VhostUserMsg msg_reply; 415 int region_i, msg_i; 416 417 VhostUserMsg msg = { 418 .hdr.request = VHOST_USER_SET_MEM_TABLE, 419 .hdr.flags = VHOST_USER_VERSION, 420 }; 421 422 if (u->region_rb_len < dev->mem->nregions) { 423 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions); 424 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset, 425 dev->mem->nregions); 426 memset(&(u->region_rb[u->region_rb_len]), '\0', 427 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len)); 428 memset(&(u->region_rb_offset[u->region_rb_len]), '\0', 429 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len)); 430 u->region_rb_len = dev->mem->nregions; 431 } 432 433 for (i = 0; i < dev->mem->nregions; ++i) { 434 struct vhost_memory_region *reg = dev->mem->regions + i; 435 ram_addr_t offset; 436 MemoryRegion *mr; 437 438 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 439 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 440 &offset); 441 fd = memory_region_get_fd(mr); 442 if (fd > 0) { 443 trace_vhost_user_set_mem_table_withfd(fd_num, mr->name, 444 reg->memory_size, 445 reg->guest_phys_addr, 446 reg->userspace_addr, offset); 447 u->region_rb_offset[i] = offset; 448 u->region_rb[i] = mr->ram_block; 449 msg.payload.memory.regions[fd_num].userspace_addr = 450 reg->userspace_addr; 451 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 452 msg.payload.memory.regions[fd_num].guest_phys_addr = 453 reg->guest_phys_addr; 454 msg.payload.memory.regions[fd_num].mmap_offset = offset; 455 assert(fd_num < VHOST_MEMORY_MAX_NREGIONS); 456 fds[fd_num++] = fd; 457 } else { 458 u->region_rb_offset[i] = 0; 459 u->region_rb[i] = NULL; 460 } 461 } 462 463 msg.payload.memory.nregions = fd_num; 464 465 if (!fd_num) { 466 error_report("Failed initializing vhost-user memory map, " 467 "consider using -object memory-backend-file share=on"); 468 return -1; 469 } 470 471 msg.hdr.size = sizeof(msg.payload.memory.nregions); 472 msg.hdr.size += sizeof(msg.payload.memory.padding); 473 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 474 475 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 476 return -1; 477 } 478 479 if (vhost_user_read(dev, &msg_reply) < 0) { 480 return -1; 481 } 482 483 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) { 484 error_report("%s: Received unexpected msg type." 485 "Expected %d received %d", __func__, 486 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request); 487 return -1; 488 } 489 /* We're using the same structure, just reusing one of the 490 * fields, so it should be the same size. 491 */ 492 if (msg_reply.hdr.size != msg.hdr.size) { 493 error_report("%s: Unexpected size for postcopy reply " 494 "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size); 495 return -1; 496 } 497 498 memset(u->postcopy_client_bases, 0, 499 sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS); 500 501 /* They're in the same order as the regions that were sent 502 * but some of the regions were skipped (above) if they 503 * didn't have fd's 504 */ 505 for (msg_i = 0, region_i = 0; 506 region_i < dev->mem->nregions; 507 region_i++) { 508 if (msg_i < fd_num && 509 msg_reply.payload.memory.regions[msg_i].guest_phys_addr == 510 dev->mem->regions[region_i].guest_phys_addr) { 511 u->postcopy_client_bases[region_i] = 512 msg_reply.payload.memory.regions[msg_i].userspace_addr; 513 trace_vhost_user_set_mem_table_postcopy( 514 msg_reply.payload.memory.regions[msg_i].userspace_addr, 515 msg.payload.memory.regions[msg_i].userspace_addr, 516 msg_i, region_i); 517 msg_i++; 518 } 519 } 520 if (msg_i != fd_num) { 521 error_report("%s: postcopy reply not fully consumed " 522 "%d vs %zd", 523 __func__, msg_i, fd_num); 524 return -1; 525 } 526 /* Now we've registered this with the postcopy code, we ack to the client, 527 * because now we're in the position to be able to deal with any faults 528 * it generates. 529 */ 530 /* TODO: Use this for failure cases as well with a bad value */ 531 msg.hdr.size = sizeof(msg.payload.u64); 532 msg.payload.u64 = 0; /* OK */ 533 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 534 return -1; 535 } 536 537 return 0; 538 } 539 540 static int vhost_user_set_mem_table(struct vhost_dev *dev, 541 struct vhost_memory *mem) 542 { 543 struct vhost_user *u = dev->opaque; 544 int fds[VHOST_MEMORY_MAX_NREGIONS]; 545 int i, fd; 546 size_t fd_num = 0; 547 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler; 548 bool reply_supported = virtio_has_feature(dev->protocol_features, 549 VHOST_USER_PROTOCOL_F_REPLY_ACK); 550 551 if (do_postcopy) { 552 /* Postcopy has enough differences that it's best done in it's own 553 * version 554 */ 555 return vhost_user_set_mem_table_postcopy(dev, mem); 556 } 557 558 VhostUserMsg msg = { 559 .hdr.request = VHOST_USER_SET_MEM_TABLE, 560 .hdr.flags = VHOST_USER_VERSION, 561 }; 562 563 if (reply_supported) { 564 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 565 } 566 567 for (i = 0; i < dev->mem->nregions; ++i) { 568 struct vhost_memory_region *reg = dev->mem->regions + i; 569 ram_addr_t offset; 570 MemoryRegion *mr; 571 572 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 573 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 574 &offset); 575 fd = memory_region_get_fd(mr); 576 if (fd > 0) { 577 if (fd_num == VHOST_MEMORY_MAX_NREGIONS) { 578 error_report("Failed preparing vhost-user memory table msg"); 579 return -1; 580 } 581 msg.payload.memory.regions[fd_num].userspace_addr = 582 reg->userspace_addr; 583 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 584 msg.payload.memory.regions[fd_num].guest_phys_addr = 585 reg->guest_phys_addr; 586 msg.payload.memory.regions[fd_num].mmap_offset = offset; 587 fds[fd_num++] = fd; 588 } 589 } 590 591 msg.payload.memory.nregions = fd_num; 592 593 if (!fd_num) { 594 error_report("Failed initializing vhost-user memory map, " 595 "consider using -object memory-backend-file share=on"); 596 return -1; 597 } 598 599 msg.hdr.size = sizeof(msg.payload.memory.nregions); 600 msg.hdr.size += sizeof(msg.payload.memory.padding); 601 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 602 603 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 604 return -1; 605 } 606 607 if (reply_supported) { 608 return process_message_reply(dev, &msg); 609 } 610 611 return 0; 612 } 613 614 static int vhost_user_set_vring_addr(struct vhost_dev *dev, 615 struct vhost_vring_addr *addr) 616 { 617 VhostUserMsg msg = { 618 .hdr.request = VHOST_USER_SET_VRING_ADDR, 619 .hdr.flags = VHOST_USER_VERSION, 620 .payload.addr = *addr, 621 .hdr.size = sizeof(msg.payload.addr), 622 }; 623 624 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 625 return -1; 626 } 627 628 return 0; 629 } 630 631 static int vhost_user_set_vring_endian(struct vhost_dev *dev, 632 struct vhost_vring_state *ring) 633 { 634 bool cross_endian = virtio_has_feature(dev->protocol_features, 635 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN); 636 VhostUserMsg msg = { 637 .hdr.request = VHOST_USER_SET_VRING_ENDIAN, 638 .hdr.flags = VHOST_USER_VERSION, 639 .payload.state = *ring, 640 .hdr.size = sizeof(msg.payload.state), 641 }; 642 643 if (!cross_endian) { 644 error_report("vhost-user trying to send unhandled ioctl"); 645 return -1; 646 } 647 648 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 649 return -1; 650 } 651 652 return 0; 653 } 654 655 static int vhost_set_vring(struct vhost_dev *dev, 656 unsigned long int request, 657 struct vhost_vring_state *ring) 658 { 659 VhostUserMsg msg = { 660 .hdr.request = request, 661 .hdr.flags = VHOST_USER_VERSION, 662 .payload.state = *ring, 663 .hdr.size = sizeof(msg.payload.state), 664 }; 665 666 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 667 return -1; 668 } 669 670 return 0; 671 } 672 673 static int vhost_user_set_vring_num(struct vhost_dev *dev, 674 struct vhost_vring_state *ring) 675 { 676 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring); 677 } 678 679 static void vhost_user_host_notifier_restore(struct vhost_dev *dev, 680 int queue_idx) 681 { 682 struct vhost_user *u = dev->opaque; 683 VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; 684 VirtIODevice *vdev = dev->vdev; 685 686 if (n->addr && !n->set) { 687 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true); 688 n->set = true; 689 } 690 } 691 692 static void vhost_user_host_notifier_remove(struct vhost_dev *dev, 693 int queue_idx) 694 { 695 struct vhost_user *u = dev->opaque; 696 VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; 697 VirtIODevice *vdev = dev->vdev; 698 699 if (n->addr && n->set) { 700 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); 701 n->set = false; 702 } 703 } 704 705 static int vhost_user_set_vring_base(struct vhost_dev *dev, 706 struct vhost_vring_state *ring) 707 { 708 vhost_user_host_notifier_restore(dev, ring->index); 709 710 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring); 711 } 712 713 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable) 714 { 715 int i; 716 717 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) { 718 return -1; 719 } 720 721 for (i = 0; i < dev->nvqs; ++i) { 722 struct vhost_vring_state state = { 723 .index = dev->vq_index + i, 724 .num = enable, 725 }; 726 727 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state); 728 } 729 730 return 0; 731 } 732 733 static int vhost_user_get_vring_base(struct vhost_dev *dev, 734 struct vhost_vring_state *ring) 735 { 736 VhostUserMsg msg = { 737 .hdr.request = VHOST_USER_GET_VRING_BASE, 738 .hdr.flags = VHOST_USER_VERSION, 739 .payload.state = *ring, 740 .hdr.size = sizeof(msg.payload.state), 741 }; 742 743 vhost_user_host_notifier_remove(dev, ring->index); 744 745 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 746 return -1; 747 } 748 749 if (vhost_user_read(dev, &msg) < 0) { 750 return -1; 751 } 752 753 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) { 754 error_report("Received unexpected msg type. Expected %d received %d", 755 VHOST_USER_GET_VRING_BASE, msg.hdr.request); 756 return -1; 757 } 758 759 if (msg.hdr.size != sizeof(msg.payload.state)) { 760 error_report("Received bad msg size."); 761 return -1; 762 } 763 764 *ring = msg.payload.state; 765 766 return 0; 767 } 768 769 static int vhost_set_vring_file(struct vhost_dev *dev, 770 VhostUserRequest request, 771 struct vhost_vring_file *file) 772 { 773 int fds[VHOST_MEMORY_MAX_NREGIONS]; 774 size_t fd_num = 0; 775 VhostUserMsg msg = { 776 .hdr.request = request, 777 .hdr.flags = VHOST_USER_VERSION, 778 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK, 779 .hdr.size = sizeof(msg.payload.u64), 780 }; 781 782 if (ioeventfd_enabled() && file->fd > 0) { 783 fds[fd_num++] = file->fd; 784 } else { 785 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK; 786 } 787 788 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 789 return -1; 790 } 791 792 return 0; 793 } 794 795 static int vhost_user_set_vring_kick(struct vhost_dev *dev, 796 struct vhost_vring_file *file) 797 { 798 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file); 799 } 800 801 static int vhost_user_set_vring_call(struct vhost_dev *dev, 802 struct vhost_vring_file *file) 803 { 804 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file); 805 } 806 807 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) 808 { 809 VhostUserMsg msg = { 810 .hdr.request = request, 811 .hdr.flags = VHOST_USER_VERSION, 812 .payload.u64 = u64, 813 .hdr.size = sizeof(msg.payload.u64), 814 }; 815 816 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 817 return -1; 818 } 819 820 return 0; 821 } 822 823 static int vhost_user_set_features(struct vhost_dev *dev, 824 uint64_t features) 825 { 826 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features); 827 } 828 829 static int vhost_user_set_protocol_features(struct vhost_dev *dev, 830 uint64_t features) 831 { 832 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features); 833 } 834 835 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64) 836 { 837 VhostUserMsg msg = { 838 .hdr.request = request, 839 .hdr.flags = VHOST_USER_VERSION, 840 }; 841 842 if (vhost_user_one_time_request(request) && dev->vq_index != 0) { 843 return 0; 844 } 845 846 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 847 return -1; 848 } 849 850 if (vhost_user_read(dev, &msg) < 0) { 851 return -1; 852 } 853 854 if (msg.hdr.request != request) { 855 error_report("Received unexpected msg type. Expected %d received %d", 856 request, msg.hdr.request); 857 return -1; 858 } 859 860 if (msg.hdr.size != sizeof(msg.payload.u64)) { 861 error_report("Received bad msg size."); 862 return -1; 863 } 864 865 *u64 = msg.payload.u64; 866 867 return 0; 868 } 869 870 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features) 871 { 872 return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features); 873 } 874 875 static int vhost_user_set_owner(struct vhost_dev *dev) 876 { 877 VhostUserMsg msg = { 878 .hdr.request = VHOST_USER_SET_OWNER, 879 .hdr.flags = VHOST_USER_VERSION, 880 }; 881 882 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 883 return -1; 884 } 885 886 return 0; 887 } 888 889 static int vhost_user_reset_device(struct vhost_dev *dev) 890 { 891 VhostUserMsg msg = { 892 .hdr.request = VHOST_USER_RESET_OWNER, 893 .hdr.flags = VHOST_USER_VERSION, 894 }; 895 896 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 897 return -1; 898 } 899 900 return 0; 901 } 902 903 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) 904 { 905 int ret = -1; 906 907 if (!dev->config_ops) { 908 return -1; 909 } 910 911 if (dev->config_ops->vhost_dev_config_notifier) { 912 ret = dev->config_ops->vhost_dev_config_notifier(dev); 913 } 914 915 return ret; 916 } 917 918 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev, 919 VhostUserVringArea *area, 920 int fd) 921 { 922 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK; 923 size_t page_size = qemu_real_host_page_size; 924 struct vhost_user *u = dev->opaque; 925 VhostUserState *user = u->user; 926 VirtIODevice *vdev = dev->vdev; 927 VhostUserHostNotifier *n; 928 void *addr; 929 char *name; 930 931 if (!virtio_has_feature(dev->protocol_features, 932 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) || 933 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) { 934 return -1; 935 } 936 937 n = &user->notifier[queue_idx]; 938 939 if (n->addr) { 940 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false); 941 object_unparent(OBJECT(&n->mr)); 942 munmap(n->addr, page_size); 943 n->addr = NULL; 944 } 945 946 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) { 947 return 0; 948 } 949 950 /* Sanity check. */ 951 if (area->size != page_size) { 952 return -1; 953 } 954 955 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, 956 fd, area->offset); 957 if (addr == MAP_FAILED) { 958 return -1; 959 } 960 961 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]", 962 user, queue_idx); 963 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, 964 page_size, addr); 965 g_free(name); 966 967 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) { 968 munmap(addr, page_size); 969 return -1; 970 } 971 972 n->addr = addr; 973 n->set = true; 974 975 return 0; 976 } 977 978 static void slave_read(void *opaque) 979 { 980 struct vhost_dev *dev = opaque; 981 struct vhost_user *u = dev->opaque; 982 VhostUserHeader hdr = { 0, }; 983 VhostUserPayload payload = { 0, }; 984 int size, ret = 0; 985 struct iovec iov; 986 struct msghdr msgh; 987 int fd[VHOST_USER_SLAVE_MAX_FDS]; 988 char control[CMSG_SPACE(sizeof(fd))]; 989 struct cmsghdr *cmsg; 990 int i, fdsize = 0; 991 992 memset(&msgh, 0, sizeof(msgh)); 993 msgh.msg_iov = &iov; 994 msgh.msg_iovlen = 1; 995 msgh.msg_control = control; 996 msgh.msg_controllen = sizeof(control); 997 998 memset(fd, -1, sizeof(fd)); 999 1000 /* Read header */ 1001 iov.iov_base = &hdr; 1002 iov.iov_len = VHOST_USER_HDR_SIZE; 1003 1004 do { 1005 size = recvmsg(u->slave_fd, &msgh, 0); 1006 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1007 1008 if (size != VHOST_USER_HDR_SIZE) { 1009 error_report("Failed to read from slave."); 1010 goto err; 1011 } 1012 1013 if (msgh.msg_flags & MSG_CTRUNC) { 1014 error_report("Truncated message."); 1015 goto err; 1016 } 1017 1018 for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL; 1019 cmsg = CMSG_NXTHDR(&msgh, cmsg)) { 1020 if (cmsg->cmsg_level == SOL_SOCKET && 1021 cmsg->cmsg_type == SCM_RIGHTS) { 1022 fdsize = cmsg->cmsg_len - CMSG_LEN(0); 1023 memcpy(fd, CMSG_DATA(cmsg), fdsize); 1024 break; 1025 } 1026 } 1027 1028 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) { 1029 error_report("Failed to read msg header." 1030 " Size %d exceeds the maximum %zu.", hdr.size, 1031 VHOST_USER_PAYLOAD_SIZE); 1032 goto err; 1033 } 1034 1035 /* Read payload */ 1036 do { 1037 size = read(u->slave_fd, &payload, hdr.size); 1038 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1039 1040 if (size != hdr.size) { 1041 error_report("Failed to read payload from slave."); 1042 goto err; 1043 } 1044 1045 switch (hdr.request) { 1046 case VHOST_USER_SLAVE_IOTLB_MSG: 1047 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb); 1048 break; 1049 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : 1050 ret = vhost_user_slave_handle_config_change(dev); 1051 break; 1052 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: 1053 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area, 1054 fd[0]); 1055 break; 1056 default: 1057 error_report("Received unexpected msg type."); 1058 ret = -EINVAL; 1059 } 1060 1061 /* Close the remaining file descriptors. */ 1062 for (i = 0; i < fdsize; i++) { 1063 if (fd[i] != -1) { 1064 close(fd[i]); 1065 } 1066 } 1067 1068 /* 1069 * REPLY_ACK feature handling. Other reply types has to be managed 1070 * directly in their request handlers. 1071 */ 1072 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) { 1073 struct iovec iovec[2]; 1074 1075 1076 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 1077 hdr.flags |= VHOST_USER_REPLY_MASK; 1078 1079 payload.u64 = !!ret; 1080 hdr.size = sizeof(payload.u64); 1081 1082 iovec[0].iov_base = &hdr; 1083 iovec[0].iov_len = VHOST_USER_HDR_SIZE; 1084 iovec[1].iov_base = &payload; 1085 iovec[1].iov_len = hdr.size; 1086 1087 do { 1088 size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec)); 1089 } while (size < 0 && (errno == EINTR || errno == EAGAIN)); 1090 1091 if (size != VHOST_USER_HDR_SIZE + hdr.size) { 1092 error_report("Failed to send msg reply to slave."); 1093 goto err; 1094 } 1095 } 1096 1097 return; 1098 1099 err: 1100 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1101 close(u->slave_fd); 1102 u->slave_fd = -1; 1103 for (i = 0; i < fdsize; i++) { 1104 if (fd[i] != -1) { 1105 close(fd[i]); 1106 } 1107 } 1108 return; 1109 } 1110 1111 static int vhost_setup_slave_channel(struct vhost_dev *dev) 1112 { 1113 VhostUserMsg msg = { 1114 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD, 1115 .hdr.flags = VHOST_USER_VERSION, 1116 }; 1117 struct vhost_user *u = dev->opaque; 1118 int sv[2], ret = 0; 1119 bool reply_supported = virtio_has_feature(dev->protocol_features, 1120 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1121 1122 if (!virtio_has_feature(dev->protocol_features, 1123 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) { 1124 return 0; 1125 } 1126 1127 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) { 1128 error_report("socketpair() failed"); 1129 return -1; 1130 } 1131 1132 u->slave_fd = sv[0]; 1133 qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev); 1134 1135 if (reply_supported) { 1136 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1137 } 1138 1139 ret = vhost_user_write(dev, &msg, &sv[1], 1); 1140 if (ret) { 1141 goto out; 1142 } 1143 1144 if (reply_supported) { 1145 ret = process_message_reply(dev, &msg); 1146 } 1147 1148 out: 1149 close(sv[1]); 1150 if (ret) { 1151 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1152 close(u->slave_fd); 1153 u->slave_fd = -1; 1154 } 1155 1156 return ret; 1157 } 1158 1159 #ifdef CONFIG_LINUX 1160 /* 1161 * Called back from the postcopy fault thread when a fault is received on our 1162 * ufd. 1163 * TODO: This is Linux specific 1164 */ 1165 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd, 1166 void *ufd) 1167 { 1168 struct vhost_dev *dev = pcfd->data; 1169 struct vhost_user *u = dev->opaque; 1170 struct uffd_msg *msg = ufd; 1171 uint64_t faultaddr = msg->arg.pagefault.address; 1172 RAMBlock *rb = NULL; 1173 uint64_t rb_offset; 1174 int i; 1175 1176 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr, 1177 dev->mem->nregions); 1178 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 1179 trace_vhost_user_postcopy_fault_handler_loop(i, 1180 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size); 1181 if (faultaddr >= u->postcopy_client_bases[i]) { 1182 /* Ofset of the fault address in the vhost region */ 1183 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i]; 1184 if (region_offset < dev->mem->regions[i].memory_size) { 1185 rb_offset = region_offset + u->region_rb_offset[i]; 1186 trace_vhost_user_postcopy_fault_handler_found(i, 1187 region_offset, rb_offset); 1188 rb = u->region_rb[i]; 1189 return postcopy_request_shared_page(pcfd, rb, faultaddr, 1190 rb_offset); 1191 } 1192 } 1193 } 1194 error_report("%s: Failed to find region for fault %" PRIx64, 1195 __func__, faultaddr); 1196 return -1; 1197 } 1198 1199 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb, 1200 uint64_t offset) 1201 { 1202 struct vhost_dev *dev = pcfd->data; 1203 struct vhost_user *u = dev->opaque; 1204 int i; 1205 1206 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset); 1207 1208 if (!u) { 1209 return 0; 1210 } 1211 /* Translate the offset into an address in the clients address space */ 1212 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 1213 if (u->region_rb[i] == rb && 1214 offset >= u->region_rb_offset[i] && 1215 offset < (u->region_rb_offset[i] + 1216 dev->mem->regions[i].memory_size)) { 1217 uint64_t client_addr = (offset - u->region_rb_offset[i]) + 1218 u->postcopy_client_bases[i]; 1219 trace_vhost_user_postcopy_waker_found(client_addr); 1220 return postcopy_wake_shared(pcfd, client_addr, rb); 1221 } 1222 } 1223 1224 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset); 1225 return 0; 1226 } 1227 #endif 1228 1229 /* 1230 * Called at the start of an inbound postcopy on reception of the 1231 * 'advise' command. 1232 */ 1233 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp) 1234 { 1235 #ifdef CONFIG_LINUX 1236 struct vhost_user *u = dev->opaque; 1237 CharBackend *chr = u->user->chr; 1238 int ufd; 1239 VhostUserMsg msg = { 1240 .hdr.request = VHOST_USER_POSTCOPY_ADVISE, 1241 .hdr.flags = VHOST_USER_VERSION, 1242 }; 1243 1244 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1245 error_setg(errp, "Failed to send postcopy_advise to vhost"); 1246 return -1; 1247 } 1248 1249 if (vhost_user_read(dev, &msg) < 0) { 1250 error_setg(errp, "Failed to get postcopy_advise reply from vhost"); 1251 return -1; 1252 } 1253 1254 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) { 1255 error_setg(errp, "Unexpected msg type. Expected %d received %d", 1256 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request); 1257 return -1; 1258 } 1259 1260 if (msg.hdr.size) { 1261 error_setg(errp, "Received bad msg size."); 1262 return -1; 1263 } 1264 ufd = qemu_chr_fe_get_msgfd(chr); 1265 if (ufd < 0) { 1266 error_setg(errp, "%s: Failed to get ufd", __func__); 1267 return -1; 1268 } 1269 qemu_set_nonblock(ufd); 1270 1271 /* register ufd with userfault thread */ 1272 u->postcopy_fd.fd = ufd; 1273 u->postcopy_fd.data = dev; 1274 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler; 1275 u->postcopy_fd.waker = vhost_user_postcopy_waker; 1276 u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */ 1277 postcopy_register_shared_ufd(&u->postcopy_fd); 1278 return 0; 1279 #else 1280 error_setg(errp, "Postcopy not supported on non-Linux systems"); 1281 return -1; 1282 #endif 1283 } 1284 1285 /* 1286 * Called at the switch to postcopy on reception of the 'listen' command. 1287 */ 1288 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp) 1289 { 1290 struct vhost_user *u = dev->opaque; 1291 int ret; 1292 VhostUserMsg msg = { 1293 .hdr.request = VHOST_USER_POSTCOPY_LISTEN, 1294 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1295 }; 1296 u->postcopy_listen = true; 1297 trace_vhost_user_postcopy_listen(); 1298 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1299 error_setg(errp, "Failed to send postcopy_listen to vhost"); 1300 return -1; 1301 } 1302 1303 ret = process_message_reply(dev, &msg); 1304 if (ret) { 1305 error_setg(errp, "Failed to receive reply to postcopy_listen"); 1306 return ret; 1307 } 1308 1309 return 0; 1310 } 1311 1312 /* 1313 * Called at the end of postcopy 1314 */ 1315 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp) 1316 { 1317 VhostUserMsg msg = { 1318 .hdr.request = VHOST_USER_POSTCOPY_END, 1319 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1320 }; 1321 int ret; 1322 struct vhost_user *u = dev->opaque; 1323 1324 trace_vhost_user_postcopy_end_entry(); 1325 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1326 error_setg(errp, "Failed to send postcopy_end to vhost"); 1327 return -1; 1328 } 1329 1330 ret = process_message_reply(dev, &msg); 1331 if (ret) { 1332 error_setg(errp, "Failed to receive reply to postcopy_end"); 1333 return ret; 1334 } 1335 postcopy_unregister_shared_ufd(&u->postcopy_fd); 1336 close(u->postcopy_fd.fd); 1337 u->postcopy_fd.handler = NULL; 1338 1339 trace_vhost_user_postcopy_end_exit(); 1340 1341 return 0; 1342 } 1343 1344 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier, 1345 void *opaque) 1346 { 1347 struct PostcopyNotifyData *pnd = opaque; 1348 struct vhost_user *u = container_of(notifier, struct vhost_user, 1349 postcopy_notifier); 1350 struct vhost_dev *dev = u->dev; 1351 1352 switch (pnd->reason) { 1353 case POSTCOPY_NOTIFY_PROBE: 1354 if (!virtio_has_feature(dev->protocol_features, 1355 VHOST_USER_PROTOCOL_F_PAGEFAULT)) { 1356 /* TODO: Get the device name into this error somehow */ 1357 error_setg(pnd->errp, 1358 "vhost-user backend not capable of postcopy"); 1359 return -ENOENT; 1360 } 1361 break; 1362 1363 case POSTCOPY_NOTIFY_INBOUND_ADVISE: 1364 return vhost_user_postcopy_advise(dev, pnd->errp); 1365 1366 case POSTCOPY_NOTIFY_INBOUND_LISTEN: 1367 return vhost_user_postcopy_listen(dev, pnd->errp); 1368 1369 case POSTCOPY_NOTIFY_INBOUND_END: 1370 return vhost_user_postcopy_end(dev, pnd->errp); 1371 1372 default: 1373 /* We ignore notifications we don't know */ 1374 break; 1375 } 1376 1377 return 0; 1378 } 1379 1380 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque) 1381 { 1382 uint64_t features, protocol_features; 1383 struct vhost_user *u; 1384 int err; 1385 1386 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1387 1388 u = g_new0(struct vhost_user, 1); 1389 u->user = opaque; 1390 u->slave_fd = -1; 1391 u->dev = dev; 1392 dev->opaque = u; 1393 1394 err = vhost_user_get_features(dev, &features); 1395 if (err < 0) { 1396 return err; 1397 } 1398 1399 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) { 1400 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; 1401 1402 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES, 1403 &protocol_features); 1404 if (err < 0) { 1405 return err; 1406 } 1407 1408 dev->protocol_features = 1409 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK; 1410 1411 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) { 1412 /* Don't acknowledge CONFIG feature if device doesn't support it */ 1413 dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG); 1414 } else if (!(protocol_features & 1415 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) { 1416 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG " 1417 "but backend does not support it."); 1418 return -1; 1419 } 1420 1421 err = vhost_user_set_protocol_features(dev, dev->protocol_features); 1422 if (err < 0) { 1423 return err; 1424 } 1425 1426 /* query the max queues we support if backend supports Multiple Queue */ 1427 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) { 1428 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM, 1429 &dev->max_queues); 1430 if (err < 0) { 1431 return err; 1432 } 1433 } 1434 1435 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) && 1436 !(virtio_has_feature(dev->protocol_features, 1437 VHOST_USER_PROTOCOL_F_SLAVE_REQ) && 1438 virtio_has_feature(dev->protocol_features, 1439 VHOST_USER_PROTOCOL_F_REPLY_ACK))) { 1440 error_report("IOMMU support requires reply-ack and " 1441 "slave-req protocol features."); 1442 return -1; 1443 } 1444 } 1445 1446 if (dev->migration_blocker == NULL && 1447 !virtio_has_feature(dev->protocol_features, 1448 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) { 1449 error_setg(&dev->migration_blocker, 1450 "Migration disabled: vhost-user backend lacks " 1451 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature."); 1452 } 1453 1454 err = vhost_setup_slave_channel(dev); 1455 if (err < 0) { 1456 return err; 1457 } 1458 1459 u->postcopy_notifier.notify = vhost_user_postcopy_notifier; 1460 postcopy_add_notifier(&u->postcopy_notifier); 1461 1462 return 0; 1463 } 1464 1465 static int vhost_user_backend_cleanup(struct vhost_dev *dev) 1466 { 1467 struct vhost_user *u; 1468 1469 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1470 1471 u = dev->opaque; 1472 if (u->postcopy_notifier.notify) { 1473 postcopy_remove_notifier(&u->postcopy_notifier); 1474 u->postcopy_notifier.notify = NULL; 1475 } 1476 u->postcopy_listen = false; 1477 if (u->postcopy_fd.handler) { 1478 postcopy_unregister_shared_ufd(&u->postcopy_fd); 1479 close(u->postcopy_fd.fd); 1480 u->postcopy_fd.handler = NULL; 1481 } 1482 if (u->slave_fd >= 0) { 1483 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1484 close(u->slave_fd); 1485 u->slave_fd = -1; 1486 } 1487 g_free(u->region_rb); 1488 u->region_rb = NULL; 1489 g_free(u->region_rb_offset); 1490 u->region_rb_offset = NULL; 1491 u->region_rb_len = 0; 1492 g_free(u); 1493 dev->opaque = 0; 1494 1495 return 0; 1496 } 1497 1498 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx) 1499 { 1500 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 1501 1502 return idx; 1503 } 1504 1505 static int vhost_user_memslots_limit(struct vhost_dev *dev) 1506 { 1507 return VHOST_MEMORY_MAX_NREGIONS; 1508 } 1509 1510 static bool vhost_user_requires_shm_log(struct vhost_dev *dev) 1511 { 1512 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1513 1514 return virtio_has_feature(dev->protocol_features, 1515 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 1516 } 1517 1518 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) 1519 { 1520 VhostUserMsg msg = { }; 1521 1522 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1523 1524 /* If guest supports GUEST_ANNOUNCE do nothing */ 1525 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) { 1526 return 0; 1527 } 1528 1529 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */ 1530 if (virtio_has_feature(dev->protocol_features, 1531 VHOST_USER_PROTOCOL_F_RARP)) { 1532 msg.hdr.request = VHOST_USER_SEND_RARP; 1533 msg.hdr.flags = VHOST_USER_VERSION; 1534 memcpy((char *)&msg.payload.u64, mac_addr, 6); 1535 msg.hdr.size = sizeof(msg.payload.u64); 1536 1537 return vhost_user_write(dev, &msg, NULL, 0); 1538 } 1539 return -1; 1540 } 1541 1542 static bool vhost_user_can_merge(struct vhost_dev *dev, 1543 uint64_t start1, uint64_t size1, 1544 uint64_t start2, uint64_t size2) 1545 { 1546 ram_addr_t offset; 1547 int mfd, rfd; 1548 MemoryRegion *mr; 1549 1550 mr = memory_region_from_host((void *)(uintptr_t)start1, &offset); 1551 mfd = memory_region_get_fd(mr); 1552 1553 mr = memory_region_from_host((void *)(uintptr_t)start2, &offset); 1554 rfd = memory_region_get_fd(mr); 1555 1556 return mfd == rfd; 1557 } 1558 1559 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu) 1560 { 1561 VhostUserMsg msg; 1562 bool reply_supported = virtio_has_feature(dev->protocol_features, 1563 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1564 1565 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) { 1566 return 0; 1567 } 1568 1569 msg.hdr.request = VHOST_USER_NET_SET_MTU; 1570 msg.payload.u64 = mtu; 1571 msg.hdr.size = sizeof(msg.payload.u64); 1572 msg.hdr.flags = VHOST_USER_VERSION; 1573 if (reply_supported) { 1574 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1575 } 1576 1577 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1578 return -1; 1579 } 1580 1581 /* If reply_ack supported, slave has to ack specified MTU is valid */ 1582 if (reply_supported) { 1583 return process_message_reply(dev, &msg); 1584 } 1585 1586 return 0; 1587 } 1588 1589 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev, 1590 struct vhost_iotlb_msg *imsg) 1591 { 1592 VhostUserMsg msg = { 1593 .hdr.request = VHOST_USER_IOTLB_MSG, 1594 .hdr.size = sizeof(msg.payload.iotlb), 1595 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1596 .payload.iotlb = *imsg, 1597 }; 1598 1599 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1600 return -EFAULT; 1601 } 1602 1603 return process_message_reply(dev, &msg); 1604 } 1605 1606 1607 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled) 1608 { 1609 /* No-op as the receive channel is not dedicated to IOTLB messages. */ 1610 } 1611 1612 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config, 1613 uint32_t config_len) 1614 { 1615 VhostUserMsg msg = { 1616 .hdr.request = VHOST_USER_GET_CONFIG, 1617 .hdr.flags = VHOST_USER_VERSION, 1618 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len, 1619 }; 1620 1621 if (!virtio_has_feature(dev->protocol_features, 1622 VHOST_USER_PROTOCOL_F_CONFIG)) { 1623 return -1; 1624 } 1625 1626 if (config_len > VHOST_USER_MAX_CONFIG_SIZE) { 1627 return -1; 1628 } 1629 1630 msg.payload.config.offset = 0; 1631 msg.payload.config.size = config_len; 1632 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1633 return -1; 1634 } 1635 1636 if (vhost_user_read(dev, &msg) < 0) { 1637 return -1; 1638 } 1639 1640 if (msg.hdr.request != VHOST_USER_GET_CONFIG) { 1641 error_report("Received unexpected msg type. Expected %d received %d", 1642 VHOST_USER_GET_CONFIG, msg.hdr.request); 1643 return -1; 1644 } 1645 1646 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) { 1647 error_report("Received bad msg size."); 1648 return -1; 1649 } 1650 1651 memcpy(config, msg.payload.config.region, config_len); 1652 1653 return 0; 1654 } 1655 1656 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data, 1657 uint32_t offset, uint32_t size, uint32_t flags) 1658 { 1659 uint8_t *p; 1660 bool reply_supported = virtio_has_feature(dev->protocol_features, 1661 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1662 1663 VhostUserMsg msg = { 1664 .hdr.request = VHOST_USER_SET_CONFIG, 1665 .hdr.flags = VHOST_USER_VERSION, 1666 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size, 1667 }; 1668 1669 if (!virtio_has_feature(dev->protocol_features, 1670 VHOST_USER_PROTOCOL_F_CONFIG)) { 1671 return -1; 1672 } 1673 1674 if (reply_supported) { 1675 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1676 } 1677 1678 if (size > VHOST_USER_MAX_CONFIG_SIZE) { 1679 return -1; 1680 } 1681 1682 msg.payload.config.offset = offset, 1683 msg.payload.config.size = size, 1684 msg.payload.config.flags = flags, 1685 p = msg.payload.config.region; 1686 memcpy(p, data, size); 1687 1688 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1689 return -1; 1690 } 1691 1692 if (reply_supported) { 1693 return process_message_reply(dev, &msg); 1694 } 1695 1696 return 0; 1697 } 1698 1699 static int vhost_user_crypto_create_session(struct vhost_dev *dev, 1700 void *session_info, 1701 uint64_t *session_id) 1702 { 1703 bool crypto_session = virtio_has_feature(dev->protocol_features, 1704 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1705 CryptoDevBackendSymSessionInfo *sess_info = session_info; 1706 VhostUserMsg msg = { 1707 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION, 1708 .hdr.flags = VHOST_USER_VERSION, 1709 .hdr.size = sizeof(msg.payload.session), 1710 }; 1711 1712 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1713 1714 if (!crypto_session) { 1715 error_report("vhost-user trying to send unhandled ioctl"); 1716 return -1; 1717 } 1718 1719 memcpy(&msg.payload.session.session_setup_data, sess_info, 1720 sizeof(CryptoDevBackendSymSessionInfo)); 1721 if (sess_info->key_len) { 1722 memcpy(&msg.payload.session.key, sess_info->cipher_key, 1723 sess_info->key_len); 1724 } 1725 if (sess_info->auth_key_len > 0) { 1726 memcpy(&msg.payload.session.auth_key, sess_info->auth_key, 1727 sess_info->auth_key_len); 1728 } 1729 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1730 error_report("vhost_user_write() return -1, create session failed"); 1731 return -1; 1732 } 1733 1734 if (vhost_user_read(dev, &msg) < 0) { 1735 error_report("vhost_user_read() return -1, create session failed"); 1736 return -1; 1737 } 1738 1739 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) { 1740 error_report("Received unexpected msg type. Expected %d received %d", 1741 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request); 1742 return -1; 1743 } 1744 1745 if (msg.hdr.size != sizeof(msg.payload.session)) { 1746 error_report("Received bad msg size."); 1747 return -1; 1748 } 1749 1750 if (msg.payload.session.session_id < 0) { 1751 error_report("Bad session id: %" PRId64 "", 1752 msg.payload.session.session_id); 1753 return -1; 1754 } 1755 *session_id = msg.payload.session.session_id; 1756 1757 return 0; 1758 } 1759 1760 static int 1761 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id) 1762 { 1763 bool crypto_session = virtio_has_feature(dev->protocol_features, 1764 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1765 VhostUserMsg msg = { 1766 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION, 1767 .hdr.flags = VHOST_USER_VERSION, 1768 .hdr.size = sizeof(msg.payload.u64), 1769 }; 1770 msg.payload.u64 = session_id; 1771 1772 if (!crypto_session) { 1773 error_report("vhost-user trying to send unhandled ioctl"); 1774 return -1; 1775 } 1776 1777 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1778 error_report("vhost_user_write() return -1, close session failed"); 1779 return -1; 1780 } 1781 1782 return 0; 1783 } 1784 1785 static bool vhost_user_mem_section_filter(struct vhost_dev *dev, 1786 MemoryRegionSection *section) 1787 { 1788 bool result; 1789 1790 result = memory_region_get_fd(section->mr) >= 0; 1791 1792 return result; 1793 } 1794 1795 static int vhost_user_get_inflight_fd(struct vhost_dev *dev, 1796 uint16_t queue_size, 1797 struct vhost_inflight *inflight) 1798 { 1799 void *addr; 1800 int fd; 1801 struct vhost_user *u = dev->opaque; 1802 CharBackend *chr = u->user->chr; 1803 VhostUserMsg msg = { 1804 .hdr.request = VHOST_USER_GET_INFLIGHT_FD, 1805 .hdr.flags = VHOST_USER_VERSION, 1806 .payload.inflight.num_queues = dev->nvqs, 1807 .payload.inflight.queue_size = queue_size, 1808 .hdr.size = sizeof(msg.payload.inflight), 1809 }; 1810 1811 if (!virtio_has_feature(dev->protocol_features, 1812 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { 1813 return 0; 1814 } 1815 1816 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1817 return -1; 1818 } 1819 1820 if (vhost_user_read(dev, &msg) < 0) { 1821 return -1; 1822 } 1823 1824 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) { 1825 error_report("Received unexpected msg type. " 1826 "Expected %d received %d", 1827 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request); 1828 return -1; 1829 } 1830 1831 if (msg.hdr.size != sizeof(msg.payload.inflight)) { 1832 error_report("Received bad msg size."); 1833 return -1; 1834 } 1835 1836 if (!msg.payload.inflight.mmap_size) { 1837 return 0; 1838 } 1839 1840 fd = qemu_chr_fe_get_msgfd(chr); 1841 if (fd < 0) { 1842 error_report("Failed to get mem fd"); 1843 return -1; 1844 } 1845 1846 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE, 1847 MAP_SHARED, fd, msg.payload.inflight.mmap_offset); 1848 1849 if (addr == MAP_FAILED) { 1850 error_report("Failed to mmap mem fd"); 1851 close(fd); 1852 return -1; 1853 } 1854 1855 inflight->addr = addr; 1856 inflight->fd = fd; 1857 inflight->size = msg.payload.inflight.mmap_size; 1858 inflight->offset = msg.payload.inflight.mmap_offset; 1859 inflight->queue_size = queue_size; 1860 1861 return 0; 1862 } 1863 1864 static int vhost_user_set_inflight_fd(struct vhost_dev *dev, 1865 struct vhost_inflight *inflight) 1866 { 1867 VhostUserMsg msg = { 1868 .hdr.request = VHOST_USER_SET_INFLIGHT_FD, 1869 .hdr.flags = VHOST_USER_VERSION, 1870 .payload.inflight.mmap_size = inflight->size, 1871 .payload.inflight.mmap_offset = inflight->offset, 1872 .payload.inflight.num_queues = dev->nvqs, 1873 .payload.inflight.queue_size = inflight->queue_size, 1874 .hdr.size = sizeof(msg.payload.inflight), 1875 }; 1876 1877 if (!virtio_has_feature(dev->protocol_features, 1878 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) { 1879 return 0; 1880 } 1881 1882 if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) { 1883 return -1; 1884 } 1885 1886 return 0; 1887 } 1888 1889 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp) 1890 { 1891 if (user->chr) { 1892 error_setg(errp, "Cannot initialize vhost-user state"); 1893 return false; 1894 } 1895 user->chr = chr; 1896 return true; 1897 } 1898 1899 void vhost_user_cleanup(VhostUserState *user) 1900 { 1901 int i; 1902 1903 if (!user->chr) { 1904 return; 1905 } 1906 1907 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { 1908 if (user->notifier[i].addr) { 1909 object_unparent(OBJECT(&user->notifier[i].mr)); 1910 munmap(user->notifier[i].addr, qemu_real_host_page_size); 1911 user->notifier[i].addr = NULL; 1912 } 1913 } 1914 user->chr = NULL; 1915 } 1916 1917 const VhostOps user_ops = { 1918 .backend_type = VHOST_BACKEND_TYPE_USER, 1919 .vhost_backend_init = vhost_user_backend_init, 1920 .vhost_backend_cleanup = vhost_user_backend_cleanup, 1921 .vhost_backend_memslots_limit = vhost_user_memslots_limit, 1922 .vhost_set_log_base = vhost_user_set_log_base, 1923 .vhost_set_mem_table = vhost_user_set_mem_table, 1924 .vhost_set_vring_addr = vhost_user_set_vring_addr, 1925 .vhost_set_vring_endian = vhost_user_set_vring_endian, 1926 .vhost_set_vring_num = vhost_user_set_vring_num, 1927 .vhost_set_vring_base = vhost_user_set_vring_base, 1928 .vhost_get_vring_base = vhost_user_get_vring_base, 1929 .vhost_set_vring_kick = vhost_user_set_vring_kick, 1930 .vhost_set_vring_call = vhost_user_set_vring_call, 1931 .vhost_set_features = vhost_user_set_features, 1932 .vhost_get_features = vhost_user_get_features, 1933 .vhost_set_owner = vhost_user_set_owner, 1934 .vhost_reset_device = vhost_user_reset_device, 1935 .vhost_get_vq_index = vhost_user_get_vq_index, 1936 .vhost_set_vring_enable = vhost_user_set_vring_enable, 1937 .vhost_requires_shm_log = vhost_user_requires_shm_log, 1938 .vhost_migration_done = vhost_user_migration_done, 1939 .vhost_backend_can_merge = vhost_user_can_merge, 1940 .vhost_net_set_mtu = vhost_user_net_set_mtu, 1941 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback, 1942 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg, 1943 .vhost_get_config = vhost_user_get_config, 1944 .vhost_set_config = vhost_user_set_config, 1945 .vhost_crypto_create_session = vhost_user_crypto_create_session, 1946 .vhost_crypto_close_session = vhost_user_crypto_close_session, 1947 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter, 1948 .vhost_get_inflight_fd = vhost_user_get_inflight_fd, 1949 .vhost_set_inflight_fd = vhost_user_set_inflight_fd, 1950 }; 1951