1 /* 2 * vhost-user 3 * 4 * Copyright (c) 2013 Virtual Open Systems Sarl. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 * 9 */ 10 11 #include "qemu/osdep.h" 12 #include "qapi/error.h" 13 #include "hw/virtio/vhost.h" 14 #include "hw/virtio/vhost-backend.h" 15 #include "hw/virtio/virtio-net.h" 16 #include "chardev/char-fe.h" 17 #include "sysemu/kvm.h" 18 #include "qemu/error-report.h" 19 #include "qemu/sockets.h" 20 #include "sysemu/cryptodev.h" 21 #include "migration/migration.h" 22 #include "migration/postcopy-ram.h" 23 #include "trace.h" 24 25 #include <sys/ioctl.h> 26 #include <sys/socket.h> 27 #include <sys/un.h> 28 #include <linux/vhost.h> 29 #include <linux/userfaultfd.h> 30 31 #define VHOST_MEMORY_MAX_NREGIONS 8 32 #define VHOST_USER_F_PROTOCOL_FEATURES 30 33 34 /* 35 * Maximum size of virtio device config space 36 */ 37 #define VHOST_USER_MAX_CONFIG_SIZE 256 38 39 enum VhostUserProtocolFeature { 40 VHOST_USER_PROTOCOL_F_MQ = 0, 41 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1, 42 VHOST_USER_PROTOCOL_F_RARP = 2, 43 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3, 44 VHOST_USER_PROTOCOL_F_NET_MTU = 4, 45 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5, 46 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6, 47 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7, 48 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8, 49 VHOST_USER_PROTOCOL_F_MAX 50 }; 51 52 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1) 53 54 typedef enum VhostUserRequest { 55 VHOST_USER_NONE = 0, 56 VHOST_USER_GET_FEATURES = 1, 57 VHOST_USER_SET_FEATURES = 2, 58 VHOST_USER_SET_OWNER = 3, 59 VHOST_USER_RESET_OWNER = 4, 60 VHOST_USER_SET_MEM_TABLE = 5, 61 VHOST_USER_SET_LOG_BASE = 6, 62 VHOST_USER_SET_LOG_FD = 7, 63 VHOST_USER_SET_VRING_NUM = 8, 64 VHOST_USER_SET_VRING_ADDR = 9, 65 VHOST_USER_SET_VRING_BASE = 10, 66 VHOST_USER_GET_VRING_BASE = 11, 67 VHOST_USER_SET_VRING_KICK = 12, 68 VHOST_USER_SET_VRING_CALL = 13, 69 VHOST_USER_SET_VRING_ERR = 14, 70 VHOST_USER_GET_PROTOCOL_FEATURES = 15, 71 VHOST_USER_SET_PROTOCOL_FEATURES = 16, 72 VHOST_USER_GET_QUEUE_NUM = 17, 73 VHOST_USER_SET_VRING_ENABLE = 18, 74 VHOST_USER_SEND_RARP = 19, 75 VHOST_USER_NET_SET_MTU = 20, 76 VHOST_USER_SET_SLAVE_REQ_FD = 21, 77 VHOST_USER_IOTLB_MSG = 22, 78 VHOST_USER_SET_VRING_ENDIAN = 23, 79 VHOST_USER_GET_CONFIG = 24, 80 VHOST_USER_SET_CONFIG = 25, 81 VHOST_USER_CREATE_CRYPTO_SESSION = 26, 82 VHOST_USER_CLOSE_CRYPTO_SESSION = 27, 83 VHOST_USER_POSTCOPY_ADVISE = 28, 84 VHOST_USER_POSTCOPY_LISTEN = 29, 85 VHOST_USER_POSTCOPY_END = 30, 86 VHOST_USER_MAX 87 } VhostUserRequest; 88 89 typedef enum VhostUserSlaveRequest { 90 VHOST_USER_SLAVE_NONE = 0, 91 VHOST_USER_SLAVE_IOTLB_MSG = 1, 92 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2, 93 VHOST_USER_SLAVE_MAX 94 } VhostUserSlaveRequest; 95 96 typedef struct VhostUserMemoryRegion { 97 uint64_t guest_phys_addr; 98 uint64_t memory_size; 99 uint64_t userspace_addr; 100 uint64_t mmap_offset; 101 } VhostUserMemoryRegion; 102 103 typedef struct VhostUserMemory { 104 uint32_t nregions; 105 uint32_t padding; 106 VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS]; 107 } VhostUserMemory; 108 109 typedef struct VhostUserLog { 110 uint64_t mmap_size; 111 uint64_t mmap_offset; 112 } VhostUserLog; 113 114 typedef struct VhostUserConfig { 115 uint32_t offset; 116 uint32_t size; 117 uint32_t flags; 118 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE]; 119 } VhostUserConfig; 120 121 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512 122 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64 123 124 typedef struct VhostUserCryptoSession { 125 /* session id for success, -1 on errors */ 126 int64_t session_id; 127 CryptoDevBackendSymSessionInfo session_setup_data; 128 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN]; 129 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN]; 130 } VhostUserCryptoSession; 131 132 static VhostUserConfig c __attribute__ ((unused)); 133 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \ 134 + sizeof(c.size) \ 135 + sizeof(c.flags)) 136 137 typedef struct { 138 VhostUserRequest request; 139 140 #define VHOST_USER_VERSION_MASK (0x3) 141 #define VHOST_USER_REPLY_MASK (0x1<<2) 142 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3) 143 uint32_t flags; 144 uint32_t size; /* the following payload size */ 145 } QEMU_PACKED VhostUserHeader; 146 147 typedef union { 148 #define VHOST_USER_VRING_IDX_MASK (0xff) 149 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8) 150 uint64_t u64; 151 struct vhost_vring_state state; 152 struct vhost_vring_addr addr; 153 VhostUserMemory memory; 154 VhostUserLog log; 155 struct vhost_iotlb_msg iotlb; 156 VhostUserConfig config; 157 VhostUserCryptoSession session; 158 } VhostUserPayload; 159 160 typedef struct VhostUserMsg { 161 VhostUserHeader hdr; 162 VhostUserPayload payload; 163 } QEMU_PACKED VhostUserMsg; 164 165 static VhostUserMsg m __attribute__ ((unused)); 166 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader)) 167 168 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload)) 169 170 /* The version of the protocol we support */ 171 #define VHOST_USER_VERSION (0x1) 172 173 struct vhost_user { 174 struct vhost_dev *dev; 175 CharBackend *chr; 176 int slave_fd; 177 NotifierWithReturn postcopy_notifier; 178 struct PostCopyFD postcopy_fd; 179 uint64_t postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS]; 180 /* Length of the region_rb and region_rb_offset arrays */ 181 size_t region_rb_len; 182 /* RAMBlock associated with a given region */ 183 RAMBlock **region_rb; 184 /* The offset from the start of the RAMBlock to the start of the 185 * vhost region. 186 */ 187 ram_addr_t *region_rb_offset; 188 189 /* True once we've entered postcopy_listen */ 190 bool postcopy_listen; 191 }; 192 193 static bool ioeventfd_enabled(void) 194 { 195 return kvm_enabled() && kvm_eventfds_enabled(); 196 } 197 198 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg) 199 { 200 struct vhost_user *u = dev->opaque; 201 CharBackend *chr = u->chr; 202 uint8_t *p = (uint8_t *) msg; 203 int r, size = VHOST_USER_HDR_SIZE; 204 205 r = qemu_chr_fe_read_all(chr, p, size); 206 if (r != size) { 207 error_report("Failed to read msg header. Read %d instead of %d." 208 " Original request %d.", r, size, msg->hdr.request); 209 goto fail; 210 } 211 212 /* validate received flags */ 213 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) { 214 error_report("Failed to read msg header." 215 " Flags 0x%x instead of 0x%x.", msg->hdr.flags, 216 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION); 217 goto fail; 218 } 219 220 /* validate message size is sane */ 221 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) { 222 error_report("Failed to read msg header." 223 " Size %d exceeds the maximum %zu.", msg->hdr.size, 224 VHOST_USER_PAYLOAD_SIZE); 225 goto fail; 226 } 227 228 if (msg->hdr.size) { 229 p += VHOST_USER_HDR_SIZE; 230 size = msg->hdr.size; 231 r = qemu_chr_fe_read_all(chr, p, size); 232 if (r != size) { 233 error_report("Failed to read msg payload." 234 " Read %d instead of %d.", r, msg->hdr.size); 235 goto fail; 236 } 237 } 238 239 return 0; 240 241 fail: 242 return -1; 243 } 244 245 static int process_message_reply(struct vhost_dev *dev, 246 const VhostUserMsg *msg) 247 { 248 VhostUserMsg msg_reply; 249 250 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) { 251 return 0; 252 } 253 254 if (vhost_user_read(dev, &msg_reply) < 0) { 255 return -1; 256 } 257 258 if (msg_reply.hdr.request != msg->hdr.request) { 259 error_report("Received unexpected msg type." 260 "Expected %d received %d", 261 msg->hdr.request, msg_reply.hdr.request); 262 return -1; 263 } 264 265 return msg_reply.payload.u64 ? -1 : 0; 266 } 267 268 static bool vhost_user_one_time_request(VhostUserRequest request) 269 { 270 switch (request) { 271 case VHOST_USER_SET_OWNER: 272 case VHOST_USER_RESET_OWNER: 273 case VHOST_USER_SET_MEM_TABLE: 274 case VHOST_USER_GET_QUEUE_NUM: 275 case VHOST_USER_NET_SET_MTU: 276 return true; 277 default: 278 return false; 279 } 280 } 281 282 /* most non-init callers ignore the error */ 283 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, 284 int *fds, int fd_num) 285 { 286 struct vhost_user *u = dev->opaque; 287 CharBackend *chr = u->chr; 288 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size; 289 290 /* 291 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE, 292 * we just need send it once in the first time. For later such 293 * request, we just ignore it. 294 */ 295 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) { 296 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 297 return 0; 298 } 299 300 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) { 301 error_report("Failed to set msg fds."); 302 return -1; 303 } 304 305 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size); 306 if (ret != size) { 307 error_report("Failed to write msg." 308 " Wrote %d instead of %d.", ret, size); 309 return -1; 310 } 311 312 return 0; 313 } 314 315 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base, 316 struct vhost_log *log) 317 { 318 int fds[VHOST_MEMORY_MAX_NREGIONS]; 319 size_t fd_num = 0; 320 bool shmfd = virtio_has_feature(dev->protocol_features, 321 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 322 VhostUserMsg msg = { 323 .hdr.request = VHOST_USER_SET_LOG_BASE, 324 .hdr.flags = VHOST_USER_VERSION, 325 .payload.log.mmap_size = log->size * sizeof(*(log->log)), 326 .payload.log.mmap_offset = 0, 327 .hdr.size = sizeof(msg.payload.log), 328 }; 329 330 if (shmfd && log->fd != -1) { 331 fds[fd_num++] = log->fd; 332 } 333 334 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 335 return -1; 336 } 337 338 if (shmfd) { 339 msg.hdr.size = 0; 340 if (vhost_user_read(dev, &msg) < 0) { 341 return -1; 342 } 343 344 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) { 345 error_report("Received unexpected msg type. " 346 "Expected %d received %d", 347 VHOST_USER_SET_LOG_BASE, msg.hdr.request); 348 return -1; 349 } 350 } 351 352 return 0; 353 } 354 355 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev, 356 struct vhost_memory *mem) 357 { 358 struct vhost_user *u = dev->opaque; 359 int fds[VHOST_MEMORY_MAX_NREGIONS]; 360 int i, fd; 361 size_t fd_num = 0; 362 bool reply_supported = virtio_has_feature(dev->protocol_features, 363 VHOST_USER_PROTOCOL_F_REPLY_ACK); 364 VhostUserMsg msg_reply; 365 int region_i, msg_i; 366 367 VhostUserMsg msg = { 368 .hdr.request = VHOST_USER_SET_MEM_TABLE, 369 .hdr.flags = VHOST_USER_VERSION, 370 }; 371 372 if (reply_supported) { 373 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 374 } 375 376 if (u->region_rb_len < dev->mem->nregions) { 377 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions); 378 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset, 379 dev->mem->nregions); 380 memset(&(u->region_rb[u->region_rb_len]), '\0', 381 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len)); 382 memset(&(u->region_rb_offset[u->region_rb_len]), '\0', 383 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len)); 384 u->region_rb_len = dev->mem->nregions; 385 } 386 387 for (i = 0; i < dev->mem->nregions; ++i) { 388 struct vhost_memory_region *reg = dev->mem->regions + i; 389 ram_addr_t offset; 390 MemoryRegion *mr; 391 392 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 393 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 394 &offset); 395 fd = memory_region_get_fd(mr); 396 if (fd > 0) { 397 trace_vhost_user_set_mem_table_withfd(fd_num, mr->name, 398 reg->memory_size, 399 reg->guest_phys_addr, 400 reg->userspace_addr, offset); 401 u->region_rb_offset[i] = offset; 402 u->region_rb[i] = mr->ram_block; 403 msg.payload.memory.regions[fd_num].userspace_addr = 404 reg->userspace_addr; 405 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 406 msg.payload.memory.regions[fd_num].guest_phys_addr = 407 reg->guest_phys_addr; 408 msg.payload.memory.regions[fd_num].mmap_offset = offset; 409 assert(fd_num < VHOST_MEMORY_MAX_NREGIONS); 410 fds[fd_num++] = fd; 411 } else { 412 u->region_rb_offset[i] = 0; 413 u->region_rb[i] = NULL; 414 } 415 } 416 417 msg.payload.memory.nregions = fd_num; 418 419 if (!fd_num) { 420 error_report("Failed initializing vhost-user memory map, " 421 "consider using -object memory-backend-file share=on"); 422 return -1; 423 } 424 425 msg.hdr.size = sizeof(msg.payload.memory.nregions); 426 msg.hdr.size += sizeof(msg.payload.memory.padding); 427 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 428 429 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 430 return -1; 431 } 432 433 if (vhost_user_read(dev, &msg_reply) < 0) { 434 return -1; 435 } 436 437 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) { 438 error_report("%s: Received unexpected msg type." 439 "Expected %d received %d", __func__, 440 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request); 441 return -1; 442 } 443 /* We're using the same structure, just reusing one of the 444 * fields, so it should be the same size. 445 */ 446 if (msg_reply.hdr.size != msg.hdr.size) { 447 error_report("%s: Unexpected size for postcopy reply " 448 "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size); 449 return -1; 450 } 451 452 memset(u->postcopy_client_bases, 0, 453 sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS); 454 455 /* They're in the same order as the regions that were sent 456 * but some of the regions were skipped (above) if they 457 * didn't have fd's 458 */ 459 for (msg_i = 0, region_i = 0; 460 region_i < dev->mem->nregions; 461 region_i++) { 462 if (msg_i < fd_num && 463 msg_reply.payload.memory.regions[msg_i].guest_phys_addr == 464 dev->mem->regions[region_i].guest_phys_addr) { 465 u->postcopy_client_bases[region_i] = 466 msg_reply.payload.memory.regions[msg_i].userspace_addr; 467 trace_vhost_user_set_mem_table_postcopy( 468 msg_reply.payload.memory.regions[msg_i].userspace_addr, 469 msg.payload.memory.regions[msg_i].userspace_addr, 470 msg_i, region_i); 471 msg_i++; 472 } 473 } 474 if (msg_i != fd_num) { 475 error_report("%s: postcopy reply not fully consumed " 476 "%d vs %zd", 477 __func__, msg_i, fd_num); 478 return -1; 479 } 480 /* Now we've registered this with the postcopy code, we ack to the client, 481 * because now we're in the position to be able to deal with any faults 482 * it generates. 483 */ 484 /* TODO: Use this for failure cases as well with a bad value */ 485 msg.hdr.size = sizeof(msg.payload.u64); 486 msg.payload.u64 = 0; /* OK */ 487 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 488 return -1; 489 } 490 491 if (reply_supported) { 492 return process_message_reply(dev, &msg); 493 } 494 495 return 0; 496 } 497 498 static int vhost_user_set_mem_table(struct vhost_dev *dev, 499 struct vhost_memory *mem) 500 { 501 struct vhost_user *u = dev->opaque; 502 int fds[VHOST_MEMORY_MAX_NREGIONS]; 503 int i, fd; 504 size_t fd_num = 0; 505 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler; 506 bool reply_supported = virtio_has_feature(dev->protocol_features, 507 VHOST_USER_PROTOCOL_F_REPLY_ACK) && 508 !do_postcopy; 509 510 if (do_postcopy) { 511 /* Postcopy has enough differences that it's best done in it's own 512 * version 513 */ 514 return vhost_user_set_mem_table_postcopy(dev, mem); 515 } 516 517 VhostUserMsg msg = { 518 .hdr.request = VHOST_USER_SET_MEM_TABLE, 519 .hdr.flags = VHOST_USER_VERSION, 520 }; 521 522 if (reply_supported) { 523 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 524 } 525 526 for (i = 0; i < dev->mem->nregions; ++i) { 527 struct vhost_memory_region *reg = dev->mem->regions + i; 528 ram_addr_t offset; 529 MemoryRegion *mr; 530 531 assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); 532 mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr, 533 &offset); 534 fd = memory_region_get_fd(mr); 535 if (fd > 0) { 536 if (fd_num == VHOST_MEMORY_MAX_NREGIONS) { 537 error_report("Failed preparing vhost-user memory table msg"); 538 return -1; 539 } 540 msg.payload.memory.regions[fd_num].userspace_addr = 541 reg->userspace_addr; 542 msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; 543 msg.payload.memory.regions[fd_num].guest_phys_addr = 544 reg->guest_phys_addr; 545 msg.payload.memory.regions[fd_num].mmap_offset = offset; 546 fds[fd_num++] = fd; 547 } 548 } 549 550 msg.payload.memory.nregions = fd_num; 551 552 if (!fd_num) { 553 error_report("Failed initializing vhost-user memory map, " 554 "consider using -object memory-backend-file share=on"); 555 return -1; 556 } 557 558 msg.hdr.size = sizeof(msg.payload.memory.nregions); 559 msg.hdr.size += sizeof(msg.payload.memory.padding); 560 msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion); 561 562 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 563 return -1; 564 } 565 566 if (reply_supported) { 567 return process_message_reply(dev, &msg); 568 } 569 570 return 0; 571 } 572 573 static int vhost_user_set_vring_addr(struct vhost_dev *dev, 574 struct vhost_vring_addr *addr) 575 { 576 VhostUserMsg msg = { 577 .hdr.request = VHOST_USER_SET_VRING_ADDR, 578 .hdr.flags = VHOST_USER_VERSION, 579 .payload.addr = *addr, 580 .hdr.size = sizeof(msg.payload.addr), 581 }; 582 583 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 584 return -1; 585 } 586 587 return 0; 588 } 589 590 static int vhost_user_set_vring_endian(struct vhost_dev *dev, 591 struct vhost_vring_state *ring) 592 { 593 bool cross_endian = virtio_has_feature(dev->protocol_features, 594 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN); 595 VhostUserMsg msg = { 596 .hdr.request = VHOST_USER_SET_VRING_ENDIAN, 597 .hdr.flags = VHOST_USER_VERSION, 598 .payload.state = *ring, 599 .hdr.size = sizeof(msg.payload.state), 600 }; 601 602 if (!cross_endian) { 603 error_report("vhost-user trying to send unhandled ioctl"); 604 return -1; 605 } 606 607 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 608 return -1; 609 } 610 611 return 0; 612 } 613 614 static int vhost_set_vring(struct vhost_dev *dev, 615 unsigned long int request, 616 struct vhost_vring_state *ring) 617 { 618 VhostUserMsg msg = { 619 .hdr.request = request, 620 .hdr.flags = VHOST_USER_VERSION, 621 .payload.state = *ring, 622 .hdr.size = sizeof(msg.payload.state), 623 }; 624 625 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 626 return -1; 627 } 628 629 return 0; 630 } 631 632 static int vhost_user_set_vring_num(struct vhost_dev *dev, 633 struct vhost_vring_state *ring) 634 { 635 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring); 636 } 637 638 static int vhost_user_set_vring_base(struct vhost_dev *dev, 639 struct vhost_vring_state *ring) 640 { 641 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring); 642 } 643 644 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable) 645 { 646 int i; 647 648 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) { 649 return -1; 650 } 651 652 for (i = 0; i < dev->nvqs; ++i) { 653 struct vhost_vring_state state = { 654 .index = dev->vq_index + i, 655 .num = enable, 656 }; 657 658 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state); 659 } 660 661 return 0; 662 } 663 664 static int vhost_user_get_vring_base(struct vhost_dev *dev, 665 struct vhost_vring_state *ring) 666 { 667 VhostUserMsg msg = { 668 .hdr.request = VHOST_USER_GET_VRING_BASE, 669 .hdr.flags = VHOST_USER_VERSION, 670 .payload.state = *ring, 671 .hdr.size = sizeof(msg.payload.state), 672 }; 673 674 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 675 return -1; 676 } 677 678 if (vhost_user_read(dev, &msg) < 0) { 679 return -1; 680 } 681 682 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) { 683 error_report("Received unexpected msg type. Expected %d received %d", 684 VHOST_USER_GET_VRING_BASE, msg.hdr.request); 685 return -1; 686 } 687 688 if (msg.hdr.size != sizeof(msg.payload.state)) { 689 error_report("Received bad msg size."); 690 return -1; 691 } 692 693 *ring = msg.payload.state; 694 695 return 0; 696 } 697 698 static int vhost_set_vring_file(struct vhost_dev *dev, 699 VhostUserRequest request, 700 struct vhost_vring_file *file) 701 { 702 int fds[VHOST_MEMORY_MAX_NREGIONS]; 703 size_t fd_num = 0; 704 VhostUserMsg msg = { 705 .hdr.request = request, 706 .hdr.flags = VHOST_USER_VERSION, 707 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK, 708 .hdr.size = sizeof(msg.payload.u64), 709 }; 710 711 if (ioeventfd_enabled() && file->fd > 0) { 712 fds[fd_num++] = file->fd; 713 } else { 714 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK; 715 } 716 717 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) { 718 return -1; 719 } 720 721 return 0; 722 } 723 724 static int vhost_user_set_vring_kick(struct vhost_dev *dev, 725 struct vhost_vring_file *file) 726 { 727 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file); 728 } 729 730 static int vhost_user_set_vring_call(struct vhost_dev *dev, 731 struct vhost_vring_file *file) 732 { 733 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file); 734 } 735 736 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) 737 { 738 VhostUserMsg msg = { 739 .hdr.request = request, 740 .hdr.flags = VHOST_USER_VERSION, 741 .payload.u64 = u64, 742 .hdr.size = sizeof(msg.payload.u64), 743 }; 744 745 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 746 return -1; 747 } 748 749 return 0; 750 } 751 752 static int vhost_user_set_features(struct vhost_dev *dev, 753 uint64_t features) 754 { 755 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features); 756 } 757 758 static int vhost_user_set_protocol_features(struct vhost_dev *dev, 759 uint64_t features) 760 { 761 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features); 762 } 763 764 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64) 765 { 766 VhostUserMsg msg = { 767 .hdr.request = request, 768 .hdr.flags = VHOST_USER_VERSION, 769 }; 770 771 if (vhost_user_one_time_request(request) && dev->vq_index != 0) { 772 return 0; 773 } 774 775 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 776 return -1; 777 } 778 779 if (vhost_user_read(dev, &msg) < 0) { 780 return -1; 781 } 782 783 if (msg.hdr.request != request) { 784 error_report("Received unexpected msg type. Expected %d received %d", 785 request, msg.hdr.request); 786 return -1; 787 } 788 789 if (msg.hdr.size != sizeof(msg.payload.u64)) { 790 error_report("Received bad msg size."); 791 return -1; 792 } 793 794 *u64 = msg.payload.u64; 795 796 return 0; 797 } 798 799 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features) 800 { 801 return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features); 802 } 803 804 static int vhost_user_set_owner(struct vhost_dev *dev) 805 { 806 VhostUserMsg msg = { 807 .hdr.request = VHOST_USER_SET_OWNER, 808 .hdr.flags = VHOST_USER_VERSION, 809 }; 810 811 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 812 return -1; 813 } 814 815 return 0; 816 } 817 818 static int vhost_user_reset_device(struct vhost_dev *dev) 819 { 820 VhostUserMsg msg = { 821 .hdr.request = VHOST_USER_RESET_OWNER, 822 .hdr.flags = VHOST_USER_VERSION, 823 }; 824 825 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 826 return -1; 827 } 828 829 return 0; 830 } 831 832 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) 833 { 834 int ret = -1; 835 836 if (!dev->config_ops) { 837 return -1; 838 } 839 840 if (dev->config_ops->vhost_dev_config_notifier) { 841 ret = dev->config_ops->vhost_dev_config_notifier(dev); 842 } 843 844 return ret; 845 } 846 847 static void slave_read(void *opaque) 848 { 849 struct vhost_dev *dev = opaque; 850 struct vhost_user *u = dev->opaque; 851 VhostUserHeader hdr = { 0, }; 852 VhostUserPayload payload = { 0, }; 853 int size, ret = 0; 854 855 /* Read header */ 856 size = read(u->slave_fd, &hdr, VHOST_USER_HDR_SIZE); 857 if (size != VHOST_USER_HDR_SIZE) { 858 error_report("Failed to read from slave."); 859 goto err; 860 } 861 862 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) { 863 error_report("Failed to read msg header." 864 " Size %d exceeds the maximum %zu.", hdr.size, 865 VHOST_USER_PAYLOAD_SIZE); 866 goto err; 867 } 868 869 /* Read payload */ 870 size = read(u->slave_fd, &payload, hdr.size); 871 if (size != hdr.size) { 872 error_report("Failed to read payload from slave."); 873 goto err; 874 } 875 876 switch (hdr.request) { 877 case VHOST_USER_SLAVE_IOTLB_MSG: 878 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb); 879 break; 880 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : 881 ret = vhost_user_slave_handle_config_change(dev); 882 break; 883 default: 884 error_report("Received unexpected msg type."); 885 ret = -EINVAL; 886 } 887 888 /* 889 * REPLY_ACK feature handling. Other reply types has to be managed 890 * directly in their request handlers. 891 */ 892 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) { 893 struct iovec iovec[2]; 894 895 896 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK; 897 hdr.flags |= VHOST_USER_REPLY_MASK; 898 899 payload.u64 = !!ret; 900 hdr.size = sizeof(payload.u64); 901 902 iovec[0].iov_base = &hdr; 903 iovec[0].iov_len = VHOST_USER_HDR_SIZE; 904 iovec[1].iov_base = &payload; 905 iovec[1].iov_len = hdr.size; 906 907 size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec)); 908 if (size != VHOST_USER_HDR_SIZE + hdr.size) { 909 error_report("Failed to send msg reply to slave."); 910 goto err; 911 } 912 } 913 914 return; 915 916 err: 917 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 918 close(u->slave_fd); 919 u->slave_fd = -1; 920 return; 921 } 922 923 static int vhost_setup_slave_channel(struct vhost_dev *dev) 924 { 925 VhostUserMsg msg = { 926 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD, 927 .hdr.flags = VHOST_USER_VERSION, 928 }; 929 struct vhost_user *u = dev->opaque; 930 int sv[2], ret = 0; 931 bool reply_supported = virtio_has_feature(dev->protocol_features, 932 VHOST_USER_PROTOCOL_F_REPLY_ACK); 933 934 if (!virtio_has_feature(dev->protocol_features, 935 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) { 936 return 0; 937 } 938 939 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) { 940 error_report("socketpair() failed"); 941 return -1; 942 } 943 944 u->slave_fd = sv[0]; 945 qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev); 946 947 if (reply_supported) { 948 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 949 } 950 951 ret = vhost_user_write(dev, &msg, &sv[1], 1); 952 if (ret) { 953 goto out; 954 } 955 956 if (reply_supported) { 957 ret = process_message_reply(dev, &msg); 958 } 959 960 out: 961 close(sv[1]); 962 if (ret) { 963 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 964 close(u->slave_fd); 965 u->slave_fd = -1; 966 } 967 968 return ret; 969 } 970 971 /* 972 * Called back from the postcopy fault thread when a fault is received on our 973 * ufd. 974 * TODO: This is Linux specific 975 */ 976 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd, 977 void *ufd) 978 { 979 struct vhost_dev *dev = pcfd->data; 980 struct vhost_user *u = dev->opaque; 981 struct uffd_msg *msg = ufd; 982 uint64_t faultaddr = msg->arg.pagefault.address; 983 RAMBlock *rb = NULL; 984 uint64_t rb_offset; 985 int i; 986 987 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr, 988 dev->mem->nregions); 989 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 990 trace_vhost_user_postcopy_fault_handler_loop(i, 991 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size); 992 if (faultaddr >= u->postcopy_client_bases[i]) { 993 /* Ofset of the fault address in the vhost region */ 994 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i]; 995 if (region_offset < dev->mem->regions[i].memory_size) { 996 rb_offset = region_offset + u->region_rb_offset[i]; 997 trace_vhost_user_postcopy_fault_handler_found(i, 998 region_offset, rb_offset); 999 rb = u->region_rb[i]; 1000 return postcopy_request_shared_page(pcfd, rb, faultaddr, 1001 rb_offset); 1002 } 1003 } 1004 } 1005 error_report("%s: Failed to find region for fault %" PRIx64, 1006 __func__, faultaddr); 1007 return -1; 1008 } 1009 1010 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb, 1011 uint64_t offset) 1012 { 1013 struct vhost_dev *dev = pcfd->data; 1014 struct vhost_user *u = dev->opaque; 1015 int i; 1016 1017 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset); 1018 1019 if (!u) { 1020 return 0; 1021 } 1022 /* Translate the offset into an address in the clients address space */ 1023 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) { 1024 if (u->region_rb[i] == rb && 1025 offset >= u->region_rb_offset[i] && 1026 offset < (u->region_rb_offset[i] + 1027 dev->mem->regions[i].memory_size)) { 1028 uint64_t client_addr = (offset - u->region_rb_offset[i]) + 1029 u->postcopy_client_bases[i]; 1030 trace_vhost_user_postcopy_waker_found(client_addr); 1031 return postcopy_wake_shared(pcfd, client_addr, rb); 1032 } 1033 } 1034 1035 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset); 1036 return 0; 1037 } 1038 1039 /* 1040 * Called at the start of an inbound postcopy on reception of the 1041 * 'advise' command. 1042 */ 1043 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp) 1044 { 1045 struct vhost_user *u = dev->opaque; 1046 CharBackend *chr = u->chr; 1047 int ufd; 1048 VhostUserMsg msg = { 1049 .hdr.request = VHOST_USER_POSTCOPY_ADVISE, 1050 .hdr.flags = VHOST_USER_VERSION, 1051 }; 1052 1053 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1054 error_setg(errp, "Failed to send postcopy_advise to vhost"); 1055 return -1; 1056 } 1057 1058 if (vhost_user_read(dev, &msg) < 0) { 1059 error_setg(errp, "Failed to get postcopy_advise reply from vhost"); 1060 return -1; 1061 } 1062 1063 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) { 1064 error_setg(errp, "Unexpected msg type. Expected %d received %d", 1065 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request); 1066 return -1; 1067 } 1068 1069 if (msg.hdr.size) { 1070 error_setg(errp, "Received bad msg size."); 1071 return -1; 1072 } 1073 ufd = qemu_chr_fe_get_msgfd(chr); 1074 if (ufd < 0) { 1075 error_setg(errp, "%s: Failed to get ufd", __func__); 1076 return -1; 1077 } 1078 fcntl(ufd, F_SETFL, O_NONBLOCK); 1079 1080 /* register ufd with userfault thread */ 1081 u->postcopy_fd.fd = ufd; 1082 u->postcopy_fd.data = dev; 1083 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler; 1084 u->postcopy_fd.waker = vhost_user_postcopy_waker; 1085 u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */ 1086 postcopy_register_shared_ufd(&u->postcopy_fd); 1087 return 0; 1088 } 1089 1090 /* 1091 * Called at the switch to postcopy on reception of the 'listen' command. 1092 */ 1093 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp) 1094 { 1095 struct vhost_user *u = dev->opaque; 1096 int ret; 1097 VhostUserMsg msg = { 1098 .hdr.request = VHOST_USER_POSTCOPY_LISTEN, 1099 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1100 }; 1101 u->postcopy_listen = true; 1102 trace_vhost_user_postcopy_listen(); 1103 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1104 error_setg(errp, "Failed to send postcopy_listen to vhost"); 1105 return -1; 1106 } 1107 1108 ret = process_message_reply(dev, &msg); 1109 if (ret) { 1110 error_setg(errp, "Failed to receive reply to postcopy_listen"); 1111 return ret; 1112 } 1113 1114 return 0; 1115 } 1116 1117 /* 1118 * Called at the end of postcopy 1119 */ 1120 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp) 1121 { 1122 VhostUserMsg msg = { 1123 .hdr.request = VHOST_USER_POSTCOPY_END, 1124 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1125 }; 1126 int ret; 1127 struct vhost_user *u = dev->opaque; 1128 1129 trace_vhost_user_postcopy_end_entry(); 1130 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1131 error_setg(errp, "Failed to send postcopy_end to vhost"); 1132 return -1; 1133 } 1134 1135 ret = process_message_reply(dev, &msg); 1136 if (ret) { 1137 error_setg(errp, "Failed to receive reply to postcopy_end"); 1138 return ret; 1139 } 1140 postcopy_unregister_shared_ufd(&u->postcopy_fd); 1141 u->postcopy_fd.handler = NULL; 1142 1143 trace_vhost_user_postcopy_end_exit(); 1144 1145 return 0; 1146 } 1147 1148 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier, 1149 void *opaque) 1150 { 1151 struct PostcopyNotifyData *pnd = opaque; 1152 struct vhost_user *u = container_of(notifier, struct vhost_user, 1153 postcopy_notifier); 1154 struct vhost_dev *dev = u->dev; 1155 1156 switch (pnd->reason) { 1157 case POSTCOPY_NOTIFY_PROBE: 1158 if (!virtio_has_feature(dev->protocol_features, 1159 VHOST_USER_PROTOCOL_F_PAGEFAULT)) { 1160 /* TODO: Get the device name into this error somehow */ 1161 error_setg(pnd->errp, 1162 "vhost-user backend not capable of postcopy"); 1163 return -ENOENT; 1164 } 1165 break; 1166 1167 case POSTCOPY_NOTIFY_INBOUND_ADVISE: 1168 return vhost_user_postcopy_advise(dev, pnd->errp); 1169 1170 case POSTCOPY_NOTIFY_INBOUND_LISTEN: 1171 return vhost_user_postcopy_listen(dev, pnd->errp); 1172 1173 case POSTCOPY_NOTIFY_INBOUND_END: 1174 return vhost_user_postcopy_end(dev, pnd->errp); 1175 1176 default: 1177 /* We ignore notifications we don't know */ 1178 break; 1179 } 1180 1181 return 0; 1182 } 1183 1184 static int vhost_user_init(struct vhost_dev *dev, void *opaque) 1185 { 1186 uint64_t features, protocol_features; 1187 struct vhost_user *u; 1188 int err; 1189 1190 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1191 1192 u = g_new0(struct vhost_user, 1); 1193 u->chr = opaque; 1194 u->slave_fd = -1; 1195 u->dev = dev; 1196 dev->opaque = u; 1197 1198 err = vhost_user_get_features(dev, &features); 1199 if (err < 0) { 1200 return err; 1201 } 1202 1203 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) { 1204 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES; 1205 1206 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES, 1207 &protocol_features); 1208 if (err < 0) { 1209 return err; 1210 } 1211 1212 dev->protocol_features = 1213 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK; 1214 err = vhost_user_set_protocol_features(dev, dev->protocol_features); 1215 if (err < 0) { 1216 return err; 1217 } 1218 1219 /* query the max queues we support if backend supports Multiple Queue */ 1220 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) { 1221 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM, 1222 &dev->max_queues); 1223 if (err < 0) { 1224 return err; 1225 } 1226 } 1227 1228 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) && 1229 !(virtio_has_feature(dev->protocol_features, 1230 VHOST_USER_PROTOCOL_F_SLAVE_REQ) && 1231 virtio_has_feature(dev->protocol_features, 1232 VHOST_USER_PROTOCOL_F_REPLY_ACK))) { 1233 error_report("IOMMU support requires reply-ack and " 1234 "slave-req protocol features."); 1235 return -1; 1236 } 1237 } 1238 1239 if (dev->migration_blocker == NULL && 1240 !virtio_has_feature(dev->protocol_features, 1241 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) { 1242 error_setg(&dev->migration_blocker, 1243 "Migration disabled: vhost-user backend lacks " 1244 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature."); 1245 } 1246 1247 err = vhost_setup_slave_channel(dev); 1248 if (err < 0) { 1249 return err; 1250 } 1251 1252 u->postcopy_notifier.notify = vhost_user_postcopy_notifier; 1253 postcopy_add_notifier(&u->postcopy_notifier); 1254 1255 return 0; 1256 } 1257 1258 static int vhost_user_cleanup(struct vhost_dev *dev) 1259 { 1260 struct vhost_user *u; 1261 1262 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1263 1264 u = dev->opaque; 1265 if (u->postcopy_notifier.notify) { 1266 postcopy_remove_notifier(&u->postcopy_notifier); 1267 u->postcopy_notifier.notify = NULL; 1268 } 1269 if (u->slave_fd >= 0) { 1270 qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL); 1271 close(u->slave_fd); 1272 u->slave_fd = -1; 1273 } 1274 g_free(u->region_rb); 1275 u->region_rb = NULL; 1276 g_free(u->region_rb_offset); 1277 u->region_rb_offset = NULL; 1278 u->region_rb_len = 0; 1279 g_free(u); 1280 dev->opaque = 0; 1281 1282 return 0; 1283 } 1284 1285 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx) 1286 { 1287 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs); 1288 1289 return idx; 1290 } 1291 1292 static int vhost_user_memslots_limit(struct vhost_dev *dev) 1293 { 1294 return VHOST_MEMORY_MAX_NREGIONS; 1295 } 1296 1297 static bool vhost_user_requires_shm_log(struct vhost_dev *dev) 1298 { 1299 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1300 1301 return virtio_has_feature(dev->protocol_features, 1302 VHOST_USER_PROTOCOL_F_LOG_SHMFD); 1303 } 1304 1305 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr) 1306 { 1307 VhostUserMsg msg = { 0 }; 1308 1309 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1310 1311 /* If guest supports GUEST_ANNOUNCE do nothing */ 1312 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) { 1313 return 0; 1314 } 1315 1316 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */ 1317 if (virtio_has_feature(dev->protocol_features, 1318 VHOST_USER_PROTOCOL_F_RARP)) { 1319 msg.hdr.request = VHOST_USER_SEND_RARP; 1320 msg.hdr.flags = VHOST_USER_VERSION; 1321 memcpy((char *)&msg.payload.u64, mac_addr, 6); 1322 msg.hdr.size = sizeof(msg.payload.u64); 1323 1324 return vhost_user_write(dev, &msg, NULL, 0); 1325 } 1326 return -1; 1327 } 1328 1329 static bool vhost_user_can_merge(struct vhost_dev *dev, 1330 uint64_t start1, uint64_t size1, 1331 uint64_t start2, uint64_t size2) 1332 { 1333 ram_addr_t offset; 1334 int mfd, rfd; 1335 MemoryRegion *mr; 1336 1337 mr = memory_region_from_host((void *)(uintptr_t)start1, &offset); 1338 mfd = memory_region_get_fd(mr); 1339 1340 mr = memory_region_from_host((void *)(uintptr_t)start2, &offset); 1341 rfd = memory_region_get_fd(mr); 1342 1343 return mfd == rfd; 1344 } 1345 1346 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu) 1347 { 1348 VhostUserMsg msg; 1349 bool reply_supported = virtio_has_feature(dev->protocol_features, 1350 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1351 1352 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) { 1353 return 0; 1354 } 1355 1356 msg.hdr.request = VHOST_USER_NET_SET_MTU; 1357 msg.payload.u64 = mtu; 1358 msg.hdr.size = sizeof(msg.payload.u64); 1359 msg.hdr.flags = VHOST_USER_VERSION; 1360 if (reply_supported) { 1361 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1362 } 1363 1364 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1365 return -1; 1366 } 1367 1368 /* If reply_ack supported, slave has to ack specified MTU is valid */ 1369 if (reply_supported) { 1370 return process_message_reply(dev, &msg); 1371 } 1372 1373 return 0; 1374 } 1375 1376 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev, 1377 struct vhost_iotlb_msg *imsg) 1378 { 1379 VhostUserMsg msg = { 1380 .hdr.request = VHOST_USER_IOTLB_MSG, 1381 .hdr.size = sizeof(msg.payload.iotlb), 1382 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK, 1383 .payload.iotlb = *imsg, 1384 }; 1385 1386 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1387 return -EFAULT; 1388 } 1389 1390 return process_message_reply(dev, &msg); 1391 } 1392 1393 1394 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled) 1395 { 1396 /* No-op as the receive channel is not dedicated to IOTLB messages. */ 1397 } 1398 1399 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config, 1400 uint32_t config_len) 1401 { 1402 VhostUserMsg msg = { 1403 .hdr.request = VHOST_USER_GET_CONFIG, 1404 .hdr.flags = VHOST_USER_VERSION, 1405 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len, 1406 }; 1407 1408 if (config_len > VHOST_USER_MAX_CONFIG_SIZE) { 1409 return -1; 1410 } 1411 1412 msg.payload.config.offset = 0; 1413 msg.payload.config.size = config_len; 1414 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1415 return -1; 1416 } 1417 1418 if (vhost_user_read(dev, &msg) < 0) { 1419 return -1; 1420 } 1421 1422 if (msg.hdr.request != VHOST_USER_GET_CONFIG) { 1423 error_report("Received unexpected msg type. Expected %d received %d", 1424 VHOST_USER_GET_CONFIG, msg.hdr.request); 1425 return -1; 1426 } 1427 1428 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) { 1429 error_report("Received bad msg size."); 1430 return -1; 1431 } 1432 1433 memcpy(config, msg.payload.config.region, config_len); 1434 1435 return 0; 1436 } 1437 1438 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data, 1439 uint32_t offset, uint32_t size, uint32_t flags) 1440 { 1441 uint8_t *p; 1442 bool reply_supported = virtio_has_feature(dev->protocol_features, 1443 VHOST_USER_PROTOCOL_F_REPLY_ACK); 1444 1445 VhostUserMsg msg = { 1446 .hdr.request = VHOST_USER_SET_CONFIG, 1447 .hdr.flags = VHOST_USER_VERSION, 1448 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size, 1449 }; 1450 1451 if (reply_supported) { 1452 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK; 1453 } 1454 1455 if (size > VHOST_USER_MAX_CONFIG_SIZE) { 1456 return -1; 1457 } 1458 1459 msg.payload.config.offset = offset, 1460 msg.payload.config.size = size, 1461 msg.payload.config.flags = flags, 1462 p = msg.payload.config.region; 1463 memcpy(p, data, size); 1464 1465 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1466 return -1; 1467 } 1468 1469 if (reply_supported) { 1470 return process_message_reply(dev, &msg); 1471 } 1472 1473 return 0; 1474 } 1475 1476 static int vhost_user_crypto_create_session(struct vhost_dev *dev, 1477 void *session_info, 1478 uint64_t *session_id) 1479 { 1480 bool crypto_session = virtio_has_feature(dev->protocol_features, 1481 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1482 CryptoDevBackendSymSessionInfo *sess_info = session_info; 1483 VhostUserMsg msg = { 1484 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION, 1485 .hdr.flags = VHOST_USER_VERSION, 1486 .hdr.size = sizeof(msg.payload.session), 1487 }; 1488 1489 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER); 1490 1491 if (!crypto_session) { 1492 error_report("vhost-user trying to send unhandled ioctl"); 1493 return -1; 1494 } 1495 1496 memcpy(&msg.payload.session.session_setup_data, sess_info, 1497 sizeof(CryptoDevBackendSymSessionInfo)); 1498 if (sess_info->key_len) { 1499 memcpy(&msg.payload.session.key, sess_info->cipher_key, 1500 sess_info->key_len); 1501 } 1502 if (sess_info->auth_key_len > 0) { 1503 memcpy(&msg.payload.session.auth_key, sess_info->auth_key, 1504 sess_info->auth_key_len); 1505 } 1506 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1507 error_report("vhost_user_write() return -1, create session failed"); 1508 return -1; 1509 } 1510 1511 if (vhost_user_read(dev, &msg) < 0) { 1512 error_report("vhost_user_read() return -1, create session failed"); 1513 return -1; 1514 } 1515 1516 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) { 1517 error_report("Received unexpected msg type. Expected %d received %d", 1518 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request); 1519 return -1; 1520 } 1521 1522 if (msg.hdr.size != sizeof(msg.payload.session)) { 1523 error_report("Received bad msg size."); 1524 return -1; 1525 } 1526 1527 if (msg.payload.session.session_id < 0) { 1528 error_report("Bad session id: %" PRId64 "", 1529 msg.payload.session.session_id); 1530 return -1; 1531 } 1532 *session_id = msg.payload.session.session_id; 1533 1534 return 0; 1535 } 1536 1537 static int 1538 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id) 1539 { 1540 bool crypto_session = virtio_has_feature(dev->protocol_features, 1541 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION); 1542 VhostUserMsg msg = { 1543 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION, 1544 .hdr.flags = VHOST_USER_VERSION, 1545 .hdr.size = sizeof(msg.payload.u64), 1546 }; 1547 msg.payload.u64 = session_id; 1548 1549 if (!crypto_session) { 1550 error_report("vhost-user trying to send unhandled ioctl"); 1551 return -1; 1552 } 1553 1554 if (vhost_user_write(dev, &msg, NULL, 0) < 0) { 1555 error_report("vhost_user_write() return -1, close session failed"); 1556 return -1; 1557 } 1558 1559 return 0; 1560 } 1561 1562 const VhostOps user_ops = { 1563 .backend_type = VHOST_BACKEND_TYPE_USER, 1564 .vhost_backend_init = vhost_user_init, 1565 .vhost_backend_cleanup = vhost_user_cleanup, 1566 .vhost_backend_memslots_limit = vhost_user_memslots_limit, 1567 .vhost_set_log_base = vhost_user_set_log_base, 1568 .vhost_set_mem_table = vhost_user_set_mem_table, 1569 .vhost_set_vring_addr = vhost_user_set_vring_addr, 1570 .vhost_set_vring_endian = vhost_user_set_vring_endian, 1571 .vhost_set_vring_num = vhost_user_set_vring_num, 1572 .vhost_set_vring_base = vhost_user_set_vring_base, 1573 .vhost_get_vring_base = vhost_user_get_vring_base, 1574 .vhost_set_vring_kick = vhost_user_set_vring_kick, 1575 .vhost_set_vring_call = vhost_user_set_vring_call, 1576 .vhost_set_features = vhost_user_set_features, 1577 .vhost_get_features = vhost_user_get_features, 1578 .vhost_set_owner = vhost_user_set_owner, 1579 .vhost_reset_device = vhost_user_reset_device, 1580 .vhost_get_vq_index = vhost_user_get_vq_index, 1581 .vhost_set_vring_enable = vhost_user_set_vring_enable, 1582 .vhost_requires_shm_log = vhost_user_requires_shm_log, 1583 .vhost_migration_done = vhost_user_migration_done, 1584 .vhost_backend_can_merge = vhost_user_can_merge, 1585 .vhost_net_set_mtu = vhost_user_net_set_mtu, 1586 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback, 1587 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg, 1588 .vhost_get_config = vhost_user_get_config, 1589 .vhost_set_config = vhost_user_set_config, 1590 .vhost_crypto_create_session = vhost_user_crypto_create_session, 1591 .vhost_crypto_close_session = vhost_user_crypto_close_session, 1592 }; 1593