1 /* 2 * Postcopy migration for RAM 3 * 4 * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Dave Gilbert <dgilbert@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * Postcopy is a migration technique where the execution flips from the 16 * source to the destination before all the data has been copied. 17 */ 18 19 #include "qemu/osdep.h" 20 #include "exec/target_page.h" 21 #include "migration.h" 22 #include "qemu-file.h" 23 #include "savevm.h" 24 #include "postcopy-ram.h" 25 #include "ram.h" 26 #include "qapi/error.h" 27 #include "qemu/notify.h" 28 #include "sysemu/sysemu.h" 29 #include "sysemu/balloon.h" 30 #include "qemu/error-report.h" 31 #include "trace.h" 32 33 /* Arbitrary limit on size of each discard command, 34 * keeps them around ~200 bytes 35 */ 36 #define MAX_DISCARDS_PER_COMMAND 12 37 38 struct PostcopyDiscardState { 39 const char *ramblock_name; 40 uint16_t cur_entry; 41 /* 42 * Start and length of a discard range (bytes) 43 */ 44 uint64_t start_list[MAX_DISCARDS_PER_COMMAND]; 45 uint64_t length_list[MAX_DISCARDS_PER_COMMAND]; 46 unsigned int nsentwords; 47 unsigned int nsentcmds; 48 }; 49 50 static NotifierWithReturnList postcopy_notifier_list; 51 52 void postcopy_infrastructure_init(void) 53 { 54 notifier_with_return_list_init(&postcopy_notifier_list); 55 } 56 57 void postcopy_add_notifier(NotifierWithReturn *nn) 58 { 59 notifier_with_return_list_add(&postcopy_notifier_list, nn); 60 } 61 62 void postcopy_remove_notifier(NotifierWithReturn *n) 63 { 64 notifier_with_return_remove(n); 65 } 66 67 int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp) 68 { 69 struct PostcopyNotifyData pnd; 70 pnd.reason = reason; 71 pnd.errp = errp; 72 73 return notifier_with_return_list_notify(&postcopy_notifier_list, 74 &pnd); 75 } 76 77 /* Postcopy needs to detect accesses to pages that haven't yet been copied 78 * across, and efficiently map new pages in, the techniques for doing this 79 * are target OS specific. 80 */ 81 #if defined(__linux__) 82 83 #include <poll.h> 84 #include <sys/ioctl.h> 85 #include <sys/syscall.h> 86 #include <asm/types.h> /* for __u64 */ 87 #endif 88 89 #if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD) 90 #include <sys/eventfd.h> 91 #include <linux/userfaultfd.h> 92 93 typedef struct PostcopyBlocktimeContext { 94 /* time when page fault initiated per vCPU */ 95 uint32_t *page_fault_vcpu_time; 96 /* page address per vCPU */ 97 uintptr_t *vcpu_addr; 98 uint32_t total_blocktime; 99 /* blocktime per vCPU */ 100 uint32_t *vcpu_blocktime; 101 /* point in time when last page fault was initiated */ 102 uint32_t last_begin; 103 /* number of vCPU are suspended */ 104 int smp_cpus_down; 105 uint64_t start_time; 106 107 /* 108 * Handler for exit event, necessary for 109 * releasing whole blocktime_ctx 110 */ 111 Notifier exit_notifier; 112 } PostcopyBlocktimeContext; 113 114 static void destroy_blocktime_context(struct PostcopyBlocktimeContext *ctx) 115 { 116 g_free(ctx->page_fault_vcpu_time); 117 g_free(ctx->vcpu_addr); 118 g_free(ctx->vcpu_blocktime); 119 g_free(ctx); 120 } 121 122 static void migration_exit_cb(Notifier *n, void *data) 123 { 124 PostcopyBlocktimeContext *ctx = container_of(n, PostcopyBlocktimeContext, 125 exit_notifier); 126 destroy_blocktime_context(ctx); 127 } 128 129 static struct PostcopyBlocktimeContext *blocktime_context_new(void) 130 { 131 PostcopyBlocktimeContext *ctx = g_new0(PostcopyBlocktimeContext, 1); 132 ctx->page_fault_vcpu_time = g_new0(uint32_t, smp_cpus); 133 ctx->vcpu_addr = g_new0(uintptr_t, smp_cpus); 134 ctx->vcpu_blocktime = g_new0(uint32_t, smp_cpus); 135 136 ctx->exit_notifier.notify = migration_exit_cb; 137 ctx->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 138 qemu_add_exit_notifier(&ctx->exit_notifier); 139 return ctx; 140 } 141 142 static uint32List *get_vcpu_blocktime_list(PostcopyBlocktimeContext *ctx) 143 { 144 uint32List *list = NULL, *entry = NULL; 145 int i; 146 147 for (i = smp_cpus - 1; i >= 0; i--) { 148 entry = g_new0(uint32List, 1); 149 entry->value = ctx->vcpu_blocktime[i]; 150 entry->next = list; 151 list = entry; 152 } 153 154 return list; 155 } 156 157 /* 158 * This function just populates MigrationInfo from postcopy's 159 * blocktime context. It will not populate MigrationInfo, 160 * unless postcopy-blocktime capability was set. 161 * 162 * @info: pointer to MigrationInfo to populate 163 */ 164 void fill_destination_postcopy_migration_info(MigrationInfo *info) 165 { 166 MigrationIncomingState *mis = migration_incoming_get_current(); 167 PostcopyBlocktimeContext *bc = mis->blocktime_ctx; 168 169 if (!bc) { 170 return; 171 } 172 173 info->has_postcopy_blocktime = true; 174 info->postcopy_blocktime = bc->total_blocktime; 175 info->has_postcopy_vcpu_blocktime = true; 176 info->postcopy_vcpu_blocktime = get_vcpu_blocktime_list(bc); 177 } 178 179 static uint32_t get_postcopy_total_blocktime(void) 180 { 181 MigrationIncomingState *mis = migration_incoming_get_current(); 182 PostcopyBlocktimeContext *bc = mis->blocktime_ctx; 183 184 if (!bc) { 185 return 0; 186 } 187 188 return bc->total_blocktime; 189 } 190 191 /** 192 * receive_ufd_features: check userfault fd features, to request only supported 193 * features in the future. 194 * 195 * Returns: true on success 196 * 197 * __NR_userfaultfd - should be checked before 198 * @features: out parameter will contain uffdio_api.features provided by kernel 199 * in case of success 200 */ 201 static bool receive_ufd_features(uint64_t *features) 202 { 203 struct uffdio_api api_struct = {0}; 204 int ufd; 205 bool ret = true; 206 207 /* if we are here __NR_userfaultfd should exists */ 208 ufd = syscall(__NR_userfaultfd, O_CLOEXEC); 209 if (ufd == -1) { 210 error_report("%s: syscall __NR_userfaultfd failed: %s", __func__, 211 strerror(errno)); 212 return false; 213 } 214 215 /* ask features */ 216 api_struct.api = UFFD_API; 217 api_struct.features = 0; 218 if (ioctl(ufd, UFFDIO_API, &api_struct)) { 219 error_report("%s: UFFDIO_API failed: %s", __func__, 220 strerror(errno)); 221 ret = false; 222 goto release_ufd; 223 } 224 225 *features = api_struct.features; 226 227 release_ufd: 228 close(ufd); 229 return ret; 230 } 231 232 /** 233 * request_ufd_features: this function should be called only once on a newly 234 * opened ufd, subsequent calls will lead to error. 235 * 236 * Returns: true on succes 237 * 238 * @ufd: fd obtained from userfaultfd syscall 239 * @features: bit mask see UFFD_API_FEATURES 240 */ 241 static bool request_ufd_features(int ufd, uint64_t features) 242 { 243 struct uffdio_api api_struct = {0}; 244 uint64_t ioctl_mask; 245 246 api_struct.api = UFFD_API; 247 api_struct.features = features; 248 if (ioctl(ufd, UFFDIO_API, &api_struct)) { 249 error_report("%s failed: UFFDIO_API failed: %s", __func__, 250 strerror(errno)); 251 return false; 252 } 253 254 ioctl_mask = (__u64)1 << _UFFDIO_REGISTER | 255 (__u64)1 << _UFFDIO_UNREGISTER; 256 if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { 257 error_report("Missing userfault features: %" PRIx64, 258 (uint64_t)(~api_struct.ioctls & ioctl_mask)); 259 return false; 260 } 261 262 return true; 263 } 264 265 static bool ufd_check_and_apply(int ufd, MigrationIncomingState *mis) 266 { 267 uint64_t asked_features = 0; 268 static uint64_t supported_features; 269 270 /* 271 * it's not possible to 272 * request UFFD_API twice per one fd 273 * userfault fd features is persistent 274 */ 275 if (!supported_features) { 276 if (!receive_ufd_features(&supported_features)) { 277 error_report("%s failed", __func__); 278 return false; 279 } 280 } 281 282 #ifdef UFFD_FEATURE_THREAD_ID 283 if (migrate_postcopy_blocktime() && mis && 284 UFFD_FEATURE_THREAD_ID & supported_features) { 285 /* kernel supports that feature */ 286 /* don't create blocktime_context if it exists */ 287 if (!mis->blocktime_ctx) { 288 mis->blocktime_ctx = blocktime_context_new(); 289 } 290 291 asked_features |= UFFD_FEATURE_THREAD_ID; 292 } 293 #endif 294 295 /* 296 * request features, even if asked_features is 0, due to 297 * kernel expects UFFD_API before UFFDIO_REGISTER, per 298 * userfault file descriptor 299 */ 300 if (!request_ufd_features(ufd, asked_features)) { 301 error_report("%s failed: features %" PRIu64, __func__, 302 asked_features); 303 return false; 304 } 305 306 if (getpagesize() != ram_pagesize_summary()) { 307 bool have_hp = false; 308 /* We've got a huge page */ 309 #ifdef UFFD_FEATURE_MISSING_HUGETLBFS 310 have_hp = supported_features & UFFD_FEATURE_MISSING_HUGETLBFS; 311 #endif 312 if (!have_hp) { 313 error_report("Userfault on this host does not support huge pages"); 314 return false; 315 } 316 } 317 return true; 318 } 319 320 /* Callback from postcopy_ram_supported_by_host block iterator. 321 */ 322 static int test_ramblock_postcopiable(RAMBlock *rb, void *opaque) 323 { 324 const char *block_name = qemu_ram_get_idstr(rb); 325 ram_addr_t length = qemu_ram_get_used_length(rb); 326 size_t pagesize = qemu_ram_pagesize(rb); 327 328 if (length % pagesize) { 329 error_report("Postcopy requires RAM blocks to be a page size multiple," 330 " block %s is 0x" RAM_ADDR_FMT " bytes with a " 331 "page size of 0x%zx", block_name, length, pagesize); 332 return 1; 333 } 334 return 0; 335 } 336 337 /* 338 * Note: This has the side effect of munlock'ing all of RAM, that's 339 * normally fine since if the postcopy succeeds it gets turned back on at the 340 * end. 341 */ 342 bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) 343 { 344 long pagesize = getpagesize(); 345 int ufd = -1; 346 bool ret = false; /* Error unless we change it */ 347 void *testarea = NULL; 348 struct uffdio_register reg_struct; 349 struct uffdio_range range_struct; 350 uint64_t feature_mask; 351 Error *local_err = NULL; 352 353 if (qemu_target_page_size() > pagesize) { 354 error_report("Target page size bigger than host page size"); 355 goto out; 356 } 357 358 ufd = syscall(__NR_userfaultfd, O_CLOEXEC); 359 if (ufd == -1) { 360 error_report("%s: userfaultfd not available: %s", __func__, 361 strerror(errno)); 362 goto out; 363 } 364 365 /* Give devices a chance to object */ 366 if (postcopy_notify(POSTCOPY_NOTIFY_PROBE, &local_err)) { 367 error_report_err(local_err); 368 goto out; 369 } 370 371 /* Version and features check */ 372 if (!ufd_check_and_apply(ufd, mis)) { 373 goto out; 374 } 375 376 /* We don't support postcopy with shared RAM yet */ 377 if (foreach_not_ignored_block(test_ramblock_postcopiable, NULL)) { 378 goto out; 379 } 380 381 /* 382 * userfault and mlock don't go together; we'll put it back later if 383 * it was enabled. 384 */ 385 if (munlockall()) { 386 error_report("%s: munlockall: %s", __func__, strerror(errno)); 387 return -1; 388 } 389 390 /* 391 * We need to check that the ops we need are supported on anon memory 392 * To do that we need to register a chunk and see the flags that 393 * are returned. 394 */ 395 testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | 396 MAP_ANONYMOUS, -1, 0); 397 if (testarea == MAP_FAILED) { 398 error_report("%s: Failed to map test area: %s", __func__, 399 strerror(errno)); 400 goto out; 401 } 402 g_assert(((size_t)testarea & (pagesize-1)) == 0); 403 404 reg_struct.range.start = (uintptr_t)testarea; 405 reg_struct.range.len = pagesize; 406 reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; 407 408 if (ioctl(ufd, UFFDIO_REGISTER, ®_struct)) { 409 error_report("%s userfault register: %s", __func__, strerror(errno)); 410 goto out; 411 } 412 413 range_struct.start = (uintptr_t)testarea; 414 range_struct.len = pagesize; 415 if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) { 416 error_report("%s userfault unregister: %s", __func__, strerror(errno)); 417 goto out; 418 } 419 420 feature_mask = (__u64)1 << _UFFDIO_WAKE | 421 (__u64)1 << _UFFDIO_COPY | 422 (__u64)1 << _UFFDIO_ZEROPAGE; 423 if ((reg_struct.ioctls & feature_mask) != feature_mask) { 424 error_report("Missing userfault map features: %" PRIx64, 425 (uint64_t)(~reg_struct.ioctls & feature_mask)); 426 goto out; 427 } 428 429 /* Success! */ 430 ret = true; 431 out: 432 if (testarea) { 433 munmap(testarea, pagesize); 434 } 435 if (ufd != -1) { 436 close(ufd); 437 } 438 return ret; 439 } 440 441 /* 442 * Setup an area of RAM so that it *can* be used for postcopy later; this 443 * must be done right at the start prior to pre-copy. 444 * opaque should be the MIS. 445 */ 446 static int init_range(RAMBlock *rb, void *opaque) 447 { 448 const char *block_name = qemu_ram_get_idstr(rb); 449 void *host_addr = qemu_ram_get_host_addr(rb); 450 ram_addr_t offset = qemu_ram_get_offset(rb); 451 ram_addr_t length = qemu_ram_get_used_length(rb); 452 trace_postcopy_init_range(block_name, host_addr, offset, length); 453 454 /* 455 * We need the whole of RAM to be truly empty for postcopy, so things 456 * like ROMs and any data tables built during init must be zero'd 457 * - we're going to get the copy from the source anyway. 458 * (Precopy will just overwrite this data, so doesn't need the discard) 459 */ 460 if (ram_discard_range(block_name, 0, length)) { 461 return -1; 462 } 463 464 return 0; 465 } 466 467 /* 468 * At the end of migration, undo the effects of init_range 469 * opaque should be the MIS. 470 */ 471 static int cleanup_range(RAMBlock *rb, void *opaque) 472 { 473 const char *block_name = qemu_ram_get_idstr(rb); 474 void *host_addr = qemu_ram_get_host_addr(rb); 475 ram_addr_t offset = qemu_ram_get_offset(rb); 476 ram_addr_t length = qemu_ram_get_used_length(rb); 477 MigrationIncomingState *mis = opaque; 478 struct uffdio_range range_struct; 479 trace_postcopy_cleanup_range(block_name, host_addr, offset, length); 480 481 /* 482 * We turned off hugepage for the precopy stage with postcopy enabled 483 * we can turn it back on now. 484 */ 485 qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE); 486 487 /* 488 * We can also turn off userfault now since we should have all the 489 * pages. It can be useful to leave it on to debug postcopy 490 * if you're not sure it's always getting every page. 491 */ 492 range_struct.start = (uintptr_t)host_addr; 493 range_struct.len = length; 494 495 if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { 496 error_report("%s: userfault unregister %s", __func__, strerror(errno)); 497 498 return -1; 499 } 500 501 return 0; 502 } 503 504 /* 505 * Initialise postcopy-ram, setting the RAM to a state where we can go into 506 * postcopy later; must be called prior to any precopy. 507 * called from arch_init's similarly named ram_postcopy_incoming_init 508 */ 509 int postcopy_ram_incoming_init(MigrationIncomingState *mis) 510 { 511 if (foreach_not_ignored_block(init_range, NULL)) { 512 return -1; 513 } 514 515 return 0; 516 } 517 518 /* 519 * Manage a single vote to the QEMU balloon inhibitor for all postcopy usage, 520 * last caller wins. 521 */ 522 static void postcopy_balloon_inhibit(bool state) 523 { 524 static bool cur_state = false; 525 526 if (state != cur_state) { 527 qemu_balloon_inhibit(state); 528 cur_state = state; 529 } 530 } 531 532 /* 533 * At the end of a migration where postcopy_ram_incoming_init was called. 534 */ 535 int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) 536 { 537 trace_postcopy_ram_incoming_cleanup_entry(); 538 539 if (mis->have_fault_thread) { 540 Error *local_err = NULL; 541 542 /* Let the fault thread quit */ 543 atomic_set(&mis->fault_thread_quit, 1); 544 postcopy_fault_thread_notify(mis); 545 trace_postcopy_ram_incoming_cleanup_join(); 546 qemu_thread_join(&mis->fault_thread); 547 548 if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) { 549 error_report_err(local_err); 550 return -1; 551 } 552 553 if (foreach_not_ignored_block(cleanup_range, mis)) { 554 return -1; 555 } 556 557 trace_postcopy_ram_incoming_cleanup_closeuf(); 558 close(mis->userfault_fd); 559 close(mis->userfault_event_fd); 560 mis->have_fault_thread = false; 561 } 562 563 postcopy_balloon_inhibit(false); 564 565 if (enable_mlock) { 566 if (os_mlock() < 0) { 567 error_report("mlock: %s", strerror(errno)); 568 /* 569 * It doesn't feel right to fail at this point, we have a valid 570 * VM state. 571 */ 572 } 573 } 574 575 postcopy_state_set(POSTCOPY_INCOMING_END); 576 577 if (mis->postcopy_tmp_page) { 578 munmap(mis->postcopy_tmp_page, mis->largest_page_size); 579 mis->postcopy_tmp_page = NULL; 580 } 581 if (mis->postcopy_tmp_zero_page) { 582 munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size); 583 mis->postcopy_tmp_zero_page = NULL; 584 } 585 trace_postcopy_ram_incoming_cleanup_blocktime( 586 get_postcopy_total_blocktime()); 587 588 trace_postcopy_ram_incoming_cleanup_exit(); 589 return 0; 590 } 591 592 /* 593 * Disable huge pages on an area 594 */ 595 static int nhp_range(RAMBlock *rb, void *opaque) 596 { 597 const char *block_name = qemu_ram_get_idstr(rb); 598 void *host_addr = qemu_ram_get_host_addr(rb); 599 ram_addr_t offset = qemu_ram_get_offset(rb); 600 ram_addr_t length = qemu_ram_get_used_length(rb); 601 trace_postcopy_nhp_range(block_name, host_addr, offset, length); 602 603 /* 604 * Before we do discards we need to ensure those discards really 605 * do delete areas of the page, even if THP thinks a hugepage would 606 * be a good idea, so force hugepages off. 607 */ 608 qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE); 609 610 return 0; 611 } 612 613 /* 614 * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard 615 * however leaving it until after precopy means that most of the precopy 616 * data is still THPd 617 */ 618 int postcopy_ram_prepare_discard(MigrationIncomingState *mis) 619 { 620 if (foreach_not_ignored_block(nhp_range, mis)) { 621 return -1; 622 } 623 624 postcopy_state_set(POSTCOPY_INCOMING_DISCARD); 625 626 return 0; 627 } 628 629 /* 630 * Mark the given area of RAM as requiring notification to unwritten areas 631 * Used as a callback on foreach_not_ignored_block. 632 * host_addr: Base of area to mark 633 * offset: Offset in the whole ram arena 634 * length: Length of the section 635 * opaque: MigrationIncomingState pointer 636 * Returns 0 on success 637 */ 638 static int ram_block_enable_notify(RAMBlock *rb, void *opaque) 639 { 640 MigrationIncomingState *mis = opaque; 641 struct uffdio_register reg_struct; 642 643 reg_struct.range.start = (uintptr_t)qemu_ram_get_host_addr(rb); 644 reg_struct.range.len = qemu_ram_get_used_length(rb); 645 reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; 646 647 /* Now tell our userfault_fd that it's responsible for this area */ 648 if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { 649 error_report("%s userfault register: %s", __func__, strerror(errno)); 650 return -1; 651 } 652 if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) { 653 error_report("%s userfault: Region doesn't support COPY", __func__); 654 return -1; 655 } 656 if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) { 657 qemu_ram_set_uf_zeroable(rb); 658 } 659 660 return 0; 661 } 662 663 int postcopy_wake_shared(struct PostCopyFD *pcfd, 664 uint64_t client_addr, 665 RAMBlock *rb) 666 { 667 size_t pagesize = qemu_ram_pagesize(rb); 668 struct uffdio_range range; 669 int ret; 670 trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb)); 671 range.start = client_addr & ~(pagesize - 1); 672 range.len = pagesize; 673 ret = ioctl(pcfd->fd, UFFDIO_WAKE, &range); 674 if (ret) { 675 error_report("%s: Failed to wake: %zx in %s (%s)", 676 __func__, (size_t)client_addr, qemu_ram_get_idstr(rb), 677 strerror(errno)); 678 } 679 return ret; 680 } 681 682 /* 683 * Callback from shared fault handlers to ask for a page, 684 * the page must be specified by a RAMBlock and an offset in that rb 685 * Note: Only for use by shared fault handlers (in fault thread) 686 */ 687 int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, 688 uint64_t client_addr, uint64_t rb_offset) 689 { 690 size_t pagesize = qemu_ram_pagesize(rb); 691 uint64_t aligned_rbo = rb_offset & ~(pagesize - 1); 692 MigrationIncomingState *mis = migration_incoming_get_current(); 693 694 trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb), 695 rb_offset); 696 if (ramblock_recv_bitmap_test_byte_offset(rb, aligned_rbo)) { 697 trace_postcopy_request_shared_page_present(pcfd->idstr, 698 qemu_ram_get_idstr(rb), rb_offset); 699 return postcopy_wake_shared(pcfd, client_addr, rb); 700 } 701 if (rb != mis->last_rb) { 702 mis->last_rb = rb; 703 migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb), 704 aligned_rbo, pagesize); 705 } else { 706 /* Save some space */ 707 migrate_send_rp_req_pages(mis, NULL, aligned_rbo, pagesize); 708 } 709 return 0; 710 } 711 712 static int get_mem_fault_cpu_index(uint32_t pid) 713 { 714 CPUState *cpu_iter; 715 716 CPU_FOREACH(cpu_iter) { 717 if (cpu_iter->thread_id == pid) { 718 trace_get_mem_fault_cpu_index(cpu_iter->cpu_index, pid); 719 return cpu_iter->cpu_index; 720 } 721 } 722 trace_get_mem_fault_cpu_index(-1, pid); 723 return -1; 724 } 725 726 static uint32_t get_low_time_offset(PostcopyBlocktimeContext *dc) 727 { 728 int64_t start_time_offset = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - 729 dc->start_time; 730 return start_time_offset < 1 ? 1 : start_time_offset & UINT32_MAX; 731 } 732 733 /* 734 * This function is being called when pagefault occurs. It 735 * tracks down vCPU blocking time. 736 * 737 * @addr: faulted host virtual address 738 * @ptid: faulted process thread id 739 * @rb: ramblock appropriate to addr 740 */ 741 static void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid, 742 RAMBlock *rb) 743 { 744 int cpu, already_received; 745 MigrationIncomingState *mis = migration_incoming_get_current(); 746 PostcopyBlocktimeContext *dc = mis->blocktime_ctx; 747 uint32_t low_time_offset; 748 749 if (!dc || ptid == 0) { 750 return; 751 } 752 cpu = get_mem_fault_cpu_index(ptid); 753 if (cpu < 0) { 754 return; 755 } 756 757 low_time_offset = get_low_time_offset(dc); 758 if (dc->vcpu_addr[cpu] == 0) { 759 atomic_inc(&dc->smp_cpus_down); 760 } 761 762 atomic_xchg(&dc->last_begin, low_time_offset); 763 atomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset); 764 atomic_xchg(&dc->vcpu_addr[cpu], addr); 765 766 /* check it here, not at the begining of the function, 767 * due to, check could accur early than bitmap_set in 768 * qemu_ufd_copy_ioctl */ 769 already_received = ramblock_recv_bitmap_test(rb, (void *)addr); 770 if (already_received) { 771 atomic_xchg(&dc->vcpu_addr[cpu], 0); 772 atomic_xchg(&dc->page_fault_vcpu_time[cpu], 0); 773 atomic_dec(&dc->smp_cpus_down); 774 } 775 trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu], 776 cpu, already_received); 777 } 778 779 /* 780 * This function just provide calculated blocktime per cpu and trace it. 781 * Total blocktime is calculated in mark_postcopy_blocktime_end. 782 * 783 * 784 * Assume we have 3 CPU 785 * 786 * S1 E1 S1 E1 787 * -----***********------------xxx***************------------------------> CPU1 788 * 789 * S2 E2 790 * ------------****************xxx---------------------------------------> CPU2 791 * 792 * S3 E3 793 * ------------------------****xxx********-------------------------------> CPU3 794 * 795 * We have sequence S1,S2,E1,S3,S1,E2,E3,E1 796 * S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3 797 * S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 - 798 * it's a part of total blocktime. 799 * S1 - here is last_begin 800 * Legend of the picture is following: 801 * * - means blocktime per vCPU 802 * x - means overlapped blocktime (total blocktime) 803 * 804 * @addr: host virtual address 805 */ 806 static void mark_postcopy_blocktime_end(uintptr_t addr) 807 { 808 MigrationIncomingState *mis = migration_incoming_get_current(); 809 PostcopyBlocktimeContext *dc = mis->blocktime_ctx; 810 int i, affected_cpu = 0; 811 bool vcpu_total_blocktime = false; 812 uint32_t read_vcpu_time, low_time_offset; 813 814 if (!dc) { 815 return; 816 } 817 818 low_time_offset = get_low_time_offset(dc); 819 /* lookup cpu, to clear it, 820 * that algorithm looks straighforward, but it's not 821 * optimal, more optimal algorithm is keeping tree or hash 822 * where key is address value is a list of */ 823 for (i = 0; i < smp_cpus; i++) { 824 uint32_t vcpu_blocktime = 0; 825 826 read_vcpu_time = atomic_fetch_add(&dc->page_fault_vcpu_time[i], 0); 827 if (atomic_fetch_add(&dc->vcpu_addr[i], 0) != addr || 828 read_vcpu_time == 0) { 829 continue; 830 } 831 atomic_xchg(&dc->vcpu_addr[i], 0); 832 vcpu_blocktime = low_time_offset - read_vcpu_time; 833 affected_cpu += 1; 834 /* we need to know is that mark_postcopy_end was due to 835 * faulted page, another possible case it's prefetched 836 * page and in that case we shouldn't be here */ 837 if (!vcpu_total_blocktime && 838 atomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) { 839 vcpu_total_blocktime = true; 840 } 841 /* continue cycle, due to one page could affect several vCPUs */ 842 dc->vcpu_blocktime[i] += vcpu_blocktime; 843 } 844 845 atomic_sub(&dc->smp_cpus_down, affected_cpu); 846 if (vcpu_total_blocktime) { 847 dc->total_blocktime += low_time_offset - atomic_fetch_add( 848 &dc->last_begin, 0); 849 } 850 trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime, 851 affected_cpu); 852 } 853 854 static bool postcopy_pause_fault_thread(MigrationIncomingState *mis) 855 { 856 trace_postcopy_pause_fault_thread(); 857 858 qemu_sem_wait(&mis->postcopy_pause_sem_fault); 859 860 trace_postcopy_pause_fault_thread_continued(); 861 862 return true; 863 } 864 865 /* 866 * Handle faults detected by the USERFAULT markings 867 */ 868 static void *postcopy_ram_fault_thread(void *opaque) 869 { 870 MigrationIncomingState *mis = opaque; 871 struct uffd_msg msg; 872 int ret; 873 size_t index; 874 RAMBlock *rb = NULL; 875 876 trace_postcopy_ram_fault_thread_entry(); 877 rcu_register_thread(); 878 mis->last_rb = NULL; /* last RAMBlock we sent part of */ 879 qemu_sem_post(&mis->fault_thread_sem); 880 881 struct pollfd *pfd; 882 size_t pfd_len = 2 + mis->postcopy_remote_fds->len; 883 884 pfd = g_new0(struct pollfd, pfd_len); 885 886 pfd[0].fd = mis->userfault_fd; 887 pfd[0].events = POLLIN; 888 pfd[1].fd = mis->userfault_event_fd; 889 pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */ 890 trace_postcopy_ram_fault_thread_fds_core(pfd[0].fd, pfd[1].fd); 891 for (index = 0; index < mis->postcopy_remote_fds->len; index++) { 892 struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds, 893 struct PostCopyFD, index); 894 pfd[2 + index].fd = pcfd->fd; 895 pfd[2 + index].events = POLLIN; 896 trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr, 897 pcfd->fd); 898 } 899 900 while (true) { 901 ram_addr_t rb_offset; 902 int poll_result; 903 904 /* 905 * We're mainly waiting for the kernel to give us a faulting HVA, 906 * however we can be told to quit via userfault_quit_fd which is 907 * an eventfd 908 */ 909 910 poll_result = poll(pfd, pfd_len, -1 /* Wait forever */); 911 if (poll_result == -1) { 912 error_report("%s: userfault poll: %s", __func__, strerror(errno)); 913 break; 914 } 915 916 if (!mis->to_src_file) { 917 /* 918 * Possibly someone tells us that the return path is 919 * broken already using the event. We should hold until 920 * the channel is rebuilt. 921 */ 922 if (postcopy_pause_fault_thread(mis)) { 923 mis->last_rb = NULL; 924 /* Continue to read the userfaultfd */ 925 } else { 926 error_report("%s: paused but don't allow to continue", 927 __func__); 928 break; 929 } 930 } 931 932 if (pfd[1].revents) { 933 uint64_t tmp64 = 0; 934 935 /* Consume the signal */ 936 if (read(mis->userfault_event_fd, &tmp64, 8) != 8) { 937 /* Nothing obviously nicer than posting this error. */ 938 error_report("%s: read() failed", __func__); 939 } 940 941 if (atomic_read(&mis->fault_thread_quit)) { 942 trace_postcopy_ram_fault_thread_quit(); 943 break; 944 } 945 } 946 947 if (pfd[0].revents) { 948 poll_result--; 949 ret = read(mis->userfault_fd, &msg, sizeof(msg)); 950 if (ret != sizeof(msg)) { 951 if (errno == EAGAIN) { 952 /* 953 * if a wake up happens on the other thread just after 954 * the poll, there is nothing to read. 955 */ 956 continue; 957 } 958 if (ret < 0) { 959 error_report("%s: Failed to read full userfault " 960 "message: %s", 961 __func__, strerror(errno)); 962 break; 963 } else { 964 error_report("%s: Read %d bytes from userfaultfd " 965 "expected %zd", 966 __func__, ret, sizeof(msg)); 967 break; /* Lost alignment, don't know what we'd read next */ 968 } 969 } 970 if (msg.event != UFFD_EVENT_PAGEFAULT) { 971 error_report("%s: Read unexpected event %ud from userfaultfd", 972 __func__, msg.event); 973 continue; /* It's not a page fault, shouldn't happen */ 974 } 975 976 rb = qemu_ram_block_from_host( 977 (void *)(uintptr_t)msg.arg.pagefault.address, 978 true, &rb_offset); 979 if (!rb) { 980 error_report("postcopy_ram_fault_thread: Fault outside guest: %" 981 PRIx64, (uint64_t)msg.arg.pagefault.address); 982 break; 983 } 984 985 rb_offset &= ~(qemu_ram_pagesize(rb) - 1); 986 trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address, 987 qemu_ram_get_idstr(rb), 988 rb_offset, 989 msg.arg.pagefault.feat.ptid); 990 mark_postcopy_blocktime_begin( 991 (uintptr_t)(msg.arg.pagefault.address), 992 msg.arg.pagefault.feat.ptid, rb); 993 994 retry: 995 /* 996 * Send the request to the source - we want to request one 997 * of our host page sizes (which is >= TPS) 998 */ 999 if (rb != mis->last_rb) { 1000 mis->last_rb = rb; 1001 ret = migrate_send_rp_req_pages(mis, 1002 qemu_ram_get_idstr(rb), 1003 rb_offset, 1004 qemu_ram_pagesize(rb)); 1005 } else { 1006 /* Save some space */ 1007 ret = migrate_send_rp_req_pages(mis, 1008 NULL, 1009 rb_offset, 1010 qemu_ram_pagesize(rb)); 1011 } 1012 1013 if (ret) { 1014 /* May be network failure, try to wait for recovery */ 1015 if (ret == -EIO && postcopy_pause_fault_thread(mis)) { 1016 /* We got reconnected somehow, try to continue */ 1017 mis->last_rb = NULL; 1018 goto retry; 1019 } else { 1020 /* This is a unavoidable fault */ 1021 error_report("%s: migrate_send_rp_req_pages() get %d", 1022 __func__, ret); 1023 break; 1024 } 1025 } 1026 } 1027 1028 /* Now handle any requests from external processes on shared memory */ 1029 /* TODO: May need to handle devices deregistering during postcopy */ 1030 for (index = 2; index < pfd_len && poll_result; index++) { 1031 if (pfd[index].revents) { 1032 struct PostCopyFD *pcfd = 1033 &g_array_index(mis->postcopy_remote_fds, 1034 struct PostCopyFD, index - 2); 1035 1036 poll_result--; 1037 if (pfd[index].revents & POLLERR) { 1038 error_report("%s: POLLERR on poll %zd fd=%d", 1039 __func__, index, pcfd->fd); 1040 pfd[index].events = 0; 1041 continue; 1042 } 1043 1044 ret = read(pcfd->fd, &msg, sizeof(msg)); 1045 if (ret != sizeof(msg)) { 1046 if (errno == EAGAIN) { 1047 /* 1048 * if a wake up happens on the other thread just after 1049 * the poll, there is nothing to read. 1050 */ 1051 continue; 1052 } 1053 if (ret < 0) { 1054 error_report("%s: Failed to read full userfault " 1055 "message: %s (shared) revents=%d", 1056 __func__, strerror(errno), 1057 pfd[index].revents); 1058 /*TODO: Could just disable this sharer */ 1059 break; 1060 } else { 1061 error_report("%s: Read %d bytes from userfaultfd " 1062 "expected %zd (shared)", 1063 __func__, ret, sizeof(msg)); 1064 /*TODO: Could just disable this sharer */ 1065 break; /*Lost alignment,don't know what we'd read next*/ 1066 } 1067 } 1068 if (msg.event != UFFD_EVENT_PAGEFAULT) { 1069 error_report("%s: Read unexpected event %ud " 1070 "from userfaultfd (shared)", 1071 __func__, msg.event); 1072 continue; /* It's not a page fault, shouldn't happen */ 1073 } 1074 /* Call the device handler registered with us */ 1075 ret = pcfd->handler(pcfd, &msg); 1076 if (ret) { 1077 error_report("%s: Failed to resolve shared fault on %zd/%s", 1078 __func__, index, pcfd->idstr); 1079 /* TODO: Fail? Disable this sharer? */ 1080 } 1081 } 1082 } 1083 } 1084 rcu_unregister_thread(); 1085 trace_postcopy_ram_fault_thread_exit(); 1086 g_free(pfd); 1087 return NULL; 1088 } 1089 1090 int postcopy_ram_enable_notify(MigrationIncomingState *mis) 1091 { 1092 /* Open the fd for the kernel to give us userfaults */ 1093 mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); 1094 if (mis->userfault_fd == -1) { 1095 error_report("%s: Failed to open userfault fd: %s", __func__, 1096 strerror(errno)); 1097 return -1; 1098 } 1099 1100 /* 1101 * Although the host check already tested the API, we need to 1102 * do the check again as an ABI handshake on the new fd. 1103 */ 1104 if (!ufd_check_and_apply(mis->userfault_fd, mis)) { 1105 return -1; 1106 } 1107 1108 /* Now an eventfd we use to tell the fault-thread to quit */ 1109 mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC); 1110 if (mis->userfault_event_fd == -1) { 1111 error_report("%s: Opening userfault_event_fd: %s", __func__, 1112 strerror(errno)); 1113 close(mis->userfault_fd); 1114 return -1; 1115 } 1116 1117 qemu_sem_init(&mis->fault_thread_sem, 0); 1118 qemu_thread_create(&mis->fault_thread, "postcopy/fault", 1119 postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE); 1120 qemu_sem_wait(&mis->fault_thread_sem); 1121 qemu_sem_destroy(&mis->fault_thread_sem); 1122 mis->have_fault_thread = true; 1123 1124 /* Mark so that we get notified of accesses to unwritten areas */ 1125 if (foreach_not_ignored_block(ram_block_enable_notify, mis)) { 1126 error_report("ram_block_enable_notify failed"); 1127 return -1; 1128 } 1129 1130 /* 1131 * Ballooning can mark pages as absent while we're postcopying 1132 * that would cause false userfaults. 1133 */ 1134 postcopy_balloon_inhibit(true); 1135 1136 trace_postcopy_ram_enable_notify(); 1137 1138 return 0; 1139 } 1140 1141 static int qemu_ufd_copy_ioctl(int userfault_fd, void *host_addr, 1142 void *from_addr, uint64_t pagesize, RAMBlock *rb) 1143 { 1144 int ret; 1145 if (from_addr) { 1146 struct uffdio_copy copy_struct; 1147 copy_struct.dst = (uint64_t)(uintptr_t)host_addr; 1148 copy_struct.src = (uint64_t)(uintptr_t)from_addr; 1149 copy_struct.len = pagesize; 1150 copy_struct.mode = 0; 1151 ret = ioctl(userfault_fd, UFFDIO_COPY, ©_struct); 1152 } else { 1153 struct uffdio_zeropage zero_struct; 1154 zero_struct.range.start = (uint64_t)(uintptr_t)host_addr; 1155 zero_struct.range.len = pagesize; 1156 zero_struct.mode = 0; 1157 ret = ioctl(userfault_fd, UFFDIO_ZEROPAGE, &zero_struct); 1158 } 1159 if (!ret) { 1160 ramblock_recv_bitmap_set_range(rb, host_addr, 1161 pagesize / qemu_target_page_size()); 1162 mark_postcopy_blocktime_end((uintptr_t)host_addr); 1163 1164 } 1165 return ret; 1166 } 1167 1168 int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset) 1169 { 1170 int i; 1171 MigrationIncomingState *mis = migration_incoming_get_current(); 1172 GArray *pcrfds = mis->postcopy_remote_fds; 1173 1174 for (i = 0; i < pcrfds->len; i++) { 1175 struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); 1176 int ret = cur->waker(cur, rb, offset); 1177 if (ret) { 1178 return ret; 1179 } 1180 } 1181 return 0; 1182 } 1183 1184 /* 1185 * Place a host page (from) at (host) atomically 1186 * returns 0 on success 1187 */ 1188 int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, 1189 RAMBlock *rb) 1190 { 1191 size_t pagesize = qemu_ram_pagesize(rb); 1192 1193 /* copy also acks to the kernel waking the stalled thread up 1194 * TODO: We can inhibit that ack and only do it if it was requested 1195 * which would be slightly cheaper, but we'd have to be careful 1196 * of the order of updating our page state. 1197 */ 1198 if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, from, pagesize, rb)) { 1199 int e = errno; 1200 error_report("%s: %s copy host: %p from: %p (size: %zd)", 1201 __func__, strerror(e), host, from, pagesize); 1202 1203 return -e; 1204 } 1205 1206 trace_postcopy_place_page(host); 1207 return postcopy_notify_shared_wake(rb, 1208 qemu_ram_block_host_offset(rb, host)); 1209 } 1210 1211 /* 1212 * Place a zero page at (host) atomically 1213 * returns 0 on success 1214 */ 1215 int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, 1216 RAMBlock *rb) 1217 { 1218 size_t pagesize = qemu_ram_pagesize(rb); 1219 trace_postcopy_place_page_zero(host); 1220 1221 /* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE 1222 * but it's not available for everything (e.g. hugetlbpages) 1223 */ 1224 if (qemu_ram_is_uf_zeroable(rb)) { 1225 if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, pagesize, rb)) { 1226 int e = errno; 1227 error_report("%s: %s zero host: %p", 1228 __func__, strerror(e), host); 1229 1230 return -e; 1231 } 1232 return postcopy_notify_shared_wake(rb, 1233 qemu_ram_block_host_offset(rb, 1234 host)); 1235 } else { 1236 /* The kernel can't use UFFDIO_ZEROPAGE for hugepages */ 1237 if (!mis->postcopy_tmp_zero_page) { 1238 mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size, 1239 PROT_READ | PROT_WRITE, 1240 MAP_PRIVATE | MAP_ANONYMOUS, 1241 -1, 0); 1242 if (mis->postcopy_tmp_zero_page == MAP_FAILED) { 1243 int e = errno; 1244 mis->postcopy_tmp_zero_page = NULL; 1245 error_report("%s: %s mapping large zero page", 1246 __func__, strerror(e)); 1247 return -e; 1248 } 1249 memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size); 1250 } 1251 return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, 1252 rb); 1253 } 1254 } 1255 1256 /* 1257 * Returns a target page of memory that can be mapped at a later point in time 1258 * using postcopy_place_page 1259 * The same address is used repeatedly, postcopy_place_page just takes the 1260 * backing page away. 1261 * Returns: Pointer to allocated page 1262 * 1263 */ 1264 void *postcopy_get_tmp_page(MigrationIncomingState *mis) 1265 { 1266 if (!mis->postcopy_tmp_page) { 1267 mis->postcopy_tmp_page = mmap(NULL, mis->largest_page_size, 1268 PROT_READ | PROT_WRITE, MAP_PRIVATE | 1269 MAP_ANONYMOUS, -1, 0); 1270 if (mis->postcopy_tmp_page == MAP_FAILED) { 1271 mis->postcopy_tmp_page = NULL; 1272 error_report("%s: %s", __func__, strerror(errno)); 1273 return NULL; 1274 } 1275 } 1276 1277 return mis->postcopy_tmp_page; 1278 } 1279 1280 #else 1281 /* No target OS support, stubs just fail */ 1282 void fill_destination_postcopy_migration_info(MigrationInfo *info) 1283 { 1284 } 1285 1286 bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) 1287 { 1288 error_report("%s: No OS support", __func__); 1289 return false; 1290 } 1291 1292 int postcopy_ram_incoming_init(MigrationIncomingState *mis) 1293 { 1294 error_report("postcopy_ram_incoming_init: No OS support"); 1295 return -1; 1296 } 1297 1298 int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) 1299 { 1300 assert(0); 1301 return -1; 1302 } 1303 1304 int postcopy_ram_prepare_discard(MigrationIncomingState *mis) 1305 { 1306 assert(0); 1307 return -1; 1308 } 1309 1310 int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, 1311 uint64_t client_addr, uint64_t rb_offset) 1312 { 1313 assert(0); 1314 return -1; 1315 } 1316 1317 int postcopy_ram_enable_notify(MigrationIncomingState *mis) 1318 { 1319 assert(0); 1320 return -1; 1321 } 1322 1323 int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, 1324 RAMBlock *rb) 1325 { 1326 assert(0); 1327 return -1; 1328 } 1329 1330 int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, 1331 RAMBlock *rb) 1332 { 1333 assert(0); 1334 return -1; 1335 } 1336 1337 void *postcopy_get_tmp_page(MigrationIncomingState *mis) 1338 { 1339 assert(0); 1340 return NULL; 1341 } 1342 1343 int postcopy_wake_shared(struct PostCopyFD *pcfd, 1344 uint64_t client_addr, 1345 RAMBlock *rb) 1346 { 1347 assert(0); 1348 return -1; 1349 } 1350 #endif 1351 1352 /* ------------------------------------------------------------------------- */ 1353 1354 void postcopy_fault_thread_notify(MigrationIncomingState *mis) 1355 { 1356 uint64_t tmp64 = 1; 1357 1358 /* 1359 * Wakeup the fault_thread. It's an eventfd that should currently 1360 * be at 0, we're going to increment it to 1 1361 */ 1362 if (write(mis->userfault_event_fd, &tmp64, 8) != 8) { 1363 /* Not much we can do here, but may as well report it */ 1364 error_report("%s: incrementing failed: %s", __func__, 1365 strerror(errno)); 1366 } 1367 } 1368 1369 /** 1370 * postcopy_discard_send_init: Called at the start of each RAMBlock before 1371 * asking to discard individual ranges. 1372 * 1373 * @ms: The current migration state. 1374 * @offset: the bitmap offset of the named RAMBlock in the migration 1375 * bitmap. 1376 * @name: RAMBlock that discards will operate on. 1377 * 1378 * returns: a new PDS. 1379 */ 1380 PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms, 1381 const char *name) 1382 { 1383 PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState)); 1384 1385 if (res) { 1386 res->ramblock_name = name; 1387 } 1388 1389 return res; 1390 } 1391 1392 /** 1393 * postcopy_discard_send_range: Called by the bitmap code for each chunk to 1394 * discard. May send a discard message, may just leave it queued to 1395 * be sent later. 1396 * 1397 * @ms: Current migration state. 1398 * @pds: Structure initialised by postcopy_discard_send_init(). 1399 * @start,@length: a range of pages in the migration bitmap in the 1400 * RAM block passed to postcopy_discard_send_init() (length=1 is one page) 1401 */ 1402 void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds, 1403 unsigned long start, unsigned long length) 1404 { 1405 size_t tp_size = qemu_target_page_size(); 1406 /* Convert to byte offsets within the RAM block */ 1407 pds->start_list[pds->cur_entry] = start * tp_size; 1408 pds->length_list[pds->cur_entry] = length * tp_size; 1409 trace_postcopy_discard_send_range(pds->ramblock_name, start, length); 1410 pds->cur_entry++; 1411 pds->nsentwords++; 1412 1413 if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) { 1414 /* Full set, ship it! */ 1415 qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, 1416 pds->ramblock_name, 1417 pds->cur_entry, 1418 pds->start_list, 1419 pds->length_list); 1420 pds->nsentcmds++; 1421 pds->cur_entry = 0; 1422 } 1423 } 1424 1425 /** 1426 * postcopy_discard_send_finish: Called at the end of each RAMBlock by the 1427 * bitmap code. Sends any outstanding discard messages, frees the PDS 1428 * 1429 * @ms: Current migration state. 1430 * @pds: Structure initialised by postcopy_discard_send_init(). 1431 */ 1432 void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds) 1433 { 1434 /* Anything unsent? */ 1435 if (pds->cur_entry) { 1436 qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, 1437 pds->ramblock_name, 1438 pds->cur_entry, 1439 pds->start_list, 1440 pds->length_list); 1441 pds->nsentcmds++; 1442 } 1443 1444 trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords, 1445 pds->nsentcmds); 1446 1447 g_free(pds); 1448 } 1449 1450 /* 1451 * Current state of incoming postcopy; note this is not part of 1452 * MigrationIncomingState since it's state is used during cleanup 1453 * at the end as MIS is being freed. 1454 */ 1455 static PostcopyState incoming_postcopy_state; 1456 1457 PostcopyState postcopy_state_get(void) 1458 { 1459 return atomic_mb_read(&incoming_postcopy_state); 1460 } 1461 1462 /* Set the state and return the old state */ 1463 PostcopyState postcopy_state_set(PostcopyState new_state) 1464 { 1465 return atomic_xchg(&incoming_postcopy_state, new_state); 1466 } 1467 1468 /* Register a handler for external shared memory postcopy 1469 * called on the destination. 1470 */ 1471 void postcopy_register_shared_ufd(struct PostCopyFD *pcfd) 1472 { 1473 MigrationIncomingState *mis = migration_incoming_get_current(); 1474 1475 mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds, 1476 *pcfd); 1477 } 1478 1479 /* Unregister a handler for external shared memory postcopy 1480 */ 1481 void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd) 1482 { 1483 guint i; 1484 MigrationIncomingState *mis = migration_incoming_get_current(); 1485 GArray *pcrfds = mis->postcopy_remote_fds; 1486 1487 for (i = 0; i < pcrfds->len; i++) { 1488 struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); 1489 if (cur->fd == pcfd->fd) { 1490 mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i); 1491 return; 1492 } 1493 } 1494 } 1495