1 /* 2 * QEMU live migration 3 * 4 * Copyright IBM, Corp. 2008 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu-common.h" 17 #include "qemu/error-report.h" 18 #include "qemu/main-loop.h" 19 #include "migration/migration.h" 20 #include "migration/qemu-file.h" 21 #include "sysemu/sysemu.h" 22 #include "block/block.h" 23 #include "qapi/qmp/qerror.h" 24 #include "qemu/sockets.h" 25 #include "migration/block.h" 26 #include "qemu/thread.h" 27 #include "qmp-commands.h" 28 #include "trace.h" 29 #include "qapi/util.h" 30 #include "qapi-event.h" 31 32 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */ 33 34 /* Amount of time to allocate to each "chunk" of bandwidth-throttled 35 * data. */ 36 #define BUFFER_DELAY 100 37 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) 38 39 /* Default compression thread count */ 40 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8 41 /* Default decompression thread count, usually decompression is at 42 * least 4 times as fast as compression.*/ 43 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 44 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */ 45 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 46 47 /* Migration XBZRLE default cache size */ 48 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) 49 50 static NotifierList migration_state_notifiers = 51 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); 52 53 static bool deferred_incoming; 54 55 /* When we add fault tolerance, we could have several 56 migrations at once. For now we don't need to add 57 dynamic creation of migration */ 58 59 /* For outgoing */ 60 MigrationState *migrate_get_current(void) 61 { 62 static MigrationState current_migration = { 63 .state = MIGRATION_STATUS_NONE, 64 .bandwidth_limit = MAX_THROTTLE, 65 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE, 66 .mbps = -1, 67 .parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = 68 DEFAULT_MIGRATE_COMPRESS_LEVEL, 69 .parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 70 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT, 71 .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 72 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT, 73 }; 74 75 return ¤t_migration; 76 } 77 78 /* For incoming */ 79 static MigrationIncomingState *mis_current; 80 81 MigrationIncomingState *migration_incoming_get_current(void) 82 { 83 return mis_current; 84 } 85 86 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f) 87 { 88 mis_current = g_malloc0(sizeof(MigrationIncomingState)); 89 mis_current->file = f; 90 QLIST_INIT(&mis_current->loadvm_handlers); 91 92 return mis_current; 93 } 94 95 void migration_incoming_state_destroy(void) 96 { 97 loadvm_free_handlers(mis_current); 98 g_free(mis_current); 99 mis_current = NULL; 100 } 101 102 103 typedef struct { 104 bool optional; 105 uint32_t size; 106 uint8_t runstate[100]; 107 RunState state; 108 bool received; 109 } GlobalState; 110 111 static GlobalState global_state; 112 113 int global_state_store(void) 114 { 115 if (!runstate_store((char *)global_state.runstate, 116 sizeof(global_state.runstate))) { 117 error_report("runstate name too big: %s", global_state.runstate); 118 trace_migrate_state_too_big(); 119 return -EINVAL; 120 } 121 return 0; 122 } 123 124 static bool global_state_received(void) 125 { 126 return global_state.received; 127 } 128 129 static RunState global_state_get_runstate(void) 130 { 131 return global_state.state; 132 } 133 134 void global_state_set_optional(void) 135 { 136 global_state.optional = true; 137 } 138 139 static bool global_state_needed(void *opaque) 140 { 141 GlobalState *s = opaque; 142 char *runstate = (char *)s->runstate; 143 144 /* If it is not optional, it is mandatory */ 145 146 if (s->optional == false) { 147 return true; 148 } 149 150 /* If state is running or paused, it is not needed */ 151 152 if (strcmp(runstate, "running") == 0 || 153 strcmp(runstate, "paused") == 0) { 154 return false; 155 } 156 157 /* for any other state it is needed */ 158 return true; 159 } 160 161 static int global_state_post_load(void *opaque, int version_id) 162 { 163 GlobalState *s = opaque; 164 Error *local_err = NULL; 165 int r; 166 char *runstate = (char *)s->runstate; 167 168 s->received = true; 169 trace_migrate_global_state_post_load(runstate); 170 171 r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX, 172 -1, &local_err); 173 174 if (r == -1) { 175 if (local_err) { 176 error_report_err(local_err); 177 } 178 return -EINVAL; 179 } 180 s->state = r; 181 182 return 0; 183 } 184 185 static void global_state_pre_save(void *opaque) 186 { 187 GlobalState *s = opaque; 188 189 trace_migrate_global_state_pre_save((char *)s->runstate); 190 s->size = strlen((char *)s->runstate) + 1; 191 } 192 193 static const VMStateDescription vmstate_globalstate = { 194 .name = "globalstate", 195 .version_id = 1, 196 .minimum_version_id = 1, 197 .post_load = global_state_post_load, 198 .pre_save = global_state_pre_save, 199 .needed = global_state_needed, 200 .fields = (VMStateField[]) { 201 VMSTATE_UINT32(size, GlobalState), 202 VMSTATE_BUFFER(runstate, GlobalState), 203 VMSTATE_END_OF_LIST() 204 }, 205 }; 206 207 void register_global_state(void) 208 { 209 /* We would use it independently that we receive it */ 210 strcpy((char *)&global_state.runstate, ""); 211 global_state.received = false; 212 vmstate_register(NULL, 0, &vmstate_globalstate, &global_state); 213 } 214 215 static void migrate_generate_event(int new_state) 216 { 217 if (migrate_use_events()) { 218 qapi_event_send_migration(new_state, &error_abort); 219 } 220 } 221 222 /* 223 * Called on -incoming with a defer: uri. 224 * The migration can be started later after any parameters have been 225 * changed. 226 */ 227 static void deferred_incoming_migration(Error **errp) 228 { 229 if (deferred_incoming) { 230 error_setg(errp, "Incoming migration already deferred"); 231 } 232 deferred_incoming = true; 233 } 234 235 void qemu_start_incoming_migration(const char *uri, Error **errp) 236 { 237 const char *p; 238 239 qapi_event_send_migration(MIGRATION_STATUS_SETUP, &error_abort); 240 if (!strcmp(uri, "defer")) { 241 deferred_incoming_migration(errp); 242 } else if (strstart(uri, "tcp:", &p)) { 243 tcp_start_incoming_migration(p, errp); 244 #ifdef CONFIG_RDMA 245 } else if (strstart(uri, "rdma:", &p)) { 246 rdma_start_incoming_migration(p, errp); 247 #endif 248 #if !defined(WIN32) 249 } else if (strstart(uri, "exec:", &p)) { 250 exec_start_incoming_migration(p, errp); 251 } else if (strstart(uri, "unix:", &p)) { 252 unix_start_incoming_migration(p, errp); 253 } else if (strstart(uri, "fd:", &p)) { 254 fd_start_incoming_migration(p, errp); 255 #endif 256 } else { 257 error_setg(errp, "unknown migration protocol: %s", uri); 258 } 259 } 260 261 static void process_incoming_migration_co(void *opaque) 262 { 263 QEMUFile *f = opaque; 264 Error *local_err = NULL; 265 int ret; 266 267 migration_incoming_state_new(f); 268 migrate_generate_event(MIGRATION_STATUS_ACTIVE); 269 ret = qemu_loadvm_state(f); 270 271 qemu_fclose(f); 272 free_xbzrle_decoded_buf(); 273 migration_incoming_state_destroy(); 274 275 if (ret < 0) { 276 migrate_generate_event(MIGRATION_STATUS_FAILED); 277 error_report("load of migration failed: %s", strerror(-ret)); 278 migrate_decompress_threads_join(); 279 exit(EXIT_FAILURE); 280 } 281 migrate_generate_event(MIGRATION_STATUS_COMPLETED); 282 qemu_announce_self(); 283 284 /* Make sure all file formats flush their mutable metadata */ 285 bdrv_invalidate_cache_all(&local_err); 286 if (local_err) { 287 error_report_err(local_err); 288 migrate_decompress_threads_join(); 289 exit(EXIT_FAILURE); 290 } 291 292 /* If global state section was not received or we are in running 293 state, we need to obey autostart. Any other state is set with 294 runstate_set. */ 295 296 if (!global_state_received() || 297 global_state_get_runstate() == RUN_STATE_RUNNING) { 298 if (autostart) { 299 vm_start(); 300 } else { 301 runstate_set(RUN_STATE_PAUSED); 302 } 303 } else { 304 runstate_set(global_state_get_runstate()); 305 } 306 migrate_decompress_threads_join(); 307 } 308 309 void process_incoming_migration(QEMUFile *f) 310 { 311 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co); 312 int fd = qemu_get_fd(f); 313 314 assert(fd != -1); 315 migrate_decompress_threads_create(); 316 qemu_set_nonblock(fd); 317 qemu_coroutine_enter(co, f); 318 } 319 320 /* amount of nanoseconds we are willing to wait for migration to be down. 321 * the choice of nanoseconds is because it is the maximum resolution that 322 * get_clock() can achieve. It is an internal measure. All user-visible 323 * units must be in seconds */ 324 static uint64_t max_downtime = 300000000; 325 326 uint64_t migrate_max_downtime(void) 327 { 328 return max_downtime; 329 } 330 331 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) 332 { 333 MigrationCapabilityStatusList *head = NULL; 334 MigrationCapabilityStatusList *caps; 335 MigrationState *s = migrate_get_current(); 336 int i; 337 338 caps = NULL; /* silence compiler warning */ 339 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) { 340 if (head == NULL) { 341 head = g_malloc0(sizeof(*caps)); 342 caps = head; 343 } else { 344 caps->next = g_malloc0(sizeof(*caps)); 345 caps = caps->next; 346 } 347 caps->value = 348 g_malloc(sizeof(*caps->value)); 349 caps->value->capability = i; 350 caps->value->state = s->enabled_capabilities[i]; 351 } 352 353 return head; 354 } 355 356 MigrationParameters *qmp_query_migrate_parameters(Error **errp) 357 { 358 MigrationParameters *params; 359 MigrationState *s = migrate_get_current(); 360 361 params = g_malloc0(sizeof(*params)); 362 params->compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 363 params->compress_threads = 364 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 365 params->decompress_threads = 366 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 367 368 return params; 369 } 370 371 static void get_xbzrle_cache_stats(MigrationInfo *info) 372 { 373 if (migrate_use_xbzrle()) { 374 info->has_xbzrle_cache = true; 375 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache)); 376 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size(); 377 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred(); 378 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred(); 379 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss(); 380 info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate(); 381 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow(); 382 } 383 } 384 385 MigrationInfo *qmp_query_migrate(Error **errp) 386 { 387 MigrationInfo *info = g_malloc0(sizeof(*info)); 388 MigrationState *s = migrate_get_current(); 389 390 switch (s->state) { 391 case MIGRATION_STATUS_NONE: 392 /* no migration has happened ever */ 393 break; 394 case MIGRATION_STATUS_SETUP: 395 info->has_status = true; 396 info->has_total_time = false; 397 break; 398 case MIGRATION_STATUS_ACTIVE: 399 case MIGRATION_STATUS_CANCELLING: 400 info->has_status = true; 401 info->has_total_time = true; 402 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) 403 - s->total_time; 404 info->has_expected_downtime = true; 405 info->expected_downtime = s->expected_downtime; 406 info->has_setup_time = true; 407 info->setup_time = s->setup_time; 408 409 info->has_ram = true; 410 info->ram = g_malloc0(sizeof(*info->ram)); 411 info->ram->transferred = ram_bytes_transferred(); 412 info->ram->remaining = ram_bytes_remaining(); 413 info->ram->total = ram_bytes_total(); 414 info->ram->duplicate = dup_mig_pages_transferred(); 415 info->ram->skipped = skipped_mig_pages_transferred(); 416 info->ram->normal = norm_mig_pages_transferred(); 417 info->ram->normal_bytes = norm_mig_bytes_transferred(); 418 info->ram->dirty_pages_rate = s->dirty_pages_rate; 419 info->ram->mbps = s->mbps; 420 info->ram->dirty_sync_count = s->dirty_sync_count; 421 422 if (blk_mig_active()) { 423 info->has_disk = true; 424 info->disk = g_malloc0(sizeof(*info->disk)); 425 info->disk->transferred = blk_mig_bytes_transferred(); 426 info->disk->remaining = blk_mig_bytes_remaining(); 427 info->disk->total = blk_mig_bytes_total(); 428 } 429 430 get_xbzrle_cache_stats(info); 431 break; 432 case MIGRATION_STATUS_COMPLETED: 433 get_xbzrle_cache_stats(info); 434 435 info->has_status = true; 436 info->has_total_time = true; 437 info->total_time = s->total_time; 438 info->has_downtime = true; 439 info->downtime = s->downtime; 440 info->has_setup_time = true; 441 info->setup_time = s->setup_time; 442 443 info->has_ram = true; 444 info->ram = g_malloc0(sizeof(*info->ram)); 445 info->ram->transferred = ram_bytes_transferred(); 446 info->ram->remaining = 0; 447 info->ram->total = ram_bytes_total(); 448 info->ram->duplicate = dup_mig_pages_transferred(); 449 info->ram->skipped = skipped_mig_pages_transferred(); 450 info->ram->normal = norm_mig_pages_transferred(); 451 info->ram->normal_bytes = norm_mig_bytes_transferred(); 452 info->ram->mbps = s->mbps; 453 info->ram->dirty_sync_count = s->dirty_sync_count; 454 break; 455 case MIGRATION_STATUS_FAILED: 456 info->has_status = true; 457 break; 458 case MIGRATION_STATUS_CANCELLED: 459 info->has_status = true; 460 break; 461 } 462 info->status = s->state; 463 464 return info; 465 } 466 467 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, 468 Error **errp) 469 { 470 MigrationState *s = migrate_get_current(); 471 MigrationCapabilityStatusList *cap; 472 473 if (s->state == MIGRATION_STATUS_ACTIVE || 474 s->state == MIGRATION_STATUS_SETUP) { 475 error_setg(errp, QERR_MIGRATION_ACTIVE); 476 return; 477 } 478 479 for (cap = params; cap; cap = cap->next) { 480 s->enabled_capabilities[cap->value->capability] = cap->value->state; 481 } 482 } 483 484 void qmp_migrate_set_parameters(bool has_compress_level, 485 int64_t compress_level, 486 bool has_compress_threads, 487 int64_t compress_threads, 488 bool has_decompress_threads, 489 int64_t decompress_threads, Error **errp) 490 { 491 MigrationState *s = migrate_get_current(); 492 493 if (has_compress_level && (compress_level < 0 || compress_level > 9)) { 494 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level", 495 "is invalid, it should be in the range of 0 to 9"); 496 return; 497 } 498 if (has_compress_threads && 499 (compress_threads < 1 || compress_threads > 255)) { 500 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 501 "compress_threads", 502 "is invalid, it should be in the range of 1 to 255"); 503 return; 504 } 505 if (has_decompress_threads && 506 (decompress_threads < 1 || decompress_threads > 255)) { 507 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 508 "decompress_threads", 509 "is invalid, it should be in the range of 1 to 255"); 510 return; 511 } 512 513 if (has_compress_level) { 514 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 515 } 516 if (has_compress_threads) { 517 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = compress_threads; 518 } 519 if (has_decompress_threads) { 520 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 521 decompress_threads; 522 } 523 } 524 525 /* shared migration helpers */ 526 527 static void migrate_set_state(MigrationState *s, int old_state, int new_state) 528 { 529 if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) { 530 trace_migrate_set_state(new_state); 531 migrate_generate_event(new_state); 532 } 533 } 534 535 static void migrate_fd_cleanup(void *opaque) 536 { 537 MigrationState *s = opaque; 538 539 qemu_bh_delete(s->cleanup_bh); 540 s->cleanup_bh = NULL; 541 542 if (s->file) { 543 trace_migrate_fd_cleanup(); 544 qemu_mutex_unlock_iothread(); 545 qemu_thread_join(&s->thread); 546 qemu_mutex_lock_iothread(); 547 548 migrate_compress_threads_join(); 549 qemu_fclose(s->file); 550 s->file = NULL; 551 } 552 553 assert(s->state != MIGRATION_STATUS_ACTIVE); 554 555 if (s->state != MIGRATION_STATUS_COMPLETED) { 556 qemu_savevm_state_cancel(); 557 if (s->state == MIGRATION_STATUS_CANCELLING) { 558 migrate_set_state(s, MIGRATION_STATUS_CANCELLING, 559 MIGRATION_STATUS_CANCELLED); 560 } 561 } 562 563 notifier_list_notify(&migration_state_notifiers, s); 564 } 565 566 void migrate_fd_error(MigrationState *s) 567 { 568 trace_migrate_fd_error(); 569 assert(s->file == NULL); 570 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 571 notifier_list_notify(&migration_state_notifiers, s); 572 } 573 574 static void migrate_fd_cancel(MigrationState *s) 575 { 576 int old_state ; 577 QEMUFile *f = migrate_get_current()->file; 578 trace_migrate_fd_cancel(); 579 580 do { 581 old_state = s->state; 582 if (old_state != MIGRATION_STATUS_SETUP && 583 old_state != MIGRATION_STATUS_ACTIVE) { 584 break; 585 } 586 migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING); 587 } while (s->state != MIGRATION_STATUS_CANCELLING); 588 589 /* 590 * If we're unlucky the migration code might be stuck somewhere in a 591 * send/write while the network has failed and is waiting to timeout; 592 * if we've got shutdown(2) available then we can force it to quit. 593 * The outgoing qemu file gets closed in migrate_fd_cleanup that is 594 * called in a bh, so there is no race against this cancel. 595 */ 596 if (s->state == MIGRATION_STATUS_CANCELLING && f) { 597 qemu_file_shutdown(f); 598 } 599 } 600 601 void add_migration_state_change_notifier(Notifier *notify) 602 { 603 notifier_list_add(&migration_state_notifiers, notify); 604 } 605 606 void remove_migration_state_change_notifier(Notifier *notify) 607 { 608 notifier_remove(notify); 609 } 610 611 bool migration_in_setup(MigrationState *s) 612 { 613 return s->state == MIGRATION_STATUS_SETUP; 614 } 615 616 bool migration_has_finished(MigrationState *s) 617 { 618 return s->state == MIGRATION_STATUS_COMPLETED; 619 } 620 621 bool migration_has_failed(MigrationState *s) 622 { 623 return (s->state == MIGRATION_STATUS_CANCELLED || 624 s->state == MIGRATION_STATUS_FAILED); 625 } 626 627 static MigrationState *migrate_init(const MigrationParams *params) 628 { 629 MigrationState *s = migrate_get_current(); 630 int64_t bandwidth_limit = s->bandwidth_limit; 631 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX]; 632 int64_t xbzrle_cache_size = s->xbzrle_cache_size; 633 int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 634 int compress_thread_count = 635 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 636 int decompress_thread_count = 637 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 638 639 memcpy(enabled_capabilities, s->enabled_capabilities, 640 sizeof(enabled_capabilities)); 641 642 memset(s, 0, sizeof(*s)); 643 s->params = *params; 644 memcpy(s->enabled_capabilities, enabled_capabilities, 645 sizeof(enabled_capabilities)); 646 s->xbzrle_cache_size = xbzrle_cache_size; 647 648 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 649 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 650 compress_thread_count; 651 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 652 decompress_thread_count; 653 s->bandwidth_limit = bandwidth_limit; 654 migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); 655 656 s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 657 return s; 658 } 659 660 static GSList *migration_blockers; 661 662 void migrate_add_blocker(Error *reason) 663 { 664 migration_blockers = g_slist_prepend(migration_blockers, reason); 665 } 666 667 void migrate_del_blocker(Error *reason) 668 { 669 migration_blockers = g_slist_remove(migration_blockers, reason); 670 } 671 672 void qmp_migrate_incoming(const char *uri, Error **errp) 673 { 674 Error *local_err = NULL; 675 static bool once = true; 676 677 if (!deferred_incoming) { 678 error_setg(errp, "For use with '-incoming defer'"); 679 return; 680 } 681 if (!once) { 682 error_setg(errp, "The incoming migration has already been started"); 683 } 684 685 qemu_start_incoming_migration(uri, &local_err); 686 687 if (local_err) { 688 error_propagate(errp, local_err); 689 return; 690 } 691 692 once = false; 693 } 694 695 void qmp_migrate(const char *uri, bool has_blk, bool blk, 696 bool has_inc, bool inc, bool has_detach, bool detach, 697 Error **errp) 698 { 699 Error *local_err = NULL; 700 MigrationState *s = migrate_get_current(); 701 MigrationParams params; 702 const char *p; 703 704 params.blk = has_blk && blk; 705 params.shared = has_inc && inc; 706 707 if (s->state == MIGRATION_STATUS_ACTIVE || 708 s->state == MIGRATION_STATUS_SETUP || 709 s->state == MIGRATION_STATUS_CANCELLING) { 710 error_setg(errp, QERR_MIGRATION_ACTIVE); 711 return; 712 } 713 if (runstate_check(RUN_STATE_INMIGRATE)) { 714 error_setg(errp, "Guest is waiting for an incoming migration"); 715 return; 716 } 717 718 if (qemu_savevm_state_blocked(errp)) { 719 return; 720 } 721 722 if (migration_blockers) { 723 *errp = error_copy(migration_blockers->data); 724 return; 725 } 726 727 /* We are starting a new migration, so we want to start in a clean 728 state. This change is only needed if previous migration 729 failed/was cancelled. We don't use migrate_set_state() because 730 we are setting the initial state, not changing it. */ 731 s->state = MIGRATION_STATUS_NONE; 732 733 s = migrate_init(¶ms); 734 735 if (strstart(uri, "tcp:", &p)) { 736 tcp_start_outgoing_migration(s, p, &local_err); 737 #ifdef CONFIG_RDMA 738 } else if (strstart(uri, "rdma:", &p)) { 739 rdma_start_outgoing_migration(s, p, &local_err); 740 #endif 741 #if !defined(WIN32) 742 } else if (strstart(uri, "exec:", &p)) { 743 exec_start_outgoing_migration(s, p, &local_err); 744 } else if (strstart(uri, "unix:", &p)) { 745 unix_start_outgoing_migration(s, p, &local_err); 746 } else if (strstart(uri, "fd:", &p)) { 747 fd_start_outgoing_migration(s, p, &local_err); 748 #endif 749 } else { 750 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", 751 "a valid migration protocol"); 752 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 753 return; 754 } 755 756 if (local_err) { 757 migrate_fd_error(s); 758 error_propagate(errp, local_err); 759 return; 760 } 761 } 762 763 void qmp_migrate_cancel(Error **errp) 764 { 765 migrate_fd_cancel(migrate_get_current()); 766 } 767 768 void qmp_migrate_set_cache_size(int64_t value, Error **errp) 769 { 770 MigrationState *s = migrate_get_current(); 771 int64_t new_size; 772 773 /* Check for truncation */ 774 if (value != (size_t)value) { 775 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 776 "exceeding address space"); 777 return; 778 } 779 780 /* Cache should not be larger than guest ram size */ 781 if (value > ram_bytes_total()) { 782 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 783 "exceeds guest ram size "); 784 return; 785 } 786 787 new_size = xbzrle_cache_resize(value); 788 if (new_size < 0) { 789 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 790 "is smaller than page size"); 791 return; 792 } 793 794 s->xbzrle_cache_size = new_size; 795 } 796 797 int64_t qmp_query_migrate_cache_size(Error **errp) 798 { 799 return migrate_xbzrle_cache_size(); 800 } 801 802 void qmp_migrate_set_speed(int64_t value, Error **errp) 803 { 804 MigrationState *s; 805 806 if (value < 0) { 807 value = 0; 808 } 809 if (value > SIZE_MAX) { 810 value = SIZE_MAX; 811 } 812 813 s = migrate_get_current(); 814 s->bandwidth_limit = value; 815 if (s->file) { 816 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO); 817 } 818 } 819 820 void qmp_migrate_set_downtime(double value, Error **errp) 821 { 822 value *= 1e9; 823 value = MAX(0, MIN(UINT64_MAX, value)); 824 max_downtime = (uint64_t)value; 825 } 826 827 bool migrate_auto_converge(void) 828 { 829 MigrationState *s; 830 831 s = migrate_get_current(); 832 833 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE]; 834 } 835 836 bool migrate_zero_blocks(void) 837 { 838 MigrationState *s; 839 840 s = migrate_get_current(); 841 842 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS]; 843 } 844 845 bool migrate_use_compression(void) 846 { 847 MigrationState *s; 848 849 s = migrate_get_current(); 850 851 return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS]; 852 } 853 854 int migrate_compress_level(void) 855 { 856 MigrationState *s; 857 858 s = migrate_get_current(); 859 860 return s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 861 } 862 863 int migrate_compress_threads(void) 864 { 865 MigrationState *s; 866 867 s = migrate_get_current(); 868 869 return s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 870 } 871 872 int migrate_decompress_threads(void) 873 { 874 MigrationState *s; 875 876 s = migrate_get_current(); 877 878 return s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 879 } 880 881 bool migrate_use_events(void) 882 { 883 MigrationState *s; 884 885 s = migrate_get_current(); 886 887 return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS]; 888 } 889 890 int migrate_use_xbzrle(void) 891 { 892 MigrationState *s; 893 894 s = migrate_get_current(); 895 896 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE]; 897 } 898 899 int64_t migrate_xbzrle_cache_size(void) 900 { 901 MigrationState *s; 902 903 s = migrate_get_current(); 904 905 return s->xbzrle_cache_size; 906 } 907 908 /* migration thread support */ 909 910 static void *migration_thread(void *opaque) 911 { 912 MigrationState *s = opaque; 913 int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 914 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); 915 int64_t initial_bytes = 0; 916 int64_t max_size = 0; 917 int64_t start_time = initial_time; 918 bool old_vm_running = false; 919 920 qemu_savevm_state_header(s->file); 921 qemu_savevm_state_begin(s->file, &s->params); 922 923 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; 924 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); 925 926 while (s->state == MIGRATION_STATUS_ACTIVE) { 927 int64_t current_time; 928 uint64_t pending_size; 929 930 if (!qemu_file_rate_limit(s->file)) { 931 pending_size = qemu_savevm_state_pending(s->file, max_size); 932 trace_migrate_pending(pending_size, max_size); 933 if (pending_size && pending_size >= max_size) { 934 qemu_savevm_state_iterate(s->file); 935 } else { 936 int ret; 937 938 qemu_mutex_lock_iothread(); 939 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 940 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); 941 old_vm_running = runstate_is_running(); 942 943 ret = global_state_store(); 944 if (!ret) { 945 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); 946 if (ret >= 0) { 947 qemu_file_set_rate_limit(s->file, INT64_MAX); 948 qemu_savevm_state_complete(s->file); 949 } 950 } 951 qemu_mutex_unlock_iothread(); 952 953 if (ret < 0) { 954 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 955 MIGRATION_STATUS_FAILED); 956 break; 957 } 958 959 if (!qemu_file_get_error(s->file)) { 960 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 961 MIGRATION_STATUS_COMPLETED); 962 break; 963 } 964 } 965 } 966 967 if (qemu_file_get_error(s->file)) { 968 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 969 MIGRATION_STATUS_FAILED); 970 break; 971 } 972 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 973 if (current_time >= initial_time + BUFFER_DELAY) { 974 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes; 975 uint64_t time_spent = current_time - initial_time; 976 double bandwidth = transferred_bytes / time_spent; 977 max_size = bandwidth * migrate_max_downtime() / 1000000; 978 979 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) / 980 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1; 981 982 trace_migrate_transferred(transferred_bytes, time_spent, 983 bandwidth, max_size); 984 /* if we haven't sent anything, we don't want to recalculate 985 10000 is a small enough number for our purposes */ 986 if (s->dirty_bytes_rate && transferred_bytes > 10000) { 987 s->expected_downtime = s->dirty_bytes_rate / bandwidth; 988 } 989 990 qemu_file_reset_rate_limit(s->file); 991 initial_time = current_time; 992 initial_bytes = qemu_ftell(s->file); 993 } 994 if (qemu_file_rate_limit(s->file)) { 995 /* usleep expects microseconds */ 996 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000); 997 } 998 } 999 1000 qemu_mutex_lock_iothread(); 1001 if (s->state == MIGRATION_STATUS_COMPLETED) { 1002 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1003 uint64_t transferred_bytes = qemu_ftell(s->file); 1004 s->total_time = end_time - s->total_time; 1005 s->downtime = end_time - start_time; 1006 if (s->total_time) { 1007 s->mbps = (((double) transferred_bytes * 8.0) / 1008 ((double) s->total_time)) / 1000; 1009 } 1010 runstate_set(RUN_STATE_POSTMIGRATE); 1011 } else { 1012 if (old_vm_running) { 1013 vm_start(); 1014 } 1015 } 1016 qemu_bh_schedule(s->cleanup_bh); 1017 qemu_mutex_unlock_iothread(); 1018 1019 return NULL; 1020 } 1021 1022 void migrate_fd_connect(MigrationState *s) 1023 { 1024 /* This is a best 1st approximation. ns to ms */ 1025 s->expected_downtime = max_downtime/1000000; 1026 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s); 1027 1028 qemu_file_set_rate_limit(s->file, 1029 s->bandwidth_limit / XFER_LIMIT_RATIO); 1030 1031 /* Notify before starting migration thread */ 1032 notifier_list_notify(&migration_state_notifiers, s); 1033 1034 migrate_compress_threads_create(); 1035 qemu_thread_create(&s->thread, "migration", migration_thread, s, 1036 QEMU_THREAD_JOINABLE); 1037 } 1038