1 /* 2 * QEMU live migration 3 * 4 * Copyright IBM, Corp. 2008 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu-common.h" 17 #include "qemu/error-report.h" 18 #include "qemu/main-loop.h" 19 #include "migration/migration.h" 20 #include "migration/qemu-file.h" 21 #include "sysemu/sysemu.h" 22 #include "block/block.h" 23 #include "qapi/qmp/qerror.h" 24 #include "qemu/sockets.h" 25 #include "qemu/rcu.h" 26 #include "migration/block.h" 27 #include "qemu/thread.h" 28 #include "qmp-commands.h" 29 #include "trace.h" 30 #include "qapi/util.h" 31 #include "qapi-event.h" 32 33 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */ 34 35 /* Amount of time to allocate to each "chunk" of bandwidth-throttled 36 * data. */ 37 #define BUFFER_DELAY 100 38 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) 39 40 /* Default compression thread count */ 41 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8 42 /* Default decompression thread count, usually decompression is at 43 * least 4 times as fast as compression.*/ 44 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 45 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */ 46 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 47 48 /* Migration XBZRLE default cache size */ 49 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) 50 51 static NotifierList migration_state_notifiers = 52 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); 53 54 static bool deferred_incoming; 55 56 /* When we add fault tolerance, we could have several 57 migrations at once. For now we don't need to add 58 dynamic creation of migration */ 59 60 /* For outgoing */ 61 MigrationState *migrate_get_current(void) 62 { 63 static MigrationState current_migration = { 64 .state = MIGRATION_STATUS_NONE, 65 .bandwidth_limit = MAX_THROTTLE, 66 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE, 67 .mbps = -1, 68 .parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = 69 DEFAULT_MIGRATE_COMPRESS_LEVEL, 70 .parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 71 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT, 72 .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 73 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT, 74 }; 75 76 return ¤t_migration; 77 } 78 79 /* For incoming */ 80 static MigrationIncomingState *mis_current; 81 82 MigrationIncomingState *migration_incoming_get_current(void) 83 { 84 return mis_current; 85 } 86 87 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f) 88 { 89 mis_current = g_malloc0(sizeof(MigrationIncomingState)); 90 mis_current->file = f; 91 QLIST_INIT(&mis_current->loadvm_handlers); 92 93 return mis_current; 94 } 95 96 void migration_incoming_state_destroy(void) 97 { 98 loadvm_free_handlers(mis_current); 99 g_free(mis_current); 100 mis_current = NULL; 101 } 102 103 104 typedef struct { 105 bool optional; 106 uint32_t size; 107 uint8_t runstate[100]; 108 RunState state; 109 bool received; 110 } GlobalState; 111 112 static GlobalState global_state; 113 114 int global_state_store(void) 115 { 116 if (!runstate_store((char *)global_state.runstate, 117 sizeof(global_state.runstate))) { 118 error_report("runstate name too big: %s", global_state.runstate); 119 trace_migrate_state_too_big(); 120 return -EINVAL; 121 } 122 return 0; 123 } 124 125 static bool global_state_received(void) 126 { 127 return global_state.received; 128 } 129 130 static RunState global_state_get_runstate(void) 131 { 132 return global_state.state; 133 } 134 135 void global_state_set_optional(void) 136 { 137 global_state.optional = true; 138 } 139 140 static bool global_state_needed(void *opaque) 141 { 142 GlobalState *s = opaque; 143 char *runstate = (char *)s->runstate; 144 145 /* If it is not optional, it is mandatory */ 146 147 if (s->optional == false) { 148 return true; 149 } 150 151 /* If state is running or paused, it is not needed */ 152 153 if (strcmp(runstate, "running") == 0 || 154 strcmp(runstate, "paused") == 0) { 155 return false; 156 } 157 158 /* for any other state it is needed */ 159 return true; 160 } 161 162 static int global_state_post_load(void *opaque, int version_id) 163 { 164 GlobalState *s = opaque; 165 Error *local_err = NULL; 166 int r; 167 char *runstate = (char *)s->runstate; 168 169 s->received = true; 170 trace_migrate_global_state_post_load(runstate); 171 172 r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX, 173 -1, &local_err); 174 175 if (r == -1) { 176 if (local_err) { 177 error_report_err(local_err); 178 } 179 return -EINVAL; 180 } 181 s->state = r; 182 183 return 0; 184 } 185 186 static void global_state_pre_save(void *opaque) 187 { 188 GlobalState *s = opaque; 189 190 trace_migrate_global_state_pre_save((char *)s->runstate); 191 s->size = strlen((char *)s->runstate) + 1; 192 } 193 194 static const VMStateDescription vmstate_globalstate = { 195 .name = "globalstate", 196 .version_id = 1, 197 .minimum_version_id = 1, 198 .post_load = global_state_post_load, 199 .pre_save = global_state_pre_save, 200 .needed = global_state_needed, 201 .fields = (VMStateField[]) { 202 VMSTATE_UINT32(size, GlobalState), 203 VMSTATE_BUFFER(runstate, GlobalState), 204 VMSTATE_END_OF_LIST() 205 }, 206 }; 207 208 void register_global_state(void) 209 { 210 /* We would use it independently that we receive it */ 211 strcpy((char *)&global_state.runstate, ""); 212 global_state.received = false; 213 vmstate_register(NULL, 0, &vmstate_globalstate, &global_state); 214 } 215 216 static void migrate_generate_event(int new_state) 217 { 218 if (migrate_use_events()) { 219 qapi_event_send_migration(new_state, &error_abort); 220 } 221 } 222 223 /* 224 * Called on -incoming with a defer: uri. 225 * The migration can be started later after any parameters have been 226 * changed. 227 */ 228 static void deferred_incoming_migration(Error **errp) 229 { 230 if (deferred_incoming) { 231 error_setg(errp, "Incoming migration already deferred"); 232 } 233 deferred_incoming = true; 234 } 235 236 void qemu_start_incoming_migration(const char *uri, Error **errp) 237 { 238 const char *p; 239 240 qapi_event_send_migration(MIGRATION_STATUS_SETUP, &error_abort); 241 if (!strcmp(uri, "defer")) { 242 deferred_incoming_migration(errp); 243 } else if (strstart(uri, "tcp:", &p)) { 244 tcp_start_incoming_migration(p, errp); 245 #ifdef CONFIG_RDMA 246 } else if (strstart(uri, "rdma:", &p)) { 247 rdma_start_incoming_migration(p, errp); 248 #endif 249 #if !defined(WIN32) 250 } else if (strstart(uri, "exec:", &p)) { 251 exec_start_incoming_migration(p, errp); 252 } else if (strstart(uri, "unix:", &p)) { 253 unix_start_incoming_migration(p, errp); 254 } else if (strstart(uri, "fd:", &p)) { 255 fd_start_incoming_migration(p, errp); 256 #endif 257 } else { 258 error_setg(errp, "unknown migration protocol: %s", uri); 259 } 260 } 261 262 static void process_incoming_migration_co(void *opaque) 263 { 264 QEMUFile *f = opaque; 265 Error *local_err = NULL; 266 int ret; 267 268 migration_incoming_state_new(f); 269 migrate_generate_event(MIGRATION_STATUS_ACTIVE); 270 ret = qemu_loadvm_state(f); 271 272 qemu_fclose(f); 273 free_xbzrle_decoded_buf(); 274 migration_incoming_state_destroy(); 275 276 if (ret < 0) { 277 migrate_generate_event(MIGRATION_STATUS_FAILED); 278 error_report("load of migration failed: %s", strerror(-ret)); 279 migrate_decompress_threads_join(); 280 exit(EXIT_FAILURE); 281 } 282 migrate_generate_event(MIGRATION_STATUS_COMPLETED); 283 qemu_announce_self(); 284 285 /* Make sure all file formats flush their mutable metadata */ 286 bdrv_invalidate_cache_all(&local_err); 287 if (local_err) { 288 error_report_err(local_err); 289 migrate_decompress_threads_join(); 290 exit(EXIT_FAILURE); 291 } 292 293 /* If global state section was not received or we are in running 294 state, we need to obey autostart. Any other state is set with 295 runstate_set. */ 296 297 if (!global_state_received() || 298 global_state_get_runstate() == RUN_STATE_RUNNING) { 299 if (autostart) { 300 vm_start(); 301 } else { 302 runstate_set(RUN_STATE_PAUSED); 303 } 304 } else { 305 runstate_set(global_state_get_runstate()); 306 } 307 migrate_decompress_threads_join(); 308 } 309 310 void process_incoming_migration(QEMUFile *f) 311 { 312 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co); 313 int fd = qemu_get_fd(f); 314 315 assert(fd != -1); 316 migrate_decompress_threads_create(); 317 qemu_set_nonblock(fd); 318 qemu_coroutine_enter(co, f); 319 } 320 321 /* amount of nanoseconds we are willing to wait for migration to be down. 322 * the choice of nanoseconds is because it is the maximum resolution that 323 * get_clock() can achieve. It is an internal measure. All user-visible 324 * units must be in seconds */ 325 static uint64_t max_downtime = 300000000; 326 327 uint64_t migrate_max_downtime(void) 328 { 329 return max_downtime; 330 } 331 332 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) 333 { 334 MigrationCapabilityStatusList *head = NULL; 335 MigrationCapabilityStatusList *caps; 336 MigrationState *s = migrate_get_current(); 337 int i; 338 339 caps = NULL; /* silence compiler warning */ 340 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) { 341 if (head == NULL) { 342 head = g_malloc0(sizeof(*caps)); 343 caps = head; 344 } else { 345 caps->next = g_malloc0(sizeof(*caps)); 346 caps = caps->next; 347 } 348 caps->value = 349 g_malloc(sizeof(*caps->value)); 350 caps->value->capability = i; 351 caps->value->state = s->enabled_capabilities[i]; 352 } 353 354 return head; 355 } 356 357 MigrationParameters *qmp_query_migrate_parameters(Error **errp) 358 { 359 MigrationParameters *params; 360 MigrationState *s = migrate_get_current(); 361 362 params = g_malloc0(sizeof(*params)); 363 params->compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 364 params->compress_threads = 365 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 366 params->decompress_threads = 367 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 368 369 return params; 370 } 371 372 static void get_xbzrle_cache_stats(MigrationInfo *info) 373 { 374 if (migrate_use_xbzrle()) { 375 info->has_xbzrle_cache = true; 376 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache)); 377 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size(); 378 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred(); 379 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred(); 380 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss(); 381 info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate(); 382 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow(); 383 } 384 } 385 386 MigrationInfo *qmp_query_migrate(Error **errp) 387 { 388 MigrationInfo *info = g_malloc0(sizeof(*info)); 389 MigrationState *s = migrate_get_current(); 390 391 switch (s->state) { 392 case MIGRATION_STATUS_NONE: 393 /* no migration has happened ever */ 394 break; 395 case MIGRATION_STATUS_SETUP: 396 info->has_status = true; 397 info->has_total_time = false; 398 break; 399 case MIGRATION_STATUS_ACTIVE: 400 case MIGRATION_STATUS_CANCELLING: 401 info->has_status = true; 402 info->has_total_time = true; 403 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) 404 - s->total_time; 405 info->has_expected_downtime = true; 406 info->expected_downtime = s->expected_downtime; 407 info->has_setup_time = true; 408 info->setup_time = s->setup_time; 409 410 info->has_ram = true; 411 info->ram = g_malloc0(sizeof(*info->ram)); 412 info->ram->transferred = ram_bytes_transferred(); 413 info->ram->remaining = ram_bytes_remaining(); 414 info->ram->total = ram_bytes_total(); 415 info->ram->duplicate = dup_mig_pages_transferred(); 416 info->ram->skipped = skipped_mig_pages_transferred(); 417 info->ram->normal = norm_mig_pages_transferred(); 418 info->ram->normal_bytes = norm_mig_bytes_transferred(); 419 info->ram->dirty_pages_rate = s->dirty_pages_rate; 420 info->ram->mbps = s->mbps; 421 info->ram->dirty_sync_count = s->dirty_sync_count; 422 423 if (blk_mig_active()) { 424 info->has_disk = true; 425 info->disk = g_malloc0(sizeof(*info->disk)); 426 info->disk->transferred = blk_mig_bytes_transferred(); 427 info->disk->remaining = blk_mig_bytes_remaining(); 428 info->disk->total = blk_mig_bytes_total(); 429 } 430 431 get_xbzrle_cache_stats(info); 432 break; 433 case MIGRATION_STATUS_COMPLETED: 434 get_xbzrle_cache_stats(info); 435 436 info->has_status = true; 437 info->has_total_time = true; 438 info->total_time = s->total_time; 439 info->has_downtime = true; 440 info->downtime = s->downtime; 441 info->has_setup_time = true; 442 info->setup_time = s->setup_time; 443 444 info->has_ram = true; 445 info->ram = g_malloc0(sizeof(*info->ram)); 446 info->ram->transferred = ram_bytes_transferred(); 447 info->ram->remaining = 0; 448 info->ram->total = ram_bytes_total(); 449 info->ram->duplicate = dup_mig_pages_transferred(); 450 info->ram->skipped = skipped_mig_pages_transferred(); 451 info->ram->normal = norm_mig_pages_transferred(); 452 info->ram->normal_bytes = norm_mig_bytes_transferred(); 453 info->ram->mbps = s->mbps; 454 info->ram->dirty_sync_count = s->dirty_sync_count; 455 break; 456 case MIGRATION_STATUS_FAILED: 457 info->has_status = true; 458 break; 459 case MIGRATION_STATUS_CANCELLED: 460 info->has_status = true; 461 break; 462 } 463 info->status = s->state; 464 465 return info; 466 } 467 468 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, 469 Error **errp) 470 { 471 MigrationState *s = migrate_get_current(); 472 MigrationCapabilityStatusList *cap; 473 474 if (s->state == MIGRATION_STATUS_ACTIVE || 475 s->state == MIGRATION_STATUS_SETUP) { 476 error_setg(errp, QERR_MIGRATION_ACTIVE); 477 return; 478 } 479 480 for (cap = params; cap; cap = cap->next) { 481 s->enabled_capabilities[cap->value->capability] = cap->value->state; 482 } 483 } 484 485 void qmp_migrate_set_parameters(bool has_compress_level, 486 int64_t compress_level, 487 bool has_compress_threads, 488 int64_t compress_threads, 489 bool has_decompress_threads, 490 int64_t decompress_threads, Error **errp) 491 { 492 MigrationState *s = migrate_get_current(); 493 494 if (has_compress_level && (compress_level < 0 || compress_level > 9)) { 495 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level", 496 "is invalid, it should be in the range of 0 to 9"); 497 return; 498 } 499 if (has_compress_threads && 500 (compress_threads < 1 || compress_threads > 255)) { 501 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 502 "compress_threads", 503 "is invalid, it should be in the range of 1 to 255"); 504 return; 505 } 506 if (has_decompress_threads && 507 (decompress_threads < 1 || decompress_threads > 255)) { 508 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 509 "decompress_threads", 510 "is invalid, it should be in the range of 1 to 255"); 511 return; 512 } 513 514 if (has_compress_level) { 515 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 516 } 517 if (has_compress_threads) { 518 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = compress_threads; 519 } 520 if (has_decompress_threads) { 521 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 522 decompress_threads; 523 } 524 } 525 526 /* shared migration helpers */ 527 528 static void migrate_set_state(MigrationState *s, int old_state, int new_state) 529 { 530 if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) { 531 trace_migrate_set_state(new_state); 532 migrate_generate_event(new_state); 533 } 534 } 535 536 static void migrate_fd_cleanup(void *opaque) 537 { 538 MigrationState *s = opaque; 539 540 qemu_bh_delete(s->cleanup_bh); 541 s->cleanup_bh = NULL; 542 543 if (s->file) { 544 trace_migrate_fd_cleanup(); 545 qemu_mutex_unlock_iothread(); 546 qemu_thread_join(&s->thread); 547 qemu_mutex_lock_iothread(); 548 549 migrate_compress_threads_join(); 550 qemu_fclose(s->file); 551 s->file = NULL; 552 } 553 554 assert(s->state != MIGRATION_STATUS_ACTIVE); 555 556 if (s->state != MIGRATION_STATUS_COMPLETED) { 557 qemu_savevm_state_cancel(); 558 if (s->state == MIGRATION_STATUS_CANCELLING) { 559 migrate_set_state(s, MIGRATION_STATUS_CANCELLING, 560 MIGRATION_STATUS_CANCELLED); 561 } 562 } 563 564 notifier_list_notify(&migration_state_notifiers, s); 565 } 566 567 void migrate_fd_error(MigrationState *s) 568 { 569 trace_migrate_fd_error(); 570 assert(s->file == NULL); 571 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 572 notifier_list_notify(&migration_state_notifiers, s); 573 } 574 575 static void migrate_fd_cancel(MigrationState *s) 576 { 577 int old_state ; 578 QEMUFile *f = migrate_get_current()->file; 579 trace_migrate_fd_cancel(); 580 581 do { 582 old_state = s->state; 583 if (old_state != MIGRATION_STATUS_SETUP && 584 old_state != MIGRATION_STATUS_ACTIVE) { 585 break; 586 } 587 migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING); 588 } while (s->state != MIGRATION_STATUS_CANCELLING); 589 590 /* 591 * If we're unlucky the migration code might be stuck somewhere in a 592 * send/write while the network has failed and is waiting to timeout; 593 * if we've got shutdown(2) available then we can force it to quit. 594 * The outgoing qemu file gets closed in migrate_fd_cleanup that is 595 * called in a bh, so there is no race against this cancel. 596 */ 597 if (s->state == MIGRATION_STATUS_CANCELLING && f) { 598 qemu_file_shutdown(f); 599 } 600 } 601 602 void add_migration_state_change_notifier(Notifier *notify) 603 { 604 notifier_list_add(&migration_state_notifiers, notify); 605 } 606 607 void remove_migration_state_change_notifier(Notifier *notify) 608 { 609 notifier_remove(notify); 610 } 611 612 bool migration_in_setup(MigrationState *s) 613 { 614 return s->state == MIGRATION_STATUS_SETUP; 615 } 616 617 bool migration_has_finished(MigrationState *s) 618 { 619 return s->state == MIGRATION_STATUS_COMPLETED; 620 } 621 622 bool migration_has_failed(MigrationState *s) 623 { 624 return (s->state == MIGRATION_STATUS_CANCELLED || 625 s->state == MIGRATION_STATUS_FAILED); 626 } 627 628 static MigrationState *migrate_init(const MigrationParams *params) 629 { 630 MigrationState *s = migrate_get_current(); 631 int64_t bandwidth_limit = s->bandwidth_limit; 632 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX]; 633 int64_t xbzrle_cache_size = s->xbzrle_cache_size; 634 int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 635 int compress_thread_count = 636 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 637 int decompress_thread_count = 638 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 639 640 memcpy(enabled_capabilities, s->enabled_capabilities, 641 sizeof(enabled_capabilities)); 642 643 memset(s, 0, sizeof(*s)); 644 s->params = *params; 645 memcpy(s->enabled_capabilities, enabled_capabilities, 646 sizeof(enabled_capabilities)); 647 s->xbzrle_cache_size = xbzrle_cache_size; 648 649 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 650 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 651 compress_thread_count; 652 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 653 decompress_thread_count; 654 s->bandwidth_limit = bandwidth_limit; 655 migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); 656 657 s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 658 return s; 659 } 660 661 static GSList *migration_blockers; 662 663 void migrate_add_blocker(Error *reason) 664 { 665 migration_blockers = g_slist_prepend(migration_blockers, reason); 666 } 667 668 void migrate_del_blocker(Error *reason) 669 { 670 migration_blockers = g_slist_remove(migration_blockers, reason); 671 } 672 673 void qmp_migrate_incoming(const char *uri, Error **errp) 674 { 675 Error *local_err = NULL; 676 static bool once = true; 677 678 if (!deferred_incoming) { 679 error_setg(errp, "For use with '-incoming defer'"); 680 return; 681 } 682 if (!once) { 683 error_setg(errp, "The incoming migration has already been started"); 684 } 685 686 qemu_start_incoming_migration(uri, &local_err); 687 688 if (local_err) { 689 error_propagate(errp, local_err); 690 return; 691 } 692 693 once = false; 694 } 695 696 void qmp_migrate(const char *uri, bool has_blk, bool blk, 697 bool has_inc, bool inc, bool has_detach, bool detach, 698 Error **errp) 699 { 700 Error *local_err = NULL; 701 MigrationState *s = migrate_get_current(); 702 MigrationParams params; 703 const char *p; 704 705 params.blk = has_blk && blk; 706 params.shared = has_inc && inc; 707 708 if (s->state == MIGRATION_STATUS_ACTIVE || 709 s->state == MIGRATION_STATUS_SETUP || 710 s->state == MIGRATION_STATUS_CANCELLING) { 711 error_setg(errp, QERR_MIGRATION_ACTIVE); 712 return; 713 } 714 if (runstate_check(RUN_STATE_INMIGRATE)) { 715 error_setg(errp, "Guest is waiting for an incoming migration"); 716 return; 717 } 718 719 if (qemu_savevm_state_blocked(errp)) { 720 return; 721 } 722 723 if (migration_blockers) { 724 *errp = error_copy(migration_blockers->data); 725 return; 726 } 727 728 /* We are starting a new migration, so we want to start in a clean 729 state. This change is only needed if previous migration 730 failed/was cancelled. We don't use migrate_set_state() because 731 we are setting the initial state, not changing it. */ 732 s->state = MIGRATION_STATUS_NONE; 733 734 s = migrate_init(¶ms); 735 736 if (strstart(uri, "tcp:", &p)) { 737 tcp_start_outgoing_migration(s, p, &local_err); 738 #ifdef CONFIG_RDMA 739 } else if (strstart(uri, "rdma:", &p)) { 740 rdma_start_outgoing_migration(s, p, &local_err); 741 #endif 742 #if !defined(WIN32) 743 } else if (strstart(uri, "exec:", &p)) { 744 exec_start_outgoing_migration(s, p, &local_err); 745 } else if (strstart(uri, "unix:", &p)) { 746 unix_start_outgoing_migration(s, p, &local_err); 747 } else if (strstart(uri, "fd:", &p)) { 748 fd_start_outgoing_migration(s, p, &local_err); 749 #endif 750 } else { 751 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", 752 "a valid migration protocol"); 753 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 754 return; 755 } 756 757 if (local_err) { 758 migrate_fd_error(s); 759 error_propagate(errp, local_err); 760 return; 761 } 762 } 763 764 void qmp_migrate_cancel(Error **errp) 765 { 766 migrate_fd_cancel(migrate_get_current()); 767 } 768 769 void qmp_migrate_set_cache_size(int64_t value, Error **errp) 770 { 771 MigrationState *s = migrate_get_current(); 772 int64_t new_size; 773 774 /* Check for truncation */ 775 if (value != (size_t)value) { 776 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 777 "exceeding address space"); 778 return; 779 } 780 781 /* Cache should not be larger than guest ram size */ 782 if (value > ram_bytes_total()) { 783 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 784 "exceeds guest ram size "); 785 return; 786 } 787 788 new_size = xbzrle_cache_resize(value); 789 if (new_size < 0) { 790 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 791 "is smaller than page size"); 792 return; 793 } 794 795 s->xbzrle_cache_size = new_size; 796 } 797 798 int64_t qmp_query_migrate_cache_size(Error **errp) 799 { 800 return migrate_xbzrle_cache_size(); 801 } 802 803 void qmp_migrate_set_speed(int64_t value, Error **errp) 804 { 805 MigrationState *s; 806 807 if (value < 0) { 808 value = 0; 809 } 810 if (value > SIZE_MAX) { 811 value = SIZE_MAX; 812 } 813 814 s = migrate_get_current(); 815 s->bandwidth_limit = value; 816 if (s->file) { 817 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO); 818 } 819 } 820 821 void qmp_migrate_set_downtime(double value, Error **errp) 822 { 823 value *= 1e9; 824 value = MAX(0, MIN(UINT64_MAX, value)); 825 max_downtime = (uint64_t)value; 826 } 827 828 bool migrate_auto_converge(void) 829 { 830 MigrationState *s; 831 832 s = migrate_get_current(); 833 834 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE]; 835 } 836 837 bool migrate_zero_blocks(void) 838 { 839 MigrationState *s; 840 841 s = migrate_get_current(); 842 843 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS]; 844 } 845 846 bool migrate_use_compression(void) 847 { 848 MigrationState *s; 849 850 s = migrate_get_current(); 851 852 return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS]; 853 } 854 855 int migrate_compress_level(void) 856 { 857 MigrationState *s; 858 859 s = migrate_get_current(); 860 861 return s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 862 } 863 864 int migrate_compress_threads(void) 865 { 866 MigrationState *s; 867 868 s = migrate_get_current(); 869 870 return s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 871 } 872 873 int migrate_decompress_threads(void) 874 { 875 MigrationState *s; 876 877 s = migrate_get_current(); 878 879 return s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 880 } 881 882 bool migrate_use_events(void) 883 { 884 MigrationState *s; 885 886 s = migrate_get_current(); 887 888 return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS]; 889 } 890 891 int migrate_use_xbzrle(void) 892 { 893 MigrationState *s; 894 895 s = migrate_get_current(); 896 897 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE]; 898 } 899 900 int64_t migrate_xbzrle_cache_size(void) 901 { 902 MigrationState *s; 903 904 s = migrate_get_current(); 905 906 return s->xbzrle_cache_size; 907 } 908 909 /* migration thread support */ 910 911 static void *migration_thread(void *opaque) 912 { 913 MigrationState *s = opaque; 914 int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 915 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); 916 int64_t initial_bytes = 0; 917 int64_t max_size = 0; 918 int64_t start_time = initial_time; 919 bool old_vm_running = false; 920 921 rcu_register_thread(); 922 923 qemu_savevm_state_header(s->file); 924 qemu_savevm_state_begin(s->file, &s->params); 925 926 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; 927 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); 928 929 while (s->state == MIGRATION_STATUS_ACTIVE) { 930 int64_t current_time; 931 uint64_t pending_size; 932 933 if (!qemu_file_rate_limit(s->file)) { 934 pending_size = qemu_savevm_state_pending(s->file, max_size); 935 trace_migrate_pending(pending_size, max_size); 936 if (pending_size && pending_size >= max_size) { 937 qemu_savevm_state_iterate(s->file); 938 } else { 939 int ret; 940 941 qemu_mutex_lock_iothread(); 942 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 943 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); 944 old_vm_running = runstate_is_running(); 945 946 ret = global_state_store(); 947 if (!ret) { 948 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); 949 if (ret >= 0) { 950 qemu_file_set_rate_limit(s->file, INT64_MAX); 951 qemu_savevm_state_complete(s->file); 952 } 953 } 954 qemu_mutex_unlock_iothread(); 955 956 if (ret < 0) { 957 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 958 MIGRATION_STATUS_FAILED); 959 break; 960 } 961 962 if (!qemu_file_get_error(s->file)) { 963 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 964 MIGRATION_STATUS_COMPLETED); 965 break; 966 } 967 } 968 } 969 970 if (qemu_file_get_error(s->file)) { 971 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 972 MIGRATION_STATUS_FAILED); 973 break; 974 } 975 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 976 if (current_time >= initial_time + BUFFER_DELAY) { 977 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes; 978 uint64_t time_spent = current_time - initial_time; 979 double bandwidth = transferred_bytes / time_spent; 980 max_size = bandwidth * migrate_max_downtime() / 1000000; 981 982 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) / 983 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1; 984 985 trace_migrate_transferred(transferred_bytes, time_spent, 986 bandwidth, max_size); 987 /* if we haven't sent anything, we don't want to recalculate 988 10000 is a small enough number for our purposes */ 989 if (s->dirty_bytes_rate && transferred_bytes > 10000) { 990 s->expected_downtime = s->dirty_bytes_rate / bandwidth; 991 } 992 993 qemu_file_reset_rate_limit(s->file); 994 initial_time = current_time; 995 initial_bytes = qemu_ftell(s->file); 996 } 997 if (qemu_file_rate_limit(s->file)) { 998 /* usleep expects microseconds */ 999 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000); 1000 } 1001 } 1002 1003 qemu_mutex_lock_iothread(); 1004 if (s->state == MIGRATION_STATUS_COMPLETED) { 1005 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 1006 uint64_t transferred_bytes = qemu_ftell(s->file); 1007 s->total_time = end_time - s->total_time; 1008 s->downtime = end_time - start_time; 1009 if (s->total_time) { 1010 s->mbps = (((double) transferred_bytes * 8.0) / 1011 ((double) s->total_time)) / 1000; 1012 } 1013 runstate_set(RUN_STATE_POSTMIGRATE); 1014 } else { 1015 if (old_vm_running) { 1016 vm_start(); 1017 } 1018 } 1019 qemu_bh_schedule(s->cleanup_bh); 1020 qemu_mutex_unlock_iothread(); 1021 1022 rcu_unregister_thread(); 1023 return NULL; 1024 } 1025 1026 void migrate_fd_connect(MigrationState *s) 1027 { 1028 /* This is a best 1st approximation. ns to ms */ 1029 s->expected_downtime = max_downtime/1000000; 1030 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s); 1031 1032 qemu_file_set_rate_limit(s->file, 1033 s->bandwidth_limit / XFER_LIMIT_RATIO); 1034 1035 /* Notify before starting migration thread */ 1036 notifier_list_notify(&migration_state_notifiers, s); 1037 1038 migrate_compress_threads_create(); 1039 qemu_thread_create(&s->thread, "migration", migration_thread, s, 1040 QEMU_THREAD_JOINABLE); 1041 } 1042