1 /* 2 * QEMU live migration 3 * 4 * Copyright IBM, Corp. 2008 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu-common.h" 17 #include "qemu/error-report.h" 18 #include "qemu/main-loop.h" 19 #include "migration/migration.h" 20 #include "migration/qemu-file.h" 21 #include "sysemu/sysemu.h" 22 #include "block/block.h" 23 #include "qapi/qmp/qerror.h" 24 #include "qemu/sockets.h" 25 #include "migration/block.h" 26 #include "qemu/thread.h" 27 #include "qmp-commands.h" 28 #include "trace.h" 29 #include "qapi/util.h" 30 #include "qapi-event.h" 31 32 #define MAX_THROTTLE (32 << 20) /* Migration speed throttling */ 33 34 /* Amount of time to allocate to each "chunk" of bandwidth-throttled 35 * data. */ 36 #define BUFFER_DELAY 100 37 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) 38 39 /* Default compression thread count */ 40 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8 41 /* Default decompression thread count, usually decompression is at 42 * least 4 times as fast as compression.*/ 43 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 44 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */ 45 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 46 47 /* Migration XBZRLE default cache size */ 48 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) 49 50 static NotifierList migration_state_notifiers = 51 NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); 52 53 static bool deferred_incoming; 54 55 /* When we add fault tolerance, we could have several 56 migrations at once. For now we don't need to add 57 dynamic creation of migration */ 58 59 /* For outgoing */ 60 MigrationState *migrate_get_current(void) 61 { 62 static MigrationState current_migration = { 63 .state = MIGRATION_STATUS_NONE, 64 .bandwidth_limit = MAX_THROTTLE, 65 .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE, 66 .mbps = -1, 67 .parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = 68 DEFAULT_MIGRATE_COMPRESS_LEVEL, 69 .parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 70 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT, 71 .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 72 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT, 73 }; 74 75 return ¤t_migration; 76 } 77 78 /* For incoming */ 79 static MigrationIncomingState *mis_current; 80 81 MigrationIncomingState *migration_incoming_get_current(void) 82 { 83 return mis_current; 84 } 85 86 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f) 87 { 88 mis_current = g_malloc0(sizeof(MigrationIncomingState)); 89 mis_current->file = f; 90 QLIST_INIT(&mis_current->loadvm_handlers); 91 92 return mis_current; 93 } 94 95 void migration_incoming_state_destroy(void) 96 { 97 loadvm_free_handlers(mis_current); 98 g_free(mis_current); 99 mis_current = NULL; 100 } 101 102 103 typedef struct { 104 bool optional; 105 uint32_t size; 106 uint8_t runstate[100]; 107 } GlobalState; 108 109 static GlobalState global_state; 110 111 static int global_state_store(void) 112 { 113 if (!runstate_store((char *)global_state.runstate, 114 sizeof(global_state.runstate))) { 115 error_report("runstate name too big: %s", global_state.runstate); 116 trace_migrate_state_too_big(); 117 return -EINVAL; 118 } 119 return 0; 120 } 121 122 static char *global_state_get_runstate(void) 123 { 124 return (char *)global_state.runstate; 125 } 126 127 void global_state_set_optional(void) 128 { 129 global_state.optional = true; 130 } 131 132 static bool global_state_needed(void *opaque) 133 { 134 GlobalState *s = opaque; 135 char *runstate = (char *)s->runstate; 136 137 /* If it is not optional, it is mandatory */ 138 139 if (s->optional == false) { 140 return true; 141 } 142 143 /* If state is running or paused, it is not needed */ 144 145 if (strcmp(runstate, "running") == 0 || 146 strcmp(runstate, "paused") == 0) { 147 return false; 148 } 149 150 /* for any other state it is needed */ 151 return true; 152 } 153 154 static int global_state_post_load(void *opaque, int version_id) 155 { 156 GlobalState *s = opaque; 157 int ret = 0; 158 char *runstate = (char *)s->runstate; 159 160 trace_migrate_global_state_post_load(runstate); 161 162 if (strcmp(runstate, "running") != 0) { 163 Error *local_err = NULL; 164 int r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX, 165 -1, &local_err); 166 167 if (r == -1) { 168 if (local_err) { 169 error_report_err(local_err); 170 } 171 return -EINVAL; 172 } 173 ret = vm_stop_force_state(r); 174 } 175 176 return ret; 177 } 178 179 static void global_state_pre_save(void *opaque) 180 { 181 GlobalState *s = opaque; 182 183 trace_migrate_global_state_pre_save((char *)s->runstate); 184 s->size = strlen((char *)s->runstate) + 1; 185 } 186 187 static const VMStateDescription vmstate_globalstate = { 188 .name = "globalstate", 189 .version_id = 1, 190 .minimum_version_id = 1, 191 .post_load = global_state_post_load, 192 .pre_save = global_state_pre_save, 193 .needed = global_state_needed, 194 .fields = (VMStateField[]) { 195 VMSTATE_UINT32(size, GlobalState), 196 VMSTATE_BUFFER(runstate, GlobalState), 197 VMSTATE_END_OF_LIST() 198 }, 199 }; 200 201 void register_global_state(void) 202 { 203 /* We would use it independently that we receive it */ 204 strcpy((char *)&global_state.runstate, ""); 205 vmstate_register(NULL, 0, &vmstate_globalstate, &global_state); 206 } 207 208 static void migrate_generate_event(int new_state) 209 { 210 if (migrate_use_events()) { 211 qapi_event_send_migration(new_state, &error_abort); 212 trace_migrate_set_state(new_state); 213 } 214 } 215 216 /* 217 * Called on -incoming with a defer: uri. 218 * The migration can be started later after any parameters have been 219 * changed. 220 */ 221 static void deferred_incoming_migration(Error **errp) 222 { 223 if (deferred_incoming) { 224 error_setg(errp, "Incoming migration already deferred"); 225 } 226 deferred_incoming = true; 227 } 228 229 void qemu_start_incoming_migration(const char *uri, Error **errp) 230 { 231 const char *p; 232 233 qapi_event_send_migration(MIGRATION_STATUS_SETUP, &error_abort); 234 if (!strcmp(uri, "defer")) { 235 deferred_incoming_migration(errp); 236 } else if (strstart(uri, "tcp:", &p)) { 237 tcp_start_incoming_migration(p, errp); 238 #ifdef CONFIG_RDMA 239 } else if (strstart(uri, "rdma:", &p)) { 240 rdma_start_incoming_migration(p, errp); 241 #endif 242 #if !defined(WIN32) 243 } else if (strstart(uri, "exec:", &p)) { 244 exec_start_incoming_migration(p, errp); 245 } else if (strstart(uri, "unix:", &p)) { 246 unix_start_incoming_migration(p, errp); 247 } else if (strstart(uri, "fd:", &p)) { 248 fd_start_incoming_migration(p, errp); 249 #endif 250 } else { 251 error_setg(errp, "unknown migration protocol: %s", uri); 252 } 253 } 254 255 static void process_incoming_migration_co(void *opaque) 256 { 257 QEMUFile *f = opaque; 258 Error *local_err = NULL; 259 int ret; 260 261 migration_incoming_state_new(f); 262 migrate_generate_event(MIGRATION_STATUS_ACTIVE); 263 ret = qemu_loadvm_state(f); 264 265 qemu_fclose(f); 266 free_xbzrle_decoded_buf(); 267 migration_incoming_state_destroy(); 268 269 if (ret < 0) { 270 migrate_generate_event(MIGRATION_STATUS_FAILED); 271 error_report("load of migration failed: %s", strerror(-ret)); 272 migrate_decompress_threads_join(); 273 exit(EXIT_FAILURE); 274 } 275 migrate_generate_event(MIGRATION_STATUS_COMPLETED); 276 qemu_announce_self(); 277 278 /* Make sure all file formats flush their mutable metadata */ 279 bdrv_invalidate_cache_all(&local_err); 280 if (local_err) { 281 error_report_err(local_err); 282 migrate_decompress_threads_join(); 283 exit(EXIT_FAILURE); 284 } 285 286 /* runstate == "" means that we haven't received it through the 287 * wire, so we obey autostart. runstate == runing means that we 288 * need to run it, we need to make sure that we do it after 289 * everything else has finished. Every other state change is done 290 * at the post_load function */ 291 292 if (strcmp(global_state_get_runstate(), "running") == 0) { 293 vm_start(); 294 } else if (strcmp(global_state_get_runstate(), "") == 0) { 295 if (autostart) { 296 vm_start(); 297 } else { 298 runstate_set(RUN_STATE_PAUSED); 299 } 300 } 301 migrate_decompress_threads_join(); 302 } 303 304 void process_incoming_migration(QEMUFile *f) 305 { 306 Coroutine *co = qemu_coroutine_create(process_incoming_migration_co); 307 int fd = qemu_get_fd(f); 308 309 assert(fd != -1); 310 migrate_decompress_threads_create(); 311 qemu_set_nonblock(fd); 312 qemu_coroutine_enter(co, f); 313 } 314 315 /* amount of nanoseconds we are willing to wait for migration to be down. 316 * the choice of nanoseconds is because it is the maximum resolution that 317 * get_clock() can achieve. It is an internal measure. All user-visible 318 * units must be in seconds */ 319 static uint64_t max_downtime = 300000000; 320 321 uint64_t migrate_max_downtime(void) 322 { 323 return max_downtime; 324 } 325 326 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) 327 { 328 MigrationCapabilityStatusList *head = NULL; 329 MigrationCapabilityStatusList *caps; 330 MigrationState *s = migrate_get_current(); 331 int i; 332 333 caps = NULL; /* silence compiler warning */ 334 for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) { 335 if (head == NULL) { 336 head = g_malloc0(sizeof(*caps)); 337 caps = head; 338 } else { 339 caps->next = g_malloc0(sizeof(*caps)); 340 caps = caps->next; 341 } 342 caps->value = 343 g_malloc(sizeof(*caps->value)); 344 caps->value->capability = i; 345 caps->value->state = s->enabled_capabilities[i]; 346 } 347 348 return head; 349 } 350 351 MigrationParameters *qmp_query_migrate_parameters(Error **errp) 352 { 353 MigrationParameters *params; 354 MigrationState *s = migrate_get_current(); 355 356 params = g_malloc0(sizeof(*params)); 357 params->compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 358 params->compress_threads = 359 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 360 params->decompress_threads = 361 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 362 363 return params; 364 } 365 366 static void get_xbzrle_cache_stats(MigrationInfo *info) 367 { 368 if (migrate_use_xbzrle()) { 369 info->has_xbzrle_cache = true; 370 info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache)); 371 info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size(); 372 info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred(); 373 info->xbzrle_cache->pages = xbzrle_mig_pages_transferred(); 374 info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss(); 375 info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate(); 376 info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow(); 377 } 378 } 379 380 MigrationInfo *qmp_query_migrate(Error **errp) 381 { 382 MigrationInfo *info = g_malloc0(sizeof(*info)); 383 MigrationState *s = migrate_get_current(); 384 385 switch (s->state) { 386 case MIGRATION_STATUS_NONE: 387 /* no migration has happened ever */ 388 break; 389 case MIGRATION_STATUS_SETUP: 390 info->has_status = true; 391 info->has_total_time = false; 392 break; 393 case MIGRATION_STATUS_ACTIVE: 394 case MIGRATION_STATUS_CANCELLING: 395 info->has_status = true; 396 info->has_total_time = true; 397 info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) 398 - s->total_time; 399 info->has_expected_downtime = true; 400 info->expected_downtime = s->expected_downtime; 401 info->has_setup_time = true; 402 info->setup_time = s->setup_time; 403 404 info->has_ram = true; 405 info->ram = g_malloc0(sizeof(*info->ram)); 406 info->ram->transferred = ram_bytes_transferred(); 407 info->ram->remaining = ram_bytes_remaining(); 408 info->ram->total = ram_bytes_total(); 409 info->ram->duplicate = dup_mig_pages_transferred(); 410 info->ram->skipped = skipped_mig_pages_transferred(); 411 info->ram->normal = norm_mig_pages_transferred(); 412 info->ram->normal_bytes = norm_mig_bytes_transferred(); 413 info->ram->dirty_pages_rate = s->dirty_pages_rate; 414 info->ram->mbps = s->mbps; 415 info->ram->dirty_sync_count = s->dirty_sync_count; 416 417 if (blk_mig_active()) { 418 info->has_disk = true; 419 info->disk = g_malloc0(sizeof(*info->disk)); 420 info->disk->transferred = blk_mig_bytes_transferred(); 421 info->disk->remaining = blk_mig_bytes_remaining(); 422 info->disk->total = blk_mig_bytes_total(); 423 } 424 425 get_xbzrle_cache_stats(info); 426 break; 427 case MIGRATION_STATUS_COMPLETED: 428 get_xbzrle_cache_stats(info); 429 430 info->has_status = true; 431 info->has_total_time = true; 432 info->total_time = s->total_time; 433 info->has_downtime = true; 434 info->downtime = s->downtime; 435 info->has_setup_time = true; 436 info->setup_time = s->setup_time; 437 438 info->has_ram = true; 439 info->ram = g_malloc0(sizeof(*info->ram)); 440 info->ram->transferred = ram_bytes_transferred(); 441 info->ram->remaining = 0; 442 info->ram->total = ram_bytes_total(); 443 info->ram->duplicate = dup_mig_pages_transferred(); 444 info->ram->skipped = skipped_mig_pages_transferred(); 445 info->ram->normal = norm_mig_pages_transferred(); 446 info->ram->normal_bytes = norm_mig_bytes_transferred(); 447 info->ram->mbps = s->mbps; 448 info->ram->dirty_sync_count = s->dirty_sync_count; 449 break; 450 case MIGRATION_STATUS_FAILED: 451 info->has_status = true; 452 break; 453 case MIGRATION_STATUS_CANCELLED: 454 info->has_status = true; 455 break; 456 } 457 info->status = s->state; 458 459 return info; 460 } 461 462 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, 463 Error **errp) 464 { 465 MigrationState *s = migrate_get_current(); 466 MigrationCapabilityStatusList *cap; 467 468 if (s->state == MIGRATION_STATUS_ACTIVE || 469 s->state == MIGRATION_STATUS_SETUP) { 470 error_setg(errp, QERR_MIGRATION_ACTIVE); 471 return; 472 } 473 474 for (cap = params; cap; cap = cap->next) { 475 s->enabled_capabilities[cap->value->capability] = cap->value->state; 476 } 477 } 478 479 void qmp_migrate_set_parameters(bool has_compress_level, 480 int64_t compress_level, 481 bool has_compress_threads, 482 int64_t compress_threads, 483 bool has_decompress_threads, 484 int64_t decompress_threads, Error **errp) 485 { 486 MigrationState *s = migrate_get_current(); 487 488 if (has_compress_level && (compress_level < 0 || compress_level > 9)) { 489 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level", 490 "is invalid, it should be in the range of 0 to 9"); 491 return; 492 } 493 if (has_compress_threads && 494 (compress_threads < 1 || compress_threads > 255)) { 495 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 496 "compress_threads", 497 "is invalid, it should be in the range of 1 to 255"); 498 return; 499 } 500 if (has_decompress_threads && 501 (decompress_threads < 1 || decompress_threads > 255)) { 502 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, 503 "decompress_threads", 504 "is invalid, it should be in the range of 1 to 255"); 505 return; 506 } 507 508 if (has_compress_level) { 509 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 510 } 511 if (has_compress_threads) { 512 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = compress_threads; 513 } 514 if (has_decompress_threads) { 515 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 516 decompress_threads; 517 } 518 } 519 520 /* shared migration helpers */ 521 522 static void migrate_set_state(MigrationState *s, int old_state, int new_state) 523 { 524 if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) { 525 migrate_generate_event(new_state); 526 } 527 } 528 529 static void migrate_fd_cleanup(void *opaque) 530 { 531 MigrationState *s = opaque; 532 533 qemu_bh_delete(s->cleanup_bh); 534 s->cleanup_bh = NULL; 535 536 if (s->file) { 537 trace_migrate_fd_cleanup(); 538 qemu_mutex_unlock_iothread(); 539 qemu_thread_join(&s->thread); 540 qemu_mutex_lock_iothread(); 541 542 migrate_compress_threads_join(); 543 qemu_fclose(s->file); 544 s->file = NULL; 545 } 546 547 assert(s->state != MIGRATION_STATUS_ACTIVE); 548 549 if (s->state != MIGRATION_STATUS_COMPLETED) { 550 qemu_savevm_state_cancel(); 551 if (s->state == MIGRATION_STATUS_CANCELLING) { 552 migrate_set_state(s, MIGRATION_STATUS_CANCELLING, 553 MIGRATION_STATUS_CANCELLED); 554 } 555 } 556 557 notifier_list_notify(&migration_state_notifiers, s); 558 } 559 560 void migrate_fd_error(MigrationState *s) 561 { 562 trace_migrate_fd_error(); 563 assert(s->file == NULL); 564 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 565 notifier_list_notify(&migration_state_notifiers, s); 566 } 567 568 static void migrate_fd_cancel(MigrationState *s) 569 { 570 int old_state ; 571 QEMUFile *f = migrate_get_current()->file; 572 trace_migrate_fd_cancel(); 573 574 do { 575 old_state = s->state; 576 if (old_state != MIGRATION_STATUS_SETUP && 577 old_state != MIGRATION_STATUS_ACTIVE) { 578 break; 579 } 580 migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING); 581 } while (s->state != MIGRATION_STATUS_CANCELLING); 582 583 /* 584 * If we're unlucky the migration code might be stuck somewhere in a 585 * send/write while the network has failed and is waiting to timeout; 586 * if we've got shutdown(2) available then we can force it to quit. 587 * The outgoing qemu file gets closed in migrate_fd_cleanup that is 588 * called in a bh, so there is no race against this cancel. 589 */ 590 if (s->state == MIGRATION_STATUS_CANCELLING && f) { 591 qemu_file_shutdown(f); 592 } 593 } 594 595 void add_migration_state_change_notifier(Notifier *notify) 596 { 597 notifier_list_add(&migration_state_notifiers, notify); 598 } 599 600 void remove_migration_state_change_notifier(Notifier *notify) 601 { 602 notifier_remove(notify); 603 } 604 605 bool migration_in_setup(MigrationState *s) 606 { 607 return s->state == MIGRATION_STATUS_SETUP; 608 } 609 610 bool migration_has_finished(MigrationState *s) 611 { 612 return s->state == MIGRATION_STATUS_COMPLETED; 613 } 614 615 bool migration_has_failed(MigrationState *s) 616 { 617 return (s->state == MIGRATION_STATUS_CANCELLED || 618 s->state == MIGRATION_STATUS_FAILED); 619 } 620 621 static MigrationState *migrate_init(const MigrationParams *params) 622 { 623 MigrationState *s = migrate_get_current(); 624 int64_t bandwidth_limit = s->bandwidth_limit; 625 bool enabled_capabilities[MIGRATION_CAPABILITY_MAX]; 626 int64_t xbzrle_cache_size = s->xbzrle_cache_size; 627 int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 628 int compress_thread_count = 629 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 630 int decompress_thread_count = 631 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 632 633 memcpy(enabled_capabilities, s->enabled_capabilities, 634 sizeof(enabled_capabilities)); 635 636 memset(s, 0, sizeof(*s)); 637 s->params = *params; 638 memcpy(s->enabled_capabilities, enabled_capabilities, 639 sizeof(enabled_capabilities)); 640 s->xbzrle_cache_size = xbzrle_cache_size; 641 642 s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; 643 s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = 644 compress_thread_count; 645 s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = 646 decompress_thread_count; 647 s->bandwidth_limit = bandwidth_limit; 648 migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); 649 650 s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 651 return s; 652 } 653 654 static GSList *migration_blockers; 655 656 void migrate_add_blocker(Error *reason) 657 { 658 migration_blockers = g_slist_prepend(migration_blockers, reason); 659 } 660 661 void migrate_del_blocker(Error *reason) 662 { 663 migration_blockers = g_slist_remove(migration_blockers, reason); 664 } 665 666 void qmp_migrate_incoming(const char *uri, Error **errp) 667 { 668 Error *local_err = NULL; 669 static bool once = true; 670 671 if (!deferred_incoming) { 672 error_setg(errp, "For use with '-incoming defer'"); 673 return; 674 } 675 if (!once) { 676 error_setg(errp, "The incoming migration has already been started"); 677 } 678 679 qemu_start_incoming_migration(uri, &local_err); 680 681 if (local_err) { 682 error_propagate(errp, local_err); 683 return; 684 } 685 686 once = false; 687 } 688 689 void qmp_migrate(const char *uri, bool has_blk, bool blk, 690 bool has_inc, bool inc, bool has_detach, bool detach, 691 Error **errp) 692 { 693 Error *local_err = NULL; 694 MigrationState *s = migrate_get_current(); 695 MigrationParams params; 696 const char *p; 697 698 params.blk = has_blk && blk; 699 params.shared = has_inc && inc; 700 701 if (s->state == MIGRATION_STATUS_ACTIVE || 702 s->state == MIGRATION_STATUS_SETUP || 703 s->state == MIGRATION_STATUS_CANCELLING) { 704 error_setg(errp, QERR_MIGRATION_ACTIVE); 705 return; 706 } 707 if (runstate_check(RUN_STATE_INMIGRATE)) { 708 error_setg(errp, "Guest is waiting for an incoming migration"); 709 return; 710 } 711 712 if (qemu_savevm_state_blocked(errp)) { 713 return; 714 } 715 716 if (migration_blockers) { 717 *errp = error_copy(migration_blockers->data); 718 return; 719 } 720 721 /* We are starting a new migration, so we want to start in a clean 722 state. This change is only needed if previous migration 723 failed/was cancelled. We don't use migrate_set_state() because 724 we are setting the initial state, not changing it. */ 725 s->state = MIGRATION_STATUS_NONE; 726 727 s = migrate_init(¶ms); 728 729 if (strstart(uri, "tcp:", &p)) { 730 tcp_start_outgoing_migration(s, p, &local_err); 731 #ifdef CONFIG_RDMA 732 } else if (strstart(uri, "rdma:", &p)) { 733 rdma_start_outgoing_migration(s, p, &local_err); 734 #endif 735 #if !defined(WIN32) 736 } else if (strstart(uri, "exec:", &p)) { 737 exec_start_outgoing_migration(s, p, &local_err); 738 } else if (strstart(uri, "unix:", &p)) { 739 unix_start_outgoing_migration(s, p, &local_err); 740 } else if (strstart(uri, "fd:", &p)) { 741 fd_start_outgoing_migration(s, p, &local_err); 742 #endif 743 } else { 744 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", 745 "a valid migration protocol"); 746 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); 747 return; 748 } 749 750 if (local_err) { 751 migrate_fd_error(s); 752 error_propagate(errp, local_err); 753 return; 754 } 755 } 756 757 void qmp_migrate_cancel(Error **errp) 758 { 759 migrate_fd_cancel(migrate_get_current()); 760 } 761 762 void qmp_migrate_set_cache_size(int64_t value, Error **errp) 763 { 764 MigrationState *s = migrate_get_current(); 765 int64_t new_size; 766 767 /* Check for truncation */ 768 if (value != (size_t)value) { 769 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 770 "exceeding address space"); 771 return; 772 } 773 774 /* Cache should not be larger than guest ram size */ 775 if (value > ram_bytes_total()) { 776 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 777 "exceeds guest ram size "); 778 return; 779 } 780 781 new_size = xbzrle_cache_resize(value); 782 if (new_size < 0) { 783 error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", 784 "is smaller than page size"); 785 return; 786 } 787 788 s->xbzrle_cache_size = new_size; 789 } 790 791 int64_t qmp_query_migrate_cache_size(Error **errp) 792 { 793 return migrate_xbzrle_cache_size(); 794 } 795 796 void qmp_migrate_set_speed(int64_t value, Error **errp) 797 { 798 MigrationState *s; 799 800 if (value < 0) { 801 value = 0; 802 } 803 if (value > SIZE_MAX) { 804 value = SIZE_MAX; 805 } 806 807 s = migrate_get_current(); 808 s->bandwidth_limit = value; 809 if (s->file) { 810 qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO); 811 } 812 } 813 814 void qmp_migrate_set_downtime(double value, Error **errp) 815 { 816 value *= 1e9; 817 value = MAX(0, MIN(UINT64_MAX, value)); 818 max_downtime = (uint64_t)value; 819 } 820 821 bool migrate_auto_converge(void) 822 { 823 MigrationState *s; 824 825 s = migrate_get_current(); 826 827 return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE]; 828 } 829 830 bool migrate_zero_blocks(void) 831 { 832 MigrationState *s; 833 834 s = migrate_get_current(); 835 836 return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS]; 837 } 838 839 bool migrate_use_compression(void) 840 { 841 MigrationState *s; 842 843 s = migrate_get_current(); 844 845 return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS]; 846 } 847 848 int migrate_compress_level(void) 849 { 850 MigrationState *s; 851 852 s = migrate_get_current(); 853 854 return s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; 855 } 856 857 int migrate_compress_threads(void) 858 { 859 MigrationState *s; 860 861 s = migrate_get_current(); 862 863 return s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; 864 } 865 866 int migrate_decompress_threads(void) 867 { 868 MigrationState *s; 869 870 s = migrate_get_current(); 871 872 return s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; 873 } 874 875 bool migrate_use_events(void) 876 { 877 MigrationState *s; 878 879 s = migrate_get_current(); 880 881 return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS]; 882 } 883 884 int migrate_use_xbzrle(void) 885 { 886 MigrationState *s; 887 888 s = migrate_get_current(); 889 890 return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE]; 891 } 892 893 int64_t migrate_xbzrle_cache_size(void) 894 { 895 MigrationState *s; 896 897 s = migrate_get_current(); 898 899 return s->xbzrle_cache_size; 900 } 901 902 /* migration thread support */ 903 904 static void *migration_thread(void *opaque) 905 { 906 MigrationState *s = opaque; 907 int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 908 int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); 909 int64_t initial_bytes = 0; 910 int64_t max_size = 0; 911 int64_t start_time = initial_time; 912 bool old_vm_running = false; 913 914 qemu_savevm_state_header(s->file); 915 qemu_savevm_state_begin(s->file, &s->params); 916 917 s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; 918 migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); 919 920 while (s->state == MIGRATION_STATUS_ACTIVE) { 921 int64_t current_time; 922 uint64_t pending_size; 923 924 if (!qemu_file_rate_limit(s->file)) { 925 pending_size = qemu_savevm_state_pending(s->file, max_size); 926 trace_migrate_pending(pending_size, max_size); 927 if (pending_size && pending_size >= max_size) { 928 qemu_savevm_state_iterate(s->file); 929 } else { 930 int ret; 931 932 qemu_mutex_lock_iothread(); 933 start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 934 qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); 935 old_vm_running = runstate_is_running(); 936 937 ret = global_state_store(); 938 if (!ret) { 939 ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); 940 if (ret >= 0) { 941 qemu_file_set_rate_limit(s->file, INT64_MAX); 942 qemu_savevm_state_complete(s->file); 943 } 944 } 945 qemu_mutex_unlock_iothread(); 946 947 if (ret < 0) { 948 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 949 MIGRATION_STATUS_FAILED); 950 break; 951 } 952 953 if (!qemu_file_get_error(s->file)) { 954 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 955 MIGRATION_STATUS_COMPLETED); 956 break; 957 } 958 } 959 } 960 961 if (qemu_file_get_error(s->file)) { 962 migrate_set_state(s, MIGRATION_STATUS_ACTIVE, 963 MIGRATION_STATUS_FAILED); 964 break; 965 } 966 current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 967 if (current_time >= initial_time + BUFFER_DELAY) { 968 uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes; 969 uint64_t time_spent = current_time - initial_time; 970 double bandwidth = transferred_bytes / time_spent; 971 max_size = bandwidth * migrate_max_downtime() / 1000000; 972 973 s->mbps = time_spent ? (((double) transferred_bytes * 8.0) / 974 ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1; 975 976 trace_migrate_transferred(transferred_bytes, time_spent, 977 bandwidth, max_size); 978 /* if we haven't sent anything, we don't want to recalculate 979 10000 is a small enough number for our purposes */ 980 if (s->dirty_bytes_rate && transferred_bytes > 10000) { 981 s->expected_downtime = s->dirty_bytes_rate / bandwidth; 982 } 983 984 qemu_file_reset_rate_limit(s->file); 985 initial_time = current_time; 986 initial_bytes = qemu_ftell(s->file); 987 } 988 if (qemu_file_rate_limit(s->file)) { 989 /* usleep expects microseconds */ 990 g_usleep((initial_time + BUFFER_DELAY - current_time)*1000); 991 } 992 } 993 994 qemu_mutex_lock_iothread(); 995 if (s->state == MIGRATION_STATUS_COMPLETED) { 996 int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); 997 uint64_t transferred_bytes = qemu_ftell(s->file); 998 s->total_time = end_time - s->total_time; 999 s->downtime = end_time - start_time; 1000 if (s->total_time) { 1001 s->mbps = (((double) transferred_bytes * 8.0) / 1002 ((double) s->total_time)) / 1000; 1003 } 1004 runstate_set(RUN_STATE_POSTMIGRATE); 1005 } else { 1006 if (old_vm_running) { 1007 vm_start(); 1008 } 1009 } 1010 qemu_bh_schedule(s->cleanup_bh); 1011 qemu_mutex_unlock_iothread(); 1012 1013 return NULL; 1014 } 1015 1016 void migrate_fd_connect(MigrationState *s) 1017 { 1018 /* This is a best 1st approximation. ns to ms */ 1019 s->expected_downtime = max_downtime/1000000; 1020 s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s); 1021 1022 qemu_file_set_rate_limit(s->file, 1023 s->bandwidth_limit / XFER_LIMIT_RATIO); 1024 1025 /* Notify before starting migration thread */ 1026 notifier_list_notify(&migration_state_notifiers, s); 1027 1028 migrate_compress_threads_create(); 1029 qemu_thread_create(&s->thread, "migration", migration_thread, s, 1030 QEMU_THREAD_JOINABLE); 1031 } 1032