xref: /openbmc/qemu/migration/migration.c (revision b14df228)
1 /*
2  * QEMU live migration
3  *
4  * Copyright IBM, Corp. 2008
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  * Contributions after 2012-01-13 are licensed under the terms of the
13  * GNU GPL, version 2 or (at your option) any later version.
14  */
15 
16 #include "qemu/osdep.h"
17 #include "qemu/cutils.h"
18 #include "qemu/error-report.h"
19 #include "qemu/main-loop.h"
20 #include "migration/blocker.h"
21 #include "exec.h"
22 #include "fd.h"
23 #include "socket.h"
24 #include "sysemu/runstate.h"
25 #include "sysemu/sysemu.h"
26 #include "sysemu/cpu-throttle.h"
27 #include "rdma.h"
28 #include "ram.h"
29 #include "migration/global_state.h"
30 #include "migration/misc.h"
31 #include "migration.h"
32 #include "savevm.h"
33 #include "qemu-file.h"
34 #include "migration/vmstate.h"
35 #include "block/block.h"
36 #include "qapi/error.h"
37 #include "qapi/clone-visitor.h"
38 #include "qapi/qapi-visit-migration.h"
39 #include "qapi/qapi-visit-sockets.h"
40 #include "qapi/qapi-commands-migration.h"
41 #include "qapi/qapi-events-migration.h"
42 #include "qapi/qmp/qerror.h"
43 #include "qapi/qmp/qnull.h"
44 #include "qemu/rcu.h"
45 #include "block.h"
46 #include "postcopy-ram.h"
47 #include "qemu/thread.h"
48 #include "trace.h"
49 #include "exec/target_page.h"
50 #include "io/channel-buffer.h"
51 #include "io/channel-tls.h"
52 #include "migration/colo.h"
53 #include "hw/boards.h"
54 #include "hw/qdev-properties.h"
55 #include "hw/qdev-properties-system.h"
56 #include "monitor/monitor.h"
57 #include "net/announce.h"
58 #include "qemu/queue.h"
59 #include "multifd.h"
60 #include "qemu/yank.h"
61 #include "sysemu/cpus.h"
62 #include "yank_functions.h"
63 #include "sysemu/qtest.h"
64 
65 #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
66 
67 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
68  * data. */
69 #define BUFFER_DELAY     100
70 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
71 
72 /* Time in milliseconds we are allowed to stop the source,
73  * for sending the last part */
74 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
75 
76 /* Maximum migrate downtime set to 2000 seconds */
77 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
78 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
79 
80 /* Default compression thread count */
81 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
82 /* Default decompression thread count, usually decompression is at
83  * least 4 times as fast as compression.*/
84 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
85 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
86 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
87 /* Define default autoconverge cpu throttle migration parameters */
88 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
89 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
90 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
91 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
92 
93 /* Migration XBZRLE default cache size */
94 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
95 
96 /* The delay time (in ms) between two COLO checkpoints */
97 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
98 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
99 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
100 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
101 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
102 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
103 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
104 
105 /* Background transfer rate for postcopy, 0 means unlimited, note
106  * that page requests can still exceed this limit.
107  */
108 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
109 
110 /*
111  * Parameters for self_announce_delay giving a stream of RARP/ARP
112  * packets after migration.
113  */
114 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
115 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
116 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
117 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
118 
119 static NotifierList migration_state_notifiers =
120     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
121 
122 /* Messages sent on the return path from destination to source */
123 enum mig_rp_message_type {
124     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
125     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
126     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
127 
128     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
129     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
130     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
131     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
132 
133     MIG_RP_MSG_MAX
134 };
135 
136 /* Migration capabilities set */
137 struct MigrateCapsSet {
138     int size;                       /* Capability set size */
139     MigrationCapability caps[];     /* Variadic array of capabilities */
140 };
141 typedef struct MigrateCapsSet MigrateCapsSet;
142 
143 /* Define and initialize MigrateCapsSet */
144 #define INITIALIZE_MIGRATE_CAPS_SET(_name, ...)   \
145     MigrateCapsSet _name = {    \
146         .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \
147         .caps = { __VA_ARGS__ } \
148     }
149 
150 /* Background-snapshot compatibility check list */
151 static const
152 INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
153     MIGRATION_CAPABILITY_POSTCOPY_RAM,
154     MIGRATION_CAPABILITY_DIRTY_BITMAPS,
155     MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME,
156     MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE,
157     MIGRATION_CAPABILITY_RETURN_PATH,
158     MIGRATION_CAPABILITY_MULTIFD,
159     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
160     MIGRATION_CAPABILITY_AUTO_CONVERGE,
161     MIGRATION_CAPABILITY_RELEASE_RAM,
162     MIGRATION_CAPABILITY_RDMA_PIN_ALL,
163     MIGRATION_CAPABILITY_COMPRESS,
164     MIGRATION_CAPABILITY_XBZRLE,
165     MIGRATION_CAPABILITY_X_COLO,
166     MIGRATION_CAPABILITY_VALIDATE_UUID,
167     MIGRATION_CAPABILITY_ZERO_COPY_SEND);
168 
169 /* When we add fault tolerance, we could have several
170    migrations at once.  For now we don't need to add
171    dynamic creation of migration */
172 
173 static MigrationState *current_migration;
174 static MigrationIncomingState *current_incoming;
175 
176 static GSList *migration_blockers;
177 
178 static bool migration_object_check(MigrationState *ms, Error **errp);
179 static int migration_maybe_pause(MigrationState *s,
180                                  int *current_active_state,
181                                  int new_state);
182 static void migrate_fd_cancel(MigrationState *s);
183 
184 static bool migrate_allow_multi_channels = true;
185 
186 void migrate_protocol_allow_multi_channels(bool allow)
187 {
188     migrate_allow_multi_channels = allow;
189 }
190 
191 bool migrate_multi_channels_is_allowed(void)
192 {
193     return migrate_allow_multi_channels;
194 }
195 
196 static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp)
197 {
198     uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp;
199 
200     return (a > b) - (a < b);
201 }
202 
203 void migration_object_init(void)
204 {
205     /* This can only be called once. */
206     assert(!current_migration);
207     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
208 
209     /*
210      * Init the migrate incoming object as well no matter whether
211      * we'll use it or not.
212      */
213     assert(!current_incoming);
214     current_incoming = g_new0(MigrationIncomingState, 1);
215     current_incoming->state = MIGRATION_STATUS_NONE;
216     current_incoming->postcopy_remote_fds =
217         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
218     qemu_mutex_init(&current_incoming->rp_mutex);
219     qemu_mutex_init(&current_incoming->postcopy_prio_thread_mutex);
220     qemu_event_init(&current_incoming->main_thread_load_event, false);
221     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
222     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
223     qemu_sem_init(&current_incoming->postcopy_pause_sem_fast_load, 0);
224     qemu_mutex_init(&current_incoming->page_request_mutex);
225     current_incoming->page_requested = g_tree_new(page_request_addr_cmp);
226 
227     migration_object_check(current_migration, &error_fatal);
228 
229     blk_mig_init();
230     ram_mig_init();
231     dirty_bitmap_mig_init();
232 }
233 
234 void migration_cancel(const Error *error)
235 {
236     if (error) {
237         migrate_set_error(current_migration, error);
238     }
239     migrate_fd_cancel(current_migration);
240 }
241 
242 void migration_shutdown(void)
243 {
244     /*
245      * When the QEMU main thread exit, the COLO thread
246      * may wait a semaphore. So, we should wakeup the
247      * COLO thread before migration shutdown.
248      */
249     colo_shutdown();
250     /*
251      * Cancel the current migration - that will (eventually)
252      * stop the migration using this structure
253      */
254     migration_cancel(NULL);
255     object_unref(OBJECT(current_migration));
256 
257     /*
258      * Cancel outgoing migration of dirty bitmaps. It should
259      * at least unref used block nodes.
260      */
261     dirty_bitmap_mig_cancel_outgoing();
262 
263     /*
264      * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
265      * are non-critical data, and their loss never considered as
266      * something serious.
267      */
268     dirty_bitmap_mig_cancel_incoming();
269 }
270 
271 /* For outgoing */
272 MigrationState *migrate_get_current(void)
273 {
274     /* This can only be called after the object created. */
275     assert(current_migration);
276     return current_migration;
277 }
278 
279 MigrationIncomingState *migration_incoming_get_current(void)
280 {
281     assert(current_incoming);
282     return current_incoming;
283 }
284 
285 void migration_incoming_transport_cleanup(MigrationIncomingState *mis)
286 {
287     if (mis->socket_address_list) {
288         qapi_free_SocketAddressList(mis->socket_address_list);
289         mis->socket_address_list = NULL;
290     }
291 
292     if (mis->transport_cleanup) {
293         mis->transport_cleanup(mis->transport_data);
294         mis->transport_data = mis->transport_cleanup = NULL;
295     }
296 }
297 
298 void migration_incoming_state_destroy(void)
299 {
300     struct MigrationIncomingState *mis = migration_incoming_get_current();
301 
302     if (mis->to_src_file) {
303         /* Tell source that we are done */
304         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
305         qemu_fclose(mis->to_src_file);
306         mis->to_src_file = NULL;
307     }
308 
309     if (mis->from_src_file) {
310         migration_ioc_unregister_yank_from_file(mis->from_src_file);
311         qemu_fclose(mis->from_src_file);
312         mis->from_src_file = NULL;
313     }
314     if (mis->postcopy_remote_fds) {
315         g_array_free(mis->postcopy_remote_fds, TRUE);
316         mis->postcopy_remote_fds = NULL;
317     }
318 
319     migration_incoming_transport_cleanup(mis);
320     qemu_event_reset(&mis->main_thread_load_event);
321 
322     if (mis->page_requested) {
323         g_tree_destroy(mis->page_requested);
324         mis->page_requested = NULL;
325     }
326 
327     if (mis->postcopy_qemufile_dst) {
328         migration_ioc_unregister_yank_from_file(mis->postcopy_qemufile_dst);
329         qemu_fclose(mis->postcopy_qemufile_dst);
330         mis->postcopy_qemufile_dst = NULL;
331     }
332 
333     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
334 }
335 
336 static void migrate_generate_event(int new_state)
337 {
338     if (migrate_use_events()) {
339         qapi_event_send_migration(new_state);
340     }
341 }
342 
343 static bool migrate_late_block_activate(void)
344 {
345     MigrationState *s;
346 
347     s = migrate_get_current();
348 
349     return s->enabled_capabilities[
350         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
351 }
352 
353 /*
354  * Send a message on the return channel back to the source
355  * of the migration.
356  */
357 static int migrate_send_rp_message(MigrationIncomingState *mis,
358                                    enum mig_rp_message_type message_type,
359                                    uint16_t len, void *data)
360 {
361     int ret = 0;
362 
363     trace_migrate_send_rp_message((int)message_type, len);
364     QEMU_LOCK_GUARD(&mis->rp_mutex);
365 
366     /*
367      * It's possible that the file handle got lost due to network
368      * failures.
369      */
370     if (!mis->to_src_file) {
371         ret = -EIO;
372         return ret;
373     }
374 
375     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
376     qemu_put_be16(mis->to_src_file, len);
377     qemu_put_buffer(mis->to_src_file, data, len);
378     qemu_fflush(mis->to_src_file);
379 
380     /* It's possible that qemu file got error during sending */
381     ret = qemu_file_get_error(mis->to_src_file);
382 
383     return ret;
384 }
385 
386 /* Request one page from the source VM at the given start address.
387  *   rb: the RAMBlock to request the page in
388  *   Start: Address offset within the RB
389  *   Len: Length in bytes required - must be a multiple of pagesize
390  */
391 int migrate_send_rp_message_req_pages(MigrationIncomingState *mis,
392                                       RAMBlock *rb, ram_addr_t start)
393 {
394     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
395     size_t msglen = 12; /* start + len */
396     size_t len = qemu_ram_pagesize(rb);
397     enum mig_rp_message_type msg_type;
398     const char *rbname;
399     int rbname_len;
400 
401     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
402     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
403 
404     /*
405      * We maintain the last ramblock that we requested for page.  Note that we
406      * don't need locking because this function will only be called within the
407      * postcopy ram fault thread.
408      */
409     if (rb != mis->last_rb) {
410         mis->last_rb = rb;
411 
412         rbname = qemu_ram_get_idstr(rb);
413         rbname_len = strlen(rbname);
414 
415         assert(rbname_len < 256);
416 
417         bufc[msglen++] = rbname_len;
418         memcpy(bufc + msglen, rbname, rbname_len);
419         msglen += rbname_len;
420         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
421     } else {
422         msg_type = MIG_RP_MSG_REQ_PAGES;
423     }
424 
425     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
426 }
427 
428 int migrate_send_rp_req_pages(MigrationIncomingState *mis,
429                               RAMBlock *rb, ram_addr_t start, uint64_t haddr)
430 {
431     void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb));
432     bool received = false;
433 
434     WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) {
435         received = ramblock_recv_bitmap_test_byte_offset(rb, start);
436         if (!received && !g_tree_lookup(mis->page_requested, aligned)) {
437             /*
438              * The page has not been received, and it's not yet in the page
439              * request list.  Queue it.  Set the value of element to 1, so that
440              * things like g_tree_lookup() will return TRUE (1) when found.
441              */
442             g_tree_insert(mis->page_requested, aligned, (gpointer)1);
443             mis->page_requested_count++;
444             trace_postcopy_page_req_add(aligned, mis->page_requested_count);
445         }
446     }
447 
448     /*
449      * If the page is there, skip sending the message.  We don't even need the
450      * lock because as long as the page arrived, it'll be there forever.
451      */
452     if (received) {
453         return 0;
454     }
455 
456     return migrate_send_rp_message_req_pages(mis, rb, start);
457 }
458 
459 static bool migration_colo_enabled;
460 bool migration_incoming_colo_enabled(void)
461 {
462     return migration_colo_enabled;
463 }
464 
465 void migration_incoming_disable_colo(void)
466 {
467     ram_block_discard_disable(false);
468     migration_colo_enabled = false;
469 }
470 
471 int migration_incoming_enable_colo(void)
472 {
473     if (ram_block_discard_disable(true)) {
474         error_report("COLO: cannot disable RAM discard");
475         return -EBUSY;
476     }
477     migration_colo_enabled = true;
478     return 0;
479 }
480 
481 void migrate_add_address(SocketAddress *address)
482 {
483     MigrationIncomingState *mis = migration_incoming_get_current();
484 
485     QAPI_LIST_PREPEND(mis->socket_address_list,
486                       QAPI_CLONE(SocketAddress, address));
487 }
488 
489 static void qemu_start_incoming_migration(const char *uri, Error **errp)
490 {
491     const char *p = NULL;
492 
493     migrate_protocol_allow_multi_channels(false); /* reset it anyway */
494     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
495     if (strstart(uri, "tcp:", &p) ||
496         strstart(uri, "unix:", NULL) ||
497         strstart(uri, "vsock:", NULL)) {
498         migrate_protocol_allow_multi_channels(true);
499         socket_start_incoming_migration(p ? p : uri, errp);
500 #ifdef CONFIG_RDMA
501     } else if (strstart(uri, "rdma:", &p)) {
502         rdma_start_incoming_migration(p, errp);
503 #endif
504     } else if (strstart(uri, "exec:", &p)) {
505         exec_start_incoming_migration(p, errp);
506     } else if (strstart(uri, "fd:", &p)) {
507         fd_start_incoming_migration(p, errp);
508     } else {
509         error_setg(errp, "unknown migration protocol: %s", uri);
510     }
511 }
512 
513 static void process_incoming_migration_bh(void *opaque)
514 {
515     Error *local_err = NULL;
516     MigrationIncomingState *mis = opaque;
517 
518     /* If capability late_block_activate is set:
519      * Only fire up the block code now if we're going to restart the
520      * VM, else 'cont' will do it.
521      * This causes file locking to happen; so we don't want it to happen
522      * unless we really are starting the VM.
523      */
524     if (!migrate_late_block_activate() ||
525          (autostart && (!global_state_received() ||
526             global_state_get_runstate() == RUN_STATE_RUNNING))) {
527         /* Make sure all file formats throw away their mutable metadata.
528          * If we get an error here, just don't restart the VM yet. */
529         bdrv_activate_all(&local_err);
530         if (local_err) {
531             error_report_err(local_err);
532             local_err = NULL;
533             autostart = false;
534         }
535     }
536 
537     /*
538      * This must happen after all error conditions are dealt with and
539      * we're sure the VM is going to be running on this host.
540      */
541     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
542 
543     if (multifd_load_cleanup(&local_err) != 0) {
544         error_report_err(local_err);
545         autostart = false;
546     }
547     /* If global state section was not received or we are in running
548        state, we need to obey autostart. Any other state is set with
549        runstate_set. */
550 
551     dirty_bitmap_mig_before_vm_start();
552 
553     if (!global_state_received() ||
554         global_state_get_runstate() == RUN_STATE_RUNNING) {
555         if (autostart) {
556             vm_start();
557         } else {
558             runstate_set(RUN_STATE_PAUSED);
559         }
560     } else if (migration_incoming_colo_enabled()) {
561         migration_incoming_disable_colo();
562         vm_start();
563     } else {
564         runstate_set(global_state_get_runstate());
565     }
566     /*
567      * This must happen after any state changes since as soon as an external
568      * observer sees this event they might start to prod at the VM assuming
569      * it's ready to use.
570      */
571     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
572                       MIGRATION_STATUS_COMPLETED);
573     qemu_bh_delete(mis->bh);
574     migration_incoming_state_destroy();
575 }
576 
577 static void process_incoming_migration_co(void *opaque)
578 {
579     MigrationIncomingState *mis = migration_incoming_get_current();
580     PostcopyState ps;
581     int ret;
582     Error *local_err = NULL;
583 
584     assert(mis->from_src_file);
585     mis->migration_incoming_co = qemu_coroutine_self();
586     mis->largest_page_size = qemu_ram_pagesize_largest();
587     postcopy_state_set(POSTCOPY_INCOMING_NONE);
588     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
589                       MIGRATION_STATUS_ACTIVE);
590     ret = qemu_loadvm_state(mis->from_src_file);
591 
592     ps = postcopy_state_get();
593     trace_process_incoming_migration_co_end(ret, ps);
594     if (ps != POSTCOPY_INCOMING_NONE) {
595         if (ps == POSTCOPY_INCOMING_ADVISE) {
596             /*
597              * Where a migration had postcopy enabled (and thus went to advise)
598              * but managed to complete within the precopy period, we can use
599              * the normal exit.
600              */
601             postcopy_ram_incoming_cleanup(mis);
602         } else if (ret >= 0) {
603             /*
604              * Postcopy was started, cleanup should happen at the end of the
605              * postcopy thread.
606              */
607             trace_process_incoming_migration_co_postcopy_end_main();
608             return;
609         }
610         /* Else if something went wrong then just fall out of the normal exit */
611     }
612 
613     /* we get COLO info, and know if we are in COLO mode */
614     if (!ret && migration_incoming_colo_enabled()) {
615         /* Make sure all file formats throw away their mutable metadata */
616         bdrv_activate_all(&local_err);
617         if (local_err) {
618             error_report_err(local_err);
619             goto fail;
620         }
621 
622         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
623              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
624         mis->have_colo_incoming_thread = true;
625         qemu_coroutine_yield();
626 
627         qemu_mutex_unlock_iothread();
628         /* Wait checkpoint incoming thread exit before free resource */
629         qemu_thread_join(&mis->colo_incoming_thread);
630         qemu_mutex_lock_iothread();
631         /* We hold the global iothread lock, so it is safe here */
632         colo_release_ram_cache();
633     }
634 
635     if (ret < 0) {
636         error_report("load of migration failed: %s", strerror(-ret));
637         goto fail;
638     }
639     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
640     qemu_bh_schedule(mis->bh);
641     mis->migration_incoming_co = NULL;
642     return;
643 fail:
644     local_err = NULL;
645     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
646                       MIGRATION_STATUS_FAILED);
647     qemu_fclose(mis->from_src_file);
648     if (multifd_load_cleanup(&local_err) != 0) {
649         error_report_err(local_err);
650     }
651     exit(EXIT_FAILURE);
652 }
653 
654 /**
655  * migration_incoming_setup: Setup incoming migration
656  * @f: file for main migration channel
657  * @errp: where to put errors
658  *
659  * Returns: %true on success, %false on error.
660  */
661 static bool migration_incoming_setup(QEMUFile *f, Error **errp)
662 {
663     MigrationIncomingState *mis = migration_incoming_get_current();
664 
665     if (multifd_load_setup(errp) != 0) {
666         return false;
667     }
668 
669     if (!mis->from_src_file) {
670         mis->from_src_file = f;
671     }
672     qemu_file_set_blocking(f, false);
673     return true;
674 }
675 
676 void migration_incoming_process(void)
677 {
678     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
679     qemu_coroutine_enter(co);
680 }
681 
682 /* Returns true if recovered from a paused migration, otherwise false */
683 static bool postcopy_try_recover(void)
684 {
685     MigrationIncomingState *mis = migration_incoming_get_current();
686 
687     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
688         /* Resumed from a paused postcopy migration */
689 
690         /* This should be set already in migration_incoming_setup() */
691         assert(mis->from_src_file);
692         /* Postcopy has standalone thread to do vm load */
693         qemu_file_set_blocking(mis->from_src_file, true);
694 
695         /* Re-configure the return path */
696         mis->to_src_file = qemu_file_get_return_path(mis->from_src_file);
697 
698         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
699                           MIGRATION_STATUS_POSTCOPY_RECOVER);
700 
701         /*
702          * Here, we only wake up the main loading thread (while the
703          * rest threads will still be waiting), so that we can receive
704          * commands from source now, and answer it if needed. The
705          * rest threads will be woken up afterwards until we are sure
706          * that source is ready to reply to page requests.
707          */
708         qemu_sem_post(&mis->postcopy_pause_sem_dst);
709         return true;
710     }
711 
712     return false;
713 }
714 
715 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
716 {
717     if (!migration_incoming_setup(f, errp)) {
718         return;
719     }
720     if (postcopy_try_recover()) {
721         return;
722     }
723     migration_incoming_process();
724 }
725 
726 static bool migration_needs_multiple_sockets(void)
727 {
728     return migrate_use_multifd() || migrate_postcopy_preempt();
729 }
730 
731 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
732 {
733     MigrationIncomingState *mis = migration_incoming_get_current();
734     Error *local_err = NULL;
735     bool start_migration;
736     QEMUFile *f;
737 
738     if (!mis->from_src_file) {
739         /* The first connection (multifd may have multiple) */
740         f = qemu_file_new_input(ioc);
741 
742         if (!migration_incoming_setup(f, errp)) {
743             return;
744         }
745 
746         /*
747          * Common migration only needs one channel, so we can start
748          * right now.  Some features need more than one channel, we wait.
749          */
750         start_migration = !migration_needs_multiple_sockets();
751     } else {
752         /* Multiple connections */
753         assert(migration_needs_multiple_sockets());
754         if (migrate_use_multifd()) {
755             start_migration = multifd_recv_new_channel(ioc, &local_err);
756         } else {
757             assert(migrate_postcopy_preempt());
758             f = qemu_file_new_input(ioc);
759             start_migration = postcopy_preempt_new_channel(mis, f);
760         }
761         if (local_err) {
762             error_propagate(errp, local_err);
763             return;
764         }
765     }
766 
767     if (start_migration) {
768         /* If it's a recovery, we're done */
769         if (postcopy_try_recover()) {
770             return;
771         }
772         migration_incoming_process();
773     }
774 }
775 
776 /**
777  * @migration_has_all_channels: We have received all channels that we need
778  *
779  * Returns true when we have got connections to all the channels that
780  * we need for migration.
781  */
782 bool migration_has_all_channels(void)
783 {
784     MigrationIncomingState *mis = migration_incoming_get_current();
785 
786     if (!mis->from_src_file) {
787         return false;
788     }
789 
790     if (migrate_use_multifd()) {
791         return multifd_recv_all_channels_created();
792     }
793 
794     if (migrate_postcopy_preempt()) {
795         return mis->postcopy_qemufile_dst != NULL;
796     }
797 
798     return true;
799 }
800 
801 /*
802  * Send a 'SHUT' message on the return channel with the given value
803  * to indicate that we've finished with the RP.  Non-0 value indicates
804  * error.
805  */
806 void migrate_send_rp_shut(MigrationIncomingState *mis,
807                           uint32_t value)
808 {
809     uint32_t buf;
810 
811     buf = cpu_to_be32(value);
812     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
813 }
814 
815 /*
816  * Send a 'PONG' message on the return channel with the given value
817  * (normally in response to a 'PING')
818  */
819 void migrate_send_rp_pong(MigrationIncomingState *mis,
820                           uint32_t value)
821 {
822     uint32_t buf;
823 
824     buf = cpu_to_be32(value);
825     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
826 }
827 
828 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
829                                  char *block_name)
830 {
831     char buf[512];
832     int len;
833     int64_t res;
834 
835     /*
836      * First, we send the header part. It contains only the len of
837      * idstr, and the idstr itself.
838      */
839     len = strlen(block_name);
840     buf[0] = len;
841     memcpy(buf + 1, block_name, len);
842 
843     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
844         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
845                      __func__);
846         return;
847     }
848 
849     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
850 
851     /*
852      * Next, we dump the received bitmap to the stream.
853      *
854      * TODO: currently we are safe since we are the only one that is
855      * using the to_src_file handle (fault thread is still paused),
856      * and it's ok even not taking the mutex. However the best way is
857      * to take the lock before sending the message header, and release
858      * the lock after sending the bitmap.
859      */
860     qemu_mutex_lock(&mis->rp_mutex);
861     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
862     qemu_mutex_unlock(&mis->rp_mutex);
863 
864     trace_migrate_send_rp_recv_bitmap(block_name, res);
865 }
866 
867 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
868 {
869     uint32_t buf;
870 
871     buf = cpu_to_be32(value);
872     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
873 }
874 
875 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
876 {
877     MigrationCapabilityStatusList *head = NULL, **tail = &head;
878     MigrationCapabilityStatus *caps;
879     MigrationState *s = migrate_get_current();
880     int i;
881 
882     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
883 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
884         if (i == MIGRATION_CAPABILITY_BLOCK) {
885             continue;
886         }
887 #endif
888         caps = g_malloc0(sizeof(*caps));
889         caps->capability = i;
890         caps->state = s->enabled_capabilities[i];
891         QAPI_LIST_APPEND(tail, caps);
892     }
893 
894     return head;
895 }
896 
897 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
898 {
899     MigrationParameters *params;
900     MigrationState *s = migrate_get_current();
901 
902     /* TODO use QAPI_CLONE() instead of duplicating it inline */
903     params = g_malloc0(sizeof(*params));
904     params->has_compress_level = true;
905     params->compress_level = s->parameters.compress_level;
906     params->has_compress_threads = true;
907     params->compress_threads = s->parameters.compress_threads;
908     params->has_compress_wait_thread = true;
909     params->compress_wait_thread = s->parameters.compress_wait_thread;
910     params->has_decompress_threads = true;
911     params->decompress_threads = s->parameters.decompress_threads;
912     params->has_throttle_trigger_threshold = true;
913     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
914     params->has_cpu_throttle_initial = true;
915     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
916     params->has_cpu_throttle_increment = true;
917     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
918     params->has_cpu_throttle_tailslow = true;
919     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
920     params->has_tls_creds = true;
921     params->tls_creds = g_strdup(s->parameters.tls_creds);
922     params->has_tls_hostname = true;
923     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
924     params->has_tls_authz = true;
925     params->tls_authz = g_strdup(s->parameters.tls_authz ?
926                                  s->parameters.tls_authz : "");
927     params->has_max_bandwidth = true;
928     params->max_bandwidth = s->parameters.max_bandwidth;
929     params->has_downtime_limit = true;
930     params->downtime_limit = s->parameters.downtime_limit;
931     params->has_x_checkpoint_delay = true;
932     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
933     params->has_block_incremental = true;
934     params->block_incremental = s->parameters.block_incremental;
935     params->has_multifd_channels = true;
936     params->multifd_channels = s->parameters.multifd_channels;
937     params->has_multifd_compression = true;
938     params->multifd_compression = s->parameters.multifd_compression;
939     params->has_multifd_zlib_level = true;
940     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
941     params->has_multifd_zstd_level = true;
942     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
943     params->has_xbzrle_cache_size = true;
944     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
945     params->has_max_postcopy_bandwidth = true;
946     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
947     params->has_max_cpu_throttle = true;
948     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
949     params->has_announce_initial = true;
950     params->announce_initial = s->parameters.announce_initial;
951     params->has_announce_max = true;
952     params->announce_max = s->parameters.announce_max;
953     params->has_announce_rounds = true;
954     params->announce_rounds = s->parameters.announce_rounds;
955     params->has_announce_step = true;
956     params->announce_step = s->parameters.announce_step;
957 
958     if (s->parameters.has_block_bitmap_mapping) {
959         params->has_block_bitmap_mapping = true;
960         params->block_bitmap_mapping =
961             QAPI_CLONE(BitmapMigrationNodeAliasList,
962                        s->parameters.block_bitmap_mapping);
963     }
964 
965     return params;
966 }
967 
968 AnnounceParameters *migrate_announce_params(void)
969 {
970     static AnnounceParameters ap;
971 
972     MigrationState *s = migrate_get_current();
973 
974     ap.initial = s->parameters.announce_initial;
975     ap.max = s->parameters.announce_max;
976     ap.rounds = s->parameters.announce_rounds;
977     ap.step = s->parameters.announce_step;
978 
979     return &ap;
980 }
981 
982 /*
983  * Return true if we're already in the middle of a migration
984  * (i.e. any of the active or setup states)
985  */
986 bool migration_is_setup_or_active(int state)
987 {
988     switch (state) {
989     case MIGRATION_STATUS_ACTIVE:
990     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
991     case MIGRATION_STATUS_POSTCOPY_PAUSED:
992     case MIGRATION_STATUS_POSTCOPY_RECOVER:
993     case MIGRATION_STATUS_SETUP:
994     case MIGRATION_STATUS_PRE_SWITCHOVER:
995     case MIGRATION_STATUS_DEVICE:
996     case MIGRATION_STATUS_WAIT_UNPLUG:
997     case MIGRATION_STATUS_COLO:
998         return true;
999 
1000     default:
1001         return false;
1002 
1003     }
1004 }
1005 
1006 bool migration_is_running(int state)
1007 {
1008     switch (state) {
1009     case MIGRATION_STATUS_ACTIVE:
1010     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1011     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1012     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1013     case MIGRATION_STATUS_SETUP:
1014     case MIGRATION_STATUS_PRE_SWITCHOVER:
1015     case MIGRATION_STATUS_DEVICE:
1016     case MIGRATION_STATUS_WAIT_UNPLUG:
1017     case MIGRATION_STATUS_CANCELLING:
1018         return true;
1019 
1020     default:
1021         return false;
1022 
1023     }
1024 }
1025 
1026 static void populate_time_info(MigrationInfo *info, MigrationState *s)
1027 {
1028     info->has_status = true;
1029     info->has_setup_time = true;
1030     info->setup_time = s->setup_time;
1031     if (s->state == MIGRATION_STATUS_COMPLETED) {
1032         info->has_total_time = true;
1033         info->total_time = s->total_time;
1034         info->has_downtime = true;
1035         info->downtime = s->downtime;
1036     } else {
1037         info->has_total_time = true;
1038         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
1039                            s->start_time;
1040         info->has_expected_downtime = true;
1041         info->expected_downtime = s->expected_downtime;
1042     }
1043 }
1044 
1045 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
1046 {
1047     size_t page_size = qemu_target_page_size();
1048 
1049     info->has_ram = true;
1050     info->ram = g_malloc0(sizeof(*info->ram));
1051     info->ram->transferred = ram_counters.transferred;
1052     info->ram->total = ram_bytes_total();
1053     info->ram->duplicate = ram_counters.duplicate;
1054     /* legacy value.  It is not used anymore */
1055     info->ram->skipped = 0;
1056     info->ram->normal = ram_counters.normal;
1057     info->ram->normal_bytes = ram_counters.normal * page_size;
1058     info->ram->mbps = s->mbps;
1059     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
1060     info->ram->dirty_sync_missed_zero_copy =
1061             ram_counters.dirty_sync_missed_zero_copy;
1062     info->ram->postcopy_requests = ram_counters.postcopy_requests;
1063     info->ram->page_size = page_size;
1064     info->ram->multifd_bytes = ram_counters.multifd_bytes;
1065     info->ram->pages_per_second = s->pages_per_second;
1066     info->ram->precopy_bytes = ram_counters.precopy_bytes;
1067     info->ram->downtime_bytes = ram_counters.downtime_bytes;
1068     info->ram->postcopy_bytes = ram_counters.postcopy_bytes;
1069 
1070     if (migrate_use_xbzrle()) {
1071         info->has_xbzrle_cache = true;
1072         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
1073         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
1074         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
1075         info->xbzrle_cache->pages = xbzrle_counters.pages;
1076         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
1077         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
1078         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
1079         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
1080     }
1081 
1082     if (migrate_use_compression()) {
1083         info->has_compression = true;
1084         info->compression = g_malloc0(sizeof(*info->compression));
1085         info->compression->pages = compression_counters.pages;
1086         info->compression->busy = compression_counters.busy;
1087         info->compression->busy_rate = compression_counters.busy_rate;
1088         info->compression->compressed_size =
1089                                     compression_counters.compressed_size;
1090         info->compression->compression_rate =
1091                                     compression_counters.compression_rate;
1092     }
1093 
1094     if (cpu_throttle_active()) {
1095         info->has_cpu_throttle_percentage = true;
1096         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
1097     }
1098 
1099     if (s->state != MIGRATION_STATUS_COMPLETED) {
1100         info->ram->remaining = ram_bytes_remaining();
1101         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
1102     }
1103 }
1104 
1105 static void populate_disk_info(MigrationInfo *info)
1106 {
1107     if (blk_mig_active()) {
1108         info->has_disk = true;
1109         info->disk = g_malloc0(sizeof(*info->disk));
1110         info->disk->transferred = blk_mig_bytes_transferred();
1111         info->disk->remaining = blk_mig_bytes_remaining();
1112         info->disk->total = blk_mig_bytes_total();
1113     }
1114 }
1115 
1116 static void fill_source_migration_info(MigrationInfo *info)
1117 {
1118     MigrationState *s = migrate_get_current();
1119     int state = qatomic_read(&s->state);
1120     GSList *cur_blocker = migration_blockers;
1121 
1122     info->blocked_reasons = NULL;
1123 
1124     /*
1125      * There are two types of reasons a migration might be blocked;
1126      * a) devices marked in VMState as non-migratable, and
1127      * b) Explicit migration blockers
1128      * We need to add both of them here.
1129      */
1130     qemu_savevm_non_migratable_list(&info->blocked_reasons);
1131 
1132     while (cur_blocker) {
1133         QAPI_LIST_PREPEND(info->blocked_reasons,
1134                           g_strdup(error_get_pretty(cur_blocker->data)));
1135         cur_blocker = g_slist_next(cur_blocker);
1136     }
1137     info->has_blocked_reasons = info->blocked_reasons != NULL;
1138 
1139     switch (state) {
1140     case MIGRATION_STATUS_NONE:
1141         /* no migration has happened ever */
1142         /* do not overwrite destination migration status */
1143         return;
1144     case MIGRATION_STATUS_SETUP:
1145         info->has_status = true;
1146         info->has_total_time = false;
1147         break;
1148     case MIGRATION_STATUS_ACTIVE:
1149     case MIGRATION_STATUS_CANCELLING:
1150     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1151     case MIGRATION_STATUS_PRE_SWITCHOVER:
1152     case MIGRATION_STATUS_DEVICE:
1153     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1154     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1155         /* TODO add some postcopy stats */
1156         populate_time_info(info, s);
1157         populate_ram_info(info, s);
1158         populate_disk_info(info);
1159         populate_vfio_info(info);
1160         break;
1161     case MIGRATION_STATUS_COLO:
1162         info->has_status = true;
1163         /* TODO: display COLO specific information (checkpoint info etc.) */
1164         break;
1165     case MIGRATION_STATUS_COMPLETED:
1166         populate_time_info(info, s);
1167         populate_ram_info(info, s);
1168         populate_vfio_info(info);
1169         break;
1170     case MIGRATION_STATUS_FAILED:
1171         info->has_status = true;
1172         if (s->error) {
1173             info->has_error_desc = true;
1174             info->error_desc = g_strdup(error_get_pretty(s->error));
1175         }
1176         break;
1177     case MIGRATION_STATUS_CANCELLED:
1178         info->has_status = true;
1179         break;
1180     case MIGRATION_STATUS_WAIT_UNPLUG:
1181         info->has_status = true;
1182         break;
1183     }
1184     info->status = state;
1185 }
1186 
1187 typedef enum WriteTrackingSupport {
1188     WT_SUPPORT_UNKNOWN = 0,
1189     WT_SUPPORT_ABSENT,
1190     WT_SUPPORT_AVAILABLE,
1191     WT_SUPPORT_COMPATIBLE
1192 } WriteTrackingSupport;
1193 
1194 static
1195 WriteTrackingSupport migrate_query_write_tracking(void)
1196 {
1197     /* Check if kernel supports required UFFD features */
1198     if (!ram_write_tracking_available()) {
1199         return WT_SUPPORT_ABSENT;
1200     }
1201     /*
1202      * Check if current memory configuration is
1203      * compatible with required UFFD features.
1204      */
1205     if (!ram_write_tracking_compatible()) {
1206         return WT_SUPPORT_AVAILABLE;
1207     }
1208 
1209     return WT_SUPPORT_COMPATIBLE;
1210 }
1211 
1212 /**
1213  * @migration_caps_check - check capability validity
1214  *
1215  * @cap_list: old capability list, array of bool
1216  * @params: new capabilities to be applied soon
1217  * @errp: set *errp if the check failed, with reason
1218  *
1219  * Returns true if check passed, otherwise false.
1220  */
1221 static bool migrate_caps_check(bool *cap_list,
1222                                MigrationCapabilityStatusList *params,
1223                                Error **errp)
1224 {
1225     MigrationCapabilityStatusList *cap;
1226     bool old_postcopy_cap;
1227     MigrationIncomingState *mis = migration_incoming_get_current();
1228 
1229     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1230 
1231     for (cap = params; cap; cap = cap->next) {
1232         cap_list[cap->value->capability] = cap->value->state;
1233     }
1234 
1235 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1236     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1237         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1238                    "block migration");
1239         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1240         return false;
1241     }
1242 #endif
1243 
1244 #ifndef CONFIG_REPLICATION
1245     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1246         error_setg(errp, "QEMU compiled without replication module"
1247                    " can't enable COLO");
1248         error_append_hint(errp, "Please enable replication before COLO.\n");
1249         return false;
1250     }
1251 #endif
1252 
1253     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1254         /* This check is reasonably expensive, so only when it's being
1255          * set the first time, also it's only the destination that needs
1256          * special support.
1257          */
1258         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1259             !postcopy_ram_supported_by_host(mis)) {
1260             /* postcopy_ram_supported_by_host will have emitted a more
1261              * detailed message
1262              */
1263             error_setg(errp, "Postcopy is not supported");
1264             return false;
1265         }
1266 
1267         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1268             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1269             return false;
1270         }
1271     }
1272 
1273     if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) {
1274         WriteTrackingSupport wt_support;
1275         int idx;
1276         /*
1277          * Check if 'background-snapshot' capability is supported by
1278          * host kernel and compatible with guest memory configuration.
1279          */
1280         wt_support = migrate_query_write_tracking();
1281         if (wt_support < WT_SUPPORT_AVAILABLE) {
1282             error_setg(errp, "Background-snapshot is not supported by host kernel");
1283             return false;
1284         }
1285         if (wt_support < WT_SUPPORT_COMPATIBLE) {
1286             error_setg(errp, "Background-snapshot is not compatible "
1287                     "with guest memory configuration");
1288             return false;
1289         }
1290 
1291         /*
1292          * Check if there are any migration capabilities
1293          * incompatible with 'background-snapshot'.
1294          */
1295         for (idx = 0; idx < check_caps_background_snapshot.size; idx++) {
1296             int incomp_cap = check_caps_background_snapshot.caps[idx];
1297             if (cap_list[incomp_cap]) {
1298                 error_setg(errp,
1299                         "Background-snapshot is not compatible with %s",
1300                         MigrationCapability_str(incomp_cap));
1301                 return false;
1302             }
1303         }
1304     }
1305 
1306 #ifdef CONFIG_LINUX
1307     if (cap_list[MIGRATION_CAPABILITY_ZERO_COPY_SEND] &&
1308         (!cap_list[MIGRATION_CAPABILITY_MULTIFD] ||
1309          cap_list[MIGRATION_CAPABILITY_COMPRESS] ||
1310          cap_list[MIGRATION_CAPABILITY_XBZRLE] ||
1311          migrate_multifd_compression() ||
1312          migrate_use_tls())) {
1313         error_setg(errp,
1314                    "Zero copy only available for non-compressed non-TLS multifd migration");
1315         return false;
1316     }
1317 #else
1318     if (cap_list[MIGRATION_CAPABILITY_ZERO_COPY_SEND]) {
1319         error_setg(errp,
1320                    "Zero copy currently only available on Linux");
1321         return false;
1322     }
1323 #endif
1324 
1325 
1326     /* incoming side only */
1327     if (runstate_check(RUN_STATE_INMIGRATE) &&
1328         !migrate_multi_channels_is_allowed() &&
1329         cap_list[MIGRATION_CAPABILITY_MULTIFD]) {
1330         error_setg(errp, "multifd is not supported by current protocol");
1331         return false;
1332     }
1333 
1334     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT]) {
1335         if (!cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1336             error_setg(errp, "Postcopy preempt requires postcopy-ram");
1337             return false;
1338         }
1339     }
1340 
1341     return true;
1342 }
1343 
1344 static void fill_destination_migration_info(MigrationInfo *info)
1345 {
1346     MigrationIncomingState *mis = migration_incoming_get_current();
1347 
1348     if (mis->socket_address_list) {
1349         info->has_socket_address = true;
1350         info->socket_address =
1351             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1352     }
1353 
1354     switch (mis->state) {
1355     case MIGRATION_STATUS_NONE:
1356         return;
1357     case MIGRATION_STATUS_SETUP:
1358     case MIGRATION_STATUS_CANCELLING:
1359     case MIGRATION_STATUS_CANCELLED:
1360     case MIGRATION_STATUS_ACTIVE:
1361     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1362     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1363     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1364     case MIGRATION_STATUS_FAILED:
1365     case MIGRATION_STATUS_COLO:
1366         info->has_status = true;
1367         break;
1368     case MIGRATION_STATUS_COMPLETED:
1369         info->has_status = true;
1370         fill_destination_postcopy_migration_info(info);
1371         break;
1372     }
1373     info->status = mis->state;
1374 }
1375 
1376 MigrationInfo *qmp_query_migrate(Error **errp)
1377 {
1378     MigrationInfo *info = g_malloc0(sizeof(*info));
1379 
1380     fill_destination_migration_info(info);
1381     fill_source_migration_info(info);
1382 
1383     return info;
1384 }
1385 
1386 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1387                                   Error **errp)
1388 {
1389     MigrationState *s = migrate_get_current();
1390     MigrationCapabilityStatusList *cap;
1391     bool cap_list[MIGRATION_CAPABILITY__MAX];
1392 
1393     if (migration_is_running(s->state)) {
1394         error_setg(errp, QERR_MIGRATION_ACTIVE);
1395         return;
1396     }
1397 
1398     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1399     if (!migrate_caps_check(cap_list, params, errp)) {
1400         return;
1401     }
1402 
1403     for (cap = params; cap; cap = cap->next) {
1404         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1405     }
1406 }
1407 
1408 /*
1409  * Check whether the parameters are valid. Error will be put into errp
1410  * (if provided). Return true if valid, otherwise false.
1411  */
1412 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1413 {
1414     if (params->has_compress_level &&
1415         (params->compress_level > 9)) {
1416         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1417                    "a value between 0 and 9");
1418         return false;
1419     }
1420 
1421     if (params->has_compress_threads && (params->compress_threads < 1)) {
1422         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1423                    "compress_threads",
1424                    "a value between 1 and 255");
1425         return false;
1426     }
1427 
1428     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1429         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1430                    "decompress_threads",
1431                    "a value between 1 and 255");
1432         return false;
1433     }
1434 
1435     if (params->has_throttle_trigger_threshold &&
1436         (params->throttle_trigger_threshold < 1 ||
1437          params->throttle_trigger_threshold > 100)) {
1438         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1439                    "throttle_trigger_threshold",
1440                    "an integer in the range of 1 to 100");
1441         return false;
1442     }
1443 
1444     if (params->has_cpu_throttle_initial &&
1445         (params->cpu_throttle_initial < 1 ||
1446          params->cpu_throttle_initial > 99)) {
1447         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1448                    "cpu_throttle_initial",
1449                    "an integer in the range of 1 to 99");
1450         return false;
1451     }
1452 
1453     if (params->has_cpu_throttle_increment &&
1454         (params->cpu_throttle_increment < 1 ||
1455          params->cpu_throttle_increment > 99)) {
1456         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1457                    "cpu_throttle_increment",
1458                    "an integer in the range of 1 to 99");
1459         return false;
1460     }
1461 
1462     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1463         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1464                    "max_bandwidth",
1465                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1466                    " bytes/second");
1467         return false;
1468     }
1469 
1470     if (params->has_downtime_limit &&
1471         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1472         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1473                    "downtime_limit",
1474                    "an integer in the range of 0 to "
1475                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1476         return false;
1477     }
1478 
1479     /* x_checkpoint_delay is now always positive */
1480 
1481     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1482         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1483                    "multifd_channels",
1484                    "a value between 1 and 255");
1485         return false;
1486     }
1487 
1488     if (params->has_multifd_zlib_level &&
1489         (params->multifd_zlib_level > 9)) {
1490         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1491                    "a value between 0 and 9");
1492         return false;
1493     }
1494 
1495     if (params->has_multifd_zstd_level &&
1496         (params->multifd_zstd_level > 20)) {
1497         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1498                    "a value between 0 and 20");
1499         return false;
1500     }
1501 
1502     if (params->has_xbzrle_cache_size &&
1503         (params->xbzrle_cache_size < qemu_target_page_size() ||
1504          !is_power_of_2(params->xbzrle_cache_size))) {
1505         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1506                    "xbzrle_cache_size",
1507                    "a power of two no less than the target page size");
1508         return false;
1509     }
1510 
1511     if (params->has_max_cpu_throttle &&
1512         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1513          params->max_cpu_throttle > 99)) {
1514         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1515                    "max_cpu_throttle",
1516                    "an integer in the range of cpu_throttle_initial to 99");
1517         return false;
1518     }
1519 
1520     if (params->has_announce_initial &&
1521         params->announce_initial > 100000) {
1522         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1523                    "announce_initial",
1524                    "a value between 0 and 100000");
1525         return false;
1526     }
1527     if (params->has_announce_max &&
1528         params->announce_max > 100000) {
1529         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1530                    "announce_max",
1531                    "a value between 0 and 100000");
1532        return false;
1533     }
1534     if (params->has_announce_rounds &&
1535         params->announce_rounds > 1000) {
1536         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1537                    "announce_rounds",
1538                    "a value between 0 and 1000");
1539        return false;
1540     }
1541     if (params->has_announce_step &&
1542         (params->announce_step < 1 ||
1543         params->announce_step > 10000)) {
1544         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1545                    "announce_step",
1546                    "a value between 0 and 10000");
1547        return false;
1548     }
1549 
1550     if (params->has_block_bitmap_mapping &&
1551         !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) {
1552         error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: ");
1553         return false;
1554     }
1555 
1556 #ifdef CONFIG_LINUX
1557     if (migrate_use_zero_copy_send() &&
1558         ((params->has_multifd_compression && params->multifd_compression) ||
1559          (params->has_tls_creds && params->tls_creds && *params->tls_creds))) {
1560         error_setg(errp,
1561                    "Zero copy only available for non-compressed non-TLS multifd migration");
1562         return false;
1563     }
1564 #endif
1565 
1566     return true;
1567 }
1568 
1569 static void migrate_params_test_apply(MigrateSetParameters *params,
1570                                       MigrationParameters *dest)
1571 {
1572     *dest = migrate_get_current()->parameters;
1573 
1574     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1575 
1576     if (params->has_compress_level) {
1577         dest->compress_level = params->compress_level;
1578     }
1579 
1580     if (params->has_compress_threads) {
1581         dest->compress_threads = params->compress_threads;
1582     }
1583 
1584     if (params->has_compress_wait_thread) {
1585         dest->compress_wait_thread = params->compress_wait_thread;
1586     }
1587 
1588     if (params->has_decompress_threads) {
1589         dest->decompress_threads = params->decompress_threads;
1590     }
1591 
1592     if (params->has_throttle_trigger_threshold) {
1593         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1594     }
1595 
1596     if (params->has_cpu_throttle_initial) {
1597         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1598     }
1599 
1600     if (params->has_cpu_throttle_increment) {
1601         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1602     }
1603 
1604     if (params->has_cpu_throttle_tailslow) {
1605         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1606     }
1607 
1608     if (params->has_tls_creds) {
1609         assert(params->tls_creds->type == QTYPE_QSTRING);
1610         dest->tls_creds = params->tls_creds->u.s;
1611     }
1612 
1613     if (params->has_tls_hostname) {
1614         assert(params->tls_hostname->type == QTYPE_QSTRING);
1615         dest->tls_hostname = params->tls_hostname->u.s;
1616     }
1617 
1618     if (params->has_max_bandwidth) {
1619         dest->max_bandwidth = params->max_bandwidth;
1620     }
1621 
1622     if (params->has_downtime_limit) {
1623         dest->downtime_limit = params->downtime_limit;
1624     }
1625 
1626     if (params->has_x_checkpoint_delay) {
1627         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1628     }
1629 
1630     if (params->has_block_incremental) {
1631         dest->block_incremental = params->block_incremental;
1632     }
1633     if (params->has_multifd_channels) {
1634         dest->multifd_channels = params->multifd_channels;
1635     }
1636     if (params->has_multifd_compression) {
1637         dest->multifd_compression = params->multifd_compression;
1638     }
1639     if (params->has_xbzrle_cache_size) {
1640         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1641     }
1642     if (params->has_max_postcopy_bandwidth) {
1643         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1644     }
1645     if (params->has_max_cpu_throttle) {
1646         dest->max_cpu_throttle = params->max_cpu_throttle;
1647     }
1648     if (params->has_announce_initial) {
1649         dest->announce_initial = params->announce_initial;
1650     }
1651     if (params->has_announce_max) {
1652         dest->announce_max = params->announce_max;
1653     }
1654     if (params->has_announce_rounds) {
1655         dest->announce_rounds = params->announce_rounds;
1656     }
1657     if (params->has_announce_step) {
1658         dest->announce_step = params->announce_step;
1659     }
1660 
1661     if (params->has_block_bitmap_mapping) {
1662         dest->has_block_bitmap_mapping = true;
1663         dest->block_bitmap_mapping = params->block_bitmap_mapping;
1664     }
1665 }
1666 
1667 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1668 {
1669     MigrationState *s = migrate_get_current();
1670 
1671     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1672 
1673     if (params->has_compress_level) {
1674         s->parameters.compress_level = params->compress_level;
1675     }
1676 
1677     if (params->has_compress_threads) {
1678         s->parameters.compress_threads = params->compress_threads;
1679     }
1680 
1681     if (params->has_compress_wait_thread) {
1682         s->parameters.compress_wait_thread = params->compress_wait_thread;
1683     }
1684 
1685     if (params->has_decompress_threads) {
1686         s->parameters.decompress_threads = params->decompress_threads;
1687     }
1688 
1689     if (params->has_throttle_trigger_threshold) {
1690         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1691     }
1692 
1693     if (params->has_cpu_throttle_initial) {
1694         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1695     }
1696 
1697     if (params->has_cpu_throttle_increment) {
1698         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1699     }
1700 
1701     if (params->has_cpu_throttle_tailslow) {
1702         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1703     }
1704 
1705     if (params->has_tls_creds) {
1706         g_free(s->parameters.tls_creds);
1707         assert(params->tls_creds->type == QTYPE_QSTRING);
1708         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1709     }
1710 
1711     if (params->has_tls_hostname) {
1712         g_free(s->parameters.tls_hostname);
1713         assert(params->tls_hostname->type == QTYPE_QSTRING);
1714         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1715     }
1716 
1717     if (params->has_tls_authz) {
1718         g_free(s->parameters.tls_authz);
1719         assert(params->tls_authz->type == QTYPE_QSTRING);
1720         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1721     }
1722 
1723     if (params->has_max_bandwidth) {
1724         s->parameters.max_bandwidth = params->max_bandwidth;
1725         if (s->to_dst_file && !migration_in_postcopy()) {
1726             qemu_file_set_rate_limit(s->to_dst_file,
1727                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1728         }
1729     }
1730 
1731     if (params->has_downtime_limit) {
1732         s->parameters.downtime_limit = params->downtime_limit;
1733     }
1734 
1735     if (params->has_x_checkpoint_delay) {
1736         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1737         if (migration_in_colo_state()) {
1738             colo_checkpoint_notify(s);
1739         }
1740     }
1741 
1742     if (params->has_block_incremental) {
1743         s->parameters.block_incremental = params->block_incremental;
1744     }
1745     if (params->has_multifd_channels) {
1746         s->parameters.multifd_channels = params->multifd_channels;
1747     }
1748     if (params->has_multifd_compression) {
1749         s->parameters.multifd_compression = params->multifd_compression;
1750     }
1751     if (params->has_xbzrle_cache_size) {
1752         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1753         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1754     }
1755     if (params->has_max_postcopy_bandwidth) {
1756         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1757         if (s->to_dst_file && migration_in_postcopy()) {
1758             qemu_file_set_rate_limit(s->to_dst_file,
1759                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1760         }
1761     }
1762     if (params->has_max_cpu_throttle) {
1763         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1764     }
1765     if (params->has_announce_initial) {
1766         s->parameters.announce_initial = params->announce_initial;
1767     }
1768     if (params->has_announce_max) {
1769         s->parameters.announce_max = params->announce_max;
1770     }
1771     if (params->has_announce_rounds) {
1772         s->parameters.announce_rounds = params->announce_rounds;
1773     }
1774     if (params->has_announce_step) {
1775         s->parameters.announce_step = params->announce_step;
1776     }
1777 
1778     if (params->has_block_bitmap_mapping) {
1779         qapi_free_BitmapMigrationNodeAliasList(
1780             s->parameters.block_bitmap_mapping);
1781 
1782         s->parameters.has_block_bitmap_mapping = true;
1783         s->parameters.block_bitmap_mapping =
1784             QAPI_CLONE(BitmapMigrationNodeAliasList,
1785                        params->block_bitmap_mapping);
1786     }
1787 }
1788 
1789 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1790 {
1791     MigrationParameters tmp;
1792 
1793     /* TODO Rewrite "" to null instead */
1794     if (params->has_tls_creds
1795         && params->tls_creds->type == QTYPE_QNULL) {
1796         qobject_unref(params->tls_creds->u.n);
1797         params->tls_creds->type = QTYPE_QSTRING;
1798         params->tls_creds->u.s = strdup("");
1799     }
1800     /* TODO Rewrite "" to null instead */
1801     if (params->has_tls_hostname
1802         && params->tls_hostname->type == QTYPE_QNULL) {
1803         qobject_unref(params->tls_hostname->u.n);
1804         params->tls_hostname->type = QTYPE_QSTRING;
1805         params->tls_hostname->u.s = strdup("");
1806     }
1807 
1808     migrate_params_test_apply(params, &tmp);
1809 
1810     if (!migrate_params_check(&tmp, errp)) {
1811         /* Invalid parameter */
1812         return;
1813     }
1814 
1815     migrate_params_apply(params, errp);
1816 }
1817 
1818 
1819 void qmp_migrate_start_postcopy(Error **errp)
1820 {
1821     MigrationState *s = migrate_get_current();
1822 
1823     if (!migrate_postcopy()) {
1824         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1825                          " the start of migration");
1826         return;
1827     }
1828 
1829     if (s->state == MIGRATION_STATUS_NONE) {
1830         error_setg(errp, "Postcopy must be started after migration has been"
1831                          " started");
1832         return;
1833     }
1834     /*
1835      * we don't error if migration has finished since that would be racy
1836      * with issuing this command.
1837      */
1838     qatomic_set(&s->start_postcopy, true);
1839 }
1840 
1841 /* shared migration helpers */
1842 
1843 void migrate_set_state(int *state, int old_state, int new_state)
1844 {
1845     assert(new_state < MIGRATION_STATUS__MAX);
1846     if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
1847         trace_migrate_set_state(MigrationStatus_str(new_state));
1848         migrate_generate_event(new_state);
1849     }
1850 }
1851 
1852 static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index,
1853                                                   bool state)
1854 {
1855     MigrationCapabilityStatus *cap;
1856 
1857     cap = g_new0(MigrationCapabilityStatus, 1);
1858     cap->capability = index;
1859     cap->state = state;
1860 
1861     return cap;
1862 }
1863 
1864 void migrate_set_block_enabled(bool value, Error **errp)
1865 {
1866     MigrationCapabilityStatusList *cap = NULL;
1867 
1868     QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value));
1869     qmp_migrate_set_capabilities(cap, errp);
1870     qapi_free_MigrationCapabilityStatusList(cap);
1871 }
1872 
1873 static void migrate_set_block_incremental(MigrationState *s, bool value)
1874 {
1875     s->parameters.block_incremental = value;
1876 }
1877 
1878 static void block_cleanup_parameters(MigrationState *s)
1879 {
1880     if (s->must_remove_block_options) {
1881         /* setting to false can never fail */
1882         migrate_set_block_enabled(false, &error_abort);
1883         migrate_set_block_incremental(s, false);
1884         s->must_remove_block_options = false;
1885     }
1886 }
1887 
1888 static void migrate_fd_cleanup(MigrationState *s)
1889 {
1890     qemu_bh_delete(s->cleanup_bh);
1891     s->cleanup_bh = NULL;
1892 
1893     g_free(s->hostname);
1894     s->hostname = NULL;
1895 
1896     qemu_savevm_state_cleanup();
1897 
1898     if (s->to_dst_file) {
1899         QEMUFile *tmp;
1900 
1901         trace_migrate_fd_cleanup();
1902         qemu_mutex_unlock_iothread();
1903         if (s->migration_thread_running) {
1904             qemu_thread_join(&s->thread);
1905             s->migration_thread_running = false;
1906         }
1907         qemu_mutex_lock_iothread();
1908 
1909         multifd_save_cleanup();
1910         qemu_mutex_lock(&s->qemu_file_lock);
1911         tmp = s->to_dst_file;
1912         s->to_dst_file = NULL;
1913         qemu_mutex_unlock(&s->qemu_file_lock);
1914         /*
1915          * Close the file handle without the lock to make sure the
1916          * critical section won't block for long.
1917          */
1918         migration_ioc_unregister_yank_from_file(tmp);
1919         qemu_fclose(tmp);
1920     }
1921 
1922     if (s->postcopy_qemufile_src) {
1923         migration_ioc_unregister_yank_from_file(s->postcopy_qemufile_src);
1924         qemu_fclose(s->postcopy_qemufile_src);
1925         s->postcopy_qemufile_src = NULL;
1926     }
1927 
1928     assert(!migration_is_active(s));
1929 
1930     if (s->state == MIGRATION_STATUS_CANCELLING) {
1931         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1932                           MIGRATION_STATUS_CANCELLED);
1933     }
1934 
1935     if (s->error) {
1936         /* It is used on info migrate.  We can't free it */
1937         error_report_err(error_copy(s->error));
1938     }
1939     notifier_list_notify(&migration_state_notifiers, s);
1940     block_cleanup_parameters(s);
1941     yank_unregister_instance(MIGRATION_YANK_INSTANCE);
1942 }
1943 
1944 static void migrate_fd_cleanup_schedule(MigrationState *s)
1945 {
1946     /*
1947      * Ref the state for bh, because it may be called when
1948      * there're already no other refs
1949      */
1950     object_ref(OBJECT(s));
1951     qemu_bh_schedule(s->cleanup_bh);
1952 }
1953 
1954 static void migrate_fd_cleanup_bh(void *opaque)
1955 {
1956     MigrationState *s = opaque;
1957     migrate_fd_cleanup(s);
1958     object_unref(OBJECT(s));
1959 }
1960 
1961 void migrate_set_error(MigrationState *s, const Error *error)
1962 {
1963     QEMU_LOCK_GUARD(&s->error_mutex);
1964     if (!s->error) {
1965         s->error = error_copy(error);
1966     }
1967 }
1968 
1969 static void migrate_error_free(MigrationState *s)
1970 {
1971     QEMU_LOCK_GUARD(&s->error_mutex);
1972     if (s->error) {
1973         error_free(s->error);
1974         s->error = NULL;
1975     }
1976 }
1977 
1978 void migrate_fd_error(MigrationState *s, const Error *error)
1979 {
1980     trace_migrate_fd_error(error_get_pretty(error));
1981     assert(s->to_dst_file == NULL);
1982     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1983                       MIGRATION_STATUS_FAILED);
1984     migrate_set_error(s, error);
1985 }
1986 
1987 static void migrate_fd_cancel(MigrationState *s)
1988 {
1989     int old_state ;
1990     QEMUFile *f = migrate_get_current()->to_dst_file;
1991     trace_migrate_fd_cancel();
1992 
1993     WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) {
1994         if (s->rp_state.from_dst_file) {
1995             /* shutdown the rp socket, so causing the rp thread to shutdown */
1996             qemu_file_shutdown(s->rp_state.from_dst_file);
1997         }
1998     }
1999 
2000     do {
2001         old_state = s->state;
2002         if (!migration_is_running(old_state)) {
2003             break;
2004         }
2005         /* If the migration is paused, kick it out of the pause */
2006         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
2007             qemu_sem_post(&s->pause_sem);
2008         }
2009         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
2010     } while (s->state != MIGRATION_STATUS_CANCELLING);
2011 
2012     /*
2013      * If we're unlucky the migration code might be stuck somewhere in a
2014      * send/write while the network has failed and is waiting to timeout;
2015      * if we've got shutdown(2) available then we can force it to quit.
2016      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
2017      * called in a bh, so there is no race against this cancel.
2018      */
2019     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
2020         qemu_file_shutdown(f);
2021     }
2022     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
2023         Error *local_err = NULL;
2024 
2025         bdrv_activate_all(&local_err);
2026         if (local_err) {
2027             error_report_err(local_err);
2028         } else {
2029             s->block_inactive = false;
2030         }
2031     }
2032 }
2033 
2034 void add_migration_state_change_notifier(Notifier *notify)
2035 {
2036     notifier_list_add(&migration_state_notifiers, notify);
2037 }
2038 
2039 void remove_migration_state_change_notifier(Notifier *notify)
2040 {
2041     notifier_remove(notify);
2042 }
2043 
2044 bool migration_in_setup(MigrationState *s)
2045 {
2046     return s->state == MIGRATION_STATUS_SETUP;
2047 }
2048 
2049 bool migration_has_finished(MigrationState *s)
2050 {
2051     return s->state == MIGRATION_STATUS_COMPLETED;
2052 }
2053 
2054 bool migration_has_failed(MigrationState *s)
2055 {
2056     return (s->state == MIGRATION_STATUS_CANCELLED ||
2057             s->state == MIGRATION_STATUS_FAILED);
2058 }
2059 
2060 bool migration_in_postcopy(void)
2061 {
2062     MigrationState *s = migrate_get_current();
2063 
2064     switch (s->state) {
2065     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2066     case MIGRATION_STATUS_POSTCOPY_PAUSED:
2067     case MIGRATION_STATUS_POSTCOPY_RECOVER:
2068         return true;
2069     default:
2070         return false;
2071     }
2072 }
2073 
2074 bool migration_in_postcopy_after_devices(MigrationState *s)
2075 {
2076     return migration_in_postcopy() && s->postcopy_after_devices;
2077 }
2078 
2079 bool migration_in_incoming_postcopy(void)
2080 {
2081     PostcopyState ps = postcopy_state_get();
2082 
2083     return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END;
2084 }
2085 
2086 bool migration_in_bg_snapshot(void)
2087 {
2088     MigrationState *s = migrate_get_current();
2089 
2090     return migrate_background_snapshot() &&
2091             migration_is_setup_or_active(s->state);
2092 }
2093 
2094 bool migration_is_idle(void)
2095 {
2096     MigrationState *s = current_migration;
2097 
2098     if (!s) {
2099         return true;
2100     }
2101 
2102     switch (s->state) {
2103     case MIGRATION_STATUS_NONE:
2104     case MIGRATION_STATUS_CANCELLED:
2105     case MIGRATION_STATUS_COMPLETED:
2106     case MIGRATION_STATUS_FAILED:
2107         return true;
2108     case MIGRATION_STATUS_SETUP:
2109     case MIGRATION_STATUS_CANCELLING:
2110     case MIGRATION_STATUS_ACTIVE:
2111     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
2112     case MIGRATION_STATUS_COLO:
2113     case MIGRATION_STATUS_PRE_SWITCHOVER:
2114     case MIGRATION_STATUS_DEVICE:
2115     case MIGRATION_STATUS_WAIT_UNPLUG:
2116         return false;
2117     case MIGRATION_STATUS__MAX:
2118         g_assert_not_reached();
2119     }
2120 
2121     return false;
2122 }
2123 
2124 bool migration_is_active(MigrationState *s)
2125 {
2126     return (s->state == MIGRATION_STATUS_ACTIVE ||
2127             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
2128 }
2129 
2130 void migrate_init(MigrationState *s)
2131 {
2132     /*
2133      * Reinitialise all migration state, except
2134      * parameters/capabilities that the user set, and
2135      * locks.
2136      */
2137     s->cleanup_bh = 0;
2138     s->vm_start_bh = 0;
2139     s->to_dst_file = NULL;
2140     s->state = MIGRATION_STATUS_NONE;
2141     s->rp_state.from_dst_file = NULL;
2142     s->rp_state.error = false;
2143     s->mbps = 0.0;
2144     s->pages_per_second = 0.0;
2145     s->downtime = 0;
2146     s->expected_downtime = 0;
2147     s->setup_time = 0;
2148     s->start_postcopy = false;
2149     s->postcopy_after_devices = false;
2150     s->migration_thread_running = false;
2151     error_free(s->error);
2152     s->error = NULL;
2153     s->hostname = NULL;
2154 
2155     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
2156 
2157     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2158     s->total_time = 0;
2159     s->vm_was_running = false;
2160     s->iteration_initial_bytes = 0;
2161     s->threshold_size = 0;
2162 }
2163 
2164 int migrate_add_blocker_internal(Error *reason, Error **errp)
2165 {
2166     /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */
2167     if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) {
2168         error_propagate_prepend(errp, error_copy(reason),
2169                                 "disallowing migration blocker "
2170                                 "(migration/snapshot in progress) for: ");
2171         return -EBUSY;
2172     }
2173 
2174     migration_blockers = g_slist_prepend(migration_blockers, reason);
2175     return 0;
2176 }
2177 
2178 int migrate_add_blocker(Error *reason, Error **errp)
2179 {
2180     if (only_migratable) {
2181         error_propagate_prepend(errp, error_copy(reason),
2182                                 "disallowing migration blocker "
2183                                 "(--only-migratable) for: ");
2184         return -EACCES;
2185     }
2186 
2187     return migrate_add_blocker_internal(reason, errp);
2188 }
2189 
2190 void migrate_del_blocker(Error *reason)
2191 {
2192     migration_blockers = g_slist_remove(migration_blockers, reason);
2193 }
2194 
2195 void qmp_migrate_incoming(const char *uri, Error **errp)
2196 {
2197     Error *local_err = NULL;
2198     static bool once = true;
2199 
2200     if (!once) {
2201         error_setg(errp, "The incoming migration has already been started");
2202         return;
2203     }
2204     if (!runstate_check(RUN_STATE_INMIGRATE)) {
2205         error_setg(errp, "'-incoming' was not specified on the command line");
2206         return;
2207     }
2208 
2209     if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2210         return;
2211     }
2212 
2213     qemu_start_incoming_migration(uri, &local_err);
2214 
2215     if (local_err) {
2216         yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2217         error_propagate(errp, local_err);
2218         return;
2219     }
2220 
2221     once = false;
2222 }
2223 
2224 void qmp_migrate_recover(const char *uri, Error **errp)
2225 {
2226     MigrationIncomingState *mis = migration_incoming_get_current();
2227 
2228     /*
2229      * Don't even bother to use ERRP_GUARD() as it _must_ always be set by
2230      * callers (no one should ignore a recover failure); if there is, it's a
2231      * programming error.
2232      */
2233     assert(errp);
2234 
2235     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2236         error_setg(errp, "Migrate recover can only be run "
2237                    "when postcopy is paused.");
2238         return;
2239     }
2240 
2241     /* If there's an existing transport, release it */
2242     migration_incoming_transport_cleanup(mis);
2243 
2244     /*
2245      * Note that this call will never start a real migration; it will
2246      * only re-setup the migration stream and poke existing migration
2247      * to continue using that newly established channel.
2248      */
2249     qemu_start_incoming_migration(uri, errp);
2250 }
2251 
2252 void qmp_migrate_pause(Error **errp)
2253 {
2254     MigrationState *ms = migrate_get_current();
2255     MigrationIncomingState *mis = migration_incoming_get_current();
2256     int ret;
2257 
2258     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2259         /* Source side, during postcopy */
2260         qemu_mutex_lock(&ms->qemu_file_lock);
2261         ret = qemu_file_shutdown(ms->to_dst_file);
2262         qemu_mutex_unlock(&ms->qemu_file_lock);
2263         if (ret) {
2264             error_setg(errp, "Failed to pause source migration");
2265         }
2266         return;
2267     }
2268 
2269     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2270         ret = qemu_file_shutdown(mis->from_src_file);
2271         if (ret) {
2272             error_setg(errp, "Failed to pause destination migration");
2273         }
2274         return;
2275     }
2276 
2277     error_setg(errp, "migrate-pause is currently only supported "
2278                "during postcopy-active state");
2279 }
2280 
2281 bool migration_is_blocked(Error **errp)
2282 {
2283     if (qemu_savevm_state_blocked(errp)) {
2284         return true;
2285     }
2286 
2287     if (migration_blockers) {
2288         error_propagate(errp, error_copy(migration_blockers->data));
2289         return true;
2290     }
2291 
2292     return false;
2293 }
2294 
2295 /* Returns true if continue to migrate, or false if error detected */
2296 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
2297                             bool resume, Error **errp)
2298 {
2299     Error *local_err = NULL;
2300 
2301     if (resume) {
2302         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
2303             error_setg(errp, "Cannot resume if there is no "
2304                        "paused migration");
2305             return false;
2306         }
2307 
2308         /*
2309          * Postcopy recovery won't work well with release-ram
2310          * capability since release-ram will drop the page buffer as
2311          * long as the page is put into the send buffer.  So if there
2312          * is a network failure happened, any page buffers that have
2313          * not yet reached the destination VM but have already been
2314          * sent from the source VM will be lost forever.  Let's refuse
2315          * the client from resuming such a postcopy migration.
2316          * Luckily release-ram was designed to only be used when src
2317          * and destination VMs are on the same host, so it should be
2318          * fine.
2319          */
2320         if (migrate_release_ram()) {
2321             error_setg(errp, "Postcopy recovery cannot work "
2322                        "when release-ram capability is set");
2323             return false;
2324         }
2325 
2326         /* This is a resume, skip init status */
2327         return true;
2328     }
2329 
2330     if (migration_is_running(s->state)) {
2331         error_setg(errp, QERR_MIGRATION_ACTIVE);
2332         return false;
2333     }
2334 
2335     if (runstate_check(RUN_STATE_INMIGRATE)) {
2336         error_setg(errp, "Guest is waiting for an incoming migration");
2337         return false;
2338     }
2339 
2340     if (runstate_check(RUN_STATE_POSTMIGRATE)) {
2341         error_setg(errp, "Can't migrate the vm that was paused due to "
2342                    "previous migration");
2343         return false;
2344     }
2345 
2346     if (migration_is_blocked(errp)) {
2347         return false;
2348     }
2349 
2350     if (blk || blk_inc) {
2351         if (migrate_colo_enabled()) {
2352             error_setg(errp, "No disk migration is required in COLO mode");
2353             return false;
2354         }
2355         if (migrate_use_block() || migrate_use_block_incremental()) {
2356             error_setg(errp, "Command options are incompatible with "
2357                        "current migration capabilities");
2358             return false;
2359         }
2360         migrate_set_block_enabled(true, &local_err);
2361         if (local_err) {
2362             error_propagate(errp, local_err);
2363             return false;
2364         }
2365         s->must_remove_block_options = true;
2366     }
2367 
2368     if (blk_inc) {
2369         migrate_set_block_incremental(s, true);
2370     }
2371 
2372     migrate_init(s);
2373     /*
2374      * set ram_counters compression_counters memory to zero for a
2375      * new migration
2376      */
2377     memset(&ram_counters, 0, sizeof(ram_counters));
2378     memset(&compression_counters, 0, sizeof(compression_counters));
2379 
2380     return true;
2381 }
2382 
2383 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2384                  bool has_inc, bool inc, bool has_detach, bool detach,
2385                  bool has_resume, bool resume, Error **errp)
2386 {
2387     Error *local_err = NULL;
2388     MigrationState *s = migrate_get_current();
2389     const char *p = NULL;
2390 
2391     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2392                          has_resume && resume, errp)) {
2393         /* Error detected, put into errp */
2394         return;
2395     }
2396 
2397     if (!(has_resume && resume)) {
2398         if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) {
2399             return;
2400         }
2401     }
2402 
2403     migrate_protocol_allow_multi_channels(false);
2404     if (strstart(uri, "tcp:", &p) ||
2405         strstart(uri, "unix:", NULL) ||
2406         strstart(uri, "vsock:", NULL)) {
2407         migrate_protocol_allow_multi_channels(true);
2408         socket_start_outgoing_migration(s, p ? p : uri, &local_err);
2409 #ifdef CONFIG_RDMA
2410     } else if (strstart(uri, "rdma:", &p)) {
2411         rdma_start_outgoing_migration(s, p, &local_err);
2412 #endif
2413     } else if (strstart(uri, "exec:", &p)) {
2414         exec_start_outgoing_migration(s, p, &local_err);
2415     } else if (strstart(uri, "fd:", &p)) {
2416         fd_start_outgoing_migration(s, p, &local_err);
2417     } else {
2418         if (!(has_resume && resume)) {
2419             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2420         }
2421         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2422                    "a valid migration protocol");
2423         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2424                           MIGRATION_STATUS_FAILED);
2425         block_cleanup_parameters(s);
2426         return;
2427     }
2428 
2429     if (local_err) {
2430         if (!(has_resume && resume)) {
2431             yank_unregister_instance(MIGRATION_YANK_INSTANCE);
2432         }
2433         migrate_fd_error(s, local_err);
2434         error_propagate(errp, local_err);
2435         return;
2436     }
2437 }
2438 
2439 void qmp_migrate_cancel(Error **errp)
2440 {
2441     migration_cancel(NULL);
2442 }
2443 
2444 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2445 {
2446     MigrationState *s = migrate_get_current();
2447     if (s->state != state) {
2448         error_setg(errp,  "Migration not in expected state: %s",
2449                    MigrationStatus_str(s->state));
2450         return;
2451     }
2452     qemu_sem_post(&s->pause_sem);
2453 }
2454 
2455 bool migrate_release_ram(void)
2456 {
2457     MigrationState *s;
2458 
2459     s = migrate_get_current();
2460 
2461     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2462 }
2463 
2464 bool migrate_postcopy_ram(void)
2465 {
2466     MigrationState *s;
2467 
2468     s = migrate_get_current();
2469 
2470     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2471 }
2472 
2473 bool migrate_postcopy(void)
2474 {
2475     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2476 }
2477 
2478 bool migrate_auto_converge(void)
2479 {
2480     MigrationState *s;
2481 
2482     s = migrate_get_current();
2483 
2484     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2485 }
2486 
2487 bool migrate_zero_blocks(void)
2488 {
2489     MigrationState *s;
2490 
2491     s = migrate_get_current();
2492 
2493     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2494 }
2495 
2496 bool migrate_postcopy_blocktime(void)
2497 {
2498     MigrationState *s;
2499 
2500     s = migrate_get_current();
2501 
2502     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2503 }
2504 
2505 bool migrate_use_compression(void)
2506 {
2507     MigrationState *s;
2508 
2509     s = migrate_get_current();
2510 
2511     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2512 }
2513 
2514 int migrate_compress_level(void)
2515 {
2516     MigrationState *s;
2517 
2518     s = migrate_get_current();
2519 
2520     return s->parameters.compress_level;
2521 }
2522 
2523 int migrate_compress_threads(void)
2524 {
2525     MigrationState *s;
2526 
2527     s = migrate_get_current();
2528 
2529     return s->parameters.compress_threads;
2530 }
2531 
2532 int migrate_compress_wait_thread(void)
2533 {
2534     MigrationState *s;
2535 
2536     s = migrate_get_current();
2537 
2538     return s->parameters.compress_wait_thread;
2539 }
2540 
2541 int migrate_decompress_threads(void)
2542 {
2543     MigrationState *s;
2544 
2545     s = migrate_get_current();
2546 
2547     return s->parameters.decompress_threads;
2548 }
2549 
2550 bool migrate_dirty_bitmaps(void)
2551 {
2552     MigrationState *s;
2553 
2554     s = migrate_get_current();
2555 
2556     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2557 }
2558 
2559 bool migrate_ignore_shared(void)
2560 {
2561     MigrationState *s;
2562 
2563     s = migrate_get_current();
2564 
2565     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2566 }
2567 
2568 bool migrate_validate_uuid(void)
2569 {
2570     MigrationState *s;
2571 
2572     s = migrate_get_current();
2573 
2574     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2575 }
2576 
2577 bool migrate_use_events(void)
2578 {
2579     MigrationState *s;
2580 
2581     s = migrate_get_current();
2582 
2583     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2584 }
2585 
2586 bool migrate_use_multifd(void)
2587 {
2588     MigrationState *s;
2589 
2590     s = migrate_get_current();
2591 
2592     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2593 }
2594 
2595 bool migrate_pause_before_switchover(void)
2596 {
2597     MigrationState *s;
2598 
2599     s = migrate_get_current();
2600 
2601     return s->enabled_capabilities[
2602         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2603 }
2604 
2605 int migrate_multifd_channels(void)
2606 {
2607     MigrationState *s;
2608 
2609     s = migrate_get_current();
2610 
2611     return s->parameters.multifd_channels;
2612 }
2613 
2614 MultiFDCompression migrate_multifd_compression(void)
2615 {
2616     MigrationState *s;
2617 
2618     s = migrate_get_current();
2619 
2620     assert(s->parameters.multifd_compression < MULTIFD_COMPRESSION__MAX);
2621     return s->parameters.multifd_compression;
2622 }
2623 
2624 int migrate_multifd_zlib_level(void)
2625 {
2626     MigrationState *s;
2627 
2628     s = migrate_get_current();
2629 
2630     return s->parameters.multifd_zlib_level;
2631 }
2632 
2633 int migrate_multifd_zstd_level(void)
2634 {
2635     MigrationState *s;
2636 
2637     s = migrate_get_current();
2638 
2639     return s->parameters.multifd_zstd_level;
2640 }
2641 
2642 #ifdef CONFIG_LINUX
2643 bool migrate_use_zero_copy_send(void)
2644 {
2645     MigrationState *s;
2646 
2647     s = migrate_get_current();
2648 
2649     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_COPY_SEND];
2650 }
2651 #endif
2652 
2653 int migrate_use_tls(void)
2654 {
2655     MigrationState *s;
2656 
2657     s = migrate_get_current();
2658 
2659     return s->parameters.tls_creds && *s->parameters.tls_creds;
2660 }
2661 
2662 int migrate_use_xbzrle(void)
2663 {
2664     MigrationState *s;
2665 
2666     s = migrate_get_current();
2667 
2668     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2669 }
2670 
2671 uint64_t migrate_xbzrle_cache_size(void)
2672 {
2673     MigrationState *s;
2674 
2675     s = migrate_get_current();
2676 
2677     return s->parameters.xbzrle_cache_size;
2678 }
2679 
2680 static int64_t migrate_max_postcopy_bandwidth(void)
2681 {
2682     MigrationState *s;
2683 
2684     s = migrate_get_current();
2685 
2686     return s->parameters.max_postcopy_bandwidth;
2687 }
2688 
2689 bool migrate_use_block(void)
2690 {
2691     MigrationState *s;
2692 
2693     s = migrate_get_current();
2694 
2695     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2696 }
2697 
2698 bool migrate_use_return_path(void)
2699 {
2700     MigrationState *s;
2701 
2702     s = migrate_get_current();
2703 
2704     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2705 }
2706 
2707 bool migrate_use_block_incremental(void)
2708 {
2709     MigrationState *s;
2710 
2711     s = migrate_get_current();
2712 
2713     return s->parameters.block_incremental;
2714 }
2715 
2716 bool migrate_background_snapshot(void)
2717 {
2718     MigrationState *s;
2719 
2720     s = migrate_get_current();
2721 
2722     return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT];
2723 }
2724 
2725 bool migrate_postcopy_preempt(void)
2726 {
2727     MigrationState *s;
2728 
2729     s = migrate_get_current();
2730 
2731     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT];
2732 }
2733 
2734 /* migration thread support */
2735 /*
2736  * Something bad happened to the RP stream, mark an error
2737  * The caller shall print or trace something to indicate why
2738  */
2739 static void mark_source_rp_bad(MigrationState *s)
2740 {
2741     s->rp_state.error = true;
2742 }
2743 
2744 static struct rp_cmd_args {
2745     ssize_t     len; /* -1 = variable */
2746     const char *name;
2747 } rp_cmd_args[] = {
2748     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2749     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2750     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2751     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2752     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2753     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2754     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2755     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2756 };
2757 
2758 /*
2759  * Process a request for pages received on the return path,
2760  * We're allowed to send more than requested (e.g. to round to our page size)
2761  * and we don't need to send pages that have already been sent.
2762  */
2763 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2764                                        ram_addr_t start, size_t len)
2765 {
2766     long our_host_ps = qemu_real_host_page_size();
2767 
2768     trace_migrate_handle_rp_req_pages(rbname, start, len);
2769 
2770     /*
2771      * Since we currently insist on matching page sizes, just sanity check
2772      * we're being asked for whole host pages.
2773      */
2774     if (!QEMU_IS_ALIGNED(start, our_host_ps) ||
2775         !QEMU_IS_ALIGNED(len, our_host_ps)) {
2776         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2777                      " len: %zd", __func__, start, len);
2778         mark_source_rp_bad(ms);
2779         return;
2780     }
2781 
2782     if (ram_save_queue_pages(rbname, start, len)) {
2783         mark_source_rp_bad(ms);
2784     }
2785 }
2786 
2787 /* Return true to retry, false to quit */
2788 static bool postcopy_pause_return_path_thread(MigrationState *s)
2789 {
2790     trace_postcopy_pause_return_path();
2791 
2792     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2793 
2794     trace_postcopy_pause_return_path_continued();
2795 
2796     return true;
2797 }
2798 
2799 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2800 {
2801     RAMBlock *block = qemu_ram_block_by_name(block_name);
2802 
2803     if (!block) {
2804         error_report("%s: invalid block name '%s'", __func__, block_name);
2805         return -EINVAL;
2806     }
2807 
2808     /* Fetch the received bitmap and refresh the dirty bitmap */
2809     return ram_dirty_bitmap_reload(s, block);
2810 }
2811 
2812 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2813 {
2814     trace_source_return_path_thread_resume_ack(value);
2815 
2816     if (value != MIGRATION_RESUME_ACK_VALUE) {
2817         error_report("%s: illegal resume_ack value %"PRIu32,
2818                      __func__, value);
2819         return -1;
2820     }
2821 
2822     /* Now both sides are active. */
2823     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2824                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2825 
2826     /* Notify send thread that time to continue send pages */
2827     qemu_sem_post(&s->rp_state.rp_sem);
2828 
2829     return 0;
2830 }
2831 
2832 /* Release ms->rp_state.from_dst_file in a safe way */
2833 static void migration_release_from_dst_file(MigrationState *ms)
2834 {
2835     QEMUFile *file;
2836 
2837     WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
2838         /*
2839          * Reset the from_dst_file pointer first before releasing it, as we
2840          * can't block within lock section
2841          */
2842         file = ms->rp_state.from_dst_file;
2843         ms->rp_state.from_dst_file = NULL;
2844     }
2845 
2846     qemu_fclose(file);
2847 }
2848 
2849 /*
2850  * Handles messages sent on the return path towards the source VM
2851  *
2852  */
2853 static void *source_return_path_thread(void *opaque)
2854 {
2855     MigrationState *ms = opaque;
2856     QEMUFile *rp = ms->rp_state.from_dst_file;
2857     uint16_t header_len, header_type;
2858     uint8_t buf[512];
2859     uint32_t tmp32, sibling_error;
2860     ram_addr_t start = 0; /* =0 to silence warning */
2861     size_t  len = 0, expected_len;
2862     int res;
2863 
2864     trace_source_return_path_thread_entry();
2865     rcu_register_thread();
2866 
2867 retry:
2868     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2869            migration_is_setup_or_active(ms->state)) {
2870         trace_source_return_path_thread_loop_top();
2871         header_type = qemu_get_be16(rp);
2872         header_len = qemu_get_be16(rp);
2873 
2874         if (qemu_file_get_error(rp)) {
2875             mark_source_rp_bad(ms);
2876             goto out;
2877         }
2878 
2879         if (header_type >= MIG_RP_MSG_MAX ||
2880             header_type == MIG_RP_MSG_INVALID) {
2881             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2882                          header_type, header_len);
2883             mark_source_rp_bad(ms);
2884             goto out;
2885         }
2886 
2887         if ((rp_cmd_args[header_type].len != -1 &&
2888             header_len != rp_cmd_args[header_type].len) ||
2889             header_len > sizeof(buf)) {
2890             error_report("RP: Received '%s' message (0x%04x) with"
2891                          "incorrect length %d expecting %zu",
2892                          rp_cmd_args[header_type].name, header_type, header_len,
2893                          (size_t)rp_cmd_args[header_type].len);
2894             mark_source_rp_bad(ms);
2895             goto out;
2896         }
2897 
2898         /* We know we've got a valid header by this point */
2899         res = qemu_get_buffer(rp, buf, header_len);
2900         if (res != header_len) {
2901             error_report("RP: Failed reading data for message 0x%04x"
2902                          " read %d expected %d",
2903                          header_type, res, header_len);
2904             mark_source_rp_bad(ms);
2905             goto out;
2906         }
2907 
2908         /* OK, we have the message and the data */
2909         switch (header_type) {
2910         case MIG_RP_MSG_SHUT:
2911             sibling_error = ldl_be_p(buf);
2912             trace_source_return_path_thread_shut(sibling_error);
2913             if (sibling_error) {
2914                 error_report("RP: Sibling indicated error %d", sibling_error);
2915                 mark_source_rp_bad(ms);
2916             }
2917             /*
2918              * We'll let the main thread deal with closing the RP
2919              * we could do a shutdown(2) on it, but we're the only user
2920              * anyway, so there's nothing gained.
2921              */
2922             goto out;
2923 
2924         case MIG_RP_MSG_PONG:
2925             tmp32 = ldl_be_p(buf);
2926             trace_source_return_path_thread_pong(tmp32);
2927             break;
2928 
2929         case MIG_RP_MSG_REQ_PAGES:
2930             start = ldq_be_p(buf);
2931             len = ldl_be_p(buf + 8);
2932             migrate_handle_rp_req_pages(ms, NULL, start, len);
2933             break;
2934 
2935         case MIG_RP_MSG_REQ_PAGES_ID:
2936             expected_len = 12 + 1; /* header + termination */
2937 
2938             if (header_len >= expected_len) {
2939                 start = ldq_be_p(buf);
2940                 len = ldl_be_p(buf + 8);
2941                 /* Now we expect an idstr */
2942                 tmp32 = buf[12]; /* Length of the following idstr */
2943                 buf[13 + tmp32] = '\0';
2944                 expected_len += tmp32;
2945             }
2946             if (header_len != expected_len) {
2947                 error_report("RP: Req_Page_id with length %d expecting %zd",
2948                              header_len, expected_len);
2949                 mark_source_rp_bad(ms);
2950                 goto out;
2951             }
2952             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2953             break;
2954 
2955         case MIG_RP_MSG_RECV_BITMAP:
2956             if (header_len < 1) {
2957                 error_report("%s: missing block name", __func__);
2958                 mark_source_rp_bad(ms);
2959                 goto out;
2960             }
2961             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2962             buf[buf[0] + 1] = '\0';
2963             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2964                 mark_source_rp_bad(ms);
2965                 goto out;
2966             }
2967             break;
2968 
2969         case MIG_RP_MSG_RESUME_ACK:
2970             tmp32 = ldl_be_p(buf);
2971             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2972                 mark_source_rp_bad(ms);
2973                 goto out;
2974             }
2975             break;
2976 
2977         default:
2978             break;
2979         }
2980     }
2981 
2982 out:
2983     res = qemu_file_get_error(rp);
2984     if (res) {
2985         if (res && migration_in_postcopy()) {
2986             /*
2987              * Maybe there is something we can do: it looks like a
2988              * network down issue, and we pause for a recovery.
2989              */
2990             migration_release_from_dst_file(ms);
2991             rp = NULL;
2992             if (postcopy_pause_return_path_thread(ms)) {
2993                 /*
2994                  * Reload rp, reset the rest.  Referencing it is safe since
2995                  * it's reset only by us above, or when migration completes
2996                  */
2997                 rp = ms->rp_state.from_dst_file;
2998                 ms->rp_state.error = false;
2999                 goto retry;
3000             }
3001         }
3002 
3003         trace_source_return_path_thread_bad_end();
3004         mark_source_rp_bad(ms);
3005     }
3006 
3007     trace_source_return_path_thread_end();
3008     migration_release_from_dst_file(ms);
3009     rcu_unregister_thread();
3010     return NULL;
3011 }
3012 
3013 static int open_return_path_on_source(MigrationState *ms,
3014                                       bool create_thread)
3015 {
3016     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
3017     if (!ms->rp_state.from_dst_file) {
3018         return -1;
3019     }
3020 
3021     trace_open_return_path_on_source();
3022 
3023     if (!create_thread) {
3024         /* We're done */
3025         return 0;
3026     }
3027 
3028     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
3029                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
3030     ms->rp_state.rp_thread_created = true;
3031 
3032     trace_open_return_path_on_source_continue();
3033 
3034     return 0;
3035 }
3036 
3037 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
3038 static int await_return_path_close_on_source(MigrationState *ms)
3039 {
3040     /*
3041      * If this is a normal exit then the destination will send a SHUT and the
3042      * rp_thread will exit, however if there's an error we need to cause
3043      * it to exit.
3044      */
3045     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
3046         /*
3047          * shutdown(2), if we have it, will cause it to unblock if it's stuck
3048          * waiting for the destination.
3049          */
3050         qemu_file_shutdown(ms->rp_state.from_dst_file);
3051         mark_source_rp_bad(ms);
3052     }
3053     trace_await_return_path_close_on_source_joining();
3054     qemu_thread_join(&ms->rp_state.rp_thread);
3055     ms->rp_state.rp_thread_created = false;
3056     trace_await_return_path_close_on_source_close();
3057     return ms->rp_state.error;
3058 }
3059 
3060 /*
3061  * Switch from normal iteration to postcopy
3062  * Returns non-0 on error
3063  */
3064 static int postcopy_start(MigrationState *ms)
3065 {
3066     int ret;
3067     QIOChannelBuffer *bioc;
3068     QEMUFile *fb;
3069     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3070     int64_t bandwidth = migrate_max_postcopy_bandwidth();
3071     bool restart_block = false;
3072     int cur_state = MIGRATION_STATUS_ACTIVE;
3073 
3074     if (postcopy_preempt_wait_channel(ms)) {
3075         migrate_set_state(&ms->state, ms->state, MIGRATION_STATUS_FAILED);
3076         return -1;
3077     }
3078 
3079     if (!migrate_pause_before_switchover()) {
3080         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
3081                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
3082     }
3083 
3084     trace_postcopy_start();
3085     qemu_mutex_lock_iothread();
3086     trace_postcopy_start_set_run();
3087 
3088     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3089     global_state_store();
3090     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3091     if (ret < 0) {
3092         goto fail;
3093     }
3094 
3095     ret = migration_maybe_pause(ms, &cur_state,
3096                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
3097     if (ret < 0) {
3098         goto fail;
3099     }
3100 
3101     ret = bdrv_inactivate_all();
3102     if (ret < 0) {
3103         goto fail;
3104     }
3105     restart_block = true;
3106 
3107     /*
3108      * Cause any non-postcopiable, but iterative devices to
3109      * send out their final data.
3110      */
3111     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
3112 
3113     /*
3114      * in Finish migrate and with the io-lock held everything should
3115      * be quiet, but we've potentially still got dirty pages and we
3116      * need to tell the destination to throw any pages it's already received
3117      * that are dirty
3118      */
3119     if (migrate_postcopy_ram()) {
3120         ram_postcopy_send_discard_bitmap(ms);
3121     }
3122 
3123     /*
3124      * send rest of state - note things that are doing postcopy
3125      * will notice we're in POSTCOPY_ACTIVE and not actually
3126      * wrap their state up here
3127      */
3128     /* 0 max-postcopy-bandwidth means unlimited */
3129     if (!bandwidth) {
3130         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
3131     } else {
3132         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
3133     }
3134     if (migrate_postcopy_ram()) {
3135         /* Ping just for debugging, helps line traces up */
3136         qemu_savevm_send_ping(ms->to_dst_file, 2);
3137     }
3138 
3139     /*
3140      * While loading the device state we may trigger page transfer
3141      * requests and the fd must be free to process those, and thus
3142      * the destination must read the whole device state off the fd before
3143      * it starts processing it.  Unfortunately the ad-hoc migration format
3144      * doesn't allow the destination to know the size to read without fully
3145      * parsing it through each devices load-state code (especially the open
3146      * coded devices that use get/put).
3147      * So we wrap the device state up in a package with a length at the start;
3148      * to do this we use a qemu_buf to hold the whole of the device state.
3149      */
3150     bioc = qio_channel_buffer_new(4096);
3151     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
3152     fb = qemu_file_new_output(QIO_CHANNEL(bioc));
3153     object_unref(OBJECT(bioc));
3154 
3155     /*
3156      * Make sure the receiver can get incoming pages before we send the rest
3157      * of the state
3158      */
3159     qemu_savevm_send_postcopy_listen(fb);
3160 
3161     qemu_savevm_state_complete_precopy(fb, false, false);
3162     if (migrate_postcopy_ram()) {
3163         qemu_savevm_send_ping(fb, 3);
3164     }
3165 
3166     qemu_savevm_send_postcopy_run(fb);
3167 
3168     /* <><> end of stuff going into the package */
3169 
3170     /* Last point of recovery; as soon as we send the package the destination
3171      * can open devices and potentially start running.
3172      * Lets just check again we've not got any errors.
3173      */
3174     ret = qemu_file_get_error(ms->to_dst_file);
3175     if (ret) {
3176         error_report("postcopy_start: Migration stream errored (pre package)");
3177         goto fail_closefb;
3178     }
3179 
3180     restart_block = false;
3181 
3182     /* Now send that blob */
3183     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
3184         goto fail_closefb;
3185     }
3186     qemu_fclose(fb);
3187 
3188     /* Send a notify to give a chance for anything that needs to happen
3189      * at the transition to postcopy and after the device state; in particular
3190      * spice needs to trigger a transition now
3191      */
3192     ms->postcopy_after_devices = true;
3193     notifier_list_notify(&migration_state_notifiers, ms);
3194 
3195     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
3196 
3197     qemu_mutex_unlock_iothread();
3198 
3199     if (migrate_postcopy_ram()) {
3200         /*
3201          * Although this ping is just for debug, it could potentially be
3202          * used for getting a better measurement of downtime at the source.
3203          */
3204         qemu_savevm_send_ping(ms->to_dst_file, 4);
3205     }
3206 
3207     if (migrate_release_ram()) {
3208         ram_postcopy_migrated_memory_release(ms);
3209     }
3210 
3211     ret = qemu_file_get_error(ms->to_dst_file);
3212     if (ret) {
3213         error_report("postcopy_start: Migration stream errored");
3214         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3215                               MIGRATION_STATUS_FAILED);
3216     }
3217 
3218     trace_postcopy_preempt_enabled(migrate_postcopy_preempt());
3219 
3220     return ret;
3221 
3222 fail_closefb:
3223     qemu_fclose(fb);
3224 fail:
3225     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
3226                           MIGRATION_STATUS_FAILED);
3227     if (restart_block) {
3228         /* A failure happened early enough that we know the destination hasn't
3229          * accessed block devices, so we're safe to recover.
3230          */
3231         Error *local_err = NULL;
3232 
3233         bdrv_activate_all(&local_err);
3234         if (local_err) {
3235             error_report_err(local_err);
3236         }
3237     }
3238     qemu_mutex_unlock_iothread();
3239     return -1;
3240 }
3241 
3242 /**
3243  * migration_maybe_pause: Pause if required to by
3244  * migrate_pause_before_switchover called with the iothread locked
3245  * Returns: 0 on success
3246  */
3247 static int migration_maybe_pause(MigrationState *s,
3248                                  int *current_active_state,
3249                                  int new_state)
3250 {
3251     if (!migrate_pause_before_switchover()) {
3252         return 0;
3253     }
3254 
3255     /* Since leaving this state is not atomic with posting the semaphore
3256      * it's possible that someone could have issued multiple migrate_continue
3257      * and the semaphore is incorrectly positive at this point;
3258      * the docs say it's undefined to reinit a semaphore that's already
3259      * init'd, so use timedwait to eat up any existing posts.
3260      */
3261     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
3262         /* This block intentionally left blank */
3263     }
3264 
3265     /*
3266      * If the migration is cancelled when it is in the completion phase,
3267      * the migration state is set to MIGRATION_STATUS_CANCELLING.
3268      * So we don't need to wait a semaphore, otherwise we would always
3269      * wait for the 'pause_sem' semaphore.
3270      */
3271     if (s->state != MIGRATION_STATUS_CANCELLING) {
3272         qemu_mutex_unlock_iothread();
3273         migrate_set_state(&s->state, *current_active_state,
3274                           MIGRATION_STATUS_PRE_SWITCHOVER);
3275         qemu_sem_wait(&s->pause_sem);
3276         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
3277                           new_state);
3278         *current_active_state = new_state;
3279         qemu_mutex_lock_iothread();
3280     }
3281 
3282     return s->state == new_state ? 0 : -EINVAL;
3283 }
3284 
3285 /**
3286  * migration_completion: Used by migration_thread when there's not much left.
3287  *   The caller 'breaks' the loop when this returns.
3288  *
3289  * @s: Current migration state
3290  */
3291 static void migration_completion(MigrationState *s)
3292 {
3293     int ret;
3294     int current_active_state = s->state;
3295 
3296     if (s->state == MIGRATION_STATUS_ACTIVE) {
3297         qemu_mutex_lock_iothread();
3298         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3299         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
3300         s->vm_was_running = runstate_is_running();
3301         ret = global_state_store();
3302 
3303         if (!ret) {
3304             bool inactivate = !migrate_colo_enabled();
3305             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
3306             trace_migration_completion_vm_stop(ret);
3307             if (ret >= 0) {
3308                 ret = migration_maybe_pause(s, &current_active_state,
3309                                             MIGRATION_STATUS_DEVICE);
3310             }
3311             if (ret >= 0) {
3312                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
3313                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
3314                                                          inactivate);
3315             }
3316             if (inactivate && ret >= 0) {
3317                 s->block_inactive = true;
3318             }
3319         }
3320         qemu_mutex_unlock_iothread();
3321 
3322         if (ret < 0) {
3323             goto fail;
3324         }
3325     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3326         trace_migration_completion_postcopy_end();
3327 
3328         qemu_mutex_lock_iothread();
3329         qemu_savevm_state_complete_postcopy(s->to_dst_file);
3330         qemu_mutex_unlock_iothread();
3331 
3332         /* Shutdown the postcopy fast path thread */
3333         if (migrate_postcopy_preempt()) {
3334             postcopy_preempt_shutdown_file(s);
3335         }
3336 
3337         trace_migration_completion_postcopy_end_after_complete();
3338     } else {
3339         goto fail;
3340     }
3341 
3342     /*
3343      * If rp was opened we must clean up the thread before
3344      * cleaning everything else up (since if there are no failures
3345      * it will wait for the destination to send it's status in
3346      * a SHUT command).
3347      */
3348     if (s->rp_state.rp_thread_created) {
3349         int rp_error;
3350         trace_migration_return_path_end_before();
3351         rp_error = await_return_path_close_on_source(s);
3352         trace_migration_return_path_end_after(rp_error);
3353         if (rp_error) {
3354             goto fail_invalidate;
3355         }
3356     }
3357 
3358     if (qemu_file_get_error(s->to_dst_file)) {
3359         trace_migration_completion_file_err();
3360         goto fail_invalidate;
3361     }
3362 
3363     if (migrate_colo_enabled() && s->state == MIGRATION_STATUS_ACTIVE) {
3364         /* COLO does not support postcopy */
3365         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
3366                           MIGRATION_STATUS_COLO);
3367     } else {
3368         migrate_set_state(&s->state, current_active_state,
3369                           MIGRATION_STATUS_COMPLETED);
3370     }
3371 
3372     return;
3373 
3374 fail_invalidate:
3375     /* If not doing postcopy, vm_start() will be called: let's regain
3376      * control on images.
3377      */
3378     if (s->state == MIGRATION_STATUS_ACTIVE ||
3379         s->state == MIGRATION_STATUS_DEVICE) {
3380         Error *local_err = NULL;
3381 
3382         qemu_mutex_lock_iothread();
3383         bdrv_activate_all(&local_err);
3384         if (local_err) {
3385             error_report_err(local_err);
3386         } else {
3387             s->block_inactive = false;
3388         }
3389         qemu_mutex_unlock_iothread();
3390     }
3391 
3392 fail:
3393     migrate_set_state(&s->state, current_active_state,
3394                       MIGRATION_STATUS_FAILED);
3395 }
3396 
3397 /**
3398  * bg_migration_completion: Used by bg_migration_thread when after all the
3399  *   RAM has been saved. The caller 'breaks' the loop when this returns.
3400  *
3401  * @s: Current migration state
3402  */
3403 static void bg_migration_completion(MigrationState *s)
3404 {
3405     int current_active_state = s->state;
3406 
3407     /*
3408      * Stop tracking RAM writes - un-protect memory, un-register UFFD
3409      * memory ranges, flush kernel wait queues and wake up threads
3410      * waiting for write fault to be resolved.
3411      */
3412     ram_write_tracking_stop();
3413 
3414     if (s->state == MIGRATION_STATUS_ACTIVE) {
3415         /*
3416          * By this moment we have RAM content saved into the migration stream.
3417          * The next step is to flush the non-RAM content (device state)
3418          * right after the ram content. The device state has been stored into
3419          * the temporary buffer before RAM saving started.
3420          */
3421         qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage);
3422         qemu_fflush(s->to_dst_file);
3423     } else if (s->state == MIGRATION_STATUS_CANCELLING) {
3424         goto fail;
3425     }
3426 
3427     if (qemu_file_get_error(s->to_dst_file)) {
3428         trace_migration_completion_file_err();
3429         goto fail;
3430     }
3431 
3432     migrate_set_state(&s->state, current_active_state,
3433                       MIGRATION_STATUS_COMPLETED);
3434     return;
3435 
3436 fail:
3437     migrate_set_state(&s->state, current_active_state,
3438                       MIGRATION_STATUS_FAILED);
3439 }
3440 
3441 bool migrate_colo_enabled(void)
3442 {
3443     MigrationState *s = migrate_get_current();
3444     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3445 }
3446 
3447 typedef enum MigThrError {
3448     /* No error detected */
3449     MIG_THR_ERR_NONE = 0,
3450     /* Detected error, but resumed successfully */
3451     MIG_THR_ERR_RECOVERED = 1,
3452     /* Detected fatal error, need to exit */
3453     MIG_THR_ERR_FATAL = 2,
3454 } MigThrError;
3455 
3456 static int postcopy_resume_handshake(MigrationState *s)
3457 {
3458     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3459 
3460     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3461         qemu_sem_wait(&s->rp_state.rp_sem);
3462     }
3463 
3464     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3465         return 0;
3466     }
3467 
3468     return -1;
3469 }
3470 
3471 /* Return zero if success, or <0 for error */
3472 static int postcopy_do_resume(MigrationState *s)
3473 {
3474     int ret;
3475 
3476     /*
3477      * Call all the resume_prepare() hooks, so that modules can be
3478      * ready for the migration resume.
3479      */
3480     ret = qemu_savevm_state_resume_prepare(s);
3481     if (ret) {
3482         error_report("%s: resume_prepare() failure detected: %d",
3483                      __func__, ret);
3484         return ret;
3485     }
3486 
3487     /*
3488      * Last handshake with destination on the resume (destination will
3489      * switch to postcopy-active afterwards)
3490      */
3491     ret = postcopy_resume_handshake(s);
3492     if (ret) {
3493         error_report("%s: handshake failed: %d", __func__, ret);
3494         return ret;
3495     }
3496 
3497     return 0;
3498 }
3499 
3500 /*
3501  * We don't return until we are in a safe state to continue current
3502  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3503  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3504  */
3505 static MigThrError postcopy_pause(MigrationState *s)
3506 {
3507     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3508 
3509     while (true) {
3510         QEMUFile *file;
3511 
3512         /*
3513          * Current channel is possibly broken. Release it.  Note that this is
3514          * guaranteed even without lock because to_dst_file should only be
3515          * modified by the migration thread.  That also guarantees that the
3516          * unregister of yank is safe too without the lock.  It should be safe
3517          * even to be within the qemu_file_lock, but we didn't do that to avoid
3518          * taking more mutex (yank_lock) within qemu_file_lock.  TL;DR: we make
3519          * the qemu_file_lock critical section as small as possible.
3520          */
3521         assert(s->to_dst_file);
3522         migration_ioc_unregister_yank_from_file(s->to_dst_file);
3523         qemu_mutex_lock(&s->qemu_file_lock);
3524         file = s->to_dst_file;
3525         s->to_dst_file = NULL;
3526         qemu_mutex_unlock(&s->qemu_file_lock);
3527 
3528         qemu_file_shutdown(file);
3529         qemu_fclose(file);
3530 
3531         /*
3532          * Do the same to postcopy fast path socket too if there is.  No
3533          * locking needed because no racer as long as we do this before setting
3534          * status to paused.
3535          */
3536         if (s->postcopy_qemufile_src) {
3537             migration_ioc_unregister_yank_from_file(s->postcopy_qemufile_src);
3538             qemu_file_shutdown(s->postcopy_qemufile_src);
3539             qemu_fclose(s->postcopy_qemufile_src);
3540             s->postcopy_qemufile_src = NULL;
3541         }
3542 
3543         migrate_set_state(&s->state, s->state,
3544                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3545 
3546         error_report("Detected IO failure for postcopy. "
3547                      "Migration paused.");
3548 
3549         /*
3550          * We wait until things fixed up. Then someone will setup the
3551          * status back for us.
3552          */
3553         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3554             qemu_sem_wait(&s->postcopy_pause_sem);
3555         }
3556 
3557         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3558             /* Woken up by a recover procedure. Give it a shot */
3559 
3560             if (postcopy_preempt_wait_channel(s)) {
3561                 /*
3562                  * Preempt enabled, and new channel create failed; loop
3563                  * back to wait for another recovery.
3564                  */
3565                 continue;
3566             }
3567 
3568             /*
3569              * Firstly, let's wake up the return path now, with a new
3570              * return path channel.
3571              */
3572             qemu_sem_post(&s->postcopy_pause_rp_sem);
3573 
3574             /* Do the resume logic */
3575             if (postcopy_do_resume(s) == 0) {
3576                 /* Let's continue! */
3577                 trace_postcopy_pause_continued();
3578                 return MIG_THR_ERR_RECOVERED;
3579             } else {
3580                 /*
3581                  * Something wrong happened during the recovery, let's
3582                  * pause again. Pause is always better than throwing
3583                  * data away.
3584                  */
3585                 continue;
3586             }
3587         } else {
3588             /* This is not right... Time to quit. */
3589             return MIG_THR_ERR_FATAL;
3590         }
3591     }
3592 }
3593 
3594 static MigThrError migration_detect_error(MigrationState *s)
3595 {
3596     int ret;
3597     int state = s->state;
3598     Error *local_error = NULL;
3599 
3600     if (state == MIGRATION_STATUS_CANCELLING ||
3601         state == MIGRATION_STATUS_CANCELLED) {
3602         /* End the migration, but don't set the state to failed */
3603         return MIG_THR_ERR_FATAL;
3604     }
3605 
3606     /*
3607      * Try to detect any file errors.  Note that postcopy_qemufile_src will
3608      * be NULL when postcopy preempt is not enabled.
3609      */
3610     ret = qemu_file_get_error_obj_any(s->to_dst_file,
3611                                       s->postcopy_qemufile_src,
3612                                       &local_error);
3613     if (!ret) {
3614         /* Everything is fine */
3615         assert(!local_error);
3616         return MIG_THR_ERR_NONE;
3617     }
3618 
3619     if (local_error) {
3620         migrate_set_error(s, local_error);
3621         error_free(local_error);
3622     }
3623 
3624     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret) {
3625         /*
3626          * For postcopy, we allow the network to be down for a
3627          * while. After that, it can be continued by a
3628          * recovery phase.
3629          */
3630         return postcopy_pause(s);
3631     } else {
3632         /*
3633          * For precopy (or postcopy with error outside IO), we fail
3634          * with no time.
3635          */
3636         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3637         trace_migration_thread_file_err();
3638 
3639         /* Time to stop the migration, now. */
3640         return MIG_THR_ERR_FATAL;
3641     }
3642 }
3643 
3644 /* How many bytes have we transferred since the beginning of the migration */
3645 static uint64_t migration_total_bytes(MigrationState *s)
3646 {
3647     return qemu_file_total_transferred(s->to_dst_file) +
3648         ram_counters.multifd_bytes;
3649 }
3650 
3651 static void migration_calculate_complete(MigrationState *s)
3652 {
3653     uint64_t bytes = migration_total_bytes(s);
3654     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3655     int64_t transfer_time;
3656 
3657     s->total_time = end_time - s->start_time;
3658     if (!s->downtime) {
3659         /*
3660          * It's still not set, so we are precopy migration.  For
3661          * postcopy, downtime is calculated during postcopy_start().
3662          */
3663         s->downtime = end_time - s->downtime_start;
3664     }
3665 
3666     transfer_time = s->total_time - s->setup_time;
3667     if (transfer_time) {
3668         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3669     }
3670 }
3671 
3672 static void update_iteration_initial_status(MigrationState *s)
3673 {
3674     /*
3675      * Update these three fields at the same time to avoid mismatch info lead
3676      * wrong speed calculation.
3677      */
3678     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3679     s->iteration_initial_bytes = migration_total_bytes(s);
3680     s->iteration_initial_pages = ram_get_total_transferred_pages();
3681 }
3682 
3683 static void migration_update_counters(MigrationState *s,
3684                                       int64_t current_time)
3685 {
3686     uint64_t transferred, transferred_pages, time_spent;
3687     uint64_t current_bytes; /* bytes transferred since the beginning */
3688     double bandwidth;
3689 
3690     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3691         return;
3692     }
3693 
3694     current_bytes = migration_total_bytes(s);
3695     transferred = current_bytes - s->iteration_initial_bytes;
3696     time_spent = current_time - s->iteration_start_time;
3697     bandwidth = (double)transferred / time_spent;
3698     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3699 
3700     s->mbps = (((double) transferred * 8.0) /
3701                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3702 
3703     transferred_pages = ram_get_total_transferred_pages() -
3704                             s->iteration_initial_pages;
3705     s->pages_per_second = (double) transferred_pages /
3706                              (((double) time_spent / 1000.0));
3707 
3708     /*
3709      * if we haven't sent anything, we don't want to
3710      * recalculate. 10000 is a small enough number for our purposes
3711      */
3712     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3713         s->expected_downtime = ram_counters.remaining / bandwidth;
3714     }
3715 
3716     qemu_file_reset_rate_limit(s->to_dst_file);
3717 
3718     update_iteration_initial_status(s);
3719 
3720     trace_migrate_transferred(transferred, time_spent,
3721                               bandwidth, s->threshold_size);
3722 }
3723 
3724 /* Migration thread iteration status */
3725 typedef enum {
3726     MIG_ITERATE_RESUME,         /* Resume current iteration */
3727     MIG_ITERATE_SKIP,           /* Skip current iteration */
3728     MIG_ITERATE_BREAK,          /* Break the loop */
3729 } MigIterateState;
3730 
3731 /*
3732  * Return true if continue to the next iteration directly, false
3733  * otherwise.
3734  */
3735 static MigIterateState migration_iteration_run(MigrationState *s)
3736 {
3737     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3738     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3739 
3740     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3741                               &pend_compat, &pend_post);
3742     pending_size = pend_pre + pend_compat + pend_post;
3743 
3744     trace_migrate_pending(pending_size, s->threshold_size,
3745                           pend_pre, pend_compat, pend_post);
3746 
3747     if (pending_size && pending_size >= s->threshold_size) {
3748         /* Still a significant amount to transfer */
3749         if (!in_postcopy && pend_pre <= s->threshold_size &&
3750             qatomic_read(&s->start_postcopy)) {
3751             if (postcopy_start(s)) {
3752                 error_report("%s: postcopy failed to start", __func__);
3753             }
3754             return MIG_ITERATE_SKIP;
3755         }
3756         /* Just another iteration step */
3757         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3758     } else {
3759         trace_migration_thread_low_pending(pending_size);
3760         migration_completion(s);
3761         return MIG_ITERATE_BREAK;
3762     }
3763 
3764     return MIG_ITERATE_RESUME;
3765 }
3766 
3767 static void migration_iteration_finish(MigrationState *s)
3768 {
3769     /* If we enabled cpu throttling for auto-converge, turn it off. */
3770     cpu_throttle_stop();
3771 
3772     qemu_mutex_lock_iothread();
3773     switch (s->state) {
3774     case MIGRATION_STATUS_COMPLETED:
3775         migration_calculate_complete(s);
3776         runstate_set(RUN_STATE_POSTMIGRATE);
3777         break;
3778     case MIGRATION_STATUS_COLO:
3779         if (!migrate_colo_enabled()) {
3780             error_report("%s: critical error: calling COLO code without "
3781                          "COLO enabled", __func__);
3782         }
3783         migrate_start_colo_process(s);
3784         s->vm_was_running = true;
3785         /* Fallthrough */
3786     case MIGRATION_STATUS_FAILED:
3787     case MIGRATION_STATUS_CANCELLED:
3788     case MIGRATION_STATUS_CANCELLING:
3789         if (s->vm_was_running) {
3790             if (!runstate_check(RUN_STATE_SHUTDOWN)) {
3791                 vm_start();
3792             }
3793         } else {
3794             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3795                 runstate_set(RUN_STATE_POSTMIGRATE);
3796             }
3797         }
3798         break;
3799 
3800     default:
3801         /* Should not reach here, but if so, forgive the VM. */
3802         error_report("%s: Unknown ending state %d", __func__, s->state);
3803         break;
3804     }
3805     migrate_fd_cleanup_schedule(s);
3806     qemu_mutex_unlock_iothread();
3807 }
3808 
3809 static void bg_migration_iteration_finish(MigrationState *s)
3810 {
3811     qemu_mutex_lock_iothread();
3812     switch (s->state) {
3813     case MIGRATION_STATUS_COMPLETED:
3814         migration_calculate_complete(s);
3815         break;
3816 
3817     case MIGRATION_STATUS_ACTIVE:
3818     case MIGRATION_STATUS_FAILED:
3819     case MIGRATION_STATUS_CANCELLED:
3820     case MIGRATION_STATUS_CANCELLING:
3821         break;
3822 
3823     default:
3824         /* Should not reach here, but if so, forgive the VM. */
3825         error_report("%s: Unknown ending state %d", __func__, s->state);
3826         break;
3827     }
3828 
3829     migrate_fd_cleanup_schedule(s);
3830     qemu_mutex_unlock_iothread();
3831 }
3832 
3833 /*
3834  * Return true if continue to the next iteration directly, false
3835  * otherwise.
3836  */
3837 static MigIterateState bg_migration_iteration_run(MigrationState *s)
3838 {
3839     int res;
3840 
3841     res = qemu_savevm_state_iterate(s->to_dst_file, false);
3842     if (res > 0) {
3843         bg_migration_completion(s);
3844         return MIG_ITERATE_BREAK;
3845     }
3846 
3847     return MIG_ITERATE_RESUME;
3848 }
3849 
3850 void migration_make_urgent_request(void)
3851 {
3852     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3853 }
3854 
3855 void migration_consume_urgent_request(void)
3856 {
3857     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3858 }
3859 
3860 /* Returns true if the rate limiting was broken by an urgent request */
3861 bool migration_rate_limit(void)
3862 {
3863     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3864     MigrationState *s = migrate_get_current();
3865 
3866     bool urgent = false;
3867     migration_update_counters(s, now);
3868     if (qemu_file_rate_limit(s->to_dst_file)) {
3869 
3870         if (qemu_file_get_error(s->to_dst_file)) {
3871             return false;
3872         }
3873         /*
3874          * Wait for a delay to do rate limiting OR
3875          * something urgent to post the semaphore.
3876          */
3877         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3878         trace_migration_rate_limit_pre(ms);
3879         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3880             /*
3881              * We were woken by one or more urgent things but
3882              * the timedwait will have consumed one of them.
3883              * The service routine for the urgent wake will dec
3884              * the semaphore itself for each item it consumes,
3885              * so add this one we just eat back.
3886              */
3887             qemu_sem_post(&s->rate_limit_sem);
3888             urgent = true;
3889         }
3890         trace_migration_rate_limit_post(urgent);
3891     }
3892     return urgent;
3893 }
3894 
3895 /*
3896  * if failover devices are present, wait they are completely
3897  * unplugged
3898  */
3899 
3900 static void qemu_savevm_wait_unplug(MigrationState *s, int old_state,
3901                                     int new_state)
3902 {
3903     if (qemu_savevm_state_guest_unplug_pending()) {
3904         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG);
3905 
3906         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3907                qemu_savevm_state_guest_unplug_pending()) {
3908             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3909         }
3910         if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) {
3911             int timeout = 120; /* 30 seconds */
3912             /*
3913              * migration has been canceled
3914              * but as we have started an unplug we must wait the end
3915              * to be able to plug back the card
3916              */
3917             while (timeout-- && qemu_savevm_state_guest_unplug_pending()) {
3918                 qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3919             }
3920             if (qemu_savevm_state_guest_unplug_pending() &&
3921                 !qtest_enabled()) {
3922                 warn_report("migration: partially unplugged device on "
3923                             "failure");
3924             }
3925         }
3926 
3927         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state);
3928     } else {
3929         migrate_set_state(&s->state, old_state, new_state);
3930     }
3931 }
3932 
3933 /*
3934  * Master migration thread on the source VM.
3935  * It drives the migration and pumps the data down the outgoing channel.
3936  */
3937 static void *migration_thread(void *opaque)
3938 {
3939     MigrationState *s = opaque;
3940     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3941     MigThrError thr_error;
3942     bool urgent = false;
3943 
3944     rcu_register_thread();
3945 
3946     object_ref(OBJECT(s));
3947     update_iteration_initial_status(s);
3948 
3949     qemu_savevm_state_header(s->to_dst_file);
3950 
3951     /*
3952      * If we opened the return path, we need to make sure dst has it
3953      * opened as well.
3954      */
3955     if (s->rp_state.rp_thread_created) {
3956         /* Now tell the dest that it should open its end so it can reply */
3957         qemu_savevm_send_open_return_path(s->to_dst_file);
3958 
3959         /* And do a ping that will make stuff easier to debug */
3960         qemu_savevm_send_ping(s->to_dst_file, 1);
3961     }
3962 
3963     if (migrate_postcopy()) {
3964         /*
3965          * Tell the destination that we *might* want to do postcopy later;
3966          * if the other end can't do postcopy it should fail now, nice and
3967          * early.
3968          */
3969         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3970     }
3971 
3972     if (migrate_colo_enabled()) {
3973         /* Notify migration destination that we enable COLO */
3974         qemu_savevm_send_colo_enable(s->to_dst_file);
3975     }
3976 
3977     qemu_savevm_state_setup(s->to_dst_file);
3978 
3979     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
3980                                MIGRATION_STATUS_ACTIVE);
3981 
3982     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3983 
3984     trace_migration_thread_setup_complete();
3985 
3986     while (migration_is_active(s)) {
3987         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3988             MigIterateState iter_state = migration_iteration_run(s);
3989             if (iter_state == MIG_ITERATE_SKIP) {
3990                 continue;
3991             } else if (iter_state == MIG_ITERATE_BREAK) {
3992                 break;
3993             }
3994         }
3995 
3996         /*
3997          * Try to detect any kind of failures, and see whether we
3998          * should stop the migration now.
3999          */
4000         thr_error = migration_detect_error(s);
4001         if (thr_error == MIG_THR_ERR_FATAL) {
4002             /* Stop migration */
4003             break;
4004         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
4005             /*
4006              * Just recovered from a e.g. network failure, reset all
4007              * the local variables. This is important to avoid
4008              * breaking transferred_bytes and bandwidth calculation
4009              */
4010             update_iteration_initial_status(s);
4011         }
4012 
4013         urgent = migration_rate_limit();
4014     }
4015 
4016     trace_migration_thread_after_loop();
4017     migration_iteration_finish(s);
4018     object_unref(OBJECT(s));
4019     rcu_unregister_thread();
4020     return NULL;
4021 }
4022 
4023 static void bg_migration_vm_start_bh(void *opaque)
4024 {
4025     MigrationState *s = opaque;
4026 
4027     qemu_bh_delete(s->vm_start_bh);
4028     s->vm_start_bh = NULL;
4029 
4030     vm_start();
4031     s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start;
4032 }
4033 
4034 /**
4035  * Background snapshot thread, based on live migration code.
4036  * This is an alternative implementation of live migration mechanism
4037  * introduced specifically to support background snapshots.
4038  *
4039  * It takes advantage of userfault_fd write protection mechanism introduced
4040  * in v5.7 kernel. Compared to existing dirty page logging migration much
4041  * lesser stream traffic is produced resulting in smaller snapshot images,
4042  * simply cause of no page duplicates can get into the stream.
4043  *
4044  * Another key point is that generated vmstate stream reflects machine state
4045  * 'frozen' at the beginning of snapshot creation compared to dirty page logging
4046  * mechanism, which effectively results in that saved snapshot is the state of VM
4047  * at the end of the process.
4048  */
4049 static void *bg_migration_thread(void *opaque)
4050 {
4051     MigrationState *s = opaque;
4052     int64_t setup_start;
4053     MigThrError thr_error;
4054     QEMUFile *fb;
4055     bool early_fail = true;
4056 
4057     rcu_register_thread();
4058     object_ref(OBJECT(s));
4059 
4060     qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
4061 
4062     setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
4063     /*
4064      * We want to save vmstate for the moment when migration has been
4065      * initiated but also we want to save RAM content while VM is running.
4066      * The RAM content should appear first in the vmstate. So, we first
4067      * stash the non-RAM part of the vmstate to the temporary buffer,
4068      * then write RAM part of the vmstate to the migration stream
4069      * with vCPUs running and, finally, write stashed non-RAM part of
4070      * the vmstate from the buffer to the migration stream.
4071      */
4072     s->bioc = qio_channel_buffer_new(512 * 1024);
4073     qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer");
4074     fb = qemu_file_new_output(QIO_CHANNEL(s->bioc));
4075     object_unref(OBJECT(s->bioc));
4076 
4077     update_iteration_initial_status(s);
4078 
4079     /*
4080      * Prepare for tracking memory writes with UFFD-WP - populate
4081      * RAM pages before protecting.
4082      */
4083 #ifdef __linux__
4084     ram_write_tracking_prepare();
4085 #endif
4086 
4087     qemu_savevm_state_header(s->to_dst_file);
4088     qemu_savevm_state_setup(s->to_dst_file);
4089 
4090     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
4091                                MIGRATION_STATUS_ACTIVE);
4092 
4093     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
4094 
4095     trace_migration_thread_setup_complete();
4096     s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
4097 
4098     qemu_mutex_lock_iothread();
4099 
4100     /*
4101      * If VM is currently in suspended state, then, to make a valid runstate
4102      * transition in vm_stop_force_state() we need to wakeup it up.
4103      */
4104     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
4105     s->vm_was_running = runstate_is_running();
4106 
4107     if (global_state_store()) {
4108         goto fail;
4109     }
4110     /* Forcibly stop VM before saving state of vCPUs and devices */
4111     if (vm_stop_force_state(RUN_STATE_PAUSED)) {
4112         goto fail;
4113     }
4114     /*
4115      * Put vCPUs in sync with shadow context structures, then
4116      * save their state to channel-buffer along with devices.
4117      */
4118     cpu_synchronize_all_states();
4119     if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) {
4120         goto fail;
4121     }
4122     /*
4123      * Since we are going to get non-iterable state data directly
4124      * from s->bioc->data, explicit flush is needed here.
4125      */
4126     qemu_fflush(fb);
4127 
4128     /* Now initialize UFFD context and start tracking RAM writes */
4129     if (ram_write_tracking_start()) {
4130         goto fail;
4131     }
4132     early_fail = false;
4133 
4134     /*
4135      * Start VM from BH handler to avoid write-fault lock here.
4136      * UFFD-WP protection for the whole RAM is already enabled so
4137      * calling VM state change notifiers from vm_start() would initiate
4138      * writes to virtio VQs memory which is in write-protected region.
4139      */
4140     s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s);
4141     qemu_bh_schedule(s->vm_start_bh);
4142 
4143     qemu_mutex_unlock_iothread();
4144 
4145     while (migration_is_active(s)) {
4146         MigIterateState iter_state = bg_migration_iteration_run(s);
4147         if (iter_state == MIG_ITERATE_SKIP) {
4148             continue;
4149         } else if (iter_state == MIG_ITERATE_BREAK) {
4150             break;
4151         }
4152 
4153         /*
4154          * Try to detect any kind of failures, and see whether we
4155          * should stop the migration now.
4156          */
4157         thr_error = migration_detect_error(s);
4158         if (thr_error == MIG_THR_ERR_FATAL) {
4159             /* Stop migration */
4160             break;
4161         }
4162 
4163         migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
4164     }
4165 
4166     trace_migration_thread_after_loop();
4167 
4168 fail:
4169     if (early_fail) {
4170         migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
4171                 MIGRATION_STATUS_FAILED);
4172         qemu_mutex_unlock_iothread();
4173     }
4174 
4175     bg_migration_iteration_finish(s);
4176 
4177     qemu_fclose(fb);
4178     object_unref(OBJECT(s));
4179     rcu_unregister_thread();
4180 
4181     return NULL;
4182 }
4183 
4184 void migrate_fd_connect(MigrationState *s, Error *error_in)
4185 {
4186     Error *local_err = NULL;
4187     int64_t rate_limit;
4188     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
4189 
4190     /*
4191      * If there's a previous error, free it and prepare for another one.
4192      * Meanwhile if migration completes successfully, there won't have an error
4193      * dumped when calling migrate_fd_cleanup().
4194      */
4195     migrate_error_free(s);
4196 
4197     s->expected_downtime = s->parameters.downtime_limit;
4198     if (resume) {
4199         assert(s->cleanup_bh);
4200     } else {
4201         assert(!s->cleanup_bh);
4202         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
4203     }
4204     if (error_in) {
4205         migrate_fd_error(s, error_in);
4206         if (resume) {
4207             /*
4208              * Don't do cleanup for resume if channel is invalid, but only dump
4209              * the error.  We wait for another channel connect from the user.
4210              * The error_report still gives HMP user a hint on what failed.
4211              * It's normally done in migrate_fd_cleanup(), but call it here
4212              * explicitly.
4213              */
4214             error_report_err(error_copy(s->error));
4215         } else {
4216             migrate_fd_cleanup(s);
4217         }
4218         return;
4219     }
4220 
4221     if (resume) {
4222         /* This is a resumed migration */
4223         rate_limit = s->parameters.max_postcopy_bandwidth /
4224             XFER_LIMIT_RATIO;
4225     } else {
4226         /* This is a fresh new migration */
4227         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
4228 
4229         /* Notify before starting migration thread */
4230         notifier_list_notify(&migration_state_notifiers, s);
4231     }
4232 
4233     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
4234     qemu_file_set_blocking(s->to_dst_file, true);
4235 
4236     /*
4237      * Open the return path. For postcopy, it is used exclusively. For
4238      * precopy, only if user specified "return-path" capability would
4239      * QEMU uses the return path.
4240      */
4241     if (migrate_postcopy_ram() || migrate_use_return_path()) {
4242         if (open_return_path_on_source(s, !resume)) {
4243             error_report("Unable to open return-path for postcopy");
4244             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
4245             migrate_fd_cleanup(s);
4246             return;
4247         }
4248     }
4249 
4250     /* This needs to be done before resuming a postcopy */
4251     if (postcopy_preempt_setup(s, &local_err)) {
4252         error_report_err(local_err);
4253         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4254                           MIGRATION_STATUS_FAILED);
4255         migrate_fd_cleanup(s);
4256         return;
4257     }
4258 
4259     if (resume) {
4260         /* Wakeup the main migration thread to do the recovery */
4261         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
4262                           MIGRATION_STATUS_POSTCOPY_RECOVER);
4263         qemu_sem_post(&s->postcopy_pause_sem);
4264         return;
4265     }
4266 
4267     if (multifd_save_setup(&local_err) != 0) {
4268         error_report_err(local_err);
4269         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
4270                           MIGRATION_STATUS_FAILED);
4271         migrate_fd_cleanup(s);
4272         return;
4273     }
4274 
4275     if (migrate_background_snapshot()) {
4276         qemu_thread_create(&s->thread, "bg_snapshot",
4277                 bg_migration_thread, s, QEMU_THREAD_JOINABLE);
4278     } else {
4279         qemu_thread_create(&s->thread, "live_migration",
4280                 migration_thread, s, QEMU_THREAD_JOINABLE);
4281     }
4282     s->migration_thread_running = true;
4283 }
4284 
4285 void migration_global_dump(Monitor *mon)
4286 {
4287     MigrationState *ms = migrate_get_current();
4288 
4289     monitor_printf(mon, "globals:\n");
4290     monitor_printf(mon, "store-global-state: %s\n",
4291                    ms->store_global_state ? "on" : "off");
4292     monitor_printf(mon, "only-migratable: %s\n",
4293                    only_migratable ? "on" : "off");
4294     monitor_printf(mon, "send-configuration: %s\n",
4295                    ms->send_configuration ? "on" : "off");
4296     monitor_printf(mon, "send-section-footer: %s\n",
4297                    ms->send_section_footer ? "on" : "off");
4298     monitor_printf(mon, "decompress-error-check: %s\n",
4299                    ms->decompress_error_check ? "on" : "off");
4300     monitor_printf(mon, "clear-bitmap-shift: %u\n",
4301                    ms->clear_bitmap_shift);
4302 }
4303 
4304 #define DEFINE_PROP_MIG_CAP(name, x)             \
4305     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
4306 
4307 static Property migration_properties[] = {
4308     DEFINE_PROP_BOOL("store-global-state", MigrationState,
4309                      store_global_state, true),
4310     DEFINE_PROP_BOOL("send-configuration", MigrationState,
4311                      send_configuration, true),
4312     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
4313                      send_section_footer, true),
4314     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
4315                       decompress_error_check, true),
4316     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
4317                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
4318 
4319     /* Migration parameters */
4320     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
4321                       parameters.compress_level,
4322                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
4323     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
4324                       parameters.compress_threads,
4325                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
4326     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
4327                       parameters.compress_wait_thread, true),
4328     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
4329                       parameters.decompress_threads,
4330                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
4331     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
4332                       parameters.throttle_trigger_threshold,
4333                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
4334     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
4335                       parameters.cpu_throttle_initial,
4336                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
4337     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
4338                       parameters.cpu_throttle_increment,
4339                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
4340     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
4341                       parameters.cpu_throttle_tailslow, false),
4342     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
4343                       parameters.max_bandwidth, MAX_THROTTLE),
4344     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
4345                       parameters.downtime_limit,
4346                       DEFAULT_MIGRATE_SET_DOWNTIME),
4347     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
4348                       parameters.x_checkpoint_delay,
4349                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
4350     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
4351                       parameters.multifd_channels,
4352                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
4353     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
4354                       parameters.multifd_compression,
4355                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
4356     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
4357                       parameters.multifd_zlib_level,
4358                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
4359     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
4360                       parameters.multifd_zstd_level,
4361                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
4362     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
4363                       parameters.xbzrle_cache_size,
4364                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
4365     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
4366                       parameters.max_postcopy_bandwidth,
4367                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
4368     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
4369                       parameters.max_cpu_throttle,
4370                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
4371     DEFINE_PROP_SIZE("announce-initial", MigrationState,
4372                       parameters.announce_initial,
4373                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
4374     DEFINE_PROP_SIZE("announce-max", MigrationState,
4375                       parameters.announce_max,
4376                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
4377     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
4378                       parameters.announce_rounds,
4379                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
4380     DEFINE_PROP_SIZE("announce-step", MigrationState,
4381                       parameters.announce_step,
4382                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
4383     DEFINE_PROP_BOOL("x-postcopy-preempt-break-huge", MigrationState,
4384                       postcopy_preempt_break_huge, true),
4385     DEFINE_PROP_STRING("tls-creds", MigrationState, parameters.tls_creds),
4386     DEFINE_PROP_STRING("tls-hostname", MigrationState, parameters.tls_hostname),
4387     DEFINE_PROP_STRING("tls-authz", MigrationState, parameters.tls_authz),
4388 
4389     /* Migration capabilities */
4390     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
4391     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
4392     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
4393     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
4394     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
4395     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
4396     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
4397     DEFINE_PROP_MIG_CAP("x-postcopy-preempt",
4398                         MIGRATION_CAPABILITY_POSTCOPY_PREEMPT),
4399     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
4400     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
4401     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
4402     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
4403     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
4404     DEFINE_PROP_MIG_CAP("x-background-snapshot",
4405             MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
4406 #ifdef CONFIG_LINUX
4407     DEFINE_PROP_MIG_CAP("x-zero-copy-send",
4408             MIGRATION_CAPABILITY_ZERO_COPY_SEND),
4409 #endif
4410 
4411     DEFINE_PROP_END_OF_LIST(),
4412 };
4413 
4414 static void migration_class_init(ObjectClass *klass, void *data)
4415 {
4416     DeviceClass *dc = DEVICE_CLASS(klass);
4417 
4418     dc->user_creatable = false;
4419     device_class_set_props(dc, migration_properties);
4420 }
4421 
4422 static void migration_instance_finalize(Object *obj)
4423 {
4424     MigrationState *ms = MIGRATION_OBJ(obj);
4425 
4426     qemu_mutex_destroy(&ms->error_mutex);
4427     qemu_mutex_destroy(&ms->qemu_file_lock);
4428     qemu_sem_destroy(&ms->wait_unplug_sem);
4429     qemu_sem_destroy(&ms->rate_limit_sem);
4430     qemu_sem_destroy(&ms->pause_sem);
4431     qemu_sem_destroy(&ms->postcopy_pause_sem);
4432     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
4433     qemu_sem_destroy(&ms->rp_state.rp_sem);
4434     qemu_sem_destroy(&ms->postcopy_qemufile_src_sem);
4435     error_free(ms->error);
4436 }
4437 
4438 static void migration_instance_init(Object *obj)
4439 {
4440     MigrationState *ms = MIGRATION_OBJ(obj);
4441     MigrationParameters *params = &ms->parameters;
4442 
4443     ms->state = MIGRATION_STATUS_NONE;
4444     ms->mbps = -1;
4445     ms->pages_per_second = -1;
4446     qemu_sem_init(&ms->pause_sem, 0);
4447     qemu_mutex_init(&ms->error_mutex);
4448 
4449     params->tls_hostname = g_strdup("");
4450     params->tls_creds = g_strdup("");
4451 
4452     /* Set has_* up only for parameter checks */
4453     params->has_compress_level = true;
4454     params->has_compress_threads = true;
4455     params->has_compress_wait_thread = true;
4456     params->has_decompress_threads = true;
4457     params->has_throttle_trigger_threshold = true;
4458     params->has_cpu_throttle_initial = true;
4459     params->has_cpu_throttle_increment = true;
4460     params->has_cpu_throttle_tailslow = true;
4461     params->has_max_bandwidth = true;
4462     params->has_downtime_limit = true;
4463     params->has_x_checkpoint_delay = true;
4464     params->has_block_incremental = true;
4465     params->has_multifd_channels = true;
4466     params->has_multifd_compression = true;
4467     params->has_multifd_zlib_level = true;
4468     params->has_multifd_zstd_level = true;
4469     params->has_xbzrle_cache_size = true;
4470     params->has_max_postcopy_bandwidth = true;
4471     params->has_max_cpu_throttle = true;
4472     params->has_announce_initial = true;
4473     params->has_announce_max = true;
4474     params->has_announce_rounds = true;
4475     params->has_announce_step = true;
4476     params->has_tls_creds = true;
4477     params->has_tls_hostname = true;
4478     params->has_tls_authz = true;
4479 
4480     qemu_sem_init(&ms->postcopy_pause_sem, 0);
4481     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
4482     qemu_sem_init(&ms->rp_state.rp_sem, 0);
4483     qemu_sem_init(&ms->rate_limit_sem, 0);
4484     qemu_sem_init(&ms->wait_unplug_sem, 0);
4485     qemu_sem_init(&ms->postcopy_qemufile_src_sem, 0);
4486     qemu_mutex_init(&ms->qemu_file_lock);
4487 }
4488 
4489 /*
4490  * Return true if check pass, false otherwise. Error will be put
4491  * inside errp if provided.
4492  */
4493 static bool migration_object_check(MigrationState *ms, Error **errp)
4494 {
4495     MigrationCapabilityStatusList *head = NULL;
4496     /* Assuming all off */
4497     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
4498     int i;
4499 
4500     if (!migrate_params_check(&ms->parameters, errp)) {
4501         return false;
4502     }
4503 
4504     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
4505         if (ms->enabled_capabilities[i]) {
4506             QAPI_LIST_PREPEND(head, migrate_cap_add(i, true));
4507         }
4508     }
4509 
4510     ret = migrate_caps_check(cap_list, head, errp);
4511 
4512     /* It works with head == NULL */
4513     qapi_free_MigrationCapabilityStatusList(head);
4514 
4515     return ret;
4516 }
4517 
4518 static const TypeInfo migration_type = {
4519     .name = TYPE_MIGRATION,
4520     /*
4521      * NOTE: TYPE_MIGRATION is not really a device, as the object is
4522      * not created using qdev_new(), it is not attached to the qdev
4523      * device tree, and it is never realized.
4524      *
4525      * TODO: Make this TYPE_OBJECT once QOM provides something like
4526      * TYPE_DEVICE's "-global" properties.
4527      */
4528     .parent = TYPE_DEVICE,
4529     .class_init = migration_class_init,
4530     .class_size = sizeof(MigrationClass),
4531     .instance_size = sizeof(MigrationState),
4532     .instance_init = migration_instance_init,
4533     .instance_finalize = migration_instance_finalize,
4534 };
4535 
4536 static void register_migration_types(void)
4537 {
4538     type_register_static(&migration_type);
4539 }
4540 
4541 type_init(register_migration_types);
4542