xref: /openbmc/qemu/migration/ram.c (revision b278b60d)
1 /*
2  * QEMU System Emulator
3  *
4  * Copyright (c) 2003-2008 Fabrice Bellard
5  * Copyright (c) 2011-2015 Red Hat Inc
6  *
7  * Authors:
8  *  Juan Quintela <quintela@redhat.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a copy
11  * of this software and associated documentation files (the "Software"), to deal
12  * in the Software without restriction, including without limitation the rights
13  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14  * copies of the Software, and to permit persons to whom the Software is
15  * furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included in
18  * all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26  * THE SOFTWARE.
27  */
28 
29 #include "qemu/osdep.h"
30 #include "qemu/cutils.h"
31 #include "qemu/bitops.h"
32 #include "qemu/bitmap.h"
33 #include "qemu/madvise.h"
34 #include "qemu/main-loop.h"
35 #include "xbzrle.h"
36 #include "ram.h"
37 #include "migration.h"
38 #include "migration-stats.h"
39 #include "migration/register.h"
40 #include "migration/misc.h"
41 #include "qemu-file.h"
42 #include "postcopy-ram.h"
43 #include "page_cache.h"
44 #include "qemu/error-report.h"
45 #include "qapi/error.h"
46 #include "qapi/qapi-types-migration.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qapi-commands-migration.h"
49 #include "qapi/qmp/qerror.h"
50 #include "trace.h"
51 #include "exec/ram_addr.h"
52 #include "exec/target_page.h"
53 #include "qemu/rcu_queue.h"
54 #include "migration/colo.h"
55 #include "sysemu/cpu-throttle.h"
56 #include "savevm.h"
57 #include "qemu/iov.h"
58 #include "multifd.h"
59 #include "sysemu/runstate.h"
60 #include "rdma.h"
61 #include "options.h"
62 #include "sysemu/dirtylimit.h"
63 #include "sysemu/kvm.h"
64 
65 #include "hw/boards.h" /* for machine_dump_guest_core() */
66 
67 #if defined(__linux__)
68 #include "qemu/userfaultfd.h"
69 #endif /* defined(__linux__) */
70 
71 /***********************************************************/
72 /* ram save/restore */
73 
74 /*
75  * RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it
76  * worked for pages that were filled with the same char.  We switched
77  * it to only search for the zero value.  And to avoid confusion with
78  * RAM_SAVE_FLAG_COMPRESS_PAGE just rename it.
79  *
80  * RAM_SAVE_FLAG_FULL was obsoleted in 2009.
81  *
82  * RAM_SAVE_FLAG_COMPRESS_PAGE (0x100) was removed in QEMU 9.1.
83  */
84 #define RAM_SAVE_FLAG_FULL     0x01
85 #define RAM_SAVE_FLAG_ZERO     0x02
86 #define RAM_SAVE_FLAG_MEM_SIZE 0x04
87 #define RAM_SAVE_FLAG_PAGE     0x08
88 #define RAM_SAVE_FLAG_EOS      0x10
89 #define RAM_SAVE_FLAG_CONTINUE 0x20
90 #define RAM_SAVE_FLAG_XBZRLE   0x40
91 /* 0x80 is reserved in rdma.h for RAM_SAVE_FLAG_HOOK */
92 #define RAM_SAVE_FLAG_MULTIFD_FLUSH    0x200
93 /* We can't use any flag that is bigger than 0x200 */
94 
95 /*
96  * mapped-ram migration supports O_DIRECT, so we need to make sure the
97  * userspace buffer, the IO operation size and the file offset are
98  * aligned according to the underlying device's block size. The first
99  * two are already aligned to page size, but we need to add padding to
100  * the file to align the offset.  We cannot read the block size
101  * dynamically because the migration file can be moved between
102  * different systems, so use 1M to cover most block sizes and to keep
103  * the file offset aligned at page size as well.
104  */
105 #define MAPPED_RAM_FILE_OFFSET_ALIGNMENT 0x100000
106 
107 /*
108  * When doing mapped-ram migration, this is the amount we read from
109  * the pages region in the migration file at a time.
110  */
111 #define MAPPED_RAM_LOAD_BUF_SIZE 0x100000
112 
113 XBZRLECacheStats xbzrle_counters;
114 
115 /* used by the search for pages to send */
116 struct PageSearchStatus {
117     /* The migration channel used for a specific host page */
118     QEMUFile    *pss_channel;
119     /* Last block from where we have sent data */
120     RAMBlock *last_sent_block;
121     /* Current block being searched */
122     RAMBlock    *block;
123     /* Current page to search from */
124     unsigned long page;
125     /* Set once we wrap around */
126     bool         complete_round;
127     /* Whether we're sending a host page */
128     bool          host_page_sending;
129     /* The start/end of current host page.  Invalid if host_page_sending==false */
130     unsigned long host_page_start;
131     unsigned long host_page_end;
132 };
133 typedef struct PageSearchStatus PageSearchStatus;
134 
135 /* struct contains XBZRLE cache and a static page
136    used by the compression */
137 static struct {
138     /* buffer used for XBZRLE encoding */
139     uint8_t *encoded_buf;
140     /* buffer for storing page content */
141     uint8_t *current_buf;
142     /* Cache for XBZRLE, Protected by lock. */
143     PageCache *cache;
144     QemuMutex lock;
145     /* it will store a page full of zeros */
146     uint8_t *zero_target_page;
147     /* buffer used for XBZRLE decoding */
148     uint8_t *decoded_buf;
149 } XBZRLE;
150 
151 static void XBZRLE_cache_lock(void)
152 {
153     if (migrate_xbzrle()) {
154         qemu_mutex_lock(&XBZRLE.lock);
155     }
156 }
157 
158 static void XBZRLE_cache_unlock(void)
159 {
160     if (migrate_xbzrle()) {
161         qemu_mutex_unlock(&XBZRLE.lock);
162     }
163 }
164 
165 /**
166  * xbzrle_cache_resize: resize the xbzrle cache
167  *
168  * This function is called from migrate_params_apply in main
169  * thread, possibly while a migration is in progress.  A running
170  * migration may be using the cache and might finish during this call,
171  * hence changes to the cache are protected by XBZRLE.lock().
172  *
173  * Returns 0 for success or -1 for error
174  *
175  * @new_size: new cache size
176  * @errp: set *errp if the check failed, with reason
177  */
178 int xbzrle_cache_resize(uint64_t new_size, Error **errp)
179 {
180     PageCache *new_cache;
181     int64_t ret = 0;
182 
183     /* Check for truncation */
184     if (new_size != (size_t)new_size) {
185         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
186                    "exceeding address space");
187         return -1;
188     }
189 
190     if (new_size == migrate_xbzrle_cache_size()) {
191         /* nothing to do */
192         return 0;
193     }
194 
195     XBZRLE_cache_lock();
196 
197     if (XBZRLE.cache != NULL) {
198         new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp);
199         if (!new_cache) {
200             ret = -1;
201             goto out;
202         }
203 
204         cache_fini(XBZRLE.cache);
205         XBZRLE.cache = new_cache;
206     }
207 out:
208     XBZRLE_cache_unlock();
209     return ret;
210 }
211 
212 static bool postcopy_preempt_active(void)
213 {
214     return migrate_postcopy_preempt() && migration_in_postcopy();
215 }
216 
217 bool migrate_ram_is_ignored(RAMBlock *block)
218 {
219     return !qemu_ram_is_migratable(block) ||
220            (migrate_ignore_shared() && qemu_ram_is_shared(block)
221                                     && qemu_ram_is_named_file(block));
222 }
223 
224 #undef RAMBLOCK_FOREACH
225 
226 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
227 {
228     RAMBlock *block;
229     int ret = 0;
230 
231     RCU_READ_LOCK_GUARD();
232 
233     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
234         ret = func(block, opaque);
235         if (ret) {
236             break;
237         }
238     }
239     return ret;
240 }
241 
242 static void ramblock_recv_map_init(void)
243 {
244     RAMBlock *rb;
245 
246     RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
247         assert(!rb->receivedmap);
248         rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits());
249     }
250 }
251 
252 int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr)
253 {
254     return test_bit(ramblock_recv_bitmap_offset(host_addr, rb),
255                     rb->receivedmap);
256 }
257 
258 bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset)
259 {
260     return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
261 }
262 
263 void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr)
264 {
265     set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap);
266 }
267 
268 void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr,
269                                     size_t nr)
270 {
271     bitmap_set_atomic(rb->receivedmap,
272                       ramblock_recv_bitmap_offset(host_addr, rb),
273                       nr);
274 }
275 
276 void ramblock_recv_bitmap_set_offset(RAMBlock *rb, uint64_t byte_offset)
277 {
278     set_bit_atomic(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap);
279 }
280 #define  RAMBLOCK_RECV_BITMAP_ENDING  (0x0123456789abcdefULL)
281 
282 /*
283  * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes).
284  *
285  * Returns >0 if success with sent bytes, or <0 if error.
286  */
287 int64_t ramblock_recv_bitmap_send(QEMUFile *file,
288                                   const char *block_name)
289 {
290     RAMBlock *block = qemu_ram_block_by_name(block_name);
291     unsigned long *le_bitmap, nbits;
292     uint64_t size;
293 
294     if (!block) {
295         error_report("%s: invalid block name: %s", __func__, block_name);
296         return -1;
297     }
298 
299     nbits = block->postcopy_length >> TARGET_PAGE_BITS;
300 
301     /*
302      * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit
303      * machines we may need 4 more bytes for padding (see below
304      * comment). So extend it a bit before hand.
305      */
306     le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
307 
308     /*
309      * Always use little endian when sending the bitmap. This is
310      * required that when source and destination VMs are not using the
311      * same endianness. (Note: big endian won't work.)
312      */
313     bitmap_to_le(le_bitmap, block->receivedmap, nbits);
314 
315     /* Size of the bitmap, in bytes */
316     size = DIV_ROUND_UP(nbits, 8);
317 
318     /*
319      * size is always aligned to 8 bytes for 64bit machines, but it
320      * may not be true for 32bit machines. We need this padding to
321      * make sure the migration can survive even between 32bit and
322      * 64bit machines.
323      */
324     size = ROUND_UP(size, 8);
325 
326     qemu_put_be64(file, size);
327     qemu_put_buffer(file, (const uint8_t *)le_bitmap, size);
328     g_free(le_bitmap);
329     /*
330      * Mark as an end, in case the middle part is screwed up due to
331      * some "mysterious" reason.
332      */
333     qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING);
334     int ret = qemu_fflush(file);
335     if (ret) {
336         return ret;
337     }
338 
339     return size + sizeof(size);
340 }
341 
342 /*
343  * An outstanding page request, on the source, having been received
344  * and queued
345  */
346 struct RAMSrcPageRequest {
347     RAMBlock *rb;
348     hwaddr    offset;
349     hwaddr    len;
350 
351     QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req;
352 };
353 
354 /* State of RAM for migration */
355 struct RAMState {
356     /*
357      * PageSearchStatus structures for the channels when send pages.
358      * Protected by the bitmap_mutex.
359      */
360     PageSearchStatus pss[RAM_CHANNEL_MAX];
361     /* UFFD file descriptor, used in 'write-tracking' migration */
362     int uffdio_fd;
363     /* total ram size in bytes */
364     uint64_t ram_bytes_total;
365     /* Last block that we have visited searching for dirty pages */
366     RAMBlock *last_seen_block;
367     /* Last dirty target page we have sent */
368     ram_addr_t last_page;
369     /* last ram version we have seen */
370     uint32_t last_version;
371     /* How many times we have dirty too many pages */
372     int dirty_rate_high_cnt;
373     /* these variables are used for bitmap sync */
374     /* last time we did a full bitmap_sync */
375     int64_t time_last_bitmap_sync;
376     /* bytes transferred at start_time */
377     uint64_t bytes_xfer_prev;
378     /* number of dirty pages since start_time */
379     uint64_t num_dirty_pages_period;
380     /* xbzrle misses since the beginning of the period */
381     uint64_t xbzrle_cache_miss_prev;
382     /* Amount of xbzrle pages since the beginning of the period */
383     uint64_t xbzrle_pages_prev;
384     /* Amount of xbzrle encoded bytes since the beginning of the period */
385     uint64_t xbzrle_bytes_prev;
386     /* Are we really using XBZRLE (e.g., after the first round). */
387     bool xbzrle_started;
388     /* Are we on the last stage of migration */
389     bool last_stage;
390 
391     /* total handled target pages at the beginning of period */
392     uint64_t target_page_count_prev;
393     /* total handled target pages since start */
394     uint64_t target_page_count;
395     /* number of dirty bits in the bitmap */
396     uint64_t migration_dirty_pages;
397     /*
398      * Protects:
399      * - dirty/clear bitmap
400      * - migration_dirty_pages
401      * - pss structures
402      */
403     QemuMutex bitmap_mutex;
404     /* The RAMBlock used in the last src_page_requests */
405     RAMBlock *last_req_rb;
406     /* Queue of outstanding page requests from the destination */
407     QemuMutex src_page_req_mutex;
408     QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
409 
410     /*
411      * This is only used when postcopy is in recovery phase, to communicate
412      * between the migration thread and the return path thread on dirty
413      * bitmap synchronizations.  This field is unused in other stages of
414      * RAM migration.
415      */
416     unsigned int postcopy_bmap_sync_requested;
417 };
418 typedef struct RAMState RAMState;
419 
420 static RAMState *ram_state;
421 
422 static NotifierWithReturnList precopy_notifier_list;
423 
424 /* Whether postcopy has queued requests? */
425 static bool postcopy_has_request(RAMState *rs)
426 {
427     return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests);
428 }
429 
430 void precopy_infrastructure_init(void)
431 {
432     notifier_with_return_list_init(&precopy_notifier_list);
433 }
434 
435 void precopy_add_notifier(NotifierWithReturn *n)
436 {
437     notifier_with_return_list_add(&precopy_notifier_list, n);
438 }
439 
440 void precopy_remove_notifier(NotifierWithReturn *n)
441 {
442     notifier_with_return_remove(n);
443 }
444 
445 int precopy_notify(PrecopyNotifyReason reason, Error **errp)
446 {
447     PrecopyNotifyData pnd;
448     pnd.reason = reason;
449 
450     return notifier_with_return_list_notify(&precopy_notifier_list, &pnd, errp);
451 }
452 
453 uint64_t ram_bytes_remaining(void)
454 {
455     return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) :
456                        0;
457 }
458 
459 void ram_transferred_add(uint64_t bytes)
460 {
461     if (runstate_is_running()) {
462         stat64_add(&mig_stats.precopy_bytes, bytes);
463     } else if (migration_in_postcopy()) {
464         stat64_add(&mig_stats.postcopy_bytes, bytes);
465     } else {
466         stat64_add(&mig_stats.downtime_bytes, bytes);
467     }
468 }
469 
470 struct MigrationOps {
471     int (*ram_save_target_page)(RAMState *rs, PageSearchStatus *pss);
472 };
473 typedef struct MigrationOps MigrationOps;
474 
475 MigrationOps *migration_ops;
476 
477 static int ram_save_host_page_urgent(PageSearchStatus *pss);
478 
479 /* NOTE: page is the PFN not real ram_addr_t. */
480 static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page)
481 {
482     pss->block = rb;
483     pss->page = page;
484     pss->complete_round = false;
485 }
486 
487 /*
488  * Check whether two PSSs are actively sending the same page.  Return true
489  * if it is, false otherwise.
490  */
491 static bool pss_overlap(PageSearchStatus *pss1, PageSearchStatus *pss2)
492 {
493     return pss1->host_page_sending && pss2->host_page_sending &&
494         (pss1->host_page_start == pss2->host_page_start);
495 }
496 
497 /**
498  * save_page_header: write page header to wire
499  *
500  * If this is the 1st block, it also writes the block identification
501  *
502  * Returns the number of bytes written
503  *
504  * @pss: current PSS channel status
505  * @block: block that contains the page we want to send
506  * @offset: offset inside the block for the page
507  *          in the lower bits, it contains flags
508  */
509 static size_t save_page_header(PageSearchStatus *pss, QEMUFile *f,
510                                RAMBlock *block, ram_addr_t offset)
511 {
512     size_t size, len;
513     bool same_block = (block == pss->last_sent_block);
514 
515     if (same_block) {
516         offset |= RAM_SAVE_FLAG_CONTINUE;
517     }
518     qemu_put_be64(f, offset);
519     size = 8;
520 
521     if (!same_block) {
522         len = strlen(block->idstr);
523         qemu_put_byte(f, len);
524         qemu_put_buffer(f, (uint8_t *)block->idstr, len);
525         size += 1 + len;
526         pss->last_sent_block = block;
527     }
528     return size;
529 }
530 
531 /**
532  * mig_throttle_guest_down: throttle down the guest
533  *
534  * Reduce amount of guest cpu execution to hopefully slow down memory
535  * writes. If guest dirty memory rate is reduced below the rate at
536  * which we can transfer pages to the destination then we should be
537  * able to complete migration. Some workloads dirty memory way too
538  * fast and will not effectively converge, even with auto-converge.
539  */
540 static void mig_throttle_guest_down(uint64_t bytes_dirty_period,
541                                     uint64_t bytes_dirty_threshold)
542 {
543     uint64_t pct_initial = migrate_cpu_throttle_initial();
544     uint64_t pct_increment = migrate_cpu_throttle_increment();
545     bool pct_tailslow = migrate_cpu_throttle_tailslow();
546     int pct_max = migrate_max_cpu_throttle();
547 
548     uint64_t throttle_now = cpu_throttle_get_percentage();
549     uint64_t cpu_now, cpu_ideal, throttle_inc;
550 
551     /* We have not started throttling yet. Let's start it. */
552     if (!cpu_throttle_active()) {
553         cpu_throttle_set(pct_initial);
554     } else {
555         /* Throttling already on, just increase the rate */
556         if (!pct_tailslow) {
557             throttle_inc = pct_increment;
558         } else {
559             /* Compute the ideal CPU percentage used by Guest, which may
560              * make the dirty rate match the dirty rate threshold. */
561             cpu_now = 100 - throttle_now;
562             cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 /
563                         bytes_dirty_period);
564             throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment);
565         }
566         cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max));
567     }
568 }
569 
570 void mig_throttle_counter_reset(void)
571 {
572     RAMState *rs = ram_state;
573 
574     rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
575     rs->num_dirty_pages_period = 0;
576     rs->bytes_xfer_prev = migration_transferred_bytes();
577 }
578 
579 /**
580  * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
581  *
582  * @current_addr: address for the zero page
583  *
584  * Update the xbzrle cache to reflect a page that's been sent as all 0.
585  * The important thing is that a stale (not-yet-0'd) page be replaced
586  * by the new data.
587  * As a bonus, if the page wasn't in the cache it gets added so that
588  * when a small write is made into the 0'd page it gets XBZRLE sent.
589  */
590 static void xbzrle_cache_zero_page(ram_addr_t current_addr)
591 {
592     /* We don't care if this fails to allocate a new cache page
593      * as long as it updated an old one */
594     cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page,
595                  stat64_get(&mig_stats.dirty_sync_count));
596 }
597 
598 #define ENCODING_FLAG_XBZRLE 0x1
599 
600 /**
601  * save_xbzrle_page: compress and send current page
602  *
603  * Returns: 1 means that we wrote the page
604  *          0 means that page is identical to the one already sent
605  *          -1 means that xbzrle would be longer than normal
606  *
607  * @rs: current RAM state
608  * @pss: current PSS channel
609  * @current_data: pointer to the address of the page contents
610  * @current_addr: addr of the page
611  * @block: block that contains the page we want to send
612  * @offset: offset inside the block for the page
613  */
614 static int save_xbzrle_page(RAMState *rs, PageSearchStatus *pss,
615                             uint8_t **current_data, ram_addr_t current_addr,
616                             RAMBlock *block, ram_addr_t offset)
617 {
618     int encoded_len = 0, bytes_xbzrle;
619     uint8_t *prev_cached_page;
620     QEMUFile *file = pss->pss_channel;
621     uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
622 
623     if (!cache_is_cached(XBZRLE.cache, current_addr, generation)) {
624         xbzrle_counters.cache_miss++;
625         if (!rs->last_stage) {
626             if (cache_insert(XBZRLE.cache, current_addr, *current_data,
627                              generation) == -1) {
628                 return -1;
629             } else {
630                 /* update *current_data when the page has been
631                    inserted into cache */
632                 *current_data = get_cached_data(XBZRLE.cache, current_addr);
633             }
634         }
635         return -1;
636     }
637 
638     /*
639      * Reaching here means the page has hit the xbzrle cache, no matter what
640      * encoding result it is (normal encoding, overflow or skipping the page),
641      * count the page as encoded. This is used to calculate the encoding rate.
642      *
643      * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB,
644      * 2nd page turns out to be skipped (i.e. no new bytes written to the
645      * page), the overall encoding rate will be 8KB / 2KB = 4, which has the
646      * skipped page included. In this way, the encoding rate can tell if the
647      * guest page is good for xbzrle encoding.
648      */
649     xbzrle_counters.pages++;
650     prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
651 
652     /* save current buffer into memory */
653     memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
654 
655     /* XBZRLE encoding (if there is no overflow) */
656     encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
657                                        TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
658                                        TARGET_PAGE_SIZE);
659 
660     /*
661      * Update the cache contents, so that it corresponds to the data
662      * sent, in all cases except where we skip the page.
663      */
664     if (!rs->last_stage && encoded_len != 0) {
665         memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
666         /*
667          * In the case where we couldn't compress, ensure that the caller
668          * sends the data from the cache, since the guest might have
669          * changed the RAM since we copied it.
670          */
671         *current_data = prev_cached_page;
672     }
673 
674     if (encoded_len == 0) {
675         trace_save_xbzrle_page_skipping();
676         return 0;
677     } else if (encoded_len == -1) {
678         trace_save_xbzrle_page_overflow();
679         xbzrle_counters.overflow++;
680         xbzrle_counters.bytes += TARGET_PAGE_SIZE;
681         return -1;
682     }
683 
684     /* Send XBZRLE based compressed page */
685     bytes_xbzrle = save_page_header(pss, pss->pss_channel, block,
686                                     offset | RAM_SAVE_FLAG_XBZRLE);
687     qemu_put_byte(file, ENCODING_FLAG_XBZRLE);
688     qemu_put_be16(file, encoded_len);
689     qemu_put_buffer(file, XBZRLE.encoded_buf, encoded_len);
690     bytes_xbzrle += encoded_len + 1 + 2;
691     /*
692      * The xbzrle encoded bytes don't count the 8 byte header with
693      * RAM_SAVE_FLAG_CONTINUE.
694      */
695     xbzrle_counters.bytes += bytes_xbzrle - 8;
696     ram_transferred_add(bytes_xbzrle);
697 
698     return 1;
699 }
700 
701 /**
702  * pss_find_next_dirty: find the next dirty page of current ramblock
703  *
704  * This function updates pss->page to point to the next dirty page index
705  * within the ramblock to migrate, or the end of ramblock when nothing
706  * found.  Note that when pss->host_page_sending==true it means we're
707  * during sending a host page, so we won't look for dirty page that is
708  * outside the host page boundary.
709  *
710  * @pss: the current page search status
711  */
712 static void pss_find_next_dirty(PageSearchStatus *pss)
713 {
714     RAMBlock *rb = pss->block;
715     unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
716     unsigned long *bitmap = rb->bmap;
717 
718     if (migrate_ram_is_ignored(rb)) {
719         /* Points directly to the end, so we know no dirty page */
720         pss->page = size;
721         return;
722     }
723 
724     /*
725      * If during sending a host page, only look for dirty pages within the
726      * current host page being send.
727      */
728     if (pss->host_page_sending) {
729         assert(pss->host_page_end);
730         size = MIN(size, pss->host_page_end);
731     }
732 
733     pss->page = find_next_bit(bitmap, size, pss->page);
734 }
735 
736 static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb,
737                                                        unsigned long page)
738 {
739     uint8_t shift;
740     hwaddr size, start;
741 
742     if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) {
743         return;
744     }
745 
746     shift = rb->clear_bmap_shift;
747     /*
748      * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this
749      * can make things easier sometimes since then start address
750      * of the small chunk will always be 64 pages aligned so the
751      * bitmap will always be aligned to unsigned long. We should
752      * even be able to remove this restriction but I'm simply
753      * keeping it.
754      */
755     assert(shift >= 6);
756 
757     size = 1ULL << (TARGET_PAGE_BITS + shift);
758     start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size);
759     trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page);
760     memory_region_clear_dirty_bitmap(rb->mr, start, size);
761 }
762 
763 static void
764 migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb,
765                                                  unsigned long start,
766                                                  unsigned long npages)
767 {
768     unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift;
769     unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages);
770     unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages);
771 
772     /*
773      * Clear pages from start to start + npages - 1, so the end boundary is
774      * exclusive.
775      */
776     for (i = chunk_start; i < chunk_end; i += chunk_pages) {
777         migration_clear_memory_region_dirty_bitmap(rb, i);
778     }
779 }
780 
781 /*
782  * colo_bitmap_find_diry:find contiguous dirty pages from start
783  *
784  * Returns the page offset within memory region of the start of the contiguout
785  * dirty page
786  *
787  * @rs: current RAM state
788  * @rb: RAMBlock where to search for dirty pages
789  * @start: page where we start the search
790  * @num: the number of contiguous dirty pages
791  */
792 static inline
793 unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb,
794                                      unsigned long start, unsigned long *num)
795 {
796     unsigned long size = rb->used_length >> TARGET_PAGE_BITS;
797     unsigned long *bitmap = rb->bmap;
798     unsigned long first, next;
799 
800     *num = 0;
801 
802     if (migrate_ram_is_ignored(rb)) {
803         return size;
804     }
805 
806     first = find_next_bit(bitmap, size, start);
807     if (first >= size) {
808         return first;
809     }
810     next = find_next_zero_bit(bitmap, size, first + 1);
811     assert(next >= first);
812     *num = next - first;
813     return first;
814 }
815 
816 static inline bool migration_bitmap_clear_dirty(RAMState *rs,
817                                                 RAMBlock *rb,
818                                                 unsigned long page)
819 {
820     bool ret;
821 
822     /*
823      * Clear dirty bitmap if needed.  This _must_ be called before we
824      * send any of the page in the chunk because we need to make sure
825      * we can capture further page content changes when we sync dirty
826      * log the next time.  So as long as we are going to send any of
827      * the page in the chunk we clear the remote dirty bitmap for all.
828      * Clearing it earlier won't be a problem, but too late will.
829      */
830     migration_clear_memory_region_dirty_bitmap(rb, page);
831 
832     ret = test_and_clear_bit(page, rb->bmap);
833     if (ret) {
834         rs->migration_dirty_pages--;
835     }
836 
837     return ret;
838 }
839 
840 static void dirty_bitmap_clear_section(MemoryRegionSection *section,
841                                        void *opaque)
842 {
843     const hwaddr offset = section->offset_within_region;
844     const hwaddr size = int128_get64(section->size);
845     const unsigned long start = offset >> TARGET_PAGE_BITS;
846     const unsigned long npages = size >> TARGET_PAGE_BITS;
847     RAMBlock *rb = section->mr->ram_block;
848     uint64_t *cleared_bits = opaque;
849 
850     /*
851      * We don't grab ram_state->bitmap_mutex because we expect to run
852      * only when starting migration or during postcopy recovery where
853      * we don't have concurrent access.
854      */
855     if (!migration_in_postcopy() && !migrate_background_snapshot()) {
856         migration_clear_memory_region_dirty_bitmap_range(rb, start, npages);
857     }
858     *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages);
859     bitmap_clear(rb->bmap, start, npages);
860 }
861 
862 /*
863  * Exclude all dirty pages from migration that fall into a discarded range as
864  * managed by a RamDiscardManager responsible for the mapped memory region of
865  * the RAMBlock. Clear the corresponding bits in the dirty bitmaps.
866  *
867  * Discarded pages ("logically unplugged") have undefined content and must
868  * not get migrated, because even reading these pages for migration might
869  * result in undesired behavior.
870  *
871  * Returns the number of cleared bits in the RAMBlock dirty bitmap.
872  *
873  * Note: The result is only stable while migrating (precopy/postcopy).
874  */
875 static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb)
876 {
877     uint64_t cleared_bits = 0;
878 
879     if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) {
880         RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
881         MemoryRegionSection section = {
882             .mr = rb->mr,
883             .offset_within_region = 0,
884             .size = int128_make64(qemu_ram_get_used_length(rb)),
885         };
886 
887         ram_discard_manager_replay_discarded(rdm, &section,
888                                              dirty_bitmap_clear_section,
889                                              &cleared_bits);
890     }
891     return cleared_bits;
892 }
893 
894 /*
895  * Check if a host-page aligned page falls into a discarded range as managed by
896  * a RamDiscardManager responsible for the mapped memory region of the RAMBlock.
897  *
898  * Note: The result is only stable while migrating (precopy/postcopy).
899  */
900 bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start)
901 {
902     if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
903         RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
904         MemoryRegionSection section = {
905             .mr = rb->mr,
906             .offset_within_region = start,
907             .size = int128_make64(qemu_ram_pagesize(rb)),
908         };
909 
910         return !ram_discard_manager_is_populated(rdm, &section);
911     }
912     return false;
913 }
914 
915 /* Called with RCU critical section */
916 static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb)
917 {
918     uint64_t new_dirty_pages =
919         cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length);
920 
921     rs->migration_dirty_pages += new_dirty_pages;
922     rs->num_dirty_pages_period += new_dirty_pages;
923 }
924 
925 /**
926  * ram_pagesize_summary: calculate all the pagesizes of a VM
927  *
928  * Returns a summary bitmap of the page sizes of all RAMBlocks
929  *
930  * For VMs with just normal pages this is equivalent to the host page
931  * size. If it's got some huge pages then it's the OR of all the
932  * different page sizes.
933  */
934 uint64_t ram_pagesize_summary(void)
935 {
936     RAMBlock *block;
937     uint64_t summary = 0;
938 
939     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
940         summary |= block->page_size;
941     }
942 
943     return summary;
944 }
945 
946 uint64_t ram_get_total_transferred_pages(void)
947 {
948     return stat64_get(&mig_stats.normal_pages) +
949         stat64_get(&mig_stats.zero_pages) +
950         xbzrle_counters.pages;
951 }
952 
953 static void migration_update_rates(RAMState *rs, int64_t end_time)
954 {
955     uint64_t page_count = rs->target_page_count - rs->target_page_count_prev;
956 
957     /* calculate period counters */
958     stat64_set(&mig_stats.dirty_pages_rate,
959                rs->num_dirty_pages_period * 1000 /
960                (end_time - rs->time_last_bitmap_sync));
961 
962     if (!page_count) {
963         return;
964     }
965 
966     if (migrate_xbzrle()) {
967         double encoded_size, unencoded_size;
968 
969         xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss -
970             rs->xbzrle_cache_miss_prev) / page_count;
971         rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss;
972         unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) *
973                          TARGET_PAGE_SIZE;
974         encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev;
975         if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) {
976             xbzrle_counters.encoding_rate = 0;
977         } else {
978             xbzrle_counters.encoding_rate = unencoded_size / encoded_size;
979         }
980         rs->xbzrle_pages_prev = xbzrle_counters.pages;
981         rs->xbzrle_bytes_prev = xbzrle_counters.bytes;
982     }
983 }
984 
985 /*
986  * Enable dirty-limit to throttle down the guest
987  */
988 static void migration_dirty_limit_guest(void)
989 {
990     /*
991      * dirty page rate quota for all vCPUs fetched from
992      * migration parameter 'vcpu_dirty_limit'
993      */
994     static int64_t quota_dirtyrate;
995     MigrationState *s = migrate_get_current();
996 
997     /*
998      * If dirty limit already enabled and migration parameter
999      * vcpu-dirty-limit untouched.
1000      */
1001     if (dirtylimit_in_service() &&
1002         quota_dirtyrate == s->parameters.vcpu_dirty_limit) {
1003         return;
1004     }
1005 
1006     quota_dirtyrate = s->parameters.vcpu_dirty_limit;
1007 
1008     /*
1009      * Set all vCPU a quota dirtyrate, note that the second
1010      * parameter will be ignored if setting all vCPU for the vm
1011      */
1012     qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL);
1013     trace_migration_dirty_limit_guest(quota_dirtyrate);
1014 }
1015 
1016 static void migration_trigger_throttle(RAMState *rs)
1017 {
1018     uint64_t threshold = migrate_throttle_trigger_threshold();
1019     uint64_t bytes_xfer_period =
1020         migration_transferred_bytes() - rs->bytes_xfer_prev;
1021     uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE;
1022     uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100;
1023 
1024     /*
1025      * The following detection logic can be refined later. For now:
1026      * Check to see if the ratio between dirtied bytes and the approx.
1027      * amount of bytes that just got transferred since the last time
1028      * we were in this routine reaches the threshold. If that happens
1029      * twice, start or increase throttling.
1030      */
1031     if ((bytes_dirty_period > bytes_dirty_threshold) &&
1032         (++rs->dirty_rate_high_cnt >= 2)) {
1033         rs->dirty_rate_high_cnt = 0;
1034         if (migrate_auto_converge()) {
1035             trace_migration_throttle();
1036             mig_throttle_guest_down(bytes_dirty_period,
1037                                     bytes_dirty_threshold);
1038         } else if (migrate_dirty_limit()) {
1039             migration_dirty_limit_guest();
1040         }
1041     }
1042 }
1043 
1044 static void migration_bitmap_sync(RAMState *rs, bool last_stage)
1045 {
1046     RAMBlock *block;
1047     int64_t end_time;
1048 
1049     stat64_add(&mig_stats.dirty_sync_count, 1);
1050 
1051     if (!rs->time_last_bitmap_sync) {
1052         rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1053     }
1054 
1055     trace_migration_bitmap_sync_start();
1056     memory_global_dirty_log_sync(last_stage);
1057 
1058     WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
1059         WITH_RCU_READ_LOCK_GUARD() {
1060             RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1061                 ramblock_sync_dirty_bitmap(rs, block);
1062             }
1063             stat64_set(&mig_stats.dirty_bytes_last_sync, ram_bytes_remaining());
1064         }
1065     }
1066 
1067     memory_global_after_dirty_log_sync();
1068     trace_migration_bitmap_sync_end(rs->num_dirty_pages_period);
1069 
1070     end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1071 
1072     /* more than 1 second = 1000 millisecons */
1073     if (end_time > rs->time_last_bitmap_sync + 1000) {
1074         migration_trigger_throttle(rs);
1075 
1076         migration_update_rates(rs, end_time);
1077 
1078         rs->target_page_count_prev = rs->target_page_count;
1079 
1080         /* reset period counters */
1081         rs->time_last_bitmap_sync = end_time;
1082         rs->num_dirty_pages_period = 0;
1083         rs->bytes_xfer_prev = migration_transferred_bytes();
1084     }
1085     if (migrate_events()) {
1086         uint64_t generation = stat64_get(&mig_stats.dirty_sync_count);
1087         qapi_event_send_migration_pass(generation);
1088     }
1089 }
1090 
1091 void migration_bitmap_sync_precopy(bool last_stage)
1092 {
1093     Error *local_err = NULL;
1094     assert(ram_state);
1095 
1096     /*
1097      * The current notifier usage is just an optimization to migration, so we
1098      * don't stop the normal migration process in the error case.
1099      */
1100     if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) {
1101         error_report_err(local_err);
1102         local_err = NULL;
1103     }
1104 
1105     migration_bitmap_sync(ram_state, last_stage);
1106 
1107     if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) {
1108         error_report_err(local_err);
1109     }
1110 }
1111 
1112 void ram_release_page(const char *rbname, uint64_t offset)
1113 {
1114     if (!migrate_release_ram() || !migration_in_postcopy()) {
1115         return;
1116     }
1117 
1118     ram_discard_range(rbname, offset, TARGET_PAGE_SIZE);
1119 }
1120 
1121 /**
1122  * save_zero_page: send the zero page to the stream
1123  *
1124  * Returns the number of pages written.
1125  *
1126  * @rs: current RAM state
1127  * @pss: current PSS channel
1128  * @offset: offset inside the block for the page
1129  */
1130 static int save_zero_page(RAMState *rs, PageSearchStatus *pss,
1131                           ram_addr_t offset)
1132 {
1133     uint8_t *p = pss->block->host + offset;
1134     QEMUFile *file = pss->pss_channel;
1135     int len = 0;
1136 
1137     if (migrate_zero_page_detection() == ZERO_PAGE_DETECTION_NONE) {
1138         return 0;
1139     }
1140 
1141     if (!buffer_is_zero(p, TARGET_PAGE_SIZE)) {
1142         return 0;
1143     }
1144 
1145     stat64_add(&mig_stats.zero_pages, 1);
1146 
1147     if (migrate_mapped_ram()) {
1148         /* zero pages are not transferred with mapped-ram */
1149         clear_bit_atomic(offset >> TARGET_PAGE_BITS, pss->block->file_bmap);
1150         return 1;
1151     }
1152 
1153     len += save_page_header(pss, file, pss->block, offset | RAM_SAVE_FLAG_ZERO);
1154     qemu_put_byte(file, 0);
1155     len += 1;
1156     ram_release_page(pss->block->idstr, offset);
1157     ram_transferred_add(len);
1158 
1159     /*
1160      * Must let xbzrle know, otherwise a previous (now 0'd) cached
1161      * page would be stale.
1162      */
1163     if (rs->xbzrle_started) {
1164         XBZRLE_cache_lock();
1165         xbzrle_cache_zero_page(pss->block->offset + offset);
1166         XBZRLE_cache_unlock();
1167     }
1168 
1169     return len;
1170 }
1171 
1172 /*
1173  * @pages: the number of pages written by the control path,
1174  *        < 0 - error
1175  *        > 0 - number of pages written
1176  *
1177  * Return true if the pages has been saved, otherwise false is returned.
1178  */
1179 static bool control_save_page(PageSearchStatus *pss,
1180                               ram_addr_t offset, int *pages)
1181 {
1182     int ret;
1183 
1184     ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset,
1185                                  TARGET_PAGE_SIZE);
1186     if (ret == RAM_SAVE_CONTROL_NOT_SUPP) {
1187         return false;
1188     }
1189 
1190     if (ret == RAM_SAVE_CONTROL_DELAYED) {
1191         *pages = 1;
1192         return true;
1193     }
1194     *pages = ret;
1195     return true;
1196 }
1197 
1198 /*
1199  * directly send the page to the stream
1200  *
1201  * Returns the number of pages written.
1202  *
1203  * @pss: current PSS channel
1204  * @block: block that contains the page we want to send
1205  * @offset: offset inside the block for the page
1206  * @buf: the page to be sent
1207  * @async: send to page asyncly
1208  */
1209 static int save_normal_page(PageSearchStatus *pss, RAMBlock *block,
1210                             ram_addr_t offset, uint8_t *buf, bool async)
1211 {
1212     QEMUFile *file = pss->pss_channel;
1213 
1214     if (migrate_mapped_ram()) {
1215         qemu_put_buffer_at(file, buf, TARGET_PAGE_SIZE,
1216                            block->pages_offset + offset);
1217         set_bit(offset >> TARGET_PAGE_BITS, block->file_bmap);
1218     } else {
1219         ram_transferred_add(save_page_header(pss, pss->pss_channel, block,
1220                                              offset | RAM_SAVE_FLAG_PAGE));
1221         if (async) {
1222             qemu_put_buffer_async(file, buf, TARGET_PAGE_SIZE,
1223                                   migrate_release_ram() &&
1224                                   migration_in_postcopy());
1225         } else {
1226             qemu_put_buffer(file, buf, TARGET_PAGE_SIZE);
1227         }
1228     }
1229     ram_transferred_add(TARGET_PAGE_SIZE);
1230     stat64_add(&mig_stats.normal_pages, 1);
1231     return 1;
1232 }
1233 
1234 /**
1235  * ram_save_page: send the given page to the stream
1236  *
1237  * Returns the number of pages written.
1238  *          < 0 - error
1239  *          >=0 - Number of pages written - this might legally be 0
1240  *                if xbzrle noticed the page was the same.
1241  *
1242  * @rs: current RAM state
1243  * @block: block that contains the page we want to send
1244  * @offset: offset inside the block for the page
1245  */
1246 static int ram_save_page(RAMState *rs, PageSearchStatus *pss)
1247 {
1248     int pages = -1;
1249     uint8_t *p;
1250     bool send_async = true;
1251     RAMBlock *block = pss->block;
1252     ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1253     ram_addr_t current_addr = block->offset + offset;
1254 
1255     p = block->host + offset;
1256     trace_ram_save_page(block->idstr, (uint64_t)offset, p);
1257 
1258     XBZRLE_cache_lock();
1259     if (rs->xbzrle_started && !migration_in_postcopy()) {
1260         pages = save_xbzrle_page(rs, pss, &p, current_addr,
1261                                  block, offset);
1262         if (!rs->last_stage) {
1263             /* Can't send this cached data async, since the cache page
1264              * might get updated before it gets to the wire
1265              */
1266             send_async = false;
1267         }
1268     }
1269 
1270     /* XBZRLE overflow or normal page */
1271     if (pages == -1) {
1272         pages = save_normal_page(pss, block, offset, p, send_async);
1273     }
1274 
1275     XBZRLE_cache_unlock();
1276 
1277     return pages;
1278 }
1279 
1280 static int ram_save_multifd_page(RAMBlock *block, ram_addr_t offset)
1281 {
1282     if (!multifd_queue_page(block, offset)) {
1283         return -1;
1284     }
1285 
1286     return 1;
1287 }
1288 
1289 
1290 #define PAGE_ALL_CLEAN 0
1291 #define PAGE_TRY_AGAIN 1
1292 #define PAGE_DIRTY_FOUND 2
1293 /**
1294  * find_dirty_block: find the next dirty page and update any state
1295  * associated with the search process.
1296  *
1297  * Returns:
1298  *         <0: An error happened
1299  *         PAGE_ALL_CLEAN: no dirty page found, give up
1300  *         PAGE_TRY_AGAIN: no dirty page found, retry for next block
1301  *         PAGE_DIRTY_FOUND: dirty page found
1302  *
1303  * @rs: current RAM state
1304  * @pss: data about the state of the current dirty page scan
1305  * @again: set to false if the search has scanned the whole of RAM
1306  */
1307 static int find_dirty_block(RAMState *rs, PageSearchStatus *pss)
1308 {
1309     /* Update pss->page for the next dirty bit in ramblock */
1310     pss_find_next_dirty(pss);
1311 
1312     if (pss->complete_round && pss->block == rs->last_seen_block &&
1313         pss->page >= rs->last_page) {
1314         /*
1315          * We've been once around the RAM and haven't found anything.
1316          * Give up.
1317          */
1318         return PAGE_ALL_CLEAN;
1319     }
1320     if (!offset_in_ramblock(pss->block,
1321                             ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) {
1322         /* Didn't find anything in this RAM Block */
1323         pss->page = 0;
1324         pss->block = QLIST_NEXT_RCU(pss->block, next);
1325         if (!pss->block) {
1326             if (migrate_multifd() &&
1327                 (!migrate_multifd_flush_after_each_section() ||
1328                  migrate_mapped_ram())) {
1329                 QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel;
1330                 int ret = multifd_ram_flush_and_sync();
1331                 if (ret < 0) {
1332                     return ret;
1333                 }
1334 
1335                 if (!migrate_mapped_ram()) {
1336                     qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
1337                     qemu_fflush(f);
1338                 }
1339             }
1340 
1341             /* Hit the end of the list */
1342             pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
1343             /* Flag that we've looped */
1344             pss->complete_round = true;
1345             /* After the first round, enable XBZRLE. */
1346             if (migrate_xbzrle()) {
1347                 rs->xbzrle_started = true;
1348             }
1349         }
1350         /* Didn't find anything this time, but try again on the new block */
1351         return PAGE_TRY_AGAIN;
1352     } else {
1353         /* We've found something */
1354         return PAGE_DIRTY_FOUND;
1355     }
1356 }
1357 
1358 /**
1359  * unqueue_page: gets a page of the queue
1360  *
1361  * Helper for 'get_queued_page' - gets a page off the queue
1362  *
1363  * Returns the block of the page (or NULL if none available)
1364  *
1365  * @rs: current RAM state
1366  * @offset: used to return the offset within the RAMBlock
1367  */
1368 static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset)
1369 {
1370     struct RAMSrcPageRequest *entry;
1371     RAMBlock *block = NULL;
1372 
1373     if (!postcopy_has_request(rs)) {
1374         return NULL;
1375     }
1376 
1377     QEMU_LOCK_GUARD(&rs->src_page_req_mutex);
1378 
1379     /*
1380      * This should _never_ change even after we take the lock, because no one
1381      * should be taking anything off the request list other than us.
1382      */
1383     assert(postcopy_has_request(rs));
1384 
1385     entry = QSIMPLEQ_FIRST(&rs->src_page_requests);
1386     block = entry->rb;
1387     *offset = entry->offset;
1388 
1389     if (entry->len > TARGET_PAGE_SIZE) {
1390         entry->len -= TARGET_PAGE_SIZE;
1391         entry->offset += TARGET_PAGE_SIZE;
1392     } else {
1393         memory_region_unref(block->mr);
1394         QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1395         g_free(entry);
1396         migration_consume_urgent_request();
1397     }
1398 
1399     return block;
1400 }
1401 
1402 #if defined(__linux__)
1403 /**
1404  * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1405  *   is found, return RAM block pointer and page offset
1406  *
1407  * Returns pointer to the RAMBlock containing faulting page,
1408  *   NULL if no write faults are pending
1409  *
1410  * @rs: current RAM state
1411  * @offset: page offset from the beginning of the block
1412  */
1413 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1414 {
1415     struct uffd_msg uffd_msg;
1416     void *page_address;
1417     RAMBlock *block;
1418     int res;
1419 
1420     if (!migrate_background_snapshot()) {
1421         return NULL;
1422     }
1423 
1424     res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1);
1425     if (res <= 0) {
1426         return NULL;
1427     }
1428 
1429     page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address;
1430     block = qemu_ram_block_from_host(page_address, false, offset);
1431     assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0);
1432     return block;
1433 }
1434 
1435 /**
1436  * ram_save_release_protection: release UFFD write protection after
1437  *   a range of pages has been saved
1438  *
1439  * @rs: current RAM state
1440  * @pss: page-search-status structure
1441  * @start_page: index of the first page in the range relative to pss->block
1442  *
1443  * Returns 0 on success, negative value in case of an error
1444 */
1445 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1446         unsigned long start_page)
1447 {
1448     int res = 0;
1449 
1450     /* Check if page is from UFFD-managed region. */
1451     if (pss->block->flags & RAM_UF_WRITEPROTECT) {
1452         void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS);
1453         uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS;
1454 
1455         /* Flush async buffers before un-protect. */
1456         qemu_fflush(pss->pss_channel);
1457         /* Un-protect memory range. */
1458         res = uffd_change_protection(rs->uffdio_fd, page_address, run_length,
1459                 false, false);
1460     }
1461 
1462     return res;
1463 }
1464 
1465 /* ram_write_tracking_available: check if kernel supports required UFFD features
1466  *
1467  * Returns true if supports, false otherwise
1468  */
1469 bool ram_write_tracking_available(void)
1470 {
1471     uint64_t uffd_features;
1472     int res;
1473 
1474     res = uffd_query_features(&uffd_features);
1475     return (res == 0 &&
1476             (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0);
1477 }
1478 
1479 /* ram_write_tracking_compatible: check if guest configuration is
1480  *   compatible with 'write-tracking'
1481  *
1482  * Returns true if compatible, false otherwise
1483  */
1484 bool ram_write_tracking_compatible(void)
1485 {
1486     const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT);
1487     int uffd_fd;
1488     RAMBlock *block;
1489     bool ret = false;
1490 
1491     /* Open UFFD file descriptor */
1492     uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false);
1493     if (uffd_fd < 0) {
1494         return false;
1495     }
1496 
1497     RCU_READ_LOCK_GUARD();
1498 
1499     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1500         uint64_t uffd_ioctls;
1501 
1502         /* Nothing to do with read-only and MMIO-writable regions */
1503         if (block->mr->readonly || block->mr->rom_device) {
1504             continue;
1505         }
1506         /* Try to register block memory via UFFD-IO to track writes */
1507         if (uffd_register_memory(uffd_fd, block->host, block->max_length,
1508                 UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) {
1509             goto out;
1510         }
1511         if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) {
1512             goto out;
1513         }
1514     }
1515     ret = true;
1516 
1517 out:
1518     uffd_close_fd(uffd_fd);
1519     return ret;
1520 }
1521 
1522 static inline void populate_read_range(RAMBlock *block, ram_addr_t offset,
1523                                        ram_addr_t size)
1524 {
1525     const ram_addr_t end = offset + size;
1526 
1527     /*
1528      * We read one byte of each page; this will preallocate page tables if
1529      * required and populate the shared zeropage on MAP_PRIVATE anonymous memory
1530      * where no page was populated yet. This might require adaption when
1531      * supporting other mappings, like shmem.
1532      */
1533     for (; offset < end; offset += block->page_size) {
1534         char tmp = *((char *)block->host + offset);
1535 
1536         /* Don't optimize the read out */
1537         asm volatile("" : "+r" (tmp));
1538     }
1539 }
1540 
1541 static inline int populate_read_section(MemoryRegionSection *section,
1542                                         void *opaque)
1543 {
1544     const hwaddr size = int128_get64(section->size);
1545     hwaddr offset = section->offset_within_region;
1546     RAMBlock *block = section->mr->ram_block;
1547 
1548     populate_read_range(block, offset, size);
1549     return 0;
1550 }
1551 
1552 /*
1553  * ram_block_populate_read: preallocate page tables and populate pages in the
1554  *   RAM block by reading a byte of each page.
1555  *
1556  * Since it's solely used for userfault_fd WP feature, here we just
1557  *   hardcode page size to qemu_real_host_page_size.
1558  *
1559  * @block: RAM block to populate
1560  */
1561 static void ram_block_populate_read(RAMBlock *rb)
1562 {
1563     /*
1564      * Skip populating all pages that fall into a discarded range as managed by
1565      * a RamDiscardManager responsible for the mapped memory region of the
1566      * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock
1567      * must not get populated automatically. We don't have to track
1568      * modifications via userfaultfd WP reliably, because these pages will
1569      * not be part of the migration stream either way -- see
1570      * ramblock_dirty_bitmap_exclude_discarded_pages().
1571      *
1572      * Note: The result is only stable while migrating (precopy/postcopy).
1573      */
1574     if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1575         RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1576         MemoryRegionSection section = {
1577             .mr = rb->mr,
1578             .offset_within_region = 0,
1579             .size = rb->mr->size,
1580         };
1581 
1582         ram_discard_manager_replay_populated(rdm, &section,
1583                                              populate_read_section, NULL);
1584     } else {
1585         populate_read_range(rb, 0, rb->used_length);
1586     }
1587 }
1588 
1589 /*
1590  * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1591  */
1592 void ram_write_tracking_prepare(void)
1593 {
1594     RAMBlock *block;
1595 
1596     RCU_READ_LOCK_GUARD();
1597 
1598     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1599         /* Nothing to do with read-only and MMIO-writable regions */
1600         if (block->mr->readonly || block->mr->rom_device) {
1601             continue;
1602         }
1603 
1604         /*
1605          * Populate pages of the RAM block before enabling userfault_fd
1606          * write protection.
1607          *
1608          * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with
1609          * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip
1610          * pages with pte_none() entries in page table.
1611          */
1612         ram_block_populate_read(block);
1613     }
1614 }
1615 
1616 static inline int uffd_protect_section(MemoryRegionSection *section,
1617                                        void *opaque)
1618 {
1619     const hwaddr size = int128_get64(section->size);
1620     const hwaddr offset = section->offset_within_region;
1621     RAMBlock *rb = section->mr->ram_block;
1622     int uffd_fd = (uintptr_t)opaque;
1623 
1624     return uffd_change_protection(uffd_fd, rb->host + offset, size, true,
1625                                   false);
1626 }
1627 
1628 static int ram_block_uffd_protect(RAMBlock *rb, int uffd_fd)
1629 {
1630     assert(rb->flags & RAM_UF_WRITEPROTECT);
1631 
1632     /* See ram_block_populate_read() */
1633     if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) {
1634         RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr);
1635         MemoryRegionSection section = {
1636             .mr = rb->mr,
1637             .offset_within_region = 0,
1638             .size = rb->mr->size,
1639         };
1640 
1641         return ram_discard_manager_replay_populated(rdm, &section,
1642                                                     uffd_protect_section,
1643                                                     (void *)(uintptr_t)uffd_fd);
1644     }
1645     return uffd_change_protection(uffd_fd, rb->host,
1646                                   rb->used_length, true, false);
1647 }
1648 
1649 /*
1650  * ram_write_tracking_start: start UFFD-WP memory tracking
1651  *
1652  * Returns 0 for success or negative value in case of error
1653  */
1654 int ram_write_tracking_start(void)
1655 {
1656     int uffd_fd;
1657     RAMState *rs = ram_state;
1658     RAMBlock *block;
1659 
1660     /* Open UFFD file descriptor */
1661     uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true);
1662     if (uffd_fd < 0) {
1663         return uffd_fd;
1664     }
1665     rs->uffdio_fd = uffd_fd;
1666 
1667     RCU_READ_LOCK_GUARD();
1668 
1669     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1670         /* Nothing to do with read-only and MMIO-writable regions */
1671         if (block->mr->readonly || block->mr->rom_device) {
1672             continue;
1673         }
1674 
1675         /* Register block memory with UFFD to track writes */
1676         if (uffd_register_memory(rs->uffdio_fd, block->host,
1677                 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) {
1678             goto fail;
1679         }
1680         block->flags |= RAM_UF_WRITEPROTECT;
1681         memory_region_ref(block->mr);
1682 
1683         /* Apply UFFD write protection to the block memory range */
1684         if (ram_block_uffd_protect(block, uffd_fd)) {
1685             goto fail;
1686         }
1687 
1688         trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size,
1689                 block->host, block->max_length);
1690     }
1691 
1692     return 0;
1693 
1694 fail:
1695     error_report("ram_write_tracking_start() failed: restoring initial memory state");
1696 
1697     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1698         if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1699             continue;
1700         }
1701         uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1702         /* Cleanup flags and remove reference */
1703         block->flags &= ~RAM_UF_WRITEPROTECT;
1704         memory_region_unref(block->mr);
1705     }
1706 
1707     uffd_close_fd(uffd_fd);
1708     rs->uffdio_fd = -1;
1709     return -1;
1710 }
1711 
1712 /**
1713  * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1714  */
1715 void ram_write_tracking_stop(void)
1716 {
1717     RAMState *rs = ram_state;
1718     RAMBlock *block;
1719 
1720     RCU_READ_LOCK_GUARD();
1721 
1722     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
1723         if ((block->flags & RAM_UF_WRITEPROTECT) == 0) {
1724             continue;
1725         }
1726         uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length);
1727 
1728         trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size,
1729                 block->host, block->max_length);
1730 
1731         /* Cleanup flags and remove reference */
1732         block->flags &= ~RAM_UF_WRITEPROTECT;
1733         memory_region_unref(block->mr);
1734     }
1735 
1736     /* Finally close UFFD file descriptor */
1737     uffd_close_fd(rs->uffdio_fd);
1738     rs->uffdio_fd = -1;
1739 }
1740 
1741 #else
1742 /* No target OS support, stubs just fail or ignore */
1743 
1744 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset)
1745 {
1746     (void) rs;
1747     (void) offset;
1748 
1749     return NULL;
1750 }
1751 
1752 static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss,
1753         unsigned long start_page)
1754 {
1755     (void) rs;
1756     (void) pss;
1757     (void) start_page;
1758 
1759     return 0;
1760 }
1761 
1762 bool ram_write_tracking_available(void)
1763 {
1764     return false;
1765 }
1766 
1767 bool ram_write_tracking_compatible(void)
1768 {
1769     g_assert_not_reached();
1770 }
1771 
1772 int ram_write_tracking_start(void)
1773 {
1774     g_assert_not_reached();
1775 }
1776 
1777 void ram_write_tracking_stop(void)
1778 {
1779     g_assert_not_reached();
1780 }
1781 #endif /* defined(__linux__) */
1782 
1783 /**
1784  * get_queued_page: unqueue a page from the postcopy requests
1785  *
1786  * Skips pages that are already sent (!dirty)
1787  *
1788  * Returns true if a queued page is found
1789  *
1790  * @rs: current RAM state
1791  * @pss: data about the state of the current dirty page scan
1792  */
1793 static bool get_queued_page(RAMState *rs, PageSearchStatus *pss)
1794 {
1795     RAMBlock  *block;
1796     ram_addr_t offset;
1797     bool dirty = false;
1798 
1799     do {
1800         block = unqueue_page(rs, &offset);
1801         /*
1802          * We're sending this page, and since it's postcopy nothing else
1803          * will dirty it, and we must make sure it doesn't get sent again
1804          * even if this queue request was received after the background
1805          * search already sent it.
1806          */
1807         if (block) {
1808             unsigned long page;
1809 
1810             page = offset >> TARGET_PAGE_BITS;
1811             dirty = test_bit(page, block->bmap);
1812             if (!dirty) {
1813                 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset,
1814                                                 page);
1815             } else {
1816                 trace_get_queued_page(block->idstr, (uint64_t)offset, page);
1817             }
1818         }
1819 
1820     } while (block && !dirty);
1821 
1822     if (!block) {
1823         /*
1824          * Poll write faults too if background snapshot is enabled; that's
1825          * when we have vcpus got blocked by the write protected pages.
1826          */
1827         block = poll_fault_page(rs, &offset);
1828     }
1829 
1830     if (block) {
1831         /*
1832          * We want the background search to continue from the queued page
1833          * since the guest is likely to want other pages near to the page
1834          * it just requested.
1835          */
1836         pss->block = block;
1837         pss->page = offset >> TARGET_PAGE_BITS;
1838 
1839         /*
1840          * This unqueued page would break the "one round" check, even is
1841          * really rare.
1842          */
1843         pss->complete_round = false;
1844     }
1845 
1846     return !!block;
1847 }
1848 
1849 /**
1850  * migration_page_queue_free: drop any remaining pages in the ram
1851  * request queue
1852  *
1853  * It should be empty at the end anyway, but in error cases there may
1854  * be some left.  in case that there is any page left, we drop it.
1855  *
1856  */
1857 static void migration_page_queue_free(RAMState *rs)
1858 {
1859     struct RAMSrcPageRequest *mspr, *next_mspr;
1860     /* This queue generally should be empty - but in the case of a failed
1861      * migration might have some droppings in.
1862      */
1863     RCU_READ_LOCK_GUARD();
1864     QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) {
1865         memory_region_unref(mspr->rb->mr);
1866         QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req);
1867         g_free(mspr);
1868     }
1869 }
1870 
1871 /**
1872  * ram_save_queue_pages: queue the page for transmission
1873  *
1874  * A request from postcopy destination for example.
1875  *
1876  * Returns zero on success or negative on error
1877  *
1878  * @rbname: Name of the RAMBLock of the request. NULL means the
1879  *          same that last one.
1880  * @start: starting address from the start of the RAMBlock
1881  * @len: length (in bytes) to send
1882  */
1883 int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len,
1884                          Error **errp)
1885 {
1886     RAMBlock *ramblock;
1887     RAMState *rs = ram_state;
1888 
1889     stat64_add(&mig_stats.postcopy_requests, 1);
1890     RCU_READ_LOCK_GUARD();
1891 
1892     if (!rbname) {
1893         /* Reuse last RAMBlock */
1894         ramblock = rs->last_req_rb;
1895 
1896         if (!ramblock) {
1897             /*
1898              * Shouldn't happen, we can't reuse the last RAMBlock if
1899              * it's the 1st request.
1900              */
1901             error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no previous block");
1902             return -1;
1903         }
1904     } else {
1905         ramblock = qemu_ram_block_by_name(rbname);
1906 
1907         if (!ramblock) {
1908             /* We shouldn't be asked for a non-existent RAMBlock */
1909             error_setg(errp, "MIG_RP_MSG_REQ_PAGES has no block '%s'", rbname);
1910             return -1;
1911         }
1912         rs->last_req_rb = ramblock;
1913     }
1914     trace_ram_save_queue_pages(ramblock->idstr, start, len);
1915     if (!offset_in_ramblock(ramblock, start + len - 1)) {
1916         error_setg(errp, "MIG_RP_MSG_REQ_PAGES request overrun, "
1917                    "start=" RAM_ADDR_FMT " len="
1918                    RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
1919                    start, len, ramblock->used_length);
1920         return -1;
1921     }
1922 
1923     /*
1924      * When with postcopy preempt, we send back the page directly in the
1925      * rp-return thread.
1926      */
1927     if (postcopy_preempt_active()) {
1928         ram_addr_t page_start = start >> TARGET_PAGE_BITS;
1929         size_t page_size = qemu_ram_pagesize(ramblock);
1930         PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY];
1931         int ret = 0;
1932 
1933         qemu_mutex_lock(&rs->bitmap_mutex);
1934 
1935         pss_init(pss, ramblock, page_start);
1936         /*
1937          * Always use the preempt channel, and make sure it's there.  It's
1938          * safe to access without lock, because when rp-thread is running
1939          * we should be the only one who operates on the qemufile
1940          */
1941         pss->pss_channel = migrate_get_current()->postcopy_qemufile_src;
1942         assert(pss->pss_channel);
1943 
1944         /*
1945          * It must be either one or multiple of host page size.  Just
1946          * assert; if something wrong we're mostly split brain anyway.
1947          */
1948         assert(len % page_size == 0);
1949         while (len) {
1950             if (ram_save_host_page_urgent(pss)) {
1951                 error_setg(errp, "ram_save_host_page_urgent() failed: "
1952                            "ramblock=%s, start_addr=0x"RAM_ADDR_FMT,
1953                            ramblock->idstr, start);
1954                 ret = -1;
1955                 break;
1956             }
1957             /*
1958              * NOTE: after ram_save_host_page_urgent() succeeded, pss->page
1959              * will automatically be moved and point to the next host page
1960              * we're going to send, so no need to update here.
1961              *
1962              * Normally QEMU never sends >1 host page in requests, so
1963              * logically we don't even need that as the loop should only
1964              * run once, but just to be consistent.
1965              */
1966             len -= page_size;
1967         };
1968         qemu_mutex_unlock(&rs->bitmap_mutex);
1969 
1970         return ret;
1971     }
1972 
1973     struct RAMSrcPageRequest *new_entry =
1974         g_new0(struct RAMSrcPageRequest, 1);
1975     new_entry->rb = ramblock;
1976     new_entry->offset = start;
1977     new_entry->len = len;
1978 
1979     memory_region_ref(ramblock->mr);
1980     qemu_mutex_lock(&rs->src_page_req_mutex);
1981     QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req);
1982     migration_make_urgent_request();
1983     qemu_mutex_unlock(&rs->src_page_req_mutex);
1984 
1985     return 0;
1986 }
1987 
1988 /**
1989  * ram_save_target_page_legacy: save one target page
1990  *
1991  * Returns the number of pages written
1992  *
1993  * @rs: current RAM state
1994  * @pss: data about the page we want to send
1995  */
1996 static int ram_save_target_page_legacy(RAMState *rs, PageSearchStatus *pss)
1997 {
1998     ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
1999     int res;
2000 
2001     if (control_save_page(pss, offset, &res)) {
2002         return res;
2003     }
2004 
2005     if (save_zero_page(rs, pss, offset)) {
2006         return 1;
2007     }
2008 
2009     return ram_save_page(rs, pss);
2010 }
2011 
2012 /**
2013  * ram_save_target_page_multifd: send one target page to multifd workers
2014  *
2015  * Returns 1 if the page was queued, -1 otherwise.
2016  *
2017  * @rs: current RAM state
2018  * @pss: data about the page we want to send
2019  */
2020 static int ram_save_target_page_multifd(RAMState *rs, PageSearchStatus *pss)
2021 {
2022     RAMBlock *block = pss->block;
2023     ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2024 
2025     /*
2026      * While using multifd live migration, we still need to handle zero
2027      * page checking on the migration main thread.
2028      */
2029     if (migrate_zero_page_detection() == ZERO_PAGE_DETECTION_LEGACY) {
2030         if (save_zero_page(rs, pss, offset)) {
2031             return 1;
2032         }
2033     }
2034 
2035     return ram_save_multifd_page(block, offset);
2036 }
2037 
2038 /* Should be called before sending a host page */
2039 static void pss_host_page_prepare(PageSearchStatus *pss)
2040 {
2041     /* How many guest pages are there in one host page? */
2042     size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2043 
2044     pss->host_page_sending = true;
2045     if (guest_pfns <= 1) {
2046         /*
2047          * This covers both when guest psize == host psize, or when guest
2048          * has larger psize than the host (guest_pfns==0).
2049          *
2050          * For the latter, we always send one whole guest page per
2051          * iteration of the host page (example: an Alpha VM on x86 host
2052          * will have guest psize 8K while host psize 4K).
2053          */
2054         pss->host_page_start = pss->page;
2055         pss->host_page_end = pss->page + 1;
2056     } else {
2057         /*
2058          * The host page spans over multiple guest pages, we send them
2059          * within the same host page iteration.
2060          */
2061         pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns);
2062         pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns);
2063     }
2064 }
2065 
2066 /*
2067  * Whether the page pointed by PSS is within the host page being sent.
2068  * Must be called after a previous pss_host_page_prepare().
2069  */
2070 static bool pss_within_range(PageSearchStatus *pss)
2071 {
2072     ram_addr_t ram_addr;
2073 
2074     assert(pss->host_page_sending);
2075 
2076     /* Over host-page boundary? */
2077     if (pss->page >= pss->host_page_end) {
2078         return false;
2079     }
2080 
2081     ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS;
2082 
2083     return offset_in_ramblock(pss->block, ram_addr);
2084 }
2085 
2086 static void pss_host_page_finish(PageSearchStatus *pss)
2087 {
2088     pss->host_page_sending = false;
2089     /* This is not needed, but just to reset it */
2090     pss->host_page_start = pss->host_page_end = 0;
2091 }
2092 
2093 /*
2094  * Send an urgent host page specified by `pss'.  Need to be called with
2095  * bitmap_mutex held.
2096  *
2097  * Returns 0 if save host page succeeded, false otherwise.
2098  */
2099 static int ram_save_host_page_urgent(PageSearchStatus *pss)
2100 {
2101     bool page_dirty, sent = false;
2102     RAMState *rs = ram_state;
2103     int ret = 0;
2104 
2105     trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page);
2106     pss_host_page_prepare(pss);
2107 
2108     /*
2109      * If precopy is sending the same page, let it be done in precopy, or
2110      * we could send the same page in two channels and none of them will
2111      * receive the whole page.
2112      */
2113     if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) {
2114         trace_postcopy_preempt_hit(pss->block->idstr,
2115                                    pss->page << TARGET_PAGE_BITS);
2116         return 0;
2117     }
2118 
2119     do {
2120         page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2121 
2122         if (page_dirty) {
2123             /* Be strict to return code; it must be 1, or what else? */
2124             if (migration_ops->ram_save_target_page(rs, pss) != 1) {
2125                 error_report_once("%s: ram_save_target_page failed", __func__);
2126                 ret = -1;
2127                 goto out;
2128             }
2129             sent = true;
2130         }
2131         pss_find_next_dirty(pss);
2132     } while (pss_within_range(pss));
2133 out:
2134     pss_host_page_finish(pss);
2135     /* For urgent requests, flush immediately if sent */
2136     if (sent) {
2137         qemu_fflush(pss->pss_channel);
2138     }
2139     return ret;
2140 }
2141 
2142 /**
2143  * ram_save_host_page: save a whole host page
2144  *
2145  * Starting at *offset send pages up to the end of the current host
2146  * page. It's valid for the initial offset to point into the middle of
2147  * a host page in which case the remainder of the hostpage is sent.
2148  * Only dirty target pages are sent. Note that the host page size may
2149  * be a huge page for this block.
2150  *
2151  * The saving stops at the boundary of the used_length of the block
2152  * if the RAMBlock isn't a multiple of the host page size.
2153  *
2154  * The caller must be with ram_state.bitmap_mutex held to call this
2155  * function.  Note that this function can temporarily release the lock, but
2156  * when the function is returned it'll make sure the lock is still held.
2157  *
2158  * Returns the number of pages written or negative on error
2159  *
2160  * @rs: current RAM state
2161  * @pss: data about the page we want to send
2162  */
2163 static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss)
2164 {
2165     bool page_dirty, preempt_active = postcopy_preempt_active();
2166     int tmppages, pages = 0;
2167     size_t pagesize_bits =
2168         qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS;
2169     unsigned long start_page = pss->page;
2170     int res;
2171 
2172     if (migrate_ram_is_ignored(pss->block)) {
2173         error_report("block %s should not be migrated !", pss->block->idstr);
2174         return 0;
2175     }
2176 
2177     /* Update host page boundary information */
2178     pss_host_page_prepare(pss);
2179 
2180     do {
2181         page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page);
2182 
2183         /* Check the pages is dirty and if it is send it */
2184         if (page_dirty) {
2185             /*
2186              * Properly yield the lock only in postcopy preempt mode
2187              * because both migration thread and rp-return thread can
2188              * operate on the bitmaps.
2189              */
2190             if (preempt_active) {
2191                 qemu_mutex_unlock(&rs->bitmap_mutex);
2192             }
2193             tmppages = migration_ops->ram_save_target_page(rs, pss);
2194             if (tmppages >= 0) {
2195                 pages += tmppages;
2196                 /*
2197                  * Allow rate limiting to happen in the middle of huge pages if
2198                  * something is sent in the current iteration.
2199                  */
2200                 if (pagesize_bits > 1 && tmppages > 0) {
2201                     migration_rate_limit();
2202                 }
2203             }
2204             if (preempt_active) {
2205                 qemu_mutex_lock(&rs->bitmap_mutex);
2206             }
2207         } else {
2208             tmppages = 0;
2209         }
2210 
2211         if (tmppages < 0) {
2212             pss_host_page_finish(pss);
2213             return tmppages;
2214         }
2215 
2216         pss_find_next_dirty(pss);
2217     } while (pss_within_range(pss));
2218 
2219     pss_host_page_finish(pss);
2220 
2221     res = ram_save_release_protection(rs, pss, start_page);
2222     return (res < 0 ? res : pages);
2223 }
2224 
2225 /**
2226  * ram_find_and_save_block: finds a dirty page and sends it to f
2227  *
2228  * Called within an RCU critical section.
2229  *
2230  * Returns the number of pages written where zero means no dirty pages,
2231  * or negative on error
2232  *
2233  * @rs: current RAM state
2234  *
2235  * On systems where host-page-size > target-page-size it will send all the
2236  * pages in a host page that are dirty.
2237  */
2238 static int ram_find_and_save_block(RAMState *rs)
2239 {
2240     PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY];
2241     int pages = 0;
2242 
2243     /* No dirty page as there is zero RAM */
2244     if (!rs->ram_bytes_total) {
2245         return pages;
2246     }
2247 
2248     /*
2249      * Always keep last_seen_block/last_page valid during this procedure,
2250      * because find_dirty_block() relies on these values (e.g., we compare
2251      * last_seen_block with pss.block to see whether we searched all the
2252      * ramblocks) to detect the completion of migration.  Having NULL value
2253      * of last_seen_block can conditionally cause below loop to run forever.
2254      */
2255     if (!rs->last_seen_block) {
2256         rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks);
2257         rs->last_page = 0;
2258     }
2259 
2260     pss_init(pss, rs->last_seen_block, rs->last_page);
2261 
2262     while (true){
2263         if (!get_queued_page(rs, pss)) {
2264             /* priority queue empty, so just search for something dirty */
2265             int res = find_dirty_block(rs, pss);
2266             if (res != PAGE_DIRTY_FOUND) {
2267                 if (res == PAGE_ALL_CLEAN) {
2268                     break;
2269                 } else if (res == PAGE_TRY_AGAIN) {
2270                     continue;
2271                 } else if (res < 0) {
2272                     pages = res;
2273                     break;
2274                 }
2275             }
2276         }
2277         pages = ram_save_host_page(rs, pss);
2278         if (pages) {
2279             break;
2280         }
2281     }
2282 
2283     rs->last_seen_block = pss->block;
2284     rs->last_page = pss->page;
2285 
2286     return pages;
2287 }
2288 
2289 static uint64_t ram_bytes_total_with_ignored(void)
2290 {
2291     RAMBlock *block;
2292     uint64_t total = 0;
2293 
2294     RCU_READ_LOCK_GUARD();
2295 
2296     RAMBLOCK_FOREACH_MIGRATABLE(block) {
2297         total += block->used_length;
2298     }
2299     return total;
2300 }
2301 
2302 uint64_t ram_bytes_total(void)
2303 {
2304     RAMBlock *block;
2305     uint64_t total = 0;
2306 
2307     RCU_READ_LOCK_GUARD();
2308 
2309     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2310         total += block->used_length;
2311     }
2312     return total;
2313 }
2314 
2315 static void xbzrle_load_setup(void)
2316 {
2317     XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE);
2318 }
2319 
2320 static void xbzrle_load_cleanup(void)
2321 {
2322     g_free(XBZRLE.decoded_buf);
2323     XBZRLE.decoded_buf = NULL;
2324 }
2325 
2326 static void ram_state_cleanup(RAMState **rsp)
2327 {
2328     if (*rsp) {
2329         migration_page_queue_free(*rsp);
2330         qemu_mutex_destroy(&(*rsp)->bitmap_mutex);
2331         qemu_mutex_destroy(&(*rsp)->src_page_req_mutex);
2332         g_free(*rsp);
2333         *rsp = NULL;
2334     }
2335 }
2336 
2337 static void xbzrle_cleanup(void)
2338 {
2339     XBZRLE_cache_lock();
2340     if (XBZRLE.cache) {
2341         cache_fini(XBZRLE.cache);
2342         g_free(XBZRLE.encoded_buf);
2343         g_free(XBZRLE.current_buf);
2344         g_free(XBZRLE.zero_target_page);
2345         XBZRLE.cache = NULL;
2346         XBZRLE.encoded_buf = NULL;
2347         XBZRLE.current_buf = NULL;
2348         XBZRLE.zero_target_page = NULL;
2349     }
2350     XBZRLE_cache_unlock();
2351 }
2352 
2353 static void ram_bitmaps_destroy(void)
2354 {
2355     RAMBlock *block;
2356 
2357     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2358         g_free(block->clear_bmap);
2359         block->clear_bmap = NULL;
2360         g_free(block->bmap);
2361         block->bmap = NULL;
2362         g_free(block->file_bmap);
2363         block->file_bmap = NULL;
2364     }
2365 }
2366 
2367 static void ram_save_cleanup(void *opaque)
2368 {
2369     RAMState **rsp = opaque;
2370 
2371     /* We don't use dirty log with background snapshots */
2372     if (!migrate_background_snapshot()) {
2373         /* caller have hold BQL or is in a bh, so there is
2374          * no writing race against the migration bitmap
2375          */
2376         if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) {
2377             /*
2378              * do not stop dirty log without starting it, since
2379              * memory_global_dirty_log_stop will assert that
2380              * memory_global_dirty_log_start/stop used in pairs
2381              */
2382             memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
2383         }
2384     }
2385 
2386     ram_bitmaps_destroy();
2387 
2388     xbzrle_cleanup();
2389     multifd_ram_save_cleanup();
2390     ram_state_cleanup(rsp);
2391     g_free(migration_ops);
2392     migration_ops = NULL;
2393 }
2394 
2395 static void ram_state_reset(RAMState *rs)
2396 {
2397     int i;
2398 
2399     for (i = 0; i < RAM_CHANNEL_MAX; i++) {
2400         rs->pss[i].last_sent_block = NULL;
2401     }
2402 
2403     rs->last_seen_block = NULL;
2404     rs->last_page = 0;
2405     rs->last_version = ram_list.version;
2406     rs->xbzrle_started = false;
2407 }
2408 
2409 #define MAX_WAIT 50 /* ms, half buffered_file limit */
2410 
2411 /* **** functions for postcopy ***** */
2412 
2413 void ram_postcopy_migrated_memory_release(MigrationState *ms)
2414 {
2415     struct RAMBlock *block;
2416 
2417     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2418         unsigned long *bitmap = block->bmap;
2419         unsigned long range = block->used_length >> TARGET_PAGE_BITS;
2420         unsigned long run_start = find_next_zero_bit(bitmap, range, 0);
2421 
2422         while (run_start < range) {
2423             unsigned long run_end = find_next_bit(bitmap, range, run_start + 1);
2424             ram_discard_range(block->idstr,
2425                               ((ram_addr_t)run_start) << TARGET_PAGE_BITS,
2426                               ((ram_addr_t)(run_end - run_start))
2427                                 << TARGET_PAGE_BITS);
2428             run_start = find_next_zero_bit(bitmap, range, run_end + 1);
2429         }
2430     }
2431 }
2432 
2433 /**
2434  * postcopy_send_discard_bm_ram: discard a RAMBlock
2435  *
2436  * Callback from postcopy_each_ram_send_discard for each RAMBlock
2437  *
2438  * @ms: current migration state
2439  * @block: RAMBlock to discard
2440  */
2441 static void postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block)
2442 {
2443     unsigned long end = block->used_length >> TARGET_PAGE_BITS;
2444     unsigned long current;
2445     unsigned long *bitmap = block->bmap;
2446 
2447     for (current = 0; current < end; ) {
2448         unsigned long one = find_next_bit(bitmap, end, current);
2449         unsigned long zero, discard_length;
2450 
2451         if (one >= end) {
2452             break;
2453         }
2454 
2455         zero = find_next_zero_bit(bitmap, end, one + 1);
2456 
2457         if (zero >= end) {
2458             discard_length = end - one;
2459         } else {
2460             discard_length = zero - one;
2461         }
2462         postcopy_discard_send_range(ms, one, discard_length);
2463         current = one + discard_length;
2464     }
2465 }
2466 
2467 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block);
2468 
2469 /**
2470  * postcopy_each_ram_send_discard: discard all RAMBlocks
2471  *
2472  * Utility for the outgoing postcopy code.
2473  *   Calls postcopy_send_discard_bm_ram for each RAMBlock
2474  *   passing it bitmap indexes and name.
2475  * (qemu_ram_foreach_block ends up passing unscaled lengths
2476  *  which would mean postcopy code would have to deal with target page)
2477  *
2478  * @ms: current migration state
2479  */
2480 static void postcopy_each_ram_send_discard(MigrationState *ms)
2481 {
2482     struct RAMBlock *block;
2483 
2484     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2485         postcopy_discard_send_init(ms, block->idstr);
2486 
2487         /*
2488          * Deal with TPS != HPS and huge pages.  It discard any partially sent
2489          * host-page size chunks, mark any partially dirty host-page size
2490          * chunks as all dirty.  In this case the host-page is the host-page
2491          * for the particular RAMBlock, i.e. it might be a huge page.
2492          */
2493         postcopy_chunk_hostpages_pass(ms, block);
2494 
2495         /*
2496          * Postcopy sends chunks of bitmap over the wire, but it
2497          * just needs indexes at this point, avoids it having
2498          * target page specific code.
2499          */
2500         postcopy_send_discard_bm_ram(ms, block);
2501         postcopy_discard_send_finish(ms);
2502     }
2503 }
2504 
2505 /**
2506  * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages
2507  *
2508  * Helper for postcopy_chunk_hostpages; it's called twice to
2509  * canonicalize the two bitmaps, that are similar, but one is
2510  * inverted.
2511  *
2512  * Postcopy requires that all target pages in a hostpage are dirty or
2513  * clean, not a mix.  This function canonicalizes the bitmaps.
2514  *
2515  * @ms: current migration state
2516  * @block: block that contains the page we want to canonicalize
2517  */
2518 static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block)
2519 {
2520     RAMState *rs = ram_state;
2521     unsigned long *bitmap = block->bmap;
2522     unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE;
2523     unsigned long pages = block->used_length >> TARGET_PAGE_BITS;
2524     unsigned long run_start;
2525 
2526     if (block->page_size == TARGET_PAGE_SIZE) {
2527         /* Easy case - TPS==HPS for a non-huge page RAMBlock */
2528         return;
2529     }
2530 
2531     /* Find a dirty page */
2532     run_start = find_next_bit(bitmap, pages, 0);
2533 
2534     while (run_start < pages) {
2535 
2536         /*
2537          * If the start of this run of pages is in the middle of a host
2538          * page, then we need to fixup this host page.
2539          */
2540         if (QEMU_IS_ALIGNED(run_start, host_ratio)) {
2541             /* Find the end of this run */
2542             run_start = find_next_zero_bit(bitmap, pages, run_start + 1);
2543             /*
2544              * If the end isn't at the start of a host page, then the
2545              * run doesn't finish at the end of a host page
2546              * and we need to discard.
2547              */
2548         }
2549 
2550         if (!QEMU_IS_ALIGNED(run_start, host_ratio)) {
2551             unsigned long page;
2552             unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start,
2553                                                              host_ratio);
2554             run_start = QEMU_ALIGN_UP(run_start, host_ratio);
2555 
2556             /* Clean up the bitmap */
2557             for (page = fixup_start_addr;
2558                  page < fixup_start_addr + host_ratio; page++) {
2559                 /*
2560                  * Remark them as dirty, updating the count for any pages
2561                  * that weren't previously dirty.
2562                  */
2563                 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap);
2564             }
2565         }
2566 
2567         /* Find the next dirty page for the next iteration */
2568         run_start = find_next_bit(bitmap, pages, run_start);
2569     }
2570 }
2571 
2572 /**
2573  * ram_postcopy_send_discard_bitmap: transmit the discard bitmap
2574  *
2575  * Transmit the set of pages to be discarded after precopy to the target
2576  * these are pages that:
2577  *     a) Have been previously transmitted but are now dirty again
2578  *     b) Pages that have never been transmitted, this ensures that
2579  *        any pages on the destination that have been mapped by background
2580  *        tasks get discarded (transparent huge pages is the specific concern)
2581  * Hopefully this is pretty sparse
2582  *
2583  * @ms: current migration state
2584  */
2585 void ram_postcopy_send_discard_bitmap(MigrationState *ms)
2586 {
2587     RAMState *rs = ram_state;
2588 
2589     RCU_READ_LOCK_GUARD();
2590 
2591     /* This should be our last sync, the src is now paused */
2592     migration_bitmap_sync(rs, false);
2593 
2594     /* Easiest way to make sure we don't resume in the middle of a host-page */
2595     rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL;
2596     rs->last_seen_block = NULL;
2597     rs->last_page = 0;
2598 
2599     postcopy_each_ram_send_discard(ms);
2600 
2601     trace_ram_postcopy_send_discard_bitmap();
2602 }
2603 
2604 /**
2605  * ram_discard_range: discard dirtied pages at the beginning of postcopy
2606  *
2607  * Returns zero on success
2608  *
2609  * @rbname: name of the RAMBlock of the request. NULL means the
2610  *          same that last one.
2611  * @start: RAMBlock starting page
2612  * @length: RAMBlock size
2613  */
2614 int ram_discard_range(const char *rbname, uint64_t start, size_t length)
2615 {
2616     trace_ram_discard_range(rbname, start, length);
2617 
2618     RCU_READ_LOCK_GUARD();
2619     RAMBlock *rb = qemu_ram_block_by_name(rbname);
2620 
2621     if (!rb) {
2622         error_report("ram_discard_range: Failed to find block '%s'", rbname);
2623         return -1;
2624     }
2625 
2626     /*
2627      * On source VM, we don't need to update the received bitmap since
2628      * we don't even have one.
2629      */
2630     if (rb->receivedmap) {
2631         bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(),
2632                      length >> qemu_target_page_bits());
2633     }
2634 
2635     return ram_block_discard_range(rb, start, length);
2636 }
2637 
2638 /*
2639  * For every allocation, we will try not to crash the VM if the
2640  * allocation failed.
2641  */
2642 static bool xbzrle_init(Error **errp)
2643 {
2644     if (!migrate_xbzrle()) {
2645         return true;
2646     }
2647 
2648     XBZRLE_cache_lock();
2649 
2650     XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE);
2651     if (!XBZRLE.zero_target_page) {
2652         error_setg(errp, "%s: Error allocating zero page", __func__);
2653         goto err_out;
2654     }
2655 
2656     XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(),
2657                               TARGET_PAGE_SIZE, errp);
2658     if (!XBZRLE.cache) {
2659         goto free_zero_page;
2660     }
2661 
2662     XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
2663     if (!XBZRLE.encoded_buf) {
2664         error_setg(errp, "%s: Error allocating encoded_buf", __func__);
2665         goto free_cache;
2666     }
2667 
2668     XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
2669     if (!XBZRLE.current_buf) {
2670         error_setg(errp, "%s: Error allocating current_buf", __func__);
2671         goto free_encoded_buf;
2672     }
2673 
2674     /* We are all good */
2675     XBZRLE_cache_unlock();
2676     return true;
2677 
2678 free_encoded_buf:
2679     g_free(XBZRLE.encoded_buf);
2680     XBZRLE.encoded_buf = NULL;
2681 free_cache:
2682     cache_fini(XBZRLE.cache);
2683     XBZRLE.cache = NULL;
2684 free_zero_page:
2685     g_free(XBZRLE.zero_target_page);
2686     XBZRLE.zero_target_page = NULL;
2687 err_out:
2688     XBZRLE_cache_unlock();
2689     return false;
2690 }
2691 
2692 static bool ram_state_init(RAMState **rsp, Error **errp)
2693 {
2694     *rsp = g_try_new0(RAMState, 1);
2695 
2696     if (!*rsp) {
2697         error_setg(errp, "%s: Init ramstate fail", __func__);
2698         return false;
2699     }
2700 
2701     qemu_mutex_init(&(*rsp)->bitmap_mutex);
2702     qemu_mutex_init(&(*rsp)->src_page_req_mutex);
2703     QSIMPLEQ_INIT(&(*rsp)->src_page_requests);
2704     (*rsp)->ram_bytes_total = ram_bytes_total();
2705 
2706     /*
2707      * Count the total number of pages used by ram blocks not including any
2708      * gaps due to alignment or unplugs.
2709      * This must match with the initial values of dirty bitmap.
2710      */
2711     (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS;
2712     ram_state_reset(*rsp);
2713 
2714     return true;
2715 }
2716 
2717 static void ram_list_init_bitmaps(void)
2718 {
2719     MigrationState *ms = migrate_get_current();
2720     RAMBlock *block;
2721     unsigned long pages;
2722     uint8_t shift;
2723 
2724     /* Skip setting bitmap if there is no RAM */
2725     if (ram_bytes_total()) {
2726         shift = ms->clear_bitmap_shift;
2727         if (shift > CLEAR_BITMAP_SHIFT_MAX) {
2728             error_report("clear_bitmap_shift (%u) too big, using "
2729                          "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX);
2730             shift = CLEAR_BITMAP_SHIFT_MAX;
2731         } else if (shift < CLEAR_BITMAP_SHIFT_MIN) {
2732             error_report("clear_bitmap_shift (%u) too small, using "
2733                          "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN);
2734             shift = CLEAR_BITMAP_SHIFT_MIN;
2735         }
2736 
2737         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2738             pages = block->max_length >> TARGET_PAGE_BITS;
2739             /*
2740              * The initial dirty bitmap for migration must be set with all
2741              * ones to make sure we'll migrate every guest RAM page to
2742              * destination.
2743              * Here we set RAMBlock.bmap all to 1 because when rebegin a
2744              * new migration after a failed migration, ram_list.
2745              * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole
2746              * guest memory.
2747              */
2748             block->bmap = bitmap_new(pages);
2749             bitmap_set(block->bmap, 0, pages);
2750             if (migrate_mapped_ram()) {
2751                 block->file_bmap = bitmap_new(pages);
2752             }
2753             block->clear_bmap_shift = shift;
2754             block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift));
2755         }
2756     }
2757 }
2758 
2759 static void migration_bitmap_clear_discarded_pages(RAMState *rs)
2760 {
2761     unsigned long pages;
2762     RAMBlock *rb;
2763 
2764     RCU_READ_LOCK_GUARD();
2765 
2766     RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
2767             pages = ramblock_dirty_bitmap_clear_discarded_pages(rb);
2768             rs->migration_dirty_pages -= pages;
2769     }
2770 }
2771 
2772 static bool ram_init_bitmaps(RAMState *rs, Error **errp)
2773 {
2774     bool ret = true;
2775 
2776     qemu_mutex_lock_ramlist();
2777 
2778     WITH_RCU_READ_LOCK_GUARD() {
2779         ram_list_init_bitmaps();
2780         /* We don't use dirty log with background snapshots */
2781         if (!migrate_background_snapshot()) {
2782             ret = memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION, errp);
2783             if (!ret) {
2784                 goto out_unlock;
2785             }
2786             migration_bitmap_sync_precopy(false);
2787         }
2788     }
2789 out_unlock:
2790     qemu_mutex_unlock_ramlist();
2791 
2792     if (!ret) {
2793         ram_bitmaps_destroy();
2794         return false;
2795     }
2796 
2797     /*
2798      * After an eventual first bitmap sync, fixup the initial bitmap
2799      * containing all 1s to exclude any discarded pages from migration.
2800      */
2801     migration_bitmap_clear_discarded_pages(rs);
2802     return true;
2803 }
2804 
2805 static int ram_init_all(RAMState **rsp, Error **errp)
2806 {
2807     if (!ram_state_init(rsp, errp)) {
2808         return -1;
2809     }
2810 
2811     if (!xbzrle_init(errp)) {
2812         ram_state_cleanup(rsp);
2813         return -1;
2814     }
2815 
2816     if (!ram_init_bitmaps(*rsp, errp)) {
2817         return -1;
2818     }
2819 
2820     return 0;
2821 }
2822 
2823 static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out)
2824 {
2825     RAMBlock *block;
2826     uint64_t pages = 0;
2827 
2828     /*
2829      * Postcopy is not using xbzrle/compression, so no need for that.
2830      * Also, since source are already halted, we don't need to care
2831      * about dirty page logging as well.
2832      */
2833 
2834     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
2835         pages += bitmap_count_one(block->bmap,
2836                                   block->used_length >> TARGET_PAGE_BITS);
2837     }
2838 
2839     /* This may not be aligned with current bitmaps. Recalculate. */
2840     rs->migration_dirty_pages = pages;
2841 
2842     ram_state_reset(rs);
2843 
2844     /* Update RAMState cache of output QEMUFile */
2845     rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out;
2846 
2847     trace_ram_state_resume_prepare(pages);
2848 }
2849 
2850 /*
2851  * This function clears bits of the free pages reported by the caller from the
2852  * migration dirty bitmap. @addr is the host address corresponding to the
2853  * start of the continuous guest free pages, and @len is the total bytes of
2854  * those pages.
2855  */
2856 void qemu_guest_free_page_hint(void *addr, size_t len)
2857 {
2858     RAMBlock *block;
2859     ram_addr_t offset;
2860     size_t used_len, start, npages;
2861 
2862     /* This function is currently expected to be used during live migration */
2863     if (!migration_is_running()) {
2864         return;
2865     }
2866 
2867     for (; len > 0; len -= used_len, addr += used_len) {
2868         block = qemu_ram_block_from_host(addr, false, &offset);
2869         if (unlikely(!block || offset >= block->used_length)) {
2870             /*
2871              * The implementation might not support RAMBlock resize during
2872              * live migration, but it could happen in theory with future
2873              * updates. So we add a check here to capture that case.
2874              */
2875             error_report_once("%s unexpected error", __func__);
2876             return;
2877         }
2878 
2879         if (len <= block->used_length - offset) {
2880             used_len = len;
2881         } else {
2882             used_len = block->used_length - offset;
2883         }
2884 
2885         start = offset >> TARGET_PAGE_BITS;
2886         npages = used_len >> TARGET_PAGE_BITS;
2887 
2888         qemu_mutex_lock(&ram_state->bitmap_mutex);
2889         /*
2890          * The skipped free pages are equavalent to be sent from clear_bmap's
2891          * perspective, so clear the bits from the memory region bitmap which
2892          * are initially set. Otherwise those skipped pages will be sent in
2893          * the next round after syncing from the memory region bitmap.
2894          */
2895         migration_clear_memory_region_dirty_bitmap_range(block, start, npages);
2896         ram_state->migration_dirty_pages -=
2897                       bitmap_count_one_with_offset(block->bmap, start, npages);
2898         bitmap_clear(block->bmap, start, npages);
2899         qemu_mutex_unlock(&ram_state->bitmap_mutex);
2900     }
2901 }
2902 
2903 #define MAPPED_RAM_HDR_VERSION 1
2904 struct MappedRamHeader {
2905     uint32_t version;
2906     /*
2907      * The target's page size, so we know how many pages are in the
2908      * bitmap.
2909      */
2910     uint64_t page_size;
2911     /*
2912      * The offset in the migration file where the pages bitmap is
2913      * stored.
2914      */
2915     uint64_t bitmap_offset;
2916     /*
2917      * The offset in the migration file where the actual pages (data)
2918      * are stored.
2919      */
2920     uint64_t pages_offset;
2921 } QEMU_PACKED;
2922 typedef struct MappedRamHeader MappedRamHeader;
2923 
2924 static void mapped_ram_setup_ramblock(QEMUFile *file, RAMBlock *block)
2925 {
2926     g_autofree MappedRamHeader *header = NULL;
2927     size_t header_size, bitmap_size;
2928     long num_pages;
2929 
2930     header = g_new0(MappedRamHeader, 1);
2931     header_size = sizeof(MappedRamHeader);
2932 
2933     num_pages = block->used_length >> TARGET_PAGE_BITS;
2934     bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
2935 
2936     /*
2937      * Save the file offsets of where the bitmap and the pages should
2938      * go as they are written at the end of migration and during the
2939      * iterative phase, respectively.
2940      */
2941     block->bitmap_offset = qemu_get_offset(file) + header_size;
2942     block->pages_offset = ROUND_UP(block->bitmap_offset +
2943                                    bitmap_size,
2944                                    MAPPED_RAM_FILE_OFFSET_ALIGNMENT);
2945 
2946     header->version = cpu_to_be32(MAPPED_RAM_HDR_VERSION);
2947     header->page_size = cpu_to_be64(TARGET_PAGE_SIZE);
2948     header->bitmap_offset = cpu_to_be64(block->bitmap_offset);
2949     header->pages_offset = cpu_to_be64(block->pages_offset);
2950 
2951     qemu_put_buffer(file, (uint8_t *) header, header_size);
2952 
2953     /* prepare offset for next ramblock */
2954     qemu_set_offset(file, block->pages_offset + block->used_length, SEEK_SET);
2955 }
2956 
2957 static bool mapped_ram_read_header(QEMUFile *file, MappedRamHeader *header,
2958                                    Error **errp)
2959 {
2960     size_t ret, header_size = sizeof(MappedRamHeader);
2961 
2962     ret = qemu_get_buffer(file, (uint8_t *)header, header_size);
2963     if (ret != header_size) {
2964         error_setg(errp, "Could not read whole mapped-ram migration header "
2965                    "(expected %zd, got %zd bytes)", header_size, ret);
2966         return false;
2967     }
2968 
2969     /* migration stream is big-endian */
2970     header->version = be32_to_cpu(header->version);
2971 
2972     if (header->version > MAPPED_RAM_HDR_VERSION) {
2973         error_setg(errp, "Migration mapped-ram capability version not "
2974                    "supported (expected <= %d, got %d)", MAPPED_RAM_HDR_VERSION,
2975                    header->version);
2976         return false;
2977     }
2978 
2979     header->page_size = be64_to_cpu(header->page_size);
2980     header->bitmap_offset = be64_to_cpu(header->bitmap_offset);
2981     header->pages_offset = be64_to_cpu(header->pages_offset);
2982 
2983     return true;
2984 }
2985 
2986 /*
2987  * Each of ram_save_setup, ram_save_iterate and ram_save_complete has
2988  * long-running RCU critical section.  When rcu-reclaims in the code
2989  * start to become numerous it will be necessary to reduce the
2990  * granularity of these critical sections.
2991  */
2992 
2993 /**
2994  * ram_save_setup: Setup RAM for migration
2995  *
2996  * Returns zero to indicate success and negative for error
2997  *
2998  * @f: QEMUFile where to send the data
2999  * @opaque: RAMState pointer
3000  * @errp: pointer to Error*, to store an error if it happens.
3001  */
3002 static int ram_save_setup(QEMUFile *f, void *opaque, Error **errp)
3003 {
3004     RAMState **rsp = opaque;
3005     RAMBlock *block;
3006     int ret, max_hg_page_size;
3007 
3008     /* migration has already setup the bitmap, reuse it. */
3009     if (!migration_in_colo_state()) {
3010         if (ram_init_all(rsp, errp) != 0) {
3011             return -1;
3012         }
3013     }
3014     (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
3015 
3016     /*
3017      * ??? Mirrors the previous value of qemu_host_page_size,
3018      * but is this really what was intended for the migration?
3019      */
3020     max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
3021 
3022     WITH_RCU_READ_LOCK_GUARD() {
3023         qemu_put_be64(f, ram_bytes_total_with_ignored()
3024                          | RAM_SAVE_FLAG_MEM_SIZE);
3025 
3026         RAMBLOCK_FOREACH_MIGRATABLE(block) {
3027             qemu_put_byte(f, strlen(block->idstr));
3028             qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
3029             qemu_put_be64(f, block->used_length);
3030             if (migrate_postcopy_ram() &&
3031                 block->page_size != max_hg_page_size) {
3032                 qemu_put_be64(f, block->page_size);
3033             }
3034             if (migrate_ignore_shared()) {
3035                 qemu_put_be64(f, block->mr->addr);
3036             }
3037 
3038             if (migrate_mapped_ram()) {
3039                 mapped_ram_setup_ramblock(f, block);
3040             }
3041         }
3042     }
3043 
3044     ret = rdma_registration_start(f, RAM_CONTROL_SETUP);
3045     if (ret < 0) {
3046         error_setg(errp, "%s: failed to start RDMA registration", __func__);
3047         qemu_file_set_error(f, ret);
3048         return ret;
3049     }
3050 
3051     ret = rdma_registration_stop(f, RAM_CONTROL_SETUP);
3052     if (ret < 0) {
3053         error_setg(errp, "%s: failed to stop RDMA registration", __func__);
3054         qemu_file_set_error(f, ret);
3055         return ret;
3056     }
3057 
3058     migration_ops = g_malloc0(sizeof(MigrationOps));
3059 
3060     if (migrate_multifd()) {
3061         multifd_ram_save_setup();
3062         migration_ops->ram_save_target_page = ram_save_target_page_multifd;
3063     } else {
3064         migration_ops->ram_save_target_page = ram_save_target_page_legacy;
3065     }
3066 
3067     bql_unlock();
3068     ret = multifd_ram_flush_and_sync();
3069     bql_lock();
3070     if (ret < 0) {
3071         error_setg(errp, "%s: multifd synchronization failed", __func__);
3072         return ret;
3073     }
3074 
3075     if (migrate_multifd() && !migrate_multifd_flush_after_each_section()
3076         && !migrate_mapped_ram()) {
3077         qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH);
3078     }
3079 
3080     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3081     ret = qemu_fflush(f);
3082     if (ret < 0) {
3083         error_setg_errno(errp, -ret, "%s failed", __func__);
3084     }
3085     return ret;
3086 }
3087 
3088 static void ram_save_file_bmap(QEMUFile *f)
3089 {
3090     RAMBlock *block;
3091 
3092     RAMBLOCK_FOREACH_MIGRATABLE(block) {
3093         long num_pages = block->used_length >> TARGET_PAGE_BITS;
3094         long bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3095 
3096         qemu_put_buffer_at(f, (uint8_t *)block->file_bmap, bitmap_size,
3097                            block->bitmap_offset);
3098         ram_transferred_add(bitmap_size);
3099 
3100         /*
3101          * Free the bitmap here to catch any synchronization issues
3102          * with multifd channels. No channels should be sending pages
3103          * after we've written the bitmap to file.
3104          */
3105         g_free(block->file_bmap);
3106         block->file_bmap = NULL;
3107     }
3108 }
3109 
3110 void ramblock_set_file_bmap_atomic(RAMBlock *block, ram_addr_t offset, bool set)
3111 {
3112     if (set) {
3113         set_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3114     } else {
3115         clear_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap);
3116     }
3117 }
3118 
3119 /**
3120  * ram_save_iterate: iterative stage for migration
3121  *
3122  * Returns zero to indicate success and negative for error
3123  *
3124  * @f: QEMUFile where to send the data
3125  * @opaque: RAMState pointer
3126  */
3127 static int ram_save_iterate(QEMUFile *f, void *opaque)
3128 {
3129     RAMState **temp = opaque;
3130     RAMState *rs = *temp;
3131     int ret = 0;
3132     int i;
3133     int64_t t0;
3134     int done = 0;
3135 
3136     /*
3137      * We'll take this lock a little bit long, but it's okay for two reasons.
3138      * Firstly, the only possible other thread to take it is who calls
3139      * qemu_guest_free_page_hint(), which should be rare; secondly, see
3140      * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which
3141      * guarantees that we'll at least released it in a regular basis.
3142      */
3143     WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) {
3144         WITH_RCU_READ_LOCK_GUARD() {
3145             if (ram_list.version != rs->last_version) {
3146                 ram_state_reset(rs);
3147             }
3148 
3149             /* Read version before ram_list.blocks */
3150             smp_rmb();
3151 
3152             ret = rdma_registration_start(f, RAM_CONTROL_ROUND);
3153             if (ret < 0) {
3154                 qemu_file_set_error(f, ret);
3155                 goto out;
3156             }
3157 
3158             t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
3159             i = 0;
3160             while ((ret = migration_rate_exceeded(f)) == 0 ||
3161                    postcopy_has_request(rs)) {
3162                 int pages;
3163 
3164                 if (qemu_file_get_error(f)) {
3165                     break;
3166                 }
3167 
3168                 pages = ram_find_and_save_block(rs);
3169                 /* no more pages to sent */
3170                 if (pages == 0) {
3171                     done = 1;
3172                     break;
3173                 }
3174 
3175                 if (pages < 0) {
3176                     qemu_file_set_error(f, pages);
3177                     break;
3178                 }
3179 
3180                 rs->target_page_count += pages;
3181 
3182                 /*
3183                  * we want to check in the 1st loop, just in case it was the 1st
3184                  * time and we had to sync the dirty bitmap.
3185                  * qemu_clock_get_ns() is a bit expensive, so we only check each
3186                  * some iterations
3187                  */
3188                 if ((i & 63) == 0) {
3189                     uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) /
3190                         1000000;
3191                     if (t1 > MAX_WAIT) {
3192                         trace_ram_save_iterate_big_wait(t1, i);
3193                         break;
3194                     }
3195                 }
3196                 i++;
3197             }
3198         }
3199     }
3200 
3201     /*
3202      * Must occur before EOS (or any QEMUFile operation)
3203      * because of RDMA protocol.
3204      */
3205     ret = rdma_registration_stop(f, RAM_CONTROL_ROUND);
3206     if (ret < 0) {
3207         qemu_file_set_error(f, ret);
3208     }
3209 
3210 out:
3211     if (ret >= 0 && migration_is_running()) {
3212         if (migrate_multifd() && migrate_multifd_flush_after_each_section() &&
3213             !migrate_mapped_ram()) {
3214             ret = multifd_ram_flush_and_sync();
3215             if (ret < 0) {
3216                 return ret;
3217             }
3218         }
3219 
3220         qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3221         ram_transferred_add(8);
3222         ret = qemu_fflush(f);
3223     }
3224     if (ret < 0) {
3225         return ret;
3226     }
3227 
3228     return done;
3229 }
3230 
3231 /**
3232  * ram_save_complete: function called to send the remaining amount of ram
3233  *
3234  * Returns zero to indicate success or negative on error
3235  *
3236  * Called with the BQL
3237  *
3238  * @f: QEMUFile where to send the data
3239  * @opaque: RAMState pointer
3240  */
3241 static int ram_save_complete(QEMUFile *f, void *opaque)
3242 {
3243     RAMState **temp = opaque;
3244     RAMState *rs = *temp;
3245     int ret = 0;
3246 
3247     rs->last_stage = !migration_in_colo_state();
3248 
3249     WITH_RCU_READ_LOCK_GUARD() {
3250         if (!migration_in_postcopy()) {
3251             migration_bitmap_sync_precopy(true);
3252         }
3253 
3254         ret = rdma_registration_start(f, RAM_CONTROL_FINISH);
3255         if (ret < 0) {
3256             qemu_file_set_error(f, ret);
3257             return ret;
3258         }
3259 
3260         /* try transferring iterative blocks of memory */
3261 
3262         /* flush all remaining blocks regardless of rate limiting */
3263         qemu_mutex_lock(&rs->bitmap_mutex);
3264         while (true) {
3265             int pages;
3266 
3267             pages = ram_find_and_save_block(rs);
3268             /* no more blocks to sent */
3269             if (pages == 0) {
3270                 break;
3271             }
3272             if (pages < 0) {
3273                 qemu_mutex_unlock(&rs->bitmap_mutex);
3274                 return pages;
3275             }
3276         }
3277         qemu_mutex_unlock(&rs->bitmap_mutex);
3278 
3279         ret = rdma_registration_stop(f, RAM_CONTROL_FINISH);
3280         if (ret < 0) {
3281             qemu_file_set_error(f, ret);
3282             return ret;
3283         }
3284     }
3285 
3286     ret = multifd_ram_flush_and_sync();
3287     if (ret < 0) {
3288         return ret;
3289     }
3290 
3291     if (migrate_mapped_ram()) {
3292         ram_save_file_bmap(f);
3293 
3294         if (qemu_file_get_error(f)) {
3295             Error *local_err = NULL;
3296             int err = qemu_file_get_error_obj(f, &local_err);
3297 
3298             error_reportf_err(local_err, "Failed to write bitmap to file: ");
3299             return -err;
3300         }
3301     }
3302 
3303     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
3304     return qemu_fflush(f);
3305 }
3306 
3307 static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy,
3308                                        uint64_t *can_postcopy)
3309 {
3310     RAMState **temp = opaque;
3311     RAMState *rs = *temp;
3312 
3313     uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3314 
3315     if (migrate_postcopy_ram()) {
3316         /* We can do postcopy, and all the data is postcopiable */
3317         *can_postcopy += remaining_size;
3318     } else {
3319         *must_precopy += remaining_size;
3320     }
3321 }
3322 
3323 static void ram_state_pending_exact(void *opaque, uint64_t *must_precopy,
3324                                     uint64_t *can_postcopy)
3325 {
3326     RAMState **temp = opaque;
3327     RAMState *rs = *temp;
3328     uint64_t remaining_size;
3329 
3330     if (!migration_in_postcopy()) {
3331         bql_lock();
3332         WITH_RCU_READ_LOCK_GUARD() {
3333             migration_bitmap_sync_precopy(false);
3334         }
3335         bql_unlock();
3336     }
3337 
3338     remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE;
3339 
3340     if (migrate_postcopy_ram()) {
3341         /* We can do postcopy, and all the data is postcopiable */
3342         *can_postcopy += remaining_size;
3343     } else {
3344         *must_precopy += remaining_size;
3345     }
3346 }
3347 
3348 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
3349 {
3350     unsigned int xh_len;
3351     int xh_flags;
3352     uint8_t *loaded_data;
3353 
3354     /* extract RLE header */
3355     xh_flags = qemu_get_byte(f);
3356     xh_len = qemu_get_be16(f);
3357 
3358     if (xh_flags != ENCODING_FLAG_XBZRLE) {
3359         error_report("Failed to load XBZRLE page - wrong compression!");
3360         return -1;
3361     }
3362 
3363     if (xh_len > TARGET_PAGE_SIZE) {
3364         error_report("Failed to load XBZRLE page - len overflow!");
3365         return -1;
3366     }
3367     loaded_data = XBZRLE.decoded_buf;
3368     /* load data and decode */
3369     /* it can change loaded_data to point to an internal buffer */
3370     qemu_get_buffer_in_place(f, &loaded_data, xh_len);
3371 
3372     /* decode RLE */
3373     if (xbzrle_decode_buffer(loaded_data, xh_len, host,
3374                              TARGET_PAGE_SIZE) == -1) {
3375         error_report("Failed to load XBZRLE page - decode error!");
3376         return -1;
3377     }
3378 
3379     return 0;
3380 }
3381 
3382 /**
3383  * ram_block_from_stream: read a RAMBlock id from the migration stream
3384  *
3385  * Must be called from within a rcu critical section.
3386  *
3387  * Returns a pointer from within the RCU-protected ram_list.
3388  *
3389  * @mis: the migration incoming state pointer
3390  * @f: QEMUFile where to read the data from
3391  * @flags: Page flags (mostly to see if it's a continuation of previous block)
3392  * @channel: the channel we're using
3393  */
3394 static inline RAMBlock *ram_block_from_stream(MigrationIncomingState *mis,
3395                                               QEMUFile *f, int flags,
3396                                               int channel)
3397 {
3398     RAMBlock *block = mis->last_recv_block[channel];
3399     char id[256];
3400     uint8_t len;
3401 
3402     if (flags & RAM_SAVE_FLAG_CONTINUE) {
3403         if (!block) {
3404             error_report("Ack, bad migration stream!");
3405             return NULL;
3406         }
3407         return block;
3408     }
3409 
3410     len = qemu_get_byte(f);
3411     qemu_get_buffer(f, (uint8_t *)id, len);
3412     id[len] = 0;
3413 
3414     block = qemu_ram_block_by_name(id);
3415     if (!block) {
3416         error_report("Can't find block %s", id);
3417         return NULL;
3418     }
3419 
3420     if (migrate_ram_is_ignored(block)) {
3421         error_report("block %s should not be migrated !", id);
3422         return NULL;
3423     }
3424 
3425     mis->last_recv_block[channel] = block;
3426 
3427     return block;
3428 }
3429 
3430 static inline void *host_from_ram_block_offset(RAMBlock *block,
3431                                                ram_addr_t offset)
3432 {
3433     if (!offset_in_ramblock(block, offset)) {
3434         return NULL;
3435     }
3436 
3437     return block->host + offset;
3438 }
3439 
3440 static void *host_page_from_ram_block_offset(RAMBlock *block,
3441                                              ram_addr_t offset)
3442 {
3443     /* Note: Explicitly no check against offset_in_ramblock(). */
3444     return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset),
3445                                    block->page_size);
3446 }
3447 
3448 static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block,
3449                                                          ram_addr_t offset)
3450 {
3451     return ((uintptr_t)block->host + offset) & (block->page_size - 1);
3452 }
3453 
3454 void colo_record_bitmap(RAMBlock *block, ram_addr_t *normal, uint32_t pages)
3455 {
3456     qemu_mutex_lock(&ram_state->bitmap_mutex);
3457     for (int i = 0; i < pages; i++) {
3458         ram_addr_t offset = normal[i];
3459         ram_state->migration_dirty_pages += !test_and_set_bit(
3460                                                 offset >> TARGET_PAGE_BITS,
3461                                                 block->bmap);
3462     }
3463     qemu_mutex_unlock(&ram_state->bitmap_mutex);
3464 }
3465 
3466 static inline void *colo_cache_from_block_offset(RAMBlock *block,
3467                              ram_addr_t offset, bool record_bitmap)
3468 {
3469     if (!offset_in_ramblock(block, offset)) {
3470         return NULL;
3471     }
3472     if (!block->colo_cache) {
3473         error_report("%s: colo_cache is NULL in block :%s",
3474                      __func__, block->idstr);
3475         return NULL;
3476     }
3477 
3478     /*
3479     * During colo checkpoint, we need bitmap of these migrated pages.
3480     * It help us to decide which pages in ram cache should be flushed
3481     * into VM's RAM later.
3482     */
3483     if (record_bitmap) {
3484         colo_record_bitmap(block, &offset, 1);
3485     }
3486     return block->colo_cache + offset;
3487 }
3488 
3489 /**
3490  * ram_handle_zero: handle the zero page case
3491  *
3492  * If a page (or a whole RDMA chunk) has been
3493  * determined to be zero, then zap it.
3494  *
3495  * @host: host address for the zero page
3496  * @ch: what the page is filled from.  We only support zero
3497  * @size: size of the zero page
3498  */
3499 void ram_handle_zero(void *host, uint64_t size)
3500 {
3501     if (!buffer_is_zero(host, size)) {
3502         memset(host, 0, size);
3503     }
3504 }
3505 
3506 static void colo_init_ram_state(void)
3507 {
3508     Error *local_err = NULL;
3509 
3510     if (!ram_state_init(&ram_state, &local_err)) {
3511         error_report_err(local_err);
3512     }
3513 }
3514 
3515 /*
3516  * colo cache: this is for secondary VM, we cache the whole
3517  * memory of the secondary VM, it is need to hold the global lock
3518  * to call this helper.
3519  */
3520 int colo_init_ram_cache(void)
3521 {
3522     RAMBlock *block;
3523 
3524     WITH_RCU_READ_LOCK_GUARD() {
3525         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3526             block->colo_cache = qemu_anon_ram_alloc(block->used_length,
3527                                                     NULL, false, false);
3528             if (!block->colo_cache) {
3529                 error_report("%s: Can't alloc memory for COLO cache of block %s,"
3530                              "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
3531                              block->used_length);
3532                 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3533                     if (block->colo_cache) {
3534                         qemu_anon_ram_free(block->colo_cache, block->used_length);
3535                         block->colo_cache = NULL;
3536                     }
3537                 }
3538                 return -errno;
3539             }
3540             if (!machine_dump_guest_core(current_machine)) {
3541                 qemu_madvise(block->colo_cache, block->used_length,
3542                              QEMU_MADV_DONTDUMP);
3543             }
3544         }
3545     }
3546 
3547     /*
3548     * Record the dirty pages that sent by PVM, we use this dirty bitmap together
3549     * with to decide which page in cache should be flushed into SVM's RAM. Here
3550     * we use the same name 'ram_bitmap' as for migration.
3551     */
3552     if (ram_bytes_total()) {
3553         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3554             unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
3555             block->bmap = bitmap_new(pages);
3556         }
3557     }
3558 
3559     colo_init_ram_state();
3560     return 0;
3561 }
3562 
3563 /* TODO: duplicated with ram_init_bitmaps */
3564 void colo_incoming_start_dirty_log(void)
3565 {
3566     RAMBlock *block = NULL;
3567     Error *local_err = NULL;
3568 
3569     /* For memory_global_dirty_log_start below. */
3570     bql_lock();
3571     qemu_mutex_lock_ramlist();
3572 
3573     memory_global_dirty_log_sync(false);
3574     WITH_RCU_READ_LOCK_GUARD() {
3575         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3576             ramblock_sync_dirty_bitmap(ram_state, block);
3577             /* Discard this dirty bitmap record */
3578             bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS);
3579         }
3580         if (!memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION,
3581                                            &local_err)) {
3582             error_report_err(local_err);
3583         }
3584     }
3585     ram_state->migration_dirty_pages = 0;
3586     qemu_mutex_unlock_ramlist();
3587     bql_unlock();
3588 }
3589 
3590 /* It is need to hold the global lock to call this helper */
3591 void colo_release_ram_cache(void)
3592 {
3593     RAMBlock *block;
3594 
3595     memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
3596     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3597         g_free(block->bmap);
3598         block->bmap = NULL;
3599     }
3600 
3601     WITH_RCU_READ_LOCK_GUARD() {
3602         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3603             if (block->colo_cache) {
3604                 qemu_anon_ram_free(block->colo_cache, block->used_length);
3605                 block->colo_cache = NULL;
3606             }
3607         }
3608     }
3609     ram_state_cleanup(&ram_state);
3610 }
3611 
3612 /**
3613  * ram_load_setup: Setup RAM for migration incoming side
3614  *
3615  * Returns zero to indicate success and negative for error
3616  *
3617  * @f: QEMUFile where to receive the data
3618  * @opaque: RAMState pointer
3619  * @errp: pointer to Error*, to store an error if it happens.
3620  */
3621 static int ram_load_setup(QEMUFile *f, void *opaque, Error **errp)
3622 {
3623     xbzrle_load_setup();
3624     ramblock_recv_map_init();
3625 
3626     return 0;
3627 }
3628 
3629 static int ram_load_cleanup(void *opaque)
3630 {
3631     RAMBlock *rb;
3632 
3633     RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3634         qemu_ram_block_writeback(rb);
3635     }
3636 
3637     xbzrle_load_cleanup();
3638 
3639     RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
3640         g_free(rb->receivedmap);
3641         rb->receivedmap = NULL;
3642     }
3643 
3644     return 0;
3645 }
3646 
3647 /**
3648  * ram_postcopy_incoming_init: allocate postcopy data structures
3649  *
3650  * Returns 0 for success and negative if there was one error
3651  *
3652  * @mis: current migration incoming state
3653  *
3654  * Allocate data structures etc needed by incoming migration with
3655  * postcopy-ram. postcopy-ram's similarly names
3656  * postcopy_ram_incoming_init does the work.
3657  */
3658 int ram_postcopy_incoming_init(MigrationIncomingState *mis)
3659 {
3660     return postcopy_ram_incoming_init(mis);
3661 }
3662 
3663 /**
3664  * ram_load_postcopy: load a page in postcopy case
3665  *
3666  * Returns 0 for success or -errno in case of error
3667  *
3668  * Called in postcopy mode by ram_load().
3669  * rcu_read_lock is taken prior to this being called.
3670  *
3671  * @f: QEMUFile where to send the data
3672  * @channel: the channel to use for loading
3673  */
3674 int ram_load_postcopy(QEMUFile *f, int channel)
3675 {
3676     int flags = 0, ret = 0;
3677     bool place_needed = false;
3678     bool matches_target_page_size = false;
3679     MigrationIncomingState *mis = migration_incoming_get_current();
3680     PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel];
3681 
3682     while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
3683         ram_addr_t addr;
3684         void *page_buffer = NULL;
3685         void *place_source = NULL;
3686         RAMBlock *block = NULL;
3687         uint8_t ch;
3688 
3689         addr = qemu_get_be64(f);
3690 
3691         /*
3692          * If qemu file error, we should stop here, and then "addr"
3693          * may be invalid
3694          */
3695         ret = qemu_file_get_error(f);
3696         if (ret) {
3697             break;
3698         }
3699 
3700         flags = addr & ~TARGET_PAGE_MASK;
3701         addr &= TARGET_PAGE_MASK;
3702 
3703         trace_ram_load_postcopy_loop(channel, (uint64_t)addr, flags);
3704         if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE)) {
3705             block = ram_block_from_stream(mis, f, flags, channel);
3706             if (!block) {
3707                 ret = -EINVAL;
3708                 break;
3709             }
3710 
3711             /*
3712              * Relying on used_length is racy and can result in false positives.
3713              * We might place pages beyond used_length in case RAM was shrunk
3714              * while in postcopy, which is fine - trying to place via
3715              * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault.
3716              */
3717             if (!block->host || addr >= block->postcopy_length) {
3718                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
3719                 ret = -EINVAL;
3720                 break;
3721             }
3722             tmp_page->target_pages++;
3723             matches_target_page_size = block->page_size == TARGET_PAGE_SIZE;
3724             /*
3725              * Postcopy requires that we place whole host pages atomically;
3726              * these may be huge pages for RAMBlocks that are backed by
3727              * hugetlbfs.
3728              * To make it atomic, the data is read into a temporary page
3729              * that's moved into place later.
3730              * The migration protocol uses,  possibly smaller, target-pages
3731              * however the source ensures it always sends all the components
3732              * of a host page in one chunk.
3733              */
3734             page_buffer = tmp_page->tmp_huge_page +
3735                           host_page_offset_from_ram_block_offset(block, addr);
3736             /* If all TP are zero then we can optimise the place */
3737             if (tmp_page->target_pages == 1) {
3738                 tmp_page->host_addr =
3739                     host_page_from_ram_block_offset(block, addr);
3740             } else if (tmp_page->host_addr !=
3741                        host_page_from_ram_block_offset(block, addr)) {
3742                 /* not the 1st TP within the HP */
3743                 error_report("Non-same host page detected on channel %d: "
3744                              "Target host page %p, received host page %p "
3745                              "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)",
3746                              channel, tmp_page->host_addr,
3747                              host_page_from_ram_block_offset(block, addr),
3748                              block->idstr, addr, tmp_page->target_pages);
3749                 ret = -EINVAL;
3750                 break;
3751             }
3752 
3753             /*
3754              * If it's the last part of a host page then we place the host
3755              * page
3756              */
3757             if (tmp_page->target_pages ==
3758                 (block->page_size / TARGET_PAGE_SIZE)) {
3759                 place_needed = true;
3760             }
3761             place_source = tmp_page->tmp_huge_page;
3762         }
3763 
3764         switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
3765         case RAM_SAVE_FLAG_ZERO:
3766             ch = qemu_get_byte(f);
3767             if (ch != 0) {
3768                 error_report("Found a zero page with value %d", ch);
3769                 ret = -EINVAL;
3770                 break;
3771             }
3772             /*
3773              * Can skip to set page_buffer when
3774              * this is a zero page and (block->page_size == TARGET_PAGE_SIZE).
3775              */
3776             if (!matches_target_page_size) {
3777                 memset(page_buffer, ch, TARGET_PAGE_SIZE);
3778             }
3779             break;
3780 
3781         case RAM_SAVE_FLAG_PAGE:
3782             tmp_page->all_zero = false;
3783             if (!matches_target_page_size) {
3784                 /* For huge pages, we always use temporary buffer */
3785                 qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
3786             } else {
3787                 /*
3788                  * For small pages that matches target page size, we
3789                  * avoid the qemu_file copy.  Instead we directly use
3790                  * the buffer of QEMUFile to place the page.  Note: we
3791                  * cannot do any QEMUFile operation before using that
3792                  * buffer to make sure the buffer is valid when
3793                  * placing the page.
3794                  */
3795                 qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
3796                                          TARGET_PAGE_SIZE);
3797             }
3798             break;
3799         case RAM_SAVE_FLAG_MULTIFD_FLUSH:
3800             multifd_recv_sync_main();
3801             break;
3802         case RAM_SAVE_FLAG_EOS:
3803             /* normal exit */
3804             if (migrate_multifd() &&
3805                 migrate_multifd_flush_after_each_section()) {
3806                 multifd_recv_sync_main();
3807             }
3808             break;
3809         default:
3810             error_report("Unknown combination of migration flags: 0x%x"
3811                          " (postcopy mode)", flags);
3812             ret = -EINVAL;
3813             break;
3814         }
3815 
3816         /* Detect for any possible file errors */
3817         if (!ret && qemu_file_get_error(f)) {
3818             ret = qemu_file_get_error(f);
3819         }
3820 
3821         if (!ret && place_needed) {
3822             if (tmp_page->all_zero) {
3823                 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block);
3824             } else {
3825                 ret = postcopy_place_page(mis, tmp_page->host_addr,
3826                                           place_source, block);
3827             }
3828             place_needed = false;
3829             postcopy_temp_page_reset(tmp_page);
3830         }
3831     }
3832 
3833     return ret;
3834 }
3835 
3836 static bool postcopy_is_running(void)
3837 {
3838     PostcopyState ps = postcopy_state_get();
3839     return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
3840 }
3841 
3842 /*
3843  * Flush content of RAM cache into SVM's memory.
3844  * Only flush the pages that be dirtied by PVM or SVM or both.
3845  */
3846 void colo_flush_ram_cache(void)
3847 {
3848     RAMBlock *block = NULL;
3849     void *dst_host;
3850     void *src_host;
3851     unsigned long offset = 0;
3852 
3853     memory_global_dirty_log_sync(false);
3854     qemu_mutex_lock(&ram_state->bitmap_mutex);
3855     WITH_RCU_READ_LOCK_GUARD() {
3856         RAMBLOCK_FOREACH_NOT_IGNORED(block) {
3857             ramblock_sync_dirty_bitmap(ram_state, block);
3858         }
3859     }
3860 
3861     trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
3862     WITH_RCU_READ_LOCK_GUARD() {
3863         block = QLIST_FIRST_RCU(&ram_list.blocks);
3864 
3865         while (block) {
3866             unsigned long num = 0;
3867 
3868             offset = colo_bitmap_find_dirty(ram_state, block, offset, &num);
3869             if (!offset_in_ramblock(block,
3870                                     ((ram_addr_t)offset) << TARGET_PAGE_BITS)) {
3871                 offset = 0;
3872                 num = 0;
3873                 block = QLIST_NEXT_RCU(block, next);
3874             } else {
3875                 unsigned long i = 0;
3876 
3877                 for (i = 0; i < num; i++) {
3878                     migration_bitmap_clear_dirty(ram_state, block, offset + i);
3879                 }
3880                 dst_host = block->host
3881                          + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3882                 src_host = block->colo_cache
3883                          + (((ram_addr_t)offset) << TARGET_PAGE_BITS);
3884                 memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num);
3885                 offset += num;
3886             }
3887         }
3888     }
3889     qemu_mutex_unlock(&ram_state->bitmap_mutex);
3890     trace_colo_flush_ram_cache_end();
3891 }
3892 
3893 static size_t ram_load_multifd_pages(void *host_addr, size_t size,
3894                                      uint64_t offset)
3895 {
3896     MultiFDRecvData *data = multifd_get_recv_data();
3897 
3898     data->opaque = host_addr;
3899     data->file_offset = offset;
3900     data->size = size;
3901 
3902     if (!multifd_recv()) {
3903         return 0;
3904     }
3905 
3906     return size;
3907 }
3908 
3909 static bool read_ramblock_mapped_ram(QEMUFile *f, RAMBlock *block,
3910                                      long num_pages, unsigned long *bitmap,
3911                                      Error **errp)
3912 {
3913     ERRP_GUARD();
3914     unsigned long set_bit_idx, clear_bit_idx;
3915     ram_addr_t offset;
3916     void *host;
3917     size_t read, unread, size;
3918 
3919     for (set_bit_idx = find_first_bit(bitmap, num_pages);
3920          set_bit_idx < num_pages;
3921          set_bit_idx = find_next_bit(bitmap, num_pages, clear_bit_idx + 1)) {
3922 
3923         clear_bit_idx = find_next_zero_bit(bitmap, num_pages, set_bit_idx + 1);
3924 
3925         unread = TARGET_PAGE_SIZE * (clear_bit_idx - set_bit_idx);
3926         offset = set_bit_idx << TARGET_PAGE_BITS;
3927 
3928         while (unread > 0) {
3929             host = host_from_ram_block_offset(block, offset);
3930             if (!host) {
3931                 error_setg(errp, "page outside of ramblock %s range",
3932                            block->idstr);
3933                 return false;
3934             }
3935 
3936             size = MIN(unread, MAPPED_RAM_LOAD_BUF_SIZE);
3937 
3938             if (migrate_multifd()) {
3939                 read = ram_load_multifd_pages(host, size,
3940                                               block->pages_offset + offset);
3941             } else {
3942                 read = qemu_get_buffer_at(f, host, size,
3943                                           block->pages_offset + offset);
3944             }
3945 
3946             if (!read) {
3947                 goto err;
3948             }
3949             offset += read;
3950             unread -= read;
3951         }
3952     }
3953 
3954     return true;
3955 
3956 err:
3957     qemu_file_get_error_obj(f, errp);
3958     error_prepend(errp, "(%s) failed to read page " RAM_ADDR_FMT
3959                   "from file offset %" PRIx64 ": ", block->idstr, offset,
3960                   block->pages_offset + offset);
3961     return false;
3962 }
3963 
3964 static void parse_ramblock_mapped_ram(QEMUFile *f, RAMBlock *block,
3965                                       ram_addr_t length, Error **errp)
3966 {
3967     g_autofree unsigned long *bitmap = NULL;
3968     MappedRamHeader header;
3969     size_t bitmap_size;
3970     long num_pages;
3971 
3972     if (!mapped_ram_read_header(f, &header, errp)) {
3973         return;
3974     }
3975 
3976     block->pages_offset = header.pages_offset;
3977 
3978     /*
3979      * Check the alignment of the file region that contains pages. We
3980      * don't enforce MAPPED_RAM_FILE_OFFSET_ALIGNMENT to allow that
3981      * value to change in the future. Do only a sanity check with page
3982      * size alignment.
3983      */
3984     if (!QEMU_IS_ALIGNED(block->pages_offset, TARGET_PAGE_SIZE)) {
3985         error_setg(errp,
3986                    "Error reading ramblock %s pages, region has bad alignment",
3987                    block->idstr);
3988         return;
3989     }
3990 
3991     num_pages = length / header.page_size;
3992     bitmap_size = BITS_TO_LONGS(num_pages) * sizeof(unsigned long);
3993 
3994     bitmap = g_malloc0(bitmap_size);
3995     if (qemu_get_buffer_at(f, (uint8_t *)bitmap, bitmap_size,
3996                            header.bitmap_offset) != bitmap_size) {
3997         error_setg(errp, "Error reading dirty bitmap");
3998         return;
3999     }
4000 
4001     if (!read_ramblock_mapped_ram(f, block, num_pages, bitmap, errp)) {
4002         return;
4003     }
4004 
4005     /* Skip pages array */
4006     qemu_set_offset(f, block->pages_offset + length, SEEK_SET);
4007 
4008     return;
4009 }
4010 
4011 static int parse_ramblock(QEMUFile *f, RAMBlock *block, ram_addr_t length)
4012 {
4013     int ret = 0;
4014     /* ADVISE is earlier, it shows the source has the postcopy capability on */
4015     bool postcopy_advised = migration_incoming_postcopy_advised();
4016     int max_hg_page_size;
4017     Error *local_err = NULL;
4018 
4019     assert(block);
4020 
4021     if (migrate_mapped_ram()) {
4022         parse_ramblock_mapped_ram(f, block, length, &local_err);
4023         if (local_err) {
4024             error_report_err(local_err);
4025             return -EINVAL;
4026         }
4027         return 0;
4028     }
4029 
4030     if (!qemu_ram_is_migratable(block)) {
4031         error_report("block %s should not be migrated !", block->idstr);
4032         return -EINVAL;
4033     }
4034 
4035     if (length != block->used_length) {
4036         ret = qemu_ram_resize(block, length, &local_err);
4037         if (local_err) {
4038             error_report_err(local_err);
4039             return ret;
4040         }
4041     }
4042 
4043     /*
4044      * ??? Mirrors the previous value of qemu_host_page_size,
4045      * but is this really what was intended for the migration?
4046      */
4047     max_hg_page_size = MAX(qemu_real_host_page_size(), TARGET_PAGE_SIZE);
4048 
4049     /* For postcopy we need to check hugepage sizes match */
4050     if (postcopy_advised && migrate_postcopy_ram() &&
4051         block->page_size != max_hg_page_size) {
4052         uint64_t remote_page_size = qemu_get_be64(f);
4053         if (remote_page_size != block->page_size) {
4054             error_report("Mismatched RAM page size %s "
4055                          "(local) %zd != %" PRId64, block->idstr,
4056                          block->page_size, remote_page_size);
4057             return -EINVAL;
4058         }
4059     }
4060     if (migrate_ignore_shared()) {
4061         hwaddr addr = qemu_get_be64(f);
4062         if (migrate_ram_is_ignored(block) &&
4063             block->mr->addr != addr) {
4064             error_report("Mismatched GPAs for block %s "
4065                          "%" PRId64 "!= %" PRId64, block->idstr,
4066                          (uint64_t)addr, (uint64_t)block->mr->addr);
4067             return -EINVAL;
4068         }
4069     }
4070     ret = rdma_block_notification_handle(f, block->idstr);
4071     if (ret < 0) {
4072         qemu_file_set_error(f, ret);
4073     }
4074 
4075     return ret;
4076 }
4077 
4078 static int parse_ramblocks(QEMUFile *f, ram_addr_t total_ram_bytes)
4079 {
4080     int ret = 0;
4081 
4082     /* Synchronize RAM block list */
4083     while (!ret && total_ram_bytes) {
4084         RAMBlock *block;
4085         char id[256];
4086         ram_addr_t length;
4087         int len = qemu_get_byte(f);
4088 
4089         qemu_get_buffer(f, (uint8_t *)id, len);
4090         id[len] = 0;
4091         length = qemu_get_be64(f);
4092 
4093         block = qemu_ram_block_by_name(id);
4094         if (block) {
4095             ret = parse_ramblock(f, block, length);
4096         } else {
4097             error_report("Unknown ramblock \"%s\", cannot accept "
4098                          "migration", id);
4099             ret = -EINVAL;
4100         }
4101         total_ram_bytes -= length;
4102     }
4103 
4104     return ret;
4105 }
4106 
4107 /**
4108  * ram_load_precopy: load pages in precopy case
4109  *
4110  * Returns 0 for success or -errno in case of error
4111  *
4112  * Called in precopy mode by ram_load().
4113  * rcu_read_lock is taken prior to this being called.
4114  *
4115  * @f: QEMUFile where to send the data
4116  */
4117 static int ram_load_precopy(QEMUFile *f)
4118 {
4119     MigrationIncomingState *mis = migration_incoming_get_current();
4120     int flags = 0, ret = 0, invalid_flags = 0, i = 0;
4121 
4122     if (migrate_mapped_ram()) {
4123         invalid_flags |= (RAM_SAVE_FLAG_HOOK | RAM_SAVE_FLAG_MULTIFD_FLUSH |
4124                           RAM_SAVE_FLAG_PAGE | RAM_SAVE_FLAG_XBZRLE |
4125                           RAM_SAVE_FLAG_ZERO);
4126     }
4127 
4128     while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
4129         ram_addr_t addr;
4130         void *host = NULL, *host_bak = NULL;
4131         uint8_t ch;
4132 
4133         /*
4134          * Yield periodically to let main loop run, but an iteration of
4135          * the main loop is expensive, so do it each some iterations
4136          */
4137         if ((i & 32767) == 0 && qemu_in_coroutine()) {
4138             aio_co_schedule(qemu_get_current_aio_context(),
4139                             qemu_coroutine_self());
4140             qemu_coroutine_yield();
4141         }
4142         i++;
4143 
4144         addr = qemu_get_be64(f);
4145         ret = qemu_file_get_error(f);
4146         if (ret) {
4147             error_report("Getting RAM address failed");
4148             break;
4149         }
4150 
4151         flags = addr & ~TARGET_PAGE_MASK;
4152         addr &= TARGET_PAGE_MASK;
4153 
4154         if (flags & invalid_flags) {
4155             error_report("Unexpected RAM flags: %d", flags & invalid_flags);
4156 
4157             ret = -EINVAL;
4158             break;
4159         }
4160 
4161         if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE |
4162                      RAM_SAVE_FLAG_XBZRLE)) {
4163             RAMBlock *block = ram_block_from_stream(mis, f, flags,
4164                                                     RAM_CHANNEL_PRECOPY);
4165 
4166             host = host_from_ram_block_offset(block, addr);
4167             /*
4168              * After going into COLO stage, we should not load the page
4169              * into SVM's memory directly, we put them into colo_cache firstly.
4170              * NOTE: We need to keep a copy of SVM's ram in colo_cache.
4171              * Previously, we copied all these memory in preparing stage of COLO
4172              * while we need to stop VM, which is a time-consuming process.
4173              * Here we optimize it by a trick, back-up every page while in
4174              * migration process while COLO is enabled, though it affects the
4175              * speed of the migration, but it obviously reduce the downtime of
4176              * back-up all SVM'S memory in COLO preparing stage.
4177              */
4178             if (migration_incoming_colo_enabled()) {
4179                 if (migration_incoming_in_colo_state()) {
4180                     /* In COLO stage, put all pages into cache temporarily */
4181                     host = colo_cache_from_block_offset(block, addr, true);
4182                 } else {
4183                    /*
4184                     * In migration stage but before COLO stage,
4185                     * Put all pages into both cache and SVM's memory.
4186                     */
4187                     host_bak = colo_cache_from_block_offset(block, addr, false);
4188                 }
4189             }
4190             if (!host) {
4191                 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
4192                 ret = -EINVAL;
4193                 break;
4194             }
4195             if (!migration_incoming_in_colo_state()) {
4196                 ramblock_recv_bitmap_set(block, host);
4197             }
4198 
4199             trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
4200         }
4201 
4202         switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
4203         case RAM_SAVE_FLAG_MEM_SIZE:
4204             ret = parse_ramblocks(f, addr);
4205             /*
4206              * For mapped-ram migration (to a file) using multifd, we sync
4207              * once and for all here to make sure all tasks we queued to
4208              * multifd threads are completed, so that all the ramblocks
4209              * (including all the guest memory pages within) are fully
4210              * loaded after this sync returns.
4211              */
4212             if (migrate_mapped_ram()) {
4213                 multifd_recv_sync_main();
4214             }
4215             break;
4216 
4217         case RAM_SAVE_FLAG_ZERO:
4218             ch = qemu_get_byte(f);
4219             if (ch != 0) {
4220                 error_report("Found a zero page with value %d", ch);
4221                 ret = -EINVAL;
4222                 break;
4223             }
4224             ram_handle_zero(host, TARGET_PAGE_SIZE);
4225             break;
4226 
4227         case RAM_SAVE_FLAG_PAGE:
4228             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
4229             break;
4230 
4231         case RAM_SAVE_FLAG_XBZRLE:
4232             if (load_xbzrle(f, addr, host) < 0) {
4233                 error_report("Failed to decompress XBZRLE page at "
4234                              RAM_ADDR_FMT, addr);
4235                 ret = -EINVAL;
4236                 break;
4237             }
4238             break;
4239         case RAM_SAVE_FLAG_MULTIFD_FLUSH:
4240             multifd_recv_sync_main();
4241             break;
4242         case RAM_SAVE_FLAG_EOS:
4243             /* normal exit */
4244             if (migrate_multifd() &&
4245                 migrate_multifd_flush_after_each_section() &&
4246                 /*
4247                  * Mapped-ram migration flushes once and for all after
4248                  * parsing ramblocks. Always ignore EOS for it.
4249                  */
4250                 !migrate_mapped_ram()) {
4251                 multifd_recv_sync_main();
4252             }
4253             break;
4254         case RAM_SAVE_FLAG_HOOK:
4255             ret = rdma_registration_handle(f);
4256             if (ret < 0) {
4257                 qemu_file_set_error(f, ret);
4258             }
4259             break;
4260         default:
4261             error_report("Unknown combination of migration flags: 0x%x", flags);
4262             ret = -EINVAL;
4263         }
4264         if (!ret) {
4265             ret = qemu_file_get_error(f);
4266         }
4267         if (!ret && host_bak) {
4268             memcpy(host_bak, host, TARGET_PAGE_SIZE);
4269         }
4270     }
4271 
4272     return ret;
4273 }
4274 
4275 static int ram_load(QEMUFile *f, void *opaque, int version_id)
4276 {
4277     int ret = 0;
4278     static uint64_t seq_iter;
4279     /*
4280      * If system is running in postcopy mode, page inserts to host memory must
4281      * be atomic
4282      */
4283     bool postcopy_running = postcopy_is_running();
4284 
4285     seq_iter++;
4286 
4287     if (version_id != 4) {
4288         return -EINVAL;
4289     }
4290 
4291     /*
4292      * This RCU critical section can be very long running.
4293      * When RCU reclaims in the code start to become numerous,
4294      * it will be necessary to reduce the granularity of this
4295      * critical section.
4296      */
4297     trace_ram_load_start();
4298     WITH_RCU_READ_LOCK_GUARD() {
4299         if (postcopy_running) {
4300             /*
4301              * Note!  Here RAM_CHANNEL_PRECOPY is the precopy channel of
4302              * postcopy migration, we have another RAM_CHANNEL_POSTCOPY to
4303              * service fast page faults.
4304              */
4305             ret = ram_load_postcopy(f, RAM_CHANNEL_PRECOPY);
4306         } else {
4307             ret = ram_load_precopy(f);
4308         }
4309     }
4310     trace_ram_load_complete(ret, seq_iter);
4311 
4312     return ret;
4313 }
4314 
4315 static bool ram_has_postcopy(void *opaque)
4316 {
4317     RAMBlock *rb;
4318     RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
4319         if (ramblock_is_pmem(rb)) {
4320             info_report("Block: %s, host: %p is a nvdimm memory, postcopy"
4321                          "is not supported now!", rb->idstr, rb->host);
4322             return false;
4323         }
4324     }
4325 
4326     return migrate_postcopy_ram();
4327 }
4328 
4329 /* Sync all the dirty bitmap with destination VM.  */
4330 static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs)
4331 {
4332     RAMBlock *block;
4333     QEMUFile *file = s->to_dst_file;
4334 
4335     trace_ram_dirty_bitmap_sync_start();
4336 
4337     qatomic_set(&rs->postcopy_bmap_sync_requested, 0);
4338     RAMBLOCK_FOREACH_NOT_IGNORED(block) {
4339         qemu_savevm_send_recv_bitmap(file, block->idstr);
4340         trace_ram_dirty_bitmap_request(block->idstr);
4341         qatomic_inc(&rs->postcopy_bmap_sync_requested);
4342     }
4343 
4344     trace_ram_dirty_bitmap_sync_wait();
4345 
4346     /* Wait until all the ramblocks' dirty bitmap synced */
4347     while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
4348         if (migration_rp_wait(s)) {
4349             return -1;
4350         }
4351     }
4352 
4353     trace_ram_dirty_bitmap_sync_complete();
4354 
4355     return 0;
4356 }
4357 
4358 /*
4359  * Read the received bitmap, revert it as the initial dirty bitmap.
4360  * This is only used when the postcopy migration is paused but wants
4361  * to resume from a middle point.
4362  *
4363  * Returns true if succeeded, false for errors.
4364  */
4365 bool ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block, Error **errp)
4366 {
4367     /* from_dst_file is always valid because we're within rp_thread */
4368     QEMUFile *file = s->rp_state.from_dst_file;
4369     g_autofree unsigned long *le_bitmap = NULL;
4370     unsigned long nbits = block->used_length >> TARGET_PAGE_BITS;
4371     uint64_t local_size = DIV_ROUND_UP(nbits, 8);
4372     uint64_t size, end_mark;
4373     RAMState *rs = ram_state;
4374 
4375     trace_ram_dirty_bitmap_reload_begin(block->idstr);
4376 
4377     if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
4378         error_setg(errp, "Reload bitmap in incorrect state %s",
4379                    MigrationStatus_str(s->state));
4380         return false;
4381     }
4382 
4383     /*
4384      * Note: see comments in ramblock_recv_bitmap_send() on why we
4385      * need the endianness conversion, and the paddings.
4386      */
4387     local_size = ROUND_UP(local_size, 8);
4388 
4389     /* Add paddings */
4390     le_bitmap = bitmap_new(nbits + BITS_PER_LONG);
4391 
4392     size = qemu_get_be64(file);
4393 
4394     /* The size of the bitmap should match with our ramblock */
4395     if (size != local_size) {
4396         error_setg(errp, "ramblock '%s' bitmap size mismatch (0x%"PRIx64
4397                    " != 0x%"PRIx64")", block->idstr, size, local_size);
4398         return false;
4399     }
4400 
4401     size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size);
4402     end_mark = qemu_get_be64(file);
4403 
4404     if (qemu_file_get_error(file) || size != local_size) {
4405         error_setg(errp, "read bitmap failed for ramblock '%s': "
4406                    "(size 0x%"PRIx64", got: 0x%"PRIx64")",
4407                    block->idstr, local_size, size);
4408         return false;
4409     }
4410 
4411     if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) {
4412         error_setg(errp, "ramblock '%s' end mark incorrect: 0x%"PRIx64,
4413                    block->idstr, end_mark);
4414         return false;
4415     }
4416 
4417     /*
4418      * Endianness conversion. We are during postcopy (though paused).
4419      * The dirty bitmap won't change. We can directly modify it.
4420      */
4421     bitmap_from_le(block->bmap, le_bitmap, nbits);
4422 
4423     /*
4424      * What we received is "received bitmap". Revert it as the initial
4425      * dirty bitmap for this ramblock.
4426      */
4427     bitmap_complement(block->bmap, block->bmap, nbits);
4428 
4429     /* Clear dirty bits of discarded ranges that we don't want to migrate. */
4430     ramblock_dirty_bitmap_clear_discarded_pages(block);
4431 
4432     /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */
4433     trace_ram_dirty_bitmap_reload_complete(block->idstr);
4434 
4435     qatomic_dec(&rs->postcopy_bmap_sync_requested);
4436 
4437     /*
4438      * We succeeded to sync bitmap for current ramblock. Always kick the
4439      * migration thread to check whether all requested bitmaps are
4440      * reloaded.  NOTE: it's racy to only kick when requested==0, because
4441      * we don't know whether the migration thread may still be increasing
4442      * it.
4443      */
4444     migration_rp_kick(s);
4445 
4446     return true;
4447 }
4448 
4449 static int ram_resume_prepare(MigrationState *s, void *opaque)
4450 {
4451     RAMState *rs = *(RAMState **)opaque;
4452     int ret;
4453 
4454     ret = ram_dirty_bitmap_sync_all(s, rs);
4455     if (ret) {
4456         return ret;
4457     }
4458 
4459     ram_state_resume_prepare(rs, s->to_dst_file);
4460 
4461     return 0;
4462 }
4463 
4464 void postcopy_preempt_shutdown_file(MigrationState *s)
4465 {
4466     qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS);
4467     qemu_fflush(s->postcopy_qemufile_src);
4468 }
4469 
4470 static SaveVMHandlers savevm_ram_handlers = {
4471     .save_setup = ram_save_setup,
4472     .save_live_iterate = ram_save_iterate,
4473     .save_live_complete_postcopy = ram_save_complete,
4474     .save_live_complete_precopy = ram_save_complete,
4475     .has_postcopy = ram_has_postcopy,
4476     .state_pending_exact = ram_state_pending_exact,
4477     .state_pending_estimate = ram_state_pending_estimate,
4478     .load_state = ram_load,
4479     .save_cleanup = ram_save_cleanup,
4480     .load_setup = ram_load_setup,
4481     .load_cleanup = ram_load_cleanup,
4482     .resume_prepare = ram_resume_prepare,
4483 };
4484 
4485 static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host,
4486                                       size_t old_size, size_t new_size)
4487 {
4488     PostcopyState ps = postcopy_state_get();
4489     ram_addr_t offset;
4490     RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset);
4491     Error *err = NULL;
4492 
4493     if (!rb) {
4494         error_report("RAM block not found");
4495         return;
4496     }
4497 
4498     if (migrate_ram_is_ignored(rb)) {
4499         return;
4500     }
4501 
4502     if (migration_is_running()) {
4503         /*
4504          * Precopy code on the source cannot deal with the size of RAM blocks
4505          * changing at random points in time - especially after sending the
4506          * RAM block sizes in the migration stream, they must no longer change.
4507          * Abort and indicate a proper reason.
4508          */
4509         error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr);
4510         migration_cancel(err);
4511         error_free(err);
4512     }
4513 
4514     switch (ps) {
4515     case POSTCOPY_INCOMING_ADVISE:
4516         /*
4517          * Update what ram_postcopy_incoming_init()->init_range() does at the
4518          * time postcopy was advised. Syncing RAM blocks with the source will
4519          * result in RAM resizes.
4520          */
4521         if (old_size < new_size) {
4522             if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) {
4523                 error_report("RAM block '%s' discard of resized RAM failed",
4524                              rb->idstr);
4525             }
4526         }
4527         rb->postcopy_length = new_size;
4528         break;
4529     case POSTCOPY_INCOMING_NONE:
4530     case POSTCOPY_INCOMING_RUNNING:
4531     case POSTCOPY_INCOMING_END:
4532         /*
4533          * Once our guest is running, postcopy does no longer care about
4534          * resizes. When growing, the new memory was not available on the
4535          * source, no handler needed.
4536          */
4537         break;
4538     default:
4539         error_report("RAM block '%s' resized during postcopy state: %d",
4540                      rb->idstr, ps);
4541         exit(-1);
4542     }
4543 }
4544 
4545 static RAMBlockNotifier ram_mig_ram_notifier = {
4546     .ram_block_resized = ram_mig_ram_block_resized,
4547 };
4548 
4549 void ram_mig_init(void)
4550 {
4551     qemu_mutex_init(&XBZRLE.lock);
4552     register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state);
4553     ram_block_notifier_add(&ram_mig_ram_notifier);
4554 }
4555