Lines Matching +full:page +full:- +full:offset
4 * Copyright (c) 2003-2008 Fabrice Bellard
5 * Copyright (c) 2011-2015 Red Hat Inc
34 #include "qemu/main-loop.h"
38 #include "migration-stats.h"
41 #include "qemu-file.h"
42 #include "postcopy-ram.h"
44 #include "qemu/error-report.h"
46 #include "qapi/qapi-types-migration.h"
47 #include "qapi/qapi-events-migration.h"
48 #include "qapi/qapi-commands-migration.h"
55 #include "sysemu/cpu-throttle.h"
96 * mapped-ram migration supports O_DIRECT, so we need to make sure the
97 * userspace buffer, the IO operation size and the file offset are
99 * two are already aligned to page size, but we need to add padding to
100 * the file to align the offset. We cannot read the block size
103 * the file offset aligned at page size as well.
108 * When doing mapped-ram migration, this is the amount we read from
117 /* The migration channel used for a specific host page */
123 /* Current page to search from */
124 unsigned long page; member
127 /* Whether we're sending a host page */
129 /* The start/end of current host page. Invalid if host_page_sending==false */
135 /* struct contains XBZRLE cache and a static page
140 /* buffer for storing page content */
145 /* it will store a page full of zeros */
173 * Returns 0 for success or -1 for error
187 return -1; in xbzrle_cache_resize()
200 ret = -1; in xbzrle_cache_resize()
247 assert(!rb->receivedmap); in ramblock_recv_map_init()
248 rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits()); in ramblock_recv_map_init()
255 rb->receivedmap); in ramblock_recv_bitmap_test()
260 return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap); in ramblock_recv_bitmap_test_byte_offset()
265 set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap); in ramblock_recv_bitmap_set()
271 bitmap_set_atomic(rb->receivedmap, in ramblock_recv_bitmap_set_range()
278 set_bit_atomic(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap); in ramblock_recv_bitmap_set_offset()
296 return -1; in ramblock_recv_bitmap_send()
299 nbits = block->postcopy_length >> TARGET_PAGE_BITS; in ramblock_recv_bitmap_send()
313 bitmap_to_le(le_bitmap, block->receivedmap, nbits); in ramblock_recv_bitmap_send()
343 * An outstanding page request, on the source, having been received
348 hwaddr offset; member
361 /* UFFD file descriptor, used in 'write-tracking' migration */
367 /* Last dirty target page we have sent */
399 * - dirty/clear bitmap
400 * - migration_dirty_pages
401 * - pss structures
406 /* Queue of outstanding page requests from the destination */
427 return !QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests); in postcopy_has_request()
455 return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) : in ram_bytes_remaining()
479 /* NOTE: page is the PFN not real ram_addr_t. */
480 static void pss_init(PageSearchStatus *pss, RAMBlock *rb, ram_addr_t page) in pss_init() argument
482 pss->block = rb; in pss_init()
483 pss->page = page; in pss_init()
484 pss->complete_round = false; in pss_init()
488 * Check whether two PSSs are actively sending the same page. Return true
493 return pss1->host_page_sending && pss2->host_page_sending && in pss_overlap()
494 (pss1->host_page_start == pss2->host_page_start); in pss_overlap()
498 * save_page_header: write page header to wire
505 * @block: block that contains the page we want to send
506 * @offset: offset inside the block for the page
510 RAMBlock *block, ram_addr_t offset) in save_page_header() argument
513 bool same_block = (block == pss->last_sent_block); in save_page_header()
516 offset |= RAM_SAVE_FLAG_CONTINUE; in save_page_header()
518 qemu_put_be64(f, offset); in save_page_header()
522 len = strlen(block->idstr); in save_page_header()
524 qemu_put_buffer(f, (uint8_t *)block->idstr, len); in save_page_header()
526 pss->last_sent_block = block; in save_page_header()
538 * fast and will not effectively converge, even with auto-converge.
561 cpu_now = 100 - throttle_now; in mig_throttle_guest_down()
564 throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment); in mig_throttle_guest_down()
574 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); in mig_throttle_counter_reset()
575 rs->num_dirty_pages_period = 0; in mig_throttle_counter_reset()
576 rs->bytes_xfer_prev = migration_transferred_bytes(); in mig_throttle_counter_reset()
580 * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache
582 * @current_addr: address for the zero page
584 * Update the xbzrle cache to reflect a page that's been sent as all 0.
585 * The important thing is that a stale (not-yet-0'd) page be replaced
587 * As a bonus, if the page wasn't in the cache it gets added so that
588 * when a small write is made into the 0'd page it gets XBZRLE sent.
592 /* We don't care if this fails to allocate a new cache page in xbzrle_cache_zero_page()
601 * save_xbzrle_page: compress and send current page
603 * Returns: 1 means that we wrote the page
604 * 0 means that page is identical to the one already sent
605 * -1 means that xbzrle would be longer than normal
609 * @current_data: pointer to the address of the page contents
610 * @current_addr: addr of the page
611 * @block: block that contains the page we want to send
612 * @offset: offset inside the block for the page
616 RAMBlock *block, ram_addr_t offset) in save_xbzrle_page() argument
620 QEMUFile *file = pss->pss_channel; in save_xbzrle_page()
625 if (!rs->last_stage) { in save_xbzrle_page()
627 generation) == -1) { in save_xbzrle_page()
628 return -1; in save_xbzrle_page()
630 /* update *current_data when the page has been in save_xbzrle_page()
635 return -1; in save_xbzrle_page()
639 * Reaching here means the page has hit the xbzrle cache, no matter what in save_xbzrle_page()
640 * encoding result it is (normal encoding, overflow or skipping the page), in save_xbzrle_page()
641 * count the page as encoded. This is used to calculate the encoding rate. in save_xbzrle_page()
643 * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB, in save_xbzrle_page()
644 * 2nd page turns out to be skipped (i.e. no new bytes written to the in save_xbzrle_page()
645 * page), the overall encoding rate will be 8KB / 2KB = 4, which has the in save_xbzrle_page()
646 * skipped page included. In this way, the encoding rate can tell if the in save_xbzrle_page()
647 * guest page is good for xbzrle encoding. in save_xbzrle_page()
662 * sent, in all cases except where we skip the page. in save_xbzrle_page()
664 if (!rs->last_stage && encoded_len != 0) { in save_xbzrle_page()
677 } else if (encoded_len == -1) { in save_xbzrle_page()
681 return -1; in save_xbzrle_page()
684 /* Send XBZRLE based compressed page */ in save_xbzrle_page()
685 bytes_xbzrle = save_page_header(pss, pss->pss_channel, block, in save_xbzrle_page()
686 offset | RAM_SAVE_FLAG_XBZRLE); in save_xbzrle_page()
695 xbzrle_counters.bytes += bytes_xbzrle - 8; in save_xbzrle_page()
702 * pss_find_next_dirty: find the next dirty page of current ramblock
704 * This function updates pss->page to point to the next dirty page index
706 * found. Note that when pss->host_page_sending==true it means we're
707 * during sending a host page, so we won't look for dirty page that is
708 * outside the host page boundary.
710 * @pss: the current page search status
714 RAMBlock *rb = pss->block; in pss_find_next_dirty()
715 unsigned long size = rb->used_length >> TARGET_PAGE_BITS; in pss_find_next_dirty()
716 unsigned long *bitmap = rb->bmap; in pss_find_next_dirty()
719 /* Points directly to the end, so we know no dirty page */ in pss_find_next_dirty()
720 pss->page = size; in pss_find_next_dirty()
725 * If during sending a host page, only look for dirty pages within the in pss_find_next_dirty()
726 * current host page being send. in pss_find_next_dirty()
728 if (pss->host_page_sending) { in pss_find_next_dirty()
729 assert(pss->host_page_end); in pss_find_next_dirty()
730 size = MIN(size, pss->host_page_end); in pss_find_next_dirty()
733 pss->page = find_next_bit(bitmap, size, pss->page); in pss_find_next_dirty()
737 unsigned long page) in migration_clear_memory_region_dirty_bitmap() argument
742 if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) { in migration_clear_memory_region_dirty_bitmap()
746 shift = rb->clear_bmap_shift; in migration_clear_memory_region_dirty_bitmap()
758 start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size); in migration_clear_memory_region_dirty_bitmap()
759 trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page); in migration_clear_memory_region_dirty_bitmap()
760 memory_region_clear_dirty_bitmap(rb->mr, start, size); in migration_clear_memory_region_dirty_bitmap()
768 unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift; in migration_clear_memory_region_dirty_bitmap_range()
773 * Clear pages from start to start + npages - 1, so the end boundary is in migration_clear_memory_region_dirty_bitmap_range()
784 * Returns the page offset within memory region of the start of the contiguout
785 * dirty page
789 * @start: page where we start the search
796 unsigned long size = rb->used_length >> TARGET_PAGE_BITS; in colo_bitmap_find_dirty()
797 unsigned long *bitmap = rb->bmap; in colo_bitmap_find_dirty()
812 *num = next - first; in colo_bitmap_find_dirty()
818 unsigned long page) in migration_bitmap_clear_dirty() argument
824 * send any of the page in the chunk because we need to make sure in migration_bitmap_clear_dirty()
825 * we can capture further page content changes when we sync dirty in migration_bitmap_clear_dirty()
827 * the page in the chunk we clear the remote dirty bitmap for all. in migration_bitmap_clear_dirty()
830 migration_clear_memory_region_dirty_bitmap(rb, page); in migration_bitmap_clear_dirty()
832 ret = test_and_clear_bit(page, rb->bmap); in migration_bitmap_clear_dirty()
834 rs->migration_dirty_pages--; in migration_bitmap_clear_dirty()
843 const hwaddr offset = section->offset_within_region; in dirty_bitmap_clear_section() local
844 const hwaddr size = int128_get64(section->size); in dirty_bitmap_clear_section()
845 const unsigned long start = offset >> TARGET_PAGE_BITS; in dirty_bitmap_clear_section()
847 RAMBlock *rb = section->mr->ram_block; in dirty_bitmap_clear_section()
851 * We don't grab ram_state->bitmap_mutex because we expect to run in dirty_bitmap_clear_section()
858 *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages); in dirty_bitmap_clear_section()
859 bitmap_clear(rb->bmap, start, npages); in dirty_bitmap_clear_section()
879 if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) { in ramblock_dirty_bitmap_clear_discarded_pages()
880 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); in ramblock_dirty_bitmap_clear_discarded_pages()
882 .mr = rb->mr, in ramblock_dirty_bitmap_clear_discarded_pages()
895 * Check if a host-page aligned page falls into a discarded range as managed by
902 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) { in ramblock_page_is_discarded()
903 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); in ramblock_page_is_discarded()
905 .mr = rb->mr, in ramblock_page_is_discarded()
919 cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); in ramblock_sync_dirty_bitmap()
921 rs->migration_dirty_pages += new_dirty_pages; in ramblock_sync_dirty_bitmap()
922 rs->num_dirty_pages_period += new_dirty_pages; in ramblock_sync_dirty_bitmap()
928 * Returns a summary bitmap of the page sizes of all RAMBlocks
930 * For VMs with just normal pages this is equivalent to the host page
932 * different page sizes.
940 summary |= block->page_size; in ram_pagesize_summary()
955 uint64_t page_count = rs->target_page_count - rs->target_page_count_prev; in migration_update_rates()
959 rs->num_dirty_pages_period * 1000 / in migration_update_rates()
960 (end_time - rs->time_last_bitmap_sync)); in migration_update_rates()
969 xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss - in migration_update_rates()
970 rs->xbzrle_cache_miss_prev) / page_count; in migration_update_rates()
971 rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss; in migration_update_rates()
972 unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) * in migration_update_rates()
974 encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev; in migration_update_rates()
975 if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) { in migration_update_rates()
980 rs->xbzrle_pages_prev = xbzrle_counters.pages; in migration_update_rates()
981 rs->xbzrle_bytes_prev = xbzrle_counters.bytes; in migration_update_rates()
986 * Enable dirty-limit to throttle down the guest
991 * dirty page rate quota for all vCPUs fetched from in migration_dirty_limit_guest()
999 * vcpu-dirty-limit untouched. in migration_dirty_limit_guest()
1002 quota_dirtyrate == s->parameters.vcpu_dirty_limit) { in migration_dirty_limit_guest()
1006 quota_dirtyrate = s->parameters.vcpu_dirty_limit; in migration_dirty_limit_guest()
1012 qmp_set_vcpu_dirty_limit(false, -1, quota_dirtyrate, NULL); in migration_dirty_limit_guest()
1020 migration_transferred_bytes() - rs->bytes_xfer_prev; in migration_trigger_throttle()
1021 uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE; in migration_trigger_throttle()
1032 (++rs->dirty_rate_high_cnt >= 2)) { in migration_trigger_throttle()
1033 rs->dirty_rate_high_cnt = 0; in migration_trigger_throttle()
1051 if (!rs->time_last_bitmap_sync) { in migration_bitmap_sync()
1052 rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); in migration_bitmap_sync()
1058 WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) { in migration_bitmap_sync()
1068 trace_migration_bitmap_sync_end(rs->num_dirty_pages_period); in migration_bitmap_sync()
1073 if (end_time > rs->time_last_bitmap_sync + 1000) { in migration_bitmap_sync()
1078 rs->target_page_count_prev = rs->target_page_count; in migration_bitmap_sync()
1081 rs->time_last_bitmap_sync = end_time; in migration_bitmap_sync()
1082 rs->num_dirty_pages_period = 0; in migration_bitmap_sync()
1083 rs->bytes_xfer_prev = migration_transferred_bytes(); in migration_bitmap_sync()
1112 void ram_release_page(const char *rbname, uint64_t offset) in ram_release_page() argument
1118 ram_discard_range(rbname, offset, TARGET_PAGE_SIZE); in ram_release_page()
1122 * save_zero_page: send the zero page to the stream
1128 * @offset: offset inside the block for the page
1131 ram_addr_t offset) in save_zero_page() argument
1133 uint8_t *p = pss->block->host + offset; in save_zero_page()
1134 QEMUFile *file = pss->pss_channel; in save_zero_page()
1148 /* zero pages are not transferred with mapped-ram */ in save_zero_page()
1149 clear_bit_atomic(offset >> TARGET_PAGE_BITS, pss->block->file_bmap); in save_zero_page()
1153 len += save_page_header(pss, file, pss->block, offset | RAM_SAVE_FLAG_ZERO); in save_zero_page()
1156 ram_release_page(pss->block->idstr, offset); in save_zero_page()
1161 * page would be stale. in save_zero_page()
1163 if (rs->xbzrle_started) { in save_zero_page()
1165 xbzrle_cache_zero_page(pss->block->offset + offset); in save_zero_page()
1174 * < 0 - error
1175 * > 0 - number of pages written
1180 ram_addr_t offset, int *pages) in control_save_page() argument
1184 ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset, in control_save_page()
1199 * directly send the page to the stream
1204 * @block: block that contains the page we want to send
1205 * @offset: offset inside the block for the page
1206 * @buf: the page to be sent
1207 * @async: send to page asyncly
1210 ram_addr_t offset, uint8_t *buf, bool async) in save_normal_page() argument
1212 QEMUFile *file = pss->pss_channel; in save_normal_page()
1216 block->pages_offset + offset); in save_normal_page()
1217 set_bit(offset >> TARGET_PAGE_BITS, block->file_bmap); in save_normal_page()
1219 ram_transferred_add(save_page_header(pss, pss->pss_channel, block, in save_normal_page()
1220 offset | RAM_SAVE_FLAG_PAGE)); in save_normal_page()
1235 * ram_save_page: send the given page to the stream
1238 * < 0 - error
1239 * >=0 - Number of pages written - this might legally be 0
1240 * if xbzrle noticed the page was the same.
1243 * @block: block that contains the page we want to send
1244 * @offset: offset inside the block for the page
1248 int pages = -1; in ram_save_page()
1251 RAMBlock *block = pss->block; in ram_save_page()
1252 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; in ram_save_page() local
1253 ram_addr_t current_addr = block->offset + offset; in ram_save_page()
1255 p = block->host + offset; in ram_save_page()
1256 trace_ram_save_page(block->idstr, (uint64_t)offset, p); in ram_save_page()
1259 if (rs->xbzrle_started && !migration_in_postcopy()) { in ram_save_page()
1261 block, offset); in ram_save_page()
1262 if (!rs->last_stage) { in ram_save_page()
1263 /* Can't send this cached data async, since the cache page in ram_save_page()
1270 /* XBZRLE overflow or normal page */ in ram_save_page()
1271 if (pages == -1) { in ram_save_page()
1272 pages = save_normal_page(pss, block, offset, p, send_async); in ram_save_page()
1280 static int ram_save_multifd_page(RAMBlock *block, ram_addr_t offset) in ram_save_multifd_page() argument
1282 if (!multifd_queue_page(block, offset)) { in ram_save_multifd_page()
1283 return -1; in ram_save_multifd_page()
1294 * find_dirty_block: find the next dirty page and update any state
1299 * PAGE_ALL_CLEAN: no dirty page found, give up
1300 * PAGE_TRY_AGAIN: no dirty page found, retry for next block
1301 * PAGE_DIRTY_FOUND: dirty page found
1304 * @pss: data about the state of the current dirty page scan
1309 /* Update pss->page for the next dirty bit in ramblock */ in find_dirty_block()
1312 if (pss->complete_round && pss->block == rs->last_seen_block && in find_dirty_block()
1313 pss->page >= rs->last_page) { in find_dirty_block()
1320 if (!offset_in_ramblock(pss->block, in find_dirty_block()
1321 ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) { in find_dirty_block()
1323 pss->page = 0; in find_dirty_block()
1324 pss->block = QLIST_NEXT_RCU(pss->block, next); in find_dirty_block()
1325 if (!pss->block) { in find_dirty_block()
1329 QEMUFile *f = rs->pss[RAM_CHANNEL_PRECOPY].pss_channel; in find_dirty_block()
1342 pss->block = QLIST_FIRST_RCU(&ram_list.blocks); in find_dirty_block()
1344 pss->complete_round = true; in find_dirty_block()
1347 rs->xbzrle_started = true; in find_dirty_block()
1359 * unqueue_page: gets a page of the queue
1361 * Helper for 'get_queued_page' - gets a page off the queue
1363 * Returns the block of the page (or NULL if none available)
1366 * @offset: used to return the offset within the RAMBlock
1368 static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset) in unqueue_page() argument
1377 QEMU_LOCK_GUARD(&rs->src_page_req_mutex); in unqueue_page()
1385 entry = QSIMPLEQ_FIRST(&rs->src_page_requests); in unqueue_page()
1386 block = entry->rb; in unqueue_page()
1387 *offset = entry->offset; in unqueue_page()
1389 if (entry->len > TARGET_PAGE_SIZE) { in unqueue_page()
1390 entry->len -= TARGET_PAGE_SIZE; in unqueue_page()
1391 entry->offset += TARGET_PAGE_SIZE; in unqueue_page()
1393 memory_region_unref(block->mr); in unqueue_page()
1394 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req); in unqueue_page()
1404 * poll_fault_page: try to get next UFFD write fault page and, if pending fault
1405 * is found, return RAM block pointer and page offset
1407 * Returns pointer to the RAMBlock containing faulting page,
1411 * @offset: page offset from the beginning of the block
1413 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset) in poll_fault_page() argument
1424 res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1); in poll_fault_page()
1430 block = qemu_ram_block_from_host(page_address, false, offset); in poll_fault_page()
1431 assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0); in poll_fault_page()
1440 * @pss: page-search-status structure
1441 * @start_page: index of the first page in the range relative to pss->block
1450 /* Check if page is from UFFD-managed region. */ in ram_save_release_protection()
1451 if (pss->block->flags & RAM_UF_WRITEPROTECT) { in ram_save_release_protection()
1452 void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS); in ram_save_release_protection()
1453 uint64_t run_length = (pss->page - start_page) << TARGET_PAGE_BITS; in ram_save_release_protection()
1455 /* Flush async buffers before un-protect. */ in ram_save_release_protection()
1456 qemu_fflush(pss->pss_channel); in ram_save_release_protection()
1457 /* Un-protect memory range. */ in ram_save_release_protection()
1458 res = uffd_change_protection(rs->uffdio_fd, page_address, run_length, in ram_save_release_protection()
1480 * compatible with 'write-tracking'
1502 /* Nothing to do with read-only and MMIO-writable regions */ in ram_write_tracking_compatible()
1503 if (block->mr->readonly || block->mr->rom_device) { in ram_write_tracking_compatible()
1506 /* Try to register block memory via UFFD-IO to track writes */ in ram_write_tracking_compatible()
1507 if (uffd_register_memory(uffd_fd, block->host, block->max_length, in ram_write_tracking_compatible()
1522 static inline void populate_read_range(RAMBlock *block, ram_addr_t offset, in populate_read_range() argument
1525 const ram_addr_t end = offset + size; in populate_read_range()
1528 * We read one byte of each page; this will preallocate page tables if in populate_read_range()
1530 * where no page was populated yet. This might require adaption when in populate_read_range()
1533 for (; offset < end; offset += block->page_size) { in populate_read_range()
1534 char tmp = *((char *)block->host + offset); in populate_read_range()
1544 const hwaddr size = int128_get64(section->size); in populate_read_section()
1545 hwaddr offset = section->offset_within_region; in populate_read_section() local
1546 RAMBlock *block = section->mr->ram_block; in populate_read_section()
1548 populate_read_range(block, offset, size); in populate_read_section()
1553 * ram_block_populate_read: preallocate page tables and populate pages in the
1554 * RAM block by reading a byte of each page.
1557 * hardcode page size to qemu_real_host_page_size.
1569 * not be part of the migration stream either way -- see in ram_block_populate_read()
1574 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) { in ram_block_populate_read()
1575 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); in ram_block_populate_read()
1577 .mr = rb->mr, in ram_block_populate_read()
1579 .size = rb->mr->size, in ram_block_populate_read()
1585 populate_read_range(rb, 0, rb->used_length); in ram_block_populate_read()
1590 * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking
1599 /* Nothing to do with read-only and MMIO-writable regions */ in ram_write_tracking_prepare()
1600 if (block->mr->readonly || block->mr->rom_device) { in ram_write_tracking_prepare()
1610 * pages with pte_none() entries in page table. in ram_write_tracking_prepare()
1619 const hwaddr size = int128_get64(section->size); in uffd_protect_section()
1620 const hwaddr offset = section->offset_within_region; in uffd_protect_section() local
1621 RAMBlock *rb = section->mr->ram_block; in uffd_protect_section()
1624 return uffd_change_protection(uffd_fd, rb->host + offset, size, true, in uffd_protect_section()
1630 assert(rb->flags & RAM_UF_WRITEPROTECT); in ram_block_uffd_protect()
1633 if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) { in ram_block_uffd_protect()
1634 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); in ram_block_uffd_protect()
1636 .mr = rb->mr, in ram_block_uffd_protect()
1638 .size = rb->mr->size, in ram_block_uffd_protect()
1645 return uffd_change_protection(uffd_fd, rb->host, in ram_block_uffd_protect()
1646 rb->used_length, true, false); in ram_block_uffd_protect()
1650 * ram_write_tracking_start: start UFFD-WP memory tracking
1665 rs->uffdio_fd = uffd_fd; in ram_write_tracking_start()
1670 /* Nothing to do with read-only and MMIO-writable regions */ in ram_write_tracking_start()
1671 if (block->mr->readonly || block->mr->rom_device) { in ram_write_tracking_start()
1676 if (uffd_register_memory(rs->uffdio_fd, block->host, in ram_write_tracking_start()
1677 block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) { in ram_write_tracking_start()
1680 block->flags |= RAM_UF_WRITEPROTECT; in ram_write_tracking_start()
1681 memory_region_ref(block->mr); in ram_write_tracking_start()
1688 trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size, in ram_write_tracking_start()
1689 block->host, block->max_length); in ram_write_tracking_start()
1698 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) { in ram_write_tracking_start()
1701 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length); in ram_write_tracking_start()
1703 block->flags &= ~RAM_UF_WRITEPROTECT; in ram_write_tracking_start()
1704 memory_region_unref(block->mr); in ram_write_tracking_start()
1708 rs->uffdio_fd = -1; in ram_write_tracking_start()
1709 return -1; in ram_write_tracking_start()
1713 * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection
1723 if ((block->flags & RAM_UF_WRITEPROTECT) == 0) { in ram_write_tracking_stop()
1726 uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length); in ram_write_tracking_stop()
1728 trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size, in ram_write_tracking_stop()
1729 block->host, block->max_length); in ram_write_tracking_stop()
1732 block->flags &= ~RAM_UF_WRITEPROTECT; in ram_write_tracking_stop()
1733 memory_region_unref(block->mr); in ram_write_tracking_stop()
1737 uffd_close_fd(rs->uffdio_fd); in ram_write_tracking_stop()
1738 rs->uffdio_fd = -1; in ram_write_tracking_stop()
1744 static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset) in poll_fault_page() argument
1747 (void) offset; in poll_fault_page()
1784 * get_queued_page: unqueue a page from the postcopy requests
1788 * Returns true if a queued page is found
1791 * @pss: data about the state of the current dirty page scan
1796 ram_addr_t offset; in get_queued_page() local
1800 block = unqueue_page(rs, &offset); in get_queued_page()
1802 * We're sending this page, and since it's postcopy nothing else in get_queued_page()
1808 unsigned long page; in get_queued_page() local
1810 page = offset >> TARGET_PAGE_BITS; in get_queued_page()
1811 dirty = test_bit(page, block->bmap); in get_queued_page()
1813 trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset, in get_queued_page()
1814 page); in get_queued_page()
1816 trace_get_queued_page(block->idstr, (uint64_t)offset, page); in get_queued_page()
1827 block = poll_fault_page(rs, &offset); in get_queued_page()
1832 * We want the background search to continue from the queued page in get_queued_page()
1833 * since the guest is likely to want other pages near to the page in get_queued_page()
1836 pss->block = block; in get_queued_page()
1837 pss->page = offset >> TARGET_PAGE_BITS; in get_queued_page()
1840 * This unqueued page would break the "one round" check, even is in get_queued_page()
1843 pss->complete_round = false; in get_queued_page()
1854 * be some left. in case that there is any page left, we drop it.
1860 /* This queue generally should be empty - but in the case of a failed in migration_page_queue_free()
1864 QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) { in migration_page_queue_free()
1865 memory_region_unref(mspr->rb->mr); in migration_page_queue_free()
1866 QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req); in migration_page_queue_free()
1872 * ram_save_queue_pages: queue the page for transmission
1894 ramblock = rs->last_req_rb; in ram_save_queue_pages()
1902 return -1; in ram_save_queue_pages()
1908 /* We shouldn't be asked for a non-existent RAMBlock */ in ram_save_queue_pages()
1910 return -1; in ram_save_queue_pages()
1912 rs->last_req_rb = ramblock; in ram_save_queue_pages()
1914 trace_ram_save_queue_pages(ramblock->idstr, start, len); in ram_save_queue_pages()
1915 if (!offset_in_ramblock(ramblock, start + len - 1)) { in ram_save_queue_pages()
1919 start, len, ramblock->used_length); in ram_save_queue_pages()
1920 return -1; in ram_save_queue_pages()
1924 * When with postcopy preempt, we send back the page directly in the in ram_save_queue_pages()
1925 * rp-return thread. in ram_save_queue_pages()
1930 PageSearchStatus *pss = &ram_state->pss[RAM_CHANNEL_POSTCOPY]; in ram_save_queue_pages()
1933 qemu_mutex_lock(&rs->bitmap_mutex); in ram_save_queue_pages()
1938 * safe to access without lock, because when rp-thread is running in ram_save_queue_pages()
1941 pss->pss_channel = migrate_get_current()->postcopy_qemufile_src; in ram_save_queue_pages()
1942 assert(pss->pss_channel); in ram_save_queue_pages()
1945 * It must be either one or multiple of host page size. Just in ram_save_queue_pages()
1953 ramblock->idstr, start); in ram_save_queue_pages()
1954 ret = -1; in ram_save_queue_pages()
1958 * NOTE: after ram_save_host_page_urgent() succeeded, pss->page in ram_save_queue_pages()
1959 * will automatically be moved and point to the next host page in ram_save_queue_pages()
1962 * Normally QEMU never sends >1 host page in requests, so in ram_save_queue_pages()
1966 len -= page_size; in ram_save_queue_pages()
1968 qemu_mutex_unlock(&rs->bitmap_mutex); in ram_save_queue_pages()
1975 new_entry->rb = ramblock; in ram_save_queue_pages()
1976 new_entry->offset = start; in ram_save_queue_pages()
1977 new_entry->len = len; in ram_save_queue_pages()
1979 memory_region_ref(ramblock->mr); in ram_save_queue_pages()
1980 qemu_mutex_lock(&rs->src_page_req_mutex); in ram_save_queue_pages()
1981 QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req); in ram_save_queue_pages()
1983 qemu_mutex_unlock(&rs->src_page_req_mutex); in ram_save_queue_pages()
1989 * ram_save_target_page_legacy: save one target page
1994 * @pss: data about the page we want to send
1998 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; in ram_save_target_page_legacy() local
2001 if (control_save_page(pss, offset, &res)) { in ram_save_target_page_legacy()
2005 if (save_zero_page(rs, pss, offset)) { in ram_save_target_page_legacy()
2013 * ram_save_target_page_multifd: send one target page to multifd workers
2015 * Returns 1 if the page was queued, -1 otherwise.
2018 * @pss: data about the page we want to send
2022 RAMBlock *block = pss->block; in ram_save_target_page_multifd()
2023 ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; in ram_save_target_page_multifd() local
2027 * page checking on the migration main thread. in ram_save_target_page_multifd()
2030 if (save_zero_page(rs, pss, offset)) { in ram_save_target_page_multifd()
2035 return ram_save_multifd_page(block, offset); in ram_save_target_page_multifd()
2038 /* Should be called before sending a host page */
2041 /* How many guest pages are there in one host page? */ in pss_host_page_prepare()
2042 size_t guest_pfns = qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS; in pss_host_page_prepare()
2044 pss->host_page_sending = true; in pss_host_page_prepare()
2050 * For the latter, we always send one whole guest page per in pss_host_page_prepare()
2051 * iteration of the host page (example: an Alpha VM on x86 host in pss_host_page_prepare()
2054 pss->host_page_start = pss->page; in pss_host_page_prepare()
2055 pss->host_page_end = pss->page + 1; in pss_host_page_prepare()
2058 * The host page spans over multiple guest pages, we send them in pss_host_page_prepare()
2059 * within the same host page iteration. in pss_host_page_prepare()
2061 pss->host_page_start = ROUND_DOWN(pss->page, guest_pfns); in pss_host_page_prepare()
2062 pss->host_page_end = ROUND_UP(pss->page + 1, guest_pfns); in pss_host_page_prepare()
2067 * Whether the page pointed by PSS is within the host page being sent.
2074 assert(pss->host_page_sending); in pss_within_range()
2076 /* Over host-page boundary? */ in pss_within_range()
2077 if (pss->page >= pss->host_page_end) { in pss_within_range()
2081 ram_addr = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; in pss_within_range()
2083 return offset_in_ramblock(pss->block, ram_addr); in pss_within_range()
2088 pss->host_page_sending = false; in pss_host_page_finish()
2090 pss->host_page_start = pss->host_page_end = 0; in pss_host_page_finish()
2094 * Send an urgent host page specified by `pss'. Need to be called with
2097 * Returns 0 if save host page succeeded, false otherwise.
2105 trace_postcopy_preempt_send_host_page(pss->block->idstr, pss->page); in ram_save_host_page_urgent()
2109 * If precopy is sending the same page, let it be done in precopy, or in ram_save_host_page_urgent()
2110 * we could send the same page in two channels and none of them will in ram_save_host_page_urgent()
2111 * receive the whole page. in ram_save_host_page_urgent()
2113 if (pss_overlap(pss, &ram_state->pss[RAM_CHANNEL_PRECOPY])) { in ram_save_host_page_urgent()
2114 trace_postcopy_preempt_hit(pss->block->idstr, in ram_save_host_page_urgent()
2115 pss->page << TARGET_PAGE_BITS); in ram_save_host_page_urgent()
2120 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page); in ram_save_host_page_urgent()
2124 if (migration_ops->ram_save_target_page(rs, pss) != 1) { in ram_save_host_page_urgent()
2126 ret = -1; in ram_save_host_page_urgent()
2137 qemu_fflush(pss->pss_channel); in ram_save_host_page_urgent()
2143 * ram_save_host_page: save a whole host page
2145 * Starting at *offset send pages up to the end of the current host
2146 * page. It's valid for the initial offset to point into the middle of
2147 * a host page in which case the remainder of the hostpage is sent.
2148 * Only dirty target pages are sent. Note that the host page size may
2149 * be a huge page for this block.
2152 * if the RAMBlock isn't a multiple of the host page size.
2161 * @pss: data about the page we want to send
2168 qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS; in ram_save_host_page()
2169 unsigned long start_page = pss->page; in ram_save_host_page()
2172 if (migrate_ram_is_ignored(pss->block)) { in ram_save_host_page()
2173 error_report("block %s should not be migrated !", pss->block->idstr); in ram_save_host_page()
2177 /* Update host page boundary information */ in ram_save_host_page()
2181 page_dirty = migration_bitmap_clear_dirty(rs, pss->block, pss->page); in ram_save_host_page()
2187 * because both migration thread and rp-return thread can in ram_save_host_page()
2191 qemu_mutex_unlock(&rs->bitmap_mutex); in ram_save_host_page()
2193 tmppages = migration_ops->ram_save_target_page(rs, pss); in ram_save_host_page()
2205 qemu_mutex_lock(&rs->bitmap_mutex); in ram_save_host_page()
2226 * ram_find_and_save_block: finds a dirty page and sends it to f
2235 * On systems where host-page-size > target-page-size it will send all the
2236 * pages in a host page that are dirty.
2240 PageSearchStatus *pss = &rs->pss[RAM_CHANNEL_PRECOPY]; in ram_find_and_save_block()
2243 /* No dirty page as there is zero RAM */ in ram_find_and_save_block()
2244 if (!rs->ram_bytes_total) { in ram_find_and_save_block()
2255 if (!rs->last_seen_block) { in ram_find_and_save_block()
2256 rs->last_seen_block = QLIST_FIRST_RCU(&ram_list.blocks); in ram_find_and_save_block()
2257 rs->last_page = 0; in ram_find_and_save_block()
2260 pss_init(pss, rs->last_seen_block, rs->last_page); in ram_find_and_save_block()
2283 rs->last_seen_block = pss->block; in ram_find_and_save_block()
2284 rs->last_page = pss->page; in ram_find_and_save_block()
2297 total += block->used_length; in ram_bytes_total_with_ignored()
2310 total += block->used_length; in ram_bytes_total()
2330 qemu_mutex_destroy(&(*rsp)->bitmap_mutex); in ram_state_cleanup()
2331 qemu_mutex_destroy(&(*rsp)->src_page_req_mutex); in ram_state_cleanup()
2358 g_free(block->clear_bmap); in ram_bitmaps_destroy()
2359 block->clear_bmap = NULL; in ram_bitmaps_destroy()
2360 g_free(block->bmap); in ram_bitmaps_destroy()
2361 block->bmap = NULL; in ram_bitmaps_destroy()
2362 g_free(block->file_bmap); in ram_bitmaps_destroy()
2363 block->file_bmap = NULL; in ram_bitmaps_destroy()
2400 rs->pss[i].last_sent_block = NULL; in ram_state_reset()
2403 rs->last_seen_block = NULL; in ram_state_reset()
2404 rs->last_page = 0; in ram_state_reset()
2405 rs->last_version = ram_list.version; in ram_state_reset()
2406 rs->xbzrle_started = false; in ram_state_reset()
2418 unsigned long *bitmap = block->bmap; in ram_postcopy_migrated_memory_release()
2419 unsigned long range = block->used_length >> TARGET_PAGE_BITS; in ram_postcopy_migrated_memory_release()
2424 ram_discard_range(block->idstr, in ram_postcopy_migrated_memory_release()
2426 ((ram_addr_t)(run_end - run_start)) in ram_postcopy_migrated_memory_release()
2443 unsigned long end = block->used_length >> TARGET_PAGE_BITS; in postcopy_send_discard_bm_ram()
2445 unsigned long *bitmap = block->bmap; in postcopy_send_discard_bm_ram()
2458 discard_length = end - one; in postcopy_send_discard_bm_ram()
2460 discard_length = zero - one; in postcopy_send_discard_bm_ram()
2476 * which would mean postcopy code would have to deal with target page)
2485 postcopy_discard_send_init(ms, block->idstr); in postcopy_each_ram_send_discard()
2489 * host-page size chunks, mark any partially dirty host-page size in postcopy_each_ram_send_discard()
2490 * chunks as all dirty. In this case the host-page is the host-page in postcopy_each_ram_send_discard()
2491 * for the particular RAMBlock, i.e. it might be a huge page. in postcopy_each_ram_send_discard()
2498 * target page specific code. in postcopy_each_ram_send_discard()
2516 * @block: block that contains the page we want to canonicalize
2521 unsigned long *bitmap = block->bmap; in postcopy_chunk_hostpages_pass()
2522 unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE; in postcopy_chunk_hostpages_pass()
2523 unsigned long pages = block->used_length >> TARGET_PAGE_BITS; in postcopy_chunk_hostpages_pass()
2526 if (block->page_size == TARGET_PAGE_SIZE) { in postcopy_chunk_hostpages_pass()
2527 /* Easy case - TPS==HPS for a non-huge page RAMBlock */ in postcopy_chunk_hostpages_pass()
2531 /* Find a dirty page */ in postcopy_chunk_hostpages_pass()
2538 * page, then we need to fixup this host page. in postcopy_chunk_hostpages_pass()
2544 * If the end isn't at the start of a host page, then the in postcopy_chunk_hostpages_pass()
2545 * run doesn't finish at the end of a host page in postcopy_chunk_hostpages_pass()
2551 unsigned long page; in postcopy_chunk_hostpages_pass() local
2557 for (page = fixup_start_addr; in postcopy_chunk_hostpages_pass()
2558 page < fixup_start_addr + host_ratio; page++) { in postcopy_chunk_hostpages_pass()
2563 rs->migration_dirty_pages += !test_and_set_bit(page, bitmap); in postcopy_chunk_hostpages_pass()
2567 /* Find the next dirty page for the next iteration */ in postcopy_chunk_hostpages_pass()
2594 /* Easiest way to make sure we don't resume in the middle of a host-page */ in ram_postcopy_send_discard_bitmap()
2595 rs->pss[RAM_CHANNEL_PRECOPY].last_sent_block = NULL; in ram_postcopy_send_discard_bitmap()
2596 rs->last_seen_block = NULL; in ram_postcopy_send_discard_bitmap()
2597 rs->last_page = 0; in ram_postcopy_send_discard_bitmap()
2611 * @start: RAMBlock starting page
2623 return -1; in ram_discard_range()
2630 if (rb->receivedmap) { in ram_discard_range()
2631 bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(), in ram_discard_range()
2652 error_setg(errp, "%s: Error allocating zero page", __func__); in xbzrle_init()
2701 qemu_mutex_init(&(*rsp)->bitmap_mutex); in ram_state_init()
2702 qemu_mutex_init(&(*rsp)->src_page_req_mutex); in ram_state_init()
2703 QSIMPLEQ_INIT(&(*rsp)->src_page_requests); in ram_state_init()
2704 (*rsp)->ram_bytes_total = ram_bytes_total(); in ram_state_init()
2711 (*rsp)->migration_dirty_pages = (*rsp)->ram_bytes_total >> TARGET_PAGE_BITS; in ram_state_init()
2726 shift = ms->clear_bitmap_shift; in ram_list_init_bitmaps()
2738 pages = block->max_length >> TARGET_PAGE_BITS; in ram_list_init_bitmaps()
2741 * ones to make sure we'll migrate every guest RAM page to in ram_list_init_bitmaps()
2748 block->bmap = bitmap_new(pages); in ram_list_init_bitmaps()
2749 bitmap_set(block->bmap, 0, pages); in ram_list_init_bitmaps()
2751 block->file_bmap = bitmap_new(pages); in ram_list_init_bitmaps()
2753 block->clear_bmap_shift = shift; in ram_list_init_bitmaps()
2754 block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift)); in ram_list_init_bitmaps()
2768 rs->migration_dirty_pages -= pages; in migration_bitmap_clear_discarded_pages()
2808 return -1; in ram_init_all()
2813 return -1; in ram_init_all()
2817 return -1; in ram_init_all()
2831 * about dirty page logging as well. in ram_state_resume_prepare()
2835 pages += bitmap_count_one(block->bmap, in ram_state_resume_prepare()
2836 block->used_length >> TARGET_PAGE_BITS); in ram_state_resume_prepare()
2840 rs->migration_dirty_pages = pages; in ram_state_resume_prepare()
2845 rs->pss[RAM_CHANNEL_PRECOPY].pss_channel = out; in ram_state_resume_prepare()
2859 ram_addr_t offset; in qemu_guest_free_page_hint() local
2867 for (; len > 0; len -= used_len, addr += used_len) { in qemu_guest_free_page_hint()
2868 block = qemu_ram_block_from_host(addr, false, &offset); in qemu_guest_free_page_hint()
2869 if (unlikely(!block || offset >= block->used_length)) { in qemu_guest_free_page_hint()
2879 if (len <= block->used_length - offset) { in qemu_guest_free_page_hint()
2882 used_len = block->used_length - offset; in qemu_guest_free_page_hint()
2885 start = offset >> TARGET_PAGE_BITS; in qemu_guest_free_page_hint()
2888 qemu_mutex_lock(&ram_state->bitmap_mutex); in qemu_guest_free_page_hint()
2896 ram_state->migration_dirty_pages -= in qemu_guest_free_page_hint()
2897 bitmap_count_one_with_offset(block->bmap, start, npages); in qemu_guest_free_page_hint()
2898 bitmap_clear(block->bmap, start, npages); in qemu_guest_free_page_hint()
2899 qemu_mutex_unlock(&ram_state->bitmap_mutex); in qemu_guest_free_page_hint()
2907 * The target's page size, so we know how many pages are in the
2912 * The offset in the migration file where the pages bitmap is
2917 * The offset in the migration file where the actual pages (data)
2933 num_pages = block->used_length >> TARGET_PAGE_BITS; in mapped_ram_setup_ramblock()
2941 block->bitmap_offset = qemu_get_offset(file) + header_size; in mapped_ram_setup_ramblock()
2942 block->pages_offset = ROUND_UP(block->bitmap_offset + in mapped_ram_setup_ramblock()
2946 header->version = cpu_to_be32(MAPPED_RAM_HDR_VERSION); in mapped_ram_setup_ramblock()
2947 header->page_size = cpu_to_be64(TARGET_PAGE_SIZE); in mapped_ram_setup_ramblock()
2948 header->bitmap_offset = cpu_to_be64(block->bitmap_offset); in mapped_ram_setup_ramblock()
2949 header->pages_offset = cpu_to_be64(block->pages_offset); in mapped_ram_setup_ramblock()
2953 /* prepare offset for next ramblock */ in mapped_ram_setup_ramblock()
2954 qemu_set_offset(file, block->pages_offset + block->used_length, SEEK_SET); in mapped_ram_setup_ramblock()
2964 error_setg(errp, "Could not read whole mapped-ram migration header " in mapped_ram_read_header()
2969 /* migration stream is big-endian */ in mapped_ram_read_header()
2970 header->version = be32_to_cpu(header->version); in mapped_ram_read_header()
2972 if (header->version > MAPPED_RAM_HDR_VERSION) { in mapped_ram_read_header()
2973 error_setg(errp, "Migration mapped-ram capability version not " in mapped_ram_read_header()
2975 header->version); in mapped_ram_read_header()
2979 header->page_size = be64_to_cpu(header->page_size); in mapped_ram_read_header()
2980 header->bitmap_offset = be64_to_cpu(header->bitmap_offset); in mapped_ram_read_header()
2981 header->pages_offset = be64_to_cpu(header->pages_offset); in mapped_ram_read_header()
2988 * long-running RCU critical section. When rcu-reclaims in the code
3011 return -1; in ram_save_setup()
3014 (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f; in ram_save_setup()
3027 qemu_put_byte(f, strlen(block->idstr)); in ram_save_setup()
3028 qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr)); in ram_save_setup()
3029 qemu_put_be64(f, block->used_length); in ram_save_setup()
3031 block->page_size != max_hg_page_size) { in ram_save_setup()
3032 qemu_put_be64(f, block->page_size); in ram_save_setup()
3035 qemu_put_be64(f, block->mr->addr); in ram_save_setup()
3062 migration_ops->ram_save_target_page = ram_save_target_page_multifd; in ram_save_setup()
3064 migration_ops->ram_save_target_page = ram_save_target_page_legacy; in ram_save_setup()
3083 error_setg_errno(errp, -ret, "%s failed", __func__); in ram_save_setup()
3093 long num_pages = block->used_length >> TARGET_PAGE_BITS; in ram_save_file_bmap()
3096 qemu_put_buffer_at(f, (uint8_t *)block->file_bmap, bitmap_size, in ram_save_file_bmap()
3097 block->bitmap_offset); in ram_save_file_bmap()
3105 g_free(block->file_bmap); in ram_save_file_bmap()
3106 block->file_bmap = NULL; in ram_save_file_bmap()
3110 void ramblock_set_file_bmap_atomic(RAMBlock *block, ram_addr_t offset, bool set) in ramblock_set_file_bmap_atomic() argument
3113 set_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap); in ramblock_set_file_bmap_atomic()
3115 clear_bit_atomic(offset >> TARGET_PAGE_BITS, block->file_bmap); in ramblock_set_file_bmap_atomic()
3143 WITH_QEMU_LOCK_GUARD(&rs->bitmap_mutex) { in ram_save_iterate()
3145 if (ram_list.version != rs->last_version) { in ram_save_iterate()
3180 rs->target_page_count += pages; in ram_save_iterate()
3189 uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / in ram_save_iterate()
3247 rs->last_stage = !migration_in_colo_state(); in ram_save_complete()
3263 qemu_mutex_lock(&rs->bitmap_mutex); in ram_save_complete()
3273 qemu_mutex_unlock(&rs->bitmap_mutex); in ram_save_complete()
3277 qemu_mutex_unlock(&rs->bitmap_mutex); in ram_save_complete()
3299 return -err; in ram_save_complete()
3313 uint64_t remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; in ram_state_pending_estimate()
3338 remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; in ram_state_pending_exact()
3359 error_report("Failed to load XBZRLE page - wrong compression!"); in load_xbzrle()
3360 return -1; in load_xbzrle()
3364 error_report("Failed to load XBZRLE page - len overflow!"); in load_xbzrle()
3365 return -1; in load_xbzrle()
3374 TARGET_PAGE_SIZE) == -1) { in load_xbzrle()
3375 error_report("Failed to load XBZRLE page - decode error!"); in load_xbzrle()
3376 return -1; in load_xbzrle()
3387 * Returns a pointer from within the RCU-protected ram_list.
3391 * @flags: Page flags (mostly to see if it's a continuation of previous block)
3398 RAMBlock *block = mis->last_recv_block[channel]; in ram_block_from_stream()
3425 mis->last_recv_block[channel] = block; in ram_block_from_stream()
3431 ram_addr_t offset) in host_from_ram_block_offset() argument
3433 if (!offset_in_ramblock(block, offset)) { in host_from_ram_block_offset()
3437 return block->host + offset; in host_from_ram_block_offset()
3441 ram_addr_t offset) in host_page_from_ram_block_offset() argument
3444 return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset), in host_page_from_ram_block_offset()
3445 block->page_size); in host_page_from_ram_block_offset()
3449 ram_addr_t offset) in host_page_offset_from_ram_block_offset() argument
3451 return ((uintptr_t)block->host + offset) & (block->page_size - 1); in host_page_offset_from_ram_block_offset()
3456 qemu_mutex_lock(&ram_state->bitmap_mutex); in colo_record_bitmap()
3458 ram_addr_t offset = normal[i]; in colo_record_bitmap() local
3459 ram_state->migration_dirty_pages += !test_and_set_bit( in colo_record_bitmap()
3460 offset >> TARGET_PAGE_BITS, in colo_record_bitmap()
3461 block->bmap); in colo_record_bitmap()
3463 qemu_mutex_unlock(&ram_state->bitmap_mutex); in colo_record_bitmap()
3467 ram_addr_t offset, bool record_bitmap) in colo_cache_from_block_offset() argument
3469 if (!offset_in_ramblock(block, offset)) { in colo_cache_from_block_offset()
3472 if (!block->colo_cache) { in colo_cache_from_block_offset()
3474 __func__, block->idstr); in colo_cache_from_block_offset()
3484 colo_record_bitmap(block, &offset, 1); in colo_cache_from_block_offset()
3486 return block->colo_cache + offset; in colo_cache_from_block_offset()
3490 * ram_handle_zero: handle the zero page case
3492 * If a page (or a whole RDMA chunk) has been
3495 * @host: host address for the zero page
3496 * @ch: what the page is filled from. We only support zero
3497 * @size: size of the zero page
3526 block->colo_cache = qemu_anon_ram_alloc(block->used_length, in colo_init_ram_cache()
3528 if (!block->colo_cache) { in colo_init_ram_cache()
3530 "size 0x" RAM_ADDR_FMT, __func__, block->idstr, in colo_init_ram_cache()
3531 block->used_length); in colo_init_ram_cache()
3533 if (block->colo_cache) { in colo_init_ram_cache()
3534 qemu_anon_ram_free(block->colo_cache, block->used_length); in colo_init_ram_cache()
3535 block->colo_cache = NULL; in colo_init_ram_cache()
3538 return -errno; in colo_init_ram_cache()
3541 qemu_madvise(block->colo_cache, block->used_length, in colo_init_ram_cache()
3549 * with to decide which page in cache should be flushed into SVM's RAM. Here in colo_init_ram_cache()
3554 unsigned long pages = block->max_length >> TARGET_PAGE_BITS; in colo_init_ram_cache()
3555 block->bmap = bitmap_new(pages); in colo_init_ram_cache()
3578 bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS); in colo_incoming_start_dirty_log()
3585 ram_state->migration_dirty_pages = 0; in colo_incoming_start_dirty_log()
3597 g_free(block->bmap); in colo_release_ram_cache()
3598 block->bmap = NULL; in colo_release_ram_cache()
3603 if (block->colo_cache) { in colo_release_ram_cache()
3604 qemu_anon_ram_free(block->colo_cache, block->used_length); in colo_release_ram_cache()
3605 block->colo_cache = NULL; in colo_release_ram_cache()
3640 g_free(rb->receivedmap); in ram_load_cleanup()
3641 rb->receivedmap = NULL; in ram_load_cleanup()
3655 * postcopy-ram. postcopy-ram's similarly names
3664 * ram_load_postcopy: load a page in postcopy case
3666 * Returns 0 for success or -errno in case of error
3680 PostcopyTmpPage *tmp_page = &mis->postcopy_tmp_pages[channel]; in ram_load_postcopy()
3707 ret = -EINVAL; in ram_load_postcopy()
3714 * while in postcopy, which is fine - trying to place via in ram_load_postcopy()
3717 if (!block->host || addr >= block->postcopy_length) { in ram_load_postcopy()
3718 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); in ram_load_postcopy()
3719 ret = -EINVAL; in ram_load_postcopy()
3722 tmp_page->target_pages++; in ram_load_postcopy()
3723 matches_target_page_size = block->page_size == TARGET_PAGE_SIZE; in ram_load_postcopy()
3728 * To make it atomic, the data is read into a temporary page in ram_load_postcopy()
3730 * The migration protocol uses, possibly smaller, target-pages in ram_load_postcopy()
3732 * of a host page in one chunk. in ram_load_postcopy()
3734 page_buffer = tmp_page->tmp_huge_page + in ram_load_postcopy()
3737 if (tmp_page->target_pages == 1) { in ram_load_postcopy()
3738 tmp_page->host_addr = in ram_load_postcopy()
3740 } else if (tmp_page->host_addr != in ram_load_postcopy()
3743 error_report("Non-same host page detected on channel %d: " in ram_load_postcopy()
3744 "Target host page %p, received host page %p " in ram_load_postcopy()
3745 "(rb %s offset 0x"RAM_ADDR_FMT" target_pages %d)", in ram_load_postcopy()
3746 channel, tmp_page->host_addr, in ram_load_postcopy()
3748 block->idstr, addr, tmp_page->target_pages); in ram_load_postcopy()
3749 ret = -EINVAL; in ram_load_postcopy()
3754 * If it's the last part of a host page then we place the host in ram_load_postcopy()
3755 * page in ram_load_postcopy()
3757 if (tmp_page->target_pages == in ram_load_postcopy()
3758 (block->page_size / TARGET_PAGE_SIZE)) { in ram_load_postcopy()
3761 place_source = tmp_page->tmp_huge_page; in ram_load_postcopy()
3768 error_report("Found a zero page with value %d", ch); in ram_load_postcopy()
3769 ret = -EINVAL; in ram_load_postcopy()
3774 * this is a zero page and (block->page_size == TARGET_PAGE_SIZE). in ram_load_postcopy()
3782 tmp_page->all_zero = false; in ram_load_postcopy()
3788 * For small pages that matches target page size, we in ram_load_postcopy()
3790 * the buffer of QEMUFile to place the page. Note: we in ram_load_postcopy()
3793 * placing the page. in ram_load_postcopy()
3812 ret = -EINVAL; in ram_load_postcopy()
3822 if (tmp_page->all_zero) { in ram_load_postcopy()
3823 ret = postcopy_place_page_zero(mis, tmp_page->host_addr, block); in ram_load_postcopy()
3825 ret = postcopy_place_page(mis, tmp_page->host_addr, in ram_load_postcopy()
3851 unsigned long offset = 0; in colo_flush_ram_cache() local
3854 qemu_mutex_lock(&ram_state->bitmap_mutex); in colo_flush_ram_cache()
3861 trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages); in colo_flush_ram_cache()
3868 offset = colo_bitmap_find_dirty(ram_state, block, offset, &num); in colo_flush_ram_cache()
3870 ((ram_addr_t)offset) << TARGET_PAGE_BITS)) { in colo_flush_ram_cache()
3871 offset = 0; in colo_flush_ram_cache()
3878 migration_bitmap_clear_dirty(ram_state, block, offset + i); in colo_flush_ram_cache()
3880 dst_host = block->host in colo_flush_ram_cache()
3881 + (((ram_addr_t)offset) << TARGET_PAGE_BITS); in colo_flush_ram_cache()
3882 src_host = block->colo_cache in colo_flush_ram_cache()
3883 + (((ram_addr_t)offset) << TARGET_PAGE_BITS); in colo_flush_ram_cache()
3885 offset += num; in colo_flush_ram_cache()
3889 qemu_mutex_unlock(&ram_state->bitmap_mutex); in colo_flush_ram_cache()
3894 uint64_t offset) in ram_load_multifd_pages() argument
3898 data->opaque = host_addr; in ram_load_multifd_pages()
3899 data->file_offset = offset; in ram_load_multifd_pages()
3900 data->size = size; in ram_load_multifd_pages()
3915 ram_addr_t offset; in read_ramblock_mapped_ram() local
3925 unread = TARGET_PAGE_SIZE * (clear_bit_idx - set_bit_idx); in read_ramblock_mapped_ram()
3926 offset = set_bit_idx << TARGET_PAGE_BITS; in read_ramblock_mapped_ram()
3929 host = host_from_ram_block_offset(block, offset); in read_ramblock_mapped_ram()
3931 error_setg(errp, "page outside of ramblock %s range", in read_ramblock_mapped_ram()
3932 block->idstr); in read_ramblock_mapped_ram()
3940 block->pages_offset + offset); in read_ramblock_mapped_ram()
3943 block->pages_offset + offset); in read_ramblock_mapped_ram()
3949 offset += read; in read_ramblock_mapped_ram()
3950 unread -= read; in read_ramblock_mapped_ram()
3958 error_prepend(errp, "(%s) failed to read page " RAM_ADDR_FMT in read_ramblock_mapped_ram()
3959 "from file offset %" PRIx64 ": ", block->idstr, offset, in read_ramblock_mapped_ram()
3960 block->pages_offset + offset); in read_ramblock_mapped_ram()
3976 block->pages_offset = header.pages_offset; in parse_ramblock_mapped_ram()
3981 * value to change in the future. Do only a sanity check with page in parse_ramblock_mapped_ram()
3984 if (!QEMU_IS_ALIGNED(block->pages_offset, TARGET_PAGE_SIZE)) { in parse_ramblock_mapped_ram()
3987 block->idstr); in parse_ramblock_mapped_ram()
4006 qemu_set_offset(f, block->pages_offset + length, SEEK_SET); in parse_ramblock_mapped_ram()
4025 return -EINVAL; in parse_ramblock()
4031 error_report("block %s should not be migrated !", block->idstr); in parse_ramblock()
4032 return -EINVAL; in parse_ramblock()
4035 if (length != block->used_length) { in parse_ramblock()
4051 block->page_size != max_hg_page_size) { in parse_ramblock()
4053 if (remote_page_size != block->page_size) { in parse_ramblock()
4054 error_report("Mismatched RAM page size %s " in parse_ramblock()
4055 "(local) %zd != %" PRId64, block->idstr, in parse_ramblock()
4056 block->page_size, remote_page_size); in parse_ramblock()
4057 return -EINVAL; in parse_ramblock()
4063 block->mr->addr != addr) { in parse_ramblock()
4065 "%" PRId64 "!= %" PRId64, block->idstr, in parse_ramblock()
4066 (uint64_t)addr, (uint64_t)block->mr->addr); in parse_ramblock()
4067 return -EINVAL; in parse_ramblock()
4070 ret = rdma_block_notification_handle(f, block->idstr); in parse_ramblock()
4099 ret = -EINVAL; in parse_ramblocks()
4101 total_ram_bytes -= length; in parse_ramblocks()
4110 * Returns 0 for success or -errno in case of error
4157 ret = -EINVAL; in ram_load_precopy()
4168 * After going into COLO stage, we should not load the page in ram_load_precopy()
4172 * while we need to stop VM, which is a time-consuming process. in ram_load_precopy()
4173 * Here we optimize it by a trick, back-up every page while in in ram_load_precopy()
4176 * back-up all SVM'S memory in COLO preparing stage. in ram_load_precopy()
4191 error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); in ram_load_precopy()
4192 ret = -EINVAL; in ram_load_precopy()
4199 trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host); in ram_load_precopy()
4206 * For mapped-ram migration (to a file) using multifd, we sync in ram_load_precopy()
4220 error_report("Found a zero page with value %d", ch); in ram_load_precopy()
4221 ret = -EINVAL; in ram_load_precopy()
4233 error_report("Failed to decompress XBZRLE page at " in ram_load_precopy()
4235 ret = -EINVAL; in ram_load_precopy()
4247 * Mapped-ram migration flushes once and for all after in ram_load_precopy()
4262 ret = -EINVAL; in ram_load_precopy()
4280 * If system is running in postcopy mode, page inserts to host memory must in ram_load()
4288 return -EINVAL; in ram_load()
4303 * service fast page faults. in ram_load()
4321 "is not supported now!", rb->idstr, rb->host); in ram_has_postcopy()
4333 QEMUFile *file = s->to_dst_file; in ram_dirty_bitmap_sync_all()
4337 qatomic_set(&rs->postcopy_bmap_sync_requested, 0); in ram_dirty_bitmap_sync_all()
4339 qemu_savevm_send_recv_bitmap(file, block->idstr); in ram_dirty_bitmap_sync_all()
4340 trace_ram_dirty_bitmap_request(block->idstr); in ram_dirty_bitmap_sync_all()
4341 qatomic_inc(&rs->postcopy_bmap_sync_requested); in ram_dirty_bitmap_sync_all()
4347 while (qatomic_read(&rs->postcopy_bmap_sync_requested)) { in ram_dirty_bitmap_sync_all()
4349 return -1; in ram_dirty_bitmap_sync_all()
4368 QEMUFile *file = s->rp_state.from_dst_file; in ram_dirty_bitmap_reload()
4370 unsigned long nbits = block->used_length >> TARGET_PAGE_BITS; in ram_dirty_bitmap_reload()
4375 trace_ram_dirty_bitmap_reload_begin(block->idstr); in ram_dirty_bitmap_reload()
4377 if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) { in ram_dirty_bitmap_reload()
4379 MigrationStatus_str(s->state)); in ram_dirty_bitmap_reload()
4397 " != 0x%"PRIx64")", block->idstr, size, local_size); in ram_dirty_bitmap_reload()
4407 block->idstr, local_size, size); in ram_dirty_bitmap_reload()
4413 block->idstr, end_mark); in ram_dirty_bitmap_reload()
4421 bitmap_from_le(block->bmap, le_bitmap, nbits); in ram_dirty_bitmap_reload()
4427 bitmap_complement(block->bmap, block->bmap, nbits); in ram_dirty_bitmap_reload()
4433 trace_ram_dirty_bitmap_reload_complete(block->idstr); in ram_dirty_bitmap_reload()
4435 qatomic_dec(&rs->postcopy_bmap_sync_requested); in ram_dirty_bitmap_reload()
4459 ram_state_resume_prepare(rs, s->to_dst_file); in ram_resume_prepare()
4466 qemu_put_be64(s->postcopy_qemufile_src, RAM_SAVE_FLAG_EOS); in postcopy_preempt_shutdown_file()
4467 qemu_fflush(s->postcopy_qemufile_src); in postcopy_preempt_shutdown_file()
4489 ram_addr_t offset; in ram_mig_ram_block_resized() local
4490 RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset); in ram_mig_ram_block_resized()
4505 * changing at random points in time - especially after sending the in ram_mig_ram_block_resized()
4509 error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr); in ram_mig_ram_block_resized()
4517 * Update what ram_postcopy_incoming_init()->init_range() does at the in ram_mig_ram_block_resized()
4522 if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) { in ram_mig_ram_block_resized()
4524 rb->idstr); in ram_mig_ram_block_resized()
4527 rb->postcopy_length = new_size; in ram_mig_ram_block_resized()
4540 rb->idstr, ps); in ram_mig_ram_block_resized()
4541 exit(-1); in ram_mig_ram_block_resized()