Lines Matching +full:suspend +full:- +full:to +full:- +full:ram
10 * See the COPYING file in the top-level directory.
16 #include "qemu/error-report.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-mem.h"
30 #include "hw/qdev-properties.h"
45 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking
79 * We want to have a reasonable default block size such that
85 * The actual THP size might differ between Linux kernels, so we try to probe
87 * to disable THP in case we fail to properly probe the THP size, or if the
104 /* No THP -> no restrictions. */ in virtio_mem_thp_size()
111 * Try to probe the actual THP size, fallback to (sane but eventually in virtio_mem_thp_size()
117 /* Sanity-check the value and fallback to something reasonable. */ in virtio_mem_thp_size()
127 warn_report("Could not detect THP size, falling back to %" PRIx64 in virtio_mem_thp_size()
151 * anonymous RAM. In any other case, reading unplugged *can* populate a in virtio_mem_has_shared_zeropage()
163 * The memory block size corresponds mostly to the section size.
165 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and
167 * aligned, similar to ordinary DIMMs).
184 * Postcopy cannot handle concurrent discards and we don't want to migrate in virtio_mem_is_busy()
185 * pages on-demand with stale content when plugging new blocks. in virtio_mem_is_busy()
206 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); in virtio_mem_for_each_unplugged_range()
207 while (first_zero_bit < vmem->bitmap_size) { in virtio_mem_for_each_unplugged_range()
208 offset = first_zero_bit * vmem->block_size; in virtio_mem_for_each_unplugged_range()
209 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_range()
210 first_zero_bit + 1) - 1; in virtio_mem_for_each_unplugged_range()
211 size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; in virtio_mem_for_each_unplugged_range()
217 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_range()
230 first_bit = find_first_bit(vmem->bitmap, vmem->bitmap_size); in virtio_mem_for_each_plugged_range()
231 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_plugged_range()
232 offset = first_bit * vmem->block_size; in virtio_mem_for_each_plugged_range()
233 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_range()
234 first_bit + 1) - 1; in virtio_mem_for_each_plugged_range()
235 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_plugged_range()
241 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_range()
258 first_bit = s->offset_within_region / vmem->block_size; in virtio_mem_for_each_plugged_section()
259 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); in virtio_mem_for_each_plugged_section()
260 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_plugged_section()
263 offset = first_bit * vmem->block_size; in virtio_mem_for_each_plugged_section()
264 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_section()
265 first_bit + 1) - 1; in virtio_mem_for_each_plugged_section()
266 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_plugged_section()
275 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_plugged_section()
290 first_bit = s->offset_within_region / vmem->block_size; in virtio_mem_for_each_unplugged_section()
291 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); in virtio_mem_for_each_unplugged_section()
292 while (first_bit < vmem->bitmap_size) { in virtio_mem_for_each_unplugged_section()
295 offset = first_bit * vmem->block_size; in virtio_mem_for_each_unplugged_section()
296 last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_section()
297 first_bit + 1) - 1; in virtio_mem_for_each_unplugged_section()
298 size = (last_bit - first_bit + 1) * vmem->block_size; in virtio_mem_for_each_unplugged_section()
307 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, in virtio_mem_for_each_unplugged_section()
317 return rdl->notify_populate(rdl, s); in virtio_mem_notify_populate_cb()
324 rdl->notify_discard(rdl, s); in virtio_mem_notify_discard_cb()
333 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_unplug()
334 MemoryRegionSection tmp = *rdl->section; in virtio_mem_notify_unplug()
339 rdl->notify_discard(rdl, &tmp); in virtio_mem_notify_unplug()
349 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_plug()
350 MemoryRegionSection tmp = *rdl->section; in virtio_mem_notify_plug()
355 ret = rdl->notify_populate(rdl, &tmp); in virtio_mem_notify_plug()
362 /* Notify all already-notified listeners. */ in virtio_mem_notify_plug()
363 QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { in virtio_mem_notify_plug()
364 MemoryRegionSection tmp = *rdl2->section; in virtio_mem_notify_plug()
372 rdl2->notify_discard(rdl2, &tmp); in virtio_mem_notify_plug()
382 if (!vmem->size) { in virtio_mem_notify_unplug_all()
386 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_notify_unplug_all()
387 if (rdl->double_discard_supported) { in virtio_mem_notify_unplug_all()
388 rdl->notify_discard(rdl, rdl->section); in virtio_mem_notify_unplug_all()
390 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_notify_unplug_all()
399 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_is_range_plugged()
400 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; in virtio_mem_is_range_plugged()
403 /* We fake a shorter bitmap to avoid searching too far. */ in virtio_mem_is_range_plugged()
404 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); in virtio_mem_is_range_plugged()
411 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_is_range_unplugged()
412 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; in virtio_mem_is_range_unplugged()
415 /* We fake a shorter bitmap to avoid searching too far. */ in virtio_mem_is_range_unplugged()
416 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); in virtio_mem_is_range_unplugged()
423 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_set_range_plugged()
424 const unsigned long nbits = size / vmem->block_size; in virtio_mem_set_range_plugged()
426 bitmap_set(vmem->bitmap, bit, nbits); in virtio_mem_set_range_plugged()
432 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; in virtio_mem_set_range_unplugged()
433 const unsigned long nbits = size / vmem->block_size; in virtio_mem_set_range_unplugged()
435 bitmap_clear(vmem->bitmap, bit, nbits); in virtio_mem_set_range_unplugged()
442 VirtQueue *vq = vmem->vq; in virtio_mem_send_response()
444 trace_virtio_mem_send_response(le16_to_cpu(resp->type)); in virtio_mem_send_response()
445 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); in virtio_mem_send_response()
465 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { in virtio_mem_valid_range()
471 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { in virtio_mem_valid_range()
474 if (gpa + size > vmem->addr + vmem->usable_region_size) { in virtio_mem_valid_range()
482 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_activate_memslot()
484 assert(vmem->memslots); in virtio_mem_activate_memslot()
488 * make address space updates faster, because we don't have to loop over in virtio_mem_activate_memslot()
491 if (memory_region_is_mapped(&vmem->memslots[idx])) { in virtio_mem_activate_memslot()
494 memory_region_add_subregion(vmem->mr, memslot_offset, &vmem->memslots[idx]); in virtio_mem_activate_memslot()
499 assert(vmem->memslots); in virtio_mem_deactivate_memslot()
501 if (!memory_region_is_mapped(&vmem->memslots[idx])) { in virtio_mem_deactivate_memslot()
504 memory_region_del_subregion(vmem->mr, &vmem->memslots[idx]); in virtio_mem_deactivate_memslot()
510 const unsigned int start_idx = offset / vmem->memslot_size; in virtio_mem_activate_memslots_to_plug()
511 const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / in virtio_mem_activate_memslots_to_plug()
512 vmem->memslot_size; in virtio_mem_activate_memslots_to_plug()
515 assert(vmem->dynamic_memslots); in virtio_mem_activate_memslots_to_plug()
529 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_deactivate_unplugged_memslots()
530 const unsigned int start_idx = offset / vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
531 const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / in virtio_mem_deactivate_unplugged_memslots()
532 vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
535 assert(vmem->dynamic_memslots); in virtio_mem_deactivate_unplugged_memslots()
540 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
541 uint64_t memslot_size = vmem->memslot_size; in virtio_mem_deactivate_unplugged_memslots()
544 if (idx == vmem->nb_memslots - 1) { in virtio_mem_deactivate_unplugged_memslots()
545 memslot_size = region_size - memslot_offset; in virtio_mem_deactivate_unplugged_memslots()
550 * have to remain active if that's the case. in virtio_mem_deactivate_unplugged_memslots()
554 const uint64_t gpa = vmem->addr + memslot_offset; in virtio_mem_deactivate_unplugged_memslots()
569 const uint64_t offset = start_gpa - vmem->addr; in virtio_mem_set_block_state()
570 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_set_block_state()
574 return -EBUSY; in virtio_mem_set_block_state()
579 return -EBUSY; in virtio_mem_set_block_state()
584 if (vmem->dynamic_memslots) { in virtio_mem_set_block_state()
590 if (vmem->prealloc) { in virtio_mem_set_block_state()
591 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; in virtio_mem_set_block_state()
592 int fd = memory_region_get_fd(&vmem->memdev->mr); in virtio_mem_set_block_state()
599 * Warn only once, we don't want to fill the log with these in virtio_mem_set_block_state()
608 ret = -EBUSY; in virtio_mem_set_block_state()
622 if (vmem->dynamic_memslots) { in virtio_mem_set_block_state()
626 if (ret && vmem->dynamic_memslots) { in virtio_mem_set_block_state()
632 ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); in virtio_mem_set_block_state()
633 return -EBUSY; in virtio_mem_set_block_state()
643 const uint64_t size = nb_blocks * vmem->block_size; in virtio_mem_state_change_request()
650 if (plug && (vmem->size + size > vmem->requested_size)) { in virtio_mem_state_change_request()
665 vmem->size += size; in virtio_mem_state_change_request()
667 vmem->size -= size; in virtio_mem_state_change_request()
669 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); in virtio_mem_state_change_request()
676 const uint64_t gpa = le64_to_cpu(req->u.plug.addr); in virtio_mem_plug_request()
677 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); in virtio_mem_plug_request()
688 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); in virtio_mem_unplug_request()
689 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); in virtio_mem_unplug_request()
701 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), in virtio_mem_resize_usable_region()
704 /* The usable region size always has to be multiples of the block size. */ in virtio_mem_resize_usable_region()
705 newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); in virtio_mem_resize_usable_region()
711 if (newsize < vmem->usable_region_size && !can_shrink) { in virtio_mem_resize_usable_region()
715 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); in virtio_mem_resize_usable_region()
716 vmem->usable_region_size = newsize; in virtio_mem_resize_usable_region()
721 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_unplug_all()
722 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_unplug_all()
724 if (vmem->size) { in virtio_mem_unplug_all()
726 return -EBUSY; in virtio_mem_unplug_all()
729 return -EBUSY; in virtio_mem_unplug_all()
733 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); in virtio_mem_unplug_all()
734 vmem->size = 0; in virtio_mem_unplug_all()
735 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); in virtio_mem_unplug_all()
738 if (vmem->dynamic_memslots) { in virtio_mem_unplug_all()
744 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); in virtio_mem_unplug_all()
762 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); in virtio_mem_state_request()
763 const uint64_t gpa = le64_to_cpu(req->u.state.addr); in virtio_mem_state_request()
764 const uint64_t size = nb_blocks * vmem->block_size; in virtio_mem_state_request()
800 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { in virtio_mem_handle_request()
801 virtio_error(vdev, "virtio-mem protocol violation: invalid request" in virtio_mem_handle_request()
808 if (iov_size(elem->in_sg, elem->in_num) < in virtio_mem_handle_request()
810 virtio_error(vdev, "virtio-mem protocol violation: not enough space" in virtio_mem_handle_request()
812 iov_size(elem->in_sg, elem->in_num)); in virtio_mem_handle_request()
833 virtio_error(vdev, "virtio-mem protocol violation: unknown request" in virtio_mem_handle_request()
849 config->block_size = cpu_to_le64(vmem->block_size); in virtio_mem_get_config()
850 config->node_id = cpu_to_le16(vmem->node); in virtio_mem_get_config()
851 config->requested_size = cpu_to_le64(vmem->requested_size); in virtio_mem_get_config()
852 config->plugged_size = cpu_to_le64(vmem->size); in virtio_mem_get_config()
853 config->addr = cpu_to_le64(vmem->addr); in virtio_mem_get_config()
854 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); in virtio_mem_get_config()
855 config->usable_region_size = cpu_to_le64(vmem->usable_region_size); in virtio_mem_get_config()
864 if (ms->numa_state && acpi_builtin()) { in virtio_mem_get_features()
867 assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); in virtio_mem_get_features()
868 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { in virtio_mem_get_features()
881 return -EFAULT; in virtio_mem_validate_features()
888 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_prepare_mr()
890 assert(!vmem->mr && vmem->dynamic_memslots); in virtio_mem_prepare_mr()
891 vmem->mr = g_new0(MemoryRegion, 1); in virtio_mem_prepare_mr()
892 memory_region_init(vmem->mr, OBJECT(vmem), "virtio-mem", in virtio_mem_prepare_mr()
894 vmem->mr->align = memory_region_get_alignment(&vmem->memdev->mr); in virtio_mem_prepare_mr()
899 const uint64_t region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_prepare_memslots()
902 g_assert(!vmem->memslots && vmem->nb_memslots && vmem->dynamic_memslots); in virtio_mem_prepare_memslots()
903 vmem->memslots = g_new0(MemoryRegion, vmem->nb_memslots); in virtio_mem_prepare_memslots()
906 for (idx = 0; idx < vmem->nb_memslots; idx++) { in virtio_mem_prepare_memslots()
907 const uint64_t memslot_offset = idx * vmem->memslot_size; in virtio_mem_prepare_memslots()
908 uint64_t memslot_size = vmem->memslot_size; in virtio_mem_prepare_memslots()
912 if (idx == vmem->nb_memslots - 1) { in virtio_mem_prepare_memslots()
913 memslot_size = region_size - memslot_offset; in virtio_mem_prepare_memslots()
916 snprintf(name, sizeof(name), "memslot-%u", idx); in virtio_mem_prepare_memslots()
917 memory_region_init_alias(&vmem->memslots[idx], OBJECT(vmem), name, in virtio_mem_prepare_memslots()
918 &vmem->memdev->mr, memslot_offset, in virtio_mem_prepare_memslots()
921 * We want to be able to atomically and efficiently activate/deactivate in virtio_mem_prepare_memslots()
925 memory_region_set_unmergeable(&vmem->memslots[idx], true); in virtio_mem_prepare_memslots()
932 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; in virtio_mem_device_realize()
940 if (!vmem->memdev) { in virtio_mem_device_realize()
943 } else if (host_memory_backend_is_mapped(vmem->memdev)) { in virtio_mem_device_realize()
946 object_get_canonical_path_component(OBJECT(vmem->memdev))); in virtio_mem_device_realize()
948 } else if (!memory_region_is_ram(&vmem->memdev->mr) || in virtio_mem_device_realize()
949 memory_region_is_rom(&vmem->memdev->mr) || in virtio_mem_device_realize()
950 !vmem->memdev->mr.ram_block) { in virtio_mem_device_realize()
954 } else if (vmem->memdev->prealloc) { in virtio_mem_device_realize()
957 " virtio-mem device. ", VIRTIO_MEM_MEMDEV_PROP, in virtio_mem_device_realize()
958 object_get_canonical_path_component(OBJECT(vmem->memdev))); in virtio_mem_device_realize()
962 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || in virtio_mem_device_realize()
963 (!nb_numa_nodes && vmem->node)) { in virtio_mem_device_realize()
966 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); in virtio_mem_device_realize()
975 rb = vmem->memdev->mr.ram_block; in virtio_mem_device_realize()
979 switch (vmem->unplugged_inaccessible) { in virtio_mem_device_realize()
982 vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; in virtio_mem_device_realize()
984 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; in virtio_mem_device_realize()
989 warn_report("'%s' property set to 'off' with a memdev that does" in virtio_mem_device_realize()
998 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; in virtio_mem_device_realize()
1001 if (vmem->dynamic_memslots && in virtio_mem_device_realize()
1002 vmem->unplugged_inaccessible != ON_OFF_AUTO_ON) { in virtio_mem_device_realize()
1003 error_setg(errp, "'%s' property set to 'on' requires '%s' to be 'on'", in virtio_mem_device_realize()
1014 if (!vmem->block_size) { in virtio_mem_device_realize()
1015 vmem->block_size = virtio_mem_default_block_size(rb); in virtio_mem_device_realize()
1018 if (vmem->block_size < page_size) { in virtio_mem_device_realize()
1019 error_setg(errp, "'%s' property has to be at least the page size (0x%" in virtio_mem_device_realize()
1022 } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { in virtio_mem_device_realize()
1027 if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { in virtio_mem_device_realize()
1028 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 in virtio_mem_device_realize()
1030 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); in virtio_mem_device_realize()
1032 } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { in virtio_mem_device_realize()
1033 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 in virtio_mem_device_realize()
1035 vmem->block_size); in virtio_mem_device_realize()
1037 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), in virtio_mem_device_realize()
1038 vmem->block_size)) { in virtio_mem_device_realize()
1039 error_setg(errp, "'%s' property memdev size has to be multiples of" in virtio_mem_device_realize()
1041 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); in virtio_mem_device_realize()
1046 error_setg(errp, "Discarding RAM is disabled"); in virtio_mem_device_realize()
1054 if (memory_region_set_ram_discard_manager(&vmem->memdev->mr, in virtio_mem_device_realize()
1056 error_setg(errp, "Failed to set RamDiscardManager"); in virtio_mem_device_realize()
1062 * We don't know at this point whether shared RAM is migrated using in virtio_mem_device_realize()
1063 * QEMU or migrated using the file content. "x-ignore-shared" will be in virtio_mem_device_realize()
1074 error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); in virtio_mem_device_realize()
1075 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); in virtio_mem_device_realize()
1081 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); in virtio_mem_device_realize()
1083 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / in virtio_mem_device_realize()
1084 vmem->block_size; in virtio_mem_device_realize()
1085 vmem->bitmap = bitmap_new(vmem->bitmap_size); in virtio_mem_device_realize()
1088 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); in virtio_mem_device_realize()
1091 * With "dynamic-memslots=off" (old behavior) we always map the whole in virtio_mem_device_realize()
1092 * RAM memory region directly. in virtio_mem_device_realize()
1094 if (vmem->dynamic_memslots) { in virtio_mem_device_realize()
1095 if (!vmem->mr) { in virtio_mem_device_realize()
1098 if (vmem->nb_memslots <= 1) { in virtio_mem_device_realize()
1099 vmem->nb_memslots = 1; in virtio_mem_device_realize()
1100 vmem->memslot_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_device_realize()
1102 if (!vmem->memslots) { in virtio_mem_device_realize()
1106 assert(!vmem->mr && !vmem->nb_memslots && !vmem->memslots); in virtio_mem_device_realize()
1109 host_memory_backend_set_mapped(vmem->memdev, true); in virtio_mem_device_realize()
1110 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); in virtio_mem_device_realize()
1111 if (vmem->early_migration) { in virtio_mem_device_realize()
1117 * We only want to unplug all memory to start with a clean slate when in virtio_mem_device_realize()
1118 * it is safe for the guest -- during system resets that call in virtio_mem_device_realize()
1122 * purposes, like resetting all devices during wakeup from suspend on in virtio_mem_device_realize()
1123 * x86 based on the reset type passed to qemu_devices_reset(). in virtio_mem_device_realize()
1126 * unexpectedly losing RAM, corrupting VM state. in virtio_mem_device_realize()
1134 vmem->system_reset = VIRTIO_MEM_SYSTEM_RESET(obj); in virtio_mem_device_realize()
1135 vmem->system_reset->vmem = vmem; in virtio_mem_device_realize()
1144 qemu_unregister_resettable(OBJECT(vmem->system_reset)); in virtio_mem_device_unrealize()
1145 object_unref(OBJECT(vmem->system_reset)); in virtio_mem_device_unrealize()
1147 if (vmem->early_migration) { in virtio_mem_device_unrealize()
1151 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); in virtio_mem_device_unrealize()
1152 host_memory_backend_set_mapped(vmem->memdev, false); in virtio_mem_device_unrealize()
1155 g_free(vmem->bitmap); in virtio_mem_device_unrealize()
1160 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); in virtio_mem_device_unrealize()
1167 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_discard_range_cb()
1169 return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; in virtio_mem_discard_range_cb()
1195 if (vmem->dynamic_memslots) { in virtio_mem_post_load_bitmap()
1197 * We don't expect any active memslots at this point to deactivate: no in virtio_mem_post_load_bitmap()
1208 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { in virtio_mem_post_load_bitmap()
1209 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_post_load_bitmap()
1223 if (!vmem->early_migration) { in virtio_mem_post_load()
1231 * If shared RAM is migrated using the file content and not using QEMU, in virtio_mem_post_load()
1234 if (migrate_ram_is_ignored(vmem->memdev->mr.ram_block)) { in virtio_mem_post_load()
1238 if (vmem->prealloc && !vmem->early_migration) { in virtio_mem_post_load()
1252 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; in virtio_mem_prealloc_range_cb()
1253 int fd = memory_region_get_fd(&vmem->memdev->mr); in virtio_mem_prealloc_range_cb()
1258 return -ENOMEM; in virtio_mem_prealloc_range_cb()
1266 RAMBlock *rb = vmem->memdev->mr.ram_block; in virtio_mem_post_load_early()
1269 if (!vmem->prealloc) { in virtio_mem_post_load_early()
1274 * If shared RAM is migrated using the file content and not using QEMU, in virtio_mem_post_load_early()
1284 * memory for all plugged memory blocks, before actual RAM migration starts in virtio_mem_post_load_early()
1294 * This is tricky: postcopy wants to start with a clean slate. On in virtio_mem_post_load_early()
1296 * preallocated) RAM such that postcopy will work as expected later. in virtio_mem_post_load_early()
1298 * However, we run after POSTCOPY_INCOMING_ADVISE -- but before actual in virtio_mem_post_load_early()
1299 * RAM migration. So let's discard all memory again. This looks like an in virtio_mem_post_load_early()
1301 * were able to allocate all required backend memory once. We cannot in virtio_mem_post_load_early()
1308 return -EBUSY; in virtio_mem_post_load_early()
1313 /* Finally, update any other state to be consistent with the new bitmap. */ in virtio_mem_post_load_early()
1328 VirtIOMEM *vmem = tmp->parent; in virtio_mem_mig_sanity_checks_pre_save()
1330 tmp->addr = vmem->addr; in virtio_mem_mig_sanity_checks_pre_save()
1331 tmp->region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_mig_sanity_checks_pre_save()
1332 tmp->block_size = vmem->block_size; in virtio_mem_mig_sanity_checks_pre_save()
1333 tmp->node = vmem->node; in virtio_mem_mig_sanity_checks_pre_save()
1340 VirtIOMEM *vmem = tmp->parent; in virtio_mem_mig_sanity_checks_post_load()
1341 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_mig_sanity_checks_post_load()
1343 if (tmp->addr != vmem->addr) { in virtio_mem_mig_sanity_checks_post_load()
1344 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, in virtio_mem_mig_sanity_checks_post_load()
1345 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); in virtio_mem_mig_sanity_checks_post_load()
1346 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1352 if (tmp->region_size != new_region_size) { in virtio_mem_mig_sanity_checks_post_load()
1353 error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" in virtio_mem_mig_sanity_checks_post_load()
1354 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, in virtio_mem_mig_sanity_checks_post_load()
1356 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1358 if (tmp->block_size != vmem->block_size) { in virtio_mem_mig_sanity_checks_post_load()
1359 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, in virtio_mem_mig_sanity_checks_post_load()
1360 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, in virtio_mem_mig_sanity_checks_post_load()
1361 vmem->block_size); in virtio_mem_mig_sanity_checks_post_load()
1362 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1364 if (tmp->node != vmem->node) { in virtio_mem_mig_sanity_checks_post_load()
1365 error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, in virtio_mem_mig_sanity_checks_post_load()
1366 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); in virtio_mem_mig_sanity_checks_post_load()
1367 return -EINVAL; in virtio_mem_mig_sanity_checks_post_load()
1373 .name = "virtio-mem-device/sanity-checks",
1390 return !vmem->early_migration; in virtio_mem_vmstate_field_exists()
1394 .name = "virtio-mem-device",
1414 * such that we have have this information around before migrating any RAM
1421 * vmstate_virtio_mem_device instead -- see virtio_mem_vmstate_field_exists().
1424 .name = "virtio-mem-device-early",
1439 .name = "virtio-mem",
1451 vi->memaddr = vmem->addr; in virtio_mem_fill_device_info()
1452 vi->node = vmem->node; in virtio_mem_fill_device_info()
1453 vi->requested_size = vmem->requested_size; in virtio_mem_fill_device_info()
1454 vi->size = vmem->size; in virtio_mem_fill_device_info()
1455 vi->max_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_fill_device_info()
1456 vi->block_size = vmem->block_size; in virtio_mem_fill_device_info()
1457 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); in virtio_mem_fill_device_info()
1462 if (!vmem->memdev) { in virtio_mem_get_memory_region()
1465 } else if (vmem->dynamic_memslots) { in virtio_mem_get_memory_region()
1466 if (!vmem->mr) { in virtio_mem_get_memory_region()
1469 return vmem->mr; in virtio_mem_get_memory_region()
1472 return &vmem->memdev->mr; in virtio_mem_get_memory_region()
1481 if (!vmem->dynamic_memslots) { in virtio_mem_decide_memslots()
1486 assert(!vmem->nb_memslots); in virtio_mem_decide_memslots()
1489 if (limit <= 1 || !vmem->memdev || !vmem->memdev->mr.ram_block) { in virtio_mem_decide_memslots()
1490 vmem->nb_memslots = 1; in virtio_mem_decide_memslots()
1494 rb = vmem->memdev->mr.ram_block; in virtio_mem_decide_memslots()
1495 region_size = memory_region_size(&vmem->memdev->mr); in virtio_mem_decide_memslots()
1498 * Determine the default block size now, to determine the minimum memslot in virtio_mem_decide_memslots()
1499 * size. We want the minimum slot size to be at least the device block size. in virtio_mem_decide_memslots()
1501 if (!vmem->block_size) { in virtio_mem_decide_memslots()
1502 vmem->block_size = virtio_mem_default_block_size(rb); in virtio_mem_decide_memslots()
1505 if (vmem->block_size < qemu_ram_pagesize(rb) || in virtio_mem_decide_memslots()
1506 !QEMU_IS_ALIGNED(region_size, vmem->block_size)) { in virtio_mem_decide_memslots()
1507 vmem->nb_memslots = 1; in virtio_mem_decide_memslots()
1513 * and all memslot sizes are aligned to the device block size. in virtio_mem_decide_memslots()
1515 memslot_size = QEMU_ALIGN_UP(region_size / limit, vmem->block_size); in virtio_mem_decide_memslots()
1516 min_memslot_size = MAX(vmem->block_size, VIRTIO_MEM_MIN_MEMSLOT_SIZE); in virtio_mem_decide_memslots()
1521 vmem->memslot_size = memslot_size; in virtio_mem_decide_memslots()
1523 vmem->nb_memslots = memslots; in virtio_mem_decide_memslots()
1528 if (!vmem->dynamic_memslots) { in virtio_mem_get_memslots()
1529 /* Exactly one static RAM memory region. */ in virtio_mem_get_memslots()
1533 /* We're called after instructed to make a decision. */ in virtio_mem_get_memslots()
1534 g_assert(vmem->nb_memslots); in virtio_mem_get_memslots()
1535 return vmem->nb_memslots; in virtio_mem_get_memslots()
1541 notifier_list_add(&vmem->size_change_notifiers, notifier); in virtio_mem_add_size_change_notifier()
1554 uint64_t value = vmem->size; in virtio_mem_get_size()
1564 uint64_t value = vmem->requested_size; in virtio_mem_get_requested_size()
1584 if (DEVICE(obj)->realized) { in virtio_mem_set_requested_size()
1585 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { in virtio_mem_set_requested_size()
1586 error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 in virtio_mem_set_requested_size()
1588 vmem->block_size); in virtio_mem_set_requested_size()
1590 } else if (value > memory_region_size(&vmem->memdev->mr)) { in virtio_mem_set_requested_size()
1593 memory_region_size(&vmem->memdev->mr)); in virtio_mem_set_requested_size()
1597 if (value != vmem->requested_size) { in virtio_mem_set_requested_size()
1599 vmem->requested_size = value; in virtio_mem_set_requested_size()
1607 vmem->requested_size = value; in virtio_mem_set_requested_size()
1615 uint64_t value = vmem->block_size; in virtio_mem_get_block_size()
1622 if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { in virtio_mem_get_block_size()
1623 value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); in virtio_mem_get_block_size()
1638 if (DEVICE(obj)->realized) { in virtio_mem_set_block_size()
1648 error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, in virtio_mem_set_block_size()
1652 error_setg(errp, "'%s' property has to be a power of two", name); in virtio_mem_set_block_size()
1655 vmem->block_size = value; in virtio_mem_set_block_size()
1662 notifier_list_init(&vmem->size_change_notifiers); in virtio_mem_instance_init()
1663 QLIST_INIT(&vmem->rdl_list); in virtio_mem_instance_init()
1681 * (it's passed as the owner to memory_region_init_*()) and finalized in virtio_mem_instance_finalize()
1684 g_free(vmem->memslots); in virtio_mem_instance_finalize()
1685 vmem->memslots = NULL; in virtio_mem_instance_finalize()
1686 g_free(vmem->mr); in virtio_mem_instance_finalize()
1687 vmem->mr = NULL; in virtio_mem_instance_finalize()
1711 g_assert(mr == &vmem->memdev->mr); in virtio_mem_rdm_get_min_granularity()
1712 return vmem->block_size; in virtio_mem_rdm_get_min_granularity()
1719 uint64_t start_gpa = vmem->addr + s->offset_within_region; in virtio_mem_rdm_is_populated()
1720 uint64_t end_gpa = start_gpa + int128_get64(s->size); in virtio_mem_rdm_is_populated()
1722 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_is_populated()
1724 start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); in virtio_mem_rdm_is_populated()
1725 end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); in virtio_mem_rdm_is_populated()
1727 if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { in virtio_mem_rdm_is_populated()
1731 return virtio_mem_is_range_plugged(vmem, start_gpa, end_gpa - start_gpa); in virtio_mem_rdm_is_populated()
1743 return data->fn(s, data->opaque); in virtio_mem_rdm_replay_populated_cb()
1757 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_replay_populated()
1767 return data->fn(s, data->opaque); in virtio_mem_rdm_replay_discarded_cb()
1781 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_replay_discarded()
1793 g_assert(s->mr == &vmem->memdev->mr); in virtio_mem_rdm_register_listener()
1794 rdl->section = memory_region_section_new_copy(s); in virtio_mem_rdm_register_listener()
1796 QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); in virtio_mem_rdm_register_listener()
1797 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_rdm_register_listener()
1801 strerror(-ret)); in virtio_mem_rdm_register_listener()
1810 g_assert(rdl->section->mr == &vmem->memdev->mr); in virtio_mem_rdm_unregister_listener()
1811 if (vmem->size) { in virtio_mem_rdm_unregister_listener()
1812 if (rdl->double_discard_supported) { in virtio_mem_rdm_unregister_listener()
1813 rdl->notify_discard(rdl, rdl->section); in virtio_mem_rdm_unregister_listener()
1815 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, in virtio_mem_rdm_unregister_listener()
1820 memory_region_section_free_copy(rdl->section); in virtio_mem_rdm_unregister_listener()
1821 rdl->section = NULL; in virtio_mem_rdm_unregister_listener()
1827 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_OFF) { in virtio_mem_unplug_request_check()
1832 error_setg(errp, "virtio-mem device cannot get unplugged while" in virtio_mem_unplug_request_check()
1837 if (vmem->size) { in virtio_mem_unplug_request_check()
1838 error_setg(errp, "virtio-mem device cannot get unplugged while some" in virtio_mem_unplug_request_check()
1842 if (vmem->requested_size) { in virtio_mem_unplug_request_check()
1843 error_setg(errp, "virtio-mem device cannot get unplugged while" in virtio_mem_unplug_request_check()
1857 dc->vmsd = &vmstate_virtio_mem; in virtio_mem_class_init()
1859 set_bit(DEVICE_CATEGORY_MISC, dc->categories); in virtio_mem_class_init()
1860 vdc->realize = virtio_mem_device_realize; in virtio_mem_class_init()
1861 vdc->unrealize = virtio_mem_device_unrealize; in virtio_mem_class_init()
1862 vdc->get_config = virtio_mem_get_config; in virtio_mem_class_init()
1863 vdc->get_features = virtio_mem_get_features; in virtio_mem_class_init()
1864 vdc->validate_features = virtio_mem_validate_features; in virtio_mem_class_init()
1865 vdc->vmsd = &vmstate_virtio_mem_device; in virtio_mem_class_init()
1867 vmc->fill_device_info = virtio_mem_fill_device_info; in virtio_mem_class_init()
1868 vmc->get_memory_region = virtio_mem_get_memory_region; in virtio_mem_class_init()
1869 vmc->decide_memslots = virtio_mem_decide_memslots; in virtio_mem_class_init()
1870 vmc->get_memslots = virtio_mem_get_memslots; in virtio_mem_class_init()
1871 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; in virtio_mem_class_init()
1872 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; in virtio_mem_class_init()
1873 vmc->unplug_request_check = virtio_mem_unplug_request_check; in virtio_mem_class_init()
1875 rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; in virtio_mem_class_init()
1876 rdmc->is_populated = virtio_mem_rdm_is_populated; in virtio_mem_class_init()
1877 rdmc->replay_populated = virtio_mem_rdm_replay_populated; in virtio_mem_class_init()
1878 rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; in virtio_mem_class_init()
1879 rdmc->register_listener = virtio_mem_rdm_register_listener; in virtio_mem_class_init()
1880 rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; in virtio_mem_class_init()
1918 return &vmem_reset->reset_state; in virtio_mem_system_reset_get_state()
1924 VirtIOMEM *vmem = vmem_reset->vmem; in virtio_mem_system_reset_hold()
1927 * When waking up from standby/suspend-to-ram, do not unplug any memory. in virtio_mem_system_reset_hold()
1936 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). in virtio_mem_system_reset_hold()
1946 rc->get_state = virtio_mem_system_reset_get_state; in virtio_mem_system_reset_class_init()
1947 rc->phases.hold = virtio_mem_system_reset_hold; in virtio_mem_system_reset_class_init()