1 /* 2 * Virtio MEM device 3 * 4 * Copyright (C) 2020 Red Hat, Inc. 5 * 6 * Authors: 7 * David Hildenbrand <david@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu/iov.h" 15 #include "qemu/cutils.h" 16 #include "qemu/error-report.h" 17 #include "qemu/units.h" 18 #include "sysemu/numa.h" 19 #include "sysemu/sysemu.h" 20 #include "sysemu/reset.h" 21 #include "hw/virtio/virtio.h" 22 #include "hw/virtio/virtio-bus.h" 23 #include "hw/virtio/virtio-access.h" 24 #include "hw/virtio/virtio-mem.h" 25 #include "qapi/error.h" 26 #include "qapi/visitor.h" 27 #include "exec/ram_addr.h" 28 #include "migration/misc.h" 29 #include "hw/boards.h" 30 #include "hw/qdev-properties.h" 31 #include CONFIG_DEVICES 32 #include "trace.h" 33 34 /* 35 * We only had legacy x86 guests that did not support 36 * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests. 37 */ 38 #if defined(TARGET_X86_64) || defined(TARGET_I386) 39 #define VIRTIO_MEM_HAS_LEGACY_GUESTS 40 #endif 41 42 /* 43 * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking 44 * bitmap small. 45 */ 46 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB)) 47 48 static uint32_t virtio_mem_default_thp_size(void) 49 { 50 uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE; 51 52 #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__) 53 default_thp_size = 2 * MiB; 54 #elif defined(__aarch64__) 55 if (qemu_real_host_page_size() == 4 * KiB) { 56 default_thp_size = 2 * MiB; 57 } else if (qemu_real_host_page_size() == 16 * KiB) { 58 default_thp_size = 32 * MiB; 59 } else if (qemu_real_host_page_size() == 64 * KiB) { 60 default_thp_size = 512 * MiB; 61 } 62 #endif 63 64 return default_thp_size; 65 } 66 67 /* 68 * We want to have a reasonable default block size such that 69 * 1. We avoid splitting THPs when unplugging memory, which degrades 70 * performance. 71 * 2. We avoid placing THPs for plugged blocks that also cover unplugged 72 * blocks. 73 * 74 * The actual THP size might differ between Linux kernels, so we try to probe 75 * it. In the future (if we ever run into issues regarding 2.), we might want 76 * to disable THP in case we fail to properly probe the THP size, or if the 77 * block size is configured smaller than the THP size. 78 */ 79 static uint32_t thp_size; 80 81 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" 82 static uint32_t virtio_mem_thp_size(void) 83 { 84 gchar *content = NULL; 85 const char *endptr; 86 uint64_t tmp; 87 88 if (thp_size) { 89 return thp_size; 90 } 91 92 /* 93 * Try to probe the actual THP size, fallback to (sane but eventually 94 * incorrect) default sizes. 95 */ 96 if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) && 97 !qemu_strtou64(content, &endptr, 0, &tmp) && 98 (!endptr || *endptr == '\n')) { 99 /* Sanity-check the value and fallback to something reasonable. */ 100 if (!tmp || !is_power_of_2(tmp)) { 101 warn_report("Read unsupported THP size: %" PRIx64, tmp); 102 } else { 103 thp_size = tmp; 104 } 105 } 106 107 if (!thp_size) { 108 thp_size = virtio_mem_default_thp_size(); 109 warn_report("Could not detect THP size, falling back to %" PRIx64 110 " MiB.", thp_size / MiB); 111 } 112 113 g_free(content); 114 return thp_size; 115 } 116 117 static uint64_t virtio_mem_default_block_size(RAMBlock *rb) 118 { 119 const uint64_t page_size = qemu_ram_pagesize(rb); 120 121 /* We can have hugetlbfs with a page size smaller than the THP size. */ 122 if (page_size == qemu_real_host_page_size()) { 123 return MAX(page_size, virtio_mem_thp_size()); 124 } 125 return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE); 126 } 127 128 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 129 static bool virtio_mem_has_shared_zeropage(RAMBlock *rb) 130 { 131 /* 132 * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE 133 * anonymous RAM. In any other case, reading unplugged *can* populate a 134 * fresh page, consuming actual memory. 135 */ 136 return !qemu_ram_is_shared(rb) && rb->fd < 0 && 137 qemu_ram_pagesize(rb) == qemu_real_host_page_size(); 138 } 139 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 140 141 /* 142 * Size the usable region bigger than the requested size if possible. Esp. 143 * Linux guests will only add (aligned) memory blocks in case they fully 144 * fit into the usable region, but plug+online only a subset of the pages. 145 * The memory block size corresponds mostly to the section size. 146 * 147 * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and 148 * a section size of 512MB on arm64 (as long as the start address is properly 149 * aligned, similar to ordinary DIMMs). 150 * 151 * We can change this at any time and maybe even make it configurable if 152 * necessary (as the section size can change). But it's more likely that the 153 * section size will rather get smaller and not bigger over time. 154 */ 155 #if defined(TARGET_X86_64) || defined(TARGET_I386) 156 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) 157 #elif defined(TARGET_ARM) 158 #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB)) 159 #else 160 #error VIRTIO_MEM_USABLE_EXTENT not defined 161 #endif 162 163 static bool virtio_mem_is_busy(void) 164 { 165 /* 166 * Postcopy cannot handle concurrent discards and we don't want to migrate 167 * pages on-demand with stale content when plugging new blocks. 168 * 169 * For precopy, we don't want unplugged blocks in our migration stream, and 170 * when plugging new blocks, the page content might differ between source 171 * and destination (observable by the guest when not initializing pages 172 * after plugging them) until we're running on the destination (as we didn't 173 * migrate these blocks when they were unplugged). 174 */ 175 return migration_in_incoming_postcopy() || !migration_is_idle(); 176 } 177 178 typedef int (*virtio_mem_range_cb)(const VirtIOMEM *vmem, void *arg, 179 uint64_t offset, uint64_t size); 180 181 static int virtio_mem_for_each_unplugged_range(const VirtIOMEM *vmem, void *arg, 182 virtio_mem_range_cb cb) 183 { 184 unsigned long first_zero_bit, last_zero_bit; 185 uint64_t offset, size; 186 int ret = 0; 187 188 first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); 189 while (first_zero_bit < vmem->bitmap_size) { 190 offset = first_zero_bit * vmem->block_size; 191 last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 192 first_zero_bit + 1) - 1; 193 size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; 194 195 ret = cb(vmem, arg, offset, size); 196 if (ret) { 197 break; 198 } 199 first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 200 last_zero_bit + 2); 201 } 202 return ret; 203 } 204 205 /* 206 * Adjust the memory section to cover the intersection with the given range. 207 * 208 * Returns false if the intersection is empty, otherwise returns true. 209 */ 210 static bool virtio_mem_intersect_memory_section(MemoryRegionSection *s, 211 uint64_t offset, uint64_t size) 212 { 213 uint64_t start = MAX(s->offset_within_region, offset); 214 uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), 215 offset + size); 216 217 if (end <= start) { 218 return false; 219 } 220 221 s->offset_within_address_space += start - s->offset_within_region; 222 s->offset_within_region = start; 223 s->size = int128_make64(end - start); 224 return true; 225 } 226 227 typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg); 228 229 static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem, 230 MemoryRegionSection *s, 231 void *arg, 232 virtio_mem_section_cb cb) 233 { 234 unsigned long first_bit, last_bit; 235 uint64_t offset, size; 236 int ret = 0; 237 238 first_bit = s->offset_within_region / vmem->block_size; 239 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 240 while (first_bit < vmem->bitmap_size) { 241 MemoryRegionSection tmp = *s; 242 243 offset = first_bit * vmem->block_size; 244 last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 245 first_bit + 1) - 1; 246 size = (last_bit - first_bit + 1) * vmem->block_size; 247 248 if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { 249 break; 250 } 251 ret = cb(&tmp, arg); 252 if (ret) { 253 break; 254 } 255 first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 256 last_bit + 2); 257 } 258 return ret; 259 } 260 261 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem, 262 MemoryRegionSection *s, 263 void *arg, 264 virtio_mem_section_cb cb) 265 { 266 unsigned long first_bit, last_bit; 267 uint64_t offset, size; 268 int ret = 0; 269 270 first_bit = s->offset_within_region / vmem->block_size; 271 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); 272 while (first_bit < vmem->bitmap_size) { 273 MemoryRegionSection tmp = *s; 274 275 offset = first_bit * vmem->block_size; 276 last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, 277 first_bit + 1) - 1; 278 size = (last_bit - first_bit + 1) * vmem->block_size; 279 280 if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { 281 break; 282 } 283 ret = cb(&tmp, arg); 284 if (ret) { 285 break; 286 } 287 first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, 288 last_bit + 2); 289 } 290 return ret; 291 } 292 293 static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg) 294 { 295 RamDiscardListener *rdl = arg; 296 297 return rdl->notify_populate(rdl, s); 298 } 299 300 static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg) 301 { 302 RamDiscardListener *rdl = arg; 303 304 rdl->notify_discard(rdl, s); 305 return 0; 306 } 307 308 static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset, 309 uint64_t size) 310 { 311 RamDiscardListener *rdl; 312 313 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 314 MemoryRegionSection tmp = *rdl->section; 315 316 if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { 317 continue; 318 } 319 rdl->notify_discard(rdl, &tmp); 320 } 321 } 322 323 static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset, 324 uint64_t size) 325 { 326 RamDiscardListener *rdl, *rdl2; 327 int ret = 0; 328 329 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 330 MemoryRegionSection tmp = *rdl->section; 331 332 if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { 333 continue; 334 } 335 ret = rdl->notify_populate(rdl, &tmp); 336 if (ret) { 337 break; 338 } 339 } 340 341 if (ret) { 342 /* Notify all already-notified listeners. */ 343 QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { 344 MemoryRegionSection tmp = *rdl2->section; 345 346 if (rdl2 == rdl) { 347 break; 348 } 349 if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { 350 continue; 351 } 352 rdl2->notify_discard(rdl2, &tmp); 353 } 354 } 355 return ret; 356 } 357 358 static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem) 359 { 360 RamDiscardListener *rdl; 361 362 if (!vmem->size) { 363 return; 364 } 365 366 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 367 if (rdl->double_discard_supported) { 368 rdl->notify_discard(rdl, rdl->section); 369 } else { 370 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 371 virtio_mem_notify_discard_cb); 372 } 373 } 374 } 375 376 static bool virtio_mem_test_bitmap(const VirtIOMEM *vmem, uint64_t start_gpa, 377 uint64_t size, bool plugged) 378 { 379 const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; 380 const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; 381 unsigned long found_bit; 382 383 /* We fake a shorter bitmap to avoid searching too far. */ 384 if (plugged) { 385 found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); 386 } else { 387 found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); 388 } 389 return found_bit > last_bit; 390 } 391 392 static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, 393 uint64_t size, bool plugged) 394 { 395 const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; 396 const unsigned long nbits = size / vmem->block_size; 397 398 if (plugged) { 399 bitmap_set(vmem->bitmap, bit, nbits); 400 } else { 401 bitmap_clear(vmem->bitmap, bit, nbits); 402 } 403 } 404 405 static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, 406 struct virtio_mem_resp *resp) 407 { 408 VirtIODevice *vdev = VIRTIO_DEVICE(vmem); 409 VirtQueue *vq = vmem->vq; 410 411 trace_virtio_mem_send_response(le16_to_cpu(resp->type)); 412 iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); 413 414 virtqueue_push(vq, elem, sizeof(*resp)); 415 virtio_notify(vdev, vq); 416 } 417 418 static void virtio_mem_send_response_simple(VirtIOMEM *vmem, 419 VirtQueueElement *elem, 420 uint16_t type) 421 { 422 struct virtio_mem_resp resp = { 423 .type = cpu_to_le16(type), 424 }; 425 426 virtio_mem_send_response(vmem, elem, &resp); 427 } 428 429 static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa, 430 uint64_t size) 431 { 432 if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { 433 return false; 434 } 435 if (gpa + size < gpa || !size) { 436 return false; 437 } 438 if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { 439 return false; 440 } 441 if (gpa + size > vmem->addr + vmem->usable_region_size) { 442 return false; 443 } 444 return true; 445 } 446 447 static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, 448 uint64_t size, bool plug) 449 { 450 const uint64_t offset = start_gpa - vmem->addr; 451 RAMBlock *rb = vmem->memdev->mr.ram_block; 452 453 if (virtio_mem_is_busy()) { 454 return -EBUSY; 455 } 456 457 if (!plug) { 458 if (ram_block_discard_range(rb, offset, size)) { 459 return -EBUSY; 460 } 461 virtio_mem_notify_unplug(vmem, offset, size); 462 } else { 463 int ret = 0; 464 465 if (vmem->prealloc) { 466 void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; 467 int fd = memory_region_get_fd(&vmem->memdev->mr); 468 Error *local_err = NULL; 469 470 qemu_prealloc_mem(fd, area, size, 1, NULL, &local_err); 471 if (local_err) { 472 static bool warned; 473 474 /* 475 * Warn only once, we don't want to fill the log with these 476 * warnings. 477 */ 478 if (!warned) { 479 warn_report_err(local_err); 480 warned = true; 481 } else { 482 error_free(local_err); 483 } 484 ret = -EBUSY; 485 } 486 } 487 if (!ret) { 488 ret = virtio_mem_notify_plug(vmem, offset, size); 489 } 490 491 if (ret) { 492 /* Could be preallocation or a notifier populated memory. */ 493 ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); 494 return -EBUSY; 495 } 496 } 497 virtio_mem_set_bitmap(vmem, start_gpa, size, plug); 498 return 0; 499 } 500 501 static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, 502 uint16_t nb_blocks, bool plug) 503 { 504 const uint64_t size = nb_blocks * vmem->block_size; 505 int ret; 506 507 if (!virtio_mem_valid_range(vmem, gpa, size)) { 508 return VIRTIO_MEM_RESP_ERROR; 509 } 510 511 if (plug && (vmem->size + size > vmem->requested_size)) { 512 return VIRTIO_MEM_RESP_NACK; 513 } 514 515 /* test if really all blocks are in the opposite state */ 516 if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) { 517 return VIRTIO_MEM_RESP_ERROR; 518 } 519 520 ret = virtio_mem_set_block_state(vmem, gpa, size, plug); 521 if (ret) { 522 return VIRTIO_MEM_RESP_BUSY; 523 } 524 if (plug) { 525 vmem->size += size; 526 } else { 527 vmem->size -= size; 528 } 529 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 530 return VIRTIO_MEM_RESP_ACK; 531 } 532 533 static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 534 struct virtio_mem_req *req) 535 { 536 const uint64_t gpa = le64_to_cpu(req->u.plug.addr); 537 const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); 538 uint16_t type; 539 540 trace_virtio_mem_plug_request(gpa, nb_blocks); 541 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); 542 virtio_mem_send_response_simple(vmem, elem, type); 543 } 544 545 static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, 546 struct virtio_mem_req *req) 547 { 548 const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); 549 const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); 550 uint16_t type; 551 552 trace_virtio_mem_unplug_request(gpa, nb_blocks); 553 type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); 554 virtio_mem_send_response_simple(vmem, elem, type); 555 } 556 557 static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, 558 uint64_t requested_size, 559 bool can_shrink) 560 { 561 uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), 562 requested_size + VIRTIO_MEM_USABLE_EXTENT); 563 564 /* The usable region size always has to be multiples of the block size. */ 565 newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); 566 567 if (!requested_size) { 568 newsize = 0; 569 } 570 571 if (newsize < vmem->usable_region_size && !can_shrink) { 572 return; 573 } 574 575 trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); 576 vmem->usable_region_size = newsize; 577 } 578 579 static int virtio_mem_unplug_all(VirtIOMEM *vmem) 580 { 581 RAMBlock *rb = vmem->memdev->mr.ram_block; 582 583 if (virtio_mem_is_busy()) { 584 return -EBUSY; 585 } 586 587 if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { 588 return -EBUSY; 589 } 590 virtio_mem_notify_unplug_all(vmem); 591 592 bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); 593 if (vmem->size) { 594 vmem->size = 0; 595 notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); 596 } 597 trace_virtio_mem_unplugged_all(); 598 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 599 return 0; 600 } 601 602 static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, 603 VirtQueueElement *elem) 604 { 605 trace_virtio_mem_unplug_all_request(); 606 if (virtio_mem_unplug_all(vmem)) { 607 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); 608 } else { 609 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); 610 } 611 } 612 613 static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, 614 struct virtio_mem_req *req) 615 { 616 const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); 617 const uint64_t gpa = le64_to_cpu(req->u.state.addr); 618 const uint64_t size = nb_blocks * vmem->block_size; 619 struct virtio_mem_resp resp = { 620 .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), 621 }; 622 623 trace_virtio_mem_state_request(gpa, nb_blocks); 624 if (!virtio_mem_valid_range(vmem, gpa, size)) { 625 virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); 626 return; 627 } 628 629 if (virtio_mem_test_bitmap(vmem, gpa, size, true)) { 630 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); 631 } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) { 632 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); 633 } else { 634 resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); 635 } 636 trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); 637 virtio_mem_send_response(vmem, elem, &resp); 638 } 639 640 static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) 641 { 642 const int len = sizeof(struct virtio_mem_req); 643 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 644 VirtQueueElement *elem; 645 struct virtio_mem_req req; 646 uint16_t type; 647 648 while (true) { 649 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 650 if (!elem) { 651 return; 652 } 653 654 if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { 655 virtio_error(vdev, "virtio-mem protocol violation: invalid request" 656 " size: %d", len); 657 virtqueue_detach_element(vq, elem, 0); 658 g_free(elem); 659 return; 660 } 661 662 if (iov_size(elem->in_sg, elem->in_num) < 663 sizeof(struct virtio_mem_resp)) { 664 virtio_error(vdev, "virtio-mem protocol violation: not enough space" 665 " for response: %zu", 666 iov_size(elem->in_sg, elem->in_num)); 667 virtqueue_detach_element(vq, elem, 0); 668 g_free(elem); 669 return; 670 } 671 672 type = le16_to_cpu(req.type); 673 switch (type) { 674 case VIRTIO_MEM_REQ_PLUG: 675 virtio_mem_plug_request(vmem, elem, &req); 676 break; 677 case VIRTIO_MEM_REQ_UNPLUG: 678 virtio_mem_unplug_request(vmem, elem, &req); 679 break; 680 case VIRTIO_MEM_REQ_UNPLUG_ALL: 681 virtio_mem_unplug_all_request(vmem, elem); 682 break; 683 case VIRTIO_MEM_REQ_STATE: 684 virtio_mem_state_request(vmem, elem, &req); 685 break; 686 default: 687 virtio_error(vdev, "virtio-mem protocol violation: unknown request" 688 " type: %d", type); 689 virtqueue_detach_element(vq, elem, 0); 690 g_free(elem); 691 return; 692 } 693 694 g_free(elem); 695 } 696 } 697 698 static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) 699 { 700 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 701 struct virtio_mem_config *config = (void *) config_data; 702 703 config->block_size = cpu_to_le64(vmem->block_size); 704 config->node_id = cpu_to_le16(vmem->node); 705 config->requested_size = cpu_to_le64(vmem->requested_size); 706 config->plugged_size = cpu_to_le64(vmem->size); 707 config->addr = cpu_to_le64(vmem->addr); 708 config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); 709 config->usable_region_size = cpu_to_le64(vmem->usable_region_size); 710 } 711 712 static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, 713 Error **errp) 714 { 715 MachineState *ms = MACHINE(qdev_get_machine()); 716 VirtIOMEM *vmem = VIRTIO_MEM(vdev); 717 718 if (ms->numa_state) { 719 #if defined(CONFIG_ACPI) 720 virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); 721 #endif 722 } 723 assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); 724 if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { 725 virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE); 726 } 727 return features; 728 } 729 730 static int virtio_mem_validate_features(VirtIODevice *vdev) 731 { 732 if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) && 733 !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) { 734 return -EFAULT; 735 } 736 return 0; 737 } 738 739 static void virtio_mem_system_reset(void *opaque) 740 { 741 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 742 743 /* 744 * During usual resets, we will unplug all memory and shrink the usable 745 * region size. This is, however, not possible in all scenarios. Then, 746 * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). 747 */ 748 virtio_mem_unplug_all(vmem); 749 } 750 751 static void virtio_mem_device_realize(DeviceState *dev, Error **errp) 752 { 753 MachineState *ms = MACHINE(qdev_get_machine()); 754 int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; 755 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 756 VirtIOMEM *vmem = VIRTIO_MEM(dev); 757 uint64_t page_size; 758 RAMBlock *rb; 759 int ret; 760 761 if (!vmem->memdev) { 762 error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); 763 return; 764 } else if (host_memory_backend_is_mapped(vmem->memdev)) { 765 error_setg(errp, "'%s' property specifies a busy memdev: %s", 766 VIRTIO_MEM_MEMDEV_PROP, 767 object_get_canonical_path_component(OBJECT(vmem->memdev))); 768 return; 769 } else if (!memory_region_is_ram(&vmem->memdev->mr) || 770 memory_region_is_rom(&vmem->memdev->mr) || 771 !vmem->memdev->mr.ram_block) { 772 error_setg(errp, "'%s' property specifies an unsupported memdev", 773 VIRTIO_MEM_MEMDEV_PROP); 774 return; 775 } 776 777 if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || 778 (!nb_numa_nodes && vmem->node)) { 779 error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" 780 "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, 781 vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); 782 return; 783 } 784 785 if (enable_mlock) { 786 error_setg(errp, "Incompatible with mlock"); 787 return; 788 } 789 790 rb = vmem->memdev->mr.ram_block; 791 page_size = qemu_ram_pagesize(rb); 792 793 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 794 switch (vmem->unplugged_inaccessible) { 795 case ON_OFF_AUTO_AUTO: 796 if (virtio_mem_has_shared_zeropage(rb)) { 797 vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; 798 } else { 799 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 800 } 801 break; 802 case ON_OFF_AUTO_OFF: 803 if (!virtio_mem_has_shared_zeropage(rb)) { 804 warn_report("'%s' property set to 'off' with a memdev that does" 805 " not support the shared zeropage.", 806 VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); 807 } 808 break; 809 default: 810 break; 811 } 812 #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 813 vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; 814 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ 815 816 /* 817 * If the block size wasn't configured by the user, use a sane default. This 818 * allows using hugetlbfs backends of any page size without manual 819 * intervention. 820 */ 821 if (!vmem->block_size) { 822 vmem->block_size = virtio_mem_default_block_size(rb); 823 } 824 825 if (vmem->block_size < page_size) { 826 error_setg(errp, "'%s' property has to be at least the page size (0x%" 827 PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); 828 return; 829 } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { 830 warn_report("'%s' property is smaller than the default block size (%" 831 PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP, 832 virtio_mem_default_block_size(rb) / MiB); 833 } 834 if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { 835 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 836 ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, 837 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 838 return; 839 } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { 840 error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 841 ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, 842 vmem->block_size); 843 return; 844 } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), 845 vmem->block_size)) { 846 error_setg(errp, "'%s' property memdev size has to be multiples of" 847 "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, 848 VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); 849 return; 850 } 851 852 if (ram_block_coordinated_discard_require(true)) { 853 error_setg(errp, "Discarding RAM is disabled"); 854 return; 855 } 856 857 ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); 858 if (ret) { 859 error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); 860 ram_block_coordinated_discard_require(false); 861 return; 862 } 863 864 virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); 865 866 vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / 867 vmem->block_size; 868 vmem->bitmap = bitmap_new(vmem->bitmap_size); 869 870 virtio_init(vdev, VIRTIO_ID_MEM, sizeof(struct virtio_mem_config)); 871 vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); 872 873 host_memory_backend_set_mapped(vmem->memdev, true); 874 vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); 875 qemu_register_reset(virtio_mem_system_reset, vmem); 876 877 /* 878 * Set ourselves as RamDiscardManager before the plug handler maps the 879 * memory region and exposes it via an address space. 880 */ 881 memory_region_set_ram_discard_manager(&vmem->memdev->mr, 882 RAM_DISCARD_MANAGER(vmem)); 883 } 884 885 static void virtio_mem_device_unrealize(DeviceState *dev) 886 { 887 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 888 VirtIOMEM *vmem = VIRTIO_MEM(dev); 889 890 /* 891 * The unplug handler unmapped the memory region, it cannot be 892 * found via an address space anymore. Unset ourselves. 893 */ 894 memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); 895 qemu_unregister_reset(virtio_mem_system_reset, vmem); 896 vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); 897 host_memory_backend_set_mapped(vmem->memdev, false); 898 virtio_del_queue(vdev, 0); 899 virtio_cleanup(vdev); 900 g_free(vmem->bitmap); 901 ram_block_coordinated_discard_require(false); 902 } 903 904 static int virtio_mem_discard_range_cb(const VirtIOMEM *vmem, void *arg, 905 uint64_t offset, uint64_t size) 906 { 907 RAMBlock *rb = vmem->memdev->mr.ram_block; 908 909 return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; 910 } 911 912 static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) 913 { 914 /* Make sure all memory is really discarded after migration. */ 915 return virtio_mem_for_each_unplugged_range(vmem, NULL, 916 virtio_mem_discard_range_cb); 917 } 918 919 static int virtio_mem_post_load(void *opaque, int version_id) 920 { 921 VirtIOMEM *vmem = VIRTIO_MEM(opaque); 922 RamDiscardListener *rdl; 923 int ret; 924 925 /* 926 * We started out with all memory discarded and our memory region is mapped 927 * into an address space. Replay, now that we updated the bitmap. 928 */ 929 QLIST_FOREACH(rdl, &vmem->rdl_list, next) { 930 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 931 virtio_mem_notify_populate_cb); 932 if (ret) { 933 return ret; 934 } 935 } 936 937 if (migration_in_incoming_postcopy()) { 938 return 0; 939 } 940 941 return virtio_mem_restore_unplugged(vmem); 942 } 943 944 typedef struct VirtIOMEMMigSanityChecks { 945 VirtIOMEM *parent; 946 uint64_t addr; 947 uint64_t region_size; 948 uint64_t block_size; 949 uint32_t node; 950 } VirtIOMEMMigSanityChecks; 951 952 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) 953 { 954 VirtIOMEMMigSanityChecks *tmp = opaque; 955 VirtIOMEM *vmem = tmp->parent; 956 957 tmp->addr = vmem->addr; 958 tmp->region_size = memory_region_size(&vmem->memdev->mr); 959 tmp->block_size = vmem->block_size; 960 tmp->node = vmem->node; 961 return 0; 962 } 963 964 static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) 965 { 966 VirtIOMEMMigSanityChecks *tmp = opaque; 967 VirtIOMEM *vmem = tmp->parent; 968 const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); 969 970 if (tmp->addr != vmem->addr) { 971 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 972 VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); 973 return -EINVAL; 974 } 975 /* 976 * Note: Preparation for resizeable memory regions. The maximum size 977 * of the memory region must not change during migration. 978 */ 979 if (tmp->region_size != new_region_size) { 980 error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" 981 PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, 982 new_region_size); 983 return -EINVAL; 984 } 985 if (tmp->block_size != vmem->block_size) { 986 error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, 987 VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, 988 vmem->block_size); 989 return -EINVAL; 990 } 991 if (tmp->node != vmem->node) { 992 error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, 993 VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); 994 return -EINVAL; 995 } 996 return 0; 997 } 998 999 static const VMStateDescription vmstate_virtio_mem_sanity_checks = { 1000 .name = "virtio-mem-device/sanity-checks", 1001 .pre_save = virtio_mem_mig_sanity_checks_pre_save, 1002 .post_load = virtio_mem_mig_sanity_checks_post_load, 1003 .fields = (VMStateField[]) { 1004 VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), 1005 VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), 1006 VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), 1007 VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), 1008 VMSTATE_END_OF_LIST(), 1009 }, 1010 }; 1011 1012 static const VMStateDescription vmstate_virtio_mem_device = { 1013 .name = "virtio-mem-device", 1014 .minimum_version_id = 1, 1015 .version_id = 1, 1016 .priority = MIG_PRI_VIRTIO_MEM, 1017 .post_load = virtio_mem_post_load, 1018 .fields = (VMStateField[]) { 1019 VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, 1020 vmstate_virtio_mem_sanity_checks), 1021 VMSTATE_UINT64(usable_region_size, VirtIOMEM), 1022 VMSTATE_UINT64(size, VirtIOMEM), 1023 VMSTATE_UINT64(requested_size, VirtIOMEM), 1024 VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), 1025 VMSTATE_END_OF_LIST() 1026 }, 1027 }; 1028 1029 static const VMStateDescription vmstate_virtio_mem = { 1030 .name = "virtio-mem", 1031 .minimum_version_id = 1, 1032 .version_id = 1, 1033 .fields = (VMStateField[]) { 1034 VMSTATE_VIRTIO_DEVICE, 1035 VMSTATE_END_OF_LIST() 1036 }, 1037 }; 1038 1039 static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, 1040 VirtioMEMDeviceInfo *vi) 1041 { 1042 vi->memaddr = vmem->addr; 1043 vi->node = vmem->node; 1044 vi->requested_size = vmem->requested_size; 1045 vi->size = vmem->size; 1046 vi->max_size = memory_region_size(&vmem->memdev->mr); 1047 vi->block_size = vmem->block_size; 1048 vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); 1049 } 1050 1051 static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) 1052 { 1053 if (!vmem->memdev) { 1054 error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); 1055 return NULL; 1056 } 1057 1058 return &vmem->memdev->mr; 1059 } 1060 1061 static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, 1062 Notifier *notifier) 1063 { 1064 notifier_list_add(&vmem->size_change_notifiers, notifier); 1065 } 1066 1067 static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, 1068 Notifier *notifier) 1069 { 1070 notifier_remove(notifier); 1071 } 1072 1073 static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, 1074 void *opaque, Error **errp) 1075 { 1076 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1077 uint64_t value = vmem->size; 1078 1079 visit_type_size(v, name, &value, errp); 1080 } 1081 1082 static void virtio_mem_get_requested_size(Object *obj, Visitor *v, 1083 const char *name, void *opaque, 1084 Error **errp) 1085 { 1086 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1087 uint64_t value = vmem->requested_size; 1088 1089 visit_type_size(v, name, &value, errp); 1090 } 1091 1092 static void virtio_mem_set_requested_size(Object *obj, Visitor *v, 1093 const char *name, void *opaque, 1094 Error **errp) 1095 { 1096 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1097 uint64_t value; 1098 1099 if (!visit_type_size(v, name, &value, errp)) { 1100 return; 1101 } 1102 1103 /* 1104 * The block size and memory backend are not fixed until the device was 1105 * realized. realize() will verify these properties then. 1106 */ 1107 if (DEVICE(obj)->realized) { 1108 if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { 1109 error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 1110 ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, 1111 vmem->block_size); 1112 return; 1113 } else if (value > memory_region_size(&vmem->memdev->mr)) { 1114 error_setg(errp, "'%s' cannot exceed the memory backend size" 1115 "(0x%" PRIx64 ")", name, 1116 memory_region_size(&vmem->memdev->mr)); 1117 return; 1118 } 1119 1120 if (value != vmem->requested_size) { 1121 virtio_mem_resize_usable_region(vmem, value, false); 1122 vmem->requested_size = value; 1123 } 1124 /* 1125 * Trigger a config update so the guest gets notified. We trigger 1126 * even if the size didn't change (especially helpful for debugging). 1127 */ 1128 virtio_notify_config(VIRTIO_DEVICE(vmem)); 1129 } else { 1130 vmem->requested_size = value; 1131 } 1132 } 1133 1134 static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, 1135 void *opaque, Error **errp) 1136 { 1137 const VirtIOMEM *vmem = VIRTIO_MEM(obj); 1138 uint64_t value = vmem->block_size; 1139 1140 /* 1141 * If not configured by the user (and we're not realized yet), use the 1142 * default block size we would use with the current memory backend. 1143 */ 1144 if (!value) { 1145 if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { 1146 value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); 1147 } else { 1148 value = virtio_mem_thp_size(); 1149 } 1150 } 1151 1152 visit_type_size(v, name, &value, errp); 1153 } 1154 1155 static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, 1156 void *opaque, Error **errp) 1157 { 1158 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1159 uint64_t value; 1160 1161 if (DEVICE(obj)->realized) { 1162 error_setg(errp, "'%s' cannot be changed", name); 1163 return; 1164 } 1165 1166 if (!visit_type_size(v, name, &value, errp)) { 1167 return; 1168 } 1169 1170 if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { 1171 error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, 1172 VIRTIO_MEM_MIN_BLOCK_SIZE); 1173 return; 1174 } else if (!is_power_of_2(value)) { 1175 error_setg(errp, "'%s' property has to be a power of two", name); 1176 return; 1177 } 1178 vmem->block_size = value; 1179 } 1180 1181 static void virtio_mem_instance_init(Object *obj) 1182 { 1183 VirtIOMEM *vmem = VIRTIO_MEM(obj); 1184 1185 notifier_list_init(&vmem->size_change_notifiers); 1186 QLIST_INIT(&vmem->rdl_list); 1187 1188 object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, 1189 NULL, NULL, NULL); 1190 object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", 1191 virtio_mem_get_requested_size, 1192 virtio_mem_set_requested_size, NULL, NULL); 1193 object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", 1194 virtio_mem_get_block_size, virtio_mem_set_block_size, 1195 NULL, NULL); 1196 } 1197 1198 static Property virtio_mem_properties[] = { 1199 DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), 1200 DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), 1201 DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false), 1202 DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, 1203 TYPE_MEMORY_BACKEND, HostMemoryBackend *), 1204 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) 1205 DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM, 1206 unplugged_inaccessible, ON_OFF_AUTO_AUTO), 1207 #endif 1208 DEFINE_PROP_END_OF_LIST(), 1209 }; 1210 1211 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm, 1212 const MemoryRegion *mr) 1213 { 1214 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1215 1216 g_assert(mr == &vmem->memdev->mr); 1217 return vmem->block_size; 1218 } 1219 1220 static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm, 1221 const MemoryRegionSection *s) 1222 { 1223 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1224 uint64_t start_gpa = vmem->addr + s->offset_within_region; 1225 uint64_t end_gpa = start_gpa + int128_get64(s->size); 1226 1227 g_assert(s->mr == &vmem->memdev->mr); 1228 1229 start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); 1230 end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); 1231 1232 if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { 1233 return false; 1234 } 1235 1236 return virtio_mem_test_bitmap(vmem, start_gpa, end_gpa - start_gpa, true); 1237 } 1238 1239 struct VirtIOMEMReplayData { 1240 void *fn; 1241 void *opaque; 1242 }; 1243 1244 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg) 1245 { 1246 struct VirtIOMEMReplayData *data = arg; 1247 1248 return ((ReplayRamPopulate)data->fn)(s, data->opaque); 1249 } 1250 1251 static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm, 1252 MemoryRegionSection *s, 1253 ReplayRamPopulate replay_fn, 1254 void *opaque) 1255 { 1256 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1257 struct VirtIOMEMReplayData data = { 1258 .fn = replay_fn, 1259 .opaque = opaque, 1260 }; 1261 1262 g_assert(s->mr == &vmem->memdev->mr); 1263 return virtio_mem_for_each_plugged_section(vmem, s, &data, 1264 virtio_mem_rdm_replay_populated_cb); 1265 } 1266 1267 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s, 1268 void *arg) 1269 { 1270 struct VirtIOMEMReplayData *data = arg; 1271 1272 ((ReplayRamDiscard)data->fn)(s, data->opaque); 1273 return 0; 1274 } 1275 1276 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm, 1277 MemoryRegionSection *s, 1278 ReplayRamDiscard replay_fn, 1279 void *opaque) 1280 { 1281 const VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1282 struct VirtIOMEMReplayData data = { 1283 .fn = replay_fn, 1284 .opaque = opaque, 1285 }; 1286 1287 g_assert(s->mr == &vmem->memdev->mr); 1288 virtio_mem_for_each_unplugged_section(vmem, s, &data, 1289 virtio_mem_rdm_replay_discarded_cb); 1290 } 1291 1292 static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm, 1293 RamDiscardListener *rdl, 1294 MemoryRegionSection *s) 1295 { 1296 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1297 int ret; 1298 1299 g_assert(s->mr == &vmem->memdev->mr); 1300 rdl->section = memory_region_section_new_copy(s); 1301 1302 QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); 1303 ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1304 virtio_mem_notify_populate_cb); 1305 if (ret) { 1306 error_report("%s: Replaying plugged ranges failed: %s", __func__, 1307 strerror(-ret)); 1308 } 1309 } 1310 1311 static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm, 1312 RamDiscardListener *rdl) 1313 { 1314 VirtIOMEM *vmem = VIRTIO_MEM(rdm); 1315 1316 g_assert(rdl->section->mr == &vmem->memdev->mr); 1317 if (vmem->size) { 1318 if (rdl->double_discard_supported) { 1319 rdl->notify_discard(rdl, rdl->section); 1320 } else { 1321 virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, 1322 virtio_mem_notify_discard_cb); 1323 } 1324 } 1325 1326 memory_region_section_free_copy(rdl->section); 1327 rdl->section = NULL; 1328 QLIST_REMOVE(rdl, next); 1329 } 1330 1331 static void virtio_mem_class_init(ObjectClass *klass, void *data) 1332 { 1333 DeviceClass *dc = DEVICE_CLASS(klass); 1334 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1335 VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); 1336 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); 1337 1338 device_class_set_props(dc, virtio_mem_properties); 1339 dc->vmsd = &vmstate_virtio_mem; 1340 1341 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1342 vdc->realize = virtio_mem_device_realize; 1343 vdc->unrealize = virtio_mem_device_unrealize; 1344 vdc->get_config = virtio_mem_get_config; 1345 vdc->get_features = virtio_mem_get_features; 1346 vdc->validate_features = virtio_mem_validate_features; 1347 vdc->vmsd = &vmstate_virtio_mem_device; 1348 1349 vmc->fill_device_info = virtio_mem_fill_device_info; 1350 vmc->get_memory_region = virtio_mem_get_memory_region; 1351 vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; 1352 vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; 1353 1354 rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; 1355 rdmc->is_populated = virtio_mem_rdm_is_populated; 1356 rdmc->replay_populated = virtio_mem_rdm_replay_populated; 1357 rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; 1358 rdmc->register_listener = virtio_mem_rdm_register_listener; 1359 rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; 1360 } 1361 1362 static const TypeInfo virtio_mem_info = { 1363 .name = TYPE_VIRTIO_MEM, 1364 .parent = TYPE_VIRTIO_DEVICE, 1365 .instance_size = sizeof(VirtIOMEM), 1366 .instance_init = virtio_mem_instance_init, 1367 .class_init = virtio_mem_class_init, 1368 .class_size = sizeof(VirtIOMEMClass), 1369 .interfaces = (InterfaceInfo[]) { 1370 { TYPE_RAM_DISCARD_MANAGER }, 1371 { } 1372 }, 1373 }; 1374 1375 static void virtio_register_types(void) 1376 { 1377 type_register_static(&virtio_mem_info); 1378 } 1379 1380 type_init(virtio_register_types) 1381