xref: /openbmc/qemu/hw/virtio/virtio-mem.c (revision d6cd3ae0)
1 /*
2  * Virtio MEM device
3  *
4  * Copyright (C) 2020 Red Hat, Inc.
5  *
6  * Authors:
7  *  David Hildenbrand <david@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.
10  * See the COPYING file in the top-level directory.
11  */
12 
13 #include "qemu/osdep.h"
14 #include "qemu/iov.h"
15 #include "qemu/cutils.h"
16 #include "qemu/error-report.h"
17 #include "qemu/units.h"
18 #include "sysemu/numa.h"
19 #include "sysemu/sysemu.h"
20 #include "sysemu/reset.h"
21 #include "hw/virtio/virtio.h"
22 #include "hw/virtio/virtio-bus.h"
23 #include "hw/virtio/virtio-access.h"
24 #include "hw/virtio/virtio-mem.h"
25 #include "qapi/error.h"
26 #include "qapi/visitor.h"
27 #include "exec/ram_addr.h"
28 #include "migration/misc.h"
29 #include "hw/boards.h"
30 #include "hw/qdev-properties.h"
31 #include CONFIG_DEVICES
32 #include "trace.h"
33 
34 /*
35  * We only had legacy x86 guests that did not support
36  * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests.
37  */
38 #if defined(TARGET_X86_64) || defined(TARGET_I386)
39 #define VIRTIO_MEM_HAS_LEGACY_GUESTS
40 #endif
41 
42 /*
43  * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking
44  * bitmap small.
45  */
46 #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB))
47 
48 static uint32_t virtio_mem_default_thp_size(void)
49 {
50     uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE;
51 
52 #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__)
53     default_thp_size = 2 * MiB;
54 #elif defined(__aarch64__)
55     if (qemu_real_host_page_size() == 4 * KiB) {
56         default_thp_size = 2 * MiB;
57     } else if (qemu_real_host_page_size() == 16 * KiB) {
58         default_thp_size = 32 * MiB;
59     } else if (qemu_real_host_page_size() == 64 * KiB) {
60         default_thp_size = 512 * MiB;
61     }
62 #endif
63 
64     return default_thp_size;
65 }
66 
67 /*
68  * We want to have a reasonable default block size such that
69  * 1. We avoid splitting THPs when unplugging memory, which degrades
70  *    performance.
71  * 2. We avoid placing THPs for plugged blocks that also cover unplugged
72  *    blocks.
73  *
74  * The actual THP size might differ between Linux kernels, so we try to probe
75  * it. In the future (if we ever run into issues regarding 2.), we might want
76  * to disable THP in case we fail to properly probe the THP size, or if the
77  * block size is configured smaller than the THP size.
78  */
79 static uint32_t thp_size;
80 
81 #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
82 static uint32_t virtio_mem_thp_size(void)
83 {
84     gchar *content = NULL;
85     const char *endptr;
86     uint64_t tmp;
87 
88     if (thp_size) {
89         return thp_size;
90     }
91 
92     /*
93      * Try to probe the actual THP size, fallback to (sane but eventually
94      * incorrect) default sizes.
95      */
96     if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) &&
97         !qemu_strtou64(content, &endptr, 0, &tmp) &&
98         (!endptr || *endptr == '\n')) {
99         /* Sanity-check the value and fallback to something reasonable. */
100         if (!tmp || !is_power_of_2(tmp)) {
101             warn_report("Read unsupported THP size: %" PRIx64, tmp);
102         } else {
103             thp_size = tmp;
104         }
105     }
106 
107     if (!thp_size) {
108         thp_size = virtio_mem_default_thp_size();
109         warn_report("Could not detect THP size, falling back to %" PRIx64
110                     "  MiB.", thp_size / MiB);
111     }
112 
113     g_free(content);
114     return thp_size;
115 }
116 
117 static uint64_t virtio_mem_default_block_size(RAMBlock *rb)
118 {
119     const uint64_t page_size = qemu_ram_pagesize(rb);
120 
121     /* We can have hugetlbfs with a page size smaller than the THP size. */
122     if (page_size == qemu_real_host_page_size()) {
123         return MAX(page_size, virtio_mem_thp_size());
124     }
125     return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE);
126 }
127 
128 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
129 static bool virtio_mem_has_shared_zeropage(RAMBlock *rb)
130 {
131     /*
132      * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE
133      * anonymous RAM. In any other case, reading unplugged *can* populate a
134      * fresh page, consuming actual memory.
135      */
136     return !qemu_ram_is_shared(rb) && rb->fd < 0 &&
137            qemu_ram_pagesize(rb) == qemu_real_host_page_size();
138 }
139 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
140 
141 /*
142  * Size the usable region bigger than the requested size if possible. Esp.
143  * Linux guests will only add (aligned) memory blocks in case they fully
144  * fit into the usable region, but plug+online only a subset of the pages.
145  * The memory block size corresponds mostly to the section size.
146  *
147  * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and
148  * a section size of 512MB on arm64 (as long as the start address is properly
149  * aligned, similar to ordinary DIMMs).
150  *
151  * We can change this at any time and maybe even make it configurable if
152  * necessary (as the section size can change). But it's more likely that the
153  * section size will rather get smaller and not bigger over time.
154  */
155 #if defined(TARGET_X86_64) || defined(TARGET_I386)
156 #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB))
157 #elif defined(TARGET_ARM)
158 #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB))
159 #else
160 #error VIRTIO_MEM_USABLE_EXTENT not defined
161 #endif
162 
163 static bool virtio_mem_is_busy(void)
164 {
165     /*
166      * Postcopy cannot handle concurrent discards and we don't want to migrate
167      * pages on-demand with stale content when plugging new blocks.
168      *
169      * For precopy, we don't want unplugged blocks in our migration stream, and
170      * when plugging new blocks, the page content might differ between source
171      * and destination (observable by the guest when not initializing pages
172      * after plugging them) until we're running on the destination (as we didn't
173      * migrate these blocks when they were unplugged).
174      */
175     return migration_in_incoming_postcopy() || !migration_is_idle();
176 }
177 
178 typedef int (*virtio_mem_range_cb)(const VirtIOMEM *vmem, void *arg,
179                                    uint64_t offset, uint64_t size);
180 
181 static int virtio_mem_for_each_unplugged_range(const VirtIOMEM *vmem, void *arg,
182                                                virtio_mem_range_cb cb)
183 {
184     unsigned long first_zero_bit, last_zero_bit;
185     uint64_t offset, size;
186     int ret = 0;
187 
188     first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size);
189     while (first_zero_bit < vmem->bitmap_size) {
190         offset = first_zero_bit * vmem->block_size;
191         last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
192                                       first_zero_bit + 1) - 1;
193         size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size;
194 
195         ret = cb(vmem, arg, offset, size);
196         if (ret) {
197             break;
198         }
199         first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
200                                             last_zero_bit + 2);
201     }
202     return ret;
203 }
204 
205 /*
206  * Adjust the memory section to cover the intersection with the given range.
207  *
208  * Returns false if the intersection is empty, otherwise returns true.
209  */
210 static bool virito_mem_intersect_memory_section(MemoryRegionSection *s,
211                                                 uint64_t offset, uint64_t size)
212 {
213     uint64_t start = MAX(s->offset_within_region, offset);
214     uint64_t end = MIN(s->offset_within_region + int128_get64(s->size),
215                        offset + size);
216 
217     if (end <= start) {
218         return false;
219     }
220 
221     s->offset_within_address_space += start - s->offset_within_region;
222     s->offset_within_region = start;
223     s->size = int128_make64(end - start);
224     return true;
225 }
226 
227 typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg);
228 
229 static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem,
230                                                MemoryRegionSection *s,
231                                                void *arg,
232                                                virtio_mem_section_cb cb)
233 {
234     unsigned long first_bit, last_bit;
235     uint64_t offset, size;
236     int ret = 0;
237 
238     first_bit = s->offset_within_region / vmem->bitmap_size;
239     first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit);
240     while (first_bit < vmem->bitmap_size) {
241         MemoryRegionSection tmp = *s;
242 
243         offset = first_bit * vmem->block_size;
244         last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
245                                       first_bit + 1) - 1;
246         size = (last_bit - first_bit + 1) * vmem->block_size;
247 
248         if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
249             break;
250         }
251         ret = cb(&tmp, arg);
252         if (ret) {
253             break;
254         }
255         first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
256                                   last_bit + 2);
257     }
258     return ret;
259 }
260 
261 static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem,
262                                                  MemoryRegionSection *s,
263                                                  void *arg,
264                                                  virtio_mem_section_cb cb)
265 {
266     unsigned long first_bit, last_bit;
267     uint64_t offset, size;
268     int ret = 0;
269 
270     first_bit = s->offset_within_region / vmem->bitmap_size;
271     first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit);
272     while (first_bit < vmem->bitmap_size) {
273         MemoryRegionSection tmp = *s;
274 
275         offset = first_bit * vmem->block_size;
276         last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size,
277                                  first_bit + 1) - 1;
278         size = (last_bit - first_bit + 1) * vmem->block_size;
279 
280         if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
281             break;
282         }
283         ret = cb(&tmp, arg);
284         if (ret) {
285             break;
286         }
287         first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size,
288                                        last_bit + 2);
289     }
290     return ret;
291 }
292 
293 static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg)
294 {
295     RamDiscardListener *rdl = arg;
296 
297     return rdl->notify_populate(rdl, s);
298 }
299 
300 static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg)
301 {
302     RamDiscardListener *rdl = arg;
303 
304     rdl->notify_discard(rdl, s);
305     return 0;
306 }
307 
308 static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset,
309                                      uint64_t size)
310 {
311     RamDiscardListener *rdl;
312 
313     QLIST_FOREACH(rdl, &vmem->rdl_list, next) {
314         MemoryRegionSection tmp = *rdl->section;
315 
316         if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
317             continue;
318         }
319         rdl->notify_discard(rdl, &tmp);
320     }
321 }
322 
323 static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset,
324                                   uint64_t size)
325 {
326     RamDiscardListener *rdl, *rdl2;
327     int ret = 0;
328 
329     QLIST_FOREACH(rdl, &vmem->rdl_list, next) {
330         MemoryRegionSection tmp = *rdl->section;
331 
332         if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
333             continue;
334         }
335         ret = rdl->notify_populate(rdl, &tmp);
336         if (ret) {
337             break;
338         }
339     }
340 
341     if (ret) {
342         /* Notify all already-notified listeners. */
343         QLIST_FOREACH(rdl2, &vmem->rdl_list, next) {
344             MemoryRegionSection tmp = *rdl->section;
345 
346             if (rdl2 == rdl) {
347                 break;
348             }
349             if (!virito_mem_intersect_memory_section(&tmp, offset, size)) {
350                 continue;
351             }
352             rdl2->notify_discard(rdl2, &tmp);
353         }
354     }
355     return ret;
356 }
357 
358 static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem)
359 {
360     RamDiscardListener *rdl;
361 
362     if (!vmem->size) {
363         return;
364     }
365 
366     QLIST_FOREACH(rdl, &vmem->rdl_list, next) {
367         if (rdl->double_discard_supported) {
368             rdl->notify_discard(rdl, rdl->section);
369         } else {
370             virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl,
371                                                 virtio_mem_notify_discard_cb);
372         }
373     }
374 }
375 
376 static bool virtio_mem_test_bitmap(const VirtIOMEM *vmem, uint64_t start_gpa,
377                                    uint64_t size, bool plugged)
378 {
379     const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size;
380     const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1;
381     unsigned long found_bit;
382 
383     /* We fake a shorter bitmap to avoid searching too far. */
384     if (plugged) {
385         found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit);
386     } else {
387         found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit);
388     }
389     return found_bit > last_bit;
390 }
391 
392 static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa,
393                                   uint64_t size, bool plugged)
394 {
395     const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size;
396     const unsigned long nbits = size / vmem->block_size;
397 
398     if (plugged) {
399         bitmap_set(vmem->bitmap, bit, nbits);
400     } else {
401         bitmap_clear(vmem->bitmap, bit, nbits);
402     }
403 }
404 
405 static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem,
406                                      struct virtio_mem_resp *resp)
407 {
408     VirtIODevice *vdev = VIRTIO_DEVICE(vmem);
409     VirtQueue *vq = vmem->vq;
410 
411     trace_virtio_mem_send_response(le16_to_cpu(resp->type));
412     iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp));
413 
414     virtqueue_push(vq, elem, sizeof(*resp));
415     virtio_notify(vdev, vq);
416 }
417 
418 static void virtio_mem_send_response_simple(VirtIOMEM *vmem,
419                                             VirtQueueElement *elem,
420                                             uint16_t type)
421 {
422     struct virtio_mem_resp resp = {
423         .type = cpu_to_le16(type),
424     };
425 
426     virtio_mem_send_response(vmem, elem, &resp);
427 }
428 
429 static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa,
430                                    uint64_t size)
431 {
432     if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) {
433         return false;
434     }
435     if (gpa + size < gpa || !size) {
436         return false;
437     }
438     if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) {
439         return false;
440     }
441     if (gpa + size > vmem->addr + vmem->usable_region_size) {
442         return false;
443     }
444     return true;
445 }
446 
447 static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa,
448                                       uint64_t size, bool plug)
449 {
450     const uint64_t offset = start_gpa - vmem->addr;
451     RAMBlock *rb = vmem->memdev->mr.ram_block;
452 
453     if (virtio_mem_is_busy()) {
454         return -EBUSY;
455     }
456 
457     if (!plug) {
458         if (ram_block_discard_range(rb, offset, size)) {
459             return -EBUSY;
460         }
461         virtio_mem_notify_unplug(vmem, offset, size);
462     } else {
463         int ret = 0;
464 
465         if (vmem->prealloc) {
466             void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset;
467             int fd = memory_region_get_fd(&vmem->memdev->mr);
468             Error *local_err = NULL;
469 
470             os_mem_prealloc(fd, area, size, 1, &local_err);
471             if (local_err) {
472                 static bool warned;
473 
474                 /*
475                  * Warn only once, we don't want to fill the log with these
476                  * warnings.
477                  */
478                 if (!warned) {
479                     warn_report_err(local_err);
480                     warned = true;
481                 } else {
482                     error_free(local_err);
483                 }
484                 ret = -EBUSY;
485             }
486         }
487         if (!ret) {
488             ret = virtio_mem_notify_plug(vmem, offset, size);
489         }
490 
491         if (ret) {
492             /* Could be preallocation or a notifier populated memory. */
493             ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size);
494             return -EBUSY;
495         }
496     }
497     virtio_mem_set_bitmap(vmem, start_gpa, size, plug);
498     return 0;
499 }
500 
501 static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa,
502                                            uint16_t nb_blocks, bool plug)
503 {
504     const uint64_t size = nb_blocks * vmem->block_size;
505     int ret;
506 
507     if (!virtio_mem_valid_range(vmem, gpa, size)) {
508         return VIRTIO_MEM_RESP_ERROR;
509     }
510 
511     if (plug && (vmem->size + size > vmem->requested_size)) {
512         return VIRTIO_MEM_RESP_NACK;
513     }
514 
515     /* test if really all blocks are in the opposite state */
516     if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) {
517         return VIRTIO_MEM_RESP_ERROR;
518     }
519 
520     ret = virtio_mem_set_block_state(vmem, gpa, size, plug);
521     if (ret) {
522         return VIRTIO_MEM_RESP_BUSY;
523     }
524     if (plug) {
525         vmem->size += size;
526     } else {
527         vmem->size -= size;
528     }
529     notifier_list_notify(&vmem->size_change_notifiers, &vmem->size);
530     return VIRTIO_MEM_RESP_ACK;
531 }
532 
533 static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem,
534                                     struct virtio_mem_req *req)
535 {
536     const uint64_t gpa = le64_to_cpu(req->u.plug.addr);
537     const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks);
538     uint16_t type;
539 
540     trace_virtio_mem_plug_request(gpa, nb_blocks);
541     type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true);
542     virtio_mem_send_response_simple(vmem, elem, type);
543 }
544 
545 static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem,
546                                       struct virtio_mem_req *req)
547 {
548     const uint64_t gpa = le64_to_cpu(req->u.unplug.addr);
549     const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks);
550     uint16_t type;
551 
552     trace_virtio_mem_unplug_request(gpa, nb_blocks);
553     type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false);
554     virtio_mem_send_response_simple(vmem, elem, type);
555 }
556 
557 static void virtio_mem_resize_usable_region(VirtIOMEM *vmem,
558                                             uint64_t requested_size,
559                                             bool can_shrink)
560 {
561     uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr),
562                            requested_size + VIRTIO_MEM_USABLE_EXTENT);
563 
564     /* The usable region size always has to be multiples of the block size. */
565     newsize = QEMU_ALIGN_UP(newsize, vmem->block_size);
566 
567     if (!requested_size) {
568         newsize = 0;
569     }
570 
571     if (newsize < vmem->usable_region_size && !can_shrink) {
572         return;
573     }
574 
575     trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize);
576     vmem->usable_region_size = newsize;
577 }
578 
579 static int virtio_mem_unplug_all(VirtIOMEM *vmem)
580 {
581     RAMBlock *rb = vmem->memdev->mr.ram_block;
582 
583     if (virtio_mem_is_busy()) {
584         return -EBUSY;
585     }
586 
587     if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) {
588         return -EBUSY;
589     }
590     virtio_mem_notify_unplug_all(vmem);
591 
592     bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size);
593     if (vmem->size) {
594         vmem->size = 0;
595         notifier_list_notify(&vmem->size_change_notifiers, &vmem->size);
596     }
597     trace_virtio_mem_unplugged_all();
598     virtio_mem_resize_usable_region(vmem, vmem->requested_size, true);
599     return 0;
600 }
601 
602 static void virtio_mem_unplug_all_request(VirtIOMEM *vmem,
603                                           VirtQueueElement *elem)
604 {
605     trace_virtio_mem_unplug_all_request();
606     if (virtio_mem_unplug_all(vmem)) {
607         virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY);
608     } else {
609         virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK);
610     }
611 }
612 
613 static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem,
614                                      struct virtio_mem_req *req)
615 {
616     const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks);
617     const uint64_t gpa = le64_to_cpu(req->u.state.addr);
618     const uint64_t size = nb_blocks * vmem->block_size;
619     struct virtio_mem_resp resp = {
620         .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK),
621     };
622 
623     trace_virtio_mem_state_request(gpa, nb_blocks);
624     if (!virtio_mem_valid_range(vmem, gpa, size)) {
625         virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR);
626         return;
627     }
628 
629     if (virtio_mem_test_bitmap(vmem, gpa, size, true)) {
630         resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED);
631     } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) {
632         resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED);
633     } else {
634         resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED);
635     }
636     trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state));
637     virtio_mem_send_response(vmem, elem, &resp);
638 }
639 
640 static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq)
641 {
642     const int len = sizeof(struct virtio_mem_req);
643     VirtIOMEM *vmem = VIRTIO_MEM(vdev);
644     VirtQueueElement *elem;
645     struct virtio_mem_req req;
646     uint16_t type;
647 
648     while (true) {
649         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
650         if (!elem) {
651             return;
652         }
653 
654         if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) {
655             virtio_error(vdev, "virtio-mem protocol violation: invalid request"
656                          " size: %d", len);
657             virtqueue_detach_element(vq, elem, 0);
658             g_free(elem);
659             return;
660         }
661 
662         if (iov_size(elem->in_sg, elem->in_num) <
663             sizeof(struct virtio_mem_resp)) {
664             virtio_error(vdev, "virtio-mem protocol violation: not enough space"
665                          " for response: %zu",
666                          iov_size(elem->in_sg, elem->in_num));
667             virtqueue_detach_element(vq, elem, 0);
668             g_free(elem);
669             return;
670         }
671 
672         type = le16_to_cpu(req.type);
673         switch (type) {
674         case VIRTIO_MEM_REQ_PLUG:
675             virtio_mem_plug_request(vmem, elem, &req);
676             break;
677         case VIRTIO_MEM_REQ_UNPLUG:
678             virtio_mem_unplug_request(vmem, elem, &req);
679             break;
680         case VIRTIO_MEM_REQ_UNPLUG_ALL:
681             virtio_mem_unplug_all_request(vmem, elem);
682             break;
683         case VIRTIO_MEM_REQ_STATE:
684             virtio_mem_state_request(vmem, elem, &req);
685             break;
686         default:
687             virtio_error(vdev, "virtio-mem protocol violation: unknown request"
688                          " type: %d", type);
689             virtqueue_detach_element(vq, elem, 0);
690             g_free(elem);
691             return;
692         }
693 
694         g_free(elem);
695     }
696 }
697 
698 static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data)
699 {
700     VirtIOMEM *vmem = VIRTIO_MEM(vdev);
701     struct virtio_mem_config *config = (void *) config_data;
702 
703     config->block_size = cpu_to_le64(vmem->block_size);
704     config->node_id = cpu_to_le16(vmem->node);
705     config->requested_size = cpu_to_le64(vmem->requested_size);
706     config->plugged_size = cpu_to_le64(vmem->size);
707     config->addr = cpu_to_le64(vmem->addr);
708     config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr));
709     config->usable_region_size = cpu_to_le64(vmem->usable_region_size);
710 }
711 
712 static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features,
713                                         Error **errp)
714 {
715     MachineState *ms = MACHINE(qdev_get_machine());
716     VirtIOMEM *vmem = VIRTIO_MEM(vdev);
717 
718     if (ms->numa_state) {
719 #if defined(CONFIG_ACPI)
720         virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM);
721 #endif
722     }
723     assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO);
724     if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) {
725         virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE);
726     }
727     return features;
728 }
729 
730 static int virtio_mem_validate_features(VirtIODevice *vdev)
731 {
732     if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) &&
733         !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) {
734         return -EFAULT;
735     }
736     return 0;
737 }
738 
739 static void virtio_mem_system_reset(void *opaque)
740 {
741     VirtIOMEM *vmem = VIRTIO_MEM(opaque);
742 
743     /*
744      * During usual resets, we will unplug all memory and shrink the usable
745      * region size. This is, however, not possible in all scenarios. Then,
746      * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL).
747      */
748     virtio_mem_unplug_all(vmem);
749 }
750 
751 static void virtio_mem_device_realize(DeviceState *dev, Error **errp)
752 {
753     MachineState *ms = MACHINE(qdev_get_machine());
754     int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0;
755     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
756     VirtIOMEM *vmem = VIRTIO_MEM(dev);
757     uint64_t page_size;
758     RAMBlock *rb;
759     int ret;
760 
761     if (!vmem->memdev) {
762         error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP);
763         return;
764     } else if (host_memory_backend_is_mapped(vmem->memdev)) {
765         error_setg(errp, "'%s' property specifies a busy memdev: %s",
766                    VIRTIO_MEM_MEMDEV_PROP,
767                    object_get_canonical_path_component(OBJECT(vmem->memdev)));
768         return;
769     } else if (!memory_region_is_ram(&vmem->memdev->mr) ||
770         memory_region_is_rom(&vmem->memdev->mr) ||
771         !vmem->memdev->mr.ram_block) {
772         error_setg(errp, "'%s' property specifies an unsupported memdev",
773                    VIRTIO_MEM_MEMDEV_PROP);
774         return;
775     }
776 
777     if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) ||
778         (!nb_numa_nodes && vmem->node)) {
779         error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds"
780                    "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP,
781                    vmem->node, nb_numa_nodes ? nb_numa_nodes : 1);
782         return;
783     }
784 
785     if (enable_mlock) {
786         error_setg(errp, "Incompatible with mlock");
787         return;
788     }
789 
790     rb = vmem->memdev->mr.ram_block;
791     page_size = qemu_ram_pagesize(rb);
792 
793 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
794     switch (vmem->unplugged_inaccessible) {
795     case ON_OFF_AUTO_AUTO:
796         if (virtio_mem_has_shared_zeropage(rb)) {
797             vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF;
798         } else {
799             vmem->unplugged_inaccessible = ON_OFF_AUTO_ON;
800         }
801         break;
802     case ON_OFF_AUTO_OFF:
803         if (!virtio_mem_has_shared_zeropage(rb)) {
804             warn_report("'%s' property set to 'off' with a memdev that does"
805                         " not support the shared zeropage.",
806                         VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP);
807         }
808         break;
809     default:
810         break;
811     }
812 #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
813     vmem->unplugged_inaccessible = ON_OFF_AUTO_ON;
814 #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */
815 
816     /*
817      * If the block size wasn't configured by the user, use a sane default. This
818      * allows using hugetlbfs backends of any page size without manual
819      * intervention.
820      */
821     if (!vmem->block_size) {
822         vmem->block_size = virtio_mem_default_block_size(rb);
823     }
824 
825     if (vmem->block_size < page_size) {
826         error_setg(errp, "'%s' property has to be at least the page size (0x%"
827                    PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size);
828         return;
829     } else if (vmem->block_size < virtio_mem_default_block_size(rb)) {
830         warn_report("'%s' property is smaller than the default block size (%"
831                     PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP,
832                     virtio_mem_default_block_size(rb) / MiB);
833     }
834     if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) {
835         error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64
836                    ")", VIRTIO_MEM_REQUESTED_SIZE_PROP,
837                    VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size);
838         return;
839     } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) {
840         error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64
841                    ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP,
842                    vmem->block_size);
843         return;
844     } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr),
845                                 vmem->block_size)) {
846         error_setg(errp, "'%s' property memdev size has to be multiples of"
847                    "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP,
848                    VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size);
849         return;
850     }
851 
852     if (ram_block_coordinated_discard_require(true)) {
853         error_setg(errp, "Discarding RAM is disabled");
854         return;
855     }
856 
857     ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb));
858     if (ret) {
859         error_setg_errno(errp, -ret, "Unexpected error discarding RAM");
860         ram_block_coordinated_discard_require(false);
861         return;
862     }
863 
864     virtio_mem_resize_usable_region(vmem, vmem->requested_size, true);
865 
866     vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) /
867                         vmem->block_size;
868     vmem->bitmap = bitmap_new(vmem->bitmap_size);
869 
870     virtio_init(vdev, VIRTIO_ID_MEM, sizeof(struct virtio_mem_config));
871     vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request);
872 
873     host_memory_backend_set_mapped(vmem->memdev, true);
874     vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem));
875     qemu_register_reset(virtio_mem_system_reset, vmem);
876 
877     /*
878      * Set ourselves as RamDiscardManager before the plug handler maps the
879      * memory region and exposes it via an address space.
880      */
881     memory_region_set_ram_discard_manager(&vmem->memdev->mr,
882                                           RAM_DISCARD_MANAGER(vmem));
883 }
884 
885 static void virtio_mem_device_unrealize(DeviceState *dev)
886 {
887     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
888     VirtIOMEM *vmem = VIRTIO_MEM(dev);
889 
890     /*
891      * The unplug handler unmapped the memory region, it cannot be
892      * found via an address space anymore. Unset ourselves.
893      */
894     memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL);
895     qemu_unregister_reset(virtio_mem_system_reset, vmem);
896     vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem));
897     host_memory_backend_set_mapped(vmem->memdev, false);
898     virtio_del_queue(vdev, 0);
899     virtio_cleanup(vdev);
900     g_free(vmem->bitmap);
901     ram_block_coordinated_discard_require(false);
902 }
903 
904 static int virtio_mem_discard_range_cb(const VirtIOMEM *vmem, void *arg,
905                                        uint64_t offset, uint64_t size)
906 {
907     RAMBlock *rb = vmem->memdev->mr.ram_block;
908 
909     return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0;
910 }
911 
912 static int virtio_mem_restore_unplugged(VirtIOMEM *vmem)
913 {
914     /* Make sure all memory is really discarded after migration. */
915     return virtio_mem_for_each_unplugged_range(vmem, NULL,
916                                                virtio_mem_discard_range_cb);
917 }
918 
919 static int virtio_mem_post_load(void *opaque, int version_id)
920 {
921     VirtIOMEM *vmem = VIRTIO_MEM(opaque);
922     RamDiscardListener *rdl;
923     int ret;
924 
925     /*
926      * We started out with all memory discarded and our memory region is mapped
927      * into an address space. Replay, now that we updated the bitmap.
928      */
929     QLIST_FOREACH(rdl, &vmem->rdl_list, next) {
930         ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl,
931                                                  virtio_mem_notify_populate_cb);
932         if (ret) {
933             return ret;
934         }
935     }
936 
937     if (migration_in_incoming_postcopy()) {
938         return 0;
939     }
940 
941     return virtio_mem_restore_unplugged(vmem);
942 }
943 
944 typedef struct VirtIOMEMMigSanityChecks {
945     VirtIOMEM *parent;
946     uint64_t addr;
947     uint64_t region_size;
948     uint64_t block_size;
949     uint32_t node;
950 } VirtIOMEMMigSanityChecks;
951 
952 static int virtio_mem_mig_sanity_checks_pre_save(void *opaque)
953 {
954     VirtIOMEMMigSanityChecks *tmp = opaque;
955     VirtIOMEM *vmem = tmp->parent;
956 
957     tmp->addr = vmem->addr;
958     tmp->region_size = memory_region_size(&vmem->memdev->mr);
959     tmp->block_size = vmem->block_size;
960     tmp->node = vmem->node;
961     return 0;
962 }
963 
964 static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id)
965 {
966     VirtIOMEMMigSanityChecks *tmp = opaque;
967     VirtIOMEM *vmem = tmp->parent;
968     const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr);
969 
970     if (tmp->addr != vmem->addr) {
971         error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64,
972                      VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr);
973         return -EINVAL;
974     }
975     /*
976      * Note: Preparation for resizeable memory regions. The maximum size
977      * of the memory region must not change during migration.
978      */
979     if (tmp->region_size != new_region_size) {
980         error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%"
981                      PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size,
982                      new_region_size);
983         return -EINVAL;
984     }
985     if (tmp->block_size != vmem->block_size) {
986         error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64,
987                      VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size,
988                      vmem->block_size);
989         return -EINVAL;
990     }
991     if (tmp->node != vmem->node) {
992         error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32,
993                      VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node);
994         return -EINVAL;
995     }
996     return 0;
997 }
998 
999 static const VMStateDescription vmstate_virtio_mem_sanity_checks = {
1000     .name = "virtio-mem-device/sanity-checks",
1001     .pre_save = virtio_mem_mig_sanity_checks_pre_save,
1002     .post_load = virtio_mem_mig_sanity_checks_post_load,
1003     .fields = (VMStateField[]) {
1004         VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks),
1005         VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks),
1006         VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks),
1007         VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks),
1008         VMSTATE_END_OF_LIST(),
1009     },
1010 };
1011 
1012 static const VMStateDescription vmstate_virtio_mem_device = {
1013     .name = "virtio-mem-device",
1014     .minimum_version_id = 1,
1015     .version_id = 1,
1016     .priority = MIG_PRI_VIRTIO_MEM,
1017     .post_load = virtio_mem_post_load,
1018     .fields = (VMStateField[]) {
1019         VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks,
1020                          vmstate_virtio_mem_sanity_checks),
1021         VMSTATE_UINT64(usable_region_size, VirtIOMEM),
1022         VMSTATE_UINT64(size, VirtIOMEM),
1023         VMSTATE_UINT64(requested_size, VirtIOMEM),
1024         VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size),
1025         VMSTATE_END_OF_LIST()
1026     },
1027 };
1028 
1029 static const VMStateDescription vmstate_virtio_mem = {
1030     .name = "virtio-mem",
1031     .minimum_version_id = 1,
1032     .version_id = 1,
1033     .fields = (VMStateField[]) {
1034         VMSTATE_VIRTIO_DEVICE,
1035         VMSTATE_END_OF_LIST()
1036     },
1037 };
1038 
1039 static void virtio_mem_fill_device_info(const VirtIOMEM *vmem,
1040                                         VirtioMEMDeviceInfo *vi)
1041 {
1042     vi->memaddr = vmem->addr;
1043     vi->node = vmem->node;
1044     vi->requested_size = vmem->requested_size;
1045     vi->size = vmem->size;
1046     vi->max_size = memory_region_size(&vmem->memdev->mr);
1047     vi->block_size = vmem->block_size;
1048     vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev));
1049 }
1050 
1051 static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp)
1052 {
1053     if (!vmem->memdev) {
1054         error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP);
1055         return NULL;
1056     }
1057 
1058     return &vmem->memdev->mr;
1059 }
1060 
1061 static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem,
1062                                                 Notifier *notifier)
1063 {
1064     notifier_list_add(&vmem->size_change_notifiers, notifier);
1065 }
1066 
1067 static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem,
1068                                                    Notifier *notifier)
1069 {
1070     notifier_remove(notifier);
1071 }
1072 
1073 static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name,
1074                                 void *opaque, Error **errp)
1075 {
1076     const VirtIOMEM *vmem = VIRTIO_MEM(obj);
1077     uint64_t value = vmem->size;
1078 
1079     visit_type_size(v, name, &value, errp);
1080 }
1081 
1082 static void virtio_mem_get_requested_size(Object *obj, Visitor *v,
1083                                           const char *name, void *opaque,
1084                                           Error **errp)
1085 {
1086     const VirtIOMEM *vmem = VIRTIO_MEM(obj);
1087     uint64_t value = vmem->requested_size;
1088 
1089     visit_type_size(v, name, &value, errp);
1090 }
1091 
1092 static void virtio_mem_set_requested_size(Object *obj, Visitor *v,
1093                                           const char *name, void *opaque,
1094                                           Error **errp)
1095 {
1096     VirtIOMEM *vmem = VIRTIO_MEM(obj);
1097     Error *err = NULL;
1098     uint64_t value;
1099 
1100     visit_type_size(v, name, &value, &err);
1101     if (err) {
1102         error_propagate(errp, err);
1103         return;
1104     }
1105 
1106     /*
1107      * The block size and memory backend are not fixed until the device was
1108      * realized. realize() will verify these properties then.
1109      */
1110     if (DEVICE(obj)->realized) {
1111         if (!QEMU_IS_ALIGNED(value, vmem->block_size)) {
1112             error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64
1113                        ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP,
1114                        vmem->block_size);
1115             return;
1116         } else if (value > memory_region_size(&vmem->memdev->mr)) {
1117             error_setg(errp, "'%s' cannot exceed the memory backend size"
1118                        "(0x%" PRIx64 ")", name,
1119                        memory_region_size(&vmem->memdev->mr));
1120             return;
1121         }
1122 
1123         if (value != vmem->requested_size) {
1124             virtio_mem_resize_usable_region(vmem, value, false);
1125             vmem->requested_size = value;
1126         }
1127         /*
1128          * Trigger a config update so the guest gets notified. We trigger
1129          * even if the size didn't change (especially helpful for debugging).
1130          */
1131         virtio_notify_config(VIRTIO_DEVICE(vmem));
1132     } else {
1133         vmem->requested_size = value;
1134     }
1135 }
1136 
1137 static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name,
1138                                       void *opaque, Error **errp)
1139 {
1140     const VirtIOMEM *vmem = VIRTIO_MEM(obj);
1141     uint64_t value = vmem->block_size;
1142 
1143     /*
1144      * If not configured by the user (and we're not realized yet), use the
1145      * default block size we would use with the current memory backend.
1146      */
1147     if (!value) {
1148         if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) {
1149             value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block);
1150         } else {
1151             value = virtio_mem_thp_size();
1152         }
1153     }
1154 
1155     visit_type_size(v, name, &value, errp);
1156 }
1157 
1158 static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name,
1159                                       void *opaque, Error **errp)
1160 {
1161     VirtIOMEM *vmem = VIRTIO_MEM(obj);
1162     Error *err = NULL;
1163     uint64_t value;
1164 
1165     if (DEVICE(obj)->realized) {
1166         error_setg(errp, "'%s' cannot be changed", name);
1167         return;
1168     }
1169 
1170     visit_type_size(v, name, &value, &err);
1171     if (err) {
1172         error_propagate(errp, err);
1173         return;
1174     }
1175 
1176     if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) {
1177         error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name,
1178                    VIRTIO_MEM_MIN_BLOCK_SIZE);
1179         return;
1180     } else if (!is_power_of_2(value)) {
1181         error_setg(errp, "'%s' property has to be a power of two", name);
1182         return;
1183     }
1184     vmem->block_size = value;
1185 }
1186 
1187 static void virtio_mem_instance_init(Object *obj)
1188 {
1189     VirtIOMEM *vmem = VIRTIO_MEM(obj);
1190 
1191     notifier_list_init(&vmem->size_change_notifiers);
1192     QLIST_INIT(&vmem->rdl_list);
1193 
1194     object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size,
1195                         NULL, NULL, NULL);
1196     object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size",
1197                         virtio_mem_get_requested_size,
1198                         virtio_mem_set_requested_size, NULL, NULL);
1199     object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size",
1200                         virtio_mem_get_block_size, virtio_mem_set_block_size,
1201                         NULL, NULL);
1202 }
1203 
1204 static Property virtio_mem_properties[] = {
1205     DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0),
1206     DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0),
1207     DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false),
1208     DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev,
1209                      TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1210 #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS)
1211     DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM,
1212                             unplugged_inaccessible, ON_OFF_AUTO_AUTO),
1213 #endif
1214     DEFINE_PROP_END_OF_LIST(),
1215 };
1216 
1217 static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm,
1218                                                    const MemoryRegion *mr)
1219 {
1220     const VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1221 
1222     g_assert(mr == &vmem->memdev->mr);
1223     return vmem->block_size;
1224 }
1225 
1226 static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm,
1227                                         const MemoryRegionSection *s)
1228 {
1229     const VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1230     uint64_t start_gpa = vmem->addr + s->offset_within_region;
1231     uint64_t end_gpa = start_gpa + int128_get64(s->size);
1232 
1233     g_assert(s->mr == &vmem->memdev->mr);
1234 
1235     start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size);
1236     end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size);
1237 
1238     if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) {
1239         return false;
1240     }
1241 
1242     return virtio_mem_test_bitmap(vmem, start_gpa, end_gpa - start_gpa, true);
1243 }
1244 
1245 struct VirtIOMEMReplayData {
1246     void *fn;
1247     void *opaque;
1248 };
1249 
1250 static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg)
1251 {
1252     struct VirtIOMEMReplayData *data = arg;
1253 
1254     return ((ReplayRamPopulate)data->fn)(s, data->opaque);
1255 }
1256 
1257 static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm,
1258                                            MemoryRegionSection *s,
1259                                            ReplayRamPopulate replay_fn,
1260                                            void *opaque)
1261 {
1262     const VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1263     struct VirtIOMEMReplayData data = {
1264         .fn = replay_fn,
1265         .opaque = opaque,
1266     };
1267 
1268     g_assert(s->mr == &vmem->memdev->mr);
1269     return virtio_mem_for_each_plugged_section(vmem, s, &data,
1270                                             virtio_mem_rdm_replay_populated_cb);
1271 }
1272 
1273 static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s,
1274                                               void *arg)
1275 {
1276     struct VirtIOMEMReplayData *data = arg;
1277 
1278     ((ReplayRamDiscard)data->fn)(s, data->opaque);
1279     return 0;
1280 }
1281 
1282 static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm,
1283                                             MemoryRegionSection *s,
1284                                             ReplayRamDiscard replay_fn,
1285                                             void *opaque)
1286 {
1287     const VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1288     struct VirtIOMEMReplayData data = {
1289         .fn = replay_fn,
1290         .opaque = opaque,
1291     };
1292 
1293     g_assert(s->mr == &vmem->memdev->mr);
1294     virtio_mem_for_each_unplugged_section(vmem, s, &data,
1295                                           virtio_mem_rdm_replay_discarded_cb);
1296 }
1297 
1298 static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm,
1299                                              RamDiscardListener *rdl,
1300                                              MemoryRegionSection *s)
1301 {
1302     VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1303     int ret;
1304 
1305     g_assert(s->mr == &vmem->memdev->mr);
1306     rdl->section = memory_region_section_new_copy(s);
1307 
1308     QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next);
1309     ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl,
1310                                               virtio_mem_notify_populate_cb);
1311     if (ret) {
1312         error_report("%s: Replaying plugged ranges failed: %s", __func__,
1313                      strerror(-ret));
1314     }
1315 }
1316 
1317 static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm,
1318                                                RamDiscardListener *rdl)
1319 {
1320     VirtIOMEM *vmem = VIRTIO_MEM(rdm);
1321 
1322     g_assert(rdl->section->mr == &vmem->memdev->mr);
1323     if (vmem->size) {
1324         if (rdl->double_discard_supported) {
1325             rdl->notify_discard(rdl, rdl->section);
1326         } else {
1327             virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl,
1328                                                 virtio_mem_notify_discard_cb);
1329         }
1330     }
1331 
1332     memory_region_section_free_copy(rdl->section);
1333     rdl->section = NULL;
1334     QLIST_REMOVE(rdl, next);
1335 }
1336 
1337 static void virtio_mem_class_init(ObjectClass *klass, void *data)
1338 {
1339     DeviceClass *dc = DEVICE_CLASS(klass);
1340     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1341     VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass);
1342     RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass);
1343 
1344     device_class_set_props(dc, virtio_mem_properties);
1345     dc->vmsd = &vmstate_virtio_mem;
1346 
1347     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1348     vdc->realize = virtio_mem_device_realize;
1349     vdc->unrealize = virtio_mem_device_unrealize;
1350     vdc->get_config = virtio_mem_get_config;
1351     vdc->get_features = virtio_mem_get_features;
1352     vdc->validate_features = virtio_mem_validate_features;
1353     vdc->vmsd = &vmstate_virtio_mem_device;
1354 
1355     vmc->fill_device_info = virtio_mem_fill_device_info;
1356     vmc->get_memory_region = virtio_mem_get_memory_region;
1357     vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier;
1358     vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier;
1359 
1360     rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity;
1361     rdmc->is_populated = virtio_mem_rdm_is_populated;
1362     rdmc->replay_populated = virtio_mem_rdm_replay_populated;
1363     rdmc->replay_discarded = virtio_mem_rdm_replay_discarded;
1364     rdmc->register_listener = virtio_mem_rdm_register_listener;
1365     rdmc->unregister_listener = virtio_mem_rdm_unregister_listener;
1366 }
1367 
1368 static const TypeInfo virtio_mem_info = {
1369     .name = TYPE_VIRTIO_MEM,
1370     .parent = TYPE_VIRTIO_DEVICE,
1371     .instance_size = sizeof(VirtIOMEM),
1372     .instance_init = virtio_mem_instance_init,
1373     .class_init = virtio_mem_class_init,
1374     .class_size = sizeof(VirtIOMEMClass),
1375     .interfaces = (InterfaceInfo[]) {
1376         { TYPE_RAM_DISCARD_MANAGER },
1377         { }
1378     },
1379 };
1380 
1381 static void virtio_register_types(void)
1382 {
1383     type_register_static(&virtio_mem_info);
1384 }
1385 
1386 type_init(virtio_register_types)
1387