xref: /openbmc/qemu/include/exec/memory.h (revision a9eacf8b4dd1d13f89fb735236e32dcfafef3e93)
1 /*
2  * Physical memory management API
3  *
4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
5  *
6  * Authors:
7  *  Avi Kivity <avi@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #ifndef MEMORY_H
15 #define MEMORY_H
16 
17 #ifndef CONFIG_USER_ONLY
18 
19 #include "exec/cpu-common.h"
20 #include "exec/hwaddr.h"
21 #include "exec/memattrs.h"
22 #include "exec/memop.h"
23 #include "exec/ramlist.h"
24 #include "qemu/bswap.h"
25 #include "qemu/queue.h"
26 #include "qemu/int128.h"
27 #include "qemu/notify.h"
28 #include "qom/object.h"
29 #include "qemu/rcu.h"
30 
31 #define RAM_ADDR_INVALID (~(ram_addr_t)0)
32 
33 #define MAX_PHYS_ADDR_SPACE_BITS 62
34 #define MAX_PHYS_ADDR            (((hwaddr)1 << MAX_PHYS_ADDR_SPACE_BITS) - 1)
35 
36 #define TYPE_MEMORY_REGION "qemu:memory-region"
37 DECLARE_INSTANCE_CHECKER(MemoryRegion, MEMORY_REGION,
38                          TYPE_MEMORY_REGION)
39 
40 #define TYPE_IOMMU_MEMORY_REGION "qemu:iommu-memory-region"
41 typedef struct IOMMUMemoryRegionClass IOMMUMemoryRegionClass;
42 DECLARE_OBJ_CHECKERS(IOMMUMemoryRegion, IOMMUMemoryRegionClass,
43                      IOMMU_MEMORY_REGION, TYPE_IOMMU_MEMORY_REGION)
44 
45 extern bool global_dirty_log;
46 
47 typedef struct MemoryRegionOps MemoryRegionOps;
48 
49 struct ReservedRegion {
50     hwaddr low;
51     hwaddr high;
52     unsigned type;
53 };
54 
55 typedef struct IOMMUTLBEntry IOMMUTLBEntry;
56 
57 /* See address_space_translate: bit 0 is read, bit 1 is write.  */
58 typedef enum {
59     IOMMU_NONE = 0,
60     IOMMU_RO   = 1,
61     IOMMU_WO   = 2,
62     IOMMU_RW   = 3,
63 } IOMMUAccessFlags;
64 
65 #define IOMMU_ACCESS_FLAG(r, w) (((r) ? IOMMU_RO : 0) | ((w) ? IOMMU_WO : 0))
66 
67 struct IOMMUTLBEntry {
68     AddressSpace    *target_as;
69     hwaddr           iova;
70     hwaddr           translated_addr;
71     hwaddr           addr_mask;  /* 0xfff = 4k translation */
72     IOMMUAccessFlags perm;
73 };
74 
75 /*
76  * Bitmap for different IOMMUNotifier capabilities. Each notifier can
77  * register with one or multiple IOMMU Notifier capability bit(s).
78  */
79 typedef enum {
80     IOMMU_NOTIFIER_NONE = 0,
81     /* Notify cache invalidations */
82     IOMMU_NOTIFIER_UNMAP = 0x1,
83     /* Notify entry changes (newly created entries) */
84     IOMMU_NOTIFIER_MAP = 0x2,
85 } IOMMUNotifierFlag;
86 
87 #define IOMMU_NOTIFIER_ALL (IOMMU_NOTIFIER_MAP | IOMMU_NOTIFIER_UNMAP)
88 
89 struct IOMMUNotifier;
90 typedef void (*IOMMUNotify)(struct IOMMUNotifier *notifier,
91                             IOMMUTLBEntry *data);
92 
93 struct IOMMUNotifier {
94     IOMMUNotify notify;
95     IOMMUNotifierFlag notifier_flags;
96     /* Notify for address space range start <= addr <= end */
97     hwaddr start;
98     hwaddr end;
99     int iommu_idx;
100     QLIST_ENTRY(IOMMUNotifier) node;
101 };
102 typedef struct IOMMUNotifier IOMMUNotifier;
103 
104 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
105 #define RAM_PREALLOC   (1 << 0)
106 
107 /* RAM is mmap-ed with MAP_SHARED */
108 #define RAM_SHARED     (1 << 1)
109 
110 /* Only a portion of RAM (used_length) is actually used, and migrated.
111  * This used_length size can change across reboots.
112  */
113 #define RAM_RESIZEABLE (1 << 2)
114 
115 /* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
116  * zero the page and wake waiting processes.
117  * (Set during postcopy)
118  */
119 #define RAM_UF_ZEROPAGE (1 << 3)
120 
121 /* RAM can be migrated */
122 #define RAM_MIGRATABLE (1 << 4)
123 
124 /* RAM is a persistent kind memory */
125 #define RAM_PMEM (1 << 5)
126 
127 static inline void iommu_notifier_init(IOMMUNotifier *n, IOMMUNotify fn,
128                                        IOMMUNotifierFlag flags,
129                                        hwaddr start, hwaddr end,
130                                        int iommu_idx)
131 {
132     n->notify = fn;
133     n->notifier_flags = flags;
134     n->start = start;
135     n->end = end;
136     n->iommu_idx = iommu_idx;
137 }
138 
139 /*
140  * Memory region callbacks
141  */
142 struct MemoryRegionOps {
143     /* Read from the memory region. @addr is relative to @mr; @size is
144      * in bytes. */
145     uint64_t (*read)(void *opaque,
146                      hwaddr addr,
147                      unsigned size);
148     /* Write to the memory region. @addr is relative to @mr; @size is
149      * in bytes. */
150     void (*write)(void *opaque,
151                   hwaddr addr,
152                   uint64_t data,
153                   unsigned size);
154 
155     MemTxResult (*read_with_attrs)(void *opaque,
156                                    hwaddr addr,
157                                    uint64_t *data,
158                                    unsigned size,
159                                    MemTxAttrs attrs);
160     MemTxResult (*write_with_attrs)(void *opaque,
161                                     hwaddr addr,
162                                     uint64_t data,
163                                     unsigned size,
164                                     MemTxAttrs attrs);
165 
166     enum device_endian endianness;
167     /* Guest-visible constraints: */
168     struct {
169         /* If nonzero, specify bounds on access sizes beyond which a machine
170          * check is thrown.
171          */
172         unsigned min_access_size;
173         unsigned max_access_size;
174         /* If true, unaligned accesses are supported.  Otherwise unaligned
175          * accesses throw machine checks.
176          */
177          bool unaligned;
178         /*
179          * If present, and returns #false, the transaction is not accepted
180          * by the device (and results in machine dependent behaviour such
181          * as a machine check exception).
182          */
183         bool (*accepts)(void *opaque, hwaddr addr,
184                         unsigned size, bool is_write,
185                         MemTxAttrs attrs);
186     } valid;
187     /* Internal implementation constraints: */
188     struct {
189         /* If nonzero, specifies the minimum size implemented.  Smaller sizes
190          * will be rounded upwards and a partial result will be returned.
191          */
192         unsigned min_access_size;
193         /* If nonzero, specifies the maximum size implemented.  Larger sizes
194          * will be done as a series of accesses with smaller sizes.
195          */
196         unsigned max_access_size;
197         /* If true, unaligned accesses are supported.  Otherwise all accesses
198          * are converted to (possibly multiple) naturally aligned accesses.
199          */
200         bool unaligned;
201     } impl;
202 };
203 
204 typedef struct MemoryRegionClass {
205     /* private */
206     ObjectClass parent_class;
207 } MemoryRegionClass;
208 
209 
210 enum IOMMUMemoryRegionAttr {
211     IOMMU_ATTR_SPAPR_TCE_FD
212 };
213 
214 /*
215  * IOMMUMemoryRegionClass:
216  *
217  * All IOMMU implementations need to subclass TYPE_IOMMU_MEMORY_REGION
218  * and provide an implementation of at least the @translate method here
219  * to handle requests to the memory region. Other methods are optional.
220  *
221  * The IOMMU implementation must use the IOMMU notifier infrastructure
222  * to report whenever mappings are changed, by calling
223  * memory_region_notify_iommu() (or, if necessary, by calling
224  * memory_region_notify_one() for each registered notifier).
225  *
226  * Conceptually an IOMMU provides a mapping from input address
227  * to an output TLB entry. If the IOMMU is aware of memory transaction
228  * attributes and the output TLB entry depends on the transaction
229  * attributes, we represent this using IOMMU indexes. Each index
230  * selects a particular translation table that the IOMMU has:
231  *
232  *   @attrs_to_index returns the IOMMU index for a set of transaction attributes
233  *
234  *   @translate takes an input address and an IOMMU index
235  *
236  * and the mapping returned can only depend on the input address and the
237  * IOMMU index.
238  *
239  * Most IOMMUs don't care about the transaction attributes and support
240  * only a single IOMMU index. A more complex IOMMU might have one index
241  * for secure transactions and one for non-secure transactions.
242  */
243 struct IOMMUMemoryRegionClass {
244     /* private: */
245     MemoryRegionClass parent_class;
246 
247     /* public: */
248     /**
249      * @translate:
250      *
251      * Return a TLB entry that contains a given address.
252      *
253      * The IOMMUAccessFlags indicated via @flag are optional and may
254      * be specified as IOMMU_NONE to indicate that the caller needs
255      * the full translation information for both reads and writes. If
256      * the access flags are specified then the IOMMU implementation
257      * may use this as an optimization, to stop doing a page table
258      * walk as soon as it knows that the requested permissions are not
259      * allowed. If IOMMU_NONE is passed then the IOMMU must do the
260      * full page table walk and report the permissions in the returned
261      * IOMMUTLBEntry. (Note that this implies that an IOMMU may not
262      * return different mappings for reads and writes.)
263      *
264      * The returned information remains valid while the caller is
265      * holding the big QEMU lock or is inside an RCU critical section;
266      * if the caller wishes to cache the mapping beyond that it must
267      * register an IOMMU notifier so it can invalidate its cached
268      * information when the IOMMU mapping changes.
269      *
270      * @iommu: the IOMMUMemoryRegion
271      *
272      * @hwaddr: address to be translated within the memory region
273      *
274      * @flag: requested access permission
275      *
276      * @iommu_idx: IOMMU index for the translation
277      */
278     IOMMUTLBEntry (*translate)(IOMMUMemoryRegion *iommu, hwaddr addr,
279                                IOMMUAccessFlags flag, int iommu_idx);
280     /**
281      * @get_min_page_size:
282      *
283      * Returns minimum supported page size in bytes.
284      *
285      * If this method is not provided then the minimum is assumed to
286      * be TARGET_PAGE_SIZE.
287      *
288      * @iommu: the IOMMUMemoryRegion
289      */
290     uint64_t (*get_min_page_size)(IOMMUMemoryRegion *iommu);
291     /**
292      * @notify_flag_changed:
293      *
294      * Called when IOMMU Notifier flag changes (ie when the set of
295      * events which IOMMU users are requesting notification for changes).
296      * Optional method -- need not be provided if the IOMMU does not
297      * need to know exactly which events must be notified.
298      *
299      * @iommu: the IOMMUMemoryRegion
300      *
301      * @old_flags: events which previously needed to be notified
302      *
303      * @new_flags: events which now need to be notified
304      *
305      * Returns 0 on success, or a negative errno; in particular
306      * returns -EINVAL if the new flag bitmap is not supported by the
307      * IOMMU memory region. In case of failure, the error object
308      * must be created
309      */
310     int (*notify_flag_changed)(IOMMUMemoryRegion *iommu,
311                                IOMMUNotifierFlag old_flags,
312                                IOMMUNotifierFlag new_flags,
313                                Error **errp);
314     /**
315      * @replay:
316      *
317      * Called to handle memory_region_iommu_replay().
318      *
319      * The default implementation of memory_region_iommu_replay() is to
320      * call the IOMMU translate method for every page in the address space
321      * with flag == IOMMU_NONE and then call the notifier if translate
322      * returns a valid mapping. If this method is implemented then it
323      * overrides the default behaviour, and must provide the full semantics
324      * of memory_region_iommu_replay(), by calling @notifier for every
325      * translation present in the IOMMU.
326      *
327      * Optional method -- an IOMMU only needs to provide this method
328      * if the default is inefficient or produces undesirable side effects.
329      *
330      * Note: this is not related to record-and-replay functionality.
331      */
332     void (*replay)(IOMMUMemoryRegion *iommu, IOMMUNotifier *notifier);
333 
334     /**
335      * @get_attr:
336      *
337      * Get IOMMU misc attributes. This is an optional method that
338      * can be used to allow users of the IOMMU to get implementation-specific
339      * information. The IOMMU implements this method to handle calls
340      * by IOMMU users to memory_region_iommu_get_attr() by filling in
341      * the arbitrary data pointer for any IOMMUMemoryRegionAttr values that
342      * the IOMMU supports. If the method is unimplemented then
343      * memory_region_iommu_get_attr() will always return -EINVAL.
344      *
345      * @iommu: the IOMMUMemoryRegion
346      *
347      * @attr: attribute being queried
348      *
349      * @data: memory to fill in with the attribute data
350      *
351      * Returns 0 on success, or a negative errno; in particular
352      * returns -EINVAL for unrecognized or unimplemented attribute types.
353      */
354     int (*get_attr)(IOMMUMemoryRegion *iommu, enum IOMMUMemoryRegionAttr attr,
355                     void *data);
356 
357     /**
358      * @attrs_to_index:
359      *
360      * Return the IOMMU index to use for a given set of transaction attributes.
361      *
362      * Optional method: if an IOMMU only supports a single IOMMU index then
363      * the default implementation of memory_region_iommu_attrs_to_index()
364      * will return 0.
365      *
366      * The indexes supported by an IOMMU must be contiguous, starting at 0.
367      *
368      * @iommu: the IOMMUMemoryRegion
369      * @attrs: memory transaction attributes
370      */
371     int (*attrs_to_index)(IOMMUMemoryRegion *iommu, MemTxAttrs attrs);
372 
373     /**
374      * @num_indexes:
375      *
376      * Return the number of IOMMU indexes this IOMMU supports.
377      *
378      * Optional method: if this method is not provided, then
379      * memory_region_iommu_num_indexes() will return 1, indicating that
380      * only a single IOMMU index is supported.
381      *
382      * @iommu: the IOMMUMemoryRegion
383      */
384     int (*num_indexes)(IOMMUMemoryRegion *iommu);
385 };
386 
387 typedef struct CoalescedMemoryRange CoalescedMemoryRange;
388 typedef struct MemoryRegionIoeventfd MemoryRegionIoeventfd;
389 
390 /** MemoryRegion:
391  *
392  * A struct representing a memory region.
393  */
394 struct MemoryRegion {
395     Object parent_obj;
396 
397     /* private: */
398 
399     /* The following fields should fit in a cache line */
400     bool romd_mode;
401     bool ram;
402     bool subpage;
403     bool readonly; /* For RAM regions */
404     bool nonvolatile;
405     bool rom_device;
406     bool flush_coalesced_mmio;
407     bool global_locking;
408     uint8_t dirty_log_mask;
409     bool is_iommu;
410     RAMBlock *ram_block;
411     Object *owner;
412 
413     const MemoryRegionOps *ops;
414     void *opaque;
415     MemoryRegion *container;
416     Int128 size;
417     hwaddr addr;
418     void (*destructor)(MemoryRegion *mr);
419     uint64_t align;
420     bool terminates;
421     bool ram_device;
422     bool enabled;
423     bool warning_printed; /* For reservations */
424     uint8_t vga_logging_count;
425     MemoryRegion *alias;
426     hwaddr alias_offset;
427     int32_t priority;
428     QTAILQ_HEAD(, MemoryRegion) subregions;
429     QTAILQ_ENTRY(MemoryRegion) subregions_link;
430     QTAILQ_HEAD(, CoalescedMemoryRange) coalesced;
431     const char *name;
432     unsigned ioeventfd_nb;
433     MemoryRegionIoeventfd *ioeventfds;
434 };
435 
436 struct IOMMUMemoryRegion {
437     MemoryRegion parent_obj;
438 
439     QLIST_HEAD(, IOMMUNotifier) iommu_notify;
440     IOMMUNotifierFlag iommu_notify_flags;
441 };
442 
443 #define IOMMU_NOTIFIER_FOREACH(n, mr) \
444     QLIST_FOREACH((n), &(mr)->iommu_notify, node)
445 
446 /**
447  * MemoryListener: callbacks structure for updates to the physical memory map
448  *
449  * Allows a component to adjust to changes in the guest-visible memory map.
450  * Use with memory_listener_register() and memory_listener_unregister().
451  */
452 struct MemoryListener {
453     /**
454      * @begin:
455      *
456      * Called at the beginning of an address space update transaction.
457      * Followed by calls to #MemoryListener.region_add(),
458      * #MemoryListener.region_del(), #MemoryListener.region_nop(),
459      * #MemoryListener.log_start() and #MemoryListener.log_stop() in
460      * increasing address order.
461      *
462      * @listener: The #MemoryListener.
463      */
464     void (*begin)(MemoryListener *listener);
465 
466     /**
467      * @commit:
468      *
469      * Called at the end of an address space update transaction,
470      * after the last call to #MemoryListener.region_add(),
471      * #MemoryListener.region_del() or #MemoryListener.region_nop(),
472      * #MemoryListener.log_start() and #MemoryListener.log_stop().
473      *
474      * @listener: The #MemoryListener.
475      */
476     void (*commit)(MemoryListener *listener);
477 
478     /**
479      * @region_add:
480      *
481      * Called during an address space update transaction,
482      * for a section of the address space that is new in this address space
483      * space since the last transaction.
484      *
485      * @listener: The #MemoryListener.
486      * @section: The new #MemoryRegionSection.
487      */
488     void (*region_add)(MemoryListener *listener, MemoryRegionSection *section);
489 
490     /**
491      * @region_del:
492      *
493      * Called during an address space update transaction,
494      * for a section of the address space that has disappeared in the address
495      * space since the last transaction.
496      *
497      * @listener: The #MemoryListener.
498      * @section: The old #MemoryRegionSection.
499      */
500     void (*region_del)(MemoryListener *listener, MemoryRegionSection *section);
501 
502     /**
503      * @region_nop:
504      *
505      * Called during an address space update transaction,
506      * for a section of the address space that is in the same place in the address
507      * space as in the last transaction.
508      *
509      * @listener: The #MemoryListener.
510      * @section: The #MemoryRegionSection.
511      */
512     void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section);
513 
514     /**
515      * @log_start:
516      *
517      * Called during an address space update transaction, after
518      * one of #MemoryListener.region_add(),#MemoryListener.region_del() or
519      * #MemoryListener.region_nop(), if dirty memory logging clients have
520      * become active since the last transaction.
521      *
522      * @listener: The #MemoryListener.
523      * @section: The #MemoryRegionSection.
524      * @old: A bitmap of dirty memory logging clients that were active in
525      * the previous transaction.
526      * @new: A bitmap of dirty memory logging clients that are active in
527      * the current transaction.
528      */
529     void (*log_start)(MemoryListener *listener, MemoryRegionSection *section,
530                       int old, int new);
531 
532     /**
533      * @log_stop:
534      *
535      * Called during an address space update transaction, after
536      * one of #MemoryListener.region_add(), #MemoryListener.region_del() or
537      * #MemoryListener.region_nop() and possibly after
538      * #MemoryListener.log_start(), if dirty memory logging clients have
539      * become inactive since the last transaction.
540      *
541      * @listener: The #MemoryListener.
542      * @section: The #MemoryRegionSection.
543      * @old: A bitmap of dirty memory logging clients that were active in
544      * the previous transaction.
545      * @new: A bitmap of dirty memory logging clients that are active in
546      * the current transaction.
547      */
548     void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section,
549                      int old, int new);
550 
551     /**
552      * @log_sync:
553      *
554      * Called by memory_region_snapshot_and_clear_dirty() and
555      * memory_global_dirty_log_sync(), before accessing QEMU's "official"
556      * copy of the dirty memory bitmap for a #MemoryRegionSection.
557      *
558      * @listener: The #MemoryListener.
559      * @section: The #MemoryRegionSection.
560      */
561     void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section);
562 
563     /**
564      * @log_clear:
565      *
566      * Called before reading the dirty memory bitmap for a
567      * #MemoryRegionSection.
568      *
569      * @listener: The #MemoryListener.
570      * @section: The #MemoryRegionSection.
571      */
572     void (*log_clear)(MemoryListener *listener, MemoryRegionSection *section);
573 
574     /**
575      * @log_global_start:
576      *
577      * Called by memory_global_dirty_log_start(), which
578      * enables the %DIRTY_LOG_MIGRATION client on all memory regions in
579      * the address space.  #MemoryListener.log_global_start() is also
580      * called when a #MemoryListener is added, if global dirty logging is
581      * active at that time.
582      *
583      * @listener: The #MemoryListener.
584      */
585     void (*log_global_start)(MemoryListener *listener);
586 
587     /**
588      * @log_global_stop:
589      *
590      * Called by memory_global_dirty_log_stop(), which
591      * disables the %DIRTY_LOG_MIGRATION client on all memory regions in
592      * the address space.
593      *
594      * @listener: The #MemoryListener.
595      */
596     void (*log_global_stop)(MemoryListener *listener);
597 
598     /**
599      * @log_global_after_sync:
600      *
601      * Called after reading the dirty memory bitmap
602      * for any #MemoryRegionSection.
603      *
604      * @listener: The #MemoryListener.
605      */
606     void (*log_global_after_sync)(MemoryListener *listener);
607 
608     /**
609      * @eventfd_add:
610      *
611      * Called during an address space update transaction,
612      * for a section of the address space that has had a new ioeventfd
613      * registration since the last transaction.
614      *
615      * @listener: The #MemoryListener.
616      * @section: The new #MemoryRegionSection.
617      * @match_data: The @match_data parameter for the new ioeventfd.
618      * @data: The @data parameter for the new ioeventfd.
619      * @e: The #EventNotifier parameter for the new ioeventfd.
620      */
621     void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section,
622                         bool match_data, uint64_t data, EventNotifier *e);
623 
624     /**
625      * @eventfd_del:
626      *
627      * Called during an address space update transaction,
628      * for a section of the address space that has dropped an ioeventfd
629      * registration since the last transaction.
630      *
631      * @listener: The #MemoryListener.
632      * @section: The new #MemoryRegionSection.
633      * @match_data: The @match_data parameter for the dropped ioeventfd.
634      * @data: The @data parameter for the dropped ioeventfd.
635      * @e: The #EventNotifier parameter for the dropped ioeventfd.
636      */
637     void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
638                         bool match_data, uint64_t data, EventNotifier *e);
639 
640     /**
641      * @coalesced_io_add:
642      *
643      * Called during an address space update transaction,
644      * for a section of the address space that has had a new coalesced
645      * MMIO range registration since the last transaction.
646      *
647      * @listener: The #MemoryListener.
648      * @section: The new #MemoryRegionSection.
649      * @addr: The starting address for the coalesced MMIO range.
650      * @len: The length of the coalesced MMIO range.
651      */
652     void (*coalesced_io_add)(MemoryListener *listener, MemoryRegionSection *section,
653                                hwaddr addr, hwaddr len);
654 
655     /**
656      * @coalesced_io_del:
657      *
658      * Called during an address space update transaction,
659      * for a section of the address space that has dropped a coalesced
660      * MMIO range since the last transaction.
661      *
662      * @listener: The #MemoryListener.
663      * @section: The new #MemoryRegionSection.
664      * @addr: The starting address for the coalesced MMIO range.
665      * @len: The length of the coalesced MMIO range.
666      */
667     void (*coalesced_io_del)(MemoryListener *listener, MemoryRegionSection *section,
668                                hwaddr addr, hwaddr len);
669     /**
670      * @priority:
671      *
672      * Govern the order in which memory listeners are invoked. Lower priorities
673      * are invoked earlier for "add" or "start" callbacks, and later for "delete"
674      * or "stop" callbacks.
675      */
676     unsigned priority;
677 
678     /* private: */
679     AddressSpace *address_space;
680     QTAILQ_ENTRY(MemoryListener) link;
681     QTAILQ_ENTRY(MemoryListener) link_as;
682 };
683 
684 /**
685  * AddressSpace: describes a mapping of addresses to #MemoryRegion objects
686  */
687 struct AddressSpace {
688     /* private: */
689     struct rcu_head rcu;
690     char *name;
691     MemoryRegion *root;
692 
693     /* Accessed via RCU.  */
694     struct FlatView *current_map;
695 
696     int ioeventfd_nb;
697     struct MemoryRegionIoeventfd *ioeventfds;
698     QTAILQ_HEAD(, MemoryListener) listeners;
699     QTAILQ_ENTRY(AddressSpace) address_spaces_link;
700 };
701 
702 typedef struct AddressSpaceDispatch AddressSpaceDispatch;
703 typedef struct FlatRange FlatRange;
704 
705 /* Flattened global view of current active memory hierarchy.  Kept in sorted
706  * order.
707  */
708 struct FlatView {
709     struct rcu_head rcu;
710     unsigned ref;
711     FlatRange *ranges;
712     unsigned nr;
713     unsigned nr_allocated;
714     struct AddressSpaceDispatch *dispatch;
715     MemoryRegion *root;
716 };
717 
718 static inline FlatView *address_space_to_flatview(AddressSpace *as)
719 {
720     return qatomic_rcu_read(&as->current_map);
721 }
722 
723 
724 /**
725  * MemoryRegionSection: describes a fragment of a #MemoryRegion
726  *
727  * @mr: the region, or %NULL if empty
728  * @fv: the flat view of the address space the region is mapped in
729  * @offset_within_region: the beginning of the section, relative to @mr's start
730  * @size: the size of the section; will not exceed @mr's boundaries
731  * @offset_within_address_space: the address of the first byte of the section
732  *     relative to the region's address space
733  * @readonly: writes to this section are ignored
734  * @nonvolatile: this section is non-volatile
735  */
736 struct MemoryRegionSection {
737     Int128 size;
738     MemoryRegion *mr;
739     FlatView *fv;
740     hwaddr offset_within_region;
741     hwaddr offset_within_address_space;
742     bool readonly;
743     bool nonvolatile;
744 };
745 
746 static inline bool MemoryRegionSection_eq(MemoryRegionSection *a,
747                                           MemoryRegionSection *b)
748 {
749     return a->mr == b->mr &&
750            a->fv == b->fv &&
751            a->offset_within_region == b->offset_within_region &&
752            a->offset_within_address_space == b->offset_within_address_space &&
753            int128_eq(a->size, b->size) &&
754            a->readonly == b->readonly &&
755            a->nonvolatile == b->nonvolatile;
756 }
757 
758 /**
759  * memory_region_init: Initialize a memory region
760  *
761  * The region typically acts as a container for other memory regions.  Use
762  * memory_region_add_subregion() to add subregions.
763  *
764  * @mr: the #MemoryRegion to be initialized
765  * @owner: the object that tracks the region's reference count
766  * @name: used for debugging; not visible to the user or ABI
767  * @size: size of the region; any subregions beyond this size will be clipped
768  */
769 void memory_region_init(MemoryRegion *mr,
770                         struct Object *owner,
771                         const char *name,
772                         uint64_t size);
773 
774 /**
775  * memory_region_ref: Add 1 to a memory region's reference count
776  *
777  * Whenever memory regions are accessed outside the BQL, they need to be
778  * preserved against hot-unplug.  MemoryRegions actually do not have their
779  * own reference count; they piggyback on a QOM object, their "owner".
780  * This function adds a reference to the owner.
781  *
782  * All MemoryRegions must have an owner if they can disappear, even if the
783  * device they belong to operates exclusively under the BQL.  This is because
784  * the region could be returned at any time by memory_region_find, and this
785  * is usually under guest control.
786  *
787  * @mr: the #MemoryRegion
788  */
789 void memory_region_ref(MemoryRegion *mr);
790 
791 /**
792  * memory_region_unref: Remove 1 to a memory region's reference count
793  *
794  * Whenever memory regions are accessed outside the BQL, they need to be
795  * preserved against hot-unplug.  MemoryRegions actually do not have their
796  * own reference count; they piggyback on a QOM object, their "owner".
797  * This function removes a reference to the owner and possibly destroys it.
798  *
799  * @mr: the #MemoryRegion
800  */
801 void memory_region_unref(MemoryRegion *mr);
802 
803 /**
804  * memory_region_init_io: Initialize an I/O memory region.
805  *
806  * Accesses into the region will cause the callbacks in @ops to be called.
807  * if @size is nonzero, subregions will be clipped to @size.
808  *
809  * @mr: the #MemoryRegion to be initialized.
810  * @owner: the object that tracks the region's reference count
811  * @ops: a structure containing read and write callbacks to be used when
812  *       I/O is performed on the region.
813  * @opaque: passed to the read and write callbacks of the @ops structure.
814  * @name: used for debugging; not visible to the user or ABI
815  * @size: size of the region.
816  */
817 void memory_region_init_io(MemoryRegion *mr,
818                            struct Object *owner,
819                            const MemoryRegionOps *ops,
820                            void *opaque,
821                            const char *name,
822                            uint64_t size);
823 
824 /**
825  * memory_region_init_ram_nomigrate:  Initialize RAM memory region.  Accesses
826  *                                    into the region will modify memory
827  *                                    directly.
828  *
829  * @mr: the #MemoryRegion to be initialized.
830  * @owner: the object that tracks the region's reference count
831  * @name: Region name, becomes part of RAMBlock name used in migration stream
832  *        must be unique within any device
833  * @size: size of the region.
834  * @errp: pointer to Error*, to store an error if it happens.
835  *
836  * Note that this function does not do anything to cause the data in the
837  * RAM memory region to be migrated; that is the responsibility of the caller.
838  */
839 void memory_region_init_ram_nomigrate(MemoryRegion *mr,
840                                       struct Object *owner,
841                                       const char *name,
842                                       uint64_t size,
843                                       Error **errp);
844 
845 /**
846  * memory_region_init_ram_shared_nomigrate:  Initialize RAM memory region.
847  *                                           Accesses into the region will
848  *                                           modify memory directly.
849  *
850  * @mr: the #MemoryRegion to be initialized.
851  * @owner: the object that tracks the region's reference count
852  * @name: Region name, becomes part of RAMBlock name used in migration stream
853  *        must be unique within any device
854  * @size: size of the region.
855  * @share: allow remapping RAM to different addresses
856  * @errp: pointer to Error*, to store an error if it happens.
857  *
858  * Note that this function is similar to memory_region_init_ram_nomigrate.
859  * The only difference is part of the RAM region can be remapped.
860  */
861 void memory_region_init_ram_shared_nomigrate(MemoryRegion *mr,
862                                              struct Object *owner,
863                                              const char *name,
864                                              uint64_t size,
865                                              bool share,
866                                              Error **errp);
867 
868 /**
869  * memory_region_init_resizeable_ram:  Initialize memory region with resizeable
870  *                                     RAM.  Accesses into the region will
871  *                                     modify memory directly.  Only an initial
872  *                                     portion of this RAM is actually used.
873  *                                     The used size can change across reboots.
874  *
875  * @mr: the #MemoryRegion to be initialized.
876  * @owner: the object that tracks the region's reference count
877  * @name: Region name, becomes part of RAMBlock name used in migration stream
878  *        must be unique within any device
879  * @size: used size of the region.
880  * @max_size: max size of the region.
881  * @resized: callback to notify owner about used size change.
882  * @errp: pointer to Error*, to store an error if it happens.
883  *
884  * Note that this function does not do anything to cause the data in the
885  * RAM memory region to be migrated; that is the responsibility of the caller.
886  */
887 void memory_region_init_resizeable_ram(MemoryRegion *mr,
888                                        struct Object *owner,
889                                        const char *name,
890                                        uint64_t size,
891                                        uint64_t max_size,
892                                        void (*resized)(const char*,
893                                                        uint64_t length,
894                                                        void *host),
895                                        Error **errp);
896 #ifdef CONFIG_POSIX
897 
898 /**
899  * memory_region_init_ram_from_file:  Initialize RAM memory region with a
900  *                                    mmap-ed backend.
901  *
902  * @mr: the #MemoryRegion to be initialized.
903  * @owner: the object that tracks the region's reference count
904  * @name: Region name, becomes part of RAMBlock name used in migration stream
905  *        must be unique within any device
906  * @size: size of the region.
907  * @align: alignment of the region base address; if 0, the default alignment
908  *         (getpagesize()) will be used.
909  * @ram_flags: Memory region features:
910  *             - RAM_SHARED: memory must be mmaped with the MAP_SHARED flag
911  *             - RAM_PMEM: the memory is persistent memory
912  *             Other bits are ignored now.
913  * @path: the path in which to allocate the RAM.
914  * @errp: pointer to Error*, to store an error if it happens.
915  *
916  * Note that this function does not do anything to cause the data in the
917  * RAM memory region to be migrated; that is the responsibility of the caller.
918  */
919 void memory_region_init_ram_from_file(MemoryRegion *mr,
920                                       struct Object *owner,
921                                       const char *name,
922                                       uint64_t size,
923                                       uint64_t align,
924                                       uint32_t ram_flags,
925                                       const char *path,
926                                       Error **errp);
927 
928 /**
929  * memory_region_init_ram_from_fd:  Initialize RAM memory region with a
930  *                                  mmap-ed backend.
931  *
932  * @mr: the #MemoryRegion to be initialized.
933  * @owner: the object that tracks the region's reference count
934  * @name: the name of the region.
935  * @size: size of the region.
936  * @share: %true if memory must be mmaped with the MAP_SHARED flag
937  * @fd: the fd to mmap.
938  * @errp: pointer to Error*, to store an error if it happens.
939  *
940  * Note that this function does not do anything to cause the data in the
941  * RAM memory region to be migrated; that is the responsibility of the caller.
942  */
943 void memory_region_init_ram_from_fd(MemoryRegion *mr,
944                                     struct Object *owner,
945                                     const char *name,
946                                     uint64_t size,
947                                     bool share,
948                                     int fd,
949                                     Error **errp);
950 #endif
951 
952 /**
953  * memory_region_init_ram_ptr:  Initialize RAM memory region from a
954  *                              user-provided pointer.  Accesses into the
955  *                              region will modify memory directly.
956  *
957  * @mr: the #MemoryRegion to be initialized.
958  * @owner: the object that tracks the region's reference count
959  * @name: Region name, becomes part of RAMBlock name used in migration stream
960  *        must be unique within any device
961  * @size: size of the region.
962  * @ptr: memory to be mapped; must contain at least @size bytes.
963  *
964  * Note that this function does not do anything to cause the data in the
965  * RAM memory region to be migrated; that is the responsibility of the caller.
966  */
967 void memory_region_init_ram_ptr(MemoryRegion *mr,
968                                 struct Object *owner,
969                                 const char *name,
970                                 uint64_t size,
971                                 void *ptr);
972 
973 /**
974  * memory_region_init_ram_device_ptr:  Initialize RAM device memory region from
975  *                                     a user-provided pointer.
976  *
977  * A RAM device represents a mapping to a physical device, such as to a PCI
978  * MMIO BAR of an vfio-pci assigned device.  The memory region may be mapped
979  * into the VM address space and access to the region will modify memory
980  * directly.  However, the memory region should not be included in a memory
981  * dump (device may not be enabled/mapped at the time of the dump), and
982  * operations incompatible with manipulating MMIO should be avoided.  Replaces
983  * skip_dump flag.
984  *
985  * @mr: the #MemoryRegion to be initialized.
986  * @owner: the object that tracks the region's reference count
987  * @name: the name of the region.
988  * @size: size of the region.
989  * @ptr: memory to be mapped; must contain at least @size bytes.
990  *
991  * Note that this function does not do anything to cause the data in the
992  * RAM memory region to be migrated; that is the responsibility of the caller.
993  * (For RAM device memory regions, migrating the contents rarely makes sense.)
994  */
995 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
996                                        struct Object *owner,
997                                        const char *name,
998                                        uint64_t size,
999                                        void *ptr);
1000 
1001 /**
1002  * memory_region_init_alias: Initialize a memory region that aliases all or a
1003  *                           part of another memory region.
1004  *
1005  * @mr: the #MemoryRegion to be initialized.
1006  * @owner: the object that tracks the region's reference count
1007  * @name: used for debugging; not visible to the user or ABI
1008  * @orig: the region to be referenced; @mr will be equivalent to
1009  *        @orig between @offset and @offset + @size - 1.
1010  * @offset: start of the section in @orig to be referenced.
1011  * @size: size of the region.
1012  */
1013 void memory_region_init_alias(MemoryRegion *mr,
1014                               struct Object *owner,
1015                               const char *name,
1016                               MemoryRegion *orig,
1017                               hwaddr offset,
1018                               uint64_t size);
1019 
1020 /**
1021  * memory_region_init_rom_nomigrate: Initialize a ROM memory region.
1022  *
1023  * This has the same effect as calling memory_region_init_ram_nomigrate()
1024  * and then marking the resulting region read-only with
1025  * memory_region_set_readonly().
1026  *
1027  * Note that this function does not do anything to cause the data in the
1028  * RAM side of the memory region to be migrated; that is the responsibility
1029  * of the caller.
1030  *
1031  * @mr: the #MemoryRegion to be initialized.
1032  * @owner: the object that tracks the region's reference count
1033  * @name: Region name, becomes part of RAMBlock name used in migration stream
1034  *        must be unique within any device
1035  * @size: size of the region.
1036  * @errp: pointer to Error*, to store an error if it happens.
1037  */
1038 void memory_region_init_rom_nomigrate(MemoryRegion *mr,
1039                                       struct Object *owner,
1040                                       const char *name,
1041                                       uint64_t size,
1042                                       Error **errp);
1043 
1044 /**
1045  * memory_region_init_rom_device_nomigrate:  Initialize a ROM memory region.
1046  *                                 Writes are handled via callbacks.
1047  *
1048  * Note that this function does not do anything to cause the data in the
1049  * RAM side of the memory region to be migrated; that is the responsibility
1050  * of the caller.
1051  *
1052  * @mr: the #MemoryRegion to be initialized.
1053  * @owner: the object that tracks the region's reference count
1054  * @ops: callbacks for write access handling (must not be NULL).
1055  * @opaque: passed to the read and write callbacks of the @ops structure.
1056  * @name: Region name, becomes part of RAMBlock name used in migration stream
1057  *        must be unique within any device
1058  * @size: size of the region.
1059  * @errp: pointer to Error*, to store an error if it happens.
1060  */
1061 void memory_region_init_rom_device_nomigrate(MemoryRegion *mr,
1062                                              struct Object *owner,
1063                                              const MemoryRegionOps *ops,
1064                                              void *opaque,
1065                                              const char *name,
1066                                              uint64_t size,
1067                                              Error **errp);
1068 
1069 /**
1070  * memory_region_init_iommu: Initialize a memory region of a custom type
1071  * that translates addresses
1072  *
1073  * An IOMMU region translates addresses and forwards accesses to a target
1074  * memory region.
1075  *
1076  * The IOMMU implementation must define a subclass of TYPE_IOMMU_MEMORY_REGION.
1077  * @_iommu_mr should be a pointer to enough memory for an instance of
1078  * that subclass, @instance_size is the size of that subclass, and
1079  * @mrtypename is its name. This function will initialize @_iommu_mr as an
1080  * instance of the subclass, and its methods will then be called to handle
1081  * accesses to the memory region. See the documentation of
1082  * #IOMMUMemoryRegionClass for further details.
1083  *
1084  * @_iommu_mr: the #IOMMUMemoryRegion to be initialized
1085  * @instance_size: the IOMMUMemoryRegion subclass instance size
1086  * @mrtypename: the type name of the #IOMMUMemoryRegion
1087  * @owner: the object that tracks the region's reference count
1088  * @name: used for debugging; not visible to the user or ABI
1089  * @size: size of the region.
1090  */
1091 void memory_region_init_iommu(void *_iommu_mr,
1092                               size_t instance_size,
1093                               const char *mrtypename,
1094                               Object *owner,
1095                               const char *name,
1096                               uint64_t size);
1097 
1098 /**
1099  * memory_region_init_ram - Initialize RAM memory region.  Accesses into the
1100  *                          region will modify memory directly.
1101  *
1102  * @mr: the #MemoryRegion to be initialized
1103  * @owner: the object that tracks the region's reference count (must be
1104  *         TYPE_DEVICE or a subclass of TYPE_DEVICE, or NULL)
1105  * @name: name of the memory region
1106  * @size: size of the region in bytes
1107  * @errp: pointer to Error*, to store an error if it happens.
1108  *
1109  * This function allocates RAM for a board model or device, and
1110  * arranges for it to be migrated (by calling vmstate_register_ram()
1111  * if @owner is a DeviceState, or vmstate_register_ram_global() if
1112  * @owner is NULL).
1113  *
1114  * TODO: Currently we restrict @owner to being either NULL (for
1115  * global RAM regions with no owner) or devices, so that we can
1116  * give the RAM block a unique name for migration purposes.
1117  * We should lift this restriction and allow arbitrary Objects.
1118  * If you pass a non-NULL non-device @owner then we will assert.
1119  */
1120 void memory_region_init_ram(MemoryRegion *mr,
1121                             struct Object *owner,
1122                             const char *name,
1123                             uint64_t size,
1124                             Error **errp);
1125 
1126 /**
1127  * memory_region_init_rom: Initialize a ROM memory region.
1128  *
1129  * This has the same effect as calling memory_region_init_ram()
1130  * and then marking the resulting region read-only with
1131  * memory_region_set_readonly(). This includes arranging for the
1132  * contents to be migrated.
1133  *
1134  * TODO: Currently we restrict @owner to being either NULL (for
1135  * global RAM regions with no owner) or devices, so that we can
1136  * give the RAM block a unique name for migration purposes.
1137  * We should lift this restriction and allow arbitrary Objects.
1138  * If you pass a non-NULL non-device @owner then we will assert.
1139  *
1140  * @mr: the #MemoryRegion to be initialized.
1141  * @owner: the object that tracks the region's reference count
1142  * @name: Region name, becomes part of RAMBlock name used in migration stream
1143  *        must be unique within any device
1144  * @size: size of the region.
1145  * @errp: pointer to Error*, to store an error if it happens.
1146  */
1147 void memory_region_init_rom(MemoryRegion *mr,
1148                             struct Object *owner,
1149                             const char *name,
1150                             uint64_t size,
1151                             Error **errp);
1152 
1153 /**
1154  * memory_region_init_rom_device:  Initialize a ROM memory region.
1155  *                                 Writes are handled via callbacks.
1156  *
1157  * This function initializes a memory region backed by RAM for reads
1158  * and callbacks for writes, and arranges for the RAM backing to
1159  * be migrated (by calling vmstate_register_ram()
1160  * if @owner is a DeviceState, or vmstate_register_ram_global() if
1161  * @owner is NULL).
1162  *
1163  * TODO: Currently we restrict @owner to being either NULL (for
1164  * global RAM regions with no owner) or devices, so that we can
1165  * give the RAM block a unique name for migration purposes.
1166  * We should lift this restriction and allow arbitrary Objects.
1167  * If you pass a non-NULL non-device @owner then we will assert.
1168  *
1169  * @mr: the #MemoryRegion to be initialized.
1170  * @owner: the object that tracks the region's reference count
1171  * @ops: callbacks for write access handling (must not be NULL).
1172  * @opaque: passed to the read and write callbacks of the @ops structure.
1173  * @name: Region name, becomes part of RAMBlock name used in migration stream
1174  *        must be unique within any device
1175  * @size: size of the region.
1176  * @errp: pointer to Error*, to store an error if it happens.
1177  */
1178 void memory_region_init_rom_device(MemoryRegion *mr,
1179                                    struct Object *owner,
1180                                    const MemoryRegionOps *ops,
1181                                    void *opaque,
1182                                    const char *name,
1183                                    uint64_t size,
1184                                    Error **errp);
1185 
1186 
1187 /**
1188  * memory_region_owner: get a memory region's owner.
1189  *
1190  * @mr: the memory region being queried.
1191  */
1192 struct Object *memory_region_owner(MemoryRegion *mr);
1193 
1194 /**
1195  * memory_region_size: get a memory region's size.
1196  *
1197  * @mr: the memory region being queried.
1198  */
1199 uint64_t memory_region_size(MemoryRegion *mr);
1200 
1201 /**
1202  * memory_region_is_ram: check whether a memory region is random access
1203  *
1204  * Returns %true if a memory region is random access.
1205  *
1206  * @mr: the memory region being queried
1207  */
1208 static inline bool memory_region_is_ram(MemoryRegion *mr)
1209 {
1210     return mr->ram;
1211 }
1212 
1213 /**
1214  * memory_region_is_ram_device: check whether a memory region is a ram device
1215  *
1216  * Returns %true if a memory region is a device backed ram region
1217  *
1218  * @mr: the memory region being queried
1219  */
1220 bool memory_region_is_ram_device(MemoryRegion *mr);
1221 
1222 /**
1223  * memory_region_is_romd: check whether a memory region is in ROMD mode
1224  *
1225  * Returns %true if a memory region is a ROM device and currently set to allow
1226  * direct reads.
1227  *
1228  * @mr: the memory region being queried
1229  */
1230 static inline bool memory_region_is_romd(MemoryRegion *mr)
1231 {
1232     return mr->rom_device && mr->romd_mode;
1233 }
1234 
1235 /**
1236  * memory_region_get_iommu: check whether a memory region is an iommu
1237  *
1238  * Returns pointer to IOMMUMemoryRegion if a memory region is an iommu,
1239  * otherwise NULL.
1240  *
1241  * @mr: the memory region being queried
1242  */
1243 static inline IOMMUMemoryRegion *memory_region_get_iommu(MemoryRegion *mr)
1244 {
1245     if (mr->alias) {
1246         return memory_region_get_iommu(mr->alias);
1247     }
1248     if (mr->is_iommu) {
1249         return (IOMMUMemoryRegion *) mr;
1250     }
1251     return NULL;
1252 }
1253 
1254 /**
1255  * memory_region_get_iommu_class_nocheck: returns iommu memory region class
1256  *   if an iommu or NULL if not
1257  *
1258  * Returns pointer to IOMMUMemoryRegionClass if a memory region is an iommu,
1259  * otherwise NULL. This is fast path avoiding QOM checking, use with caution.
1260  *
1261  * @iommu_mr: the memory region being queried
1262  */
1263 static inline IOMMUMemoryRegionClass *memory_region_get_iommu_class_nocheck(
1264         IOMMUMemoryRegion *iommu_mr)
1265 {
1266     return (IOMMUMemoryRegionClass *) (((Object *)iommu_mr)->class);
1267 }
1268 
1269 #define memory_region_is_iommu(mr) (memory_region_get_iommu(mr) != NULL)
1270 
1271 /**
1272  * memory_region_iommu_get_min_page_size: get minimum supported page size
1273  * for an iommu
1274  *
1275  * Returns minimum supported page size for an iommu.
1276  *
1277  * @iommu_mr: the memory region being queried
1278  */
1279 uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr);
1280 
1281 /**
1282  * memory_region_notify_iommu: notify a change in an IOMMU translation entry.
1283  *
1284  * The notification type will be decided by entry.perm bits:
1285  *
1286  * - For UNMAP (cache invalidation) notifies: set entry.perm to IOMMU_NONE.
1287  * - For MAP (newly added entry) notifies: set entry.perm to the
1288  *   permission of the page (which is definitely !IOMMU_NONE).
1289  *
1290  * Note: for any IOMMU implementation, an in-place mapping change
1291  * should be notified with an UNMAP followed by a MAP.
1292  *
1293  * @iommu_mr: the memory region that was changed
1294  * @iommu_idx: the IOMMU index for the translation table which has changed
1295  * @entry: the new entry in the IOMMU translation table.  The entry
1296  *         replaces all old entries for the same virtual I/O address range.
1297  *         Deleted entries have .@perm == 0.
1298  */
1299 void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
1300                                 int iommu_idx,
1301                                 IOMMUTLBEntry entry);
1302 
1303 /**
1304  * memory_region_notify_one: notify a change in an IOMMU translation
1305  *                           entry to a single notifier
1306  *
1307  * This works just like memory_region_notify_iommu(), but it only
1308  * notifies a specific notifier, not all of them.
1309  *
1310  * @notifier: the notifier to be notified
1311  * @entry: the new entry in the IOMMU translation table.  The entry
1312  *         replaces all old entries for the same virtual I/O address range.
1313  *         Deleted entries have .@perm == 0.
1314  */
1315 void memory_region_notify_one(IOMMUNotifier *notifier,
1316                               IOMMUTLBEntry *entry);
1317 
1318 /**
1319  * memory_region_register_iommu_notifier: register a notifier for changes to
1320  * IOMMU translation entries.
1321  *
1322  * Returns 0 on success, or a negative errno otherwise. In particular,
1323  * -EINVAL indicates that at least one of the attributes of the notifier
1324  * is not supported (flag/range) by the IOMMU memory region. In case of error
1325  * the error object must be created.
1326  *
1327  * @mr: the memory region to observe
1328  * @n: the IOMMUNotifier to be added; the notify callback receives a
1329  *     pointer to an #IOMMUTLBEntry as the opaque value; the pointer
1330  *     ceases to be valid on exit from the notifier.
1331  * @errp: pointer to Error*, to store an error if it happens.
1332  */
1333 int memory_region_register_iommu_notifier(MemoryRegion *mr,
1334                                           IOMMUNotifier *n, Error **errp);
1335 
1336 /**
1337  * memory_region_iommu_replay: replay existing IOMMU translations to
1338  * a notifier with the minimum page granularity returned by
1339  * mr->iommu_ops->get_page_size().
1340  *
1341  * Note: this is not related to record-and-replay functionality.
1342  *
1343  * @iommu_mr: the memory region to observe
1344  * @n: the notifier to which to replay iommu mappings
1345  */
1346 void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n);
1347 
1348 /**
1349  * memory_region_unregister_iommu_notifier: unregister a notifier for
1350  * changes to IOMMU translation entries.
1351  *
1352  * @mr: the memory region which was observed and for which notity_stopped()
1353  *      needs to be called
1354  * @n: the notifier to be removed.
1355  */
1356 void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
1357                                              IOMMUNotifier *n);
1358 
1359 /**
1360  * memory_region_iommu_get_attr: return an IOMMU attr if get_attr() is
1361  * defined on the IOMMU.
1362  *
1363  * Returns 0 on success, or a negative errno otherwise. In particular,
1364  * -EINVAL indicates that the IOMMU does not support the requested
1365  * attribute.
1366  *
1367  * @iommu_mr: the memory region
1368  * @attr: the requested attribute
1369  * @data: a pointer to the requested attribute data
1370  */
1371 int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr,
1372                                  enum IOMMUMemoryRegionAttr attr,
1373                                  void *data);
1374 
1375 /**
1376  * memory_region_iommu_attrs_to_index: return the IOMMU index to
1377  * use for translations with the given memory transaction attributes.
1378  *
1379  * @iommu_mr: the memory region
1380  * @attrs: the memory transaction attributes
1381  */
1382 int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr,
1383                                        MemTxAttrs attrs);
1384 
1385 /**
1386  * memory_region_iommu_num_indexes: return the total number of IOMMU
1387  * indexes that this IOMMU supports.
1388  *
1389  * @iommu_mr: the memory region
1390  */
1391 int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr);
1392 
1393 /**
1394  * memory_region_name: get a memory region's name
1395  *
1396  * Returns the string that was used to initialize the memory region.
1397  *
1398  * @mr: the memory region being queried
1399  */
1400 const char *memory_region_name(const MemoryRegion *mr);
1401 
1402 /**
1403  * memory_region_is_logging: return whether a memory region is logging writes
1404  *
1405  * Returns %true if the memory region is logging writes for the given client
1406  *
1407  * @mr: the memory region being queried
1408  * @client: the client being queried
1409  */
1410 bool memory_region_is_logging(MemoryRegion *mr, uint8_t client);
1411 
1412 /**
1413  * memory_region_get_dirty_log_mask: return the clients for which a
1414  * memory region is logging writes.
1415  *
1416  * Returns a bitmap of clients, in which the DIRTY_MEMORY_* constants
1417  * are the bit indices.
1418  *
1419  * @mr: the memory region being queried
1420  */
1421 uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr);
1422 
1423 /**
1424  * memory_region_is_rom: check whether a memory region is ROM
1425  *
1426  * Returns %true if a memory region is read-only memory.
1427  *
1428  * @mr: the memory region being queried
1429  */
1430 static inline bool memory_region_is_rom(MemoryRegion *mr)
1431 {
1432     return mr->ram && mr->readonly;
1433 }
1434 
1435 /**
1436  * memory_region_is_nonvolatile: check whether a memory region is non-volatile
1437  *
1438  * Returns %true is a memory region is non-volatile memory.
1439  *
1440  * @mr: the memory region being queried
1441  */
1442 static inline bool memory_region_is_nonvolatile(MemoryRegion *mr)
1443 {
1444     return mr->nonvolatile;
1445 }
1446 
1447 /**
1448  * memory_region_get_fd: Get a file descriptor backing a RAM memory region.
1449  *
1450  * Returns a file descriptor backing a file-based RAM memory region,
1451  * or -1 if the region is not a file-based RAM memory region.
1452  *
1453  * @mr: the RAM or alias memory region being queried.
1454  */
1455 int memory_region_get_fd(MemoryRegion *mr);
1456 
1457 /**
1458  * memory_region_from_host: Convert a pointer into a RAM memory region
1459  * and an offset within it.
1460  *
1461  * Given a host pointer inside a RAM memory region (created with
1462  * memory_region_init_ram() or memory_region_init_ram_ptr()), return
1463  * the MemoryRegion and the offset within it.
1464  *
1465  * Use with care; by the time this function returns, the returned pointer is
1466  * not protected by RCU anymore.  If the caller is not within an RCU critical
1467  * section and does not hold the iothread lock, it must have other means of
1468  * protecting the pointer, such as a reference to the region that includes
1469  * the incoming ram_addr_t.
1470  *
1471  * @ptr: the host pointer to be converted
1472  * @offset: the offset within memory region
1473  */
1474 MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset);
1475 
1476 /**
1477  * memory_region_get_ram_ptr: Get a pointer into a RAM memory region.
1478  *
1479  * Returns a host pointer to a RAM memory region (created with
1480  * memory_region_init_ram() or memory_region_init_ram_ptr()).
1481  *
1482  * Use with care; by the time this function returns, the returned pointer is
1483  * not protected by RCU anymore.  If the caller is not within an RCU critical
1484  * section and does not hold the iothread lock, it must have other means of
1485  * protecting the pointer, such as a reference to the region that includes
1486  * the incoming ram_addr_t.
1487  *
1488  * @mr: the memory region being queried.
1489  */
1490 void *memory_region_get_ram_ptr(MemoryRegion *mr);
1491 
1492 /* memory_region_ram_resize: Resize a RAM region.
1493  *
1494  * Only legal before guest might have detected the memory size: e.g. on
1495  * incoming migration, or right after reset.
1496  *
1497  * @mr: a memory region created with @memory_region_init_resizeable_ram.
1498  * @newsize: the new size the region
1499  * @errp: pointer to Error*, to store an error if it happens.
1500  */
1501 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize,
1502                               Error **errp);
1503 
1504 /**
1505  * memory_region_msync: Synchronize selected address range of
1506  * a memory mapped region
1507  *
1508  * @mr: the memory region to be msync
1509  * @addr: the initial address of the range to be sync
1510  * @size: the size of the range to be sync
1511  */
1512 void memory_region_msync(MemoryRegion *mr, hwaddr addr, hwaddr size);
1513 
1514 /**
1515  * memory_region_writeback: Trigger cache writeback for
1516  * selected address range
1517  *
1518  * @mr: the memory region to be updated
1519  * @addr: the initial address of the range to be written back
1520  * @size: the size of the range to be written back
1521  */
1522 void memory_region_writeback(MemoryRegion *mr, hwaddr addr, hwaddr size);
1523 
1524 /**
1525  * memory_region_set_log: Turn dirty logging on or off for a region.
1526  *
1527  * Turns dirty logging on or off for a specified client (display, migration).
1528  * Only meaningful for RAM regions.
1529  *
1530  * @mr: the memory region being updated.
1531  * @log: whether dirty logging is to be enabled or disabled.
1532  * @client: the user of the logging information; %DIRTY_MEMORY_VGA only.
1533  */
1534 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client);
1535 
1536 /**
1537  * memory_region_set_dirty: Mark a range of bytes as dirty in a memory region.
1538  *
1539  * Marks a range of bytes as dirty, after it has been dirtied outside
1540  * guest code.
1541  *
1542  * @mr: the memory region being dirtied.
1543  * @addr: the address (relative to the start of the region) being dirtied.
1544  * @size: size of the range being dirtied.
1545  */
1546 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
1547                              hwaddr size);
1548 
1549 /**
1550  * memory_region_clear_dirty_bitmap - clear dirty bitmap for memory range
1551  *
1552  * This function is called when the caller wants to clear the remote
1553  * dirty bitmap of a memory range within the memory region.  This can
1554  * be used by e.g. KVM to manually clear dirty log when
1555  * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT is declared support by the host
1556  * kernel.
1557  *
1558  * @mr:     the memory region to clear the dirty log upon
1559  * @start:  start address offset within the memory region
1560  * @len:    length of the memory region to clear dirty bitmap
1561  */
1562 void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start,
1563                                       hwaddr len);
1564 
1565 /**
1566  * memory_region_snapshot_and_clear_dirty: Get a snapshot of the dirty
1567  *                                         bitmap and clear it.
1568  *
1569  * Creates a snapshot of the dirty bitmap, clears the dirty bitmap and
1570  * returns the snapshot.  The snapshot can then be used to query dirty
1571  * status, using memory_region_snapshot_get_dirty.  Snapshotting allows
1572  * querying the same page multiple times, which is especially useful for
1573  * display updates where the scanlines often are not page aligned.
1574  *
1575  * The dirty bitmap region which gets copyed into the snapshot (and
1576  * cleared afterwards) can be larger than requested.  The boundaries
1577  * are rounded up/down so complete bitmap longs (covering 64 pages on
1578  * 64bit hosts) can be copied over into the bitmap snapshot.  Which
1579  * isn't a problem for display updates as the extra pages are outside
1580  * the visible area, and in case the visible area changes a full
1581  * display redraw is due anyway.  Should other use cases for this
1582  * function emerge we might have to revisit this implementation
1583  * detail.
1584  *
1585  * Use g_free to release DirtyBitmapSnapshot.
1586  *
1587  * @mr: the memory region being queried.
1588  * @addr: the address (relative to the start of the region) being queried.
1589  * @size: the size of the range being queried.
1590  * @client: the user of the logging information; typically %DIRTY_MEMORY_VGA.
1591  */
1592 DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr,
1593                                                             hwaddr addr,
1594                                                             hwaddr size,
1595                                                             unsigned client);
1596 
1597 /**
1598  * memory_region_snapshot_get_dirty: Check whether a range of bytes is dirty
1599  *                                   in the specified dirty bitmap snapshot.
1600  *
1601  * @mr: the memory region being queried.
1602  * @snap: the dirty bitmap snapshot
1603  * @addr: the address (relative to the start of the region) being queried.
1604  * @size: the size of the range being queried.
1605  */
1606 bool memory_region_snapshot_get_dirty(MemoryRegion *mr,
1607                                       DirtyBitmapSnapshot *snap,
1608                                       hwaddr addr, hwaddr size);
1609 
1610 /**
1611  * memory_region_reset_dirty: Mark a range of pages as clean, for a specified
1612  *                            client.
1613  *
1614  * Marks a range of pages as no longer dirty.
1615  *
1616  * @mr: the region being updated.
1617  * @addr: the start of the subrange being cleaned.
1618  * @size: the size of the subrange being cleaned.
1619  * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
1620  *          %DIRTY_MEMORY_VGA.
1621  */
1622 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
1623                                hwaddr size, unsigned client);
1624 
1625 /**
1626  * memory_region_flush_rom_device: Mark a range of pages dirty and invalidate
1627  *                                 TBs (for self-modifying code).
1628  *
1629  * The MemoryRegionOps->write() callback of a ROM device must use this function
1630  * to mark byte ranges that have been modified internally, such as by directly
1631  * accessing the memory returned by memory_region_get_ram_ptr().
1632  *
1633  * This function marks the range dirty and invalidates TBs so that TCG can
1634  * detect self-modifying code.
1635  *
1636  * @mr: the region being flushed.
1637  * @addr: the start, relative to the start of the region, of the range being
1638  *        flushed.
1639  * @size: the size, in bytes, of the range being flushed.
1640  */
1641 void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size);
1642 
1643 /**
1644  * memory_region_set_readonly: Turn a memory region read-only (or read-write)
1645  *
1646  * Allows a memory region to be marked as read-only (turning it into a ROM).
1647  * only useful on RAM regions.
1648  *
1649  * @mr: the region being updated.
1650  * @readonly: whether rhe region is to be ROM or RAM.
1651  */
1652 void memory_region_set_readonly(MemoryRegion *mr, bool readonly);
1653 
1654 /**
1655  * memory_region_set_nonvolatile: Turn a memory region non-volatile
1656  *
1657  * Allows a memory region to be marked as non-volatile.
1658  * only useful on RAM regions.
1659  *
1660  * @mr: the region being updated.
1661  * @nonvolatile: whether rhe region is to be non-volatile.
1662  */
1663 void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile);
1664 
1665 /**
1666  * memory_region_rom_device_set_romd: enable/disable ROMD mode
1667  *
1668  * Allows a ROM device (initialized with memory_region_init_rom_device() to
1669  * set to ROMD mode (default) or MMIO mode.  When it is in ROMD mode, the
1670  * device is mapped to guest memory and satisfies read access directly.
1671  * When in MMIO mode, reads are forwarded to the #MemoryRegion.read function.
1672  * Writes are always handled by the #MemoryRegion.write function.
1673  *
1674  * @mr: the memory region to be updated
1675  * @romd_mode: %true to put the region into ROMD mode
1676  */
1677 void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode);
1678 
1679 /**
1680  * memory_region_set_coalescing: Enable memory coalescing for the region.
1681  *
1682  * Enabled writes to a region to be queued for later processing. MMIO ->write
1683  * callbacks may be delayed until a non-coalesced MMIO is issued.
1684  * Only useful for IO regions.  Roughly similar to write-combining hardware.
1685  *
1686  * @mr: the memory region to be write coalesced
1687  */
1688 void memory_region_set_coalescing(MemoryRegion *mr);
1689 
1690 /**
1691  * memory_region_add_coalescing: Enable memory coalescing for a sub-range of
1692  *                               a region.
1693  *
1694  * Like memory_region_set_coalescing(), but works on a sub-range of a region.
1695  * Multiple calls can be issued coalesced disjoint ranges.
1696  *
1697  * @mr: the memory region to be updated.
1698  * @offset: the start of the range within the region to be coalesced.
1699  * @size: the size of the subrange to be coalesced.
1700  */
1701 void memory_region_add_coalescing(MemoryRegion *mr,
1702                                   hwaddr offset,
1703                                   uint64_t size);
1704 
1705 /**
1706  * memory_region_clear_coalescing: Disable MMIO coalescing for the region.
1707  *
1708  * Disables any coalescing caused by memory_region_set_coalescing() or
1709  * memory_region_add_coalescing().  Roughly equivalent to uncacheble memory
1710  * hardware.
1711  *
1712  * @mr: the memory region to be updated.
1713  */
1714 void memory_region_clear_coalescing(MemoryRegion *mr);
1715 
1716 /**
1717  * memory_region_set_flush_coalesced: Enforce memory coalescing flush before
1718  *                                    accesses.
1719  *
1720  * Ensure that pending coalesced MMIO request are flushed before the memory
1721  * region is accessed. This property is automatically enabled for all regions
1722  * passed to memory_region_set_coalescing() and memory_region_add_coalescing().
1723  *
1724  * @mr: the memory region to be updated.
1725  */
1726 void memory_region_set_flush_coalesced(MemoryRegion *mr);
1727 
1728 /**
1729  * memory_region_clear_flush_coalesced: Disable memory coalescing flush before
1730  *                                      accesses.
1731  *
1732  * Clear the automatic coalesced MMIO flushing enabled via
1733  * memory_region_set_flush_coalesced. Note that this service has no effect on
1734  * memory regions that have MMIO coalescing enabled for themselves. For them,
1735  * automatic flushing will stop once coalescing is disabled.
1736  *
1737  * @mr: the memory region to be updated.
1738  */
1739 void memory_region_clear_flush_coalesced(MemoryRegion *mr);
1740 
1741 /**
1742  * memory_region_clear_global_locking: Declares that access processing does
1743  *                                     not depend on the QEMU global lock.
1744  *
1745  * By clearing this property, accesses to the memory region will be processed
1746  * outside of QEMU's global lock (unless the lock is held on when issuing the
1747  * access request). In this case, the device model implementing the access
1748  * handlers is responsible for synchronization of concurrency.
1749  *
1750  * @mr: the memory region to be updated.
1751  */
1752 void memory_region_clear_global_locking(MemoryRegion *mr);
1753 
1754 /**
1755  * memory_region_add_eventfd: Request an eventfd to be triggered when a word
1756  *                            is written to a location.
1757  *
1758  * Marks a word in an IO region (initialized with memory_region_init_io())
1759  * as a trigger for an eventfd event.  The I/O callback will not be called.
1760  * The caller must be prepared to handle failure (that is, take the required
1761  * action if the callback _is_ called).
1762  *
1763  * @mr: the memory region being updated.
1764  * @addr: the address within @mr that is to be monitored
1765  * @size: the size of the access to trigger the eventfd
1766  * @match_data: whether to match against @data, instead of just @addr
1767  * @data: the data to match against the guest write
1768  * @e: event notifier to be triggered when @addr, @size, and @data all match.
1769  **/
1770 void memory_region_add_eventfd(MemoryRegion *mr,
1771                                hwaddr addr,
1772                                unsigned size,
1773                                bool match_data,
1774                                uint64_t data,
1775                                EventNotifier *e);
1776 
1777 /**
1778  * memory_region_del_eventfd: Cancel an eventfd.
1779  *
1780  * Cancels an eventfd trigger requested by a previous
1781  * memory_region_add_eventfd() call.
1782  *
1783  * @mr: the memory region being updated.
1784  * @addr: the address within @mr that is to be monitored
1785  * @size: the size of the access to trigger the eventfd
1786  * @match_data: whether to match against @data, instead of just @addr
1787  * @data: the data to match against the guest write
1788  * @e: event notifier to be triggered when @addr, @size, and @data all match.
1789  */
1790 void memory_region_del_eventfd(MemoryRegion *mr,
1791                                hwaddr addr,
1792                                unsigned size,
1793                                bool match_data,
1794                                uint64_t data,
1795                                EventNotifier *e);
1796 
1797 /**
1798  * memory_region_add_subregion: Add a subregion to a container.
1799  *
1800  * Adds a subregion at @offset.  The subregion may not overlap with other
1801  * subregions (except for those explicitly marked as overlapping).  A region
1802  * may only be added once as a subregion (unless removed with
1803  * memory_region_del_subregion()); use memory_region_init_alias() if you
1804  * want a region to be a subregion in multiple locations.
1805  *
1806  * @mr: the region to contain the new subregion; must be a container
1807  *      initialized with memory_region_init().
1808  * @offset: the offset relative to @mr where @subregion is added.
1809  * @subregion: the subregion to be added.
1810  */
1811 void memory_region_add_subregion(MemoryRegion *mr,
1812                                  hwaddr offset,
1813                                  MemoryRegion *subregion);
1814 /**
1815  * memory_region_add_subregion_overlap: Add a subregion to a container
1816  *                                      with overlap.
1817  *
1818  * Adds a subregion at @offset.  The subregion may overlap with other
1819  * subregions.  Conflicts are resolved by having a higher @priority hide a
1820  * lower @priority. Subregions without priority are taken as @priority 0.
1821  * A region may only be added once as a subregion (unless removed with
1822  * memory_region_del_subregion()); use memory_region_init_alias() if you
1823  * want a region to be a subregion in multiple locations.
1824  *
1825  * @mr: the region to contain the new subregion; must be a container
1826  *      initialized with memory_region_init().
1827  * @offset: the offset relative to @mr where @subregion is added.
1828  * @subregion: the subregion to be added.
1829  * @priority: used for resolving overlaps; highest priority wins.
1830  */
1831 void memory_region_add_subregion_overlap(MemoryRegion *mr,
1832                                          hwaddr offset,
1833                                          MemoryRegion *subregion,
1834                                          int priority);
1835 
1836 /**
1837  * memory_region_get_ram_addr: Get the ram address associated with a memory
1838  *                             region
1839  *
1840  * @mr: the region to be queried
1841  */
1842 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr);
1843 
1844 uint64_t memory_region_get_alignment(const MemoryRegion *mr);
1845 /**
1846  * memory_region_del_subregion: Remove a subregion.
1847  *
1848  * Removes a subregion from its container.
1849  *
1850  * @mr: the container to be updated.
1851  * @subregion: the region being removed; must be a current subregion of @mr.
1852  */
1853 void memory_region_del_subregion(MemoryRegion *mr,
1854                                  MemoryRegion *subregion);
1855 
1856 /*
1857  * memory_region_set_enabled: dynamically enable or disable a region
1858  *
1859  * Enables or disables a memory region.  A disabled memory region
1860  * ignores all accesses to itself and its subregions.  It does not
1861  * obscure sibling subregions with lower priority - it simply behaves as
1862  * if it was removed from the hierarchy.
1863  *
1864  * Regions default to being enabled.
1865  *
1866  * @mr: the region to be updated
1867  * @enabled: whether to enable or disable the region
1868  */
1869 void memory_region_set_enabled(MemoryRegion *mr, bool enabled);
1870 
1871 /*
1872  * memory_region_set_address: dynamically update the address of a region
1873  *
1874  * Dynamically updates the address of a region, relative to its container.
1875  * May be used on regions are currently part of a memory hierarchy.
1876  *
1877  * @mr: the region to be updated
1878  * @addr: new address, relative to container region
1879  */
1880 void memory_region_set_address(MemoryRegion *mr, hwaddr addr);
1881 
1882 /*
1883  * memory_region_set_size: dynamically update the size of a region.
1884  *
1885  * Dynamically updates the size of a region.
1886  *
1887  * @mr: the region to be updated
1888  * @size: used size of the region.
1889  */
1890 void memory_region_set_size(MemoryRegion *mr, uint64_t size);
1891 
1892 /*
1893  * memory_region_set_alias_offset: dynamically update a memory alias's offset
1894  *
1895  * Dynamically updates the offset into the target region that an alias points
1896  * to, as if the fourth argument to memory_region_init_alias() has changed.
1897  *
1898  * @mr: the #MemoryRegion to be updated; should be an alias.
1899  * @offset: the new offset into the target memory region
1900  */
1901 void memory_region_set_alias_offset(MemoryRegion *mr,
1902                                     hwaddr offset);
1903 
1904 /**
1905  * memory_region_present: checks if an address relative to a @container
1906  * translates into #MemoryRegion within @container
1907  *
1908  * Answer whether a #MemoryRegion within @container covers the address
1909  * @addr.
1910  *
1911  * @container: a #MemoryRegion within which @addr is a relative address
1912  * @addr: the area within @container to be searched
1913  */
1914 bool memory_region_present(MemoryRegion *container, hwaddr addr);
1915 
1916 /**
1917  * memory_region_is_mapped: returns true if #MemoryRegion is mapped
1918  * into any address space.
1919  *
1920  * @mr: a #MemoryRegion which should be checked if it's mapped
1921  */
1922 bool memory_region_is_mapped(MemoryRegion *mr);
1923 
1924 /**
1925  * memory_region_find: translate an address/size relative to a
1926  * MemoryRegion into a #MemoryRegionSection.
1927  *
1928  * Locates the first #MemoryRegion within @mr that overlaps the range
1929  * given by @addr and @size.
1930  *
1931  * Returns a #MemoryRegionSection that describes a contiguous overlap.
1932  * It will have the following characteristics:
1933  * - @size = 0 iff no overlap was found
1934  * - @mr is non-%NULL iff an overlap was found
1935  *
1936  * Remember that in the return value the @offset_within_region is
1937  * relative to the returned region (in the .@mr field), not to the
1938  * @mr argument.
1939  *
1940  * Similarly, the .@offset_within_address_space is relative to the
1941  * address space that contains both regions, the passed and the
1942  * returned one.  However, in the special case where the @mr argument
1943  * has no container (and thus is the root of the address space), the
1944  * following will hold:
1945  * - @offset_within_address_space >= @addr
1946  * - @offset_within_address_space + .@size <= @addr + @size
1947  *
1948  * @mr: a MemoryRegion within which @addr is a relative address
1949  * @addr: start of the area within @as to be searched
1950  * @size: size of the area to be searched
1951  */
1952 MemoryRegionSection memory_region_find(MemoryRegion *mr,
1953                                        hwaddr addr, uint64_t size);
1954 
1955 /**
1956  * memory_global_dirty_log_sync: synchronize the dirty log for all memory
1957  *
1958  * Synchronizes the dirty page log for all address spaces.
1959  */
1960 void memory_global_dirty_log_sync(void);
1961 
1962 /**
1963  * memory_global_dirty_log_sync: synchronize the dirty log for all memory
1964  *
1965  * Synchronizes the vCPUs with a thread that is reading the dirty bitmap.
1966  * This function must be called after the dirty log bitmap is cleared, and
1967  * before dirty guest memory pages are read.  If you are using
1968  * #DirtyBitmapSnapshot, memory_region_snapshot_and_clear_dirty() takes
1969  * care of doing this.
1970  */
1971 void memory_global_after_dirty_log_sync(void);
1972 
1973 /**
1974  * memory_region_transaction_begin: Start a transaction.
1975  *
1976  * During a transaction, changes will be accumulated and made visible
1977  * only when the transaction ends (is committed).
1978  */
1979 void memory_region_transaction_begin(void);
1980 
1981 /**
1982  * memory_region_transaction_commit: Commit a transaction and make changes
1983  *                                   visible to the guest.
1984  */
1985 void memory_region_transaction_commit(void);
1986 
1987 /**
1988  * memory_listener_register: register callbacks to be called when memory
1989  *                           sections are mapped or unmapped into an address
1990  *                           space
1991  *
1992  * @listener: an object containing the callbacks to be called
1993  * @filter: if non-%NULL, only regions in this address space will be observed
1994  */
1995 void memory_listener_register(MemoryListener *listener, AddressSpace *filter);
1996 
1997 /**
1998  * memory_listener_unregister: undo the effect of memory_listener_register()
1999  *
2000  * @listener: an object containing the callbacks to be removed
2001  */
2002 void memory_listener_unregister(MemoryListener *listener);
2003 
2004 /**
2005  * memory_global_dirty_log_start: begin dirty logging for all regions
2006  */
2007 void memory_global_dirty_log_start(void);
2008 
2009 /**
2010  * memory_global_dirty_log_stop: end dirty logging for all regions
2011  */
2012 void memory_global_dirty_log_stop(void);
2013 
2014 void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled);
2015 
2016 /**
2017  * memory_region_dispatch_read: perform a read directly to the specified
2018  * MemoryRegion.
2019  *
2020  * @mr: #MemoryRegion to access
2021  * @addr: address within that region
2022  * @pval: pointer to uint64_t which the data is written to
2023  * @op: size, sign, and endianness of the memory operation
2024  * @attrs: memory transaction attributes to use for the access
2025  */
2026 MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
2027                                         hwaddr addr,
2028                                         uint64_t *pval,
2029                                         MemOp op,
2030                                         MemTxAttrs attrs);
2031 /**
2032  * memory_region_dispatch_write: perform a write directly to the specified
2033  * MemoryRegion.
2034  *
2035  * @mr: #MemoryRegion to access
2036  * @addr: address within that region
2037  * @data: data to write
2038  * @op: size, sign, and endianness of the memory operation
2039  * @attrs: memory transaction attributes to use for the access
2040  */
2041 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
2042                                          hwaddr addr,
2043                                          uint64_t data,
2044                                          MemOp op,
2045                                          MemTxAttrs attrs);
2046 
2047 /**
2048  * address_space_init: initializes an address space
2049  *
2050  * @as: an uninitialized #AddressSpace
2051  * @root: a #MemoryRegion that routes addresses for the address space
2052  * @name: an address space name.  The name is only used for debugging
2053  *        output.
2054  */
2055 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name);
2056 
2057 /**
2058  * address_space_destroy: destroy an address space
2059  *
2060  * Releases all resources associated with an address space.  After an address space
2061  * is destroyed, its root memory region (given by address_space_init()) may be destroyed
2062  * as well.
2063  *
2064  * @as: address space to be destroyed
2065  */
2066 void address_space_destroy(AddressSpace *as);
2067 
2068 /**
2069  * address_space_remove_listeners: unregister all listeners of an address space
2070  *
2071  * Removes all callbacks previously registered with memory_listener_register()
2072  * for @as.
2073  *
2074  * @as: an initialized #AddressSpace
2075  */
2076 void address_space_remove_listeners(AddressSpace *as);
2077 
2078 /**
2079  * address_space_rw: read from or write to an address space.
2080  *
2081  * Return a MemTxResult indicating whether the operation succeeded
2082  * or failed (eg unassigned memory, device rejected the transaction,
2083  * IOMMU fault).
2084  *
2085  * @as: #AddressSpace to be accessed
2086  * @addr: address within that address space
2087  * @attrs: memory transaction attributes
2088  * @buf: buffer with the data transferred
2089  * @len: the number of bytes to read or write
2090  * @is_write: indicates the transfer direction
2091  */
2092 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr,
2093                              MemTxAttrs attrs, void *buf,
2094                              hwaddr len, bool is_write);
2095 
2096 /**
2097  * address_space_write: write to address space.
2098  *
2099  * Return a MemTxResult indicating whether the operation succeeded
2100  * or failed (eg unassigned memory, device rejected the transaction,
2101  * IOMMU fault).
2102  *
2103  * @as: #AddressSpace to be accessed
2104  * @addr: address within that address space
2105  * @attrs: memory transaction attributes
2106  * @buf: buffer with the data transferred
2107  * @len: the number of bytes to write
2108  */
2109 MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
2110                                 MemTxAttrs attrs,
2111                                 const void *buf, hwaddr len);
2112 
2113 /**
2114  * address_space_write_rom: write to address space, including ROM.
2115  *
2116  * This function writes to the specified address space, but will
2117  * write data to both ROM and RAM. This is used for non-guest
2118  * writes like writes from the gdb debug stub or initial loading
2119  * of ROM contents.
2120  *
2121  * Note that portions of the write which attempt to write data to
2122  * a device will be silently ignored -- only real RAM and ROM will
2123  * be written to.
2124  *
2125  * Return a MemTxResult indicating whether the operation succeeded
2126  * or failed (eg unassigned memory, device rejected the transaction,
2127  * IOMMU fault).
2128  *
2129  * @as: #AddressSpace to be accessed
2130  * @addr: address within that address space
2131  * @attrs: memory transaction attributes
2132  * @buf: buffer with the data transferred
2133  * @len: the number of bytes to write
2134  */
2135 MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
2136                                     MemTxAttrs attrs,
2137                                     const void *buf, hwaddr len);
2138 
2139 /* address_space_ld*: load from an address space
2140  * address_space_st*: store to an address space
2141  *
2142  * These functions perform a load or store of the byte, word,
2143  * longword or quad to the specified address within the AddressSpace.
2144  * The _le suffixed functions treat the data as little endian;
2145  * _be indicates big endian; no suffix indicates "same endianness
2146  * as guest CPU".
2147  *
2148  * The "guest CPU endianness" accessors are deprecated for use outside
2149  * target-* code; devices should be CPU-agnostic and use either the LE
2150  * or the BE accessors.
2151  *
2152  * @as #AddressSpace to be accessed
2153  * @addr: address within that address space
2154  * @val: data value, for stores
2155  * @attrs: memory transaction attributes
2156  * @result: location to write the success/failure of the transaction;
2157  *   if NULL, this information is discarded
2158  */
2159 
2160 #define SUFFIX
2161 #define ARG1         as
2162 #define ARG1_DECL    AddressSpace *as
2163 #include "exec/memory_ldst.h.inc"
2164 
2165 #define SUFFIX
2166 #define ARG1         as
2167 #define ARG1_DECL    AddressSpace *as
2168 #include "exec/memory_ldst_phys.h.inc"
2169 
2170 struct MemoryRegionCache {
2171     void *ptr;
2172     hwaddr xlat;
2173     hwaddr len;
2174     FlatView *fv;
2175     MemoryRegionSection mrs;
2176     bool is_write;
2177 };
2178 
2179 #define MEMORY_REGION_CACHE_INVALID ((MemoryRegionCache) { .mrs.mr = NULL })
2180 
2181 
2182 /* address_space_ld*_cached: load from a cached #MemoryRegion
2183  * address_space_st*_cached: store into a cached #MemoryRegion
2184  *
2185  * These functions perform a load or store of the byte, word,
2186  * longword or quad to the specified address.  The address is
2187  * a physical address in the AddressSpace, but it must lie within
2188  * a #MemoryRegion that was mapped with address_space_cache_init.
2189  *
2190  * The _le suffixed functions treat the data as little endian;
2191  * _be indicates big endian; no suffix indicates "same endianness
2192  * as guest CPU".
2193  *
2194  * The "guest CPU endianness" accessors are deprecated for use outside
2195  * target-* code; devices should be CPU-agnostic and use either the LE
2196  * or the BE accessors.
2197  *
2198  * @cache: previously initialized #MemoryRegionCache to be accessed
2199  * @addr: address within the address space
2200  * @val: data value, for stores
2201  * @attrs: memory transaction attributes
2202  * @result: location to write the success/failure of the transaction;
2203  *   if NULL, this information is discarded
2204  */
2205 
2206 #define SUFFIX       _cached_slow
2207 #define ARG1         cache
2208 #define ARG1_DECL    MemoryRegionCache *cache
2209 #include "exec/memory_ldst.h.inc"
2210 
2211 /* Inline fast path for direct RAM access.  */
2212 static inline uint8_t address_space_ldub_cached(MemoryRegionCache *cache,
2213     hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
2214 {
2215     assert(addr < cache->len);
2216     if (likely(cache->ptr)) {
2217         return ldub_p(cache->ptr + addr);
2218     } else {
2219         return address_space_ldub_cached_slow(cache, addr, attrs, result);
2220     }
2221 }
2222 
2223 static inline void address_space_stb_cached(MemoryRegionCache *cache,
2224     hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
2225 {
2226     assert(addr < cache->len);
2227     if (likely(cache->ptr)) {
2228         stb_p(cache->ptr + addr, val);
2229     } else {
2230         address_space_stb_cached_slow(cache, addr, val, attrs, result);
2231     }
2232 }
2233 
2234 #define ENDIANNESS   _le
2235 #include "exec/memory_ldst_cached.h.inc"
2236 
2237 #define ENDIANNESS   _be
2238 #include "exec/memory_ldst_cached.h.inc"
2239 
2240 #define SUFFIX       _cached
2241 #define ARG1         cache
2242 #define ARG1_DECL    MemoryRegionCache *cache
2243 #include "exec/memory_ldst_phys.h.inc"
2244 
2245 /* address_space_cache_init: prepare for repeated access to a physical
2246  * memory region
2247  *
2248  * @cache: #MemoryRegionCache to be filled
2249  * @as: #AddressSpace to be accessed
2250  * @addr: address within that address space
2251  * @len: length of buffer
2252  * @is_write: indicates the transfer direction
2253  *
2254  * Will only work with RAM, and may map a subset of the requested range by
2255  * returning a value that is less than @len.  On failure, return a negative
2256  * errno value.
2257  *
2258  * Because it only works with RAM, this function can be used for
2259  * read-modify-write operations.  In this case, is_write should be %true.
2260  *
2261  * Note that addresses passed to the address_space_*_cached functions
2262  * are relative to @addr.
2263  */
2264 int64_t address_space_cache_init(MemoryRegionCache *cache,
2265                                  AddressSpace *as,
2266                                  hwaddr addr,
2267                                  hwaddr len,
2268                                  bool is_write);
2269 
2270 /**
2271  * address_space_cache_invalidate: complete a write to a #MemoryRegionCache
2272  *
2273  * @cache: The #MemoryRegionCache to operate on.
2274  * @addr: The first physical address that was written, relative to the
2275  * address that was passed to @address_space_cache_init.
2276  * @access_len: The number of bytes that were written starting at @addr.
2277  */
2278 void address_space_cache_invalidate(MemoryRegionCache *cache,
2279                                     hwaddr addr,
2280                                     hwaddr access_len);
2281 
2282 /**
2283  * address_space_cache_destroy: free a #MemoryRegionCache
2284  *
2285  * @cache: The #MemoryRegionCache whose memory should be released.
2286  */
2287 void address_space_cache_destroy(MemoryRegionCache *cache);
2288 
2289 /* address_space_get_iotlb_entry: translate an address into an IOTLB
2290  * entry. Should be called from an RCU critical section.
2291  */
2292 IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
2293                                             bool is_write, MemTxAttrs attrs);
2294 
2295 /* address_space_translate: translate an address range into an address space
2296  * into a MemoryRegion and an address range into that section.  Should be
2297  * called from an RCU critical section, to avoid that the last reference
2298  * to the returned region disappears after address_space_translate returns.
2299  *
2300  * @fv: #FlatView to be accessed
2301  * @addr: address within that address space
2302  * @xlat: pointer to address within the returned memory region section's
2303  * #MemoryRegion.
2304  * @len: pointer to length
2305  * @is_write: indicates the transfer direction
2306  * @attrs: memory attributes
2307  */
2308 MemoryRegion *flatview_translate(FlatView *fv,
2309                                  hwaddr addr, hwaddr *xlat,
2310                                  hwaddr *len, bool is_write,
2311                                  MemTxAttrs attrs);
2312 
2313 static inline MemoryRegion *address_space_translate(AddressSpace *as,
2314                                                     hwaddr addr, hwaddr *xlat,
2315                                                     hwaddr *len, bool is_write,
2316                                                     MemTxAttrs attrs)
2317 {
2318     return flatview_translate(address_space_to_flatview(as),
2319                               addr, xlat, len, is_write, attrs);
2320 }
2321 
2322 /* address_space_access_valid: check for validity of accessing an address
2323  * space range
2324  *
2325  * Check whether memory is assigned to the given address space range, and
2326  * access is permitted by any IOMMU regions that are active for the address
2327  * space.
2328  *
2329  * For now, addr and len should be aligned to a page size.  This limitation
2330  * will be lifted in the future.
2331  *
2332  * @as: #AddressSpace to be accessed
2333  * @addr: address within that address space
2334  * @len: length of the area to be checked
2335  * @is_write: indicates the transfer direction
2336  * @attrs: memory attributes
2337  */
2338 bool address_space_access_valid(AddressSpace *as, hwaddr addr, hwaddr len,
2339                                 bool is_write, MemTxAttrs attrs);
2340 
2341 /* address_space_map: map a physical memory region into a host virtual address
2342  *
2343  * May map a subset of the requested range, given by and returned in @plen.
2344  * May return %NULL and set *@plen to zero(0), if resources needed to perform
2345  * the mapping are exhausted.
2346  * Use only for reads OR writes - not for read-modify-write operations.
2347  * Use cpu_register_map_client() to know when retrying the map operation is
2348  * likely to succeed.
2349  *
2350  * @as: #AddressSpace to be accessed
2351  * @addr: address within that address space
2352  * @plen: pointer to length of buffer; updated on return
2353  * @is_write: indicates the transfer direction
2354  * @attrs: memory attributes
2355  */
2356 void *address_space_map(AddressSpace *as, hwaddr addr,
2357                         hwaddr *plen, bool is_write, MemTxAttrs attrs);
2358 
2359 /* address_space_unmap: Unmaps a memory region previously mapped by address_space_map()
2360  *
2361  * Will also mark the memory as dirty if @is_write == %true.  @access_len gives
2362  * the amount of memory that was actually read or written by the caller.
2363  *
2364  * @as: #AddressSpace used
2365  * @buffer: host pointer as returned by address_space_map()
2366  * @len: buffer length as returned by address_space_map()
2367  * @access_len: amount of data actually transferred
2368  * @is_write: indicates the transfer direction
2369  */
2370 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2371                          bool is_write, hwaddr access_len);
2372 
2373 
2374 /* Internal functions, part of the implementation of address_space_read.  */
2375 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2376                                     MemTxAttrs attrs, void *buf, hwaddr len);
2377 MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
2378                                    MemTxAttrs attrs, void *buf,
2379                                    hwaddr len, hwaddr addr1, hwaddr l,
2380                                    MemoryRegion *mr);
2381 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr);
2382 
2383 /* Internal functions, part of the implementation of address_space_read_cached
2384  * and address_space_write_cached.  */
2385 MemTxResult address_space_read_cached_slow(MemoryRegionCache *cache,
2386                                            hwaddr addr, void *buf, hwaddr len);
2387 MemTxResult address_space_write_cached_slow(MemoryRegionCache *cache,
2388                                             hwaddr addr, const void *buf,
2389                                             hwaddr len);
2390 
2391 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
2392 {
2393     if (is_write) {
2394         return memory_region_is_ram(mr) && !mr->readonly &&
2395                !mr->rom_device && !memory_region_is_ram_device(mr);
2396     } else {
2397         return (memory_region_is_ram(mr) && !memory_region_is_ram_device(mr)) ||
2398                memory_region_is_romd(mr);
2399     }
2400 }
2401 
2402 /**
2403  * address_space_read: read from an address space.
2404  *
2405  * Return a MemTxResult indicating whether the operation succeeded
2406  * or failed (eg unassigned memory, device rejected the transaction,
2407  * IOMMU fault).  Called within RCU critical section.
2408  *
2409  * @as: #AddressSpace to be accessed
2410  * @addr: address within that address space
2411  * @attrs: memory transaction attributes
2412  * @buf: buffer with the data transferred
2413  * @len: length of the data transferred
2414  */
2415 static inline __attribute__((__always_inline__))
2416 MemTxResult address_space_read(AddressSpace *as, hwaddr addr,
2417                                MemTxAttrs attrs, void *buf,
2418                                hwaddr len)
2419 {
2420     MemTxResult result = MEMTX_OK;
2421     hwaddr l, addr1;
2422     void *ptr;
2423     MemoryRegion *mr;
2424     FlatView *fv;
2425 
2426     if (__builtin_constant_p(len)) {
2427         if (len) {
2428             RCU_READ_LOCK_GUARD();
2429             fv = address_space_to_flatview(as);
2430             l = len;
2431             mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
2432             if (len == l && memory_access_is_direct(mr, false)) {
2433                 ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
2434                 memcpy(buf, ptr, len);
2435             } else {
2436                 result = flatview_read_continue(fv, addr, attrs, buf, len,
2437                                                 addr1, l, mr);
2438             }
2439         }
2440     } else {
2441         result = address_space_read_full(as, addr, attrs, buf, len);
2442     }
2443     return result;
2444 }
2445 
2446 /**
2447  * address_space_read_cached: read from a cached RAM region
2448  *
2449  * @cache: Cached region to be addressed
2450  * @addr: address relative to the base of the RAM region
2451  * @buf: buffer with the data transferred
2452  * @len: length of the data transferred
2453  */
2454 static inline MemTxResult
2455 address_space_read_cached(MemoryRegionCache *cache, hwaddr addr,
2456                           void *buf, hwaddr len)
2457 {
2458     assert(addr < cache->len && len <= cache->len - addr);
2459     if (likely(cache->ptr)) {
2460         memcpy(buf, cache->ptr + addr, len);
2461         return MEMTX_OK;
2462     } else {
2463         return address_space_read_cached_slow(cache, addr, buf, len);
2464     }
2465 }
2466 
2467 /**
2468  * address_space_write_cached: write to a cached RAM region
2469  *
2470  * @cache: Cached region to be addressed
2471  * @addr: address relative to the base of the RAM region
2472  * @buf: buffer with the data transferred
2473  * @len: length of the data transferred
2474  */
2475 static inline MemTxResult
2476 address_space_write_cached(MemoryRegionCache *cache, hwaddr addr,
2477                            const void *buf, hwaddr len)
2478 {
2479     assert(addr < cache->len && len <= cache->len - addr);
2480     if (likely(cache->ptr)) {
2481         memcpy(cache->ptr + addr, buf, len);
2482         return MEMTX_OK;
2483     } else {
2484         return address_space_write_cached_slow(cache, addr, buf, len);
2485     }
2486 }
2487 
2488 #ifdef NEED_CPU_H
2489 /* enum device_endian to MemOp.  */
2490 static inline MemOp devend_memop(enum device_endian end)
2491 {
2492     QEMU_BUILD_BUG_ON(DEVICE_HOST_ENDIAN != DEVICE_LITTLE_ENDIAN &&
2493                       DEVICE_HOST_ENDIAN != DEVICE_BIG_ENDIAN);
2494 
2495 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
2496     /* Swap if non-host endianness or native (target) endianness */
2497     return (end == DEVICE_HOST_ENDIAN) ? 0 : MO_BSWAP;
2498 #else
2499     const int non_host_endianness =
2500         DEVICE_LITTLE_ENDIAN ^ DEVICE_BIG_ENDIAN ^ DEVICE_HOST_ENDIAN;
2501 
2502     /* In this case, native (target) endianness needs no swap.  */
2503     return (end == non_host_endianness) ? MO_BSWAP : 0;
2504 #endif
2505 }
2506 #endif
2507 
2508 /*
2509  * Inhibit technologies that require discarding of pages in RAM blocks, e.g.,
2510  * to manage the actual amount of memory consumed by the VM (then, the memory
2511  * provided by RAM blocks might be bigger than the desired memory consumption).
2512  * This *must* be set if:
2513  * - Discarding parts of a RAM blocks does not result in the change being
2514  *   reflected in the VM and the pages getting freed.
2515  * - All memory in RAM blocks is pinned or duplicated, invaldiating any previous
2516  *   discards blindly.
2517  * - Discarding parts of a RAM blocks will result in integrity issues (e.g.,
2518  *   encrypted VMs).
2519  * Technologies that only temporarily pin the current working set of a
2520  * driver are fine, because we don't expect such pages to be discarded
2521  * (esp. based on guest action like balloon inflation).
2522  *
2523  * This is *not* to be used to protect from concurrent discards (esp.,
2524  * postcopy).
2525  *
2526  * Returns 0 if successful. Returns -EBUSY if a technology that relies on
2527  * discards to work reliably is active.
2528  */
2529 int ram_block_discard_disable(bool state);
2530 
2531 /*
2532  * Inhibit technologies that disable discarding of pages in RAM blocks.
2533  *
2534  * Returns 0 if successful. Returns -EBUSY if discards are already set to
2535  * broken.
2536  */
2537 int ram_block_discard_require(bool state);
2538 
2539 /*
2540  * Test if discarding of memory in ram blocks is disabled.
2541  */
2542 bool ram_block_discard_is_disabled(void);
2543 
2544 /*
2545  * Test if discarding of memory in ram blocks is required to work reliably.
2546  */
2547 bool ram_block_discard_is_required(void);
2548 
2549 #endif
2550 
2551 #endif
2552