xref: /openbmc/qemu/system/physmem.c (revision 7bd98c65e0ef871c7fe66e42a8c0b01784da1828)
1 /*
2  * RAM allocation and memory access
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "exec/page-vary.h"
22 #include "qapi/error.h"
23 
24 #include "qemu/cutils.h"
25 #include "qemu/cacheflush.h"
26 #include "qemu/hbitmap.h"
27 #include "qemu/madvise.h"
28 #include "qemu/lockable.h"
29 
30 #ifdef CONFIG_TCG
31 #include "accel/tcg/cpu-ops.h"
32 #include "accel/tcg/iommu.h"
33 #endif /* CONFIG_TCG */
34 
35 #include "exec/cputlb.h"
36 #include "exec/page-protection.h"
37 #include "exec/target_page.h"
38 #include "exec/translation-block.h"
39 #include "hw/qdev-core.h"
40 #include "hw/qdev-properties.h"
41 #include "hw/boards.h"
42 #include "system/xen.h"
43 #include "system/kvm.h"
44 #include "system/tcg.h"
45 #include "system/qtest.h"
46 #include "qemu/timer.h"
47 #include "qemu/config-file.h"
48 #include "qemu/error-report.h"
49 #include "qemu/qemu-print.h"
50 #include "qemu/log.h"
51 #include "qemu/memalign.h"
52 #include "qemu/memfd.h"
53 #include "system/memory.h"
54 #include "system/ioport.h"
55 #include "system/dma.h"
56 #include "system/hostmem.h"
57 #include "system/hw_accel.h"
58 #include "system/xen-mapcache.h"
59 #include "trace.h"
60 
61 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
62 #include <linux/falloc.h>
63 #endif
64 
65 #include "qemu/rcu_queue.h"
66 #include "qemu/main-loop.h"
67 #include "system/replay.h"
68 
69 #include "system/ram_addr.h"
70 
71 #include "qemu/pmem.h"
72 
73 #include "qapi/qapi-types-migration.h"
74 #include "migration/blocker.h"
75 #include "migration/cpr.h"
76 #include "migration/options.h"
77 #include "migration/vmstate.h"
78 
79 #include "qemu/range.h"
80 #ifndef _WIN32
81 #include "qemu/mmap-alloc.h"
82 #endif
83 
84 #include "monitor/monitor.h"
85 
86 #ifdef CONFIG_LIBDAXCTL
87 #include <daxctl/libdaxctl.h>
88 #endif
89 
90 #include "memory-internal.h"
91 
92 //#define DEBUG_SUBPAGE
93 
94 /* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
95  * are protected by the ramlist lock.
96  */
97 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
98 
99 static MemoryRegion *system_memory;
100 static MemoryRegion *system_io;
101 
102 AddressSpace address_space_io;
103 AddressSpace address_space_memory;
104 
105 static MemoryRegion io_mem_unassigned;
106 
107 typedef struct PhysPageEntry PhysPageEntry;
108 
109 struct PhysPageEntry {
110     /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
111     uint32_t skip : 6;
112      /* index into phys_sections (!skip) or phys_map_nodes (skip) */
113     uint32_t ptr : 26;
114 };
115 
116 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
117 
118 /* Size of the L2 (and L3, etc) page tables.  */
119 #define ADDR_SPACE_BITS 64
120 
121 #define P_L2_BITS 9
122 #define P_L2_SIZE (1 << P_L2_BITS)
123 
124 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
125 
126 typedef PhysPageEntry Node[P_L2_SIZE];
127 
128 typedef struct PhysPageMap {
129     struct rcu_head rcu;
130 
131     unsigned sections_nb;
132     unsigned sections_nb_alloc;
133     unsigned nodes_nb;
134     unsigned nodes_nb_alloc;
135     Node *nodes;
136     MemoryRegionSection *sections;
137 } PhysPageMap;
138 
139 struct AddressSpaceDispatch {
140     MemoryRegionSection *mru_section;
141     /* This is a multi-level map on the physical address space.
142      * The bottom level has pointers to MemoryRegionSections.
143      */
144     PhysPageEntry phys_map;
145     PhysPageMap map;
146 };
147 
148 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
149 typedef struct subpage_t {
150     MemoryRegion iomem;
151     FlatView *fv;
152     hwaddr base;
153     uint16_t sub_section[];
154 } subpage_t;
155 
156 #define PHYS_SECTION_UNASSIGNED 0
157 
158 static void io_mem_init(void);
159 static void memory_map_init(void);
160 static void tcg_log_global_after_sync(MemoryListener *listener);
161 static void tcg_commit(MemoryListener *listener);
162 static bool ram_is_cpr_compatible(RAMBlock *rb);
163 
164 /**
165  * CPUAddressSpace: all the information a CPU needs about an AddressSpace
166  * @cpu: the CPU whose AddressSpace this is
167  * @as: the AddressSpace itself
168  * @tcg_as_listener: listener for tracking changes to the AddressSpace
169  */
170 typedef struct CPUAddressSpace {
171     CPUState *cpu;
172     AddressSpace *as;
173     MemoryListener tcg_as_listener;
174 } CPUAddressSpace;
175 
176 struct DirtyBitmapSnapshot {
177     ram_addr_t start;
178     ram_addr_t end;
179     unsigned long dirty[];
180 };
181 
phys_map_node_reserve(PhysPageMap * map,unsigned nodes)182 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
183 {
184     static unsigned alloc_hint = 16;
185     if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
186         map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
187         map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
188         alloc_hint = map->nodes_nb_alloc;
189     }
190 }
191 
phys_map_node_alloc(PhysPageMap * map,bool leaf)192 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
193 {
194     unsigned i;
195     uint32_t ret;
196     PhysPageEntry e;
197     PhysPageEntry *p;
198 
199     ret = map->nodes_nb++;
200     p = map->nodes[ret];
201     assert(ret != PHYS_MAP_NODE_NIL);
202     assert(ret != map->nodes_nb_alloc);
203 
204     e.skip = leaf ? 0 : 1;
205     e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
206     for (i = 0; i < P_L2_SIZE; ++i) {
207         memcpy(&p[i], &e, sizeof(e));
208     }
209     return ret;
210 }
211 
phys_page_set_level(PhysPageMap * map,PhysPageEntry * lp,hwaddr * index,uint64_t * nb,uint16_t leaf,int level)212 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
213                                 hwaddr *index, uint64_t *nb, uint16_t leaf,
214                                 int level)
215 {
216     PhysPageEntry *p;
217     hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
218 
219     if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
220         lp->ptr = phys_map_node_alloc(map, level == 0);
221     }
222     p = map->nodes[lp->ptr];
223     lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
224 
225     while (*nb && lp < &p[P_L2_SIZE]) {
226         if ((*index & (step - 1)) == 0 && *nb >= step) {
227             lp->skip = 0;
228             lp->ptr = leaf;
229             *index += step;
230             *nb -= step;
231         } else {
232             phys_page_set_level(map, lp, index, nb, leaf, level - 1);
233         }
234         ++lp;
235     }
236 }
237 
phys_page_set(AddressSpaceDispatch * d,hwaddr index,uint64_t nb,uint16_t leaf)238 static void phys_page_set(AddressSpaceDispatch *d,
239                           hwaddr index, uint64_t nb,
240                           uint16_t leaf)
241 {
242     /* Wildly overreserve - it doesn't matter much. */
243     phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
244 
245     phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
246 }
247 
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249  * and update our entry so we can skip it and go directly to the destination.
250  */
phys_page_compact(PhysPageEntry * lp,Node * nodes)251 static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
252 {
253     unsigned valid_ptr = P_L2_SIZE;
254     int valid = 0;
255     PhysPageEntry *p;
256     int i;
257 
258     if (lp->ptr == PHYS_MAP_NODE_NIL) {
259         return;
260     }
261 
262     p = nodes[lp->ptr];
263     for (i = 0; i < P_L2_SIZE; i++) {
264         if (p[i].ptr == PHYS_MAP_NODE_NIL) {
265             continue;
266         }
267 
268         valid_ptr = i;
269         valid++;
270         if (p[i].skip) {
271             phys_page_compact(&p[i], nodes);
272         }
273     }
274 
275     /* We can only compress if there's only one child. */
276     if (valid != 1) {
277         return;
278     }
279 
280     assert(valid_ptr < P_L2_SIZE);
281 
282     /* Don't compress if it won't fit in the # of bits we have. */
283     if (P_L2_LEVELS >= (1 << 6) &&
284         lp->skip + p[valid_ptr].skip >= (1 << 6)) {
285         return;
286     }
287 
288     lp->ptr = p[valid_ptr].ptr;
289     if (!p[valid_ptr].skip) {
290         /* If our only child is a leaf, make this a leaf. */
291         /* By design, we should have made this node a leaf to begin with so we
292          * should never reach here.
293          * But since it's so simple to handle this, let's do it just in case we
294          * change this rule.
295          */
296         lp->skip = 0;
297     } else {
298         lp->skip += p[valid_ptr].skip;
299     }
300 }
301 
address_space_dispatch_compact(AddressSpaceDispatch * d)302 void address_space_dispatch_compact(AddressSpaceDispatch *d)
303 {
304     if (d->phys_map.skip) {
305         phys_page_compact(&d->phys_map, d->map.nodes);
306     }
307 }
308 
section_covers_addr(const MemoryRegionSection * section,hwaddr addr)309 static inline bool section_covers_addr(const MemoryRegionSection *section,
310                                        hwaddr addr)
311 {
312     /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
313      * the section must cover the entire address space.
314      */
315     return int128_gethi(section->size) ||
316            range_covers_byte(section->offset_within_address_space,
317                              int128_getlo(section->size), addr);
318 }
319 
phys_page_find(AddressSpaceDispatch * d,hwaddr addr)320 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
321 {
322     PhysPageEntry lp = d->phys_map, *p;
323     Node *nodes = d->map.nodes;
324     MemoryRegionSection *sections = d->map.sections;
325     hwaddr index = addr >> TARGET_PAGE_BITS;
326     int i;
327 
328     for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
329         if (lp.ptr == PHYS_MAP_NODE_NIL) {
330             return &sections[PHYS_SECTION_UNASSIGNED];
331         }
332         p = nodes[lp.ptr];
333         lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
334     }
335 
336     if (section_covers_addr(&sections[lp.ptr], addr)) {
337         return &sections[lp.ptr];
338     } else {
339         return &sections[PHYS_SECTION_UNASSIGNED];
340     }
341 }
342 
343 /* Called from RCU critical section */
address_space_lookup_region(AddressSpaceDispatch * d,hwaddr addr,bool resolve_subpage)344 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
345                                                         hwaddr addr,
346                                                         bool resolve_subpage)
347 {
348     MemoryRegionSection *section = qatomic_read(&d->mru_section);
349     subpage_t *subpage;
350 
351     if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
352         !section_covers_addr(section, addr)) {
353         section = phys_page_find(d, addr);
354         qatomic_set(&d->mru_section, section);
355     }
356     if (resolve_subpage && section->mr->subpage) {
357         subpage = container_of(section->mr, subpage_t, iomem);
358         section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
359     }
360     return section;
361 }
362 
363 /* Called from RCU critical section */
364 static MemoryRegionSection *
address_space_translate_internal(AddressSpaceDispatch * d,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool resolve_subpage)365 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
366                                  hwaddr *plen, bool resolve_subpage)
367 {
368     MemoryRegionSection *section;
369     MemoryRegion *mr;
370     Int128 diff;
371 
372     section = address_space_lookup_region(d, addr, resolve_subpage);
373     /* Compute offset within MemoryRegionSection */
374     addr -= section->offset_within_address_space;
375 
376     /* Compute offset within MemoryRegion */
377     *xlat = addr + section->offset_within_region;
378 
379     mr = section->mr;
380 
381     /* MMIO registers can be expected to perform full-width accesses based only
382      * on their address, without considering adjacent registers that could
383      * decode to completely different MemoryRegions.  When such registers
384      * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
385      * regions overlap wildly.  For this reason we cannot clamp the accesses
386      * here.
387      *
388      * If the length is small (as is the case for address_space_ldl/stl),
389      * everything works fine.  If the incoming length is large, however,
390      * the caller really has to do the clamping through memory_access_size.
391      */
392     if (memory_region_is_ram(mr)) {
393         diff = int128_sub(section->size, int128_make64(addr));
394         *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
395     }
396     return section;
397 }
398 
399 /**
400  * address_space_translate_iommu - translate an address through an IOMMU
401  * memory region and then through the target address space.
402  *
403  * @iommu_mr: the IOMMU memory region that we start the translation from
404  * @addr: the address to be translated through the MMU
405  * @xlat: the translated address offset within the destination memory region.
406  *        It cannot be %NULL.
407  * @plen_out: valid read/write length of the translated address. It
408  *            cannot be %NULL.
409  * @page_mask_out: page mask for the translated address. This
410  *            should only be meaningful for IOMMU translated
411  *            addresses, since there may be huge pages that this bit
412  *            would tell. It can be %NULL if we don't care about it.
413  * @is_write: whether the translation operation is for write
414  * @is_mmio: whether this can be MMIO, set true if it can
415  * @target_as: the address space targeted by the IOMMU
416  * @attrs: transaction attributes
417  *
418  * This function is called from RCU critical section.  It is the common
419  * part of flatview_do_translate and address_space_translate_cached.
420  */
address_space_translate_iommu(IOMMUMemoryRegion * iommu_mr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)421 static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
422                                                          hwaddr *xlat,
423                                                          hwaddr *plen_out,
424                                                          hwaddr *page_mask_out,
425                                                          bool is_write,
426                                                          bool is_mmio,
427                                                          AddressSpace **target_as,
428                                                          MemTxAttrs attrs)
429 {
430     MemoryRegionSection *section;
431     hwaddr page_mask = (hwaddr)-1;
432 
433     do {
434         hwaddr addr = *xlat;
435         IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
436         int iommu_idx = 0;
437         IOMMUTLBEntry iotlb;
438 
439         if (imrc->attrs_to_index) {
440             iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
441         }
442 
443         iotlb = imrc->translate(iommu_mr, addr, is_write ?
444                                 IOMMU_WO : IOMMU_RO, iommu_idx);
445 
446         if (!(iotlb.perm & (1 << is_write))) {
447             goto unassigned;
448         }
449 
450         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
451                 | (addr & iotlb.addr_mask));
452         page_mask &= iotlb.addr_mask;
453         *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
454         *target_as = iotlb.target_as;
455 
456         section = address_space_translate_internal(
457                 address_space_to_dispatch(iotlb.target_as), addr, xlat,
458                 plen_out, is_mmio);
459 
460         iommu_mr = memory_region_get_iommu(section->mr);
461     } while (unlikely(iommu_mr));
462 
463     if (page_mask_out) {
464         *page_mask_out = page_mask;
465     }
466     return *section;
467 
468 unassigned:
469     return (MemoryRegionSection) { .mr = &io_mem_unassigned };
470 }
471 
472 /**
473  * flatview_do_translate - translate an address in FlatView
474  *
475  * @fv: the flat view that we want to translate on
476  * @addr: the address to be translated in above address space
477  * @xlat: the translated address offset within memory region. It
478  *        cannot be @NULL.
479  * @plen_out: valid read/write length of the translated address. It
480  *            can be @NULL when we don't care about it.
481  * @page_mask_out: page mask for the translated address. This
482  *            should only be meaningful for IOMMU translated
483  *            addresses, since there may be huge pages that this bit
484  *            would tell. It can be @NULL if we don't care about it.
485  * @is_write: whether the translation operation is for write
486  * @is_mmio: whether this can be MMIO, set true if it can
487  * @target_as: the address space targeted by the IOMMU
488  * @attrs: memory transaction attributes
489  *
490  * This function is called from RCU critical section
491  */
flatview_do_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)492 static MemoryRegionSection flatview_do_translate(FlatView *fv,
493                                                  hwaddr addr,
494                                                  hwaddr *xlat,
495                                                  hwaddr *plen_out,
496                                                  hwaddr *page_mask_out,
497                                                  bool is_write,
498                                                  bool is_mmio,
499                                                  AddressSpace **target_as,
500                                                  MemTxAttrs attrs)
501 {
502     MemoryRegionSection *section;
503     IOMMUMemoryRegion *iommu_mr;
504     hwaddr plen = (hwaddr)(-1);
505 
506     if (!plen_out) {
507         plen_out = &plen;
508     }
509 
510     section = address_space_translate_internal(
511             flatview_to_dispatch(fv), addr, xlat,
512             plen_out, is_mmio);
513 
514     iommu_mr = memory_region_get_iommu(section->mr);
515     if (unlikely(iommu_mr)) {
516         return address_space_translate_iommu(iommu_mr, xlat,
517                                              plen_out, page_mask_out,
518                                              is_write, is_mmio,
519                                              target_as, attrs);
520     }
521     if (page_mask_out) {
522         /* Not behind an IOMMU, use default page size. */
523         *page_mask_out = ~TARGET_PAGE_MASK;
524     }
525 
526     return *section;
527 }
528 
529 /* Called from RCU critical section */
address_space_get_iotlb_entry(AddressSpace * as,hwaddr addr,bool is_write,MemTxAttrs attrs)530 IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
531                                             bool is_write, MemTxAttrs attrs)
532 {
533     MemoryRegionSection section;
534     hwaddr xlat, page_mask;
535 
536     /*
537      * This can never be MMIO, and we don't really care about plen,
538      * but page mask.
539      */
540     section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
541                                     NULL, &page_mask, is_write, false, &as,
542                                     attrs);
543 
544     /* Illegal translation */
545     if (section.mr == &io_mem_unassigned) {
546         goto iotlb_fail;
547     }
548 
549     /* Convert memory region offset into address space offset */
550     xlat += section.offset_within_address_space -
551         section.offset_within_region;
552 
553     return (IOMMUTLBEntry) {
554         .target_as = as,
555         .iova = addr & ~page_mask,
556         .translated_addr = xlat & ~page_mask,
557         .addr_mask = page_mask,
558         /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
559         .perm = IOMMU_RW,
560     };
561 
562 iotlb_fail:
563     return (IOMMUTLBEntry) {0};
564 }
565 
566 /* Called from RCU critical section */
flatview_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)567 MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
568                                  hwaddr *plen, bool is_write,
569                                  MemTxAttrs attrs)
570 {
571     MemoryRegion *mr;
572     MemoryRegionSection section;
573     AddressSpace *as = NULL;
574 
575     /* This can be MMIO, so setup MMIO bit. */
576     section = flatview_do_translate(fv, addr, xlat, plen, NULL,
577                                     is_write, true, &as, attrs);
578     mr = section.mr;
579 
580     if (xen_enabled() && memory_access_is_direct(mr, is_write, attrs)) {
581         hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
582         *plen = MIN(page, *plen);
583     }
584 
585     return mr;
586 }
587 
588 #ifdef CONFIG_TCG
589 
590 typedef struct TCGIOMMUNotifier {
591     IOMMUNotifier n;
592     MemoryRegion *mr;
593     CPUState *cpu;
594     int iommu_idx;
595     bool active;
596 } TCGIOMMUNotifier;
597 
tcg_iommu_unmap_notify(IOMMUNotifier * n,IOMMUTLBEntry * iotlb)598 static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
599 {
600     TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
601 
602     if (!notifier->active) {
603         return;
604     }
605     tlb_flush(notifier->cpu);
606     notifier->active = false;
607     /* We leave the notifier struct on the list to avoid reallocating it later.
608      * Generally the number of IOMMUs a CPU deals with will be small.
609      * In any case we can't unregister the iommu notifier from a notify
610      * callback.
611      */
612 }
613 
tcg_register_iommu_notifier(CPUState * cpu,IOMMUMemoryRegion * iommu_mr,int iommu_idx)614 static void tcg_register_iommu_notifier(CPUState *cpu,
615                                         IOMMUMemoryRegion *iommu_mr,
616                                         int iommu_idx)
617 {
618     /* Make sure this CPU has an IOMMU notifier registered for this
619      * IOMMU/IOMMU index combination, so that we can flush its TLB
620      * when the IOMMU tells us the mappings we've cached have changed.
621      */
622     MemoryRegion *mr = MEMORY_REGION(iommu_mr);
623     TCGIOMMUNotifier *notifier = NULL;
624     int i;
625 
626     for (i = 0; i < cpu->iommu_notifiers->len; i++) {
627         notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
628         if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
629             break;
630         }
631     }
632     if (i == cpu->iommu_notifiers->len) {
633         /* Not found, add a new entry at the end of the array */
634         cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
635         notifier = g_new0(TCGIOMMUNotifier, 1);
636         g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
637 
638         notifier->mr = mr;
639         notifier->iommu_idx = iommu_idx;
640         notifier->cpu = cpu;
641         /* Rather than trying to register interest in the specific part
642          * of the iommu's address space that we've accessed and then
643          * expand it later as subsequent accesses touch more of it, we
644          * just register interest in the whole thing, on the assumption
645          * that iommu reconfiguration will be rare.
646          */
647         iommu_notifier_init(&notifier->n,
648                             tcg_iommu_unmap_notify,
649                             IOMMU_NOTIFIER_UNMAP,
650                             0,
651                             HWADDR_MAX,
652                             iommu_idx);
653         memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
654                                               &error_fatal);
655     }
656 
657     if (!notifier->active) {
658         notifier->active = true;
659     }
660 }
661 
tcg_iommu_free_notifier_list(CPUState * cpu)662 void tcg_iommu_free_notifier_list(CPUState *cpu)
663 {
664     /* Destroy the CPU's notifier list */
665     int i;
666     TCGIOMMUNotifier *notifier;
667 
668     for (i = 0; i < cpu->iommu_notifiers->len; i++) {
669         notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
670         memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
671         g_free(notifier);
672     }
673     g_array_free(cpu->iommu_notifiers, true);
674 }
675 
tcg_iommu_init_notifier_list(CPUState * cpu)676 void tcg_iommu_init_notifier_list(CPUState *cpu)
677 {
678     cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
679 }
680 
681 /* Called from RCU critical section */
682 MemoryRegionSection *
address_space_translate_for_iotlb(CPUState * cpu,int asidx,hwaddr orig_addr,hwaddr * xlat,hwaddr * plen,MemTxAttrs attrs,int * prot)683 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
684                                   hwaddr *xlat, hwaddr *plen,
685                                   MemTxAttrs attrs, int *prot)
686 {
687     MemoryRegionSection *section;
688     IOMMUMemoryRegion *iommu_mr;
689     IOMMUMemoryRegionClass *imrc;
690     IOMMUTLBEntry iotlb;
691     int iommu_idx;
692     hwaddr addr = orig_addr;
693     AddressSpaceDispatch *d = address_space_to_dispatch(cpu->cpu_ases[asidx].as);
694 
695     for (;;) {
696         section = address_space_translate_internal(d, addr, &addr, plen, false);
697 
698         iommu_mr = memory_region_get_iommu(section->mr);
699         if (!iommu_mr) {
700             break;
701         }
702 
703         imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
704 
705         iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
706         tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
707         /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
708          * doesn't short-cut its translation table walk.
709          */
710         iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
711         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
712                 | (addr & iotlb.addr_mask));
713         /* Update the caller's prot bits to remove permissions the IOMMU
714          * is giving us a failure response for. If we get down to no
715          * permissions left at all we can give up now.
716          */
717         if (!(iotlb.perm & IOMMU_RO)) {
718             *prot &= ~(PAGE_READ | PAGE_EXEC);
719         }
720         if (!(iotlb.perm & IOMMU_WO)) {
721             *prot &= ~PAGE_WRITE;
722         }
723 
724         if (!*prot) {
725             goto translate_fail;
726         }
727 
728         d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
729     }
730 
731     assert(!memory_region_is_iommu(section->mr));
732     *xlat = addr;
733     return section;
734 
735 translate_fail:
736     /*
737      * We should be given a page-aligned address -- certainly
738      * tlb_set_page_with_attrs() does so.  The page offset of xlat
739      * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
740      * The page portion of xlat will be logged by memory_region_access_valid()
741      * when this memory access is rejected, so use the original untranslated
742      * physical address.
743      */
744     assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
745     *xlat = orig_addr;
746     return &d->map.sections[PHYS_SECTION_UNASSIGNED];
747 }
748 
iotlb_to_section(CPUState * cpu,hwaddr index,MemTxAttrs attrs)749 MemoryRegionSection *iotlb_to_section(CPUState *cpu,
750                                       hwaddr index, MemTxAttrs attrs)
751 {
752     int asidx = cpu_asidx_from_attrs(cpu, attrs);
753     CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
754     AddressSpaceDispatch *d = address_space_to_dispatch(cpuas->as);
755     int section_index = index & ~TARGET_PAGE_MASK;
756     MemoryRegionSection *ret;
757 
758     assert(section_index < d->map.sections_nb);
759     ret = d->map.sections + section_index;
760     assert(ret->mr);
761     assert(ret->mr->ops);
762 
763     return ret;
764 }
765 
766 /* Called from RCU critical section */
memory_region_section_get_iotlb(CPUState * cpu,MemoryRegionSection * section)767 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
768                                        MemoryRegionSection *section)
769 {
770     AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
771     return section - d->map.sections;
772 }
773 
774 #endif /* CONFIG_TCG */
775 
cpu_address_space_init(CPUState * cpu,int asidx,const char * prefix,MemoryRegion * mr)776 void cpu_address_space_init(CPUState *cpu, int asidx,
777                             const char *prefix, MemoryRegion *mr)
778 {
779     CPUAddressSpace *newas;
780     AddressSpace *as = g_new0(AddressSpace, 1);
781     char *as_name;
782 
783     assert(mr);
784     as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
785     address_space_init(as, mr, as_name);
786     g_free(as_name);
787 
788     /* Target code should have set num_ases before calling us */
789     assert(asidx < cpu->num_ases);
790 
791     if (asidx == 0) {
792         /* address space 0 gets the convenience alias */
793         cpu->as = as;
794     }
795 
796     if (!cpu->cpu_ases) {
797         cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
798     }
799 
800     newas = &cpu->cpu_ases[asidx];
801     newas->cpu = cpu;
802     newas->as = as;
803     if (tcg_enabled()) {
804         newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
805         newas->tcg_as_listener.commit = tcg_commit;
806         newas->tcg_as_listener.name = "tcg";
807         memory_listener_register(&newas->tcg_as_listener, as);
808     }
809 }
810 
cpu_destroy_address_spaces(CPUState * cpu)811 void cpu_destroy_address_spaces(CPUState *cpu)
812 {
813     CPUAddressSpace *cpuas;
814     int asidx;
815 
816     assert(cpu->cpu_ases);
817 
818     /* convenience alias just points to some cpu_ases[n] */
819     cpu->as = NULL;
820 
821     for (asidx = 0; asidx < cpu->num_ases; asidx++) {
822         cpuas = &cpu->cpu_ases[asidx];
823         if (!cpuas->as) {
824             /* This index was never initialized; no deinit needed */
825             continue;
826         }
827         if (tcg_enabled()) {
828             memory_listener_unregister(&cpuas->tcg_as_listener);
829         }
830         g_clear_pointer(&cpuas->as, address_space_destroy_free);
831     }
832 
833     g_clear_pointer(&cpu->cpu_ases, g_free);
834 }
835 
cpu_get_address_space(CPUState * cpu,int asidx)836 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
837 {
838     /* Return the AddressSpace corresponding to the specified index */
839     return cpu->cpu_ases[asidx].as;
840 }
841 
842 /* Called from RCU critical section */
qemu_get_ram_block(ram_addr_t addr)843 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
844 {
845     RAMBlock *block;
846 
847     block = qatomic_rcu_read(&ram_list.mru_block);
848     if (block && addr - block->offset < block->max_length) {
849         return block;
850     }
851     RAMBLOCK_FOREACH(block) {
852         if (addr - block->offset < block->max_length) {
853             goto found;
854         }
855     }
856 
857     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
858     abort();
859 
860 found:
861     /* It is safe to write mru_block outside the BQL.  This
862      * is what happens:
863      *
864      *     mru_block = xxx
865      *     rcu_read_unlock()
866      *                                        xxx removed from list
867      *                  rcu_read_lock()
868      *                  read mru_block
869      *                                        mru_block = NULL;
870      *                                        call_rcu(reclaim_ramblock, xxx);
871      *                  rcu_read_unlock()
872      *
873      * qatomic_rcu_set is not needed here.  The block was already published
874      * when it was placed into the list.  Here we're just making an extra
875      * copy of the pointer.
876      */
877     ram_list.mru_block = block;
878     return block;
879 }
880 
tlb_reset_dirty_range_all(ram_addr_t start,ram_addr_t length)881 void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
882 {
883     CPUState *cpu;
884     ram_addr_t start1;
885     RAMBlock *block;
886     ram_addr_t end;
887 
888     assert(tcg_enabled());
889     end = TARGET_PAGE_ALIGN(start + length);
890     start &= TARGET_PAGE_MASK;
891 
892     RCU_READ_LOCK_GUARD();
893     block = qemu_get_ram_block(start);
894     assert(block == qemu_get_ram_block(end - 1));
895     start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
896     CPU_FOREACH(cpu) {
897         tlb_reset_dirty(cpu, start1, length);
898     }
899 }
900 
901 /* Note: start and end must be within the same ram block.  */
cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,ram_addr_t length,unsigned client)902 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
903                                               ram_addr_t length,
904                                               unsigned client)
905 {
906     DirtyMemoryBlocks *blocks;
907     unsigned long end, page, start_page;
908     bool dirty = false;
909     RAMBlock *ramblock;
910     uint64_t mr_offset, mr_size;
911 
912     if (length == 0) {
913         return false;
914     }
915 
916     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
917     start_page = start >> TARGET_PAGE_BITS;
918     page = start_page;
919 
920     WITH_RCU_READ_LOCK_GUARD() {
921         blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
922         ramblock = qemu_get_ram_block(start);
923         /* Range sanity check on the ramblock */
924         assert(start >= ramblock->offset &&
925                start + length <= ramblock->offset + ramblock->used_length);
926 
927         while (page < end) {
928             unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
929             unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
930             unsigned long num = MIN(end - page,
931                                     DIRTY_MEMORY_BLOCK_SIZE - offset);
932 
933             dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
934                                                   offset, num);
935             page += num;
936         }
937 
938         mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
939         mr_size = (end - start_page) << TARGET_PAGE_BITS;
940         memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
941     }
942 
943     if (dirty) {
944         cpu_physical_memory_dirty_bits_cleared(start, length);
945     }
946 
947     return dirty;
948 }
949 
cpu_physical_memory_snapshot_and_clear_dirty(MemoryRegion * mr,hwaddr offset,hwaddr length,unsigned client)950 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
951     (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
952 {
953     DirtyMemoryBlocks *blocks;
954     ram_addr_t start, first, last;
955     unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
956     DirtyBitmapSnapshot *snap;
957     unsigned long page, end, dest;
958 
959     start = memory_region_get_ram_addr(mr);
960     /* We know we're only called for RAM MemoryRegions */
961     assert(start != RAM_ADDR_INVALID);
962     start += offset;
963 
964     first = QEMU_ALIGN_DOWN(start, align);
965     last  = QEMU_ALIGN_UP(start + length, align);
966 
967     snap = g_malloc0(sizeof(*snap) +
968                      ((last - first) >> (TARGET_PAGE_BITS + 3)));
969     snap->start = first;
970     snap->end   = last;
971 
972     page = first >> TARGET_PAGE_BITS;
973     end  = last  >> TARGET_PAGE_BITS;
974     dest = 0;
975 
976     WITH_RCU_READ_LOCK_GUARD() {
977         blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
978 
979         while (page < end) {
980             unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
981             unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
982             unsigned long num = MIN(end - page,
983                                     DIRTY_MEMORY_BLOCK_SIZE - ofs);
984 
985             assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
986             assert(QEMU_IS_ALIGNED(num,    (1 << BITS_PER_LEVEL)));
987             ofs >>= BITS_PER_LEVEL;
988 
989             bitmap_copy_and_clear_atomic(snap->dirty + dest,
990                                          blocks->blocks[idx] + ofs,
991                                          num);
992             page += num;
993             dest += num >> BITS_PER_LEVEL;
994         }
995     }
996 
997     cpu_physical_memory_dirty_bits_cleared(start, length);
998 
999     memory_region_clear_dirty_bitmap(mr, offset, length);
1000 
1001     return snap;
1002 }
1003 
cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot * snap,ram_addr_t start,ram_addr_t length)1004 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1005                                             ram_addr_t start,
1006                                             ram_addr_t length)
1007 {
1008     unsigned long page, end;
1009 
1010     assert(start >= snap->start);
1011     assert(start + length <= snap->end);
1012 
1013     end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1014     page = (start - snap->start) >> TARGET_PAGE_BITS;
1015 
1016     while (page < end) {
1017         if (test_bit(page, snap->dirty)) {
1018             return true;
1019         }
1020         page++;
1021     }
1022     return false;
1023 }
1024 
1025 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
1026                             uint16_t section);
1027 static subpage_t *subpage_init(FlatView *fv, hwaddr base);
1028 
phys_section_add(PhysPageMap * map,MemoryRegionSection * section)1029 static uint16_t phys_section_add(PhysPageMap *map,
1030                                  MemoryRegionSection *section)
1031 {
1032     /* The physical section number is ORed with a page-aligned
1033      * pointer to produce the iotlb entries.  Thus it should
1034      * never overflow into the page-aligned value.
1035      */
1036     assert(map->sections_nb < TARGET_PAGE_SIZE);
1037 
1038     if (map->sections_nb == map->sections_nb_alloc) {
1039         map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1040         map->sections = g_renew(MemoryRegionSection, map->sections,
1041                                 map->sections_nb_alloc);
1042     }
1043     map->sections[map->sections_nb] = *section;
1044     memory_region_ref(section->mr);
1045     return map->sections_nb++;
1046 }
1047 
phys_section_destroy(MemoryRegion * mr)1048 static void phys_section_destroy(MemoryRegion *mr)
1049 {
1050     bool have_sub_page = mr->subpage;
1051 
1052     memory_region_unref(mr);
1053 
1054     if (have_sub_page) {
1055         subpage_t *subpage = container_of(mr, subpage_t, iomem);
1056         object_unref(OBJECT(&subpage->iomem));
1057         g_free(subpage);
1058     }
1059 }
1060 
phys_sections_free(PhysPageMap * map)1061 static void phys_sections_free(PhysPageMap *map)
1062 {
1063     while (map->sections_nb > 0) {
1064         MemoryRegionSection *section = &map->sections[--map->sections_nb];
1065         phys_section_destroy(section->mr);
1066     }
1067     g_free(map->sections);
1068     g_free(map->nodes);
1069 }
1070 
register_subpage(FlatView * fv,MemoryRegionSection * section)1071 static void register_subpage(FlatView *fv, MemoryRegionSection *section)
1072 {
1073     AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1074     subpage_t *subpage;
1075     hwaddr base = section->offset_within_address_space
1076         & TARGET_PAGE_MASK;
1077     MemoryRegionSection *existing = phys_page_find(d, base);
1078     MemoryRegionSection subsection = {
1079         .offset_within_address_space = base,
1080         .size = int128_make64(TARGET_PAGE_SIZE),
1081     };
1082     hwaddr start, end;
1083 
1084     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1085 
1086     if (!(existing->mr->subpage)) {
1087         subpage = subpage_init(fv, base);
1088         subsection.fv = fv;
1089         subsection.mr = &subpage->iomem;
1090         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1091                       phys_section_add(&d->map, &subsection));
1092     } else {
1093         subpage = container_of(existing->mr, subpage_t, iomem);
1094     }
1095     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1096     end = start + int128_get64(section->size) - 1;
1097     subpage_register(subpage, start, end,
1098                      phys_section_add(&d->map, section));
1099 }
1100 
1101 
register_multipage(FlatView * fv,MemoryRegionSection * section)1102 static void register_multipage(FlatView *fv,
1103                                MemoryRegionSection *section)
1104 {
1105     AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1106     hwaddr start_addr = section->offset_within_address_space;
1107     uint16_t section_index = phys_section_add(&d->map, section);
1108     uint64_t num_pages = int128_get64(int128_rshift(section->size,
1109                                                     TARGET_PAGE_BITS));
1110 
1111     assert(num_pages);
1112     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1113 }
1114 
1115 /*
1116  * The range in *section* may look like this:
1117  *
1118  *      |s|PPPPPPP|s|
1119  *
1120  * where s stands for subpage and P for page.
1121  */
flatview_add_to_dispatch(FlatView * fv,MemoryRegionSection * section)1122 void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
1123 {
1124     MemoryRegionSection remain = *section;
1125     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1126 
1127     /* register first subpage */
1128     if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1129         uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1130                         - remain.offset_within_address_space;
1131 
1132         MemoryRegionSection now = remain;
1133         now.size = int128_min(int128_make64(left), now.size);
1134         register_subpage(fv, &now);
1135         if (int128_eq(remain.size, now.size)) {
1136             return;
1137         }
1138         remain.size = int128_sub(remain.size, now.size);
1139         remain.offset_within_address_space += int128_get64(now.size);
1140         remain.offset_within_region += int128_get64(now.size);
1141     }
1142 
1143     /* register whole pages */
1144     if (int128_ge(remain.size, page_size)) {
1145         MemoryRegionSection now = remain;
1146         now.size = int128_and(now.size, int128_neg(page_size));
1147         register_multipage(fv, &now);
1148         if (int128_eq(remain.size, now.size)) {
1149             return;
1150         }
1151         remain.size = int128_sub(remain.size, now.size);
1152         remain.offset_within_address_space += int128_get64(now.size);
1153         remain.offset_within_region += int128_get64(now.size);
1154     }
1155 
1156     /* register last subpage */
1157     register_subpage(fv, &remain);
1158 }
1159 
qemu_flush_coalesced_mmio_buffer(void)1160 void qemu_flush_coalesced_mmio_buffer(void)
1161 {
1162     if (kvm_enabled())
1163         kvm_flush_coalesced_mmio_buffer();
1164 }
1165 
qemu_mutex_lock_ramlist(void)1166 void qemu_mutex_lock_ramlist(void)
1167 {
1168     qemu_mutex_lock(&ram_list.mutex);
1169 }
1170 
qemu_mutex_unlock_ramlist(void)1171 void qemu_mutex_unlock_ramlist(void)
1172 {
1173     qemu_mutex_unlock(&ram_list.mutex);
1174 }
1175 
ram_block_format(void)1176 GString *ram_block_format(void)
1177 {
1178     RAMBlock *block;
1179     char *psize;
1180     GString *buf = g_string_new("");
1181 
1182     RCU_READ_LOCK_GUARD();
1183     g_string_append_printf(buf, "%24s %8s  %18s %18s %18s %18s %3s\n",
1184                            "Block Name", "PSize", "Offset", "Used", "Total",
1185                            "HVA", "RO");
1186 
1187     RAMBLOCK_FOREACH(block) {
1188         psize = size_to_str(block->page_size);
1189         g_string_append_printf(buf, "%24s %8s  0x%016" PRIx64 " 0x%016" PRIx64
1190                                " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1191                                block->idstr, psize,
1192                                (uint64_t)block->offset,
1193                                (uint64_t)block->used_length,
1194                                (uint64_t)block->max_length,
1195                                (uint64_t)(uintptr_t)block->host,
1196                                block->mr->readonly ? "ro" : "rw");
1197 
1198         g_free(psize);
1199     }
1200 
1201     return buf;
1202 }
1203 
find_min_backend_pagesize(Object * obj,void * opaque)1204 static int find_min_backend_pagesize(Object *obj, void *opaque)
1205 {
1206     long *hpsize_min = opaque;
1207 
1208     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1209         HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1210         long hpsize = host_memory_backend_pagesize(backend);
1211 
1212         if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
1213             *hpsize_min = hpsize;
1214         }
1215     }
1216 
1217     return 0;
1218 }
1219 
find_max_backend_pagesize(Object * obj,void * opaque)1220 static int find_max_backend_pagesize(Object *obj, void *opaque)
1221 {
1222     long *hpsize_max = opaque;
1223 
1224     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1225         HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1226         long hpsize = host_memory_backend_pagesize(backend);
1227 
1228         if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1229             *hpsize_max = hpsize;
1230         }
1231     }
1232 
1233     return 0;
1234 }
1235 
1236 /*
1237  * TODO: We assume right now that all mapped host memory backends are
1238  * used as RAM, however some might be used for different purposes.
1239  */
qemu_minrampagesize(void)1240 long qemu_minrampagesize(void)
1241 {
1242     long hpsize = LONG_MAX;
1243     Object *memdev_root = object_resolve_path("/objects", NULL);
1244 
1245     object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
1246     return hpsize;
1247 }
1248 
qemu_maxrampagesize(void)1249 long qemu_maxrampagesize(void)
1250 {
1251     long pagesize = 0;
1252     Object *memdev_root = object_resolve_path("/objects", NULL);
1253 
1254     object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
1255     return pagesize;
1256 }
1257 
1258 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
get_file_size(int fd)1259 static int64_t get_file_size(int fd)
1260 {
1261     int64_t size;
1262 #if defined(__linux__)
1263     struct stat st;
1264 
1265     if (fstat(fd, &st) < 0) {
1266         return -errno;
1267     }
1268 
1269     /* Special handling for devdax character devices */
1270     if (S_ISCHR(st.st_mode)) {
1271         g_autofree char *subsystem_path = NULL;
1272         g_autofree char *subsystem = NULL;
1273 
1274         subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1275                                          major(st.st_rdev), minor(st.st_rdev));
1276         subsystem = g_file_read_link(subsystem_path, NULL);
1277 
1278         if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1279             g_autofree char *size_path = NULL;
1280             g_autofree char *size_str = NULL;
1281 
1282             size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1283                                     major(st.st_rdev), minor(st.st_rdev));
1284 
1285             if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1286                 return g_ascii_strtoll(size_str, NULL, 0);
1287             }
1288         }
1289     }
1290 #endif /* defined(__linux__) */
1291 
1292     /* st.st_size may be zero for special files yet lseek(2) works */
1293     size = lseek(fd, 0, SEEK_END);
1294     if (size < 0) {
1295         return -errno;
1296     }
1297     return size;
1298 }
1299 
get_file_align(int fd)1300 static int64_t get_file_align(int fd)
1301 {
1302     int64_t align = -1;
1303 #if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1304     struct stat st;
1305 
1306     if (fstat(fd, &st) < 0) {
1307         return -errno;
1308     }
1309 
1310     /* Special handling for devdax character devices */
1311     if (S_ISCHR(st.st_mode)) {
1312         g_autofree char *path = NULL;
1313         g_autofree char *rpath = NULL;
1314         struct daxctl_ctx *ctx;
1315         struct daxctl_region *region;
1316         int rc = 0;
1317 
1318         path = g_strdup_printf("/sys/dev/char/%d:%d",
1319                     major(st.st_rdev), minor(st.st_rdev));
1320         rpath = realpath(path, NULL);
1321         if (!rpath) {
1322             return -errno;
1323         }
1324 
1325         rc = daxctl_new(&ctx);
1326         if (rc) {
1327             return -1;
1328         }
1329 
1330         daxctl_region_foreach(ctx, region) {
1331             if (strstr(rpath, daxctl_region_get_path(region))) {
1332                 align = daxctl_region_get_align(region);
1333                 break;
1334             }
1335         }
1336         daxctl_unref(ctx);
1337     }
1338 #endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1339 
1340     return align;
1341 }
1342 
file_ram_open(const char * path,const char * region_name,bool readonly,bool * created)1343 static int file_ram_open(const char *path,
1344                          const char *region_name,
1345                          bool readonly,
1346                          bool *created)
1347 {
1348     char *filename;
1349     char *sanitized_name;
1350     char *c;
1351     int fd = -1;
1352 
1353     *created = false;
1354     for (;;) {
1355         fd = open(path, readonly ? O_RDONLY : O_RDWR);
1356         if (fd >= 0) {
1357             /*
1358              * open(O_RDONLY) won't fail with EISDIR. Check manually if we
1359              * opened a directory and fail similarly to how we fail ENOENT
1360              * in readonly mode. Note that mkstemp() would imply O_RDWR.
1361              */
1362             if (readonly) {
1363                 struct stat file_stat;
1364 
1365                 if (fstat(fd, &file_stat)) {
1366                     close(fd);
1367                     if (errno == EINTR) {
1368                         continue;
1369                     }
1370                     return -errno;
1371                 } else if (S_ISDIR(file_stat.st_mode)) {
1372                     close(fd);
1373                     return -EISDIR;
1374                 }
1375             }
1376             /* @path names an existing file, use it */
1377             break;
1378         }
1379         if (errno == ENOENT) {
1380             if (readonly) {
1381                 /* Refuse to create new, readonly files. */
1382                 return -ENOENT;
1383             }
1384             /* @path names a file that doesn't exist, create it */
1385             fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1386             if (fd >= 0) {
1387                 *created = true;
1388                 break;
1389             }
1390         } else if (errno == EISDIR) {
1391             /* @path names a directory, create a file there */
1392             /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1393             sanitized_name = g_strdup(region_name);
1394             for (c = sanitized_name; *c != '\0'; c++) {
1395                 if (*c == '/') {
1396                     *c = '_';
1397                 }
1398             }
1399 
1400             filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1401                                        sanitized_name);
1402             g_free(sanitized_name);
1403 
1404             fd = mkstemp(filename);
1405             if (fd >= 0) {
1406                 unlink(filename);
1407                 g_free(filename);
1408                 break;
1409             }
1410             g_free(filename);
1411         }
1412         if (errno != EEXIST && errno != EINTR) {
1413             return -errno;
1414         }
1415         /*
1416          * Try again on EINTR and EEXIST.  The latter happens when
1417          * something else creates the file between our two open().
1418          */
1419     }
1420 
1421     return fd;
1422 }
1423 
file_ram_alloc(RAMBlock * block,ram_addr_t memory,int fd,bool truncate,off_t offset,Error ** errp)1424 static void *file_ram_alloc(RAMBlock *block,
1425                             ram_addr_t memory,
1426                             int fd,
1427                             bool truncate,
1428                             off_t offset,
1429                             Error **errp)
1430 {
1431     uint32_t qemu_map_flags;
1432     void *area;
1433 
1434     block->page_size = qemu_fd_getpagesize(fd);
1435     if (block->mr->align % block->page_size) {
1436         error_setg(errp, "alignment 0x%" PRIx64
1437                    " must be multiples of page size 0x%zx",
1438                    block->mr->align, block->page_size);
1439         return NULL;
1440     } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1441         error_setg(errp, "alignment 0x%" PRIx64
1442                    " must be a power of two", block->mr->align);
1443         return NULL;
1444     } else if (offset % block->page_size) {
1445         error_setg(errp, "offset 0x%" PRIx64
1446                    " must be multiples of page size 0x%zx",
1447                    offset, block->page_size);
1448         return NULL;
1449     }
1450     block->mr->align = MAX(block->page_size, block->mr->align);
1451 #if defined(__s390x__)
1452     if (kvm_enabled()) {
1453         block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1454     }
1455 #endif
1456 
1457     if (memory < block->page_size) {
1458         error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1459                    "or larger than page size 0x%zx",
1460                    memory, block->page_size);
1461         return NULL;
1462     }
1463 
1464     memory = ROUND_UP(memory, block->page_size);
1465 
1466     /*
1467      * ftruncate is not supported by hugetlbfs in older
1468      * hosts, so don't bother bailing out on errors.
1469      * If anything goes wrong with it under other filesystems,
1470      * mmap will fail.
1471      *
1472      * Do not truncate the non-empty backend file to avoid corrupting
1473      * the existing data in the file. Disabling shrinking is not
1474      * enough. For example, the current vNVDIMM implementation stores
1475      * the guest NVDIMM labels at the end of the backend file. If the
1476      * backend file is later extended, QEMU will not be able to find
1477      * those labels. Therefore, extending the non-empty backend file
1478      * is disabled as well.
1479      */
1480     if (truncate && ftruncate(fd, offset + memory)) {
1481         perror("ftruncate");
1482     }
1483 
1484     qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
1485     qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1486     qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
1487     qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
1488     area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
1489     if (area == MAP_FAILED) {
1490         error_setg_errno(errp, errno,
1491                          "unable to map backing store for guest RAM");
1492         return NULL;
1493     }
1494 
1495     block->fd = fd;
1496     block->fd_offset = offset;
1497     return area;
1498 }
1499 #endif
1500 
1501 /* Allocate space within the ram_addr_t space that governs the
1502  * dirty bitmaps.
1503  * Called with the ramlist lock held.
1504  */
find_ram_offset(ram_addr_t size)1505 static ram_addr_t find_ram_offset(ram_addr_t size)
1506 {
1507     RAMBlock *block, *next_block;
1508     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1509 
1510     assert(size != 0); /* it would hand out same offset multiple times */
1511 
1512     if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1513         return 0;
1514     }
1515 
1516     RAMBLOCK_FOREACH(block) {
1517         ram_addr_t candidate, next = RAM_ADDR_MAX;
1518 
1519         /* Align blocks to start on a 'long' in the bitmap
1520          * which makes the bitmap sync'ing take the fast path.
1521          */
1522         candidate = block->offset + block->max_length;
1523         candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
1524 
1525         /* Search for the closest following block
1526          * and find the gap.
1527          */
1528         RAMBLOCK_FOREACH(next_block) {
1529             if (next_block->offset >= candidate) {
1530                 next = MIN(next, next_block->offset);
1531             }
1532         }
1533 
1534         /* If it fits remember our place and remember the size
1535          * of gap, but keep going so that we might find a smaller
1536          * gap to fill so avoiding fragmentation.
1537          */
1538         if (next - candidate >= size && next - candidate < mingap) {
1539             offset = candidate;
1540             mingap = next - candidate;
1541         }
1542 
1543         trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
1544     }
1545 
1546     if (offset == RAM_ADDR_MAX) {
1547         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1548                 (uint64_t)size);
1549         abort();
1550     }
1551 
1552     trace_find_ram_offset(size, offset);
1553 
1554     return offset;
1555 }
1556 
qemu_ram_setup_dump(void * addr,ram_addr_t size)1557 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1558 {
1559     int ret;
1560 
1561     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1562     if (!machine_dump_guest_core(current_machine)) {
1563         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1564         if (ret) {
1565             perror("qemu_madvise");
1566             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1567                             "but dump-guest-core=off specified\n");
1568         }
1569     }
1570 }
1571 
qemu_ram_get_idstr(RAMBlock * rb)1572 const char *qemu_ram_get_idstr(RAMBlock *rb)
1573 {
1574     return rb->idstr;
1575 }
1576 
qemu_ram_get_host_addr(RAMBlock * rb)1577 void *qemu_ram_get_host_addr(RAMBlock *rb)
1578 {
1579     return rb->host;
1580 }
1581 
qemu_ram_get_offset(RAMBlock * rb)1582 ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
1583 {
1584     return rb->offset;
1585 }
1586 
qemu_ram_get_fd_offset(RAMBlock * rb)1587 ram_addr_t qemu_ram_get_fd_offset(RAMBlock *rb)
1588 {
1589     return rb->fd_offset;
1590 }
1591 
qemu_ram_get_used_length(RAMBlock * rb)1592 ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
1593 {
1594     return rb->used_length;
1595 }
1596 
qemu_ram_get_max_length(RAMBlock * rb)1597 ram_addr_t qemu_ram_get_max_length(RAMBlock *rb)
1598 {
1599     return rb->max_length;
1600 }
1601 
qemu_ram_is_shared(RAMBlock * rb)1602 bool qemu_ram_is_shared(RAMBlock *rb)
1603 {
1604     return rb->flags & RAM_SHARED;
1605 }
1606 
qemu_ram_is_noreserve(RAMBlock * rb)1607 bool qemu_ram_is_noreserve(RAMBlock *rb)
1608 {
1609     return rb->flags & RAM_NORESERVE;
1610 }
1611 
1612 /* Note: Only set at the start of postcopy */
qemu_ram_is_uf_zeroable(RAMBlock * rb)1613 bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1614 {
1615     return rb->flags & RAM_UF_ZEROPAGE;
1616 }
1617 
qemu_ram_set_uf_zeroable(RAMBlock * rb)1618 void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1619 {
1620     rb->flags |= RAM_UF_ZEROPAGE;
1621 }
1622 
qemu_ram_is_migratable(RAMBlock * rb)1623 bool qemu_ram_is_migratable(RAMBlock *rb)
1624 {
1625     return rb->flags & RAM_MIGRATABLE;
1626 }
1627 
qemu_ram_set_migratable(RAMBlock * rb)1628 void qemu_ram_set_migratable(RAMBlock *rb)
1629 {
1630     rb->flags |= RAM_MIGRATABLE;
1631 }
1632 
qemu_ram_unset_migratable(RAMBlock * rb)1633 void qemu_ram_unset_migratable(RAMBlock *rb)
1634 {
1635     rb->flags &= ~RAM_MIGRATABLE;
1636 }
1637 
qemu_ram_is_named_file(RAMBlock * rb)1638 bool qemu_ram_is_named_file(RAMBlock *rb)
1639 {
1640     return rb->flags & RAM_NAMED_FILE;
1641 }
1642 
qemu_ram_get_fd(RAMBlock * rb)1643 int qemu_ram_get_fd(RAMBlock *rb)
1644 {
1645     return rb->fd;
1646 }
1647 
1648 /* Called with the BQL held.  */
qemu_ram_set_idstr(RAMBlock * new_block,const char * name,DeviceState * dev)1649 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1650 {
1651     RAMBlock *block;
1652 
1653     assert(new_block);
1654     assert(!new_block->idstr[0]);
1655 
1656     if (dev) {
1657         char *id = qdev_get_dev_path(dev);
1658         if (id) {
1659             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1660             g_free(id);
1661         }
1662     }
1663     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1664 
1665     RCU_READ_LOCK_GUARD();
1666     RAMBLOCK_FOREACH(block) {
1667         if (block != new_block &&
1668             !strcmp(block->idstr, new_block->idstr)) {
1669             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1670                     new_block->idstr);
1671             abort();
1672         }
1673     }
1674 }
1675 
1676 /* Called with the BQL held.  */
qemu_ram_unset_idstr(RAMBlock * block)1677 void qemu_ram_unset_idstr(RAMBlock *block)
1678 {
1679     /* FIXME: arch_init.c assumes that this is not called throughout
1680      * migration.  Ignore the problem since hot-unplug during migration
1681      * does not work anyway.
1682      */
1683     if (block) {
1684         memset(block->idstr, 0, sizeof(block->idstr));
1685     }
1686 }
1687 
cpr_name(MemoryRegion * mr)1688 static char *cpr_name(MemoryRegion *mr)
1689 {
1690     const char *mr_name = memory_region_name(mr);
1691     g_autofree char *id = mr->dev ? qdev_get_dev_path(mr->dev) : NULL;
1692 
1693     if (id) {
1694         return g_strdup_printf("%s/%s", id, mr_name);
1695     } else {
1696         return g_strdup(mr_name);
1697     }
1698 }
1699 
qemu_ram_pagesize(RAMBlock * rb)1700 size_t qemu_ram_pagesize(RAMBlock *rb)
1701 {
1702     return rb->page_size;
1703 }
1704 
1705 /* Returns the largest size of page in use */
qemu_ram_pagesize_largest(void)1706 size_t qemu_ram_pagesize_largest(void)
1707 {
1708     RAMBlock *block;
1709     size_t largest = 0;
1710 
1711     RAMBLOCK_FOREACH(block) {
1712         largest = MAX(largest, qemu_ram_pagesize(block));
1713     }
1714 
1715     return largest;
1716 }
1717 
memory_try_enable_merging(void * addr,size_t len)1718 static int memory_try_enable_merging(void *addr, size_t len)
1719 {
1720     if (!machine_mem_merge(current_machine)) {
1721         /* disabled by the user */
1722         return 0;
1723     }
1724 
1725     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1726 }
1727 
1728 /*
1729  * Resizing RAM while migrating can result in the migration being canceled.
1730  * Care has to be taken if the guest might have already detected the memory.
1731  *
1732  * As memory core doesn't know how is memory accessed, it is up to
1733  * resize callback to update device state and/or add assertions to detect
1734  * misuse, if necessary.
1735  */
qemu_ram_resize(RAMBlock * block,ram_addr_t newsize,Error ** errp)1736 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1737 {
1738     const ram_addr_t oldsize = block->used_length;
1739     const ram_addr_t unaligned_size = newsize;
1740 
1741     assert(block);
1742 
1743     newsize = TARGET_PAGE_ALIGN(newsize);
1744     newsize = REAL_HOST_PAGE_ALIGN(newsize);
1745 
1746     if (block->used_length == newsize) {
1747         /*
1748          * We don't have to resize the ram block (which only knows aligned
1749          * sizes), however, we have to notify if the unaligned size changed.
1750          */
1751         if (unaligned_size != memory_region_size(block->mr)) {
1752             memory_region_set_size(block->mr, unaligned_size);
1753             if (block->resized) {
1754                 block->resized(block->idstr, unaligned_size, block->host);
1755             }
1756         }
1757         return 0;
1758     }
1759 
1760     if (!(block->flags & RAM_RESIZEABLE)) {
1761         error_setg_errno(errp, EINVAL,
1762                          "Size mismatch: %s: 0x" RAM_ADDR_FMT
1763                          " != 0x" RAM_ADDR_FMT, block->idstr,
1764                          newsize, block->used_length);
1765         return -EINVAL;
1766     }
1767 
1768     if (block->max_length < newsize) {
1769         error_setg_errno(errp, EINVAL,
1770                          "Size too large: %s: 0x" RAM_ADDR_FMT
1771                          " > 0x" RAM_ADDR_FMT, block->idstr,
1772                          newsize, block->max_length);
1773         return -EINVAL;
1774     }
1775 
1776     /* Notify before modifying the ram block and touching the bitmaps. */
1777     if (block->host) {
1778         ram_block_notify_resize(block->host, oldsize, newsize);
1779     }
1780 
1781     cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1782     block->used_length = newsize;
1783     cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1784                                         DIRTY_CLIENTS_ALL);
1785     memory_region_set_size(block->mr, unaligned_size);
1786     if (block->resized) {
1787         block->resized(block->idstr, unaligned_size, block->host);
1788     }
1789     return 0;
1790 }
1791 
1792 /*
1793  * Trigger sync on the given ram block for range [start, start + length]
1794  * with the backing store if one is available.
1795  * Otherwise no-op.
1796  * @Note: this is supposed to be a synchronous op.
1797  */
qemu_ram_msync(RAMBlock * block,ram_addr_t start,ram_addr_t length)1798 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
1799 {
1800     /* The requested range should fit in within the block range */
1801     g_assert((start + length) <= block->used_length);
1802 
1803 #ifdef CONFIG_LIBPMEM
1804     /* The lack of support for pmem should not block the sync */
1805     if (ramblock_is_pmem(block)) {
1806         void *addr = ramblock_ptr(block, start);
1807         pmem_persist(addr, length);
1808         return;
1809     }
1810 #endif
1811     if (block->fd >= 0) {
1812         /**
1813          * Case there is no support for PMEM or the memory has not been
1814          * specified as persistent (or is not one) - use the msync.
1815          * Less optimal but still achieves the same goal
1816          */
1817         void *addr = ramblock_ptr(block, start);
1818         if (qemu_msync(addr, length, block->fd)) {
1819             warn_report("%s: failed to sync memory range: start: "
1820                     RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
1821                     __func__, start, length);
1822         }
1823     }
1824 }
1825 
1826 /* Called with ram_list.mutex held */
dirty_memory_extend(ram_addr_t new_ram_size)1827 static void dirty_memory_extend(ram_addr_t new_ram_size)
1828 {
1829     unsigned int old_num_blocks = ram_list.num_dirty_blocks;
1830     unsigned int new_num_blocks = DIV_ROUND_UP(new_ram_size,
1831                                                DIRTY_MEMORY_BLOCK_SIZE);
1832     int i;
1833 
1834     /* Only need to extend if block count increased */
1835     if (new_num_blocks <= old_num_blocks) {
1836         return;
1837     }
1838 
1839     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1840         DirtyMemoryBlocks *old_blocks;
1841         DirtyMemoryBlocks *new_blocks;
1842         int j;
1843 
1844         old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
1845         new_blocks = g_malloc(sizeof(*new_blocks) +
1846                               sizeof(new_blocks->blocks[0]) * new_num_blocks);
1847 
1848         if (old_num_blocks) {
1849             memcpy(new_blocks->blocks, old_blocks->blocks,
1850                    old_num_blocks * sizeof(old_blocks->blocks[0]));
1851         }
1852 
1853         for (j = old_num_blocks; j < new_num_blocks; j++) {
1854             new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1855         }
1856 
1857         qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1858 
1859         if (old_blocks) {
1860             g_free_rcu(old_blocks, rcu);
1861         }
1862     }
1863 
1864     ram_list.num_dirty_blocks = new_num_blocks;
1865 }
1866 
ram_block_add(RAMBlock * new_block,Error ** errp)1867 static void ram_block_add(RAMBlock *new_block, Error **errp)
1868 {
1869     const bool noreserve = qemu_ram_is_noreserve(new_block);
1870     const bool shared = qemu_ram_is_shared(new_block);
1871     RAMBlock *block;
1872     RAMBlock *last_block = NULL;
1873     bool free_on_error = false;
1874     ram_addr_t ram_size;
1875     Error *err = NULL;
1876 
1877     qemu_mutex_lock_ramlist();
1878     new_block->offset = find_ram_offset(new_block->max_length);
1879 
1880     if (!new_block->host) {
1881         if (xen_enabled()) {
1882             xen_ram_alloc(new_block->offset, new_block->max_length,
1883                           new_block->mr, &err);
1884             if (err) {
1885                 error_propagate(errp, err);
1886                 qemu_mutex_unlock_ramlist();
1887                 return;
1888             }
1889         } else {
1890             new_block->host = qemu_anon_ram_alloc(new_block->max_length,
1891                                                   &new_block->mr->align,
1892                                                   shared, noreserve);
1893             if (!new_block->host) {
1894                 error_setg_errno(errp, errno,
1895                                  "cannot set up guest memory '%s'",
1896                                  memory_region_name(new_block->mr));
1897                 qemu_mutex_unlock_ramlist();
1898                 return;
1899             }
1900             memory_try_enable_merging(new_block->host, new_block->max_length);
1901             free_on_error = true;
1902         }
1903     }
1904 
1905     if (new_block->flags & RAM_GUEST_MEMFD) {
1906         int ret;
1907 
1908         if (!kvm_enabled()) {
1909             error_setg(errp, "cannot set up private guest memory for %s: KVM required",
1910                        object_get_typename(OBJECT(current_machine->cgs)));
1911             goto out_free;
1912         }
1913         assert(new_block->guest_memfd < 0);
1914 
1915         ret = ram_block_coordinated_discard_require(true);
1916         if (ret < 0) {
1917             error_setg_errno(errp, -ret,
1918                              "cannot set up private guest memory: discard currently blocked");
1919             error_append_hint(errp, "Are you using assigned devices?\n");
1920             goto out_free;
1921         }
1922 
1923         new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length,
1924                                                         0, errp);
1925         if (new_block->guest_memfd < 0) {
1926             qemu_mutex_unlock_ramlist();
1927             goto out_free;
1928         }
1929 
1930         /*
1931          * The attribute bitmap of the RamBlockAttributes is default to
1932          * discarded, which mimics the behavior of kvm_set_phys_mem() when it
1933          * calls kvm_set_memory_attributes_private(). This leads to a brief
1934          * period of inconsistency between the creation of the RAMBlock and its
1935          * mapping into the physical address space. However, this is not
1936          * problematic, as no users rely on the attribute status to perform
1937          * any actions during this interval.
1938          */
1939         new_block->attributes = ram_block_attributes_create(new_block);
1940         if (!new_block->attributes) {
1941             error_setg(errp, "Failed to create ram block attribute");
1942             close(new_block->guest_memfd);
1943             ram_block_coordinated_discard_require(false);
1944             qemu_mutex_unlock_ramlist();
1945             goto out_free;
1946         }
1947 
1948         /*
1949          * Add a specific guest_memfd blocker if a generic one would not be
1950          * added by ram_block_add_cpr_blocker.
1951          */
1952         if (ram_is_cpr_compatible(new_block)) {
1953             error_setg(&new_block->cpr_blocker,
1954                        "Memory region %s uses guest_memfd, "
1955                        "which is not supported with CPR.",
1956                        memory_region_name(new_block->mr));
1957             migrate_add_blocker_modes(&new_block->cpr_blocker, errp,
1958                                       MIG_MODE_CPR_TRANSFER, -1);
1959         }
1960     }
1961 
1962     ram_size = (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS;
1963     dirty_memory_extend(ram_size);
1964     /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
1965      * QLIST (which has an RCU-friendly variant) does not have insertion at
1966      * tail, so save the last element in last_block.
1967      */
1968     RAMBLOCK_FOREACH(block) {
1969         last_block = block;
1970         if (block->max_length < new_block->max_length) {
1971             break;
1972         }
1973     }
1974     if (block) {
1975         QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1976     } else if (last_block) {
1977         QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1978     } else { /* list is empty */
1979         QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1980     }
1981     ram_list.mru_block = NULL;
1982 
1983     /* Write list before version */
1984     smp_wmb();
1985     ram_list.version++;
1986     qemu_mutex_unlock_ramlist();
1987 
1988     cpu_physical_memory_set_dirty_range(new_block->offset,
1989                                         new_block->used_length,
1990                                         DIRTY_CLIENTS_ALL);
1991 
1992     if (new_block->host) {
1993         qemu_ram_setup_dump(new_block->host, new_block->max_length);
1994         qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1995         /*
1996          * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
1997          * Configure it unless the machine is a qtest server, in which case
1998          * KVM is not used and it may be forked (eg for fuzzing purposes).
1999          */
2000         if (!qtest_enabled()) {
2001             qemu_madvise(new_block->host, new_block->max_length,
2002                          QEMU_MADV_DONTFORK);
2003         }
2004         ram_block_notify_add(new_block->host, new_block->used_length,
2005                              new_block->max_length);
2006     }
2007     return;
2008 
2009 out_free:
2010     if (free_on_error) {
2011         qemu_anon_ram_free(new_block->host, new_block->max_length);
2012         new_block->host = NULL;
2013     }
2014 }
2015 
2016 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
qemu_ram_alloc_from_fd(ram_addr_t size,ram_addr_t max_size,qemu_ram_resize_cb resized,MemoryRegion * mr,uint32_t ram_flags,int fd,off_t offset,bool grow,Error ** errp)2017 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, ram_addr_t max_size,
2018                                  qemu_ram_resize_cb resized, MemoryRegion *mr,
2019                                  uint32_t ram_flags, int fd, off_t offset,
2020                                  bool grow,
2021                                  Error **errp)
2022 {
2023     ERRP_GUARD();
2024     RAMBlock *new_block;
2025     Error *local_err = NULL;
2026     int64_t file_size, file_align, share_flags;
2027 
2028     share_flags = ram_flags & (RAM_PRIVATE | RAM_SHARED);
2029     assert(share_flags != (RAM_SHARED | RAM_PRIVATE));
2030     ram_flags &= ~RAM_PRIVATE;
2031 
2032     /* Just support these ram flags by now. */
2033     assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE |
2034                           RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY |
2035                           RAM_READONLY_FD | RAM_GUEST_MEMFD |
2036                           RAM_RESIZEABLE)) == 0);
2037     assert(max_size >= size);
2038 
2039     if (xen_enabled()) {
2040         error_setg(errp, "-mem-path not supported with Xen");
2041         return NULL;
2042     }
2043 
2044     if (kvm_enabled() && !kvm_has_sync_mmu()) {
2045         error_setg(errp,
2046                    "host lacks kvm mmu notifiers, -mem-path unsupported");
2047         return NULL;
2048     }
2049 
2050     size = TARGET_PAGE_ALIGN(size);
2051     size = REAL_HOST_PAGE_ALIGN(size);
2052     max_size = TARGET_PAGE_ALIGN(max_size);
2053     max_size = REAL_HOST_PAGE_ALIGN(max_size);
2054 
2055     file_size = get_file_size(fd);
2056     if (file_size && file_size < offset + max_size && !grow) {
2057         error_setg(errp, "%s backing store size 0x%" PRIx64
2058                    " is too small for 'size' option 0x" RAM_ADDR_FMT
2059                    " plus 'offset' option 0x%" PRIx64,
2060                    memory_region_name(mr), file_size, max_size,
2061                    (uint64_t)offset);
2062         return NULL;
2063     }
2064 
2065     file_align = get_file_align(fd);
2066     if (file_align > 0 && file_align > mr->align) {
2067         error_setg(errp, "backing store align 0x%" PRIx64
2068                    " is larger than 'align' option 0x%" PRIx64,
2069                    file_align, mr->align);
2070         return NULL;
2071     }
2072 
2073     new_block = g_malloc0(sizeof(*new_block));
2074     new_block->mr = mr;
2075     new_block->used_length = size;
2076     new_block->max_length = max_size;
2077     new_block->resized = resized;
2078     new_block->flags = ram_flags;
2079     new_block->guest_memfd = -1;
2080     new_block->host = file_ram_alloc(new_block, max_size, fd,
2081                                      file_size < offset + max_size,
2082                                      offset, errp);
2083     if (!new_block->host) {
2084         g_free(new_block);
2085         return NULL;
2086     }
2087 
2088     ram_block_add(new_block, &local_err);
2089     if (local_err) {
2090         g_free(new_block);
2091         error_propagate(errp, local_err);
2092         return NULL;
2093     }
2094     return new_block;
2095 
2096 }
2097 
2098 
qemu_ram_alloc_from_file(ram_addr_t size,MemoryRegion * mr,uint32_t ram_flags,const char * mem_path,off_t offset,Error ** errp)2099 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
2100                                    uint32_t ram_flags, const char *mem_path,
2101                                    off_t offset, Error **errp)
2102 {
2103     int fd;
2104     bool created;
2105     RAMBlock *block;
2106 
2107     fd = file_ram_open(mem_path, memory_region_name(mr),
2108                        !!(ram_flags & RAM_READONLY_FD), &created);
2109     if (fd < 0) {
2110         error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
2111                          mem_path);
2112         if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
2113             fd == -EACCES) {
2114             /*
2115              * If we can open the file R/O (note: will never create a new file)
2116              * and we are dealing with a private mapping, there are still ways
2117              * to consume such files and get RAM instead of ROM.
2118              */
2119             fd = file_ram_open(mem_path, memory_region_name(mr), true,
2120                                &created);
2121             if (fd < 0) {
2122                 return NULL;
2123             }
2124             assert(!created);
2125             close(fd);
2126             error_append_hint(errp, "Consider opening the backing store"
2127                 " read-only but still creating writable RAM using"
2128                 " '-object memory-backend-file,readonly=on,rom=off...'"
2129                 " (see \"VM templating\" documentation)\n");
2130         }
2131         return NULL;
2132     }
2133 
2134     block = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, offset,
2135                                    false, errp);
2136     if (!block) {
2137         if (created) {
2138             unlink(mem_path);
2139         }
2140         close(fd);
2141         return NULL;
2142     }
2143 
2144     return block;
2145 }
2146 #endif
2147 
2148 #ifdef CONFIG_POSIX
2149 /*
2150  * Create MAP_SHARED RAMBlocks by mmap'ing a file descriptor, so it can be
2151  * shared with another process if CPR is being used.  Use memfd if available
2152  * because it has no size limits, else use POSIX shm.
2153  */
qemu_ram_get_shared_fd(const char * name,bool * reused,Error ** errp)2154 static int qemu_ram_get_shared_fd(const char *name, bool *reused, Error **errp)
2155 {
2156     int fd = cpr_find_fd(name, 0);
2157 
2158     if (fd >= 0) {
2159         *reused = true;
2160         return fd;
2161     }
2162 
2163     if (qemu_memfd_check(0)) {
2164         fd = qemu_memfd_create(name, 0, 0, 0, 0, errp);
2165     } else {
2166         fd = qemu_shm_alloc(0, errp);
2167     }
2168 
2169     if (fd >= 0) {
2170         cpr_save_fd(name, 0, fd);
2171     }
2172     *reused = false;
2173     return fd;
2174 }
2175 #endif
2176 
2177 static
qemu_ram_alloc_internal(ram_addr_t size,ram_addr_t max_size,qemu_ram_resize_cb resized,void * host,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2178 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2179                                   qemu_ram_resize_cb resized,
2180                                   void *host, uint32_t ram_flags,
2181                                   MemoryRegion *mr, Error **errp)
2182 {
2183     RAMBlock *new_block;
2184     Error *local_err = NULL;
2185     int align, share_flags;
2186 
2187     share_flags = ram_flags & (RAM_PRIVATE | RAM_SHARED);
2188     assert(share_flags != (RAM_SHARED | RAM_PRIVATE));
2189     ram_flags &= ~RAM_PRIVATE;
2190 
2191     assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC |
2192                           RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
2193     assert(!host ^ (ram_flags & RAM_PREALLOC));
2194     assert(max_size >= size);
2195 
2196     /* ignore RAM_SHARED for Windows and emscripten*/
2197 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
2198     if (!host) {
2199         if (!share_flags && current_machine->aux_ram_share) {
2200             ram_flags |= RAM_SHARED;
2201         }
2202         if (ram_flags & RAM_SHARED) {
2203             bool reused;
2204             g_autofree char *name = cpr_name(mr);
2205             int fd = qemu_ram_get_shared_fd(name, &reused, errp);
2206 
2207             if (fd < 0) {
2208                 return NULL;
2209             }
2210 
2211             /* Use same alignment as qemu_anon_ram_alloc */
2212             mr->align = QEMU_VMALLOC_ALIGN;
2213 
2214             /*
2215              * This can fail if the shm mount size is too small, or alloc from
2216              * fd is not supported, but previous QEMU versions that called
2217              * qemu_anon_ram_alloc for anonymous shared memory could have
2218              * succeeded.  Quietly fail and fall back.
2219              *
2220              * After cpr-transfer, new QEMU could create a memory region
2221              * with a larger max size than old, so pass reused to grow the
2222              * region if necessary.  The extra space will be usable after a
2223              * guest reset.
2224              */
2225             new_block = qemu_ram_alloc_from_fd(size, max_size, resized, mr,
2226                                                ram_flags, fd, 0, reused, NULL);
2227             if (new_block) {
2228                 trace_qemu_ram_alloc_shared(name, new_block->used_length,
2229                                             new_block->max_length, fd,
2230                                             new_block->host);
2231                 return new_block;
2232             }
2233 
2234             cpr_delete_fd(name, 0);
2235             close(fd);
2236             /* fall back to anon allocation */
2237         }
2238     }
2239 #endif
2240 
2241     align = qemu_real_host_page_size();
2242     align = MAX(align, TARGET_PAGE_SIZE);
2243     size = ROUND_UP(size, align);
2244     max_size = ROUND_UP(max_size, align);
2245 
2246     new_block = g_malloc0(sizeof(*new_block));
2247     new_block->mr = mr;
2248     new_block->resized = resized;
2249     new_block->used_length = size;
2250     new_block->max_length = max_size;
2251     new_block->fd = -1;
2252     new_block->guest_memfd = -1;
2253     new_block->page_size = qemu_real_host_page_size();
2254     new_block->host = host;
2255     new_block->flags = ram_flags;
2256     ram_block_add(new_block, &local_err);
2257     if (local_err) {
2258         g_free(new_block);
2259         error_propagate(errp, local_err);
2260         return NULL;
2261     }
2262     return new_block;
2263 }
2264 
qemu_ram_alloc_from_ptr(ram_addr_t size,void * host,MemoryRegion * mr,Error ** errp)2265 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
2266                                    MemoryRegion *mr, Error **errp)
2267 {
2268     return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2269                                    errp);
2270 }
2271 
qemu_ram_alloc(ram_addr_t size,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2272 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
2273                          MemoryRegion *mr, Error **errp)
2274 {
2275     assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE | RAM_GUEST_MEMFD |
2276                           RAM_PRIVATE)) == 0);
2277     return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
2278 }
2279 
qemu_ram_alloc_resizeable(ram_addr_t size,ram_addr_t maxsz,qemu_ram_resize_cb resized,MemoryRegion * mr,Error ** errp)2280 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
2281                                     qemu_ram_resize_cb resized,
2282                                     MemoryRegion *mr, Error **errp)
2283 {
2284     return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2285                                    RAM_RESIZEABLE, mr, errp);
2286 }
2287 
reclaim_ramblock(RAMBlock * block)2288 static void reclaim_ramblock(RAMBlock *block)
2289 {
2290     if (block->flags & RAM_PREALLOC) {
2291         ;
2292     } else if (xen_enabled()) {
2293         xen_invalidate_map_cache_entry(block->host);
2294 #if !defined(_WIN32) && !defined(EMSCRIPTEN)
2295     } else if (block->fd >= 0) {
2296         qemu_ram_munmap(block->fd, block->host, block->max_length);
2297         close(block->fd);
2298 #endif
2299     } else {
2300         qemu_anon_ram_free(block->host, block->max_length);
2301     }
2302 
2303     if (block->guest_memfd >= 0) {
2304         ram_block_attributes_destroy(block->attributes);
2305         close(block->guest_memfd);
2306         ram_block_coordinated_discard_require(false);
2307     }
2308 
2309     g_free(block);
2310 }
2311 
qemu_ram_free(RAMBlock * block)2312 void qemu_ram_free(RAMBlock *block)
2313 {
2314     g_autofree char *name = NULL;
2315 
2316     if (!block) {
2317         return;
2318     }
2319 
2320     if (block->host) {
2321         ram_block_notify_remove(block->host, block->used_length,
2322                                 block->max_length);
2323     }
2324 
2325     qemu_mutex_lock_ramlist();
2326     name = cpr_name(block->mr);
2327     cpr_delete_fd(name, 0);
2328     QLIST_REMOVE_RCU(block, next);
2329     ram_list.mru_block = NULL;
2330     /* Write list before version */
2331     smp_wmb();
2332     ram_list.version++;
2333     call_rcu(block, reclaim_ramblock, rcu);
2334     qemu_mutex_unlock_ramlist();
2335 }
2336 
2337 #ifndef _WIN32
2338 /* Simply remap the given VM memory location from start to start+length */
qemu_ram_remap_mmap(RAMBlock * block,uint64_t start,size_t length)2339 static int qemu_ram_remap_mmap(RAMBlock *block, uint64_t start, size_t length)
2340 {
2341     int flags, prot;
2342     void *area;
2343     void *host_startaddr = block->host + start;
2344 
2345     assert(block->fd < 0);
2346     flags = MAP_FIXED | MAP_ANONYMOUS;
2347     flags |= block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE;
2348     flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
2349     prot = PROT_READ;
2350     prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
2351     area = mmap(host_startaddr, length, prot, flags, -1, 0);
2352     return area != host_startaddr ? -errno : 0;
2353 }
2354 
2355 /*
2356  * qemu_ram_remap - remap a single RAM page
2357  *
2358  * @addr: address in ram_addr_t address space.
2359  *
2360  * This function will try remapping a single page of guest RAM identified by
2361  * @addr, essentially discarding memory to recover from previously poisoned
2362  * memory (MCE). The page size depends on the RAMBlock (i.e., hugetlb). @addr
2363  * does not have to point at the start of the page.
2364  *
2365  * This function is only to be used during system resets; it will kill the
2366  * VM if remapping failed.
2367  */
qemu_ram_remap(ram_addr_t addr)2368 void qemu_ram_remap(ram_addr_t addr)
2369 {
2370     RAMBlock *block;
2371     uint64_t offset;
2372     void *vaddr;
2373     size_t page_size;
2374 
2375     RAMBLOCK_FOREACH(block) {
2376         offset = addr - block->offset;
2377         if (offset < block->max_length) {
2378             /* Respect the pagesize of our RAMBlock */
2379             page_size = qemu_ram_pagesize(block);
2380             offset = QEMU_ALIGN_DOWN(offset, page_size);
2381 
2382             vaddr = ramblock_ptr(block, offset);
2383             if (block->flags & RAM_PREALLOC) {
2384                 ;
2385             } else if (xen_enabled()) {
2386                 abort();
2387             } else {
2388                 if (ram_block_discard_range(block, offset, page_size) != 0) {
2389                     /*
2390                      * Fall back to using mmap() only for anonymous mapping,
2391                      * as if a backing file is associated we may not be able
2392                      * to recover the memory in all cases.
2393                      * So don't take the risk of using only mmap and fail now.
2394                      */
2395                     if (block->fd >= 0) {
2396                         error_report("Could not remap RAM %s:%" PRIx64 "+%"
2397                                      PRIx64 " +%zx", block->idstr, offset,
2398                                      block->fd_offset, page_size);
2399                         exit(1);
2400                     }
2401                     if (qemu_ram_remap_mmap(block, offset, page_size) != 0) {
2402                         error_report("Could not remap RAM %s:%" PRIx64 " +%zx",
2403                                      block->idstr, offset, page_size);
2404                         exit(1);
2405                     }
2406                 }
2407                 memory_try_enable_merging(vaddr, page_size);
2408                 qemu_ram_setup_dump(vaddr, page_size);
2409             }
2410 
2411             break;
2412         }
2413     }
2414 }
2415 #endif /* !_WIN32 */
2416 
2417 /*
2418  * Return a host pointer to guest's ram.
2419  * For Xen, foreign mappings get created if they don't already exist.
2420  *
2421  * @block: block for the RAM to lookup (optional and may be NULL).
2422  * @addr: address within the memory region.
2423  * @size: pointer to requested size (optional and may be NULL).
2424  *        size may get modified and return a value smaller than
2425  *        what was requested.
2426  * @lock: wether to lock the mapping in xen-mapcache until invalidated.
2427  * @is_write: hint wether to map RW or RO in the xen-mapcache.
2428  *            (optional and may always be set to true).
2429  *
2430  * Called within RCU critical section.
2431  */
qemu_ram_ptr_length(RAMBlock * block,ram_addr_t addr,hwaddr * size,bool lock,bool is_write)2432 static void *qemu_ram_ptr_length(RAMBlock *block, ram_addr_t addr,
2433                                  hwaddr *size, bool lock,
2434                                  bool is_write)
2435 {
2436     hwaddr len = 0;
2437 
2438     if (size && *size == 0) {
2439         return NULL;
2440     }
2441 
2442     if (block == NULL) {
2443         block = qemu_get_ram_block(addr);
2444         addr -= block->offset;
2445     }
2446     if (size) {
2447         *size = MIN(*size, block->max_length - addr);
2448         len = *size;
2449     }
2450 
2451     if (xen_enabled() && block->host == NULL) {
2452         /* We need to check if the requested address is in the RAM
2453          * because we don't want to map the entire memory in QEMU.
2454          * In that case just map the requested area.
2455          */
2456         if (xen_mr_is_memory(block->mr)) {
2457             return xen_map_cache(block->mr, block->offset + addr,
2458                                  len, block->offset,
2459                                  lock, lock, is_write);
2460         }
2461 
2462         block->host = xen_map_cache(block->mr, block->offset,
2463                                     block->max_length,
2464                                     block->offset,
2465                                     1, lock, is_write);
2466     }
2467 
2468     return ramblock_ptr(block, addr);
2469 }
2470 
2471 /*
2472  * Return a host pointer to ram allocated with qemu_ram_alloc.
2473  * This should not be used for general purpose DMA.  Use address_space_map
2474  * or address_space_rw instead. For local memory (e.g. video ram) that the
2475  * device owns, use memory_region_get_ram_ptr.
2476  *
2477  * Called within RCU critical section.
2478  */
qemu_map_ram_ptr(RAMBlock * ram_block,ram_addr_t addr)2479 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
2480 {
2481     return qemu_ram_ptr_length(ram_block, addr, NULL, false, true);
2482 }
2483 
2484 /* Return the offset of a hostpointer within a ramblock */
qemu_ram_block_host_offset(RAMBlock * rb,void * host)2485 ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2486 {
2487     ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2488     assert((uintptr_t)host >= (uintptr_t)rb->host);
2489     assert(res < rb->max_length);
2490 
2491     return res;
2492 }
2493 
qemu_ram_block_from_host(void * ptr,bool round_offset,ram_addr_t * offset)2494 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
2495                                    ram_addr_t *offset)
2496 {
2497     RAMBlock *block;
2498     uint8_t *host = ptr;
2499 
2500     if (xen_enabled()) {
2501         ram_addr_t ram_addr;
2502         RCU_READ_LOCK_GUARD();
2503         ram_addr = xen_ram_addr_from_mapcache(ptr);
2504         if (ram_addr == RAM_ADDR_INVALID) {
2505             return NULL;
2506         }
2507 
2508         block = qemu_get_ram_block(ram_addr);
2509         if (block) {
2510             *offset = ram_addr - block->offset;
2511         }
2512         return block;
2513     }
2514 
2515     RCU_READ_LOCK_GUARD();
2516     block = qatomic_rcu_read(&ram_list.mru_block);
2517     if (block && block->host && host - block->host < block->max_length) {
2518         goto found;
2519     }
2520 
2521     RAMBLOCK_FOREACH(block) {
2522         /* This case append when the block is not mapped. */
2523         if (block->host == NULL) {
2524             continue;
2525         }
2526         if (host - block->host < block->max_length) {
2527             goto found;
2528         }
2529     }
2530 
2531     return NULL;
2532 
2533 found:
2534     *offset = (host - block->host);
2535     if (round_offset) {
2536         *offset &= TARGET_PAGE_MASK;
2537     }
2538     return block;
2539 }
2540 
2541 /*
2542  * Finds the named RAMBlock
2543  *
2544  * name: The name of RAMBlock to find
2545  *
2546  * Returns: RAMBlock (or NULL if not found)
2547  */
qemu_ram_block_by_name(const char * name)2548 RAMBlock *qemu_ram_block_by_name(const char *name)
2549 {
2550     RAMBlock *block;
2551 
2552     RAMBLOCK_FOREACH(block) {
2553         if (!strcmp(name, block->idstr)) {
2554             return block;
2555         }
2556     }
2557 
2558     return NULL;
2559 }
2560 
2561 /*
2562  * Some of the system routines need to translate from a host pointer
2563  * (typically a TLB entry) back to a ram offset.
2564  */
qemu_ram_addr_from_host(void * ptr)2565 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2566 {
2567     RAMBlock *block;
2568     ram_addr_t offset;
2569 
2570     block = qemu_ram_block_from_host(ptr, false, &offset);
2571     if (!block) {
2572         return RAM_ADDR_INVALID;
2573     }
2574 
2575     return block->offset + offset;
2576 }
2577 
qemu_ram_addr_from_host_nofail(void * ptr)2578 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2579 {
2580     ram_addr_t ram_addr;
2581 
2582     ram_addr = qemu_ram_addr_from_host(ptr);
2583     if (ram_addr == RAM_ADDR_INVALID) {
2584         error_report("Bad ram pointer %p", ptr);
2585         abort();
2586     }
2587     return ram_addr;
2588 }
2589 
2590 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2591                                  MemTxAttrs attrs, void *buf, hwaddr len);
2592 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2593                                   const void *buf, hwaddr len);
2594 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
2595                                   bool is_write, MemTxAttrs attrs);
2596 
subpage_read(void * opaque,hwaddr addr,uint64_t * data,unsigned len,MemTxAttrs attrs)2597 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2598                                 unsigned len, MemTxAttrs attrs)
2599 {
2600     subpage_t *subpage = opaque;
2601     uint8_t buf[8];
2602     MemTxResult res;
2603 
2604 #if defined(DEBUG_SUBPAGE)
2605     printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__,
2606            subpage, len, addr);
2607 #endif
2608     res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
2609     if (res) {
2610         return res;
2611     }
2612     *data = ldn_p(buf, len);
2613     return MEMTX_OK;
2614 }
2615 
subpage_write(void * opaque,hwaddr addr,uint64_t value,unsigned len,MemTxAttrs attrs)2616 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2617                                  uint64_t value, unsigned len, MemTxAttrs attrs)
2618 {
2619     subpage_t *subpage = opaque;
2620     uint8_t buf[8];
2621 
2622 #if defined(DEBUG_SUBPAGE)
2623     printf("%s: subpage %p len %u addr " HWADDR_FMT_plx
2624            " value %"PRIx64"\n",
2625            __func__, subpage, len, addr, value);
2626 #endif
2627     stn_p(buf, len, value);
2628     return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
2629 }
2630 
subpage_accepts(void * opaque,hwaddr addr,unsigned len,bool is_write,MemTxAttrs attrs)2631 static bool subpage_accepts(void *opaque, hwaddr addr,
2632                             unsigned len, bool is_write,
2633                             MemTxAttrs attrs)
2634 {
2635     subpage_t *subpage = opaque;
2636 #if defined(DEBUG_SUBPAGE)
2637     printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n",
2638            __func__, subpage, is_write ? 'w' : 'r', len, addr);
2639 #endif
2640 
2641     return flatview_access_valid(subpage->fv, addr + subpage->base,
2642                                  len, is_write, attrs);
2643 }
2644 
2645 static const MemoryRegionOps subpage_ops = {
2646     .read_with_attrs = subpage_read,
2647     .write_with_attrs = subpage_write,
2648     .impl.min_access_size = 1,
2649     .impl.max_access_size = 8,
2650     .valid.min_access_size = 1,
2651     .valid.max_access_size = 8,
2652     .valid.accepts = subpage_accepts,
2653     .endianness = DEVICE_NATIVE_ENDIAN,
2654 };
2655 
subpage_register(subpage_t * mmio,uint32_t start,uint32_t end,uint16_t section)2656 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2657                             uint16_t section)
2658 {
2659     int idx, eidx;
2660 
2661     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2662         return -1;
2663     idx = SUBPAGE_IDX(start);
2664     eidx = SUBPAGE_IDX(end);
2665 #if defined(DEBUG_SUBPAGE)
2666     printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2667            __func__, mmio, start, end, idx, eidx, section);
2668 #endif
2669     for (; idx <= eidx; idx++) {
2670         mmio->sub_section[idx] = section;
2671     }
2672 
2673     return 0;
2674 }
2675 
subpage_init(FlatView * fv,hwaddr base)2676 static subpage_t *subpage_init(FlatView *fv, hwaddr base)
2677 {
2678     subpage_t *mmio;
2679 
2680     /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
2681     mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
2682     mmio->fv = fv;
2683     mmio->base = base;
2684     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2685                           NULL, TARGET_PAGE_SIZE);
2686     mmio->iomem.subpage = true;
2687 #if defined(DEBUG_SUBPAGE)
2688     printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__,
2689            mmio, base, TARGET_PAGE_SIZE);
2690 #endif
2691 
2692     return mmio;
2693 }
2694 
dummy_section(PhysPageMap * map,FlatView * fv,MemoryRegion * mr)2695 static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
2696 {
2697     assert(fv);
2698     MemoryRegionSection section = {
2699         .fv = fv,
2700         .mr = mr,
2701         .offset_within_address_space = 0,
2702         .offset_within_region = 0,
2703         .size = int128_2_64(),
2704     };
2705 
2706     return phys_section_add(map, &section);
2707 }
2708 
io_mem_init(void)2709 static void io_mem_init(void)
2710 {
2711     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2712                           NULL, UINT64_MAX);
2713 }
2714 
address_space_dispatch_new(FlatView * fv)2715 AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
2716 {
2717     AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2718     uint16_t n;
2719 
2720     n = dummy_section(&d->map, fv, &io_mem_unassigned);
2721     assert(n == PHYS_SECTION_UNASSIGNED);
2722 
2723     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2724 
2725     return d;
2726 }
2727 
address_space_dispatch_free(AddressSpaceDispatch * d)2728 void address_space_dispatch_free(AddressSpaceDispatch *d)
2729 {
2730     phys_sections_free(&d->map);
2731     g_free(d);
2732 }
2733 
do_nothing(CPUState * cpu,run_on_cpu_data d)2734 static void do_nothing(CPUState *cpu, run_on_cpu_data d)
2735 {
2736 }
2737 
tcg_log_global_after_sync(MemoryListener * listener)2738 static void tcg_log_global_after_sync(MemoryListener *listener)
2739 {
2740     CPUAddressSpace *cpuas;
2741 
2742     /* Wait for the CPU to end the current TB.  This avoids the following
2743      * incorrect race:
2744      *
2745      *      vCPU                         migration
2746      *      ----------------------       -------------------------
2747      *      TLB check -> slow path
2748      *        notdirty_mem_write
2749      *          write to RAM
2750      *          mark dirty
2751      *                                   clear dirty flag
2752      *      TLB check -> fast path
2753      *                                   read memory
2754      *        write to RAM
2755      *
2756      * by pushing the migration thread's memory read after the vCPU thread has
2757      * written the memory.
2758      */
2759     if (replay_mode == REPLAY_MODE_NONE) {
2760         /*
2761          * VGA can make calls to this function while updating the screen.
2762          * In record/replay mode this causes a deadlock, because
2763          * run_on_cpu waits for rr mutex. Therefore no races are possible
2764          * in this case and no need for making run_on_cpu when
2765          * record/replay is enabled.
2766          */
2767         cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2768         run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
2769     }
2770 }
2771 
tcg_commit_cpu(CPUState * cpu,run_on_cpu_data data)2772 static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
2773 {
2774     tlb_flush(cpu);
2775 }
2776 
tcg_commit(MemoryListener * listener)2777 static void tcg_commit(MemoryListener *listener)
2778 {
2779     CPUAddressSpace *cpuas;
2780     CPUState *cpu;
2781 
2782     assert(tcg_enabled());
2783     /* since each CPU stores ram addresses in its TLB cache, we must
2784        reset the modified entries */
2785     cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2786     cpu = cpuas->cpu;
2787 
2788     /*
2789      * Queueing the work function will kick the cpu back to
2790      * the main loop, which will end the RCU critical section and reclaim
2791      * the memory data structures.
2792      *
2793      * That said, the listener is also called during realize, before
2794      * all of the tcg machinery for run-on is initialized: thus halt_cond.
2795      */
2796     if (cpu->halt_cond) {
2797         async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2798     } else {
2799         tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2800     }
2801 }
2802 
memory_map_init(void)2803 static void memory_map_init(void)
2804 {
2805     system_memory = g_malloc(sizeof(*system_memory));
2806 
2807     memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2808     address_space_init(&address_space_memory, system_memory, "memory");
2809 
2810     system_io = g_malloc(sizeof(*system_io));
2811     memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2812                           65536);
2813     address_space_init(&address_space_io, system_io, "I/O");
2814 }
2815 
get_system_memory(void)2816 MemoryRegion *get_system_memory(void)
2817 {
2818     return system_memory;
2819 }
2820 
get_system_io(void)2821 MemoryRegion *get_system_io(void)
2822 {
2823     return system_io;
2824 }
2825 
invalidate_and_set_dirty(MemoryRegion * mr,hwaddr addr,hwaddr length)2826 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2827                                      hwaddr length)
2828 {
2829     uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2830     ram_addr_t ramaddr = memory_region_get_ram_addr(mr);
2831 
2832     /* We know we're only called for RAM MemoryRegions */
2833     assert(ramaddr != RAM_ADDR_INVALID);
2834     addr += ramaddr;
2835 
2836     /* No early return if dirty_log_mask is or becomes 0, because
2837      * cpu_physical_memory_set_dirty_range will still call
2838      * xen_modified_memory.
2839      */
2840     if (dirty_log_mask) {
2841         dirty_log_mask =
2842             cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2843     }
2844     if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2845         assert(tcg_enabled());
2846         tb_invalidate_phys_range(NULL, addr, addr + length - 1);
2847         dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2848     }
2849     cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2850 }
2851 
memory_region_flush_rom_device(MemoryRegion * mr,hwaddr addr,hwaddr size)2852 void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
2853 {
2854     /*
2855      * In principle this function would work on other memory region types too,
2856      * but the ROM device use case is the only one where this operation is
2857      * necessary.  Other memory regions should use the
2858      * address_space_read/write() APIs.
2859      */
2860     assert(memory_region_is_romd(mr));
2861 
2862     invalidate_and_set_dirty(mr, addr, size);
2863 }
2864 
memory_access_size(MemoryRegion * mr,unsigned l,hwaddr addr)2865 int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2866 {
2867     unsigned access_size_max = mr->ops->valid.max_access_size;
2868 
2869     /* Regions are assumed to support 1-4 byte accesses unless
2870        otherwise specified.  */
2871     if (access_size_max == 0) {
2872         access_size_max = 4;
2873     }
2874 
2875     /* Bound the maximum access by the alignment of the address.  */
2876     if (!mr->ops->impl.unaligned) {
2877         unsigned align_size_max = addr & -addr;
2878         if (align_size_max != 0 && align_size_max < access_size_max) {
2879             access_size_max = align_size_max;
2880         }
2881     }
2882 
2883     /* Don't attempt accesses larger than the maximum.  */
2884     if (l > access_size_max) {
2885         l = access_size_max;
2886     }
2887     l = pow2floor(l);
2888 
2889     return l;
2890 }
2891 
prepare_mmio_access(MemoryRegion * mr)2892 bool prepare_mmio_access(MemoryRegion *mr)
2893 {
2894     bool release_lock = false;
2895 
2896     if (!bql_locked()) {
2897         bql_lock();
2898         release_lock = true;
2899     }
2900     if (mr->flush_coalesced_mmio) {
2901         qemu_flush_coalesced_mmio_buffer();
2902     }
2903 
2904     return release_lock;
2905 }
2906 
2907 /**
2908  * flatview_access_allowed
2909  * @mr: #MemoryRegion to be accessed
2910  * @attrs: memory transaction attributes
2911  * @addr: address within that memory region
2912  * @len: the number of bytes to access
2913  *
2914  * Check if a memory transaction is allowed.
2915  *
2916  * Returns: true if transaction is allowed, false if denied.
2917  */
flatview_access_allowed(MemoryRegion * mr,MemTxAttrs attrs,hwaddr addr,hwaddr len)2918 static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
2919                                     hwaddr addr, hwaddr len)
2920 {
2921     if (likely(!attrs.memory)) {
2922         return true;
2923     }
2924     if (memory_region_is_ram(mr)) {
2925         return true;
2926     }
2927     qemu_log_mask(LOG_INVALID_MEM,
2928                   "Invalid access to non-RAM device at "
2929                   "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
2930                   "region '%s'\n", addr, len, memory_region_name(mr));
2931     return false;
2932 }
2933 
flatview_write_continue_step(MemTxAttrs attrs,const uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)2934 static MemTxResult flatview_write_continue_step(MemTxAttrs attrs,
2935                                                 const uint8_t *buf,
2936                                                 hwaddr len, hwaddr mr_addr,
2937                                                 hwaddr *l, MemoryRegion *mr)
2938 {
2939     if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2940         return MEMTX_ACCESS_ERROR;
2941     }
2942 
2943     if (!memory_access_is_direct(mr, true, attrs)) {
2944         uint64_t val;
2945         MemTxResult result;
2946         bool release_lock = prepare_mmio_access(mr);
2947 
2948         *l = memory_access_size(mr, *l, mr_addr);
2949         /*
2950          * XXX: could force current_cpu to NULL to avoid
2951          * potential bugs
2952          */
2953 
2954         /*
2955          * Assure Coverity (and ourselves) that we are not going to OVERRUN
2956          * the buffer by following ldn_he_p().
2957          */
2958 #ifdef QEMU_STATIC_ANALYSIS
2959         assert((*l == 1 && len >= 1) ||
2960                (*l == 2 && len >= 2) ||
2961                (*l == 4 && len >= 4) ||
2962                (*l == 8 && len >= 8));
2963 #endif
2964         val = ldn_he_p(buf, *l);
2965         result = memory_region_dispatch_write(mr, mr_addr, val,
2966                                               size_memop(*l), attrs);
2967         if (release_lock) {
2968             bql_unlock();
2969         }
2970 
2971         return result;
2972     } else {
2973         /* RAM case */
2974         uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
2975                                                false, true);
2976 
2977         memmove(ram_ptr, buf, *l);
2978         invalidate_and_set_dirty(mr, mr_addr, *l);
2979 
2980         return MEMTX_OK;
2981     }
2982 }
2983 
2984 /* Called within RCU critical section.  */
flatview_write_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)2985 static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
2986                                            MemTxAttrs attrs,
2987                                            const void *ptr,
2988                                            hwaddr len, hwaddr mr_addr,
2989                                            hwaddr l, MemoryRegion *mr)
2990 {
2991     MemTxResult result = MEMTX_OK;
2992     const uint8_t *buf = ptr;
2993 
2994     for (;;) {
2995         result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
2996                                                mr);
2997 
2998         len -= l;
2999         buf += l;
3000         addr += l;
3001 
3002         if (!len) {
3003             break;
3004         }
3005 
3006         l = len;
3007         mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3008     }
3009 
3010     return result;
3011 }
3012 
3013 /* Called from RCU critical section.  */
flatview_write(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3014 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
3015                                   const void *buf, hwaddr len)
3016 {
3017     hwaddr l;
3018     hwaddr mr_addr;
3019     MemoryRegion *mr;
3020 
3021     l = len;
3022     mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3023     if (!flatview_access_allowed(mr, attrs, addr, len)) {
3024         return MEMTX_ACCESS_ERROR;
3025     }
3026     return flatview_write_continue(fv, addr, attrs, buf, len,
3027                                    mr_addr, l, mr);
3028 }
3029 
flatview_read_continue_step(MemTxAttrs attrs,uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)3030 static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf,
3031                                                hwaddr len, hwaddr mr_addr,
3032                                                hwaddr *l,
3033                                                MemoryRegion *mr)
3034 {
3035     if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
3036         return MEMTX_ACCESS_ERROR;
3037     }
3038 
3039     if (!memory_access_is_direct(mr, false, attrs)) {
3040         /* I/O case */
3041         uint64_t val;
3042         MemTxResult result;
3043         bool release_lock = prepare_mmio_access(mr);
3044 
3045         *l = memory_access_size(mr, *l, mr_addr);
3046         result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l),
3047                                              attrs);
3048 
3049         /*
3050          * Assure Coverity (and ourselves) that we are not going to OVERRUN
3051          * the buffer by following stn_he_p().
3052          */
3053 #ifdef QEMU_STATIC_ANALYSIS
3054         assert((*l == 1 && len >= 1) ||
3055                (*l == 2 && len >= 2) ||
3056                (*l == 4 && len >= 4) ||
3057                (*l == 8 && len >= 8));
3058 #endif
3059         stn_he_p(buf, *l, val);
3060 
3061         if (release_lock) {
3062             bql_unlock();
3063         }
3064         return result;
3065     } else {
3066         /* RAM case */
3067         uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
3068                                                false, false);
3069 
3070         memcpy(buf, ram_ptr, *l);
3071 
3072         return MEMTX_OK;
3073     }
3074 }
3075 
3076 /* Called within RCU critical section.  */
flatview_read_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3077 MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
3078                                    MemTxAttrs attrs, void *ptr,
3079                                    hwaddr len, hwaddr mr_addr, hwaddr l,
3080                                    MemoryRegion *mr)
3081 {
3082     MemTxResult result = MEMTX_OK;
3083     uint8_t *buf = ptr;
3084 
3085     fuzz_dma_read_cb(addr, len, mr);
3086     for (;;) {
3087         result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3088 
3089         len -= l;
3090         buf += l;
3091         addr += l;
3092 
3093         if (!len) {
3094             break;
3095         }
3096 
3097         l = len;
3098         mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3099     }
3100 
3101     return result;
3102 }
3103 
3104 /* Called from RCU critical section.  */
flatview_read(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)3105 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
3106                                  MemTxAttrs attrs, void *buf, hwaddr len)
3107 {
3108     hwaddr l;
3109     hwaddr mr_addr;
3110     MemoryRegion *mr;
3111 
3112     l = len;
3113     mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3114     if (!flatview_access_allowed(mr, attrs, addr, len)) {
3115         return MEMTX_ACCESS_ERROR;
3116     }
3117     return flatview_read_continue(fv, addr, attrs, buf, len,
3118                                   mr_addr, l, mr);
3119 }
3120 
address_space_read_full(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)3121 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
3122                                     MemTxAttrs attrs, void *buf, hwaddr len)
3123 {
3124     MemTxResult result = MEMTX_OK;
3125     FlatView *fv;
3126 
3127     if (len > 0) {
3128         RCU_READ_LOCK_GUARD();
3129         fv = address_space_to_flatview(as);
3130         result = flatview_read(fv, addr, attrs, buf, len);
3131     }
3132 
3133     return result;
3134 }
3135 
address_space_write(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3136 MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
3137                                 MemTxAttrs attrs,
3138                                 const void *buf, hwaddr len)
3139 {
3140     MemTxResult result = MEMTX_OK;
3141     FlatView *fv;
3142 
3143     if (len > 0) {
3144         RCU_READ_LOCK_GUARD();
3145         fv = address_space_to_flatview(as);
3146         result = flatview_write(fv, addr, attrs, buf, len);
3147     }
3148 
3149     return result;
3150 }
3151 
address_space_rw(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len,bool is_write)3152 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
3153                              void *buf, hwaddr len, bool is_write)
3154 {
3155     if (is_write) {
3156         return address_space_write(as, addr, attrs, buf, len);
3157     } else {
3158         return address_space_read_full(as, addr, attrs, buf, len);
3159     }
3160 }
3161 
address_space_set(AddressSpace * as,hwaddr addr,uint8_t c,hwaddr len,MemTxAttrs attrs)3162 MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
3163                               uint8_t c, hwaddr len, MemTxAttrs attrs)
3164 {
3165 #define FILLBUF_SIZE 512
3166     uint8_t fillbuf[FILLBUF_SIZE];
3167     int l;
3168     MemTxResult error = MEMTX_OK;
3169 
3170     memset(fillbuf, c, FILLBUF_SIZE);
3171     while (len > 0) {
3172         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
3173         error |= address_space_write(as, addr, attrs, fillbuf, l);
3174         len -= l;
3175         addr += l;
3176     }
3177 
3178     return error;
3179 }
3180 
cpu_physical_memory_rw(hwaddr addr,void * buf,hwaddr len,bool is_write)3181 void cpu_physical_memory_rw(hwaddr addr, void *buf,
3182                             hwaddr len, bool is_write)
3183 {
3184     address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3185                      buf, len, is_write);
3186 }
3187 
3188 enum write_rom_type {
3189     WRITE_DATA,
3190     FLUSH_CACHE,
3191 };
3192 
address_space_write_rom_internal(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,enum write_rom_type type)3193 static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
3194                                                            hwaddr addr,
3195                                                            MemTxAttrs attrs,
3196                                                            const void *ptr,
3197                                                            hwaddr len,
3198                                                            enum write_rom_type type)
3199 {
3200     hwaddr l;
3201     uint8_t *ram_ptr;
3202     hwaddr addr1;
3203     MemoryRegion *mr;
3204     const uint8_t *buf = ptr;
3205 
3206     RCU_READ_LOCK_GUARD();
3207     while (len > 0) {
3208         l = len;
3209         mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
3210 
3211         if (!memory_region_supports_direct_access(mr)) {
3212             l = memory_access_size(mr, l, addr1);
3213         } else {
3214             /* ROM/RAM case */
3215             ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3216             switch (type) {
3217             case WRITE_DATA:
3218                 memcpy(ram_ptr, buf, l);
3219                 invalidate_and_set_dirty(mr, addr1, l);
3220                 break;
3221             case FLUSH_CACHE:
3222                 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
3223                 break;
3224             }
3225         }
3226         len -= l;
3227         buf += l;
3228         addr += l;
3229     }
3230     return MEMTX_OK;
3231 }
3232 
3233 /* used for ROM loading : can write in RAM and ROM */
address_space_write_rom(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3234 MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
3235                                     MemTxAttrs attrs,
3236                                     const void *buf, hwaddr len)
3237 {
3238     return address_space_write_rom_internal(as, addr, attrs,
3239                                             buf, len, WRITE_DATA);
3240 }
3241 
cpu_flush_icache_range(hwaddr start,hwaddr len)3242 void cpu_flush_icache_range(hwaddr start, hwaddr len)
3243 {
3244     /*
3245      * This function should do the same thing as an icache flush that was
3246      * triggered from within the guest. For TCG we are always cache coherent,
3247      * so there is no need to flush anything. For KVM / Xen we need to flush
3248      * the host's instruction cache at least.
3249      */
3250     if (tcg_enabled()) {
3251         return;
3252     }
3253 
3254     address_space_write_rom_internal(&address_space_memory,
3255                                      start, MEMTXATTRS_UNSPECIFIED,
3256                                      NULL, len, FLUSH_CACHE);
3257 }
3258 
3259 /*
3260  * A magic value stored in the first 8 bytes of the bounce buffer struct. Used
3261  * to detect illegal pointers passed to address_space_unmap.
3262  */
3263 #define BOUNCE_BUFFER_MAGIC 0xb4017ceb4ffe12ed
3264 
3265 typedef struct {
3266     uint64_t magic;
3267     MemoryRegion *mr;
3268     hwaddr addr;
3269     size_t len;
3270     uint8_t buffer[];
3271 } BounceBuffer;
3272 
3273 static void
address_space_unregister_map_client_do(AddressSpaceMapClient * client)3274 address_space_unregister_map_client_do(AddressSpaceMapClient *client)
3275 {
3276     QLIST_REMOVE(client, link);
3277     g_free(client);
3278 }
3279 
address_space_notify_map_clients_locked(AddressSpace * as)3280 static void address_space_notify_map_clients_locked(AddressSpace *as)
3281 {
3282     AddressSpaceMapClient *client;
3283 
3284     while (!QLIST_EMPTY(&as->map_client_list)) {
3285         client = QLIST_FIRST(&as->map_client_list);
3286         qemu_bh_schedule(client->bh);
3287         address_space_unregister_map_client_do(client);
3288     }
3289 }
3290 
address_space_register_map_client(AddressSpace * as,QEMUBH * bh)3291 void address_space_register_map_client(AddressSpace *as, QEMUBH *bh)
3292 {
3293     AddressSpaceMapClient *client = g_malloc(sizeof(*client));
3294 
3295     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3296     client->bh = bh;
3297     QLIST_INSERT_HEAD(&as->map_client_list, client, link);
3298     /* Write map_client_list before reading bounce_buffer_size. */
3299     smp_mb();
3300     if (qatomic_read(&as->bounce_buffer_size) < as->max_bounce_buffer_size) {
3301         address_space_notify_map_clients_locked(as);
3302     }
3303 }
3304 
cpu_exec_init_all(void)3305 void cpu_exec_init_all(void)
3306 {
3307     qemu_mutex_init(&ram_list.mutex);
3308     /* The data structures we set up here depend on knowing the page size,
3309      * so no more changes can be made after this point.
3310      * In an ideal world, nothing we did before we had finished the
3311      * machine setup would care about the target page size, and we could
3312      * do this much later, rather than requiring board models to state
3313      * up front what their requirements are.
3314      */
3315     finalize_target_page_bits();
3316     io_mem_init();
3317     memory_map_init();
3318 }
3319 
address_space_unregister_map_client(AddressSpace * as,QEMUBH * bh)3320 void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh)
3321 {
3322     AddressSpaceMapClient *client;
3323 
3324     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3325     QLIST_FOREACH(client, &as->map_client_list, link) {
3326         if (client->bh == bh) {
3327             address_space_unregister_map_client_do(client);
3328             break;
3329         }
3330     }
3331 }
3332 
address_space_notify_map_clients(AddressSpace * as)3333 static void address_space_notify_map_clients(AddressSpace *as)
3334 {
3335     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3336     address_space_notify_map_clients_locked(as);
3337 }
3338 
flatview_access_valid(FlatView * fv,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3339 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
3340                                   bool is_write, MemTxAttrs attrs)
3341 {
3342     MemoryRegion *mr;
3343     hwaddr l, xlat;
3344 
3345     while (len > 0) {
3346         l = len;
3347         mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3348         if (!memory_access_is_direct(mr, is_write, attrs)) {
3349             l = memory_access_size(mr, l, addr);
3350             if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
3351                 return false;
3352             }
3353         }
3354 
3355         len -= l;
3356         addr += l;
3357     }
3358     return true;
3359 }
3360 
address_space_access_valid(AddressSpace * as,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3361 bool address_space_access_valid(AddressSpace *as, hwaddr addr,
3362                                 hwaddr len, bool is_write,
3363                                 MemTxAttrs attrs)
3364 {
3365     FlatView *fv;
3366 
3367     RCU_READ_LOCK_GUARD();
3368     fv = address_space_to_flatview(as);
3369     return flatview_access_valid(fv, addr, len, is_write, attrs);
3370 }
3371 
3372 static hwaddr
flatview_extend_translation(FlatView * fv,hwaddr addr,hwaddr target_len,MemoryRegion * mr,hwaddr base,hwaddr len,bool is_write,MemTxAttrs attrs)3373 flatview_extend_translation(FlatView *fv, hwaddr addr,
3374                             hwaddr target_len,
3375                             MemoryRegion *mr, hwaddr base, hwaddr len,
3376                             bool is_write, MemTxAttrs attrs)
3377 {
3378     hwaddr done = 0;
3379     hwaddr xlat;
3380     MemoryRegion *this_mr;
3381 
3382     for (;;) {
3383         target_len -= len;
3384         addr += len;
3385         done += len;
3386         if (target_len == 0) {
3387             return done;
3388         }
3389 
3390         len = target_len;
3391         this_mr = flatview_translate(fv, addr, &xlat,
3392                                      &len, is_write, attrs);
3393         if (this_mr != mr || xlat != base + done) {
3394             return done;
3395         }
3396     }
3397 }
3398 
3399 /* Map a physical memory region into a host virtual address.
3400  * May map a subset of the requested range, given by and returned in *plen.
3401  * May return NULL if resources needed to perform the mapping are exhausted.
3402  * Use only for reads OR writes - not for read-modify-write operations.
3403  * Use address_space_register_map_client() to know when retrying the map
3404  * operation is likely to succeed.
3405  */
address_space_map(AddressSpace * as,hwaddr addr,hwaddr * plen,bool is_write,MemTxAttrs attrs)3406 void *address_space_map(AddressSpace *as,
3407                         hwaddr addr,
3408                         hwaddr *plen,
3409                         bool is_write,
3410                         MemTxAttrs attrs)
3411 {
3412     hwaddr len = *plen;
3413     hwaddr l, xlat;
3414     MemoryRegion *mr;
3415     FlatView *fv;
3416 
3417     trace_address_space_map(as, addr, len, is_write, *(uint32_t *) &attrs);
3418 
3419     if (len == 0) {
3420         return NULL;
3421     }
3422 
3423     l = len;
3424     RCU_READ_LOCK_GUARD();
3425     fv = address_space_to_flatview(as);
3426     mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3427 
3428     if (!memory_access_is_direct(mr, is_write, attrs)) {
3429         size_t used = qatomic_read(&as->bounce_buffer_size);
3430         for (;;) {
3431             hwaddr alloc = MIN(as->max_bounce_buffer_size - used, l);
3432             size_t new_size = used + alloc;
3433             size_t actual =
3434                 qatomic_cmpxchg(&as->bounce_buffer_size, used, new_size);
3435             if (actual == used) {
3436                 l = alloc;
3437                 break;
3438             }
3439             used = actual;
3440         }
3441 
3442         if (l == 0) {
3443             *plen = 0;
3444             return NULL;
3445         }
3446 
3447         BounceBuffer *bounce = g_malloc0(l + sizeof(BounceBuffer));
3448         bounce->magic = BOUNCE_BUFFER_MAGIC;
3449         memory_region_ref(mr);
3450         bounce->mr = mr;
3451         bounce->addr = addr;
3452         bounce->len = l;
3453 
3454         if (!is_write) {
3455             flatview_read(fv, addr, attrs,
3456                           bounce->buffer, l);
3457         }
3458 
3459         *plen = l;
3460         return bounce->buffer;
3461     }
3462 
3463     memory_region_ref(mr);
3464     *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
3465                                         l, is_write, attrs);
3466     fuzz_dma_read_cb(addr, *plen, mr);
3467     return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write);
3468 }
3469 
3470 /* Unmaps a memory region previously mapped by address_space_map().
3471  * Will also mark the memory as dirty if is_write is true.  access_len gives
3472  * the amount of memory that was actually read or written by the caller.
3473  */
address_space_unmap(AddressSpace * as,void * buffer,hwaddr len,bool is_write,hwaddr access_len)3474 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3475                          bool is_write, hwaddr access_len)
3476 {
3477     MemoryRegion *mr;
3478     ram_addr_t addr1;
3479 
3480     mr = memory_region_from_host(buffer, &addr1);
3481     if (mr != NULL) {
3482         if (is_write) {
3483             invalidate_and_set_dirty(mr, addr1, access_len);
3484         }
3485         if (xen_enabled()) {
3486             xen_invalidate_map_cache_entry(buffer);
3487         }
3488         memory_region_unref(mr);
3489         return;
3490     }
3491 
3492 
3493     BounceBuffer *bounce = container_of(buffer, BounceBuffer, buffer);
3494     assert(bounce->magic == BOUNCE_BUFFER_MAGIC);
3495 
3496     if (is_write) {
3497         address_space_write(as, bounce->addr, MEMTXATTRS_UNSPECIFIED,
3498                             bounce->buffer, access_len);
3499     }
3500 
3501     qatomic_sub(&as->bounce_buffer_size, bounce->len);
3502     bounce->magic = ~BOUNCE_BUFFER_MAGIC;
3503     memory_region_unref(bounce->mr);
3504     g_free(bounce);
3505     /* Write bounce_buffer_size before reading map_client_list. */
3506     smp_mb();
3507     address_space_notify_map_clients(as);
3508 }
3509 
cpu_physical_memory_map(hwaddr addr,hwaddr * plen,bool is_write)3510 void *cpu_physical_memory_map(hwaddr addr,
3511                               hwaddr *plen,
3512                               bool is_write)
3513 {
3514     return address_space_map(&address_space_memory, addr, plen, is_write,
3515                              MEMTXATTRS_UNSPECIFIED);
3516 }
3517 
cpu_physical_memory_unmap(void * buffer,hwaddr len,bool is_write,hwaddr access_len)3518 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3519                                bool is_write, hwaddr access_len)
3520 {
3521     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3522 }
3523 
3524 #define ARG1_DECL                AddressSpace *as
3525 #define ARG1                     as
3526 #define SUFFIX
3527 #define TRANSLATE(...)           address_space_translate(as, __VA_ARGS__)
3528 #define RCU_READ_LOCK(...)       rcu_read_lock()
3529 #define RCU_READ_UNLOCK(...)     rcu_read_unlock()
3530 #include "memory_ldst.c.inc"
3531 
address_space_cache_init(MemoryRegionCache * cache,AddressSpace * as,hwaddr addr,hwaddr len,bool is_write)3532 int64_t address_space_cache_init(MemoryRegionCache *cache,
3533                                  AddressSpace *as,
3534                                  hwaddr addr,
3535                                  hwaddr len,
3536                                  bool is_write)
3537 {
3538     AddressSpaceDispatch *d;
3539     hwaddr l;
3540     MemoryRegion *mr;
3541     Int128 diff;
3542 
3543     assert(len > 0);
3544 
3545     l = len;
3546     cache->fv = address_space_get_flatview(as);
3547     d = flatview_to_dispatch(cache->fv);
3548     cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3549 
3550     /*
3551      * cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3552      * Take that into account to compute how many bytes are there between
3553      * cache->xlat and the end of the section.
3554      */
3555     diff = int128_sub(cache->mrs.size,
3556                       int128_make64(cache->xlat - cache->mrs.offset_within_region));
3557     l = int128_get64(int128_min(diff, int128_make64(l)));
3558 
3559     mr = cache->mrs.mr;
3560     memory_region_ref(mr);
3561     if (memory_access_is_direct(mr, is_write, MEMTXATTRS_UNSPECIFIED)) {
3562         /* We don't care about the memory attributes here as we're only
3563          * doing this if we found actual RAM, which behaves the same
3564          * regardless of attributes; so UNSPECIFIED is fine.
3565          */
3566         l = flatview_extend_translation(cache->fv, addr, len, mr,
3567                                         cache->xlat, l, is_write,
3568                                         MEMTXATTRS_UNSPECIFIED);
3569         cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true,
3570                                          is_write);
3571     } else {
3572         cache->ptr = NULL;
3573     }
3574 
3575     cache->len = l;
3576     cache->is_write = is_write;
3577     return l;
3578 }
3579 
address_space_cache_invalidate(MemoryRegionCache * cache,hwaddr addr,hwaddr access_len)3580 void address_space_cache_invalidate(MemoryRegionCache *cache,
3581                                     hwaddr addr,
3582                                     hwaddr access_len)
3583 {
3584     assert(cache->is_write);
3585     if (likely(cache->ptr)) {
3586         invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3587     }
3588 }
3589 
address_space_cache_destroy(MemoryRegionCache * cache)3590 void address_space_cache_destroy(MemoryRegionCache *cache)
3591 {
3592     if (!cache->mrs.mr) {
3593         return;
3594     }
3595 
3596     if (xen_enabled()) {
3597         xen_invalidate_map_cache_entry(cache->ptr);
3598     }
3599     memory_region_unref(cache->mrs.mr);
3600     flatview_unref(cache->fv);
3601     cache->mrs.mr = NULL;
3602     cache->fv = NULL;
3603 }
3604 
3605 /* Called from RCU critical section.  This function has the same
3606  * semantics as address_space_translate, but it only works on a
3607  * predefined range of a MemoryRegion that was mapped with
3608  * address_space_cache_init.
3609  */
address_space_translate_cached(MemoryRegionCache * cache,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)3610 static inline MemoryRegion *address_space_translate_cached(
3611     MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
3612     hwaddr *plen, bool is_write, MemTxAttrs attrs)
3613 {
3614     MemoryRegionSection section;
3615     MemoryRegion *mr;
3616     IOMMUMemoryRegion *iommu_mr;
3617     AddressSpace *target_as;
3618 
3619     assert(!cache->ptr);
3620     *xlat = addr + cache->xlat;
3621 
3622     mr = cache->mrs.mr;
3623     iommu_mr = memory_region_get_iommu(mr);
3624     if (!iommu_mr) {
3625         /* MMIO region.  */
3626         return mr;
3627     }
3628 
3629     section = address_space_translate_iommu(iommu_mr, xlat, plen,
3630                                             NULL, is_write, true,
3631                                             &target_as, attrs);
3632     return section.mr;
3633 }
3634 
3635 /* Called within RCU critical section.  */
address_space_write_continue_cached(MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3636 static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs,
3637                                                        const void *ptr,
3638                                                        hwaddr len,
3639                                                        hwaddr mr_addr,
3640                                                        hwaddr l,
3641                                                        MemoryRegion *mr)
3642 {
3643     MemTxResult result = MEMTX_OK;
3644     const uint8_t *buf = ptr;
3645 
3646     for (;;) {
3647         result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3648                                                mr);
3649 
3650         len -= l;
3651         buf += l;
3652         mr_addr += l;
3653 
3654         if (!len) {
3655             break;
3656         }
3657 
3658         l = len;
3659     }
3660 
3661     return result;
3662 }
3663 
3664 /* Called within RCU critical section.  */
address_space_read_continue_cached(MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3665 static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs,
3666                                                       void *ptr, hwaddr len,
3667                                                       hwaddr mr_addr, hwaddr l,
3668                                                       MemoryRegion *mr)
3669 {
3670     MemTxResult result = MEMTX_OK;
3671     uint8_t *buf = ptr;
3672 
3673     for (;;) {
3674         result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3675         len -= l;
3676         buf += l;
3677         mr_addr += l;
3678 
3679         if (!len) {
3680             break;
3681         }
3682         l = len;
3683     }
3684 
3685     return result;
3686 }
3687 
3688 /* Called from RCU critical section. address_space_read_cached uses this
3689  * out of line function when the target is an MMIO or IOMMU region.
3690  */
3691 MemTxResult
address_space_read_cached_slow(MemoryRegionCache * cache,hwaddr addr,void * buf,hwaddr len)3692 address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3693                                    void *buf, hwaddr len)
3694 {
3695     hwaddr mr_addr, l;
3696     MemoryRegion *mr;
3697 
3698     l = len;
3699     mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false,
3700                                         MEMTXATTRS_UNSPECIFIED);
3701     return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED,
3702                                               buf, len, mr_addr, l, mr);
3703 }
3704 
3705 /* Called from RCU critical section. address_space_write_cached uses this
3706  * out of line function when the target is an MMIO or IOMMU region.
3707  */
3708 MemTxResult
address_space_write_cached_slow(MemoryRegionCache * cache,hwaddr addr,const void * buf,hwaddr len)3709 address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3710                                     const void *buf, hwaddr len)
3711 {
3712     hwaddr mr_addr, l;
3713     MemoryRegion *mr;
3714 
3715     l = len;
3716     mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true,
3717                                         MEMTXATTRS_UNSPECIFIED);
3718     return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED,
3719                                                buf, len, mr_addr, l, mr);
3720 }
3721 
3722 #define ARG1_DECL                MemoryRegionCache *cache
3723 #define ARG1                     cache
3724 #define SUFFIX                   _cached_slow
3725 #define TRANSLATE(...)           address_space_translate_cached(cache, __VA_ARGS__)
3726 #define RCU_READ_LOCK()          ((void)0)
3727 #define RCU_READ_UNLOCK()        ((void)0)
3728 #include "memory_ldst.c.inc"
3729 
3730 /* virtual memory access for debug (includes writing to ROM) */
cpu_memory_rw_debug(CPUState * cpu,vaddr addr,void * ptr,size_t len,bool is_write)3731 int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
3732                         void *ptr, size_t len, bool is_write)
3733 {
3734     hwaddr phys_addr;
3735     vaddr l, page;
3736     uint8_t *buf = ptr;
3737 
3738     cpu_synchronize_state(cpu);
3739     while (len > 0) {
3740         int asidx;
3741         MemTxAttrs attrs;
3742         MemTxResult res;
3743 
3744         page = addr & TARGET_PAGE_MASK;
3745         phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3746         asidx = cpu_asidx_from_attrs(cpu, attrs);
3747         /* if no physical page mapped, return an error */
3748         if (phys_addr == -1)
3749             return -1;
3750         l = (page + TARGET_PAGE_SIZE) - addr;
3751         if (l > len)
3752             l = len;
3753         phys_addr += (addr & ~TARGET_PAGE_MASK);
3754         res = address_space_rw(cpu->cpu_ases[asidx].as, phys_addr, attrs, buf,
3755                                l, is_write);
3756         if (res != MEMTX_OK) {
3757             return -1;
3758         }
3759         len -= l;
3760         buf += l;
3761         addr += l;
3762     }
3763     return 0;
3764 }
3765 
cpu_physical_memory_is_io(hwaddr phys_addr)3766 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3767 {
3768     MemoryRegion*mr;
3769     hwaddr l = 1;
3770 
3771     RCU_READ_LOCK_GUARD();
3772     mr = address_space_translate(&address_space_memory,
3773                                  phys_addr, &phys_addr, &l, false,
3774                                  MEMTXATTRS_UNSPECIFIED);
3775 
3776     return !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3777 }
3778 
qemu_ram_foreach_block(RAMBlockIterFunc func,void * opaque)3779 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3780 {
3781     RAMBlock *block;
3782     int ret = 0;
3783 
3784     RCU_READ_LOCK_GUARD();
3785     RAMBLOCK_FOREACH(block) {
3786         ret = func(block, opaque);
3787         if (ret) {
3788             break;
3789         }
3790     }
3791     return ret;
3792 }
3793 
3794 /*
3795  * Unmap pages of memory from start to start+length such that
3796  * they a) read as 0, b) Trigger whatever fault mechanism
3797  * the OS provides for postcopy.
3798  * The pages must be unmapped by the end of the function.
3799  * Returns: 0 on success, none-0 on failure
3800  *
3801  */
ram_block_discard_range(RAMBlock * rb,uint64_t start,size_t length)3802 int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3803 {
3804     int ret = -1;
3805 
3806     uint8_t *host_startaddr = rb->host + start;
3807 
3808     if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
3809         error_report("%s: Unaligned start address: %p",
3810                      __func__, host_startaddr);
3811         goto err;
3812     }
3813 
3814     if ((start + length) <= rb->max_length) {
3815         bool need_madvise, need_fallocate;
3816         if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
3817             error_report("%s: Unaligned length: %zx", __func__, length);
3818             goto err;
3819         }
3820 
3821         errno = ENOTSUP; /* If we are missing MADVISE etc */
3822 
3823         /* The logic here is messy;
3824          *    madvise DONTNEED fails for hugepages
3825          *    fallocate works on hugepages and shmem
3826          *    shared anonymous memory requires madvise REMOVE
3827          */
3828         need_madvise = (rb->page_size == qemu_real_host_page_size());
3829         need_fallocate = rb->fd != -1;
3830         if (need_fallocate) {
3831             /* For a file, this causes the area of the file to be zero'd
3832              * if read, and for hugetlbfs also causes it to be unmapped
3833              * so a userfault will trigger.
3834              */
3835 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3836             /*
3837              * fallocate() will fail with readonly files. Let's print a
3838              * proper error message.
3839              */
3840             if (rb->flags & RAM_READONLY_FD) {
3841                 error_report("%s: Discarding RAM with readonly files is not"
3842                              " supported", __func__);
3843                 goto err;
3844 
3845             }
3846             /*
3847              * We'll discard data from the actual file, even though we only
3848              * have a MAP_PRIVATE mapping, possibly messing with other
3849              * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
3850              * change that behavior whithout violating the promised
3851              * semantics of ram_block_discard_range().
3852              *
3853              * Only warn, because it works as long as nobody else uses that
3854              * file.
3855              */
3856             if (!qemu_ram_is_shared(rb)) {
3857                 warn_report_once("%s: Discarding RAM"
3858                                  " in private file mappings is possibly"
3859                                  " dangerous, because it will modify the"
3860                                  " underlying file and will affect other"
3861                                  " users of the file", __func__);
3862             }
3863 
3864             ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3865                             start + rb->fd_offset, length);
3866             if (ret) {
3867                 ret = -errno;
3868                 error_report("%s: Failed to fallocate %s:%" PRIx64 "+%" PRIx64
3869                              " +%zx (%d)", __func__, rb->idstr, start,
3870                              rb->fd_offset, length, ret);
3871                 goto err;
3872             }
3873 #else
3874             ret = -ENOSYS;
3875             error_report("%s: fallocate not available/file"
3876                          "%s:%" PRIx64 "+%" PRIx64 " +%zx (%d)", __func__,
3877                          rb->idstr, start, rb->fd_offset, length, ret);
3878             goto err;
3879 #endif
3880         }
3881         if (need_madvise) {
3882             /* For normal RAM this causes it to be unmapped,
3883              * for shared memory it causes the local mapping to disappear
3884              * and to fall back on the file contents (which we just
3885              * fallocate'd away).
3886              */
3887 #if defined(CONFIG_MADVISE)
3888             if (qemu_ram_is_shared(rb) && rb->fd < 0) {
3889                 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
3890             } else {
3891                 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
3892             }
3893             if (ret) {
3894                 ret = -errno;
3895                 error_report("%s: Failed to discard range "
3896                              "%s:%" PRIx64 " +%zx (%d)",
3897                              __func__, rb->idstr, start, length, ret);
3898                 goto err;
3899             }
3900 #else
3901             ret = -ENOSYS;
3902             error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)",
3903                          __func__, rb->idstr, start, length, ret);
3904             goto err;
3905 #endif
3906         }
3907         trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
3908                                       need_madvise, need_fallocate, ret);
3909     } else {
3910         error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")",
3911                      __func__, rb->idstr, start, length, rb->max_length);
3912     }
3913 
3914 err:
3915     return ret;
3916 }
3917 
ram_block_discard_guest_memfd_range(RAMBlock * rb,uint64_t start,size_t length)3918 int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start,
3919                                         size_t length)
3920 {
3921     int ret = -1;
3922 
3923 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3924     /* ignore fd_offset with guest_memfd */
3925     ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3926                     start, length);
3927 
3928     if (ret) {
3929         ret = -errno;
3930         error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3931                      __func__, rb->idstr, start, length, ret);
3932     }
3933 #else
3934     ret = -ENOSYS;
3935     error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)",
3936                  __func__, rb->idstr, start, length, ret);
3937 #endif
3938 
3939     return ret;
3940 }
3941 
ramblock_is_pmem(RAMBlock * rb)3942 bool ramblock_is_pmem(RAMBlock *rb)
3943 {
3944     return rb->flags & RAM_PMEM;
3945 }
3946 
mtree_print_phys_entries(int start,int end,int skip,int ptr)3947 static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
3948 {
3949     if (start == end - 1) {
3950         qemu_printf("\t%3d      ", start);
3951     } else {
3952         qemu_printf("\t%3d..%-3d ", start, end - 1);
3953     }
3954     qemu_printf(" skip=%d ", skip);
3955     if (ptr == PHYS_MAP_NODE_NIL) {
3956         qemu_printf(" ptr=NIL");
3957     } else if (!skip) {
3958         qemu_printf(" ptr=#%d", ptr);
3959     } else {
3960         qemu_printf(" ptr=[%d]", ptr);
3961     }
3962     qemu_printf("\n");
3963 }
3964 
3965 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3966                            int128_sub((size), int128_one())) : 0)
3967 
mtree_print_dispatch(AddressSpaceDispatch * d,MemoryRegion * root)3968 void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
3969 {
3970     int i;
3971 
3972     qemu_printf("  Dispatch\n");
3973     qemu_printf("    Physical sections\n");
3974 
3975     for (i = 0; i < d->map.sections_nb; ++i) {
3976         MemoryRegionSection *s = d->map.sections + i;
3977         const char *names[] = { " [unassigned]", " [not dirty]",
3978                                 " [ROM]", " [watch]" };
3979 
3980         qemu_printf("      #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
3981                     " %s%s%s%s%s",
3982             i,
3983             s->offset_within_address_space,
3984             s->offset_within_address_space + MR_SIZE(s->size),
3985             s->mr->name ? s->mr->name : "(noname)",
3986             i < ARRAY_SIZE(names) ? names[i] : "",
3987             s->mr == root ? " [ROOT]" : "",
3988             s == d->mru_section ? " [MRU]" : "",
3989             s->mr->is_iommu ? " [iommu]" : "");
3990 
3991         if (s->mr->alias) {
3992             qemu_printf(" alias=%s", s->mr->alias->name ?
3993                     s->mr->alias->name : "noname");
3994         }
3995         qemu_printf("\n");
3996     }
3997 
3998     qemu_printf("    Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
3999                P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
4000     for (i = 0; i < d->map.nodes_nb; ++i) {
4001         int j, jprev;
4002         PhysPageEntry prev;
4003         Node *n = d->map.nodes + i;
4004 
4005         qemu_printf("      [%d]\n", i);
4006 
4007         for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
4008             PhysPageEntry *pe = *n + j;
4009 
4010             if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
4011                 continue;
4012             }
4013 
4014             mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4015 
4016             jprev = j;
4017             prev = *pe;
4018         }
4019 
4020         if (jprev != ARRAY_SIZE(*n)) {
4021             mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4022         }
4023     }
4024 }
4025 
4026 /* Require any discards to work. */
4027 static unsigned int ram_block_discard_required_cnt;
4028 /* Require only coordinated discards to work. */
4029 static unsigned int ram_block_coordinated_discard_required_cnt;
4030 /* Disable any discards. */
4031 static unsigned int ram_block_discard_disabled_cnt;
4032 /* Disable only uncoordinated discards. */
4033 static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
4034 static QemuMutex ram_block_discard_disable_mutex;
4035 
ram_block_discard_disable_mutex_lock(void)4036 static void ram_block_discard_disable_mutex_lock(void)
4037 {
4038     static gsize initialized;
4039 
4040     if (g_once_init_enter(&initialized)) {
4041         qemu_mutex_init(&ram_block_discard_disable_mutex);
4042         g_once_init_leave(&initialized, 1);
4043     }
4044     qemu_mutex_lock(&ram_block_discard_disable_mutex);
4045 }
4046 
ram_block_discard_disable_mutex_unlock(void)4047 static void ram_block_discard_disable_mutex_unlock(void)
4048 {
4049     qemu_mutex_unlock(&ram_block_discard_disable_mutex);
4050 }
4051 
ram_block_discard_disable(bool state)4052 int ram_block_discard_disable(bool state)
4053 {
4054     int ret = 0;
4055 
4056     ram_block_discard_disable_mutex_lock();
4057     if (!state) {
4058         ram_block_discard_disabled_cnt--;
4059     } else if (ram_block_discard_required_cnt ||
4060                ram_block_coordinated_discard_required_cnt) {
4061         ret = -EBUSY;
4062     } else {
4063         ram_block_discard_disabled_cnt++;
4064     }
4065     ram_block_discard_disable_mutex_unlock();
4066     return ret;
4067 }
4068 
ram_block_uncoordinated_discard_disable(bool state)4069 int ram_block_uncoordinated_discard_disable(bool state)
4070 {
4071     int ret = 0;
4072 
4073     ram_block_discard_disable_mutex_lock();
4074     if (!state) {
4075         ram_block_uncoordinated_discard_disabled_cnt--;
4076     } else if (ram_block_discard_required_cnt) {
4077         ret = -EBUSY;
4078     } else {
4079         ram_block_uncoordinated_discard_disabled_cnt++;
4080     }
4081     ram_block_discard_disable_mutex_unlock();
4082     return ret;
4083 }
4084 
ram_block_discard_require(bool state)4085 int ram_block_discard_require(bool state)
4086 {
4087     int ret = 0;
4088 
4089     ram_block_discard_disable_mutex_lock();
4090     if (!state) {
4091         ram_block_discard_required_cnt--;
4092     } else if (ram_block_discard_disabled_cnt ||
4093                ram_block_uncoordinated_discard_disabled_cnt) {
4094         ret = -EBUSY;
4095     } else {
4096         ram_block_discard_required_cnt++;
4097     }
4098     ram_block_discard_disable_mutex_unlock();
4099     return ret;
4100 }
4101 
ram_block_coordinated_discard_require(bool state)4102 int ram_block_coordinated_discard_require(bool state)
4103 {
4104     int ret = 0;
4105 
4106     ram_block_discard_disable_mutex_lock();
4107     if (!state) {
4108         ram_block_coordinated_discard_required_cnt--;
4109     } else if (ram_block_discard_disabled_cnt) {
4110         ret = -EBUSY;
4111     } else {
4112         ram_block_coordinated_discard_required_cnt++;
4113     }
4114     ram_block_discard_disable_mutex_unlock();
4115     return ret;
4116 }
4117 
ram_block_discard_is_disabled(void)4118 bool ram_block_discard_is_disabled(void)
4119 {
4120     return qatomic_read(&ram_block_discard_disabled_cnt) ||
4121            qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
4122 }
4123 
ram_block_discard_is_required(void)4124 bool ram_block_discard_is_required(void)
4125 {
4126     return qatomic_read(&ram_block_discard_required_cnt) ||
4127            qatomic_read(&ram_block_coordinated_discard_required_cnt);
4128 }
4129 
4130 /*
4131  * Return true if ram is compatible with CPR.  Do not exclude rom,
4132  * because the rom file could change in new QEMU.
4133  */
ram_is_cpr_compatible(RAMBlock * rb)4134 static bool ram_is_cpr_compatible(RAMBlock *rb)
4135 {
4136     MemoryRegion *mr = rb->mr;
4137 
4138     if (!mr || !memory_region_is_ram(mr)) {
4139         return true;
4140     }
4141 
4142     /* Ram device is remapped in new QEMU */
4143     if (memory_region_is_ram_device(mr)) {
4144         return true;
4145     }
4146 
4147     /*
4148      * A file descriptor is passed to new QEMU and remapped, or its backing
4149      * file is reopened and mapped.  It must be shared to avoid COW.
4150      */
4151     if (rb->fd >= 0 && qemu_ram_is_shared(rb)) {
4152         return true;
4153     }
4154 
4155     return false;
4156 }
4157 
4158 /*
4159  * Add a blocker for each volatile ram block.  This function should only be
4160  * called after we know that the block is migratable.  Non-migratable blocks
4161  * are either re-created in new QEMU, or are handled specially, or are covered
4162  * by a device-level CPR blocker.
4163  */
ram_block_add_cpr_blocker(RAMBlock * rb,Error ** errp)4164 void ram_block_add_cpr_blocker(RAMBlock *rb, Error **errp)
4165 {
4166     assert(qemu_ram_is_migratable(rb));
4167 
4168     if (ram_is_cpr_compatible(rb)) {
4169         return;
4170     }
4171 
4172     error_setg(&rb->cpr_blocker,
4173                "Memory region %s is not compatible with CPR. share=on is "
4174                "required for memory-backend objects, and aux-ram-share=on is "
4175                "required.", memory_region_name(rb->mr));
4176     migrate_add_blocker_modes(&rb->cpr_blocker, errp, MIG_MODE_CPR_TRANSFER,
4177                               -1);
4178 }
4179 
ram_block_del_cpr_blocker(RAMBlock * rb)4180 void ram_block_del_cpr_blocker(RAMBlock *rb)
4181 {
4182     migrate_del_blocker(&rb->cpr_blocker);
4183 }
4184