1 /*
2 * RAM allocation and memory access
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "exec/page-vary.h"
22 #include "qapi/error.h"
23
24 #include "qemu/cutils.h"
25 #include "qemu/cacheflush.h"
26 #include "qemu/hbitmap.h"
27 #include "qemu/madvise.h"
28 #include "qemu/lockable.h"
29
30 #ifdef CONFIG_TCG
31 #include "accel/tcg/cpu-ops.h"
32 #include "accel/tcg/iommu.h"
33 #endif /* CONFIG_TCG */
34
35 #include "exec/cputlb.h"
36 #include "exec/page-protection.h"
37 #include "exec/target_page.h"
38 #include "exec/translation-block.h"
39 #include "hw/qdev-core.h"
40 #include "hw/qdev-properties.h"
41 #include "hw/boards.h"
42 #include "system/xen.h"
43 #include "system/kvm.h"
44 #include "system/tcg.h"
45 #include "system/qtest.h"
46 #include "qemu/timer.h"
47 #include "qemu/config-file.h"
48 #include "qemu/error-report.h"
49 #include "qemu/qemu-print.h"
50 #include "qemu/log.h"
51 #include "qemu/memalign.h"
52 #include "qemu/memfd.h"
53 #include "system/memory.h"
54 #include "system/ioport.h"
55 #include "system/dma.h"
56 #include "system/hostmem.h"
57 #include "system/hw_accel.h"
58 #include "system/xen-mapcache.h"
59 #include "trace.h"
60
61 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
62 #include <linux/falloc.h>
63 #endif
64
65 #include "qemu/rcu_queue.h"
66 #include "qemu/main-loop.h"
67 #include "system/replay.h"
68
69 #include "system/ram_addr.h"
70
71 #include "qemu/pmem.h"
72
73 #include "qapi/qapi-types-migration.h"
74 #include "migration/blocker.h"
75 #include "migration/cpr.h"
76 #include "migration/options.h"
77 #include "migration/vmstate.h"
78
79 #include "qemu/range.h"
80 #ifndef _WIN32
81 #include "qemu/mmap-alloc.h"
82 #endif
83
84 #include "monitor/monitor.h"
85
86 #ifdef CONFIG_LIBDAXCTL
87 #include <daxctl/libdaxctl.h>
88 #endif
89
90 #include "memory-internal.h"
91
92 //#define DEBUG_SUBPAGE
93
94 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
95 * are protected by the ramlist lock.
96 */
97 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
98
99 static MemoryRegion *system_memory;
100 static MemoryRegion *system_io;
101
102 AddressSpace address_space_io;
103 AddressSpace address_space_memory;
104
105 static MemoryRegion io_mem_unassigned;
106
107 typedef struct PhysPageEntry PhysPageEntry;
108
109 struct PhysPageEntry {
110 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
111 uint32_t skip : 6;
112 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
113 uint32_t ptr : 26;
114 };
115
116 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
117
118 /* Size of the L2 (and L3, etc) page tables. */
119 #define ADDR_SPACE_BITS 64
120
121 #define P_L2_BITS 9
122 #define P_L2_SIZE (1 << P_L2_BITS)
123
124 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
125
126 typedef PhysPageEntry Node[P_L2_SIZE];
127
128 typedef struct PhysPageMap {
129 struct rcu_head rcu;
130
131 unsigned sections_nb;
132 unsigned sections_nb_alloc;
133 unsigned nodes_nb;
134 unsigned nodes_nb_alloc;
135 Node *nodes;
136 MemoryRegionSection *sections;
137 } PhysPageMap;
138
139 struct AddressSpaceDispatch {
140 MemoryRegionSection *mru_section;
141 /* This is a multi-level map on the physical address space.
142 * The bottom level has pointers to MemoryRegionSections.
143 */
144 PhysPageEntry phys_map;
145 PhysPageMap map;
146 };
147
148 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
149 typedef struct subpage_t {
150 MemoryRegion iomem;
151 FlatView *fv;
152 hwaddr base;
153 uint16_t sub_section[];
154 } subpage_t;
155
156 #define PHYS_SECTION_UNASSIGNED 0
157
158 static void io_mem_init(void);
159 static void memory_map_init(void);
160 static void tcg_log_global_after_sync(MemoryListener *listener);
161 static void tcg_commit(MemoryListener *listener);
162 static bool ram_is_cpr_compatible(RAMBlock *rb);
163
164 /**
165 * CPUAddressSpace: all the information a CPU needs about an AddressSpace
166 * @cpu: the CPU whose AddressSpace this is
167 * @as: the AddressSpace itself
168 * @tcg_as_listener: listener for tracking changes to the AddressSpace
169 */
170 typedef struct CPUAddressSpace {
171 CPUState *cpu;
172 AddressSpace *as;
173 MemoryListener tcg_as_listener;
174 } CPUAddressSpace;
175
176 struct DirtyBitmapSnapshot {
177 ram_addr_t start;
178 ram_addr_t end;
179 unsigned long dirty[];
180 };
181
phys_map_node_reserve(PhysPageMap * map,unsigned nodes)182 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
183 {
184 static unsigned alloc_hint = 16;
185 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
186 map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
187 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
188 alloc_hint = map->nodes_nb_alloc;
189 }
190 }
191
phys_map_node_alloc(PhysPageMap * map,bool leaf)192 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
193 {
194 unsigned i;
195 uint32_t ret;
196 PhysPageEntry e;
197 PhysPageEntry *p;
198
199 ret = map->nodes_nb++;
200 p = map->nodes[ret];
201 assert(ret != PHYS_MAP_NODE_NIL);
202 assert(ret != map->nodes_nb_alloc);
203
204 e.skip = leaf ? 0 : 1;
205 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
206 for (i = 0; i < P_L2_SIZE; ++i) {
207 memcpy(&p[i], &e, sizeof(e));
208 }
209 return ret;
210 }
211
phys_page_set_level(PhysPageMap * map,PhysPageEntry * lp,hwaddr * index,uint64_t * nb,uint16_t leaf,int level)212 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
213 hwaddr *index, uint64_t *nb, uint16_t leaf,
214 int level)
215 {
216 PhysPageEntry *p;
217 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
218
219 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
220 lp->ptr = phys_map_node_alloc(map, level == 0);
221 }
222 p = map->nodes[lp->ptr];
223 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
224
225 while (*nb && lp < &p[P_L2_SIZE]) {
226 if ((*index & (step - 1)) == 0 && *nb >= step) {
227 lp->skip = 0;
228 lp->ptr = leaf;
229 *index += step;
230 *nb -= step;
231 } else {
232 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
233 }
234 ++lp;
235 }
236 }
237
phys_page_set(AddressSpaceDispatch * d,hwaddr index,uint64_t nb,uint16_t leaf)238 static void phys_page_set(AddressSpaceDispatch *d,
239 hwaddr index, uint64_t nb,
240 uint16_t leaf)
241 {
242 /* Wildly overreserve - it doesn't matter much. */
243 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
244
245 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
246 }
247
248 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
249 * and update our entry so we can skip it and go directly to the destination.
250 */
phys_page_compact(PhysPageEntry * lp,Node * nodes)251 static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
252 {
253 unsigned valid_ptr = P_L2_SIZE;
254 int valid = 0;
255 PhysPageEntry *p;
256 int i;
257
258 if (lp->ptr == PHYS_MAP_NODE_NIL) {
259 return;
260 }
261
262 p = nodes[lp->ptr];
263 for (i = 0; i < P_L2_SIZE; i++) {
264 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
265 continue;
266 }
267
268 valid_ptr = i;
269 valid++;
270 if (p[i].skip) {
271 phys_page_compact(&p[i], nodes);
272 }
273 }
274
275 /* We can only compress if there's only one child. */
276 if (valid != 1) {
277 return;
278 }
279
280 assert(valid_ptr < P_L2_SIZE);
281
282 /* Don't compress if it won't fit in the # of bits we have. */
283 if (P_L2_LEVELS >= (1 << 6) &&
284 lp->skip + p[valid_ptr].skip >= (1 << 6)) {
285 return;
286 }
287
288 lp->ptr = p[valid_ptr].ptr;
289 if (!p[valid_ptr].skip) {
290 /* If our only child is a leaf, make this a leaf. */
291 /* By design, we should have made this node a leaf to begin with so we
292 * should never reach here.
293 * But since it's so simple to handle this, let's do it just in case we
294 * change this rule.
295 */
296 lp->skip = 0;
297 } else {
298 lp->skip += p[valid_ptr].skip;
299 }
300 }
301
address_space_dispatch_compact(AddressSpaceDispatch * d)302 void address_space_dispatch_compact(AddressSpaceDispatch *d)
303 {
304 if (d->phys_map.skip) {
305 phys_page_compact(&d->phys_map, d->map.nodes);
306 }
307 }
308
section_covers_addr(const MemoryRegionSection * section,hwaddr addr)309 static inline bool section_covers_addr(const MemoryRegionSection *section,
310 hwaddr addr)
311 {
312 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
313 * the section must cover the entire address space.
314 */
315 return int128_gethi(section->size) ||
316 range_covers_byte(section->offset_within_address_space,
317 int128_getlo(section->size), addr);
318 }
319
phys_page_find(AddressSpaceDispatch * d,hwaddr addr)320 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
321 {
322 PhysPageEntry lp = d->phys_map, *p;
323 Node *nodes = d->map.nodes;
324 MemoryRegionSection *sections = d->map.sections;
325 hwaddr index = addr >> TARGET_PAGE_BITS;
326 int i;
327
328 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
329 if (lp.ptr == PHYS_MAP_NODE_NIL) {
330 return §ions[PHYS_SECTION_UNASSIGNED];
331 }
332 p = nodes[lp.ptr];
333 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
334 }
335
336 if (section_covers_addr(§ions[lp.ptr], addr)) {
337 return §ions[lp.ptr];
338 } else {
339 return §ions[PHYS_SECTION_UNASSIGNED];
340 }
341 }
342
343 /* Called from RCU critical section */
address_space_lookup_region(AddressSpaceDispatch * d,hwaddr addr,bool resolve_subpage)344 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
345 hwaddr addr,
346 bool resolve_subpage)
347 {
348 MemoryRegionSection *section = qatomic_read(&d->mru_section);
349 subpage_t *subpage;
350
351 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
352 !section_covers_addr(section, addr)) {
353 section = phys_page_find(d, addr);
354 qatomic_set(&d->mru_section, section);
355 }
356 if (resolve_subpage && section->mr->subpage) {
357 subpage = container_of(section->mr, subpage_t, iomem);
358 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
359 }
360 return section;
361 }
362
363 /* Called from RCU critical section */
364 static MemoryRegionSection *
address_space_translate_internal(AddressSpaceDispatch * d,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool resolve_subpage)365 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
366 hwaddr *plen, bool resolve_subpage)
367 {
368 MemoryRegionSection *section;
369 MemoryRegion *mr;
370 Int128 diff;
371
372 section = address_space_lookup_region(d, addr, resolve_subpage);
373 /* Compute offset within MemoryRegionSection */
374 addr -= section->offset_within_address_space;
375
376 /* Compute offset within MemoryRegion */
377 *xlat = addr + section->offset_within_region;
378
379 mr = section->mr;
380
381 /* MMIO registers can be expected to perform full-width accesses based only
382 * on their address, without considering adjacent registers that could
383 * decode to completely different MemoryRegions. When such registers
384 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
385 * regions overlap wildly. For this reason we cannot clamp the accesses
386 * here.
387 *
388 * If the length is small (as is the case for address_space_ldl/stl),
389 * everything works fine. If the incoming length is large, however,
390 * the caller really has to do the clamping through memory_access_size.
391 */
392 if (memory_region_is_ram(mr)) {
393 diff = int128_sub(section->size, int128_make64(addr));
394 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
395 }
396 return section;
397 }
398
399 /**
400 * address_space_translate_iommu - translate an address through an IOMMU
401 * memory region and then through the target address space.
402 *
403 * @iommu_mr: the IOMMU memory region that we start the translation from
404 * @addr: the address to be translated through the MMU
405 * @xlat: the translated address offset within the destination memory region.
406 * It cannot be %NULL.
407 * @plen_out: valid read/write length of the translated address. It
408 * cannot be %NULL.
409 * @page_mask_out: page mask for the translated address. This
410 * should only be meaningful for IOMMU translated
411 * addresses, since there may be huge pages that this bit
412 * would tell. It can be %NULL if we don't care about it.
413 * @is_write: whether the translation operation is for write
414 * @is_mmio: whether this can be MMIO, set true if it can
415 * @target_as: the address space targeted by the IOMMU
416 * @attrs: transaction attributes
417 *
418 * This function is called from RCU critical section. It is the common
419 * part of flatview_do_translate and address_space_translate_cached.
420 */
address_space_translate_iommu(IOMMUMemoryRegion * iommu_mr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)421 static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
422 hwaddr *xlat,
423 hwaddr *plen_out,
424 hwaddr *page_mask_out,
425 bool is_write,
426 bool is_mmio,
427 AddressSpace **target_as,
428 MemTxAttrs attrs)
429 {
430 MemoryRegionSection *section;
431 hwaddr page_mask = (hwaddr)-1;
432
433 do {
434 hwaddr addr = *xlat;
435 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
436 int iommu_idx = 0;
437 IOMMUTLBEntry iotlb;
438
439 if (imrc->attrs_to_index) {
440 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
441 }
442
443 iotlb = imrc->translate(iommu_mr, addr, is_write ?
444 IOMMU_WO : IOMMU_RO, iommu_idx);
445
446 if (!(iotlb.perm & (1 << is_write))) {
447 goto unassigned;
448 }
449
450 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
451 | (addr & iotlb.addr_mask));
452 page_mask &= iotlb.addr_mask;
453 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
454 *target_as = iotlb.target_as;
455
456 section = address_space_translate_internal(
457 address_space_to_dispatch(iotlb.target_as), addr, xlat,
458 plen_out, is_mmio);
459
460 iommu_mr = memory_region_get_iommu(section->mr);
461 } while (unlikely(iommu_mr));
462
463 if (page_mask_out) {
464 *page_mask_out = page_mask;
465 }
466 return *section;
467
468 unassigned:
469 return (MemoryRegionSection) { .mr = &io_mem_unassigned };
470 }
471
472 /**
473 * flatview_do_translate - translate an address in FlatView
474 *
475 * @fv: the flat view that we want to translate on
476 * @addr: the address to be translated in above address space
477 * @xlat: the translated address offset within memory region. It
478 * cannot be @NULL.
479 * @plen_out: valid read/write length of the translated address. It
480 * can be @NULL when we don't care about it.
481 * @page_mask_out: page mask for the translated address. This
482 * should only be meaningful for IOMMU translated
483 * addresses, since there may be huge pages that this bit
484 * would tell. It can be @NULL if we don't care about it.
485 * @is_write: whether the translation operation is for write
486 * @is_mmio: whether this can be MMIO, set true if it can
487 * @target_as: the address space targeted by the IOMMU
488 * @attrs: memory transaction attributes
489 *
490 * This function is called from RCU critical section
491 */
flatview_do_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)492 static MemoryRegionSection flatview_do_translate(FlatView *fv,
493 hwaddr addr,
494 hwaddr *xlat,
495 hwaddr *plen_out,
496 hwaddr *page_mask_out,
497 bool is_write,
498 bool is_mmio,
499 AddressSpace **target_as,
500 MemTxAttrs attrs)
501 {
502 MemoryRegionSection *section;
503 IOMMUMemoryRegion *iommu_mr;
504 hwaddr plen = (hwaddr)(-1);
505
506 if (!plen_out) {
507 plen_out = &plen;
508 }
509
510 section = address_space_translate_internal(
511 flatview_to_dispatch(fv), addr, xlat,
512 plen_out, is_mmio);
513
514 iommu_mr = memory_region_get_iommu(section->mr);
515 if (unlikely(iommu_mr)) {
516 return address_space_translate_iommu(iommu_mr, xlat,
517 plen_out, page_mask_out,
518 is_write, is_mmio,
519 target_as, attrs);
520 }
521 if (page_mask_out) {
522 /* Not behind an IOMMU, use default page size. */
523 *page_mask_out = ~TARGET_PAGE_MASK;
524 }
525
526 return *section;
527 }
528
529 /* Called from RCU critical section */
address_space_get_iotlb_entry(AddressSpace * as,hwaddr addr,bool is_write,MemTxAttrs attrs)530 IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
531 bool is_write, MemTxAttrs attrs)
532 {
533 MemoryRegionSection section;
534 hwaddr xlat, page_mask;
535
536 /*
537 * This can never be MMIO, and we don't really care about plen,
538 * but page mask.
539 */
540 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
541 NULL, &page_mask, is_write, false, &as,
542 attrs);
543
544 /* Illegal translation */
545 if (section.mr == &io_mem_unassigned) {
546 goto iotlb_fail;
547 }
548
549 /* Convert memory region offset into address space offset */
550 xlat += section.offset_within_address_space -
551 section.offset_within_region;
552
553 return (IOMMUTLBEntry) {
554 .target_as = as,
555 .iova = addr & ~page_mask,
556 .translated_addr = xlat & ~page_mask,
557 .addr_mask = page_mask,
558 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
559 .perm = IOMMU_RW,
560 };
561
562 iotlb_fail:
563 return (IOMMUTLBEntry) {0};
564 }
565
566 /* Called from RCU critical section */
flatview_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)567 MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
568 hwaddr *plen, bool is_write,
569 MemTxAttrs attrs)
570 {
571 MemoryRegion *mr;
572 MemoryRegionSection section;
573 AddressSpace *as = NULL;
574
575 /* This can be MMIO, so setup MMIO bit. */
576 section = flatview_do_translate(fv, addr, xlat, plen, NULL,
577 is_write, true, &as, attrs);
578 mr = section.mr;
579
580 if (xen_enabled() && memory_access_is_direct(mr, is_write, attrs)) {
581 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
582 *plen = MIN(page, *plen);
583 }
584
585 return mr;
586 }
587
588 #ifdef CONFIG_TCG
589
590 typedef struct TCGIOMMUNotifier {
591 IOMMUNotifier n;
592 MemoryRegion *mr;
593 CPUState *cpu;
594 int iommu_idx;
595 bool active;
596 } TCGIOMMUNotifier;
597
tcg_iommu_unmap_notify(IOMMUNotifier * n,IOMMUTLBEntry * iotlb)598 static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
599 {
600 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
601
602 if (!notifier->active) {
603 return;
604 }
605 tlb_flush(notifier->cpu);
606 notifier->active = false;
607 /* We leave the notifier struct on the list to avoid reallocating it later.
608 * Generally the number of IOMMUs a CPU deals with will be small.
609 * In any case we can't unregister the iommu notifier from a notify
610 * callback.
611 */
612 }
613
tcg_register_iommu_notifier(CPUState * cpu,IOMMUMemoryRegion * iommu_mr,int iommu_idx)614 static void tcg_register_iommu_notifier(CPUState *cpu,
615 IOMMUMemoryRegion *iommu_mr,
616 int iommu_idx)
617 {
618 /* Make sure this CPU has an IOMMU notifier registered for this
619 * IOMMU/IOMMU index combination, so that we can flush its TLB
620 * when the IOMMU tells us the mappings we've cached have changed.
621 */
622 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
623 TCGIOMMUNotifier *notifier = NULL;
624 int i;
625
626 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
627 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
628 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
629 break;
630 }
631 }
632 if (i == cpu->iommu_notifiers->len) {
633 /* Not found, add a new entry at the end of the array */
634 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
635 notifier = g_new0(TCGIOMMUNotifier, 1);
636 g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
637
638 notifier->mr = mr;
639 notifier->iommu_idx = iommu_idx;
640 notifier->cpu = cpu;
641 /* Rather than trying to register interest in the specific part
642 * of the iommu's address space that we've accessed and then
643 * expand it later as subsequent accesses touch more of it, we
644 * just register interest in the whole thing, on the assumption
645 * that iommu reconfiguration will be rare.
646 */
647 iommu_notifier_init(¬ifier->n,
648 tcg_iommu_unmap_notify,
649 IOMMU_NOTIFIER_UNMAP,
650 0,
651 HWADDR_MAX,
652 iommu_idx);
653 memory_region_register_iommu_notifier(notifier->mr, ¬ifier->n,
654 &error_fatal);
655 }
656
657 if (!notifier->active) {
658 notifier->active = true;
659 }
660 }
661
tcg_iommu_free_notifier_list(CPUState * cpu)662 void tcg_iommu_free_notifier_list(CPUState *cpu)
663 {
664 /* Destroy the CPU's notifier list */
665 int i;
666 TCGIOMMUNotifier *notifier;
667
668 for (i = 0; i < cpu->iommu_notifiers->len; i++) {
669 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
670 memory_region_unregister_iommu_notifier(notifier->mr, ¬ifier->n);
671 g_free(notifier);
672 }
673 g_array_free(cpu->iommu_notifiers, true);
674 }
675
tcg_iommu_init_notifier_list(CPUState * cpu)676 void tcg_iommu_init_notifier_list(CPUState *cpu)
677 {
678 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
679 }
680
681 /* Called from RCU critical section */
682 MemoryRegionSection *
address_space_translate_for_iotlb(CPUState * cpu,int asidx,hwaddr orig_addr,hwaddr * xlat,hwaddr * plen,MemTxAttrs attrs,int * prot)683 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
684 hwaddr *xlat, hwaddr *plen,
685 MemTxAttrs attrs, int *prot)
686 {
687 MemoryRegionSection *section;
688 IOMMUMemoryRegion *iommu_mr;
689 IOMMUMemoryRegionClass *imrc;
690 IOMMUTLBEntry iotlb;
691 int iommu_idx;
692 hwaddr addr = orig_addr;
693 AddressSpaceDispatch *d = address_space_to_dispatch(cpu->cpu_ases[asidx].as);
694
695 for (;;) {
696 section = address_space_translate_internal(d, addr, &addr, plen, false);
697
698 iommu_mr = memory_region_get_iommu(section->mr);
699 if (!iommu_mr) {
700 break;
701 }
702
703 imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
704
705 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
706 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
707 /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
708 * doesn't short-cut its translation table walk.
709 */
710 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
711 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
712 | (addr & iotlb.addr_mask));
713 /* Update the caller's prot bits to remove permissions the IOMMU
714 * is giving us a failure response for. If we get down to no
715 * permissions left at all we can give up now.
716 */
717 if (!(iotlb.perm & IOMMU_RO)) {
718 *prot &= ~(PAGE_READ | PAGE_EXEC);
719 }
720 if (!(iotlb.perm & IOMMU_WO)) {
721 *prot &= ~PAGE_WRITE;
722 }
723
724 if (!*prot) {
725 goto translate_fail;
726 }
727
728 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
729 }
730
731 assert(!memory_region_is_iommu(section->mr));
732 *xlat = addr;
733 return section;
734
735 translate_fail:
736 /*
737 * We should be given a page-aligned address -- certainly
738 * tlb_set_page_with_attrs() does so. The page offset of xlat
739 * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
740 * The page portion of xlat will be logged by memory_region_access_valid()
741 * when this memory access is rejected, so use the original untranslated
742 * physical address.
743 */
744 assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
745 *xlat = orig_addr;
746 return &d->map.sections[PHYS_SECTION_UNASSIGNED];
747 }
748
iotlb_to_section(CPUState * cpu,hwaddr index,MemTxAttrs attrs)749 MemoryRegionSection *iotlb_to_section(CPUState *cpu,
750 hwaddr index, MemTxAttrs attrs)
751 {
752 int asidx = cpu_asidx_from_attrs(cpu, attrs);
753 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
754 AddressSpaceDispatch *d = address_space_to_dispatch(cpuas->as);
755 int section_index = index & ~TARGET_PAGE_MASK;
756 MemoryRegionSection *ret;
757
758 assert(section_index < d->map.sections_nb);
759 ret = d->map.sections + section_index;
760 assert(ret->mr);
761 assert(ret->mr->ops);
762
763 return ret;
764 }
765
766 /* Called from RCU critical section */
memory_region_section_get_iotlb(CPUState * cpu,MemoryRegionSection * section)767 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
768 MemoryRegionSection *section)
769 {
770 AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
771 return section - d->map.sections;
772 }
773
774 #endif /* CONFIG_TCG */
775
cpu_address_space_init(CPUState * cpu,int asidx,const char * prefix,MemoryRegion * mr)776 void cpu_address_space_init(CPUState *cpu, int asidx,
777 const char *prefix, MemoryRegion *mr)
778 {
779 CPUAddressSpace *newas;
780 AddressSpace *as = g_new0(AddressSpace, 1);
781 char *as_name;
782
783 assert(mr);
784 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
785 address_space_init(as, mr, as_name);
786 g_free(as_name);
787
788 /* Target code should have set num_ases before calling us */
789 assert(asidx < cpu->num_ases);
790
791 if (asidx == 0) {
792 /* address space 0 gets the convenience alias */
793 cpu->as = as;
794 }
795
796 if (!cpu->cpu_ases) {
797 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
798 }
799
800 newas = &cpu->cpu_ases[asidx];
801 newas->cpu = cpu;
802 newas->as = as;
803 if (tcg_enabled()) {
804 newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
805 newas->tcg_as_listener.commit = tcg_commit;
806 newas->tcg_as_listener.name = "tcg";
807 memory_listener_register(&newas->tcg_as_listener, as);
808 }
809 }
810
cpu_destroy_address_spaces(CPUState * cpu)811 void cpu_destroy_address_spaces(CPUState *cpu)
812 {
813 CPUAddressSpace *cpuas;
814 int asidx;
815
816 assert(cpu->cpu_ases);
817
818 /* convenience alias just points to some cpu_ases[n] */
819 cpu->as = NULL;
820
821 for (asidx = 0; asidx < cpu->num_ases; asidx++) {
822 cpuas = &cpu->cpu_ases[asidx];
823 if (!cpuas->as) {
824 /* This index was never initialized; no deinit needed */
825 continue;
826 }
827 if (tcg_enabled()) {
828 memory_listener_unregister(&cpuas->tcg_as_listener);
829 }
830 g_clear_pointer(&cpuas->as, address_space_destroy_free);
831 }
832
833 g_clear_pointer(&cpu->cpu_ases, g_free);
834 }
835
cpu_get_address_space(CPUState * cpu,int asidx)836 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
837 {
838 /* Return the AddressSpace corresponding to the specified index */
839 return cpu->cpu_ases[asidx].as;
840 }
841
842 /* Called from RCU critical section */
qemu_get_ram_block(ram_addr_t addr)843 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
844 {
845 RAMBlock *block;
846
847 block = qatomic_rcu_read(&ram_list.mru_block);
848 if (block && addr - block->offset < block->max_length) {
849 return block;
850 }
851 RAMBLOCK_FOREACH(block) {
852 if (addr - block->offset < block->max_length) {
853 goto found;
854 }
855 }
856
857 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
858 abort();
859
860 found:
861 /* It is safe to write mru_block outside the BQL. This
862 * is what happens:
863 *
864 * mru_block = xxx
865 * rcu_read_unlock()
866 * xxx removed from list
867 * rcu_read_lock()
868 * read mru_block
869 * mru_block = NULL;
870 * call_rcu(reclaim_ramblock, xxx);
871 * rcu_read_unlock()
872 *
873 * qatomic_rcu_set is not needed here. The block was already published
874 * when it was placed into the list. Here we're just making an extra
875 * copy of the pointer.
876 */
877 ram_list.mru_block = block;
878 return block;
879 }
880
tlb_reset_dirty_range_all(ram_addr_t start,ram_addr_t length)881 void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
882 {
883 CPUState *cpu;
884 ram_addr_t start1;
885 RAMBlock *block;
886 ram_addr_t end;
887
888 assert(tcg_enabled());
889 end = TARGET_PAGE_ALIGN(start + length);
890 start &= TARGET_PAGE_MASK;
891
892 RCU_READ_LOCK_GUARD();
893 block = qemu_get_ram_block(start);
894 assert(block == qemu_get_ram_block(end - 1));
895 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
896 CPU_FOREACH(cpu) {
897 tlb_reset_dirty(cpu, start1, length);
898 }
899 }
900
901 /* Note: start and end must be within the same ram block. */
cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,ram_addr_t length,unsigned client)902 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
903 ram_addr_t length,
904 unsigned client)
905 {
906 DirtyMemoryBlocks *blocks;
907 unsigned long end, page, start_page;
908 bool dirty = false;
909 RAMBlock *ramblock;
910 uint64_t mr_offset, mr_size;
911
912 if (length == 0) {
913 return false;
914 }
915
916 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
917 start_page = start >> TARGET_PAGE_BITS;
918 page = start_page;
919
920 WITH_RCU_READ_LOCK_GUARD() {
921 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
922 ramblock = qemu_get_ram_block(start);
923 /* Range sanity check on the ramblock */
924 assert(start >= ramblock->offset &&
925 start + length <= ramblock->offset + ramblock->used_length);
926
927 while (page < end) {
928 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
929 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
930 unsigned long num = MIN(end - page,
931 DIRTY_MEMORY_BLOCK_SIZE - offset);
932
933 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
934 offset, num);
935 page += num;
936 }
937
938 mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
939 mr_size = (end - start_page) << TARGET_PAGE_BITS;
940 memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
941 }
942
943 if (dirty) {
944 cpu_physical_memory_dirty_bits_cleared(start, length);
945 }
946
947 return dirty;
948 }
949
cpu_physical_memory_snapshot_and_clear_dirty(MemoryRegion * mr,hwaddr offset,hwaddr length,unsigned client)950 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
951 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
952 {
953 DirtyMemoryBlocks *blocks;
954 ram_addr_t start, first, last;
955 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
956 DirtyBitmapSnapshot *snap;
957 unsigned long page, end, dest;
958
959 start = memory_region_get_ram_addr(mr);
960 /* We know we're only called for RAM MemoryRegions */
961 assert(start != RAM_ADDR_INVALID);
962 start += offset;
963
964 first = QEMU_ALIGN_DOWN(start, align);
965 last = QEMU_ALIGN_UP(start + length, align);
966
967 snap = g_malloc0(sizeof(*snap) +
968 ((last - first) >> (TARGET_PAGE_BITS + 3)));
969 snap->start = first;
970 snap->end = last;
971
972 page = first >> TARGET_PAGE_BITS;
973 end = last >> TARGET_PAGE_BITS;
974 dest = 0;
975
976 WITH_RCU_READ_LOCK_GUARD() {
977 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
978
979 while (page < end) {
980 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
981 unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
982 unsigned long num = MIN(end - page,
983 DIRTY_MEMORY_BLOCK_SIZE - ofs);
984
985 assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
986 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL)));
987 ofs >>= BITS_PER_LEVEL;
988
989 bitmap_copy_and_clear_atomic(snap->dirty + dest,
990 blocks->blocks[idx] + ofs,
991 num);
992 page += num;
993 dest += num >> BITS_PER_LEVEL;
994 }
995 }
996
997 cpu_physical_memory_dirty_bits_cleared(start, length);
998
999 memory_region_clear_dirty_bitmap(mr, offset, length);
1000
1001 return snap;
1002 }
1003
cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot * snap,ram_addr_t start,ram_addr_t length)1004 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
1005 ram_addr_t start,
1006 ram_addr_t length)
1007 {
1008 unsigned long page, end;
1009
1010 assert(start >= snap->start);
1011 assert(start + length <= snap->end);
1012
1013 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
1014 page = (start - snap->start) >> TARGET_PAGE_BITS;
1015
1016 while (page < end) {
1017 if (test_bit(page, snap->dirty)) {
1018 return true;
1019 }
1020 page++;
1021 }
1022 return false;
1023 }
1024
1025 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
1026 uint16_t section);
1027 static subpage_t *subpage_init(FlatView *fv, hwaddr base);
1028
phys_section_add(PhysPageMap * map,MemoryRegionSection * section)1029 static uint16_t phys_section_add(PhysPageMap *map,
1030 MemoryRegionSection *section)
1031 {
1032 /* The physical section number is ORed with a page-aligned
1033 * pointer to produce the iotlb entries. Thus it should
1034 * never overflow into the page-aligned value.
1035 */
1036 assert(map->sections_nb < TARGET_PAGE_SIZE);
1037
1038 if (map->sections_nb == map->sections_nb_alloc) {
1039 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1040 map->sections = g_renew(MemoryRegionSection, map->sections,
1041 map->sections_nb_alloc);
1042 }
1043 map->sections[map->sections_nb] = *section;
1044 memory_region_ref(section->mr);
1045 return map->sections_nb++;
1046 }
1047
phys_section_destroy(MemoryRegion * mr)1048 static void phys_section_destroy(MemoryRegion *mr)
1049 {
1050 bool have_sub_page = mr->subpage;
1051
1052 memory_region_unref(mr);
1053
1054 if (have_sub_page) {
1055 subpage_t *subpage = container_of(mr, subpage_t, iomem);
1056 object_unref(OBJECT(&subpage->iomem));
1057 g_free(subpage);
1058 }
1059 }
1060
phys_sections_free(PhysPageMap * map)1061 static void phys_sections_free(PhysPageMap *map)
1062 {
1063 while (map->sections_nb > 0) {
1064 MemoryRegionSection *section = &map->sections[--map->sections_nb];
1065 phys_section_destroy(section->mr);
1066 }
1067 g_free(map->sections);
1068 g_free(map->nodes);
1069 }
1070
register_subpage(FlatView * fv,MemoryRegionSection * section)1071 static void register_subpage(FlatView *fv, MemoryRegionSection *section)
1072 {
1073 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1074 subpage_t *subpage;
1075 hwaddr base = section->offset_within_address_space
1076 & TARGET_PAGE_MASK;
1077 MemoryRegionSection *existing = phys_page_find(d, base);
1078 MemoryRegionSection subsection = {
1079 .offset_within_address_space = base,
1080 .size = int128_make64(TARGET_PAGE_SIZE),
1081 };
1082 hwaddr start, end;
1083
1084 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1085
1086 if (!(existing->mr->subpage)) {
1087 subpage = subpage_init(fv, base);
1088 subsection.fv = fv;
1089 subsection.mr = &subpage->iomem;
1090 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1091 phys_section_add(&d->map, &subsection));
1092 } else {
1093 subpage = container_of(existing->mr, subpage_t, iomem);
1094 }
1095 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1096 end = start + int128_get64(section->size) - 1;
1097 subpage_register(subpage, start, end,
1098 phys_section_add(&d->map, section));
1099 }
1100
1101
register_multipage(FlatView * fv,MemoryRegionSection * section)1102 static void register_multipage(FlatView *fv,
1103 MemoryRegionSection *section)
1104 {
1105 AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1106 hwaddr start_addr = section->offset_within_address_space;
1107 uint16_t section_index = phys_section_add(&d->map, section);
1108 uint64_t num_pages = int128_get64(int128_rshift(section->size,
1109 TARGET_PAGE_BITS));
1110
1111 assert(num_pages);
1112 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1113 }
1114
1115 /*
1116 * The range in *section* may look like this:
1117 *
1118 * |s|PPPPPPP|s|
1119 *
1120 * where s stands for subpage and P for page.
1121 */
flatview_add_to_dispatch(FlatView * fv,MemoryRegionSection * section)1122 void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
1123 {
1124 MemoryRegionSection remain = *section;
1125 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1126
1127 /* register first subpage */
1128 if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1129 uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1130 - remain.offset_within_address_space;
1131
1132 MemoryRegionSection now = remain;
1133 now.size = int128_min(int128_make64(left), now.size);
1134 register_subpage(fv, &now);
1135 if (int128_eq(remain.size, now.size)) {
1136 return;
1137 }
1138 remain.size = int128_sub(remain.size, now.size);
1139 remain.offset_within_address_space += int128_get64(now.size);
1140 remain.offset_within_region += int128_get64(now.size);
1141 }
1142
1143 /* register whole pages */
1144 if (int128_ge(remain.size, page_size)) {
1145 MemoryRegionSection now = remain;
1146 now.size = int128_and(now.size, int128_neg(page_size));
1147 register_multipage(fv, &now);
1148 if (int128_eq(remain.size, now.size)) {
1149 return;
1150 }
1151 remain.size = int128_sub(remain.size, now.size);
1152 remain.offset_within_address_space += int128_get64(now.size);
1153 remain.offset_within_region += int128_get64(now.size);
1154 }
1155
1156 /* register last subpage */
1157 register_subpage(fv, &remain);
1158 }
1159
qemu_flush_coalesced_mmio_buffer(void)1160 void qemu_flush_coalesced_mmio_buffer(void)
1161 {
1162 if (kvm_enabled())
1163 kvm_flush_coalesced_mmio_buffer();
1164 }
1165
qemu_mutex_lock_ramlist(void)1166 void qemu_mutex_lock_ramlist(void)
1167 {
1168 qemu_mutex_lock(&ram_list.mutex);
1169 }
1170
qemu_mutex_unlock_ramlist(void)1171 void qemu_mutex_unlock_ramlist(void)
1172 {
1173 qemu_mutex_unlock(&ram_list.mutex);
1174 }
1175
ram_block_format(void)1176 GString *ram_block_format(void)
1177 {
1178 RAMBlock *block;
1179 char *psize;
1180 GString *buf = g_string_new("");
1181
1182 RCU_READ_LOCK_GUARD();
1183 g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n",
1184 "Block Name", "PSize", "Offset", "Used", "Total",
1185 "HVA", "RO");
1186
1187 RAMBLOCK_FOREACH(block) {
1188 psize = size_to_str(block->page_size);
1189 g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64
1190 " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1191 block->idstr, psize,
1192 (uint64_t)block->offset,
1193 (uint64_t)block->used_length,
1194 (uint64_t)block->max_length,
1195 (uint64_t)(uintptr_t)block->host,
1196 block->mr->readonly ? "ro" : "rw");
1197
1198 g_free(psize);
1199 }
1200
1201 return buf;
1202 }
1203
find_min_backend_pagesize(Object * obj,void * opaque)1204 static int find_min_backend_pagesize(Object *obj, void *opaque)
1205 {
1206 long *hpsize_min = opaque;
1207
1208 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1209 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1210 long hpsize = host_memory_backend_pagesize(backend);
1211
1212 if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
1213 *hpsize_min = hpsize;
1214 }
1215 }
1216
1217 return 0;
1218 }
1219
find_max_backend_pagesize(Object * obj,void * opaque)1220 static int find_max_backend_pagesize(Object *obj, void *opaque)
1221 {
1222 long *hpsize_max = opaque;
1223
1224 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1225 HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1226 long hpsize = host_memory_backend_pagesize(backend);
1227
1228 if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1229 *hpsize_max = hpsize;
1230 }
1231 }
1232
1233 return 0;
1234 }
1235
1236 /*
1237 * TODO: We assume right now that all mapped host memory backends are
1238 * used as RAM, however some might be used for different purposes.
1239 */
qemu_minrampagesize(void)1240 long qemu_minrampagesize(void)
1241 {
1242 long hpsize = LONG_MAX;
1243 Object *memdev_root = object_resolve_path("/objects", NULL);
1244
1245 object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
1246 return hpsize;
1247 }
1248
qemu_maxrampagesize(void)1249 long qemu_maxrampagesize(void)
1250 {
1251 long pagesize = 0;
1252 Object *memdev_root = object_resolve_path("/objects", NULL);
1253
1254 object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
1255 return pagesize;
1256 }
1257
1258 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
get_file_size(int fd)1259 static int64_t get_file_size(int fd)
1260 {
1261 int64_t size;
1262 #if defined(__linux__)
1263 struct stat st;
1264
1265 if (fstat(fd, &st) < 0) {
1266 return -errno;
1267 }
1268
1269 /* Special handling for devdax character devices */
1270 if (S_ISCHR(st.st_mode)) {
1271 g_autofree char *subsystem_path = NULL;
1272 g_autofree char *subsystem = NULL;
1273
1274 subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1275 major(st.st_rdev), minor(st.st_rdev));
1276 subsystem = g_file_read_link(subsystem_path, NULL);
1277
1278 if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1279 g_autofree char *size_path = NULL;
1280 g_autofree char *size_str = NULL;
1281
1282 size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1283 major(st.st_rdev), minor(st.st_rdev));
1284
1285 if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1286 return g_ascii_strtoll(size_str, NULL, 0);
1287 }
1288 }
1289 }
1290 #endif /* defined(__linux__) */
1291
1292 /* st.st_size may be zero for special files yet lseek(2) works */
1293 size = lseek(fd, 0, SEEK_END);
1294 if (size < 0) {
1295 return -errno;
1296 }
1297 return size;
1298 }
1299
get_file_align(int fd)1300 static int64_t get_file_align(int fd)
1301 {
1302 int64_t align = -1;
1303 #if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1304 struct stat st;
1305
1306 if (fstat(fd, &st) < 0) {
1307 return -errno;
1308 }
1309
1310 /* Special handling for devdax character devices */
1311 if (S_ISCHR(st.st_mode)) {
1312 g_autofree char *path = NULL;
1313 g_autofree char *rpath = NULL;
1314 struct daxctl_ctx *ctx;
1315 struct daxctl_region *region;
1316 int rc = 0;
1317
1318 path = g_strdup_printf("/sys/dev/char/%d:%d",
1319 major(st.st_rdev), minor(st.st_rdev));
1320 rpath = realpath(path, NULL);
1321 if (!rpath) {
1322 return -errno;
1323 }
1324
1325 rc = daxctl_new(&ctx);
1326 if (rc) {
1327 return -1;
1328 }
1329
1330 daxctl_region_foreach(ctx, region) {
1331 if (strstr(rpath, daxctl_region_get_path(region))) {
1332 align = daxctl_region_get_align(region);
1333 break;
1334 }
1335 }
1336 daxctl_unref(ctx);
1337 }
1338 #endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1339
1340 return align;
1341 }
1342
file_ram_open(const char * path,const char * region_name,bool readonly,bool * created)1343 static int file_ram_open(const char *path,
1344 const char *region_name,
1345 bool readonly,
1346 bool *created)
1347 {
1348 char *filename;
1349 char *sanitized_name;
1350 char *c;
1351 int fd = -1;
1352
1353 *created = false;
1354 for (;;) {
1355 fd = open(path, readonly ? O_RDONLY : O_RDWR);
1356 if (fd >= 0) {
1357 /*
1358 * open(O_RDONLY) won't fail with EISDIR. Check manually if we
1359 * opened a directory and fail similarly to how we fail ENOENT
1360 * in readonly mode. Note that mkstemp() would imply O_RDWR.
1361 */
1362 if (readonly) {
1363 struct stat file_stat;
1364
1365 if (fstat(fd, &file_stat)) {
1366 close(fd);
1367 if (errno == EINTR) {
1368 continue;
1369 }
1370 return -errno;
1371 } else if (S_ISDIR(file_stat.st_mode)) {
1372 close(fd);
1373 return -EISDIR;
1374 }
1375 }
1376 /* @path names an existing file, use it */
1377 break;
1378 }
1379 if (errno == ENOENT) {
1380 if (readonly) {
1381 /* Refuse to create new, readonly files. */
1382 return -ENOENT;
1383 }
1384 /* @path names a file that doesn't exist, create it */
1385 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1386 if (fd >= 0) {
1387 *created = true;
1388 break;
1389 }
1390 } else if (errno == EISDIR) {
1391 /* @path names a directory, create a file there */
1392 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1393 sanitized_name = g_strdup(region_name);
1394 for (c = sanitized_name; *c != '\0'; c++) {
1395 if (*c == '/') {
1396 *c = '_';
1397 }
1398 }
1399
1400 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1401 sanitized_name);
1402 g_free(sanitized_name);
1403
1404 fd = mkstemp(filename);
1405 if (fd >= 0) {
1406 unlink(filename);
1407 g_free(filename);
1408 break;
1409 }
1410 g_free(filename);
1411 }
1412 if (errno != EEXIST && errno != EINTR) {
1413 return -errno;
1414 }
1415 /*
1416 * Try again on EINTR and EEXIST. The latter happens when
1417 * something else creates the file between our two open().
1418 */
1419 }
1420
1421 return fd;
1422 }
1423
file_ram_alloc(RAMBlock * block,ram_addr_t memory,int fd,bool truncate,off_t offset,Error ** errp)1424 static void *file_ram_alloc(RAMBlock *block,
1425 ram_addr_t memory,
1426 int fd,
1427 bool truncate,
1428 off_t offset,
1429 Error **errp)
1430 {
1431 uint32_t qemu_map_flags;
1432 void *area;
1433
1434 block->page_size = qemu_fd_getpagesize(fd);
1435 if (block->mr->align % block->page_size) {
1436 error_setg(errp, "alignment 0x%" PRIx64
1437 " must be multiples of page size 0x%zx",
1438 block->mr->align, block->page_size);
1439 return NULL;
1440 } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1441 error_setg(errp, "alignment 0x%" PRIx64
1442 " must be a power of two", block->mr->align);
1443 return NULL;
1444 } else if (offset % block->page_size) {
1445 error_setg(errp, "offset 0x%" PRIx64
1446 " must be multiples of page size 0x%zx",
1447 offset, block->page_size);
1448 return NULL;
1449 }
1450 block->mr->align = MAX(block->page_size, block->mr->align);
1451 #if defined(__s390x__)
1452 if (kvm_enabled()) {
1453 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1454 }
1455 #endif
1456
1457 if (memory < block->page_size) {
1458 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1459 "or larger than page size 0x%zx",
1460 memory, block->page_size);
1461 return NULL;
1462 }
1463
1464 memory = ROUND_UP(memory, block->page_size);
1465
1466 /*
1467 * ftruncate is not supported by hugetlbfs in older
1468 * hosts, so don't bother bailing out on errors.
1469 * If anything goes wrong with it under other filesystems,
1470 * mmap will fail.
1471 *
1472 * Do not truncate the non-empty backend file to avoid corrupting
1473 * the existing data in the file. Disabling shrinking is not
1474 * enough. For example, the current vNVDIMM implementation stores
1475 * the guest NVDIMM labels at the end of the backend file. If the
1476 * backend file is later extended, QEMU will not be able to find
1477 * those labels. Therefore, extending the non-empty backend file
1478 * is disabled as well.
1479 */
1480 if (truncate && ftruncate(fd, offset + memory)) {
1481 perror("ftruncate");
1482 }
1483
1484 qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
1485 qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1486 qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
1487 qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
1488 area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
1489 if (area == MAP_FAILED) {
1490 error_setg_errno(errp, errno,
1491 "unable to map backing store for guest RAM");
1492 return NULL;
1493 }
1494
1495 block->fd = fd;
1496 block->fd_offset = offset;
1497 return area;
1498 }
1499 #endif
1500
1501 /* Allocate space within the ram_addr_t space that governs the
1502 * dirty bitmaps.
1503 * Called with the ramlist lock held.
1504 */
find_ram_offset(ram_addr_t size)1505 static ram_addr_t find_ram_offset(ram_addr_t size)
1506 {
1507 RAMBlock *block, *next_block;
1508 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1509
1510 assert(size != 0); /* it would hand out same offset multiple times */
1511
1512 if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1513 return 0;
1514 }
1515
1516 RAMBLOCK_FOREACH(block) {
1517 ram_addr_t candidate, next = RAM_ADDR_MAX;
1518
1519 /* Align blocks to start on a 'long' in the bitmap
1520 * which makes the bitmap sync'ing take the fast path.
1521 */
1522 candidate = block->offset + block->max_length;
1523 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
1524
1525 /* Search for the closest following block
1526 * and find the gap.
1527 */
1528 RAMBLOCK_FOREACH(next_block) {
1529 if (next_block->offset >= candidate) {
1530 next = MIN(next, next_block->offset);
1531 }
1532 }
1533
1534 /* If it fits remember our place and remember the size
1535 * of gap, but keep going so that we might find a smaller
1536 * gap to fill so avoiding fragmentation.
1537 */
1538 if (next - candidate >= size && next - candidate < mingap) {
1539 offset = candidate;
1540 mingap = next - candidate;
1541 }
1542
1543 trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
1544 }
1545
1546 if (offset == RAM_ADDR_MAX) {
1547 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1548 (uint64_t)size);
1549 abort();
1550 }
1551
1552 trace_find_ram_offset(size, offset);
1553
1554 return offset;
1555 }
1556
qemu_ram_setup_dump(void * addr,ram_addr_t size)1557 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1558 {
1559 int ret;
1560
1561 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1562 if (!machine_dump_guest_core(current_machine)) {
1563 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1564 if (ret) {
1565 perror("qemu_madvise");
1566 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1567 "but dump-guest-core=off specified\n");
1568 }
1569 }
1570 }
1571
qemu_ram_get_idstr(RAMBlock * rb)1572 const char *qemu_ram_get_idstr(RAMBlock *rb)
1573 {
1574 return rb->idstr;
1575 }
1576
qemu_ram_get_host_addr(RAMBlock * rb)1577 void *qemu_ram_get_host_addr(RAMBlock *rb)
1578 {
1579 return rb->host;
1580 }
1581
qemu_ram_get_offset(RAMBlock * rb)1582 ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
1583 {
1584 return rb->offset;
1585 }
1586
qemu_ram_get_fd_offset(RAMBlock * rb)1587 ram_addr_t qemu_ram_get_fd_offset(RAMBlock *rb)
1588 {
1589 return rb->fd_offset;
1590 }
1591
qemu_ram_get_used_length(RAMBlock * rb)1592 ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
1593 {
1594 return rb->used_length;
1595 }
1596
qemu_ram_get_max_length(RAMBlock * rb)1597 ram_addr_t qemu_ram_get_max_length(RAMBlock *rb)
1598 {
1599 return rb->max_length;
1600 }
1601
qemu_ram_is_shared(RAMBlock * rb)1602 bool qemu_ram_is_shared(RAMBlock *rb)
1603 {
1604 return rb->flags & RAM_SHARED;
1605 }
1606
qemu_ram_is_noreserve(RAMBlock * rb)1607 bool qemu_ram_is_noreserve(RAMBlock *rb)
1608 {
1609 return rb->flags & RAM_NORESERVE;
1610 }
1611
1612 /* Note: Only set at the start of postcopy */
qemu_ram_is_uf_zeroable(RAMBlock * rb)1613 bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1614 {
1615 return rb->flags & RAM_UF_ZEROPAGE;
1616 }
1617
qemu_ram_set_uf_zeroable(RAMBlock * rb)1618 void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1619 {
1620 rb->flags |= RAM_UF_ZEROPAGE;
1621 }
1622
qemu_ram_is_migratable(RAMBlock * rb)1623 bool qemu_ram_is_migratable(RAMBlock *rb)
1624 {
1625 return rb->flags & RAM_MIGRATABLE;
1626 }
1627
qemu_ram_set_migratable(RAMBlock * rb)1628 void qemu_ram_set_migratable(RAMBlock *rb)
1629 {
1630 rb->flags |= RAM_MIGRATABLE;
1631 }
1632
qemu_ram_unset_migratable(RAMBlock * rb)1633 void qemu_ram_unset_migratable(RAMBlock *rb)
1634 {
1635 rb->flags &= ~RAM_MIGRATABLE;
1636 }
1637
qemu_ram_is_named_file(RAMBlock * rb)1638 bool qemu_ram_is_named_file(RAMBlock *rb)
1639 {
1640 return rb->flags & RAM_NAMED_FILE;
1641 }
1642
qemu_ram_get_fd(RAMBlock * rb)1643 int qemu_ram_get_fd(RAMBlock *rb)
1644 {
1645 return rb->fd;
1646 }
1647
1648 /* Called with the BQL held. */
qemu_ram_set_idstr(RAMBlock * new_block,const char * name,DeviceState * dev)1649 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1650 {
1651 RAMBlock *block;
1652
1653 assert(new_block);
1654 assert(!new_block->idstr[0]);
1655
1656 if (dev) {
1657 char *id = qdev_get_dev_path(dev);
1658 if (id) {
1659 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1660 g_free(id);
1661 }
1662 }
1663 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1664
1665 RCU_READ_LOCK_GUARD();
1666 RAMBLOCK_FOREACH(block) {
1667 if (block != new_block &&
1668 !strcmp(block->idstr, new_block->idstr)) {
1669 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1670 new_block->idstr);
1671 abort();
1672 }
1673 }
1674 }
1675
1676 /* Called with the BQL held. */
qemu_ram_unset_idstr(RAMBlock * block)1677 void qemu_ram_unset_idstr(RAMBlock *block)
1678 {
1679 /* FIXME: arch_init.c assumes that this is not called throughout
1680 * migration. Ignore the problem since hot-unplug during migration
1681 * does not work anyway.
1682 */
1683 if (block) {
1684 memset(block->idstr, 0, sizeof(block->idstr));
1685 }
1686 }
1687
cpr_name(MemoryRegion * mr)1688 static char *cpr_name(MemoryRegion *mr)
1689 {
1690 const char *mr_name = memory_region_name(mr);
1691 g_autofree char *id = mr->dev ? qdev_get_dev_path(mr->dev) : NULL;
1692
1693 if (id) {
1694 return g_strdup_printf("%s/%s", id, mr_name);
1695 } else {
1696 return g_strdup(mr_name);
1697 }
1698 }
1699
qemu_ram_pagesize(RAMBlock * rb)1700 size_t qemu_ram_pagesize(RAMBlock *rb)
1701 {
1702 return rb->page_size;
1703 }
1704
1705 /* Returns the largest size of page in use */
qemu_ram_pagesize_largest(void)1706 size_t qemu_ram_pagesize_largest(void)
1707 {
1708 RAMBlock *block;
1709 size_t largest = 0;
1710
1711 RAMBLOCK_FOREACH(block) {
1712 largest = MAX(largest, qemu_ram_pagesize(block));
1713 }
1714
1715 return largest;
1716 }
1717
memory_try_enable_merging(void * addr,size_t len)1718 static int memory_try_enable_merging(void *addr, size_t len)
1719 {
1720 if (!machine_mem_merge(current_machine)) {
1721 /* disabled by the user */
1722 return 0;
1723 }
1724
1725 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1726 }
1727
1728 /*
1729 * Resizing RAM while migrating can result in the migration being canceled.
1730 * Care has to be taken if the guest might have already detected the memory.
1731 *
1732 * As memory core doesn't know how is memory accessed, it is up to
1733 * resize callback to update device state and/or add assertions to detect
1734 * misuse, if necessary.
1735 */
qemu_ram_resize(RAMBlock * block,ram_addr_t newsize,Error ** errp)1736 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1737 {
1738 const ram_addr_t oldsize = block->used_length;
1739 const ram_addr_t unaligned_size = newsize;
1740
1741 assert(block);
1742
1743 newsize = TARGET_PAGE_ALIGN(newsize);
1744 newsize = REAL_HOST_PAGE_ALIGN(newsize);
1745
1746 if (block->used_length == newsize) {
1747 /*
1748 * We don't have to resize the ram block (which only knows aligned
1749 * sizes), however, we have to notify if the unaligned size changed.
1750 */
1751 if (unaligned_size != memory_region_size(block->mr)) {
1752 memory_region_set_size(block->mr, unaligned_size);
1753 if (block->resized) {
1754 block->resized(block->idstr, unaligned_size, block->host);
1755 }
1756 }
1757 return 0;
1758 }
1759
1760 if (!(block->flags & RAM_RESIZEABLE)) {
1761 error_setg_errno(errp, EINVAL,
1762 "Size mismatch: %s: 0x" RAM_ADDR_FMT
1763 " != 0x" RAM_ADDR_FMT, block->idstr,
1764 newsize, block->used_length);
1765 return -EINVAL;
1766 }
1767
1768 if (block->max_length < newsize) {
1769 error_setg_errno(errp, EINVAL,
1770 "Size too large: %s: 0x" RAM_ADDR_FMT
1771 " > 0x" RAM_ADDR_FMT, block->idstr,
1772 newsize, block->max_length);
1773 return -EINVAL;
1774 }
1775
1776 /* Notify before modifying the ram block and touching the bitmaps. */
1777 if (block->host) {
1778 ram_block_notify_resize(block->host, oldsize, newsize);
1779 }
1780
1781 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1782 block->used_length = newsize;
1783 cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1784 DIRTY_CLIENTS_ALL);
1785 memory_region_set_size(block->mr, unaligned_size);
1786 if (block->resized) {
1787 block->resized(block->idstr, unaligned_size, block->host);
1788 }
1789 return 0;
1790 }
1791
1792 /*
1793 * Trigger sync on the given ram block for range [start, start + length]
1794 * with the backing store if one is available.
1795 * Otherwise no-op.
1796 * @Note: this is supposed to be a synchronous op.
1797 */
qemu_ram_msync(RAMBlock * block,ram_addr_t start,ram_addr_t length)1798 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
1799 {
1800 /* The requested range should fit in within the block range */
1801 g_assert((start + length) <= block->used_length);
1802
1803 #ifdef CONFIG_LIBPMEM
1804 /* The lack of support for pmem should not block the sync */
1805 if (ramblock_is_pmem(block)) {
1806 void *addr = ramblock_ptr(block, start);
1807 pmem_persist(addr, length);
1808 return;
1809 }
1810 #endif
1811 if (block->fd >= 0) {
1812 /**
1813 * Case there is no support for PMEM or the memory has not been
1814 * specified as persistent (or is not one) - use the msync.
1815 * Less optimal but still achieves the same goal
1816 */
1817 void *addr = ramblock_ptr(block, start);
1818 if (qemu_msync(addr, length, block->fd)) {
1819 warn_report("%s: failed to sync memory range: start: "
1820 RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
1821 __func__, start, length);
1822 }
1823 }
1824 }
1825
1826 /* Called with ram_list.mutex held */
dirty_memory_extend(ram_addr_t new_ram_size)1827 static void dirty_memory_extend(ram_addr_t new_ram_size)
1828 {
1829 unsigned int old_num_blocks = ram_list.num_dirty_blocks;
1830 unsigned int new_num_blocks = DIV_ROUND_UP(new_ram_size,
1831 DIRTY_MEMORY_BLOCK_SIZE);
1832 int i;
1833
1834 /* Only need to extend if block count increased */
1835 if (new_num_blocks <= old_num_blocks) {
1836 return;
1837 }
1838
1839 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1840 DirtyMemoryBlocks *old_blocks;
1841 DirtyMemoryBlocks *new_blocks;
1842 int j;
1843
1844 old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
1845 new_blocks = g_malloc(sizeof(*new_blocks) +
1846 sizeof(new_blocks->blocks[0]) * new_num_blocks);
1847
1848 if (old_num_blocks) {
1849 memcpy(new_blocks->blocks, old_blocks->blocks,
1850 old_num_blocks * sizeof(old_blocks->blocks[0]));
1851 }
1852
1853 for (j = old_num_blocks; j < new_num_blocks; j++) {
1854 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1855 }
1856
1857 qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1858
1859 if (old_blocks) {
1860 g_free_rcu(old_blocks, rcu);
1861 }
1862 }
1863
1864 ram_list.num_dirty_blocks = new_num_blocks;
1865 }
1866
ram_block_add(RAMBlock * new_block,Error ** errp)1867 static void ram_block_add(RAMBlock *new_block, Error **errp)
1868 {
1869 const bool noreserve = qemu_ram_is_noreserve(new_block);
1870 const bool shared = qemu_ram_is_shared(new_block);
1871 RAMBlock *block;
1872 RAMBlock *last_block = NULL;
1873 bool free_on_error = false;
1874 ram_addr_t ram_size;
1875 Error *err = NULL;
1876
1877 qemu_mutex_lock_ramlist();
1878 new_block->offset = find_ram_offset(new_block->max_length);
1879
1880 if (!new_block->host) {
1881 if (xen_enabled()) {
1882 xen_ram_alloc(new_block->offset, new_block->max_length,
1883 new_block->mr, &err);
1884 if (err) {
1885 error_propagate(errp, err);
1886 qemu_mutex_unlock_ramlist();
1887 return;
1888 }
1889 } else {
1890 new_block->host = qemu_anon_ram_alloc(new_block->max_length,
1891 &new_block->mr->align,
1892 shared, noreserve);
1893 if (!new_block->host) {
1894 error_setg_errno(errp, errno,
1895 "cannot set up guest memory '%s'",
1896 memory_region_name(new_block->mr));
1897 qemu_mutex_unlock_ramlist();
1898 return;
1899 }
1900 memory_try_enable_merging(new_block->host, new_block->max_length);
1901 free_on_error = true;
1902 }
1903 }
1904
1905 if (new_block->flags & RAM_GUEST_MEMFD) {
1906 int ret;
1907
1908 if (!kvm_enabled()) {
1909 error_setg(errp, "cannot set up private guest memory for %s: KVM required",
1910 object_get_typename(OBJECT(current_machine->cgs)));
1911 goto out_free;
1912 }
1913 assert(new_block->guest_memfd < 0);
1914
1915 ret = ram_block_coordinated_discard_require(true);
1916 if (ret < 0) {
1917 error_setg_errno(errp, -ret,
1918 "cannot set up private guest memory: discard currently blocked");
1919 error_append_hint(errp, "Are you using assigned devices?\n");
1920 goto out_free;
1921 }
1922
1923 new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length,
1924 0, errp);
1925 if (new_block->guest_memfd < 0) {
1926 qemu_mutex_unlock_ramlist();
1927 goto out_free;
1928 }
1929
1930 /*
1931 * The attribute bitmap of the RamBlockAttributes is default to
1932 * discarded, which mimics the behavior of kvm_set_phys_mem() when it
1933 * calls kvm_set_memory_attributes_private(). This leads to a brief
1934 * period of inconsistency between the creation of the RAMBlock and its
1935 * mapping into the physical address space. However, this is not
1936 * problematic, as no users rely on the attribute status to perform
1937 * any actions during this interval.
1938 */
1939 new_block->attributes = ram_block_attributes_create(new_block);
1940 if (!new_block->attributes) {
1941 error_setg(errp, "Failed to create ram block attribute");
1942 close(new_block->guest_memfd);
1943 ram_block_coordinated_discard_require(false);
1944 qemu_mutex_unlock_ramlist();
1945 goto out_free;
1946 }
1947
1948 /*
1949 * Add a specific guest_memfd blocker if a generic one would not be
1950 * added by ram_block_add_cpr_blocker.
1951 */
1952 if (ram_is_cpr_compatible(new_block)) {
1953 error_setg(&new_block->cpr_blocker,
1954 "Memory region %s uses guest_memfd, "
1955 "which is not supported with CPR.",
1956 memory_region_name(new_block->mr));
1957 migrate_add_blocker_modes(&new_block->cpr_blocker, errp,
1958 MIG_MODE_CPR_TRANSFER, -1);
1959 }
1960 }
1961
1962 ram_size = (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS;
1963 dirty_memory_extend(ram_size);
1964 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1965 * QLIST (which has an RCU-friendly variant) does not have insertion at
1966 * tail, so save the last element in last_block.
1967 */
1968 RAMBLOCK_FOREACH(block) {
1969 last_block = block;
1970 if (block->max_length < new_block->max_length) {
1971 break;
1972 }
1973 }
1974 if (block) {
1975 QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1976 } else if (last_block) {
1977 QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1978 } else { /* list is empty */
1979 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1980 }
1981 ram_list.mru_block = NULL;
1982
1983 /* Write list before version */
1984 smp_wmb();
1985 ram_list.version++;
1986 qemu_mutex_unlock_ramlist();
1987
1988 cpu_physical_memory_set_dirty_range(new_block->offset,
1989 new_block->used_length,
1990 DIRTY_CLIENTS_ALL);
1991
1992 if (new_block->host) {
1993 qemu_ram_setup_dump(new_block->host, new_block->max_length);
1994 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1995 /*
1996 * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
1997 * Configure it unless the machine is a qtest server, in which case
1998 * KVM is not used and it may be forked (eg for fuzzing purposes).
1999 */
2000 if (!qtest_enabled()) {
2001 qemu_madvise(new_block->host, new_block->max_length,
2002 QEMU_MADV_DONTFORK);
2003 }
2004 ram_block_notify_add(new_block->host, new_block->used_length,
2005 new_block->max_length);
2006 }
2007 return;
2008
2009 out_free:
2010 if (free_on_error) {
2011 qemu_anon_ram_free(new_block->host, new_block->max_length);
2012 new_block->host = NULL;
2013 }
2014 }
2015
2016 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
qemu_ram_alloc_from_fd(ram_addr_t size,ram_addr_t max_size,qemu_ram_resize_cb resized,MemoryRegion * mr,uint32_t ram_flags,int fd,off_t offset,bool grow,Error ** errp)2017 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, ram_addr_t max_size,
2018 qemu_ram_resize_cb resized, MemoryRegion *mr,
2019 uint32_t ram_flags, int fd, off_t offset,
2020 bool grow,
2021 Error **errp)
2022 {
2023 ERRP_GUARD();
2024 RAMBlock *new_block;
2025 Error *local_err = NULL;
2026 int64_t file_size, file_align, share_flags;
2027
2028 share_flags = ram_flags & (RAM_PRIVATE | RAM_SHARED);
2029 assert(share_flags != (RAM_SHARED | RAM_PRIVATE));
2030 ram_flags &= ~RAM_PRIVATE;
2031
2032 /* Just support these ram flags by now. */
2033 assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE |
2034 RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY |
2035 RAM_READONLY_FD | RAM_GUEST_MEMFD |
2036 RAM_RESIZEABLE)) == 0);
2037 assert(max_size >= size);
2038
2039 if (xen_enabled()) {
2040 error_setg(errp, "-mem-path not supported with Xen");
2041 return NULL;
2042 }
2043
2044 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2045 error_setg(errp,
2046 "host lacks kvm mmu notifiers, -mem-path unsupported");
2047 return NULL;
2048 }
2049
2050 size = TARGET_PAGE_ALIGN(size);
2051 size = REAL_HOST_PAGE_ALIGN(size);
2052 max_size = TARGET_PAGE_ALIGN(max_size);
2053 max_size = REAL_HOST_PAGE_ALIGN(max_size);
2054
2055 file_size = get_file_size(fd);
2056 if (file_size && file_size < offset + max_size && !grow) {
2057 error_setg(errp, "%s backing store size 0x%" PRIx64
2058 " is too small for 'size' option 0x" RAM_ADDR_FMT
2059 " plus 'offset' option 0x%" PRIx64,
2060 memory_region_name(mr), file_size, max_size,
2061 (uint64_t)offset);
2062 return NULL;
2063 }
2064
2065 file_align = get_file_align(fd);
2066 if (file_align > 0 && file_align > mr->align) {
2067 error_setg(errp, "backing store align 0x%" PRIx64
2068 " is larger than 'align' option 0x%" PRIx64,
2069 file_align, mr->align);
2070 return NULL;
2071 }
2072
2073 new_block = g_malloc0(sizeof(*new_block));
2074 new_block->mr = mr;
2075 new_block->used_length = size;
2076 new_block->max_length = max_size;
2077 new_block->resized = resized;
2078 new_block->flags = ram_flags;
2079 new_block->guest_memfd = -1;
2080 new_block->host = file_ram_alloc(new_block, max_size, fd,
2081 file_size < offset + max_size,
2082 offset, errp);
2083 if (!new_block->host) {
2084 g_free(new_block);
2085 return NULL;
2086 }
2087
2088 ram_block_add(new_block, &local_err);
2089 if (local_err) {
2090 g_free(new_block);
2091 error_propagate(errp, local_err);
2092 return NULL;
2093 }
2094 return new_block;
2095
2096 }
2097
2098
qemu_ram_alloc_from_file(ram_addr_t size,MemoryRegion * mr,uint32_t ram_flags,const char * mem_path,off_t offset,Error ** errp)2099 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
2100 uint32_t ram_flags, const char *mem_path,
2101 off_t offset, Error **errp)
2102 {
2103 int fd;
2104 bool created;
2105 RAMBlock *block;
2106
2107 fd = file_ram_open(mem_path, memory_region_name(mr),
2108 !!(ram_flags & RAM_READONLY_FD), &created);
2109 if (fd < 0) {
2110 error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
2111 mem_path);
2112 if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
2113 fd == -EACCES) {
2114 /*
2115 * If we can open the file R/O (note: will never create a new file)
2116 * and we are dealing with a private mapping, there are still ways
2117 * to consume such files and get RAM instead of ROM.
2118 */
2119 fd = file_ram_open(mem_path, memory_region_name(mr), true,
2120 &created);
2121 if (fd < 0) {
2122 return NULL;
2123 }
2124 assert(!created);
2125 close(fd);
2126 error_append_hint(errp, "Consider opening the backing store"
2127 " read-only but still creating writable RAM using"
2128 " '-object memory-backend-file,readonly=on,rom=off...'"
2129 " (see \"VM templating\" documentation)\n");
2130 }
2131 return NULL;
2132 }
2133
2134 block = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, offset,
2135 false, errp);
2136 if (!block) {
2137 if (created) {
2138 unlink(mem_path);
2139 }
2140 close(fd);
2141 return NULL;
2142 }
2143
2144 return block;
2145 }
2146 #endif
2147
2148 #ifdef CONFIG_POSIX
2149 /*
2150 * Create MAP_SHARED RAMBlocks by mmap'ing a file descriptor, so it can be
2151 * shared with another process if CPR is being used. Use memfd if available
2152 * because it has no size limits, else use POSIX shm.
2153 */
qemu_ram_get_shared_fd(const char * name,bool * reused,Error ** errp)2154 static int qemu_ram_get_shared_fd(const char *name, bool *reused, Error **errp)
2155 {
2156 int fd = cpr_find_fd(name, 0);
2157
2158 if (fd >= 0) {
2159 *reused = true;
2160 return fd;
2161 }
2162
2163 if (qemu_memfd_check(0)) {
2164 fd = qemu_memfd_create(name, 0, 0, 0, 0, errp);
2165 } else {
2166 fd = qemu_shm_alloc(0, errp);
2167 }
2168
2169 if (fd >= 0) {
2170 cpr_save_fd(name, 0, fd);
2171 }
2172 *reused = false;
2173 return fd;
2174 }
2175 #endif
2176
2177 static
qemu_ram_alloc_internal(ram_addr_t size,ram_addr_t max_size,qemu_ram_resize_cb resized,void * host,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2178 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2179 qemu_ram_resize_cb resized,
2180 void *host, uint32_t ram_flags,
2181 MemoryRegion *mr, Error **errp)
2182 {
2183 RAMBlock *new_block;
2184 Error *local_err = NULL;
2185 int align, share_flags;
2186
2187 share_flags = ram_flags & (RAM_PRIVATE | RAM_SHARED);
2188 assert(share_flags != (RAM_SHARED | RAM_PRIVATE));
2189 ram_flags &= ~RAM_PRIVATE;
2190
2191 assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC |
2192 RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
2193 assert(!host ^ (ram_flags & RAM_PREALLOC));
2194 assert(max_size >= size);
2195
2196 /* ignore RAM_SHARED for Windows and emscripten*/
2197 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN)
2198 if (!host) {
2199 if (!share_flags && current_machine->aux_ram_share) {
2200 ram_flags |= RAM_SHARED;
2201 }
2202 if (ram_flags & RAM_SHARED) {
2203 bool reused;
2204 g_autofree char *name = cpr_name(mr);
2205 int fd = qemu_ram_get_shared_fd(name, &reused, errp);
2206
2207 if (fd < 0) {
2208 return NULL;
2209 }
2210
2211 /* Use same alignment as qemu_anon_ram_alloc */
2212 mr->align = QEMU_VMALLOC_ALIGN;
2213
2214 /*
2215 * This can fail if the shm mount size is too small, or alloc from
2216 * fd is not supported, but previous QEMU versions that called
2217 * qemu_anon_ram_alloc for anonymous shared memory could have
2218 * succeeded. Quietly fail and fall back.
2219 *
2220 * After cpr-transfer, new QEMU could create a memory region
2221 * with a larger max size than old, so pass reused to grow the
2222 * region if necessary. The extra space will be usable after a
2223 * guest reset.
2224 */
2225 new_block = qemu_ram_alloc_from_fd(size, max_size, resized, mr,
2226 ram_flags, fd, 0, reused, NULL);
2227 if (new_block) {
2228 trace_qemu_ram_alloc_shared(name, new_block->used_length,
2229 new_block->max_length, fd,
2230 new_block->host);
2231 return new_block;
2232 }
2233
2234 cpr_delete_fd(name, 0);
2235 close(fd);
2236 /* fall back to anon allocation */
2237 }
2238 }
2239 #endif
2240
2241 align = qemu_real_host_page_size();
2242 align = MAX(align, TARGET_PAGE_SIZE);
2243 size = ROUND_UP(size, align);
2244 max_size = ROUND_UP(max_size, align);
2245
2246 new_block = g_malloc0(sizeof(*new_block));
2247 new_block->mr = mr;
2248 new_block->resized = resized;
2249 new_block->used_length = size;
2250 new_block->max_length = max_size;
2251 new_block->fd = -1;
2252 new_block->guest_memfd = -1;
2253 new_block->page_size = qemu_real_host_page_size();
2254 new_block->host = host;
2255 new_block->flags = ram_flags;
2256 ram_block_add(new_block, &local_err);
2257 if (local_err) {
2258 g_free(new_block);
2259 error_propagate(errp, local_err);
2260 return NULL;
2261 }
2262 return new_block;
2263 }
2264
qemu_ram_alloc_from_ptr(ram_addr_t size,void * host,MemoryRegion * mr,Error ** errp)2265 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
2266 MemoryRegion *mr, Error **errp)
2267 {
2268 return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2269 errp);
2270 }
2271
qemu_ram_alloc(ram_addr_t size,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2272 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
2273 MemoryRegion *mr, Error **errp)
2274 {
2275 assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE | RAM_GUEST_MEMFD |
2276 RAM_PRIVATE)) == 0);
2277 return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
2278 }
2279
qemu_ram_alloc_resizeable(ram_addr_t size,ram_addr_t maxsz,qemu_ram_resize_cb resized,MemoryRegion * mr,Error ** errp)2280 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
2281 qemu_ram_resize_cb resized,
2282 MemoryRegion *mr, Error **errp)
2283 {
2284 return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2285 RAM_RESIZEABLE, mr, errp);
2286 }
2287
reclaim_ramblock(RAMBlock * block)2288 static void reclaim_ramblock(RAMBlock *block)
2289 {
2290 if (block->flags & RAM_PREALLOC) {
2291 ;
2292 } else if (xen_enabled()) {
2293 xen_invalidate_map_cache_entry(block->host);
2294 #if !defined(_WIN32) && !defined(EMSCRIPTEN)
2295 } else if (block->fd >= 0) {
2296 qemu_ram_munmap(block->fd, block->host, block->max_length);
2297 close(block->fd);
2298 #endif
2299 } else {
2300 qemu_anon_ram_free(block->host, block->max_length);
2301 }
2302
2303 if (block->guest_memfd >= 0) {
2304 ram_block_attributes_destroy(block->attributes);
2305 close(block->guest_memfd);
2306 ram_block_coordinated_discard_require(false);
2307 }
2308
2309 g_free(block);
2310 }
2311
qemu_ram_free(RAMBlock * block)2312 void qemu_ram_free(RAMBlock *block)
2313 {
2314 g_autofree char *name = NULL;
2315
2316 if (!block) {
2317 return;
2318 }
2319
2320 if (block->host) {
2321 ram_block_notify_remove(block->host, block->used_length,
2322 block->max_length);
2323 }
2324
2325 qemu_mutex_lock_ramlist();
2326 name = cpr_name(block->mr);
2327 cpr_delete_fd(name, 0);
2328 QLIST_REMOVE_RCU(block, next);
2329 ram_list.mru_block = NULL;
2330 /* Write list before version */
2331 smp_wmb();
2332 ram_list.version++;
2333 call_rcu(block, reclaim_ramblock, rcu);
2334 qemu_mutex_unlock_ramlist();
2335 }
2336
2337 #ifndef _WIN32
2338 /* Simply remap the given VM memory location from start to start+length */
qemu_ram_remap_mmap(RAMBlock * block,uint64_t start,size_t length)2339 static int qemu_ram_remap_mmap(RAMBlock *block, uint64_t start, size_t length)
2340 {
2341 int flags, prot;
2342 void *area;
2343 void *host_startaddr = block->host + start;
2344
2345 assert(block->fd < 0);
2346 flags = MAP_FIXED | MAP_ANONYMOUS;
2347 flags |= block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE;
2348 flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
2349 prot = PROT_READ;
2350 prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
2351 area = mmap(host_startaddr, length, prot, flags, -1, 0);
2352 return area != host_startaddr ? -errno : 0;
2353 }
2354
2355 /*
2356 * qemu_ram_remap - remap a single RAM page
2357 *
2358 * @addr: address in ram_addr_t address space.
2359 *
2360 * This function will try remapping a single page of guest RAM identified by
2361 * @addr, essentially discarding memory to recover from previously poisoned
2362 * memory (MCE). The page size depends on the RAMBlock (i.e., hugetlb). @addr
2363 * does not have to point at the start of the page.
2364 *
2365 * This function is only to be used during system resets; it will kill the
2366 * VM if remapping failed.
2367 */
qemu_ram_remap(ram_addr_t addr)2368 void qemu_ram_remap(ram_addr_t addr)
2369 {
2370 RAMBlock *block;
2371 uint64_t offset;
2372 void *vaddr;
2373 size_t page_size;
2374
2375 RAMBLOCK_FOREACH(block) {
2376 offset = addr - block->offset;
2377 if (offset < block->max_length) {
2378 /* Respect the pagesize of our RAMBlock */
2379 page_size = qemu_ram_pagesize(block);
2380 offset = QEMU_ALIGN_DOWN(offset, page_size);
2381
2382 vaddr = ramblock_ptr(block, offset);
2383 if (block->flags & RAM_PREALLOC) {
2384 ;
2385 } else if (xen_enabled()) {
2386 abort();
2387 } else {
2388 if (ram_block_discard_range(block, offset, page_size) != 0) {
2389 /*
2390 * Fall back to using mmap() only for anonymous mapping,
2391 * as if a backing file is associated we may not be able
2392 * to recover the memory in all cases.
2393 * So don't take the risk of using only mmap and fail now.
2394 */
2395 if (block->fd >= 0) {
2396 error_report("Could not remap RAM %s:%" PRIx64 "+%"
2397 PRIx64 " +%zx", block->idstr, offset,
2398 block->fd_offset, page_size);
2399 exit(1);
2400 }
2401 if (qemu_ram_remap_mmap(block, offset, page_size) != 0) {
2402 error_report("Could not remap RAM %s:%" PRIx64 " +%zx",
2403 block->idstr, offset, page_size);
2404 exit(1);
2405 }
2406 }
2407 memory_try_enable_merging(vaddr, page_size);
2408 qemu_ram_setup_dump(vaddr, page_size);
2409 }
2410
2411 break;
2412 }
2413 }
2414 }
2415 #endif /* !_WIN32 */
2416
2417 /*
2418 * Return a host pointer to guest's ram.
2419 * For Xen, foreign mappings get created if they don't already exist.
2420 *
2421 * @block: block for the RAM to lookup (optional and may be NULL).
2422 * @addr: address within the memory region.
2423 * @size: pointer to requested size (optional and may be NULL).
2424 * size may get modified and return a value smaller than
2425 * what was requested.
2426 * @lock: wether to lock the mapping in xen-mapcache until invalidated.
2427 * @is_write: hint wether to map RW or RO in the xen-mapcache.
2428 * (optional and may always be set to true).
2429 *
2430 * Called within RCU critical section.
2431 */
qemu_ram_ptr_length(RAMBlock * block,ram_addr_t addr,hwaddr * size,bool lock,bool is_write)2432 static void *qemu_ram_ptr_length(RAMBlock *block, ram_addr_t addr,
2433 hwaddr *size, bool lock,
2434 bool is_write)
2435 {
2436 hwaddr len = 0;
2437
2438 if (size && *size == 0) {
2439 return NULL;
2440 }
2441
2442 if (block == NULL) {
2443 block = qemu_get_ram_block(addr);
2444 addr -= block->offset;
2445 }
2446 if (size) {
2447 *size = MIN(*size, block->max_length - addr);
2448 len = *size;
2449 }
2450
2451 if (xen_enabled() && block->host == NULL) {
2452 /* We need to check if the requested address is in the RAM
2453 * because we don't want to map the entire memory in QEMU.
2454 * In that case just map the requested area.
2455 */
2456 if (xen_mr_is_memory(block->mr)) {
2457 return xen_map_cache(block->mr, block->offset + addr,
2458 len, block->offset,
2459 lock, lock, is_write);
2460 }
2461
2462 block->host = xen_map_cache(block->mr, block->offset,
2463 block->max_length,
2464 block->offset,
2465 1, lock, is_write);
2466 }
2467
2468 return ramblock_ptr(block, addr);
2469 }
2470
2471 /*
2472 * Return a host pointer to ram allocated with qemu_ram_alloc.
2473 * This should not be used for general purpose DMA. Use address_space_map
2474 * or address_space_rw instead. For local memory (e.g. video ram) that the
2475 * device owns, use memory_region_get_ram_ptr.
2476 *
2477 * Called within RCU critical section.
2478 */
qemu_map_ram_ptr(RAMBlock * ram_block,ram_addr_t addr)2479 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
2480 {
2481 return qemu_ram_ptr_length(ram_block, addr, NULL, false, true);
2482 }
2483
2484 /* Return the offset of a hostpointer within a ramblock */
qemu_ram_block_host_offset(RAMBlock * rb,void * host)2485 ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2486 {
2487 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2488 assert((uintptr_t)host >= (uintptr_t)rb->host);
2489 assert(res < rb->max_length);
2490
2491 return res;
2492 }
2493
qemu_ram_block_from_host(void * ptr,bool round_offset,ram_addr_t * offset)2494 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
2495 ram_addr_t *offset)
2496 {
2497 RAMBlock *block;
2498 uint8_t *host = ptr;
2499
2500 if (xen_enabled()) {
2501 ram_addr_t ram_addr;
2502 RCU_READ_LOCK_GUARD();
2503 ram_addr = xen_ram_addr_from_mapcache(ptr);
2504 if (ram_addr == RAM_ADDR_INVALID) {
2505 return NULL;
2506 }
2507
2508 block = qemu_get_ram_block(ram_addr);
2509 if (block) {
2510 *offset = ram_addr - block->offset;
2511 }
2512 return block;
2513 }
2514
2515 RCU_READ_LOCK_GUARD();
2516 block = qatomic_rcu_read(&ram_list.mru_block);
2517 if (block && block->host && host - block->host < block->max_length) {
2518 goto found;
2519 }
2520
2521 RAMBLOCK_FOREACH(block) {
2522 /* This case append when the block is not mapped. */
2523 if (block->host == NULL) {
2524 continue;
2525 }
2526 if (host - block->host < block->max_length) {
2527 goto found;
2528 }
2529 }
2530
2531 return NULL;
2532
2533 found:
2534 *offset = (host - block->host);
2535 if (round_offset) {
2536 *offset &= TARGET_PAGE_MASK;
2537 }
2538 return block;
2539 }
2540
2541 /*
2542 * Finds the named RAMBlock
2543 *
2544 * name: The name of RAMBlock to find
2545 *
2546 * Returns: RAMBlock (or NULL if not found)
2547 */
qemu_ram_block_by_name(const char * name)2548 RAMBlock *qemu_ram_block_by_name(const char *name)
2549 {
2550 RAMBlock *block;
2551
2552 RAMBLOCK_FOREACH(block) {
2553 if (!strcmp(name, block->idstr)) {
2554 return block;
2555 }
2556 }
2557
2558 return NULL;
2559 }
2560
2561 /*
2562 * Some of the system routines need to translate from a host pointer
2563 * (typically a TLB entry) back to a ram offset.
2564 */
qemu_ram_addr_from_host(void * ptr)2565 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2566 {
2567 RAMBlock *block;
2568 ram_addr_t offset;
2569
2570 block = qemu_ram_block_from_host(ptr, false, &offset);
2571 if (!block) {
2572 return RAM_ADDR_INVALID;
2573 }
2574
2575 return block->offset + offset;
2576 }
2577
qemu_ram_addr_from_host_nofail(void * ptr)2578 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2579 {
2580 ram_addr_t ram_addr;
2581
2582 ram_addr = qemu_ram_addr_from_host(ptr);
2583 if (ram_addr == RAM_ADDR_INVALID) {
2584 error_report("Bad ram pointer %p", ptr);
2585 abort();
2586 }
2587 return ram_addr;
2588 }
2589
2590 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2591 MemTxAttrs attrs, void *buf, hwaddr len);
2592 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2593 const void *buf, hwaddr len);
2594 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
2595 bool is_write, MemTxAttrs attrs);
2596
subpage_read(void * opaque,hwaddr addr,uint64_t * data,unsigned len,MemTxAttrs attrs)2597 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2598 unsigned len, MemTxAttrs attrs)
2599 {
2600 subpage_t *subpage = opaque;
2601 uint8_t buf[8];
2602 MemTxResult res;
2603
2604 #if defined(DEBUG_SUBPAGE)
2605 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__,
2606 subpage, len, addr);
2607 #endif
2608 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
2609 if (res) {
2610 return res;
2611 }
2612 *data = ldn_p(buf, len);
2613 return MEMTX_OK;
2614 }
2615
subpage_write(void * opaque,hwaddr addr,uint64_t value,unsigned len,MemTxAttrs attrs)2616 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2617 uint64_t value, unsigned len, MemTxAttrs attrs)
2618 {
2619 subpage_t *subpage = opaque;
2620 uint8_t buf[8];
2621
2622 #if defined(DEBUG_SUBPAGE)
2623 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx
2624 " value %"PRIx64"\n",
2625 __func__, subpage, len, addr, value);
2626 #endif
2627 stn_p(buf, len, value);
2628 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
2629 }
2630
subpage_accepts(void * opaque,hwaddr addr,unsigned len,bool is_write,MemTxAttrs attrs)2631 static bool subpage_accepts(void *opaque, hwaddr addr,
2632 unsigned len, bool is_write,
2633 MemTxAttrs attrs)
2634 {
2635 subpage_t *subpage = opaque;
2636 #if defined(DEBUG_SUBPAGE)
2637 printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n",
2638 __func__, subpage, is_write ? 'w' : 'r', len, addr);
2639 #endif
2640
2641 return flatview_access_valid(subpage->fv, addr + subpage->base,
2642 len, is_write, attrs);
2643 }
2644
2645 static const MemoryRegionOps subpage_ops = {
2646 .read_with_attrs = subpage_read,
2647 .write_with_attrs = subpage_write,
2648 .impl.min_access_size = 1,
2649 .impl.max_access_size = 8,
2650 .valid.min_access_size = 1,
2651 .valid.max_access_size = 8,
2652 .valid.accepts = subpage_accepts,
2653 .endianness = DEVICE_NATIVE_ENDIAN,
2654 };
2655
subpage_register(subpage_t * mmio,uint32_t start,uint32_t end,uint16_t section)2656 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2657 uint16_t section)
2658 {
2659 int idx, eidx;
2660
2661 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2662 return -1;
2663 idx = SUBPAGE_IDX(start);
2664 eidx = SUBPAGE_IDX(end);
2665 #if defined(DEBUG_SUBPAGE)
2666 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2667 __func__, mmio, start, end, idx, eidx, section);
2668 #endif
2669 for (; idx <= eidx; idx++) {
2670 mmio->sub_section[idx] = section;
2671 }
2672
2673 return 0;
2674 }
2675
subpage_init(FlatView * fv,hwaddr base)2676 static subpage_t *subpage_init(FlatView *fv, hwaddr base)
2677 {
2678 subpage_t *mmio;
2679
2680 /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
2681 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
2682 mmio->fv = fv;
2683 mmio->base = base;
2684 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2685 NULL, TARGET_PAGE_SIZE);
2686 mmio->iomem.subpage = true;
2687 #if defined(DEBUG_SUBPAGE)
2688 printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__,
2689 mmio, base, TARGET_PAGE_SIZE);
2690 #endif
2691
2692 return mmio;
2693 }
2694
dummy_section(PhysPageMap * map,FlatView * fv,MemoryRegion * mr)2695 static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
2696 {
2697 assert(fv);
2698 MemoryRegionSection section = {
2699 .fv = fv,
2700 .mr = mr,
2701 .offset_within_address_space = 0,
2702 .offset_within_region = 0,
2703 .size = int128_2_64(),
2704 };
2705
2706 return phys_section_add(map, §ion);
2707 }
2708
io_mem_init(void)2709 static void io_mem_init(void)
2710 {
2711 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2712 NULL, UINT64_MAX);
2713 }
2714
address_space_dispatch_new(FlatView * fv)2715 AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
2716 {
2717 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2718 uint16_t n;
2719
2720 n = dummy_section(&d->map, fv, &io_mem_unassigned);
2721 assert(n == PHYS_SECTION_UNASSIGNED);
2722
2723 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2724
2725 return d;
2726 }
2727
address_space_dispatch_free(AddressSpaceDispatch * d)2728 void address_space_dispatch_free(AddressSpaceDispatch *d)
2729 {
2730 phys_sections_free(&d->map);
2731 g_free(d);
2732 }
2733
do_nothing(CPUState * cpu,run_on_cpu_data d)2734 static void do_nothing(CPUState *cpu, run_on_cpu_data d)
2735 {
2736 }
2737
tcg_log_global_after_sync(MemoryListener * listener)2738 static void tcg_log_global_after_sync(MemoryListener *listener)
2739 {
2740 CPUAddressSpace *cpuas;
2741
2742 /* Wait for the CPU to end the current TB. This avoids the following
2743 * incorrect race:
2744 *
2745 * vCPU migration
2746 * ---------------------- -------------------------
2747 * TLB check -> slow path
2748 * notdirty_mem_write
2749 * write to RAM
2750 * mark dirty
2751 * clear dirty flag
2752 * TLB check -> fast path
2753 * read memory
2754 * write to RAM
2755 *
2756 * by pushing the migration thread's memory read after the vCPU thread has
2757 * written the memory.
2758 */
2759 if (replay_mode == REPLAY_MODE_NONE) {
2760 /*
2761 * VGA can make calls to this function while updating the screen.
2762 * In record/replay mode this causes a deadlock, because
2763 * run_on_cpu waits for rr mutex. Therefore no races are possible
2764 * in this case and no need for making run_on_cpu when
2765 * record/replay is enabled.
2766 */
2767 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2768 run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
2769 }
2770 }
2771
tcg_commit_cpu(CPUState * cpu,run_on_cpu_data data)2772 static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
2773 {
2774 tlb_flush(cpu);
2775 }
2776
tcg_commit(MemoryListener * listener)2777 static void tcg_commit(MemoryListener *listener)
2778 {
2779 CPUAddressSpace *cpuas;
2780 CPUState *cpu;
2781
2782 assert(tcg_enabled());
2783 /* since each CPU stores ram addresses in its TLB cache, we must
2784 reset the modified entries */
2785 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2786 cpu = cpuas->cpu;
2787
2788 /*
2789 * Queueing the work function will kick the cpu back to
2790 * the main loop, which will end the RCU critical section and reclaim
2791 * the memory data structures.
2792 *
2793 * That said, the listener is also called during realize, before
2794 * all of the tcg machinery for run-on is initialized: thus halt_cond.
2795 */
2796 if (cpu->halt_cond) {
2797 async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2798 } else {
2799 tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2800 }
2801 }
2802
memory_map_init(void)2803 static void memory_map_init(void)
2804 {
2805 system_memory = g_malloc(sizeof(*system_memory));
2806
2807 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2808 address_space_init(&address_space_memory, system_memory, "memory");
2809
2810 system_io = g_malloc(sizeof(*system_io));
2811 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2812 65536);
2813 address_space_init(&address_space_io, system_io, "I/O");
2814 }
2815
get_system_memory(void)2816 MemoryRegion *get_system_memory(void)
2817 {
2818 return system_memory;
2819 }
2820
get_system_io(void)2821 MemoryRegion *get_system_io(void)
2822 {
2823 return system_io;
2824 }
2825
invalidate_and_set_dirty(MemoryRegion * mr,hwaddr addr,hwaddr length)2826 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2827 hwaddr length)
2828 {
2829 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2830 ram_addr_t ramaddr = memory_region_get_ram_addr(mr);
2831
2832 /* We know we're only called for RAM MemoryRegions */
2833 assert(ramaddr != RAM_ADDR_INVALID);
2834 addr += ramaddr;
2835
2836 /* No early return if dirty_log_mask is or becomes 0, because
2837 * cpu_physical_memory_set_dirty_range will still call
2838 * xen_modified_memory.
2839 */
2840 if (dirty_log_mask) {
2841 dirty_log_mask =
2842 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2843 }
2844 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2845 assert(tcg_enabled());
2846 tb_invalidate_phys_range(NULL, addr, addr + length - 1);
2847 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2848 }
2849 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2850 }
2851
memory_region_flush_rom_device(MemoryRegion * mr,hwaddr addr,hwaddr size)2852 void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
2853 {
2854 /*
2855 * In principle this function would work on other memory region types too,
2856 * but the ROM device use case is the only one where this operation is
2857 * necessary. Other memory regions should use the
2858 * address_space_read/write() APIs.
2859 */
2860 assert(memory_region_is_romd(mr));
2861
2862 invalidate_and_set_dirty(mr, addr, size);
2863 }
2864
memory_access_size(MemoryRegion * mr,unsigned l,hwaddr addr)2865 int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2866 {
2867 unsigned access_size_max = mr->ops->valid.max_access_size;
2868
2869 /* Regions are assumed to support 1-4 byte accesses unless
2870 otherwise specified. */
2871 if (access_size_max == 0) {
2872 access_size_max = 4;
2873 }
2874
2875 /* Bound the maximum access by the alignment of the address. */
2876 if (!mr->ops->impl.unaligned) {
2877 unsigned align_size_max = addr & -addr;
2878 if (align_size_max != 0 && align_size_max < access_size_max) {
2879 access_size_max = align_size_max;
2880 }
2881 }
2882
2883 /* Don't attempt accesses larger than the maximum. */
2884 if (l > access_size_max) {
2885 l = access_size_max;
2886 }
2887 l = pow2floor(l);
2888
2889 return l;
2890 }
2891
prepare_mmio_access(MemoryRegion * mr)2892 bool prepare_mmio_access(MemoryRegion *mr)
2893 {
2894 bool release_lock = false;
2895
2896 if (!bql_locked()) {
2897 bql_lock();
2898 release_lock = true;
2899 }
2900 if (mr->flush_coalesced_mmio) {
2901 qemu_flush_coalesced_mmio_buffer();
2902 }
2903
2904 return release_lock;
2905 }
2906
2907 /**
2908 * flatview_access_allowed
2909 * @mr: #MemoryRegion to be accessed
2910 * @attrs: memory transaction attributes
2911 * @addr: address within that memory region
2912 * @len: the number of bytes to access
2913 *
2914 * Check if a memory transaction is allowed.
2915 *
2916 * Returns: true if transaction is allowed, false if denied.
2917 */
flatview_access_allowed(MemoryRegion * mr,MemTxAttrs attrs,hwaddr addr,hwaddr len)2918 static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
2919 hwaddr addr, hwaddr len)
2920 {
2921 if (likely(!attrs.memory)) {
2922 return true;
2923 }
2924 if (memory_region_is_ram(mr)) {
2925 return true;
2926 }
2927 qemu_log_mask(LOG_INVALID_MEM,
2928 "Invalid access to non-RAM device at "
2929 "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
2930 "region '%s'\n", addr, len, memory_region_name(mr));
2931 return false;
2932 }
2933
flatview_write_continue_step(MemTxAttrs attrs,const uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)2934 static MemTxResult flatview_write_continue_step(MemTxAttrs attrs,
2935 const uint8_t *buf,
2936 hwaddr len, hwaddr mr_addr,
2937 hwaddr *l, MemoryRegion *mr)
2938 {
2939 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2940 return MEMTX_ACCESS_ERROR;
2941 }
2942
2943 if (!memory_access_is_direct(mr, true, attrs)) {
2944 uint64_t val;
2945 MemTxResult result;
2946 bool release_lock = prepare_mmio_access(mr);
2947
2948 *l = memory_access_size(mr, *l, mr_addr);
2949 /*
2950 * XXX: could force current_cpu to NULL to avoid
2951 * potential bugs
2952 */
2953
2954 /*
2955 * Assure Coverity (and ourselves) that we are not going to OVERRUN
2956 * the buffer by following ldn_he_p().
2957 */
2958 #ifdef QEMU_STATIC_ANALYSIS
2959 assert((*l == 1 && len >= 1) ||
2960 (*l == 2 && len >= 2) ||
2961 (*l == 4 && len >= 4) ||
2962 (*l == 8 && len >= 8));
2963 #endif
2964 val = ldn_he_p(buf, *l);
2965 result = memory_region_dispatch_write(mr, mr_addr, val,
2966 size_memop(*l), attrs);
2967 if (release_lock) {
2968 bql_unlock();
2969 }
2970
2971 return result;
2972 } else {
2973 /* RAM case */
2974 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
2975 false, true);
2976
2977 memmove(ram_ptr, buf, *l);
2978 invalidate_and_set_dirty(mr, mr_addr, *l);
2979
2980 return MEMTX_OK;
2981 }
2982 }
2983
2984 /* Called within RCU critical section. */
flatview_write_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)2985 static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
2986 MemTxAttrs attrs,
2987 const void *ptr,
2988 hwaddr len, hwaddr mr_addr,
2989 hwaddr l, MemoryRegion *mr)
2990 {
2991 MemTxResult result = MEMTX_OK;
2992 const uint8_t *buf = ptr;
2993
2994 for (;;) {
2995 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
2996 mr);
2997
2998 len -= l;
2999 buf += l;
3000 addr += l;
3001
3002 if (!len) {
3003 break;
3004 }
3005
3006 l = len;
3007 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3008 }
3009
3010 return result;
3011 }
3012
3013 /* Called from RCU critical section. */
flatview_write(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3014 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
3015 const void *buf, hwaddr len)
3016 {
3017 hwaddr l;
3018 hwaddr mr_addr;
3019 MemoryRegion *mr;
3020
3021 l = len;
3022 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
3023 if (!flatview_access_allowed(mr, attrs, addr, len)) {
3024 return MEMTX_ACCESS_ERROR;
3025 }
3026 return flatview_write_continue(fv, addr, attrs, buf, len,
3027 mr_addr, l, mr);
3028 }
3029
flatview_read_continue_step(MemTxAttrs attrs,uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)3030 static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf,
3031 hwaddr len, hwaddr mr_addr,
3032 hwaddr *l,
3033 MemoryRegion *mr)
3034 {
3035 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
3036 return MEMTX_ACCESS_ERROR;
3037 }
3038
3039 if (!memory_access_is_direct(mr, false, attrs)) {
3040 /* I/O case */
3041 uint64_t val;
3042 MemTxResult result;
3043 bool release_lock = prepare_mmio_access(mr);
3044
3045 *l = memory_access_size(mr, *l, mr_addr);
3046 result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l),
3047 attrs);
3048
3049 /*
3050 * Assure Coverity (and ourselves) that we are not going to OVERRUN
3051 * the buffer by following stn_he_p().
3052 */
3053 #ifdef QEMU_STATIC_ANALYSIS
3054 assert((*l == 1 && len >= 1) ||
3055 (*l == 2 && len >= 2) ||
3056 (*l == 4 && len >= 4) ||
3057 (*l == 8 && len >= 8));
3058 #endif
3059 stn_he_p(buf, *l, val);
3060
3061 if (release_lock) {
3062 bql_unlock();
3063 }
3064 return result;
3065 } else {
3066 /* RAM case */
3067 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
3068 false, false);
3069
3070 memcpy(buf, ram_ptr, *l);
3071
3072 return MEMTX_OK;
3073 }
3074 }
3075
3076 /* Called within RCU critical section. */
flatview_read_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3077 MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
3078 MemTxAttrs attrs, void *ptr,
3079 hwaddr len, hwaddr mr_addr, hwaddr l,
3080 MemoryRegion *mr)
3081 {
3082 MemTxResult result = MEMTX_OK;
3083 uint8_t *buf = ptr;
3084
3085 fuzz_dma_read_cb(addr, len, mr);
3086 for (;;) {
3087 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3088
3089 len -= l;
3090 buf += l;
3091 addr += l;
3092
3093 if (!len) {
3094 break;
3095 }
3096
3097 l = len;
3098 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3099 }
3100
3101 return result;
3102 }
3103
3104 /* Called from RCU critical section. */
flatview_read(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)3105 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
3106 MemTxAttrs attrs, void *buf, hwaddr len)
3107 {
3108 hwaddr l;
3109 hwaddr mr_addr;
3110 MemoryRegion *mr;
3111
3112 l = len;
3113 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
3114 if (!flatview_access_allowed(mr, attrs, addr, len)) {
3115 return MEMTX_ACCESS_ERROR;
3116 }
3117 return flatview_read_continue(fv, addr, attrs, buf, len,
3118 mr_addr, l, mr);
3119 }
3120
address_space_read_full(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)3121 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
3122 MemTxAttrs attrs, void *buf, hwaddr len)
3123 {
3124 MemTxResult result = MEMTX_OK;
3125 FlatView *fv;
3126
3127 if (len > 0) {
3128 RCU_READ_LOCK_GUARD();
3129 fv = address_space_to_flatview(as);
3130 result = flatview_read(fv, addr, attrs, buf, len);
3131 }
3132
3133 return result;
3134 }
3135
address_space_write(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3136 MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
3137 MemTxAttrs attrs,
3138 const void *buf, hwaddr len)
3139 {
3140 MemTxResult result = MEMTX_OK;
3141 FlatView *fv;
3142
3143 if (len > 0) {
3144 RCU_READ_LOCK_GUARD();
3145 fv = address_space_to_flatview(as);
3146 result = flatview_write(fv, addr, attrs, buf, len);
3147 }
3148
3149 return result;
3150 }
3151
address_space_rw(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len,bool is_write)3152 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
3153 void *buf, hwaddr len, bool is_write)
3154 {
3155 if (is_write) {
3156 return address_space_write(as, addr, attrs, buf, len);
3157 } else {
3158 return address_space_read_full(as, addr, attrs, buf, len);
3159 }
3160 }
3161
address_space_set(AddressSpace * as,hwaddr addr,uint8_t c,hwaddr len,MemTxAttrs attrs)3162 MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
3163 uint8_t c, hwaddr len, MemTxAttrs attrs)
3164 {
3165 #define FILLBUF_SIZE 512
3166 uint8_t fillbuf[FILLBUF_SIZE];
3167 int l;
3168 MemTxResult error = MEMTX_OK;
3169
3170 memset(fillbuf, c, FILLBUF_SIZE);
3171 while (len > 0) {
3172 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
3173 error |= address_space_write(as, addr, attrs, fillbuf, l);
3174 len -= l;
3175 addr += l;
3176 }
3177
3178 return error;
3179 }
3180
cpu_physical_memory_rw(hwaddr addr,void * buf,hwaddr len,bool is_write)3181 void cpu_physical_memory_rw(hwaddr addr, void *buf,
3182 hwaddr len, bool is_write)
3183 {
3184 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3185 buf, len, is_write);
3186 }
3187
3188 enum write_rom_type {
3189 WRITE_DATA,
3190 FLUSH_CACHE,
3191 };
3192
address_space_write_rom_internal(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,enum write_rom_type type)3193 static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
3194 hwaddr addr,
3195 MemTxAttrs attrs,
3196 const void *ptr,
3197 hwaddr len,
3198 enum write_rom_type type)
3199 {
3200 hwaddr l;
3201 uint8_t *ram_ptr;
3202 hwaddr addr1;
3203 MemoryRegion *mr;
3204 const uint8_t *buf = ptr;
3205
3206 RCU_READ_LOCK_GUARD();
3207 while (len > 0) {
3208 l = len;
3209 mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
3210
3211 if (!memory_region_supports_direct_access(mr)) {
3212 l = memory_access_size(mr, l, addr1);
3213 } else {
3214 /* ROM/RAM case */
3215 ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3216 switch (type) {
3217 case WRITE_DATA:
3218 memcpy(ram_ptr, buf, l);
3219 invalidate_and_set_dirty(mr, addr1, l);
3220 break;
3221 case FLUSH_CACHE:
3222 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
3223 break;
3224 }
3225 }
3226 len -= l;
3227 buf += l;
3228 addr += l;
3229 }
3230 return MEMTX_OK;
3231 }
3232
3233 /* used for ROM loading : can write in RAM and ROM */
address_space_write_rom(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3234 MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
3235 MemTxAttrs attrs,
3236 const void *buf, hwaddr len)
3237 {
3238 return address_space_write_rom_internal(as, addr, attrs,
3239 buf, len, WRITE_DATA);
3240 }
3241
cpu_flush_icache_range(hwaddr start,hwaddr len)3242 void cpu_flush_icache_range(hwaddr start, hwaddr len)
3243 {
3244 /*
3245 * This function should do the same thing as an icache flush that was
3246 * triggered from within the guest. For TCG we are always cache coherent,
3247 * so there is no need to flush anything. For KVM / Xen we need to flush
3248 * the host's instruction cache at least.
3249 */
3250 if (tcg_enabled()) {
3251 return;
3252 }
3253
3254 address_space_write_rom_internal(&address_space_memory,
3255 start, MEMTXATTRS_UNSPECIFIED,
3256 NULL, len, FLUSH_CACHE);
3257 }
3258
3259 /*
3260 * A magic value stored in the first 8 bytes of the bounce buffer struct. Used
3261 * to detect illegal pointers passed to address_space_unmap.
3262 */
3263 #define BOUNCE_BUFFER_MAGIC 0xb4017ceb4ffe12ed
3264
3265 typedef struct {
3266 uint64_t magic;
3267 MemoryRegion *mr;
3268 hwaddr addr;
3269 size_t len;
3270 uint8_t buffer[];
3271 } BounceBuffer;
3272
3273 static void
address_space_unregister_map_client_do(AddressSpaceMapClient * client)3274 address_space_unregister_map_client_do(AddressSpaceMapClient *client)
3275 {
3276 QLIST_REMOVE(client, link);
3277 g_free(client);
3278 }
3279
address_space_notify_map_clients_locked(AddressSpace * as)3280 static void address_space_notify_map_clients_locked(AddressSpace *as)
3281 {
3282 AddressSpaceMapClient *client;
3283
3284 while (!QLIST_EMPTY(&as->map_client_list)) {
3285 client = QLIST_FIRST(&as->map_client_list);
3286 qemu_bh_schedule(client->bh);
3287 address_space_unregister_map_client_do(client);
3288 }
3289 }
3290
address_space_register_map_client(AddressSpace * as,QEMUBH * bh)3291 void address_space_register_map_client(AddressSpace *as, QEMUBH *bh)
3292 {
3293 AddressSpaceMapClient *client = g_malloc(sizeof(*client));
3294
3295 QEMU_LOCK_GUARD(&as->map_client_list_lock);
3296 client->bh = bh;
3297 QLIST_INSERT_HEAD(&as->map_client_list, client, link);
3298 /* Write map_client_list before reading bounce_buffer_size. */
3299 smp_mb();
3300 if (qatomic_read(&as->bounce_buffer_size) < as->max_bounce_buffer_size) {
3301 address_space_notify_map_clients_locked(as);
3302 }
3303 }
3304
cpu_exec_init_all(void)3305 void cpu_exec_init_all(void)
3306 {
3307 qemu_mutex_init(&ram_list.mutex);
3308 /* The data structures we set up here depend on knowing the page size,
3309 * so no more changes can be made after this point.
3310 * In an ideal world, nothing we did before we had finished the
3311 * machine setup would care about the target page size, and we could
3312 * do this much later, rather than requiring board models to state
3313 * up front what their requirements are.
3314 */
3315 finalize_target_page_bits();
3316 io_mem_init();
3317 memory_map_init();
3318 }
3319
address_space_unregister_map_client(AddressSpace * as,QEMUBH * bh)3320 void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh)
3321 {
3322 AddressSpaceMapClient *client;
3323
3324 QEMU_LOCK_GUARD(&as->map_client_list_lock);
3325 QLIST_FOREACH(client, &as->map_client_list, link) {
3326 if (client->bh == bh) {
3327 address_space_unregister_map_client_do(client);
3328 break;
3329 }
3330 }
3331 }
3332
address_space_notify_map_clients(AddressSpace * as)3333 static void address_space_notify_map_clients(AddressSpace *as)
3334 {
3335 QEMU_LOCK_GUARD(&as->map_client_list_lock);
3336 address_space_notify_map_clients_locked(as);
3337 }
3338
flatview_access_valid(FlatView * fv,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3339 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
3340 bool is_write, MemTxAttrs attrs)
3341 {
3342 MemoryRegion *mr;
3343 hwaddr l, xlat;
3344
3345 while (len > 0) {
3346 l = len;
3347 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3348 if (!memory_access_is_direct(mr, is_write, attrs)) {
3349 l = memory_access_size(mr, l, addr);
3350 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
3351 return false;
3352 }
3353 }
3354
3355 len -= l;
3356 addr += l;
3357 }
3358 return true;
3359 }
3360
address_space_access_valid(AddressSpace * as,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3361 bool address_space_access_valid(AddressSpace *as, hwaddr addr,
3362 hwaddr len, bool is_write,
3363 MemTxAttrs attrs)
3364 {
3365 FlatView *fv;
3366
3367 RCU_READ_LOCK_GUARD();
3368 fv = address_space_to_flatview(as);
3369 return flatview_access_valid(fv, addr, len, is_write, attrs);
3370 }
3371
3372 static hwaddr
flatview_extend_translation(FlatView * fv,hwaddr addr,hwaddr target_len,MemoryRegion * mr,hwaddr base,hwaddr len,bool is_write,MemTxAttrs attrs)3373 flatview_extend_translation(FlatView *fv, hwaddr addr,
3374 hwaddr target_len,
3375 MemoryRegion *mr, hwaddr base, hwaddr len,
3376 bool is_write, MemTxAttrs attrs)
3377 {
3378 hwaddr done = 0;
3379 hwaddr xlat;
3380 MemoryRegion *this_mr;
3381
3382 for (;;) {
3383 target_len -= len;
3384 addr += len;
3385 done += len;
3386 if (target_len == 0) {
3387 return done;
3388 }
3389
3390 len = target_len;
3391 this_mr = flatview_translate(fv, addr, &xlat,
3392 &len, is_write, attrs);
3393 if (this_mr != mr || xlat != base + done) {
3394 return done;
3395 }
3396 }
3397 }
3398
3399 /* Map a physical memory region into a host virtual address.
3400 * May map a subset of the requested range, given by and returned in *plen.
3401 * May return NULL if resources needed to perform the mapping are exhausted.
3402 * Use only for reads OR writes - not for read-modify-write operations.
3403 * Use address_space_register_map_client() to know when retrying the map
3404 * operation is likely to succeed.
3405 */
address_space_map(AddressSpace * as,hwaddr addr,hwaddr * plen,bool is_write,MemTxAttrs attrs)3406 void *address_space_map(AddressSpace *as,
3407 hwaddr addr,
3408 hwaddr *plen,
3409 bool is_write,
3410 MemTxAttrs attrs)
3411 {
3412 hwaddr len = *plen;
3413 hwaddr l, xlat;
3414 MemoryRegion *mr;
3415 FlatView *fv;
3416
3417 trace_address_space_map(as, addr, len, is_write, *(uint32_t *) &attrs);
3418
3419 if (len == 0) {
3420 return NULL;
3421 }
3422
3423 l = len;
3424 RCU_READ_LOCK_GUARD();
3425 fv = address_space_to_flatview(as);
3426 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3427
3428 if (!memory_access_is_direct(mr, is_write, attrs)) {
3429 size_t used = qatomic_read(&as->bounce_buffer_size);
3430 for (;;) {
3431 hwaddr alloc = MIN(as->max_bounce_buffer_size - used, l);
3432 size_t new_size = used + alloc;
3433 size_t actual =
3434 qatomic_cmpxchg(&as->bounce_buffer_size, used, new_size);
3435 if (actual == used) {
3436 l = alloc;
3437 break;
3438 }
3439 used = actual;
3440 }
3441
3442 if (l == 0) {
3443 *plen = 0;
3444 return NULL;
3445 }
3446
3447 BounceBuffer *bounce = g_malloc0(l + sizeof(BounceBuffer));
3448 bounce->magic = BOUNCE_BUFFER_MAGIC;
3449 memory_region_ref(mr);
3450 bounce->mr = mr;
3451 bounce->addr = addr;
3452 bounce->len = l;
3453
3454 if (!is_write) {
3455 flatview_read(fv, addr, attrs,
3456 bounce->buffer, l);
3457 }
3458
3459 *plen = l;
3460 return bounce->buffer;
3461 }
3462
3463 memory_region_ref(mr);
3464 *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
3465 l, is_write, attrs);
3466 fuzz_dma_read_cb(addr, *plen, mr);
3467 return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write);
3468 }
3469
3470 /* Unmaps a memory region previously mapped by address_space_map().
3471 * Will also mark the memory as dirty if is_write is true. access_len gives
3472 * the amount of memory that was actually read or written by the caller.
3473 */
address_space_unmap(AddressSpace * as,void * buffer,hwaddr len,bool is_write,hwaddr access_len)3474 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3475 bool is_write, hwaddr access_len)
3476 {
3477 MemoryRegion *mr;
3478 ram_addr_t addr1;
3479
3480 mr = memory_region_from_host(buffer, &addr1);
3481 if (mr != NULL) {
3482 if (is_write) {
3483 invalidate_and_set_dirty(mr, addr1, access_len);
3484 }
3485 if (xen_enabled()) {
3486 xen_invalidate_map_cache_entry(buffer);
3487 }
3488 memory_region_unref(mr);
3489 return;
3490 }
3491
3492
3493 BounceBuffer *bounce = container_of(buffer, BounceBuffer, buffer);
3494 assert(bounce->magic == BOUNCE_BUFFER_MAGIC);
3495
3496 if (is_write) {
3497 address_space_write(as, bounce->addr, MEMTXATTRS_UNSPECIFIED,
3498 bounce->buffer, access_len);
3499 }
3500
3501 qatomic_sub(&as->bounce_buffer_size, bounce->len);
3502 bounce->magic = ~BOUNCE_BUFFER_MAGIC;
3503 memory_region_unref(bounce->mr);
3504 g_free(bounce);
3505 /* Write bounce_buffer_size before reading map_client_list. */
3506 smp_mb();
3507 address_space_notify_map_clients(as);
3508 }
3509
cpu_physical_memory_map(hwaddr addr,hwaddr * plen,bool is_write)3510 void *cpu_physical_memory_map(hwaddr addr,
3511 hwaddr *plen,
3512 bool is_write)
3513 {
3514 return address_space_map(&address_space_memory, addr, plen, is_write,
3515 MEMTXATTRS_UNSPECIFIED);
3516 }
3517
cpu_physical_memory_unmap(void * buffer,hwaddr len,bool is_write,hwaddr access_len)3518 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3519 bool is_write, hwaddr access_len)
3520 {
3521 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3522 }
3523
3524 #define ARG1_DECL AddressSpace *as
3525 #define ARG1 as
3526 #define SUFFIX
3527 #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__)
3528 #define RCU_READ_LOCK(...) rcu_read_lock()
3529 #define RCU_READ_UNLOCK(...) rcu_read_unlock()
3530 #include "memory_ldst.c.inc"
3531
address_space_cache_init(MemoryRegionCache * cache,AddressSpace * as,hwaddr addr,hwaddr len,bool is_write)3532 int64_t address_space_cache_init(MemoryRegionCache *cache,
3533 AddressSpace *as,
3534 hwaddr addr,
3535 hwaddr len,
3536 bool is_write)
3537 {
3538 AddressSpaceDispatch *d;
3539 hwaddr l;
3540 MemoryRegion *mr;
3541 Int128 diff;
3542
3543 assert(len > 0);
3544
3545 l = len;
3546 cache->fv = address_space_get_flatview(as);
3547 d = flatview_to_dispatch(cache->fv);
3548 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3549
3550 /*
3551 * cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3552 * Take that into account to compute how many bytes are there between
3553 * cache->xlat and the end of the section.
3554 */
3555 diff = int128_sub(cache->mrs.size,
3556 int128_make64(cache->xlat - cache->mrs.offset_within_region));
3557 l = int128_get64(int128_min(diff, int128_make64(l)));
3558
3559 mr = cache->mrs.mr;
3560 memory_region_ref(mr);
3561 if (memory_access_is_direct(mr, is_write, MEMTXATTRS_UNSPECIFIED)) {
3562 /* We don't care about the memory attributes here as we're only
3563 * doing this if we found actual RAM, which behaves the same
3564 * regardless of attributes; so UNSPECIFIED is fine.
3565 */
3566 l = flatview_extend_translation(cache->fv, addr, len, mr,
3567 cache->xlat, l, is_write,
3568 MEMTXATTRS_UNSPECIFIED);
3569 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true,
3570 is_write);
3571 } else {
3572 cache->ptr = NULL;
3573 }
3574
3575 cache->len = l;
3576 cache->is_write = is_write;
3577 return l;
3578 }
3579
address_space_cache_invalidate(MemoryRegionCache * cache,hwaddr addr,hwaddr access_len)3580 void address_space_cache_invalidate(MemoryRegionCache *cache,
3581 hwaddr addr,
3582 hwaddr access_len)
3583 {
3584 assert(cache->is_write);
3585 if (likely(cache->ptr)) {
3586 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3587 }
3588 }
3589
address_space_cache_destroy(MemoryRegionCache * cache)3590 void address_space_cache_destroy(MemoryRegionCache *cache)
3591 {
3592 if (!cache->mrs.mr) {
3593 return;
3594 }
3595
3596 if (xen_enabled()) {
3597 xen_invalidate_map_cache_entry(cache->ptr);
3598 }
3599 memory_region_unref(cache->mrs.mr);
3600 flatview_unref(cache->fv);
3601 cache->mrs.mr = NULL;
3602 cache->fv = NULL;
3603 }
3604
3605 /* Called from RCU critical section. This function has the same
3606 * semantics as address_space_translate, but it only works on a
3607 * predefined range of a MemoryRegion that was mapped with
3608 * address_space_cache_init.
3609 */
address_space_translate_cached(MemoryRegionCache * cache,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)3610 static inline MemoryRegion *address_space_translate_cached(
3611 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
3612 hwaddr *plen, bool is_write, MemTxAttrs attrs)
3613 {
3614 MemoryRegionSection section;
3615 MemoryRegion *mr;
3616 IOMMUMemoryRegion *iommu_mr;
3617 AddressSpace *target_as;
3618
3619 assert(!cache->ptr);
3620 *xlat = addr + cache->xlat;
3621
3622 mr = cache->mrs.mr;
3623 iommu_mr = memory_region_get_iommu(mr);
3624 if (!iommu_mr) {
3625 /* MMIO region. */
3626 return mr;
3627 }
3628
3629 section = address_space_translate_iommu(iommu_mr, xlat, plen,
3630 NULL, is_write, true,
3631 &target_as, attrs);
3632 return section.mr;
3633 }
3634
3635 /* Called within RCU critical section. */
address_space_write_continue_cached(MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3636 static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs,
3637 const void *ptr,
3638 hwaddr len,
3639 hwaddr mr_addr,
3640 hwaddr l,
3641 MemoryRegion *mr)
3642 {
3643 MemTxResult result = MEMTX_OK;
3644 const uint8_t *buf = ptr;
3645
3646 for (;;) {
3647 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3648 mr);
3649
3650 len -= l;
3651 buf += l;
3652 mr_addr += l;
3653
3654 if (!len) {
3655 break;
3656 }
3657
3658 l = len;
3659 }
3660
3661 return result;
3662 }
3663
3664 /* Called within RCU critical section. */
address_space_read_continue_cached(MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3665 static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs,
3666 void *ptr, hwaddr len,
3667 hwaddr mr_addr, hwaddr l,
3668 MemoryRegion *mr)
3669 {
3670 MemTxResult result = MEMTX_OK;
3671 uint8_t *buf = ptr;
3672
3673 for (;;) {
3674 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3675 len -= l;
3676 buf += l;
3677 mr_addr += l;
3678
3679 if (!len) {
3680 break;
3681 }
3682 l = len;
3683 }
3684
3685 return result;
3686 }
3687
3688 /* Called from RCU critical section. address_space_read_cached uses this
3689 * out of line function when the target is an MMIO or IOMMU region.
3690 */
3691 MemTxResult
address_space_read_cached_slow(MemoryRegionCache * cache,hwaddr addr,void * buf,hwaddr len)3692 address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3693 void *buf, hwaddr len)
3694 {
3695 hwaddr mr_addr, l;
3696 MemoryRegion *mr;
3697
3698 l = len;
3699 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false,
3700 MEMTXATTRS_UNSPECIFIED);
3701 return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED,
3702 buf, len, mr_addr, l, mr);
3703 }
3704
3705 /* Called from RCU critical section. address_space_write_cached uses this
3706 * out of line function when the target is an MMIO or IOMMU region.
3707 */
3708 MemTxResult
address_space_write_cached_slow(MemoryRegionCache * cache,hwaddr addr,const void * buf,hwaddr len)3709 address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3710 const void *buf, hwaddr len)
3711 {
3712 hwaddr mr_addr, l;
3713 MemoryRegion *mr;
3714
3715 l = len;
3716 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true,
3717 MEMTXATTRS_UNSPECIFIED);
3718 return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED,
3719 buf, len, mr_addr, l, mr);
3720 }
3721
3722 #define ARG1_DECL MemoryRegionCache *cache
3723 #define ARG1 cache
3724 #define SUFFIX _cached_slow
3725 #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__)
3726 #define RCU_READ_LOCK() ((void)0)
3727 #define RCU_READ_UNLOCK() ((void)0)
3728 #include "memory_ldst.c.inc"
3729
3730 /* virtual memory access for debug (includes writing to ROM) */
cpu_memory_rw_debug(CPUState * cpu,vaddr addr,void * ptr,size_t len,bool is_write)3731 int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
3732 void *ptr, size_t len, bool is_write)
3733 {
3734 hwaddr phys_addr;
3735 vaddr l, page;
3736 uint8_t *buf = ptr;
3737
3738 cpu_synchronize_state(cpu);
3739 while (len > 0) {
3740 int asidx;
3741 MemTxAttrs attrs;
3742 MemTxResult res;
3743
3744 page = addr & TARGET_PAGE_MASK;
3745 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3746 asidx = cpu_asidx_from_attrs(cpu, attrs);
3747 /* if no physical page mapped, return an error */
3748 if (phys_addr == -1)
3749 return -1;
3750 l = (page + TARGET_PAGE_SIZE) - addr;
3751 if (l > len)
3752 l = len;
3753 phys_addr += (addr & ~TARGET_PAGE_MASK);
3754 res = address_space_rw(cpu->cpu_ases[asidx].as, phys_addr, attrs, buf,
3755 l, is_write);
3756 if (res != MEMTX_OK) {
3757 return -1;
3758 }
3759 len -= l;
3760 buf += l;
3761 addr += l;
3762 }
3763 return 0;
3764 }
3765
cpu_physical_memory_is_io(hwaddr phys_addr)3766 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3767 {
3768 MemoryRegion*mr;
3769 hwaddr l = 1;
3770
3771 RCU_READ_LOCK_GUARD();
3772 mr = address_space_translate(&address_space_memory,
3773 phys_addr, &phys_addr, &l, false,
3774 MEMTXATTRS_UNSPECIFIED);
3775
3776 return !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3777 }
3778
qemu_ram_foreach_block(RAMBlockIterFunc func,void * opaque)3779 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3780 {
3781 RAMBlock *block;
3782 int ret = 0;
3783
3784 RCU_READ_LOCK_GUARD();
3785 RAMBLOCK_FOREACH(block) {
3786 ret = func(block, opaque);
3787 if (ret) {
3788 break;
3789 }
3790 }
3791 return ret;
3792 }
3793
3794 /*
3795 * Unmap pages of memory from start to start+length such that
3796 * they a) read as 0, b) Trigger whatever fault mechanism
3797 * the OS provides for postcopy.
3798 * The pages must be unmapped by the end of the function.
3799 * Returns: 0 on success, none-0 on failure
3800 *
3801 */
ram_block_discard_range(RAMBlock * rb,uint64_t start,size_t length)3802 int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3803 {
3804 int ret = -1;
3805
3806 uint8_t *host_startaddr = rb->host + start;
3807
3808 if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
3809 error_report("%s: Unaligned start address: %p",
3810 __func__, host_startaddr);
3811 goto err;
3812 }
3813
3814 if ((start + length) <= rb->max_length) {
3815 bool need_madvise, need_fallocate;
3816 if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
3817 error_report("%s: Unaligned length: %zx", __func__, length);
3818 goto err;
3819 }
3820
3821 errno = ENOTSUP; /* If we are missing MADVISE etc */
3822
3823 /* The logic here is messy;
3824 * madvise DONTNEED fails for hugepages
3825 * fallocate works on hugepages and shmem
3826 * shared anonymous memory requires madvise REMOVE
3827 */
3828 need_madvise = (rb->page_size == qemu_real_host_page_size());
3829 need_fallocate = rb->fd != -1;
3830 if (need_fallocate) {
3831 /* For a file, this causes the area of the file to be zero'd
3832 * if read, and for hugetlbfs also causes it to be unmapped
3833 * so a userfault will trigger.
3834 */
3835 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3836 /*
3837 * fallocate() will fail with readonly files. Let's print a
3838 * proper error message.
3839 */
3840 if (rb->flags & RAM_READONLY_FD) {
3841 error_report("%s: Discarding RAM with readonly files is not"
3842 " supported", __func__);
3843 goto err;
3844
3845 }
3846 /*
3847 * We'll discard data from the actual file, even though we only
3848 * have a MAP_PRIVATE mapping, possibly messing with other
3849 * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
3850 * change that behavior whithout violating the promised
3851 * semantics of ram_block_discard_range().
3852 *
3853 * Only warn, because it works as long as nobody else uses that
3854 * file.
3855 */
3856 if (!qemu_ram_is_shared(rb)) {
3857 warn_report_once("%s: Discarding RAM"
3858 " in private file mappings is possibly"
3859 " dangerous, because it will modify the"
3860 " underlying file and will affect other"
3861 " users of the file", __func__);
3862 }
3863
3864 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3865 start + rb->fd_offset, length);
3866 if (ret) {
3867 ret = -errno;
3868 error_report("%s: Failed to fallocate %s:%" PRIx64 "+%" PRIx64
3869 " +%zx (%d)", __func__, rb->idstr, start,
3870 rb->fd_offset, length, ret);
3871 goto err;
3872 }
3873 #else
3874 ret = -ENOSYS;
3875 error_report("%s: fallocate not available/file"
3876 "%s:%" PRIx64 "+%" PRIx64 " +%zx (%d)", __func__,
3877 rb->idstr, start, rb->fd_offset, length, ret);
3878 goto err;
3879 #endif
3880 }
3881 if (need_madvise) {
3882 /* For normal RAM this causes it to be unmapped,
3883 * for shared memory it causes the local mapping to disappear
3884 * and to fall back on the file contents (which we just
3885 * fallocate'd away).
3886 */
3887 #if defined(CONFIG_MADVISE)
3888 if (qemu_ram_is_shared(rb) && rb->fd < 0) {
3889 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
3890 } else {
3891 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
3892 }
3893 if (ret) {
3894 ret = -errno;
3895 error_report("%s: Failed to discard range "
3896 "%s:%" PRIx64 " +%zx (%d)",
3897 __func__, rb->idstr, start, length, ret);
3898 goto err;
3899 }
3900 #else
3901 ret = -ENOSYS;
3902 error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)",
3903 __func__, rb->idstr, start, length, ret);
3904 goto err;
3905 #endif
3906 }
3907 trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
3908 need_madvise, need_fallocate, ret);
3909 } else {
3910 error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")",
3911 __func__, rb->idstr, start, length, rb->max_length);
3912 }
3913
3914 err:
3915 return ret;
3916 }
3917
ram_block_discard_guest_memfd_range(RAMBlock * rb,uint64_t start,size_t length)3918 int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start,
3919 size_t length)
3920 {
3921 int ret = -1;
3922
3923 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3924 /* ignore fd_offset with guest_memfd */
3925 ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3926 start, length);
3927
3928 if (ret) {
3929 ret = -errno;
3930 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3931 __func__, rb->idstr, start, length, ret);
3932 }
3933 #else
3934 ret = -ENOSYS;
3935 error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)",
3936 __func__, rb->idstr, start, length, ret);
3937 #endif
3938
3939 return ret;
3940 }
3941
ramblock_is_pmem(RAMBlock * rb)3942 bool ramblock_is_pmem(RAMBlock *rb)
3943 {
3944 return rb->flags & RAM_PMEM;
3945 }
3946
mtree_print_phys_entries(int start,int end,int skip,int ptr)3947 static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
3948 {
3949 if (start == end - 1) {
3950 qemu_printf("\t%3d ", start);
3951 } else {
3952 qemu_printf("\t%3d..%-3d ", start, end - 1);
3953 }
3954 qemu_printf(" skip=%d ", skip);
3955 if (ptr == PHYS_MAP_NODE_NIL) {
3956 qemu_printf(" ptr=NIL");
3957 } else if (!skip) {
3958 qemu_printf(" ptr=#%d", ptr);
3959 } else {
3960 qemu_printf(" ptr=[%d]", ptr);
3961 }
3962 qemu_printf("\n");
3963 }
3964
3965 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3966 int128_sub((size), int128_one())) : 0)
3967
mtree_print_dispatch(AddressSpaceDispatch * d,MemoryRegion * root)3968 void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
3969 {
3970 int i;
3971
3972 qemu_printf(" Dispatch\n");
3973 qemu_printf(" Physical sections\n");
3974
3975 for (i = 0; i < d->map.sections_nb; ++i) {
3976 MemoryRegionSection *s = d->map.sections + i;
3977 const char *names[] = { " [unassigned]", " [not dirty]",
3978 " [ROM]", " [watch]" };
3979
3980 qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
3981 " %s%s%s%s%s",
3982 i,
3983 s->offset_within_address_space,
3984 s->offset_within_address_space + MR_SIZE(s->size),
3985 s->mr->name ? s->mr->name : "(noname)",
3986 i < ARRAY_SIZE(names) ? names[i] : "",
3987 s->mr == root ? " [ROOT]" : "",
3988 s == d->mru_section ? " [MRU]" : "",
3989 s->mr->is_iommu ? " [iommu]" : "");
3990
3991 if (s->mr->alias) {
3992 qemu_printf(" alias=%s", s->mr->alias->name ?
3993 s->mr->alias->name : "noname");
3994 }
3995 qemu_printf("\n");
3996 }
3997
3998 qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
3999 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
4000 for (i = 0; i < d->map.nodes_nb; ++i) {
4001 int j, jprev;
4002 PhysPageEntry prev;
4003 Node *n = d->map.nodes + i;
4004
4005 qemu_printf(" [%d]\n", i);
4006
4007 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
4008 PhysPageEntry *pe = *n + j;
4009
4010 if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
4011 continue;
4012 }
4013
4014 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4015
4016 jprev = j;
4017 prev = *pe;
4018 }
4019
4020 if (jprev != ARRAY_SIZE(*n)) {
4021 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
4022 }
4023 }
4024 }
4025
4026 /* Require any discards to work. */
4027 static unsigned int ram_block_discard_required_cnt;
4028 /* Require only coordinated discards to work. */
4029 static unsigned int ram_block_coordinated_discard_required_cnt;
4030 /* Disable any discards. */
4031 static unsigned int ram_block_discard_disabled_cnt;
4032 /* Disable only uncoordinated discards. */
4033 static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
4034 static QemuMutex ram_block_discard_disable_mutex;
4035
ram_block_discard_disable_mutex_lock(void)4036 static void ram_block_discard_disable_mutex_lock(void)
4037 {
4038 static gsize initialized;
4039
4040 if (g_once_init_enter(&initialized)) {
4041 qemu_mutex_init(&ram_block_discard_disable_mutex);
4042 g_once_init_leave(&initialized, 1);
4043 }
4044 qemu_mutex_lock(&ram_block_discard_disable_mutex);
4045 }
4046
ram_block_discard_disable_mutex_unlock(void)4047 static void ram_block_discard_disable_mutex_unlock(void)
4048 {
4049 qemu_mutex_unlock(&ram_block_discard_disable_mutex);
4050 }
4051
ram_block_discard_disable(bool state)4052 int ram_block_discard_disable(bool state)
4053 {
4054 int ret = 0;
4055
4056 ram_block_discard_disable_mutex_lock();
4057 if (!state) {
4058 ram_block_discard_disabled_cnt--;
4059 } else if (ram_block_discard_required_cnt ||
4060 ram_block_coordinated_discard_required_cnt) {
4061 ret = -EBUSY;
4062 } else {
4063 ram_block_discard_disabled_cnt++;
4064 }
4065 ram_block_discard_disable_mutex_unlock();
4066 return ret;
4067 }
4068
ram_block_uncoordinated_discard_disable(bool state)4069 int ram_block_uncoordinated_discard_disable(bool state)
4070 {
4071 int ret = 0;
4072
4073 ram_block_discard_disable_mutex_lock();
4074 if (!state) {
4075 ram_block_uncoordinated_discard_disabled_cnt--;
4076 } else if (ram_block_discard_required_cnt) {
4077 ret = -EBUSY;
4078 } else {
4079 ram_block_uncoordinated_discard_disabled_cnt++;
4080 }
4081 ram_block_discard_disable_mutex_unlock();
4082 return ret;
4083 }
4084
ram_block_discard_require(bool state)4085 int ram_block_discard_require(bool state)
4086 {
4087 int ret = 0;
4088
4089 ram_block_discard_disable_mutex_lock();
4090 if (!state) {
4091 ram_block_discard_required_cnt--;
4092 } else if (ram_block_discard_disabled_cnt ||
4093 ram_block_uncoordinated_discard_disabled_cnt) {
4094 ret = -EBUSY;
4095 } else {
4096 ram_block_discard_required_cnt++;
4097 }
4098 ram_block_discard_disable_mutex_unlock();
4099 return ret;
4100 }
4101
ram_block_coordinated_discard_require(bool state)4102 int ram_block_coordinated_discard_require(bool state)
4103 {
4104 int ret = 0;
4105
4106 ram_block_discard_disable_mutex_lock();
4107 if (!state) {
4108 ram_block_coordinated_discard_required_cnt--;
4109 } else if (ram_block_discard_disabled_cnt) {
4110 ret = -EBUSY;
4111 } else {
4112 ram_block_coordinated_discard_required_cnt++;
4113 }
4114 ram_block_discard_disable_mutex_unlock();
4115 return ret;
4116 }
4117
ram_block_discard_is_disabled(void)4118 bool ram_block_discard_is_disabled(void)
4119 {
4120 return qatomic_read(&ram_block_discard_disabled_cnt) ||
4121 qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
4122 }
4123
ram_block_discard_is_required(void)4124 bool ram_block_discard_is_required(void)
4125 {
4126 return qatomic_read(&ram_block_discard_required_cnt) ||
4127 qatomic_read(&ram_block_coordinated_discard_required_cnt);
4128 }
4129
4130 /*
4131 * Return true if ram is compatible with CPR. Do not exclude rom,
4132 * because the rom file could change in new QEMU.
4133 */
ram_is_cpr_compatible(RAMBlock * rb)4134 static bool ram_is_cpr_compatible(RAMBlock *rb)
4135 {
4136 MemoryRegion *mr = rb->mr;
4137
4138 if (!mr || !memory_region_is_ram(mr)) {
4139 return true;
4140 }
4141
4142 /* Ram device is remapped in new QEMU */
4143 if (memory_region_is_ram_device(mr)) {
4144 return true;
4145 }
4146
4147 /*
4148 * A file descriptor is passed to new QEMU and remapped, or its backing
4149 * file is reopened and mapped. It must be shared to avoid COW.
4150 */
4151 if (rb->fd >= 0 && qemu_ram_is_shared(rb)) {
4152 return true;
4153 }
4154
4155 return false;
4156 }
4157
4158 /*
4159 * Add a blocker for each volatile ram block. This function should only be
4160 * called after we know that the block is migratable. Non-migratable blocks
4161 * are either re-created in new QEMU, or are handled specially, or are covered
4162 * by a device-level CPR blocker.
4163 */
ram_block_add_cpr_blocker(RAMBlock * rb,Error ** errp)4164 void ram_block_add_cpr_blocker(RAMBlock *rb, Error **errp)
4165 {
4166 assert(qemu_ram_is_migratable(rb));
4167
4168 if (ram_is_cpr_compatible(rb)) {
4169 return;
4170 }
4171
4172 error_setg(&rb->cpr_blocker,
4173 "Memory region %s is not compatible with CPR. share=on is "
4174 "required for memory-backend objects, and aux-ram-share=on is "
4175 "required.", memory_region_name(rb->mr));
4176 migrate_add_blocker_modes(&rb->cpr_blocker, errp, MIG_MODE_CPR_TRANSFER,
4177 -1);
4178 }
4179
ram_block_del_cpr_blocker(RAMBlock * rb)4180 void ram_block_del_cpr_blocker(RAMBlock *rb)
4181 {
4182 migrate_del_blocker(&rb->cpr_blocker);
4183 }
4184