1 /* 2 * RAM allocation and memory access 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "exec/page-vary.h" 22 #include "qapi/error.h" 23 24 #include "qemu/cutils.h" 25 #include "qemu/cacheflush.h" 26 #include "qemu/hbitmap.h" 27 #include "qemu/madvise.h" 28 29 #ifdef CONFIG_TCG 30 #include "hw/core/tcg-cpu-ops.h" 31 #endif /* CONFIG_TCG */ 32 33 #include "exec/exec-all.h" 34 #include "exec/page-protection.h" 35 #include "exec/target_page.h" 36 #include "hw/qdev-core.h" 37 #include "hw/qdev-properties.h" 38 #include "hw/boards.h" 39 #include "sysemu/xen.h" 40 #include "sysemu/kvm.h" 41 #include "sysemu/tcg.h" 42 #include "sysemu/qtest.h" 43 #include "qemu/timer.h" 44 #include "qemu/config-file.h" 45 #include "qemu/error-report.h" 46 #include "qemu/qemu-print.h" 47 #include "qemu/log.h" 48 #include "qemu/memalign.h" 49 #include "exec/memory.h" 50 #include "exec/ioport.h" 51 #include "sysemu/dma.h" 52 #include "sysemu/hostmem.h" 53 #include "sysemu/hw_accel.h" 54 #include "sysemu/xen-mapcache.h" 55 #include "trace/trace-root.h" 56 57 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE 58 #include <linux/falloc.h> 59 #endif 60 61 #include "qemu/rcu_queue.h" 62 #include "qemu/main-loop.h" 63 #include "exec/translate-all.h" 64 #include "sysemu/replay.h" 65 66 #include "exec/memory-internal.h" 67 #include "exec/ram_addr.h" 68 69 #include "qemu/pmem.h" 70 71 #include "migration/vmstate.h" 72 73 #include "qemu/range.h" 74 #ifndef _WIN32 75 #include "qemu/mmap-alloc.h" 76 #endif 77 78 #include "monitor/monitor.h" 79 80 #ifdef CONFIG_LIBDAXCTL 81 #include <daxctl/libdaxctl.h> 82 #endif 83 84 //#define DEBUG_SUBPAGE 85 86 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes 87 * are protected by the ramlist lock. 88 */ 89 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) }; 90 91 static MemoryRegion *system_memory; 92 static MemoryRegion *system_io; 93 94 AddressSpace address_space_io; 95 AddressSpace address_space_memory; 96 97 static MemoryRegion io_mem_unassigned; 98 99 typedef struct PhysPageEntry PhysPageEntry; 100 101 struct PhysPageEntry { 102 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */ 103 uint32_t skip : 6; 104 /* index into phys_sections (!skip) or phys_map_nodes (skip) */ 105 uint32_t ptr : 26; 106 }; 107 108 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6) 109 110 /* Size of the L2 (and L3, etc) page tables. */ 111 #define ADDR_SPACE_BITS 64 112 113 #define P_L2_BITS 9 114 #define P_L2_SIZE (1 << P_L2_BITS) 115 116 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1) 117 118 typedef PhysPageEntry Node[P_L2_SIZE]; 119 120 typedef struct PhysPageMap { 121 struct rcu_head rcu; 122 123 unsigned sections_nb; 124 unsigned sections_nb_alloc; 125 unsigned nodes_nb; 126 unsigned nodes_nb_alloc; 127 Node *nodes; 128 MemoryRegionSection *sections; 129 } PhysPageMap; 130 131 struct AddressSpaceDispatch { 132 MemoryRegionSection *mru_section; 133 /* This is a multi-level map on the physical address space. 134 * The bottom level has pointers to MemoryRegionSections. 135 */ 136 PhysPageEntry phys_map; 137 PhysPageMap map; 138 }; 139 140 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK) 141 typedef struct subpage_t { 142 MemoryRegion iomem; 143 FlatView *fv; 144 hwaddr base; 145 uint16_t sub_section[]; 146 } subpage_t; 147 148 #define PHYS_SECTION_UNASSIGNED 0 149 150 static void io_mem_init(void); 151 static void memory_map_init(void); 152 static void tcg_log_global_after_sync(MemoryListener *listener); 153 static void tcg_commit(MemoryListener *listener); 154 155 /** 156 * CPUAddressSpace: all the information a CPU needs about an AddressSpace 157 * @cpu: the CPU whose AddressSpace this is 158 * @as: the AddressSpace itself 159 * @memory_dispatch: its dispatch pointer (cached, RCU protected) 160 * @tcg_as_listener: listener for tracking changes to the AddressSpace 161 */ 162 typedef struct CPUAddressSpace { 163 CPUState *cpu; 164 AddressSpace *as; 165 struct AddressSpaceDispatch *memory_dispatch; 166 MemoryListener tcg_as_listener; 167 } CPUAddressSpace; 168 169 struct DirtyBitmapSnapshot { 170 ram_addr_t start; 171 ram_addr_t end; 172 unsigned long dirty[]; 173 }; 174 175 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes) 176 { 177 static unsigned alloc_hint = 16; 178 if (map->nodes_nb + nodes > map->nodes_nb_alloc) { 179 map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes); 180 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc); 181 alloc_hint = map->nodes_nb_alloc; 182 } 183 } 184 185 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf) 186 { 187 unsigned i; 188 uint32_t ret; 189 PhysPageEntry e; 190 PhysPageEntry *p; 191 192 ret = map->nodes_nb++; 193 p = map->nodes[ret]; 194 assert(ret != PHYS_MAP_NODE_NIL); 195 assert(ret != map->nodes_nb_alloc); 196 197 e.skip = leaf ? 0 : 1; 198 e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL; 199 for (i = 0; i < P_L2_SIZE; ++i) { 200 memcpy(&p[i], &e, sizeof(e)); 201 } 202 return ret; 203 } 204 205 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp, 206 hwaddr *index, uint64_t *nb, uint16_t leaf, 207 int level) 208 { 209 PhysPageEntry *p; 210 hwaddr step = (hwaddr)1 << (level * P_L2_BITS); 211 212 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) { 213 lp->ptr = phys_map_node_alloc(map, level == 0); 214 } 215 p = map->nodes[lp->ptr]; 216 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)]; 217 218 while (*nb && lp < &p[P_L2_SIZE]) { 219 if ((*index & (step - 1)) == 0 && *nb >= step) { 220 lp->skip = 0; 221 lp->ptr = leaf; 222 *index += step; 223 *nb -= step; 224 } else { 225 phys_page_set_level(map, lp, index, nb, leaf, level - 1); 226 } 227 ++lp; 228 } 229 } 230 231 static void phys_page_set(AddressSpaceDispatch *d, 232 hwaddr index, uint64_t nb, 233 uint16_t leaf) 234 { 235 /* Wildly overreserve - it doesn't matter much. */ 236 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS); 237 238 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); 239 } 240 241 /* Compact a non leaf page entry. Simply detect that the entry has a single child, 242 * and update our entry so we can skip it and go directly to the destination. 243 */ 244 static void phys_page_compact(PhysPageEntry *lp, Node *nodes) 245 { 246 unsigned valid_ptr = P_L2_SIZE; 247 int valid = 0; 248 PhysPageEntry *p; 249 int i; 250 251 if (lp->ptr == PHYS_MAP_NODE_NIL) { 252 return; 253 } 254 255 p = nodes[lp->ptr]; 256 for (i = 0; i < P_L2_SIZE; i++) { 257 if (p[i].ptr == PHYS_MAP_NODE_NIL) { 258 continue; 259 } 260 261 valid_ptr = i; 262 valid++; 263 if (p[i].skip) { 264 phys_page_compact(&p[i], nodes); 265 } 266 } 267 268 /* We can only compress if there's only one child. */ 269 if (valid != 1) { 270 return; 271 } 272 273 assert(valid_ptr < P_L2_SIZE); 274 275 /* Don't compress if it won't fit in the # of bits we have. */ 276 if (P_L2_LEVELS >= (1 << 6) && 277 lp->skip + p[valid_ptr].skip >= (1 << 6)) { 278 return; 279 } 280 281 lp->ptr = p[valid_ptr].ptr; 282 if (!p[valid_ptr].skip) { 283 /* If our only child is a leaf, make this a leaf. */ 284 /* By design, we should have made this node a leaf to begin with so we 285 * should never reach here. 286 * But since it's so simple to handle this, let's do it just in case we 287 * change this rule. 288 */ 289 lp->skip = 0; 290 } else { 291 lp->skip += p[valid_ptr].skip; 292 } 293 } 294 295 void address_space_dispatch_compact(AddressSpaceDispatch *d) 296 { 297 if (d->phys_map.skip) { 298 phys_page_compact(&d->phys_map, d->map.nodes); 299 } 300 } 301 302 static inline bool section_covers_addr(const MemoryRegionSection *section, 303 hwaddr addr) 304 { 305 /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means 306 * the section must cover the entire address space. 307 */ 308 return int128_gethi(section->size) || 309 range_covers_byte(section->offset_within_address_space, 310 int128_getlo(section->size), addr); 311 } 312 313 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr) 314 { 315 PhysPageEntry lp = d->phys_map, *p; 316 Node *nodes = d->map.nodes; 317 MemoryRegionSection *sections = d->map.sections; 318 hwaddr index = addr >> TARGET_PAGE_BITS; 319 int i; 320 321 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) { 322 if (lp.ptr == PHYS_MAP_NODE_NIL) { 323 return §ions[PHYS_SECTION_UNASSIGNED]; 324 } 325 p = nodes[lp.ptr]; 326 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)]; 327 } 328 329 if (section_covers_addr(§ions[lp.ptr], addr)) { 330 return §ions[lp.ptr]; 331 } else { 332 return §ions[PHYS_SECTION_UNASSIGNED]; 333 } 334 } 335 336 /* Called from RCU critical section */ 337 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d, 338 hwaddr addr, 339 bool resolve_subpage) 340 { 341 MemoryRegionSection *section = qatomic_read(&d->mru_section); 342 subpage_t *subpage; 343 344 if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] || 345 !section_covers_addr(section, addr)) { 346 section = phys_page_find(d, addr); 347 qatomic_set(&d->mru_section, section); 348 } 349 if (resolve_subpage && section->mr->subpage) { 350 subpage = container_of(section->mr, subpage_t, iomem); 351 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]]; 352 } 353 return section; 354 } 355 356 /* Called from RCU critical section */ 357 static MemoryRegionSection * 358 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat, 359 hwaddr *plen, bool resolve_subpage) 360 { 361 MemoryRegionSection *section; 362 MemoryRegion *mr; 363 Int128 diff; 364 365 section = address_space_lookup_region(d, addr, resolve_subpage); 366 /* Compute offset within MemoryRegionSection */ 367 addr -= section->offset_within_address_space; 368 369 /* Compute offset within MemoryRegion */ 370 *xlat = addr + section->offset_within_region; 371 372 mr = section->mr; 373 374 /* MMIO registers can be expected to perform full-width accesses based only 375 * on their address, without considering adjacent registers that could 376 * decode to completely different MemoryRegions. When such registers 377 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO 378 * regions overlap wildly. For this reason we cannot clamp the accesses 379 * here. 380 * 381 * If the length is small (as is the case for address_space_ldl/stl), 382 * everything works fine. If the incoming length is large, however, 383 * the caller really has to do the clamping through memory_access_size. 384 */ 385 if (memory_region_is_ram(mr)) { 386 diff = int128_sub(section->size, int128_make64(addr)); 387 *plen = int128_get64(int128_min(diff, int128_make64(*plen))); 388 } 389 return section; 390 } 391 392 /** 393 * address_space_translate_iommu - translate an address through an IOMMU 394 * memory region and then through the target address space. 395 * 396 * @iommu_mr: the IOMMU memory region that we start the translation from 397 * @addr: the address to be translated through the MMU 398 * @xlat: the translated address offset within the destination memory region. 399 * It cannot be %NULL. 400 * @plen_out: valid read/write length of the translated address. It 401 * cannot be %NULL. 402 * @page_mask_out: page mask for the translated address. This 403 * should only be meaningful for IOMMU translated 404 * addresses, since there may be huge pages that this bit 405 * would tell. It can be %NULL if we don't care about it. 406 * @is_write: whether the translation operation is for write 407 * @is_mmio: whether this can be MMIO, set true if it can 408 * @target_as: the address space targeted by the IOMMU 409 * @attrs: transaction attributes 410 * 411 * This function is called from RCU critical section. It is the common 412 * part of flatview_do_translate and address_space_translate_cached. 413 */ 414 static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr, 415 hwaddr *xlat, 416 hwaddr *plen_out, 417 hwaddr *page_mask_out, 418 bool is_write, 419 bool is_mmio, 420 AddressSpace **target_as, 421 MemTxAttrs attrs) 422 { 423 MemoryRegionSection *section; 424 hwaddr page_mask = (hwaddr)-1; 425 426 do { 427 hwaddr addr = *xlat; 428 IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr); 429 int iommu_idx = 0; 430 IOMMUTLBEntry iotlb; 431 432 if (imrc->attrs_to_index) { 433 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs); 434 } 435 436 iotlb = imrc->translate(iommu_mr, addr, is_write ? 437 IOMMU_WO : IOMMU_RO, iommu_idx); 438 439 if (!(iotlb.perm & (1 << is_write))) { 440 goto unassigned; 441 } 442 443 addr = ((iotlb.translated_addr & ~iotlb.addr_mask) 444 | (addr & iotlb.addr_mask)); 445 page_mask &= iotlb.addr_mask; 446 *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1); 447 *target_as = iotlb.target_as; 448 449 section = address_space_translate_internal( 450 address_space_to_dispatch(iotlb.target_as), addr, xlat, 451 plen_out, is_mmio); 452 453 iommu_mr = memory_region_get_iommu(section->mr); 454 } while (unlikely(iommu_mr)); 455 456 if (page_mask_out) { 457 *page_mask_out = page_mask; 458 } 459 return *section; 460 461 unassigned: 462 return (MemoryRegionSection) { .mr = &io_mem_unassigned }; 463 } 464 465 /** 466 * flatview_do_translate - translate an address in FlatView 467 * 468 * @fv: the flat view that we want to translate on 469 * @addr: the address to be translated in above address space 470 * @xlat: the translated address offset within memory region. It 471 * cannot be @NULL. 472 * @plen_out: valid read/write length of the translated address. It 473 * can be @NULL when we don't care about it. 474 * @page_mask_out: page mask for the translated address. This 475 * should only be meaningful for IOMMU translated 476 * addresses, since there may be huge pages that this bit 477 * would tell. It can be @NULL if we don't care about it. 478 * @is_write: whether the translation operation is for write 479 * @is_mmio: whether this can be MMIO, set true if it can 480 * @target_as: the address space targeted by the IOMMU 481 * @attrs: memory transaction attributes 482 * 483 * This function is called from RCU critical section 484 */ 485 static MemoryRegionSection flatview_do_translate(FlatView *fv, 486 hwaddr addr, 487 hwaddr *xlat, 488 hwaddr *plen_out, 489 hwaddr *page_mask_out, 490 bool is_write, 491 bool is_mmio, 492 AddressSpace **target_as, 493 MemTxAttrs attrs) 494 { 495 MemoryRegionSection *section; 496 IOMMUMemoryRegion *iommu_mr; 497 hwaddr plen = (hwaddr)(-1); 498 499 if (!plen_out) { 500 plen_out = &plen; 501 } 502 503 section = address_space_translate_internal( 504 flatview_to_dispatch(fv), addr, xlat, 505 plen_out, is_mmio); 506 507 iommu_mr = memory_region_get_iommu(section->mr); 508 if (unlikely(iommu_mr)) { 509 return address_space_translate_iommu(iommu_mr, xlat, 510 plen_out, page_mask_out, 511 is_write, is_mmio, 512 target_as, attrs); 513 } 514 if (page_mask_out) { 515 /* Not behind an IOMMU, use default page size. */ 516 *page_mask_out = ~TARGET_PAGE_MASK; 517 } 518 519 return *section; 520 } 521 522 /* Called from RCU critical section */ 523 IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr, 524 bool is_write, MemTxAttrs attrs) 525 { 526 MemoryRegionSection section; 527 hwaddr xlat, page_mask; 528 529 /* 530 * This can never be MMIO, and we don't really care about plen, 531 * but page mask. 532 */ 533 section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat, 534 NULL, &page_mask, is_write, false, &as, 535 attrs); 536 537 /* Illegal translation */ 538 if (section.mr == &io_mem_unassigned) { 539 goto iotlb_fail; 540 } 541 542 /* Convert memory region offset into address space offset */ 543 xlat += section.offset_within_address_space - 544 section.offset_within_region; 545 546 return (IOMMUTLBEntry) { 547 .target_as = as, 548 .iova = addr & ~page_mask, 549 .translated_addr = xlat & ~page_mask, 550 .addr_mask = page_mask, 551 /* IOTLBs are for DMAs, and DMA only allows on RAMs. */ 552 .perm = IOMMU_RW, 553 }; 554 555 iotlb_fail: 556 return (IOMMUTLBEntry) {0}; 557 } 558 559 /* Called from RCU critical section */ 560 MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat, 561 hwaddr *plen, bool is_write, 562 MemTxAttrs attrs) 563 { 564 MemoryRegion *mr; 565 MemoryRegionSection section; 566 AddressSpace *as = NULL; 567 568 /* This can be MMIO, so setup MMIO bit. */ 569 section = flatview_do_translate(fv, addr, xlat, plen, NULL, 570 is_write, true, &as, attrs); 571 mr = section.mr; 572 573 if (xen_enabled() && memory_access_is_direct(mr, is_write)) { 574 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr; 575 *plen = MIN(page, *plen); 576 } 577 578 return mr; 579 } 580 581 typedef struct TCGIOMMUNotifier { 582 IOMMUNotifier n; 583 MemoryRegion *mr; 584 CPUState *cpu; 585 int iommu_idx; 586 bool active; 587 } TCGIOMMUNotifier; 588 589 static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 590 { 591 TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n); 592 593 if (!notifier->active) { 594 return; 595 } 596 tlb_flush(notifier->cpu); 597 notifier->active = false; 598 /* We leave the notifier struct on the list to avoid reallocating it later. 599 * Generally the number of IOMMUs a CPU deals with will be small. 600 * In any case we can't unregister the iommu notifier from a notify 601 * callback. 602 */ 603 } 604 605 static void tcg_register_iommu_notifier(CPUState *cpu, 606 IOMMUMemoryRegion *iommu_mr, 607 int iommu_idx) 608 { 609 /* Make sure this CPU has an IOMMU notifier registered for this 610 * IOMMU/IOMMU index combination, so that we can flush its TLB 611 * when the IOMMU tells us the mappings we've cached have changed. 612 */ 613 MemoryRegion *mr = MEMORY_REGION(iommu_mr); 614 TCGIOMMUNotifier *notifier = NULL; 615 int i; 616 617 for (i = 0; i < cpu->iommu_notifiers->len; i++) { 618 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); 619 if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) { 620 break; 621 } 622 } 623 if (i == cpu->iommu_notifiers->len) { 624 /* Not found, add a new entry at the end of the array */ 625 cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1); 626 notifier = g_new0(TCGIOMMUNotifier, 1); 627 g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier; 628 629 notifier->mr = mr; 630 notifier->iommu_idx = iommu_idx; 631 notifier->cpu = cpu; 632 /* Rather than trying to register interest in the specific part 633 * of the iommu's address space that we've accessed and then 634 * expand it later as subsequent accesses touch more of it, we 635 * just register interest in the whole thing, on the assumption 636 * that iommu reconfiguration will be rare. 637 */ 638 iommu_notifier_init(¬ifier->n, 639 tcg_iommu_unmap_notify, 640 IOMMU_NOTIFIER_UNMAP, 641 0, 642 HWADDR_MAX, 643 iommu_idx); 644 memory_region_register_iommu_notifier(notifier->mr, ¬ifier->n, 645 &error_fatal); 646 } 647 648 if (!notifier->active) { 649 notifier->active = true; 650 } 651 } 652 653 void tcg_iommu_free_notifier_list(CPUState *cpu) 654 { 655 /* Destroy the CPU's notifier list */ 656 int i; 657 TCGIOMMUNotifier *notifier; 658 659 for (i = 0; i < cpu->iommu_notifiers->len; i++) { 660 notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); 661 memory_region_unregister_iommu_notifier(notifier->mr, ¬ifier->n); 662 g_free(notifier); 663 } 664 g_array_free(cpu->iommu_notifiers, true); 665 } 666 667 void tcg_iommu_init_notifier_list(CPUState *cpu) 668 { 669 cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *)); 670 } 671 672 /* Called from RCU critical section */ 673 MemoryRegionSection * 674 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr, 675 hwaddr *xlat, hwaddr *plen, 676 MemTxAttrs attrs, int *prot) 677 { 678 MemoryRegionSection *section; 679 IOMMUMemoryRegion *iommu_mr; 680 IOMMUMemoryRegionClass *imrc; 681 IOMMUTLBEntry iotlb; 682 int iommu_idx; 683 hwaddr addr = orig_addr; 684 AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch; 685 686 for (;;) { 687 section = address_space_translate_internal(d, addr, &addr, plen, false); 688 689 iommu_mr = memory_region_get_iommu(section->mr); 690 if (!iommu_mr) { 691 break; 692 } 693 694 imrc = memory_region_get_iommu_class_nocheck(iommu_mr); 695 696 iommu_idx = imrc->attrs_to_index(iommu_mr, attrs); 697 tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx); 698 /* We need all the permissions, so pass IOMMU_NONE so the IOMMU 699 * doesn't short-cut its translation table walk. 700 */ 701 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx); 702 addr = ((iotlb.translated_addr & ~iotlb.addr_mask) 703 | (addr & iotlb.addr_mask)); 704 /* Update the caller's prot bits to remove permissions the IOMMU 705 * is giving us a failure response for. If we get down to no 706 * permissions left at all we can give up now. 707 */ 708 if (!(iotlb.perm & IOMMU_RO)) { 709 *prot &= ~(PAGE_READ | PAGE_EXEC); 710 } 711 if (!(iotlb.perm & IOMMU_WO)) { 712 *prot &= ~PAGE_WRITE; 713 } 714 715 if (!*prot) { 716 goto translate_fail; 717 } 718 719 d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as)); 720 } 721 722 assert(!memory_region_is_iommu(section->mr)); 723 *xlat = addr; 724 return section; 725 726 translate_fail: 727 /* 728 * We should be given a page-aligned address -- certainly 729 * tlb_set_page_with_attrs() does so. The page offset of xlat 730 * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0. 731 * The page portion of xlat will be logged by memory_region_access_valid() 732 * when this memory access is rejected, so use the original untranslated 733 * physical address. 734 */ 735 assert((orig_addr & ~TARGET_PAGE_MASK) == 0); 736 *xlat = orig_addr; 737 return &d->map.sections[PHYS_SECTION_UNASSIGNED]; 738 } 739 740 void cpu_address_space_init(CPUState *cpu, int asidx, 741 const char *prefix, MemoryRegion *mr) 742 { 743 CPUAddressSpace *newas; 744 AddressSpace *as = g_new0(AddressSpace, 1); 745 char *as_name; 746 747 assert(mr); 748 as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index); 749 address_space_init(as, mr, as_name); 750 g_free(as_name); 751 752 /* Target code should have set num_ases before calling us */ 753 assert(asidx < cpu->num_ases); 754 755 if (asidx == 0) { 756 /* address space 0 gets the convenience alias */ 757 cpu->as = as; 758 } 759 760 /* KVM cannot currently support multiple address spaces. */ 761 assert(asidx == 0 || !kvm_enabled()); 762 763 if (!cpu->cpu_ases) { 764 cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases); 765 } 766 767 newas = &cpu->cpu_ases[asidx]; 768 newas->cpu = cpu; 769 newas->as = as; 770 if (tcg_enabled()) { 771 newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync; 772 newas->tcg_as_listener.commit = tcg_commit; 773 newas->tcg_as_listener.name = "tcg"; 774 memory_listener_register(&newas->tcg_as_listener, as); 775 } 776 } 777 778 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx) 779 { 780 /* Return the AddressSpace corresponding to the specified index */ 781 return cpu->cpu_ases[asidx].as; 782 } 783 784 /* Called from RCU critical section */ 785 static RAMBlock *qemu_get_ram_block(ram_addr_t addr) 786 { 787 RAMBlock *block; 788 789 block = qatomic_rcu_read(&ram_list.mru_block); 790 if (block && addr - block->offset < block->max_length) { 791 return block; 792 } 793 RAMBLOCK_FOREACH(block) { 794 if (addr - block->offset < block->max_length) { 795 goto found; 796 } 797 } 798 799 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr); 800 abort(); 801 802 found: 803 /* It is safe to write mru_block outside the BQL. This 804 * is what happens: 805 * 806 * mru_block = xxx 807 * rcu_read_unlock() 808 * xxx removed from list 809 * rcu_read_lock() 810 * read mru_block 811 * mru_block = NULL; 812 * call_rcu(reclaim_ramblock, xxx); 813 * rcu_read_unlock() 814 * 815 * qatomic_rcu_set is not needed here. The block was already published 816 * when it was placed into the list. Here we're just making an extra 817 * copy of the pointer. 818 */ 819 ram_list.mru_block = block; 820 return block; 821 } 822 823 void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) 824 { 825 CPUState *cpu; 826 ram_addr_t start1; 827 RAMBlock *block; 828 ram_addr_t end; 829 830 assert(tcg_enabled()); 831 end = TARGET_PAGE_ALIGN(start + length); 832 start &= TARGET_PAGE_MASK; 833 834 RCU_READ_LOCK_GUARD(); 835 block = qemu_get_ram_block(start); 836 assert(block == qemu_get_ram_block(end - 1)); 837 start1 = (uintptr_t)ramblock_ptr(block, start - block->offset); 838 CPU_FOREACH(cpu) { 839 tlb_reset_dirty(cpu, start1, length); 840 } 841 } 842 843 /* Note: start and end must be within the same ram block. */ 844 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 845 ram_addr_t length, 846 unsigned client) 847 { 848 DirtyMemoryBlocks *blocks; 849 unsigned long end, page, start_page; 850 bool dirty = false; 851 RAMBlock *ramblock; 852 uint64_t mr_offset, mr_size; 853 854 if (length == 0) { 855 return false; 856 } 857 858 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 859 start_page = start >> TARGET_PAGE_BITS; 860 page = start_page; 861 862 WITH_RCU_READ_LOCK_GUARD() { 863 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 864 ramblock = qemu_get_ram_block(start); 865 /* Range sanity check on the ramblock */ 866 assert(start >= ramblock->offset && 867 start + length <= ramblock->offset + ramblock->used_length); 868 869 while (page < end) { 870 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE; 871 unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE; 872 unsigned long num = MIN(end - page, 873 DIRTY_MEMORY_BLOCK_SIZE - offset); 874 875 dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx], 876 offset, num); 877 page += num; 878 } 879 880 mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset; 881 mr_size = (end - start_page) << TARGET_PAGE_BITS; 882 memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size); 883 } 884 885 if (dirty) { 886 cpu_physical_memory_dirty_bits_cleared(start, length); 887 } 888 889 return dirty; 890 } 891 892 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty 893 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) 894 { 895 DirtyMemoryBlocks *blocks; 896 ram_addr_t start = memory_region_get_ram_addr(mr) + offset; 897 unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL); 898 ram_addr_t first = QEMU_ALIGN_DOWN(start, align); 899 ram_addr_t last = QEMU_ALIGN_UP(start + length, align); 900 DirtyBitmapSnapshot *snap; 901 unsigned long page, end, dest; 902 903 snap = g_malloc0(sizeof(*snap) + 904 ((last - first) >> (TARGET_PAGE_BITS + 3))); 905 snap->start = first; 906 snap->end = last; 907 908 page = first >> TARGET_PAGE_BITS; 909 end = last >> TARGET_PAGE_BITS; 910 dest = 0; 911 912 WITH_RCU_READ_LOCK_GUARD() { 913 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 914 915 while (page < end) { 916 unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE; 917 unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE; 918 unsigned long num = MIN(end - page, 919 DIRTY_MEMORY_BLOCK_SIZE - ofs); 920 921 assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL))); 922 assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL))); 923 ofs >>= BITS_PER_LEVEL; 924 925 bitmap_copy_and_clear_atomic(snap->dirty + dest, 926 blocks->blocks[idx] + ofs, 927 num); 928 page += num; 929 dest += num >> BITS_PER_LEVEL; 930 } 931 } 932 933 cpu_physical_memory_dirty_bits_cleared(start, length); 934 935 memory_region_clear_dirty_bitmap(mr, offset, length); 936 937 return snap; 938 } 939 940 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, 941 ram_addr_t start, 942 ram_addr_t length) 943 { 944 unsigned long page, end; 945 946 assert(start >= snap->start); 947 assert(start + length <= snap->end); 948 949 end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS; 950 page = (start - snap->start) >> TARGET_PAGE_BITS; 951 952 while (page < end) { 953 if (test_bit(page, snap->dirty)) { 954 return true; 955 } 956 page++; 957 } 958 return false; 959 } 960 961 /* Called from RCU critical section */ 962 hwaddr memory_region_section_get_iotlb(CPUState *cpu, 963 MemoryRegionSection *section) 964 { 965 AddressSpaceDispatch *d = flatview_to_dispatch(section->fv); 966 return section - d->map.sections; 967 } 968 969 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, 970 uint16_t section); 971 static subpage_t *subpage_init(FlatView *fv, hwaddr base); 972 973 static uint16_t phys_section_add(PhysPageMap *map, 974 MemoryRegionSection *section) 975 { 976 /* The physical section number is ORed with a page-aligned 977 * pointer to produce the iotlb entries. Thus it should 978 * never overflow into the page-aligned value. 979 */ 980 assert(map->sections_nb < TARGET_PAGE_SIZE); 981 982 if (map->sections_nb == map->sections_nb_alloc) { 983 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16); 984 map->sections = g_renew(MemoryRegionSection, map->sections, 985 map->sections_nb_alloc); 986 } 987 map->sections[map->sections_nb] = *section; 988 memory_region_ref(section->mr); 989 return map->sections_nb++; 990 } 991 992 static void phys_section_destroy(MemoryRegion *mr) 993 { 994 bool have_sub_page = mr->subpage; 995 996 memory_region_unref(mr); 997 998 if (have_sub_page) { 999 subpage_t *subpage = container_of(mr, subpage_t, iomem); 1000 object_unref(OBJECT(&subpage->iomem)); 1001 g_free(subpage); 1002 } 1003 } 1004 1005 static void phys_sections_free(PhysPageMap *map) 1006 { 1007 while (map->sections_nb > 0) { 1008 MemoryRegionSection *section = &map->sections[--map->sections_nb]; 1009 phys_section_destroy(section->mr); 1010 } 1011 g_free(map->sections); 1012 g_free(map->nodes); 1013 } 1014 1015 static void register_subpage(FlatView *fv, MemoryRegionSection *section) 1016 { 1017 AddressSpaceDispatch *d = flatview_to_dispatch(fv); 1018 subpage_t *subpage; 1019 hwaddr base = section->offset_within_address_space 1020 & TARGET_PAGE_MASK; 1021 MemoryRegionSection *existing = phys_page_find(d, base); 1022 MemoryRegionSection subsection = { 1023 .offset_within_address_space = base, 1024 .size = int128_make64(TARGET_PAGE_SIZE), 1025 }; 1026 hwaddr start, end; 1027 1028 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned); 1029 1030 if (!(existing->mr->subpage)) { 1031 subpage = subpage_init(fv, base); 1032 subsection.fv = fv; 1033 subsection.mr = &subpage->iomem; 1034 phys_page_set(d, base >> TARGET_PAGE_BITS, 1, 1035 phys_section_add(&d->map, &subsection)); 1036 } else { 1037 subpage = container_of(existing->mr, subpage_t, iomem); 1038 } 1039 start = section->offset_within_address_space & ~TARGET_PAGE_MASK; 1040 end = start + int128_get64(section->size) - 1; 1041 subpage_register(subpage, start, end, 1042 phys_section_add(&d->map, section)); 1043 } 1044 1045 1046 static void register_multipage(FlatView *fv, 1047 MemoryRegionSection *section) 1048 { 1049 AddressSpaceDispatch *d = flatview_to_dispatch(fv); 1050 hwaddr start_addr = section->offset_within_address_space; 1051 uint16_t section_index = phys_section_add(&d->map, section); 1052 uint64_t num_pages = int128_get64(int128_rshift(section->size, 1053 TARGET_PAGE_BITS)); 1054 1055 assert(num_pages); 1056 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); 1057 } 1058 1059 /* 1060 * The range in *section* may look like this: 1061 * 1062 * |s|PPPPPPP|s| 1063 * 1064 * where s stands for subpage and P for page. 1065 */ 1066 void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section) 1067 { 1068 MemoryRegionSection remain = *section; 1069 Int128 page_size = int128_make64(TARGET_PAGE_SIZE); 1070 1071 /* register first subpage */ 1072 if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) { 1073 uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space) 1074 - remain.offset_within_address_space; 1075 1076 MemoryRegionSection now = remain; 1077 now.size = int128_min(int128_make64(left), now.size); 1078 register_subpage(fv, &now); 1079 if (int128_eq(remain.size, now.size)) { 1080 return; 1081 } 1082 remain.size = int128_sub(remain.size, now.size); 1083 remain.offset_within_address_space += int128_get64(now.size); 1084 remain.offset_within_region += int128_get64(now.size); 1085 } 1086 1087 /* register whole pages */ 1088 if (int128_ge(remain.size, page_size)) { 1089 MemoryRegionSection now = remain; 1090 now.size = int128_and(now.size, int128_neg(page_size)); 1091 register_multipage(fv, &now); 1092 if (int128_eq(remain.size, now.size)) { 1093 return; 1094 } 1095 remain.size = int128_sub(remain.size, now.size); 1096 remain.offset_within_address_space += int128_get64(now.size); 1097 remain.offset_within_region += int128_get64(now.size); 1098 } 1099 1100 /* register last subpage */ 1101 register_subpage(fv, &remain); 1102 } 1103 1104 void qemu_flush_coalesced_mmio_buffer(void) 1105 { 1106 if (kvm_enabled()) 1107 kvm_flush_coalesced_mmio_buffer(); 1108 } 1109 1110 void qemu_mutex_lock_ramlist(void) 1111 { 1112 qemu_mutex_lock(&ram_list.mutex); 1113 } 1114 1115 void qemu_mutex_unlock_ramlist(void) 1116 { 1117 qemu_mutex_unlock(&ram_list.mutex); 1118 } 1119 1120 GString *ram_block_format(void) 1121 { 1122 RAMBlock *block; 1123 char *psize; 1124 GString *buf = g_string_new(""); 1125 1126 RCU_READ_LOCK_GUARD(); 1127 g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n", 1128 "Block Name", "PSize", "Offset", "Used", "Total", 1129 "HVA", "RO"); 1130 1131 RAMBLOCK_FOREACH(block) { 1132 psize = size_to_str(block->page_size); 1133 g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64 1134 " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n", 1135 block->idstr, psize, 1136 (uint64_t)block->offset, 1137 (uint64_t)block->used_length, 1138 (uint64_t)block->max_length, 1139 (uint64_t)(uintptr_t)block->host, 1140 block->mr->readonly ? "ro" : "rw"); 1141 1142 g_free(psize); 1143 } 1144 1145 return buf; 1146 } 1147 1148 static int find_min_backend_pagesize(Object *obj, void *opaque) 1149 { 1150 long *hpsize_min = opaque; 1151 1152 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) { 1153 HostMemoryBackend *backend = MEMORY_BACKEND(obj); 1154 long hpsize = host_memory_backend_pagesize(backend); 1155 1156 if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) { 1157 *hpsize_min = hpsize; 1158 } 1159 } 1160 1161 return 0; 1162 } 1163 1164 static int find_max_backend_pagesize(Object *obj, void *opaque) 1165 { 1166 long *hpsize_max = opaque; 1167 1168 if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) { 1169 HostMemoryBackend *backend = MEMORY_BACKEND(obj); 1170 long hpsize = host_memory_backend_pagesize(backend); 1171 1172 if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) { 1173 *hpsize_max = hpsize; 1174 } 1175 } 1176 1177 return 0; 1178 } 1179 1180 /* 1181 * TODO: We assume right now that all mapped host memory backends are 1182 * used as RAM, however some might be used for different purposes. 1183 */ 1184 long qemu_minrampagesize(void) 1185 { 1186 long hpsize = LONG_MAX; 1187 Object *memdev_root = object_resolve_path("/objects", NULL); 1188 1189 object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize); 1190 return hpsize; 1191 } 1192 1193 long qemu_maxrampagesize(void) 1194 { 1195 long pagesize = 0; 1196 Object *memdev_root = object_resolve_path("/objects", NULL); 1197 1198 object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize); 1199 return pagesize; 1200 } 1201 1202 #ifdef CONFIG_POSIX 1203 static int64_t get_file_size(int fd) 1204 { 1205 int64_t size; 1206 #if defined(__linux__) 1207 struct stat st; 1208 1209 if (fstat(fd, &st) < 0) { 1210 return -errno; 1211 } 1212 1213 /* Special handling for devdax character devices */ 1214 if (S_ISCHR(st.st_mode)) { 1215 g_autofree char *subsystem_path = NULL; 1216 g_autofree char *subsystem = NULL; 1217 1218 subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem", 1219 major(st.st_rdev), minor(st.st_rdev)); 1220 subsystem = g_file_read_link(subsystem_path, NULL); 1221 1222 if (subsystem && g_str_has_suffix(subsystem, "/dax")) { 1223 g_autofree char *size_path = NULL; 1224 g_autofree char *size_str = NULL; 1225 1226 size_path = g_strdup_printf("/sys/dev/char/%d:%d/size", 1227 major(st.st_rdev), minor(st.st_rdev)); 1228 1229 if (g_file_get_contents(size_path, &size_str, NULL, NULL)) { 1230 return g_ascii_strtoll(size_str, NULL, 0); 1231 } 1232 } 1233 } 1234 #endif /* defined(__linux__) */ 1235 1236 /* st.st_size may be zero for special files yet lseek(2) works */ 1237 size = lseek(fd, 0, SEEK_END); 1238 if (size < 0) { 1239 return -errno; 1240 } 1241 return size; 1242 } 1243 1244 static int64_t get_file_align(int fd) 1245 { 1246 int64_t align = -1; 1247 #if defined(__linux__) && defined(CONFIG_LIBDAXCTL) 1248 struct stat st; 1249 1250 if (fstat(fd, &st) < 0) { 1251 return -errno; 1252 } 1253 1254 /* Special handling for devdax character devices */ 1255 if (S_ISCHR(st.st_mode)) { 1256 g_autofree char *path = NULL; 1257 g_autofree char *rpath = NULL; 1258 struct daxctl_ctx *ctx; 1259 struct daxctl_region *region; 1260 int rc = 0; 1261 1262 path = g_strdup_printf("/sys/dev/char/%d:%d", 1263 major(st.st_rdev), minor(st.st_rdev)); 1264 rpath = realpath(path, NULL); 1265 if (!rpath) { 1266 return -errno; 1267 } 1268 1269 rc = daxctl_new(&ctx); 1270 if (rc) { 1271 return -1; 1272 } 1273 1274 daxctl_region_foreach(ctx, region) { 1275 if (strstr(rpath, daxctl_region_get_path(region))) { 1276 align = daxctl_region_get_align(region); 1277 break; 1278 } 1279 } 1280 daxctl_unref(ctx); 1281 } 1282 #endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */ 1283 1284 return align; 1285 } 1286 1287 static int file_ram_open(const char *path, 1288 const char *region_name, 1289 bool readonly, 1290 bool *created) 1291 { 1292 char *filename; 1293 char *sanitized_name; 1294 char *c; 1295 int fd = -1; 1296 1297 *created = false; 1298 for (;;) { 1299 fd = open(path, readonly ? O_RDONLY : O_RDWR); 1300 if (fd >= 0) { 1301 /* 1302 * open(O_RDONLY) won't fail with EISDIR. Check manually if we 1303 * opened a directory and fail similarly to how we fail ENOENT 1304 * in readonly mode. Note that mkstemp() would imply O_RDWR. 1305 */ 1306 if (readonly) { 1307 struct stat file_stat; 1308 1309 if (fstat(fd, &file_stat)) { 1310 close(fd); 1311 if (errno == EINTR) { 1312 continue; 1313 } 1314 return -errno; 1315 } else if (S_ISDIR(file_stat.st_mode)) { 1316 close(fd); 1317 return -EISDIR; 1318 } 1319 } 1320 /* @path names an existing file, use it */ 1321 break; 1322 } 1323 if (errno == ENOENT) { 1324 if (readonly) { 1325 /* Refuse to create new, readonly files. */ 1326 return -ENOENT; 1327 } 1328 /* @path names a file that doesn't exist, create it */ 1329 fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644); 1330 if (fd >= 0) { 1331 *created = true; 1332 break; 1333 } 1334 } else if (errno == EISDIR) { 1335 /* @path names a directory, create a file there */ 1336 /* Make name safe to use with mkstemp by replacing '/' with '_'. */ 1337 sanitized_name = g_strdup(region_name); 1338 for (c = sanitized_name; *c != '\0'; c++) { 1339 if (*c == '/') { 1340 *c = '_'; 1341 } 1342 } 1343 1344 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path, 1345 sanitized_name); 1346 g_free(sanitized_name); 1347 1348 fd = mkstemp(filename); 1349 if (fd >= 0) { 1350 unlink(filename); 1351 g_free(filename); 1352 break; 1353 } 1354 g_free(filename); 1355 } 1356 if (errno != EEXIST && errno != EINTR) { 1357 return -errno; 1358 } 1359 /* 1360 * Try again on EINTR and EEXIST. The latter happens when 1361 * something else creates the file between our two open(). 1362 */ 1363 } 1364 1365 return fd; 1366 } 1367 1368 static void *file_ram_alloc(RAMBlock *block, 1369 ram_addr_t memory, 1370 int fd, 1371 bool truncate, 1372 off_t offset, 1373 Error **errp) 1374 { 1375 uint32_t qemu_map_flags; 1376 void *area; 1377 1378 block->page_size = qemu_fd_getpagesize(fd); 1379 if (block->mr->align % block->page_size) { 1380 error_setg(errp, "alignment 0x%" PRIx64 1381 " must be multiples of page size 0x%zx", 1382 block->mr->align, block->page_size); 1383 return NULL; 1384 } else if (block->mr->align && !is_power_of_2(block->mr->align)) { 1385 error_setg(errp, "alignment 0x%" PRIx64 1386 " must be a power of two", block->mr->align); 1387 return NULL; 1388 } else if (offset % block->page_size) { 1389 error_setg(errp, "offset 0x%" PRIx64 1390 " must be multiples of page size 0x%zx", 1391 offset, block->page_size); 1392 return NULL; 1393 } 1394 block->mr->align = MAX(block->page_size, block->mr->align); 1395 #if defined(__s390x__) 1396 if (kvm_enabled()) { 1397 block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN); 1398 } 1399 #endif 1400 1401 if (memory < block->page_size) { 1402 error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to " 1403 "or larger than page size 0x%zx", 1404 memory, block->page_size); 1405 return NULL; 1406 } 1407 1408 memory = ROUND_UP(memory, block->page_size); 1409 1410 /* 1411 * ftruncate is not supported by hugetlbfs in older 1412 * hosts, so don't bother bailing out on errors. 1413 * If anything goes wrong with it under other filesystems, 1414 * mmap will fail. 1415 * 1416 * Do not truncate the non-empty backend file to avoid corrupting 1417 * the existing data in the file. Disabling shrinking is not 1418 * enough. For example, the current vNVDIMM implementation stores 1419 * the guest NVDIMM labels at the end of the backend file. If the 1420 * backend file is later extended, QEMU will not be able to find 1421 * those labels. Therefore, extending the non-empty backend file 1422 * is disabled as well. 1423 */ 1424 if (truncate && ftruncate(fd, offset + memory)) { 1425 perror("ftruncate"); 1426 } 1427 1428 qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0; 1429 qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0; 1430 qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0; 1431 qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0; 1432 area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset); 1433 if (area == MAP_FAILED) { 1434 error_setg_errno(errp, errno, 1435 "unable to map backing store for guest RAM"); 1436 return NULL; 1437 } 1438 1439 block->fd = fd; 1440 block->fd_offset = offset; 1441 return area; 1442 } 1443 #endif 1444 1445 /* Allocate space within the ram_addr_t space that governs the 1446 * dirty bitmaps. 1447 * Called with the ramlist lock held. 1448 */ 1449 static ram_addr_t find_ram_offset(ram_addr_t size) 1450 { 1451 RAMBlock *block, *next_block; 1452 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX; 1453 1454 assert(size != 0); /* it would hand out same offset multiple times */ 1455 1456 if (QLIST_EMPTY_RCU(&ram_list.blocks)) { 1457 return 0; 1458 } 1459 1460 RAMBLOCK_FOREACH(block) { 1461 ram_addr_t candidate, next = RAM_ADDR_MAX; 1462 1463 /* Align blocks to start on a 'long' in the bitmap 1464 * which makes the bitmap sync'ing take the fast path. 1465 */ 1466 candidate = block->offset + block->max_length; 1467 candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS); 1468 1469 /* Search for the closest following block 1470 * and find the gap. 1471 */ 1472 RAMBLOCK_FOREACH(next_block) { 1473 if (next_block->offset >= candidate) { 1474 next = MIN(next, next_block->offset); 1475 } 1476 } 1477 1478 /* If it fits remember our place and remember the size 1479 * of gap, but keep going so that we might find a smaller 1480 * gap to fill so avoiding fragmentation. 1481 */ 1482 if (next - candidate >= size && next - candidate < mingap) { 1483 offset = candidate; 1484 mingap = next - candidate; 1485 } 1486 1487 trace_find_ram_offset_loop(size, candidate, offset, next, mingap); 1488 } 1489 1490 if (offset == RAM_ADDR_MAX) { 1491 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n", 1492 (uint64_t)size); 1493 abort(); 1494 } 1495 1496 trace_find_ram_offset(size, offset); 1497 1498 return offset; 1499 } 1500 1501 static unsigned long last_ram_page(void) 1502 { 1503 RAMBlock *block; 1504 ram_addr_t last = 0; 1505 1506 RCU_READ_LOCK_GUARD(); 1507 RAMBLOCK_FOREACH(block) { 1508 last = MAX(last, block->offset + block->max_length); 1509 } 1510 return last >> TARGET_PAGE_BITS; 1511 } 1512 1513 static void qemu_ram_setup_dump(void *addr, ram_addr_t size) 1514 { 1515 int ret; 1516 1517 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */ 1518 if (!machine_dump_guest_core(current_machine)) { 1519 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP); 1520 if (ret) { 1521 perror("qemu_madvise"); 1522 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, " 1523 "but dump_guest_core=off specified\n"); 1524 } 1525 } 1526 } 1527 1528 const char *qemu_ram_get_idstr(RAMBlock *rb) 1529 { 1530 return rb->idstr; 1531 } 1532 1533 void *qemu_ram_get_host_addr(RAMBlock *rb) 1534 { 1535 return rb->host; 1536 } 1537 1538 ram_addr_t qemu_ram_get_offset(RAMBlock *rb) 1539 { 1540 return rb->offset; 1541 } 1542 1543 ram_addr_t qemu_ram_get_used_length(RAMBlock *rb) 1544 { 1545 return rb->used_length; 1546 } 1547 1548 ram_addr_t qemu_ram_get_max_length(RAMBlock *rb) 1549 { 1550 return rb->max_length; 1551 } 1552 1553 bool qemu_ram_is_shared(RAMBlock *rb) 1554 { 1555 return rb->flags & RAM_SHARED; 1556 } 1557 1558 bool qemu_ram_is_noreserve(RAMBlock *rb) 1559 { 1560 return rb->flags & RAM_NORESERVE; 1561 } 1562 1563 /* Note: Only set at the start of postcopy */ 1564 bool qemu_ram_is_uf_zeroable(RAMBlock *rb) 1565 { 1566 return rb->flags & RAM_UF_ZEROPAGE; 1567 } 1568 1569 void qemu_ram_set_uf_zeroable(RAMBlock *rb) 1570 { 1571 rb->flags |= RAM_UF_ZEROPAGE; 1572 } 1573 1574 bool qemu_ram_is_migratable(RAMBlock *rb) 1575 { 1576 return rb->flags & RAM_MIGRATABLE; 1577 } 1578 1579 void qemu_ram_set_migratable(RAMBlock *rb) 1580 { 1581 rb->flags |= RAM_MIGRATABLE; 1582 } 1583 1584 void qemu_ram_unset_migratable(RAMBlock *rb) 1585 { 1586 rb->flags &= ~RAM_MIGRATABLE; 1587 } 1588 1589 bool qemu_ram_is_named_file(RAMBlock *rb) 1590 { 1591 return rb->flags & RAM_NAMED_FILE; 1592 } 1593 1594 int qemu_ram_get_fd(RAMBlock *rb) 1595 { 1596 return rb->fd; 1597 } 1598 1599 /* Called with the BQL held. */ 1600 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev) 1601 { 1602 RAMBlock *block; 1603 1604 assert(new_block); 1605 assert(!new_block->idstr[0]); 1606 1607 if (dev) { 1608 char *id = qdev_get_dev_path(dev); 1609 if (id) { 1610 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id); 1611 g_free(id); 1612 } 1613 } 1614 pstrcat(new_block->idstr, sizeof(new_block->idstr), name); 1615 1616 RCU_READ_LOCK_GUARD(); 1617 RAMBLOCK_FOREACH(block) { 1618 if (block != new_block && 1619 !strcmp(block->idstr, new_block->idstr)) { 1620 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n", 1621 new_block->idstr); 1622 abort(); 1623 } 1624 } 1625 } 1626 1627 /* Called with the BQL held. */ 1628 void qemu_ram_unset_idstr(RAMBlock *block) 1629 { 1630 /* FIXME: arch_init.c assumes that this is not called throughout 1631 * migration. Ignore the problem since hot-unplug during migration 1632 * does not work anyway. 1633 */ 1634 if (block) { 1635 memset(block->idstr, 0, sizeof(block->idstr)); 1636 } 1637 } 1638 1639 size_t qemu_ram_pagesize(RAMBlock *rb) 1640 { 1641 return rb->page_size; 1642 } 1643 1644 /* Returns the largest size of page in use */ 1645 size_t qemu_ram_pagesize_largest(void) 1646 { 1647 RAMBlock *block; 1648 size_t largest = 0; 1649 1650 RAMBLOCK_FOREACH(block) { 1651 largest = MAX(largest, qemu_ram_pagesize(block)); 1652 } 1653 1654 return largest; 1655 } 1656 1657 static int memory_try_enable_merging(void *addr, size_t len) 1658 { 1659 if (!machine_mem_merge(current_machine)) { 1660 /* disabled by the user */ 1661 return 0; 1662 } 1663 1664 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE); 1665 } 1666 1667 /* 1668 * Resizing RAM while migrating can result in the migration being canceled. 1669 * Care has to be taken if the guest might have already detected the memory. 1670 * 1671 * As memory core doesn't know how is memory accessed, it is up to 1672 * resize callback to update device state and/or add assertions to detect 1673 * misuse, if necessary. 1674 */ 1675 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) 1676 { 1677 const ram_addr_t oldsize = block->used_length; 1678 const ram_addr_t unaligned_size = newsize; 1679 1680 assert(block); 1681 1682 newsize = TARGET_PAGE_ALIGN(newsize); 1683 newsize = REAL_HOST_PAGE_ALIGN(newsize); 1684 1685 if (block->used_length == newsize) { 1686 /* 1687 * We don't have to resize the ram block (which only knows aligned 1688 * sizes), however, we have to notify if the unaligned size changed. 1689 */ 1690 if (unaligned_size != memory_region_size(block->mr)) { 1691 memory_region_set_size(block->mr, unaligned_size); 1692 if (block->resized) { 1693 block->resized(block->idstr, unaligned_size, block->host); 1694 } 1695 } 1696 return 0; 1697 } 1698 1699 if (!(block->flags & RAM_RESIZEABLE)) { 1700 error_setg_errno(errp, EINVAL, 1701 "Size mismatch: %s: 0x" RAM_ADDR_FMT 1702 " != 0x" RAM_ADDR_FMT, block->idstr, 1703 newsize, block->used_length); 1704 return -EINVAL; 1705 } 1706 1707 if (block->max_length < newsize) { 1708 error_setg_errno(errp, EINVAL, 1709 "Size too large: %s: 0x" RAM_ADDR_FMT 1710 " > 0x" RAM_ADDR_FMT, block->idstr, 1711 newsize, block->max_length); 1712 return -EINVAL; 1713 } 1714 1715 /* Notify before modifying the ram block and touching the bitmaps. */ 1716 if (block->host) { 1717 ram_block_notify_resize(block->host, oldsize, newsize); 1718 } 1719 1720 cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); 1721 block->used_length = newsize; 1722 cpu_physical_memory_set_dirty_range(block->offset, block->used_length, 1723 DIRTY_CLIENTS_ALL); 1724 memory_region_set_size(block->mr, unaligned_size); 1725 if (block->resized) { 1726 block->resized(block->idstr, unaligned_size, block->host); 1727 } 1728 return 0; 1729 } 1730 1731 /* 1732 * Trigger sync on the given ram block for range [start, start + length] 1733 * with the backing store if one is available. 1734 * Otherwise no-op. 1735 * @Note: this is supposed to be a synchronous op. 1736 */ 1737 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length) 1738 { 1739 /* The requested range should fit in within the block range */ 1740 g_assert((start + length) <= block->used_length); 1741 1742 #ifdef CONFIG_LIBPMEM 1743 /* The lack of support for pmem should not block the sync */ 1744 if (ramblock_is_pmem(block)) { 1745 void *addr = ramblock_ptr(block, start); 1746 pmem_persist(addr, length); 1747 return; 1748 } 1749 #endif 1750 if (block->fd >= 0) { 1751 /** 1752 * Case there is no support for PMEM or the memory has not been 1753 * specified as persistent (or is not one) - use the msync. 1754 * Less optimal but still achieves the same goal 1755 */ 1756 void *addr = ramblock_ptr(block, start); 1757 if (qemu_msync(addr, length, block->fd)) { 1758 warn_report("%s: failed to sync memory range: start: " 1759 RAM_ADDR_FMT " length: " RAM_ADDR_FMT, 1760 __func__, start, length); 1761 } 1762 } 1763 } 1764 1765 /* Called with ram_list.mutex held */ 1766 static void dirty_memory_extend(ram_addr_t old_ram_size, 1767 ram_addr_t new_ram_size) 1768 { 1769 ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size, 1770 DIRTY_MEMORY_BLOCK_SIZE); 1771 ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size, 1772 DIRTY_MEMORY_BLOCK_SIZE); 1773 int i; 1774 1775 /* Only need to extend if block count increased */ 1776 if (new_num_blocks <= old_num_blocks) { 1777 return; 1778 } 1779 1780 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 1781 DirtyMemoryBlocks *old_blocks; 1782 DirtyMemoryBlocks *new_blocks; 1783 int j; 1784 1785 old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]); 1786 new_blocks = g_malloc(sizeof(*new_blocks) + 1787 sizeof(new_blocks->blocks[0]) * new_num_blocks); 1788 1789 if (old_num_blocks) { 1790 memcpy(new_blocks->blocks, old_blocks->blocks, 1791 old_num_blocks * sizeof(old_blocks->blocks[0])); 1792 } 1793 1794 for (j = old_num_blocks; j < new_num_blocks; j++) { 1795 new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE); 1796 } 1797 1798 qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks); 1799 1800 if (old_blocks) { 1801 g_free_rcu(old_blocks, rcu); 1802 } 1803 } 1804 } 1805 1806 static void ram_block_add(RAMBlock *new_block, Error **errp) 1807 { 1808 const bool noreserve = qemu_ram_is_noreserve(new_block); 1809 const bool shared = qemu_ram_is_shared(new_block); 1810 RAMBlock *block; 1811 RAMBlock *last_block = NULL; 1812 bool free_on_error = false; 1813 ram_addr_t old_ram_size, new_ram_size; 1814 Error *err = NULL; 1815 1816 old_ram_size = last_ram_page(); 1817 1818 qemu_mutex_lock_ramlist(); 1819 new_block->offset = find_ram_offset(new_block->max_length); 1820 1821 if (!new_block->host) { 1822 if (xen_enabled()) { 1823 xen_ram_alloc(new_block->offset, new_block->max_length, 1824 new_block->mr, &err); 1825 if (err) { 1826 error_propagate(errp, err); 1827 qemu_mutex_unlock_ramlist(); 1828 return; 1829 } 1830 } else { 1831 new_block->host = qemu_anon_ram_alloc(new_block->max_length, 1832 &new_block->mr->align, 1833 shared, noreserve); 1834 if (!new_block->host) { 1835 error_setg_errno(errp, errno, 1836 "cannot set up guest memory '%s'", 1837 memory_region_name(new_block->mr)); 1838 qemu_mutex_unlock_ramlist(); 1839 return; 1840 } 1841 memory_try_enable_merging(new_block->host, new_block->max_length); 1842 free_on_error = true; 1843 } 1844 } 1845 1846 if (new_block->flags & RAM_GUEST_MEMFD) { 1847 assert(kvm_enabled()); 1848 assert(new_block->guest_memfd < 0); 1849 1850 if (ram_block_discard_require(true) < 0) { 1851 error_setg_errno(errp, errno, 1852 "cannot set up private guest memory: discard currently blocked"); 1853 error_append_hint(errp, "Are you using assigned devices?\n"); 1854 goto out_free; 1855 } 1856 1857 new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length, 1858 0, errp); 1859 if (new_block->guest_memfd < 0) { 1860 qemu_mutex_unlock_ramlist(); 1861 goto out_free; 1862 } 1863 } 1864 1865 new_ram_size = MAX(old_ram_size, 1866 (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS); 1867 if (new_ram_size > old_ram_size) { 1868 dirty_memory_extend(old_ram_size, new_ram_size); 1869 } 1870 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ, 1871 * QLIST (which has an RCU-friendly variant) does not have insertion at 1872 * tail, so save the last element in last_block. 1873 */ 1874 RAMBLOCK_FOREACH(block) { 1875 last_block = block; 1876 if (block->max_length < new_block->max_length) { 1877 break; 1878 } 1879 } 1880 if (block) { 1881 QLIST_INSERT_BEFORE_RCU(block, new_block, next); 1882 } else if (last_block) { 1883 QLIST_INSERT_AFTER_RCU(last_block, new_block, next); 1884 } else { /* list is empty */ 1885 QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next); 1886 } 1887 ram_list.mru_block = NULL; 1888 1889 /* Write list before version */ 1890 smp_wmb(); 1891 ram_list.version++; 1892 qemu_mutex_unlock_ramlist(); 1893 1894 cpu_physical_memory_set_dirty_range(new_block->offset, 1895 new_block->used_length, 1896 DIRTY_CLIENTS_ALL); 1897 1898 if (new_block->host) { 1899 qemu_ram_setup_dump(new_block->host, new_block->max_length); 1900 qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE); 1901 /* 1902 * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU 1903 * Configure it unless the machine is a qtest server, in which case 1904 * KVM is not used and it may be forked (eg for fuzzing purposes). 1905 */ 1906 if (!qtest_enabled()) { 1907 qemu_madvise(new_block->host, new_block->max_length, 1908 QEMU_MADV_DONTFORK); 1909 } 1910 ram_block_notify_add(new_block->host, new_block->used_length, 1911 new_block->max_length); 1912 } 1913 return; 1914 1915 out_free: 1916 if (free_on_error) { 1917 qemu_anon_ram_free(new_block->host, new_block->max_length); 1918 new_block->host = NULL; 1919 } 1920 } 1921 1922 #ifdef CONFIG_POSIX 1923 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, 1924 uint32_t ram_flags, int fd, off_t offset, 1925 Error **errp) 1926 { 1927 RAMBlock *new_block; 1928 Error *local_err = NULL; 1929 int64_t file_size, file_align; 1930 1931 /* Just support these ram flags by now. */ 1932 assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE | 1933 RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY | 1934 RAM_READONLY_FD | RAM_GUEST_MEMFD)) == 0); 1935 1936 if (xen_enabled()) { 1937 error_setg(errp, "-mem-path not supported with Xen"); 1938 return NULL; 1939 } 1940 1941 if (kvm_enabled() && !kvm_has_sync_mmu()) { 1942 error_setg(errp, 1943 "host lacks kvm mmu notifiers, -mem-path unsupported"); 1944 return NULL; 1945 } 1946 1947 size = TARGET_PAGE_ALIGN(size); 1948 size = REAL_HOST_PAGE_ALIGN(size); 1949 1950 file_size = get_file_size(fd); 1951 if (file_size > offset && file_size < (offset + size)) { 1952 error_setg(errp, "backing store size 0x%" PRIx64 1953 " does not match 'size' option 0x" RAM_ADDR_FMT, 1954 file_size, size); 1955 return NULL; 1956 } 1957 1958 file_align = get_file_align(fd); 1959 if (file_align > 0 && file_align > mr->align) { 1960 error_setg(errp, "backing store align 0x%" PRIx64 1961 " is larger than 'align' option 0x%" PRIx64, 1962 file_align, mr->align); 1963 return NULL; 1964 } 1965 1966 new_block = g_malloc0(sizeof(*new_block)); 1967 new_block->mr = mr; 1968 new_block->used_length = size; 1969 new_block->max_length = size; 1970 new_block->flags = ram_flags; 1971 new_block->guest_memfd = -1; 1972 new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset, 1973 errp); 1974 if (!new_block->host) { 1975 g_free(new_block); 1976 return NULL; 1977 } 1978 1979 ram_block_add(new_block, &local_err); 1980 if (local_err) { 1981 g_free(new_block); 1982 error_propagate(errp, local_err); 1983 return NULL; 1984 } 1985 return new_block; 1986 1987 } 1988 1989 1990 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 1991 uint32_t ram_flags, const char *mem_path, 1992 off_t offset, Error **errp) 1993 { 1994 int fd; 1995 bool created; 1996 RAMBlock *block; 1997 1998 fd = file_ram_open(mem_path, memory_region_name(mr), 1999 !!(ram_flags & RAM_READONLY_FD), &created); 2000 if (fd < 0) { 2001 error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM", 2002 mem_path); 2003 if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) && 2004 fd == -EACCES) { 2005 /* 2006 * If we can open the file R/O (note: will never create a new file) 2007 * and we are dealing with a private mapping, there are still ways 2008 * to consume such files and get RAM instead of ROM. 2009 */ 2010 fd = file_ram_open(mem_path, memory_region_name(mr), true, 2011 &created); 2012 if (fd < 0) { 2013 return NULL; 2014 } 2015 assert(!created); 2016 close(fd); 2017 error_append_hint(errp, "Consider opening the backing store" 2018 " read-only but still creating writable RAM using" 2019 " '-object memory-backend-file,readonly=on,rom=off...'" 2020 " (see \"VM templating\" documentation)\n"); 2021 } 2022 return NULL; 2023 } 2024 2025 block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp); 2026 if (!block) { 2027 if (created) { 2028 unlink(mem_path); 2029 } 2030 close(fd); 2031 return NULL; 2032 } 2033 2034 return block; 2035 } 2036 #endif 2037 2038 static 2039 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size, 2040 void (*resized)(const char*, 2041 uint64_t length, 2042 void *host), 2043 void *host, uint32_t ram_flags, 2044 MemoryRegion *mr, Error **errp) 2045 { 2046 RAMBlock *new_block; 2047 Error *local_err = NULL; 2048 int align; 2049 2050 assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC | 2051 RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0); 2052 assert(!host ^ (ram_flags & RAM_PREALLOC)); 2053 2054 align = qemu_real_host_page_size(); 2055 align = MAX(align, TARGET_PAGE_SIZE); 2056 size = ROUND_UP(size, align); 2057 max_size = ROUND_UP(max_size, align); 2058 2059 new_block = g_malloc0(sizeof(*new_block)); 2060 new_block->mr = mr; 2061 new_block->resized = resized; 2062 new_block->used_length = size; 2063 new_block->max_length = max_size; 2064 assert(max_size >= size); 2065 new_block->fd = -1; 2066 new_block->guest_memfd = -1; 2067 new_block->page_size = qemu_real_host_page_size(); 2068 new_block->host = host; 2069 new_block->flags = ram_flags; 2070 ram_block_add(new_block, &local_err); 2071 if (local_err) { 2072 g_free(new_block); 2073 error_propagate(errp, local_err); 2074 return NULL; 2075 } 2076 return new_block; 2077 } 2078 2079 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 2080 MemoryRegion *mr, Error **errp) 2081 { 2082 return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr, 2083 errp); 2084 } 2085 2086 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, 2087 MemoryRegion *mr, Error **errp) 2088 { 2089 assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0); 2090 return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp); 2091 } 2092 2093 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz, 2094 void (*resized)(const char*, 2095 uint64_t length, 2096 void *host), 2097 MemoryRegion *mr, Error **errp) 2098 { 2099 return qemu_ram_alloc_internal(size, maxsz, resized, NULL, 2100 RAM_RESIZEABLE, mr, errp); 2101 } 2102 2103 static void reclaim_ramblock(RAMBlock *block) 2104 { 2105 if (block->flags & RAM_PREALLOC) { 2106 ; 2107 } else if (xen_enabled()) { 2108 xen_invalidate_map_cache_entry(block->host); 2109 #ifndef _WIN32 2110 } else if (block->fd >= 0) { 2111 qemu_ram_munmap(block->fd, block->host, block->max_length); 2112 close(block->fd); 2113 #endif 2114 } else { 2115 qemu_anon_ram_free(block->host, block->max_length); 2116 } 2117 2118 if (block->guest_memfd >= 0) { 2119 close(block->guest_memfd); 2120 ram_block_discard_require(false); 2121 } 2122 2123 g_free(block); 2124 } 2125 2126 void qemu_ram_free(RAMBlock *block) 2127 { 2128 if (!block) { 2129 return; 2130 } 2131 2132 if (block->host) { 2133 ram_block_notify_remove(block->host, block->used_length, 2134 block->max_length); 2135 } 2136 2137 qemu_mutex_lock_ramlist(); 2138 QLIST_REMOVE_RCU(block, next); 2139 ram_list.mru_block = NULL; 2140 /* Write list before version */ 2141 smp_wmb(); 2142 ram_list.version++; 2143 call_rcu(block, reclaim_ramblock, rcu); 2144 qemu_mutex_unlock_ramlist(); 2145 } 2146 2147 #ifndef _WIN32 2148 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length) 2149 { 2150 RAMBlock *block; 2151 ram_addr_t offset; 2152 int flags; 2153 void *area, *vaddr; 2154 int prot; 2155 2156 RAMBLOCK_FOREACH(block) { 2157 offset = addr - block->offset; 2158 if (offset < block->max_length) { 2159 vaddr = ramblock_ptr(block, offset); 2160 if (block->flags & RAM_PREALLOC) { 2161 ; 2162 } else if (xen_enabled()) { 2163 abort(); 2164 } else { 2165 flags = MAP_FIXED; 2166 flags |= block->flags & RAM_SHARED ? 2167 MAP_SHARED : MAP_PRIVATE; 2168 flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0; 2169 prot = PROT_READ; 2170 prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE; 2171 if (block->fd >= 0) { 2172 area = mmap(vaddr, length, prot, flags, block->fd, 2173 offset + block->fd_offset); 2174 } else { 2175 flags |= MAP_ANONYMOUS; 2176 area = mmap(vaddr, length, prot, flags, -1, 0); 2177 } 2178 if (area != vaddr) { 2179 error_report("Could not remap addr: " 2180 RAM_ADDR_FMT "@" RAM_ADDR_FMT "", 2181 length, addr); 2182 exit(1); 2183 } 2184 memory_try_enable_merging(vaddr, length); 2185 qemu_ram_setup_dump(vaddr, length); 2186 } 2187 } 2188 } 2189 } 2190 #endif /* !_WIN32 */ 2191 2192 /* 2193 * Return a host pointer to guest's ram. 2194 * For Xen, foreign mappings get created if they don't already exist. 2195 * 2196 * @block: block for the RAM to lookup (optional and may be NULL). 2197 * @addr: address within the memory region. 2198 * @size: pointer to requested size (optional and may be NULL). 2199 * size may get modified and return a value smaller than 2200 * what was requested. 2201 * @lock: wether to lock the mapping in xen-mapcache until invalidated. 2202 * @is_write: hint wether to map RW or RO in the xen-mapcache. 2203 * (optional and may always be set to true). 2204 * 2205 * Called within RCU critical section. 2206 */ 2207 static void *qemu_ram_ptr_length(RAMBlock *block, ram_addr_t addr, 2208 hwaddr *size, bool lock, 2209 bool is_write) 2210 { 2211 hwaddr len = 0; 2212 2213 if (size && *size == 0) { 2214 return NULL; 2215 } 2216 2217 if (block == NULL) { 2218 block = qemu_get_ram_block(addr); 2219 addr -= block->offset; 2220 } 2221 if (size) { 2222 *size = MIN(*size, block->max_length - addr); 2223 len = *size; 2224 } 2225 2226 if (xen_enabled() && block->host == NULL) { 2227 /* We need to check if the requested address is in the RAM 2228 * because we don't want to map the entire memory in QEMU. 2229 * In that case just map the requested area. 2230 */ 2231 if (block->offset == 0) { 2232 return xen_map_cache(block->mr, addr, len, lock, lock, 2233 is_write); 2234 } 2235 2236 block->host = xen_map_cache(block->mr, block->offset, 2237 block->max_length, 1, 2238 lock, is_write); 2239 } 2240 2241 return ramblock_ptr(block, addr); 2242 } 2243 2244 /* 2245 * Return a host pointer to ram allocated with qemu_ram_alloc. 2246 * This should not be used for general purpose DMA. Use address_space_map 2247 * or address_space_rw instead. For local memory (e.g. video ram) that the 2248 * device owns, use memory_region_get_ram_ptr. 2249 * 2250 * Called within RCU critical section. 2251 */ 2252 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr) 2253 { 2254 return qemu_ram_ptr_length(ram_block, addr, NULL, false, true); 2255 } 2256 2257 /* Return the offset of a hostpointer within a ramblock */ 2258 ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host) 2259 { 2260 ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host; 2261 assert((uintptr_t)host >= (uintptr_t)rb->host); 2262 assert(res < rb->max_length); 2263 2264 return res; 2265 } 2266 2267 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset, 2268 ram_addr_t *offset) 2269 { 2270 RAMBlock *block; 2271 uint8_t *host = ptr; 2272 2273 if (xen_enabled()) { 2274 ram_addr_t ram_addr; 2275 RCU_READ_LOCK_GUARD(); 2276 ram_addr = xen_ram_addr_from_mapcache(ptr); 2277 block = qemu_get_ram_block(ram_addr); 2278 if (block) { 2279 *offset = ram_addr - block->offset; 2280 } 2281 return block; 2282 } 2283 2284 RCU_READ_LOCK_GUARD(); 2285 block = qatomic_rcu_read(&ram_list.mru_block); 2286 if (block && block->host && host - block->host < block->max_length) { 2287 goto found; 2288 } 2289 2290 RAMBLOCK_FOREACH(block) { 2291 /* This case append when the block is not mapped. */ 2292 if (block->host == NULL) { 2293 continue; 2294 } 2295 if (host - block->host < block->max_length) { 2296 goto found; 2297 } 2298 } 2299 2300 return NULL; 2301 2302 found: 2303 *offset = (host - block->host); 2304 if (round_offset) { 2305 *offset &= TARGET_PAGE_MASK; 2306 } 2307 return block; 2308 } 2309 2310 /* 2311 * Finds the named RAMBlock 2312 * 2313 * name: The name of RAMBlock to find 2314 * 2315 * Returns: RAMBlock (or NULL if not found) 2316 */ 2317 RAMBlock *qemu_ram_block_by_name(const char *name) 2318 { 2319 RAMBlock *block; 2320 2321 RAMBLOCK_FOREACH(block) { 2322 if (!strcmp(name, block->idstr)) { 2323 return block; 2324 } 2325 } 2326 2327 return NULL; 2328 } 2329 2330 /* 2331 * Some of the system routines need to translate from a host pointer 2332 * (typically a TLB entry) back to a ram offset. 2333 */ 2334 ram_addr_t qemu_ram_addr_from_host(void *ptr) 2335 { 2336 RAMBlock *block; 2337 ram_addr_t offset; 2338 2339 block = qemu_ram_block_from_host(ptr, false, &offset); 2340 if (!block) { 2341 return RAM_ADDR_INVALID; 2342 } 2343 2344 return block->offset + offset; 2345 } 2346 2347 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) 2348 { 2349 ram_addr_t ram_addr; 2350 2351 ram_addr = qemu_ram_addr_from_host(ptr); 2352 if (ram_addr == RAM_ADDR_INVALID) { 2353 error_report("Bad ram pointer %p", ptr); 2354 abort(); 2355 } 2356 return ram_addr; 2357 } 2358 2359 static MemTxResult flatview_read(FlatView *fv, hwaddr addr, 2360 MemTxAttrs attrs, void *buf, hwaddr len); 2361 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, 2362 const void *buf, hwaddr len); 2363 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len, 2364 bool is_write, MemTxAttrs attrs); 2365 2366 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data, 2367 unsigned len, MemTxAttrs attrs) 2368 { 2369 subpage_t *subpage = opaque; 2370 uint8_t buf[8]; 2371 MemTxResult res; 2372 2373 #if defined(DEBUG_SUBPAGE) 2374 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__, 2375 subpage, len, addr); 2376 #endif 2377 res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len); 2378 if (res) { 2379 return res; 2380 } 2381 *data = ldn_p(buf, len); 2382 return MEMTX_OK; 2383 } 2384 2385 static MemTxResult subpage_write(void *opaque, hwaddr addr, 2386 uint64_t value, unsigned len, MemTxAttrs attrs) 2387 { 2388 subpage_t *subpage = opaque; 2389 uint8_t buf[8]; 2390 2391 #if defined(DEBUG_SUBPAGE) 2392 printf("%s: subpage %p len %u addr " HWADDR_FMT_plx 2393 " value %"PRIx64"\n", 2394 __func__, subpage, len, addr, value); 2395 #endif 2396 stn_p(buf, len, value); 2397 return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len); 2398 } 2399 2400 static bool subpage_accepts(void *opaque, hwaddr addr, 2401 unsigned len, bool is_write, 2402 MemTxAttrs attrs) 2403 { 2404 subpage_t *subpage = opaque; 2405 #if defined(DEBUG_SUBPAGE) 2406 printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n", 2407 __func__, subpage, is_write ? 'w' : 'r', len, addr); 2408 #endif 2409 2410 return flatview_access_valid(subpage->fv, addr + subpage->base, 2411 len, is_write, attrs); 2412 } 2413 2414 static const MemoryRegionOps subpage_ops = { 2415 .read_with_attrs = subpage_read, 2416 .write_with_attrs = subpage_write, 2417 .impl.min_access_size = 1, 2418 .impl.max_access_size = 8, 2419 .valid.min_access_size = 1, 2420 .valid.max_access_size = 8, 2421 .valid.accepts = subpage_accepts, 2422 .endianness = DEVICE_NATIVE_ENDIAN, 2423 }; 2424 2425 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, 2426 uint16_t section) 2427 { 2428 int idx, eidx; 2429 2430 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE) 2431 return -1; 2432 idx = SUBPAGE_IDX(start); 2433 eidx = SUBPAGE_IDX(end); 2434 #if defined(DEBUG_SUBPAGE) 2435 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n", 2436 __func__, mmio, start, end, idx, eidx, section); 2437 #endif 2438 for (; idx <= eidx; idx++) { 2439 mmio->sub_section[idx] = section; 2440 } 2441 2442 return 0; 2443 } 2444 2445 static subpage_t *subpage_init(FlatView *fv, hwaddr base) 2446 { 2447 subpage_t *mmio; 2448 2449 /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */ 2450 mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t)); 2451 mmio->fv = fv; 2452 mmio->base = base; 2453 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio, 2454 NULL, TARGET_PAGE_SIZE); 2455 mmio->iomem.subpage = true; 2456 #if defined(DEBUG_SUBPAGE) 2457 printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__, 2458 mmio, base, TARGET_PAGE_SIZE); 2459 #endif 2460 2461 return mmio; 2462 } 2463 2464 static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr) 2465 { 2466 assert(fv); 2467 MemoryRegionSection section = { 2468 .fv = fv, 2469 .mr = mr, 2470 .offset_within_address_space = 0, 2471 .offset_within_region = 0, 2472 .size = int128_2_64(), 2473 }; 2474 2475 return phys_section_add(map, §ion); 2476 } 2477 2478 MemoryRegionSection *iotlb_to_section(CPUState *cpu, 2479 hwaddr index, MemTxAttrs attrs) 2480 { 2481 int asidx = cpu_asidx_from_attrs(cpu, attrs); 2482 CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx]; 2483 AddressSpaceDispatch *d = cpuas->memory_dispatch; 2484 int section_index = index & ~TARGET_PAGE_MASK; 2485 MemoryRegionSection *ret; 2486 2487 assert(section_index < d->map.sections_nb); 2488 ret = d->map.sections + section_index; 2489 assert(ret->mr); 2490 assert(ret->mr->ops); 2491 2492 return ret; 2493 } 2494 2495 static void io_mem_init(void) 2496 { 2497 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL, 2498 NULL, UINT64_MAX); 2499 } 2500 2501 AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv) 2502 { 2503 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); 2504 uint16_t n; 2505 2506 n = dummy_section(&d->map, fv, &io_mem_unassigned); 2507 assert(n == PHYS_SECTION_UNASSIGNED); 2508 2509 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; 2510 2511 return d; 2512 } 2513 2514 void address_space_dispatch_free(AddressSpaceDispatch *d) 2515 { 2516 phys_sections_free(&d->map); 2517 g_free(d); 2518 } 2519 2520 static void do_nothing(CPUState *cpu, run_on_cpu_data d) 2521 { 2522 } 2523 2524 static void tcg_log_global_after_sync(MemoryListener *listener) 2525 { 2526 CPUAddressSpace *cpuas; 2527 2528 /* Wait for the CPU to end the current TB. This avoids the following 2529 * incorrect race: 2530 * 2531 * vCPU migration 2532 * ---------------------- ------------------------- 2533 * TLB check -> slow path 2534 * notdirty_mem_write 2535 * write to RAM 2536 * mark dirty 2537 * clear dirty flag 2538 * TLB check -> fast path 2539 * read memory 2540 * write to RAM 2541 * 2542 * by pushing the migration thread's memory read after the vCPU thread has 2543 * written the memory. 2544 */ 2545 if (replay_mode == REPLAY_MODE_NONE) { 2546 /* 2547 * VGA can make calls to this function while updating the screen. 2548 * In record/replay mode this causes a deadlock, because 2549 * run_on_cpu waits for rr mutex. Therefore no races are possible 2550 * in this case and no need for making run_on_cpu when 2551 * record/replay is enabled. 2552 */ 2553 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener); 2554 run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL); 2555 } 2556 } 2557 2558 static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data) 2559 { 2560 CPUAddressSpace *cpuas = data.host_ptr; 2561 2562 cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as); 2563 tlb_flush(cpu); 2564 } 2565 2566 static void tcg_commit(MemoryListener *listener) 2567 { 2568 CPUAddressSpace *cpuas; 2569 CPUState *cpu; 2570 2571 assert(tcg_enabled()); 2572 /* since each CPU stores ram addresses in its TLB cache, we must 2573 reset the modified entries */ 2574 cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener); 2575 cpu = cpuas->cpu; 2576 2577 /* 2578 * Defer changes to as->memory_dispatch until the cpu is quiescent. 2579 * Otherwise we race between (1) other cpu threads and (2) ongoing 2580 * i/o for the current cpu thread, with data cached by mmu_lookup(). 2581 * 2582 * In addition, queueing the work function will kick the cpu back to 2583 * the main loop, which will end the RCU critical section and reclaim 2584 * the memory data structures. 2585 * 2586 * That said, the listener is also called during realize, before 2587 * all of the tcg machinery for run-on is initialized: thus halt_cond. 2588 */ 2589 if (cpu->halt_cond) { 2590 async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas)); 2591 } else { 2592 tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas)); 2593 } 2594 } 2595 2596 static void memory_map_init(void) 2597 { 2598 system_memory = g_malloc(sizeof(*system_memory)); 2599 2600 memory_region_init(system_memory, NULL, "system", UINT64_MAX); 2601 address_space_init(&address_space_memory, system_memory, "memory"); 2602 2603 system_io = g_malloc(sizeof(*system_io)); 2604 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io", 2605 65536); 2606 address_space_init(&address_space_io, system_io, "I/O"); 2607 } 2608 2609 MemoryRegion *get_system_memory(void) 2610 { 2611 return system_memory; 2612 } 2613 2614 MemoryRegion *get_system_io(void) 2615 { 2616 return system_io; 2617 } 2618 2619 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, 2620 hwaddr length) 2621 { 2622 uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); 2623 addr += memory_region_get_ram_addr(mr); 2624 2625 /* No early return if dirty_log_mask is or becomes 0, because 2626 * cpu_physical_memory_set_dirty_range will still call 2627 * xen_modified_memory. 2628 */ 2629 if (dirty_log_mask) { 2630 dirty_log_mask = 2631 cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); 2632 } 2633 if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { 2634 assert(tcg_enabled()); 2635 tb_invalidate_phys_range(addr, addr + length - 1); 2636 dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); 2637 } 2638 cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); 2639 } 2640 2641 void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size) 2642 { 2643 /* 2644 * In principle this function would work on other memory region types too, 2645 * but the ROM device use case is the only one where this operation is 2646 * necessary. Other memory regions should use the 2647 * address_space_read/write() APIs. 2648 */ 2649 assert(memory_region_is_romd(mr)); 2650 2651 invalidate_and_set_dirty(mr, addr, size); 2652 } 2653 2654 int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) 2655 { 2656 unsigned access_size_max = mr->ops->valid.max_access_size; 2657 2658 /* Regions are assumed to support 1-4 byte accesses unless 2659 otherwise specified. */ 2660 if (access_size_max == 0) { 2661 access_size_max = 4; 2662 } 2663 2664 /* Bound the maximum access by the alignment of the address. */ 2665 if (!mr->ops->impl.unaligned) { 2666 unsigned align_size_max = addr & -addr; 2667 if (align_size_max != 0 && align_size_max < access_size_max) { 2668 access_size_max = align_size_max; 2669 } 2670 } 2671 2672 /* Don't attempt accesses larger than the maximum. */ 2673 if (l > access_size_max) { 2674 l = access_size_max; 2675 } 2676 l = pow2floor(l); 2677 2678 return l; 2679 } 2680 2681 bool prepare_mmio_access(MemoryRegion *mr) 2682 { 2683 bool release_lock = false; 2684 2685 if (!bql_locked()) { 2686 bql_lock(); 2687 release_lock = true; 2688 } 2689 if (mr->flush_coalesced_mmio) { 2690 qemu_flush_coalesced_mmio_buffer(); 2691 } 2692 2693 return release_lock; 2694 } 2695 2696 /** 2697 * flatview_access_allowed 2698 * @mr: #MemoryRegion to be accessed 2699 * @attrs: memory transaction attributes 2700 * @addr: address within that memory region 2701 * @len: the number of bytes to access 2702 * 2703 * Check if a memory transaction is allowed. 2704 * 2705 * Returns: true if transaction is allowed, false if denied. 2706 */ 2707 static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs, 2708 hwaddr addr, hwaddr len) 2709 { 2710 if (likely(!attrs.memory)) { 2711 return true; 2712 } 2713 if (memory_region_is_ram(mr)) { 2714 return true; 2715 } 2716 qemu_log_mask(LOG_GUEST_ERROR, 2717 "Invalid access to non-RAM device at " 2718 "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", " 2719 "region '%s'\n", addr, len, memory_region_name(mr)); 2720 return false; 2721 } 2722 2723 static MemTxResult flatview_write_continue_step(MemTxAttrs attrs, 2724 const uint8_t *buf, 2725 hwaddr len, hwaddr mr_addr, 2726 hwaddr *l, MemoryRegion *mr) 2727 { 2728 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) { 2729 return MEMTX_ACCESS_ERROR; 2730 } 2731 2732 if (!memory_access_is_direct(mr, true)) { 2733 uint64_t val; 2734 MemTxResult result; 2735 bool release_lock = prepare_mmio_access(mr); 2736 2737 *l = memory_access_size(mr, *l, mr_addr); 2738 /* 2739 * XXX: could force current_cpu to NULL to avoid 2740 * potential bugs 2741 */ 2742 2743 /* 2744 * Assure Coverity (and ourselves) that we are not going to OVERRUN 2745 * the buffer by following ldn_he_p(). 2746 */ 2747 #ifdef QEMU_STATIC_ANALYSIS 2748 assert((*l == 1 && len >= 1) || 2749 (*l == 2 && len >= 2) || 2750 (*l == 4 && len >= 4) || 2751 (*l == 8 && len >= 8)); 2752 #endif 2753 val = ldn_he_p(buf, *l); 2754 result = memory_region_dispatch_write(mr, mr_addr, val, 2755 size_memop(*l), attrs); 2756 if (release_lock) { 2757 bql_unlock(); 2758 } 2759 2760 return result; 2761 } else { 2762 /* RAM case */ 2763 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l, 2764 false, true); 2765 2766 memmove(ram_ptr, buf, *l); 2767 invalidate_and_set_dirty(mr, mr_addr, *l); 2768 2769 return MEMTX_OK; 2770 } 2771 } 2772 2773 /* Called within RCU critical section. */ 2774 static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr, 2775 MemTxAttrs attrs, 2776 const void *ptr, 2777 hwaddr len, hwaddr mr_addr, 2778 hwaddr l, MemoryRegion *mr) 2779 { 2780 MemTxResult result = MEMTX_OK; 2781 const uint8_t *buf = ptr; 2782 2783 for (;;) { 2784 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l, 2785 mr); 2786 2787 len -= l; 2788 buf += l; 2789 addr += l; 2790 2791 if (!len) { 2792 break; 2793 } 2794 2795 l = len; 2796 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs); 2797 } 2798 2799 return result; 2800 } 2801 2802 /* Called from RCU critical section. */ 2803 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, 2804 const void *buf, hwaddr len) 2805 { 2806 hwaddr l; 2807 hwaddr mr_addr; 2808 MemoryRegion *mr; 2809 2810 l = len; 2811 mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs); 2812 if (!flatview_access_allowed(mr, attrs, addr, len)) { 2813 return MEMTX_ACCESS_ERROR; 2814 } 2815 return flatview_write_continue(fv, addr, attrs, buf, len, 2816 mr_addr, l, mr); 2817 } 2818 2819 static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf, 2820 hwaddr len, hwaddr mr_addr, 2821 hwaddr *l, 2822 MemoryRegion *mr) 2823 { 2824 if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) { 2825 return MEMTX_ACCESS_ERROR; 2826 } 2827 2828 if (!memory_access_is_direct(mr, false)) { 2829 /* I/O case */ 2830 uint64_t val; 2831 MemTxResult result; 2832 bool release_lock = prepare_mmio_access(mr); 2833 2834 *l = memory_access_size(mr, *l, mr_addr); 2835 result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l), 2836 attrs); 2837 2838 /* 2839 * Assure Coverity (and ourselves) that we are not going to OVERRUN 2840 * the buffer by following stn_he_p(). 2841 */ 2842 #ifdef QEMU_STATIC_ANALYSIS 2843 assert((*l == 1 && len >= 1) || 2844 (*l == 2 && len >= 2) || 2845 (*l == 4 && len >= 4) || 2846 (*l == 8 && len >= 8)); 2847 #endif 2848 stn_he_p(buf, *l, val); 2849 2850 if (release_lock) { 2851 bql_unlock(); 2852 } 2853 return result; 2854 } else { 2855 /* RAM case */ 2856 uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l, 2857 false, false); 2858 2859 memcpy(buf, ram_ptr, *l); 2860 2861 return MEMTX_OK; 2862 } 2863 } 2864 2865 /* Called within RCU critical section. */ 2866 MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, 2867 MemTxAttrs attrs, void *ptr, 2868 hwaddr len, hwaddr mr_addr, hwaddr l, 2869 MemoryRegion *mr) 2870 { 2871 MemTxResult result = MEMTX_OK; 2872 uint8_t *buf = ptr; 2873 2874 fuzz_dma_read_cb(addr, len, mr); 2875 for (;;) { 2876 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr); 2877 2878 len -= l; 2879 buf += l; 2880 addr += l; 2881 2882 if (!len) { 2883 break; 2884 } 2885 2886 l = len; 2887 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs); 2888 } 2889 2890 return result; 2891 } 2892 2893 /* Called from RCU critical section. */ 2894 static MemTxResult flatview_read(FlatView *fv, hwaddr addr, 2895 MemTxAttrs attrs, void *buf, hwaddr len) 2896 { 2897 hwaddr l; 2898 hwaddr mr_addr; 2899 MemoryRegion *mr; 2900 2901 l = len; 2902 mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs); 2903 if (!flatview_access_allowed(mr, attrs, addr, len)) { 2904 return MEMTX_ACCESS_ERROR; 2905 } 2906 return flatview_read_continue(fv, addr, attrs, buf, len, 2907 mr_addr, l, mr); 2908 } 2909 2910 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, 2911 MemTxAttrs attrs, void *buf, hwaddr len) 2912 { 2913 MemTxResult result = MEMTX_OK; 2914 FlatView *fv; 2915 2916 if (len > 0) { 2917 RCU_READ_LOCK_GUARD(); 2918 fv = address_space_to_flatview(as); 2919 result = flatview_read(fv, addr, attrs, buf, len); 2920 } 2921 2922 return result; 2923 } 2924 2925 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, 2926 MemTxAttrs attrs, 2927 const void *buf, hwaddr len) 2928 { 2929 MemTxResult result = MEMTX_OK; 2930 FlatView *fv; 2931 2932 if (len > 0) { 2933 RCU_READ_LOCK_GUARD(); 2934 fv = address_space_to_flatview(as); 2935 result = flatview_write(fv, addr, attrs, buf, len); 2936 } 2937 2938 return result; 2939 } 2940 2941 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, 2942 void *buf, hwaddr len, bool is_write) 2943 { 2944 if (is_write) { 2945 return address_space_write(as, addr, attrs, buf, len); 2946 } else { 2947 return address_space_read_full(as, addr, attrs, buf, len); 2948 } 2949 } 2950 2951 MemTxResult address_space_set(AddressSpace *as, hwaddr addr, 2952 uint8_t c, hwaddr len, MemTxAttrs attrs) 2953 { 2954 #define FILLBUF_SIZE 512 2955 uint8_t fillbuf[FILLBUF_SIZE]; 2956 int l; 2957 MemTxResult error = MEMTX_OK; 2958 2959 memset(fillbuf, c, FILLBUF_SIZE); 2960 while (len > 0) { 2961 l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE; 2962 error |= address_space_write(as, addr, attrs, fillbuf, l); 2963 len -= l; 2964 addr += l; 2965 } 2966 2967 return error; 2968 } 2969 2970 void cpu_physical_memory_rw(hwaddr addr, void *buf, 2971 hwaddr len, bool is_write) 2972 { 2973 address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED, 2974 buf, len, is_write); 2975 } 2976 2977 enum write_rom_type { 2978 WRITE_DATA, 2979 FLUSH_CACHE, 2980 }; 2981 2982 static inline MemTxResult address_space_write_rom_internal(AddressSpace *as, 2983 hwaddr addr, 2984 MemTxAttrs attrs, 2985 const void *ptr, 2986 hwaddr len, 2987 enum write_rom_type type) 2988 { 2989 hwaddr l; 2990 uint8_t *ram_ptr; 2991 hwaddr addr1; 2992 MemoryRegion *mr; 2993 const uint8_t *buf = ptr; 2994 2995 RCU_READ_LOCK_GUARD(); 2996 while (len > 0) { 2997 l = len; 2998 mr = address_space_translate(as, addr, &addr1, &l, true, attrs); 2999 3000 if (!(memory_region_is_ram(mr) || 3001 memory_region_is_romd(mr))) { 3002 l = memory_access_size(mr, l, addr1); 3003 } else { 3004 /* ROM/RAM case */ 3005 ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); 3006 switch (type) { 3007 case WRITE_DATA: 3008 memcpy(ram_ptr, buf, l); 3009 invalidate_and_set_dirty(mr, addr1, l); 3010 break; 3011 case FLUSH_CACHE: 3012 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l); 3013 break; 3014 } 3015 } 3016 len -= l; 3017 buf += l; 3018 addr += l; 3019 } 3020 return MEMTX_OK; 3021 } 3022 3023 /* used for ROM loading : can write in RAM and ROM */ 3024 MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr, 3025 MemTxAttrs attrs, 3026 const void *buf, hwaddr len) 3027 { 3028 return address_space_write_rom_internal(as, addr, attrs, 3029 buf, len, WRITE_DATA); 3030 } 3031 3032 void cpu_flush_icache_range(hwaddr start, hwaddr len) 3033 { 3034 /* 3035 * This function should do the same thing as an icache flush that was 3036 * triggered from within the guest. For TCG we are always cache coherent, 3037 * so there is no need to flush anything. For KVM / Xen we need to flush 3038 * the host's instruction cache at least. 3039 */ 3040 if (tcg_enabled()) { 3041 return; 3042 } 3043 3044 address_space_write_rom_internal(&address_space_memory, 3045 start, MEMTXATTRS_UNSPECIFIED, 3046 NULL, len, FLUSH_CACHE); 3047 } 3048 3049 typedef struct { 3050 MemoryRegion *mr; 3051 void *buffer; 3052 hwaddr addr; 3053 hwaddr len; 3054 bool in_use; 3055 } BounceBuffer; 3056 3057 static BounceBuffer bounce; 3058 3059 typedef struct MapClient { 3060 QEMUBH *bh; 3061 QLIST_ENTRY(MapClient) link; 3062 } MapClient; 3063 3064 QemuMutex map_client_list_lock; 3065 static QLIST_HEAD(, MapClient) map_client_list 3066 = QLIST_HEAD_INITIALIZER(map_client_list); 3067 3068 static void cpu_unregister_map_client_do(MapClient *client) 3069 { 3070 QLIST_REMOVE(client, link); 3071 g_free(client); 3072 } 3073 3074 static void cpu_notify_map_clients_locked(void) 3075 { 3076 MapClient *client; 3077 3078 while (!QLIST_EMPTY(&map_client_list)) { 3079 client = QLIST_FIRST(&map_client_list); 3080 qemu_bh_schedule(client->bh); 3081 cpu_unregister_map_client_do(client); 3082 } 3083 } 3084 3085 void cpu_register_map_client(QEMUBH *bh) 3086 { 3087 MapClient *client = g_malloc(sizeof(*client)); 3088 3089 qemu_mutex_lock(&map_client_list_lock); 3090 client->bh = bh; 3091 QLIST_INSERT_HEAD(&map_client_list, client, link); 3092 /* Write map_client_list before reading in_use. */ 3093 smp_mb(); 3094 if (!qatomic_read(&bounce.in_use)) { 3095 cpu_notify_map_clients_locked(); 3096 } 3097 qemu_mutex_unlock(&map_client_list_lock); 3098 } 3099 3100 void cpu_exec_init_all(void) 3101 { 3102 qemu_mutex_init(&ram_list.mutex); 3103 /* The data structures we set up here depend on knowing the page size, 3104 * so no more changes can be made after this point. 3105 * In an ideal world, nothing we did before we had finished the 3106 * machine setup would care about the target page size, and we could 3107 * do this much later, rather than requiring board models to state 3108 * up front what their requirements are. 3109 */ 3110 finalize_target_page_bits(); 3111 io_mem_init(); 3112 memory_map_init(); 3113 qemu_mutex_init(&map_client_list_lock); 3114 } 3115 3116 void cpu_unregister_map_client(QEMUBH *bh) 3117 { 3118 MapClient *client; 3119 3120 qemu_mutex_lock(&map_client_list_lock); 3121 QLIST_FOREACH(client, &map_client_list, link) { 3122 if (client->bh == bh) { 3123 cpu_unregister_map_client_do(client); 3124 break; 3125 } 3126 } 3127 qemu_mutex_unlock(&map_client_list_lock); 3128 } 3129 3130 static void cpu_notify_map_clients(void) 3131 { 3132 qemu_mutex_lock(&map_client_list_lock); 3133 cpu_notify_map_clients_locked(); 3134 qemu_mutex_unlock(&map_client_list_lock); 3135 } 3136 3137 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len, 3138 bool is_write, MemTxAttrs attrs) 3139 { 3140 MemoryRegion *mr; 3141 hwaddr l, xlat; 3142 3143 while (len > 0) { 3144 l = len; 3145 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs); 3146 if (!memory_access_is_direct(mr, is_write)) { 3147 l = memory_access_size(mr, l, addr); 3148 if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) { 3149 return false; 3150 } 3151 } 3152 3153 len -= l; 3154 addr += l; 3155 } 3156 return true; 3157 } 3158 3159 bool address_space_access_valid(AddressSpace *as, hwaddr addr, 3160 hwaddr len, bool is_write, 3161 MemTxAttrs attrs) 3162 { 3163 FlatView *fv; 3164 3165 RCU_READ_LOCK_GUARD(); 3166 fv = address_space_to_flatview(as); 3167 return flatview_access_valid(fv, addr, len, is_write, attrs); 3168 } 3169 3170 static hwaddr 3171 flatview_extend_translation(FlatView *fv, hwaddr addr, 3172 hwaddr target_len, 3173 MemoryRegion *mr, hwaddr base, hwaddr len, 3174 bool is_write, MemTxAttrs attrs) 3175 { 3176 hwaddr done = 0; 3177 hwaddr xlat; 3178 MemoryRegion *this_mr; 3179 3180 for (;;) { 3181 target_len -= len; 3182 addr += len; 3183 done += len; 3184 if (target_len == 0) { 3185 return done; 3186 } 3187 3188 len = target_len; 3189 this_mr = flatview_translate(fv, addr, &xlat, 3190 &len, is_write, attrs); 3191 if (this_mr != mr || xlat != base + done) { 3192 return done; 3193 } 3194 } 3195 } 3196 3197 /* Map a physical memory region into a host virtual address. 3198 * May map a subset of the requested range, given by and returned in *plen. 3199 * May return NULL if resources needed to perform the mapping are exhausted. 3200 * Use only for reads OR writes - not for read-modify-write operations. 3201 * Use cpu_register_map_client() to know when retrying the map operation is 3202 * likely to succeed. 3203 */ 3204 void *address_space_map(AddressSpace *as, 3205 hwaddr addr, 3206 hwaddr *plen, 3207 bool is_write, 3208 MemTxAttrs attrs) 3209 { 3210 hwaddr len = *plen; 3211 hwaddr l, xlat; 3212 MemoryRegion *mr; 3213 FlatView *fv; 3214 3215 if (len == 0) { 3216 return NULL; 3217 } 3218 3219 l = len; 3220 RCU_READ_LOCK_GUARD(); 3221 fv = address_space_to_flatview(as); 3222 mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs); 3223 3224 if (!memory_access_is_direct(mr, is_write)) { 3225 if (qatomic_xchg(&bounce.in_use, true)) { 3226 *plen = 0; 3227 return NULL; 3228 } 3229 /* Avoid unbounded allocations */ 3230 l = MIN(l, TARGET_PAGE_SIZE); 3231 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l); 3232 bounce.addr = addr; 3233 bounce.len = l; 3234 3235 memory_region_ref(mr); 3236 bounce.mr = mr; 3237 if (!is_write) { 3238 flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED, 3239 bounce.buffer, l); 3240 } 3241 3242 *plen = l; 3243 return bounce.buffer; 3244 } 3245 3246 3247 memory_region_ref(mr); 3248 *plen = flatview_extend_translation(fv, addr, len, mr, xlat, 3249 l, is_write, attrs); 3250 fuzz_dma_read_cb(addr, *plen, mr); 3251 return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write); 3252 } 3253 3254 /* Unmaps a memory region previously mapped by address_space_map(). 3255 * Will also mark the memory as dirty if is_write is true. access_len gives 3256 * the amount of memory that was actually read or written by the caller. 3257 */ 3258 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, 3259 bool is_write, hwaddr access_len) 3260 { 3261 if (buffer != bounce.buffer) { 3262 MemoryRegion *mr; 3263 ram_addr_t addr1; 3264 3265 mr = memory_region_from_host(buffer, &addr1); 3266 assert(mr != NULL); 3267 if (is_write) { 3268 invalidate_and_set_dirty(mr, addr1, access_len); 3269 } 3270 if (xen_enabled()) { 3271 xen_invalidate_map_cache_entry(buffer); 3272 } 3273 memory_region_unref(mr); 3274 return; 3275 } 3276 if (is_write) { 3277 address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED, 3278 bounce.buffer, access_len); 3279 } 3280 qemu_vfree(bounce.buffer); 3281 bounce.buffer = NULL; 3282 memory_region_unref(bounce.mr); 3283 /* Clear in_use before reading map_client_list. */ 3284 qatomic_set_mb(&bounce.in_use, false); 3285 cpu_notify_map_clients(); 3286 } 3287 3288 void *cpu_physical_memory_map(hwaddr addr, 3289 hwaddr *plen, 3290 bool is_write) 3291 { 3292 return address_space_map(&address_space_memory, addr, plen, is_write, 3293 MEMTXATTRS_UNSPECIFIED); 3294 } 3295 3296 void cpu_physical_memory_unmap(void *buffer, hwaddr len, 3297 bool is_write, hwaddr access_len) 3298 { 3299 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len); 3300 } 3301 3302 #define ARG1_DECL AddressSpace *as 3303 #define ARG1 as 3304 #define SUFFIX 3305 #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__) 3306 #define RCU_READ_LOCK(...) rcu_read_lock() 3307 #define RCU_READ_UNLOCK(...) rcu_read_unlock() 3308 #include "memory_ldst.c.inc" 3309 3310 int64_t address_space_cache_init(MemoryRegionCache *cache, 3311 AddressSpace *as, 3312 hwaddr addr, 3313 hwaddr len, 3314 bool is_write) 3315 { 3316 AddressSpaceDispatch *d; 3317 hwaddr l; 3318 MemoryRegion *mr; 3319 Int128 diff; 3320 3321 assert(len > 0); 3322 3323 l = len; 3324 cache->fv = address_space_get_flatview(as); 3325 d = flatview_to_dispatch(cache->fv); 3326 cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true); 3327 3328 /* 3329 * cache->xlat is now relative to cache->mrs.mr, not to the section itself. 3330 * Take that into account to compute how many bytes are there between 3331 * cache->xlat and the end of the section. 3332 */ 3333 diff = int128_sub(cache->mrs.size, 3334 int128_make64(cache->xlat - cache->mrs.offset_within_region)); 3335 l = int128_get64(int128_min(diff, int128_make64(l))); 3336 3337 mr = cache->mrs.mr; 3338 memory_region_ref(mr); 3339 if (memory_access_is_direct(mr, is_write)) { 3340 /* We don't care about the memory attributes here as we're only 3341 * doing this if we found actual RAM, which behaves the same 3342 * regardless of attributes; so UNSPECIFIED is fine. 3343 */ 3344 l = flatview_extend_translation(cache->fv, addr, len, mr, 3345 cache->xlat, l, is_write, 3346 MEMTXATTRS_UNSPECIFIED); 3347 cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true, 3348 is_write); 3349 } else { 3350 cache->ptr = NULL; 3351 } 3352 3353 cache->len = l; 3354 cache->is_write = is_write; 3355 return l; 3356 } 3357 3358 void address_space_cache_invalidate(MemoryRegionCache *cache, 3359 hwaddr addr, 3360 hwaddr access_len) 3361 { 3362 assert(cache->is_write); 3363 if (likely(cache->ptr)) { 3364 invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len); 3365 } 3366 } 3367 3368 void address_space_cache_destroy(MemoryRegionCache *cache) 3369 { 3370 if (!cache->mrs.mr) { 3371 return; 3372 } 3373 3374 if (xen_enabled()) { 3375 xen_invalidate_map_cache_entry(cache->ptr); 3376 } 3377 memory_region_unref(cache->mrs.mr); 3378 flatview_unref(cache->fv); 3379 cache->mrs.mr = NULL; 3380 cache->fv = NULL; 3381 } 3382 3383 /* Called from RCU critical section. This function has the same 3384 * semantics as address_space_translate, but it only works on a 3385 * predefined range of a MemoryRegion that was mapped with 3386 * address_space_cache_init. 3387 */ 3388 static inline MemoryRegion *address_space_translate_cached( 3389 MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat, 3390 hwaddr *plen, bool is_write, MemTxAttrs attrs) 3391 { 3392 MemoryRegionSection section; 3393 MemoryRegion *mr; 3394 IOMMUMemoryRegion *iommu_mr; 3395 AddressSpace *target_as; 3396 3397 assert(!cache->ptr); 3398 *xlat = addr + cache->xlat; 3399 3400 mr = cache->mrs.mr; 3401 iommu_mr = memory_region_get_iommu(mr); 3402 if (!iommu_mr) { 3403 /* MMIO region. */ 3404 return mr; 3405 } 3406 3407 section = address_space_translate_iommu(iommu_mr, xlat, plen, 3408 NULL, is_write, true, 3409 &target_as, attrs); 3410 return section.mr; 3411 } 3412 3413 /* Called within RCU critical section. */ 3414 static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs, 3415 const void *ptr, 3416 hwaddr len, 3417 hwaddr mr_addr, 3418 hwaddr l, 3419 MemoryRegion *mr) 3420 { 3421 MemTxResult result = MEMTX_OK; 3422 const uint8_t *buf = ptr; 3423 3424 for (;;) { 3425 result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l, 3426 mr); 3427 3428 len -= l; 3429 buf += l; 3430 mr_addr += l; 3431 3432 if (!len) { 3433 break; 3434 } 3435 3436 l = len; 3437 } 3438 3439 return result; 3440 } 3441 3442 /* Called within RCU critical section. */ 3443 static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs, 3444 void *ptr, hwaddr len, 3445 hwaddr mr_addr, hwaddr l, 3446 MemoryRegion *mr) 3447 { 3448 MemTxResult result = MEMTX_OK; 3449 uint8_t *buf = ptr; 3450 3451 for (;;) { 3452 result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr); 3453 len -= l; 3454 buf += l; 3455 mr_addr += l; 3456 3457 if (!len) { 3458 break; 3459 } 3460 l = len; 3461 } 3462 3463 return result; 3464 } 3465 3466 /* Called from RCU critical section. address_space_read_cached uses this 3467 * out of line function when the target is an MMIO or IOMMU region. 3468 */ 3469 MemTxResult 3470 address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr, 3471 void *buf, hwaddr len) 3472 { 3473 hwaddr mr_addr, l; 3474 MemoryRegion *mr; 3475 3476 l = len; 3477 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false, 3478 MEMTXATTRS_UNSPECIFIED); 3479 return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED, 3480 buf, len, mr_addr, l, mr); 3481 } 3482 3483 /* Called from RCU critical section. address_space_write_cached uses this 3484 * out of line function when the target is an MMIO or IOMMU region. 3485 */ 3486 MemTxResult 3487 address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr, 3488 const void *buf, hwaddr len) 3489 { 3490 hwaddr mr_addr, l; 3491 MemoryRegion *mr; 3492 3493 l = len; 3494 mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true, 3495 MEMTXATTRS_UNSPECIFIED); 3496 return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED, 3497 buf, len, mr_addr, l, mr); 3498 } 3499 3500 #define ARG1_DECL MemoryRegionCache *cache 3501 #define ARG1 cache 3502 #define SUFFIX _cached_slow 3503 #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__) 3504 #define RCU_READ_LOCK() ((void)0) 3505 #define RCU_READ_UNLOCK() ((void)0) 3506 #include "memory_ldst.c.inc" 3507 3508 /* virtual memory access for debug (includes writing to ROM) */ 3509 int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, 3510 void *ptr, size_t len, bool is_write) 3511 { 3512 hwaddr phys_addr; 3513 vaddr l, page; 3514 uint8_t *buf = ptr; 3515 3516 cpu_synchronize_state(cpu); 3517 while (len > 0) { 3518 int asidx; 3519 MemTxAttrs attrs; 3520 MemTxResult res; 3521 3522 page = addr & TARGET_PAGE_MASK; 3523 phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs); 3524 asidx = cpu_asidx_from_attrs(cpu, attrs); 3525 /* if no physical page mapped, return an error */ 3526 if (phys_addr == -1) 3527 return -1; 3528 l = (page + TARGET_PAGE_SIZE) - addr; 3529 if (l > len) 3530 l = len; 3531 phys_addr += (addr & ~TARGET_PAGE_MASK); 3532 if (is_write) { 3533 res = address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr, 3534 attrs, buf, l); 3535 } else { 3536 res = address_space_read(cpu->cpu_ases[asidx].as, phys_addr, 3537 attrs, buf, l); 3538 } 3539 if (res != MEMTX_OK) { 3540 return -1; 3541 } 3542 len -= l; 3543 buf += l; 3544 addr += l; 3545 } 3546 return 0; 3547 } 3548 3549 bool cpu_physical_memory_is_io(hwaddr phys_addr) 3550 { 3551 MemoryRegion*mr; 3552 hwaddr l = 1; 3553 3554 RCU_READ_LOCK_GUARD(); 3555 mr = address_space_translate(&address_space_memory, 3556 phys_addr, &phys_addr, &l, false, 3557 MEMTXATTRS_UNSPECIFIED); 3558 3559 return !(memory_region_is_ram(mr) || memory_region_is_romd(mr)); 3560 } 3561 3562 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque) 3563 { 3564 RAMBlock *block; 3565 int ret = 0; 3566 3567 RCU_READ_LOCK_GUARD(); 3568 RAMBLOCK_FOREACH(block) { 3569 ret = func(block, opaque); 3570 if (ret) { 3571 break; 3572 } 3573 } 3574 return ret; 3575 } 3576 3577 /* 3578 * Unmap pages of memory from start to start+length such that 3579 * they a) read as 0, b) Trigger whatever fault mechanism 3580 * the OS provides for postcopy. 3581 * The pages must be unmapped by the end of the function. 3582 * Returns: 0 on success, none-0 on failure 3583 * 3584 */ 3585 int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) 3586 { 3587 int ret = -1; 3588 3589 uint8_t *host_startaddr = rb->host + start; 3590 3591 if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) { 3592 error_report("%s: Unaligned start address: %p", 3593 __func__, host_startaddr); 3594 goto err; 3595 } 3596 3597 if ((start + length) <= rb->max_length) { 3598 bool need_madvise, need_fallocate; 3599 if (!QEMU_IS_ALIGNED(length, rb->page_size)) { 3600 error_report("%s: Unaligned length: %zx", __func__, length); 3601 goto err; 3602 } 3603 3604 errno = ENOTSUP; /* If we are missing MADVISE etc */ 3605 3606 /* The logic here is messy; 3607 * madvise DONTNEED fails for hugepages 3608 * fallocate works on hugepages and shmem 3609 * shared anonymous memory requires madvise REMOVE 3610 */ 3611 need_madvise = (rb->page_size == qemu_real_host_page_size()); 3612 need_fallocate = rb->fd != -1; 3613 if (need_fallocate) { 3614 /* For a file, this causes the area of the file to be zero'd 3615 * if read, and for hugetlbfs also causes it to be unmapped 3616 * so a userfault will trigger. 3617 */ 3618 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE 3619 /* 3620 * fallocate() will fail with readonly files. Let's print a 3621 * proper error message. 3622 */ 3623 if (rb->flags & RAM_READONLY_FD) { 3624 error_report("%s: Discarding RAM with readonly files is not" 3625 " supported", __func__); 3626 goto err; 3627 3628 } 3629 /* 3630 * We'll discard data from the actual file, even though we only 3631 * have a MAP_PRIVATE mapping, possibly messing with other 3632 * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to 3633 * change that behavior whithout violating the promised 3634 * semantics of ram_block_discard_range(). 3635 * 3636 * Only warn, because it works as long as nobody else uses that 3637 * file. 3638 */ 3639 if (!qemu_ram_is_shared(rb)) { 3640 warn_report_once("%s: Discarding RAM" 3641 " in private file mappings is possibly" 3642 " dangerous, because it will modify the" 3643 " underlying file and will affect other" 3644 " users of the file", __func__); 3645 } 3646 3647 ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 3648 start, length); 3649 if (ret) { 3650 ret = -errno; 3651 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)", 3652 __func__, rb->idstr, start, length, ret); 3653 goto err; 3654 } 3655 #else 3656 ret = -ENOSYS; 3657 error_report("%s: fallocate not available/file" 3658 "%s:%" PRIx64 " +%zx (%d)", 3659 __func__, rb->idstr, start, length, ret); 3660 goto err; 3661 #endif 3662 } 3663 if (need_madvise) { 3664 /* For normal RAM this causes it to be unmapped, 3665 * for shared memory it causes the local mapping to disappear 3666 * and to fall back on the file contents (which we just 3667 * fallocate'd away). 3668 */ 3669 #if defined(CONFIG_MADVISE) 3670 if (qemu_ram_is_shared(rb) && rb->fd < 0) { 3671 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE); 3672 } else { 3673 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED); 3674 } 3675 if (ret) { 3676 ret = -errno; 3677 error_report("%s: Failed to discard range " 3678 "%s:%" PRIx64 " +%zx (%d)", 3679 __func__, rb->idstr, start, length, ret); 3680 goto err; 3681 } 3682 #else 3683 ret = -ENOSYS; 3684 error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)", 3685 __func__, rb->idstr, start, length, ret); 3686 goto err; 3687 #endif 3688 } 3689 trace_ram_block_discard_range(rb->idstr, host_startaddr, length, 3690 need_madvise, need_fallocate, ret); 3691 } else { 3692 error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")", 3693 __func__, rb->idstr, start, length, rb->max_length); 3694 } 3695 3696 err: 3697 return ret; 3698 } 3699 3700 int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, 3701 size_t length) 3702 { 3703 int ret = -1; 3704 3705 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE 3706 ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 3707 start, length); 3708 3709 if (ret) { 3710 ret = -errno; 3711 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)", 3712 __func__, rb->idstr, start, length, ret); 3713 } 3714 #else 3715 ret = -ENOSYS; 3716 error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)", 3717 __func__, rb->idstr, start, length, ret); 3718 #endif 3719 3720 return ret; 3721 } 3722 3723 bool ramblock_is_pmem(RAMBlock *rb) 3724 { 3725 return rb->flags & RAM_PMEM; 3726 } 3727 3728 static void mtree_print_phys_entries(int start, int end, int skip, int ptr) 3729 { 3730 if (start == end - 1) { 3731 qemu_printf("\t%3d ", start); 3732 } else { 3733 qemu_printf("\t%3d..%-3d ", start, end - 1); 3734 } 3735 qemu_printf(" skip=%d ", skip); 3736 if (ptr == PHYS_MAP_NODE_NIL) { 3737 qemu_printf(" ptr=NIL"); 3738 } else if (!skip) { 3739 qemu_printf(" ptr=#%d", ptr); 3740 } else { 3741 qemu_printf(" ptr=[%d]", ptr); 3742 } 3743 qemu_printf("\n"); 3744 } 3745 3746 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \ 3747 int128_sub((size), int128_one())) : 0) 3748 3749 void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root) 3750 { 3751 int i; 3752 3753 qemu_printf(" Dispatch\n"); 3754 qemu_printf(" Physical sections\n"); 3755 3756 for (i = 0; i < d->map.sections_nb; ++i) { 3757 MemoryRegionSection *s = d->map.sections + i; 3758 const char *names[] = { " [unassigned]", " [not dirty]", 3759 " [ROM]", " [watch]" }; 3760 3761 qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx 3762 " %s%s%s%s%s", 3763 i, 3764 s->offset_within_address_space, 3765 s->offset_within_address_space + MR_SIZE(s->size), 3766 s->mr->name ? s->mr->name : "(noname)", 3767 i < ARRAY_SIZE(names) ? names[i] : "", 3768 s->mr == root ? " [ROOT]" : "", 3769 s == d->mru_section ? " [MRU]" : "", 3770 s->mr->is_iommu ? " [iommu]" : ""); 3771 3772 if (s->mr->alias) { 3773 qemu_printf(" alias=%s", s->mr->alias->name ? 3774 s->mr->alias->name : "noname"); 3775 } 3776 qemu_printf("\n"); 3777 } 3778 3779 qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n", 3780 P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip); 3781 for (i = 0; i < d->map.nodes_nb; ++i) { 3782 int j, jprev; 3783 PhysPageEntry prev; 3784 Node *n = d->map.nodes + i; 3785 3786 qemu_printf(" [%d]\n", i); 3787 3788 for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) { 3789 PhysPageEntry *pe = *n + j; 3790 3791 if (pe->ptr == prev.ptr && pe->skip == prev.skip) { 3792 continue; 3793 } 3794 3795 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr); 3796 3797 jprev = j; 3798 prev = *pe; 3799 } 3800 3801 if (jprev != ARRAY_SIZE(*n)) { 3802 mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr); 3803 } 3804 } 3805 } 3806 3807 /* Require any discards to work. */ 3808 static unsigned int ram_block_discard_required_cnt; 3809 /* Require only coordinated discards to work. */ 3810 static unsigned int ram_block_coordinated_discard_required_cnt; 3811 /* Disable any discards. */ 3812 static unsigned int ram_block_discard_disabled_cnt; 3813 /* Disable only uncoordinated discards. */ 3814 static unsigned int ram_block_uncoordinated_discard_disabled_cnt; 3815 static QemuMutex ram_block_discard_disable_mutex; 3816 3817 static void ram_block_discard_disable_mutex_lock(void) 3818 { 3819 static gsize initialized; 3820 3821 if (g_once_init_enter(&initialized)) { 3822 qemu_mutex_init(&ram_block_discard_disable_mutex); 3823 g_once_init_leave(&initialized, 1); 3824 } 3825 qemu_mutex_lock(&ram_block_discard_disable_mutex); 3826 } 3827 3828 static void ram_block_discard_disable_mutex_unlock(void) 3829 { 3830 qemu_mutex_unlock(&ram_block_discard_disable_mutex); 3831 } 3832 3833 int ram_block_discard_disable(bool state) 3834 { 3835 int ret = 0; 3836 3837 ram_block_discard_disable_mutex_lock(); 3838 if (!state) { 3839 ram_block_discard_disabled_cnt--; 3840 } else if (ram_block_discard_required_cnt || 3841 ram_block_coordinated_discard_required_cnt) { 3842 ret = -EBUSY; 3843 } else { 3844 ram_block_discard_disabled_cnt++; 3845 } 3846 ram_block_discard_disable_mutex_unlock(); 3847 return ret; 3848 } 3849 3850 int ram_block_uncoordinated_discard_disable(bool state) 3851 { 3852 int ret = 0; 3853 3854 ram_block_discard_disable_mutex_lock(); 3855 if (!state) { 3856 ram_block_uncoordinated_discard_disabled_cnt--; 3857 } else if (ram_block_discard_required_cnt) { 3858 ret = -EBUSY; 3859 } else { 3860 ram_block_uncoordinated_discard_disabled_cnt++; 3861 } 3862 ram_block_discard_disable_mutex_unlock(); 3863 return ret; 3864 } 3865 3866 int ram_block_discard_require(bool state) 3867 { 3868 int ret = 0; 3869 3870 ram_block_discard_disable_mutex_lock(); 3871 if (!state) { 3872 ram_block_discard_required_cnt--; 3873 } else if (ram_block_discard_disabled_cnt || 3874 ram_block_uncoordinated_discard_disabled_cnt) { 3875 ret = -EBUSY; 3876 } else { 3877 ram_block_discard_required_cnt++; 3878 } 3879 ram_block_discard_disable_mutex_unlock(); 3880 return ret; 3881 } 3882 3883 int ram_block_coordinated_discard_require(bool state) 3884 { 3885 int ret = 0; 3886 3887 ram_block_discard_disable_mutex_lock(); 3888 if (!state) { 3889 ram_block_coordinated_discard_required_cnt--; 3890 } else if (ram_block_discard_disabled_cnt) { 3891 ret = -EBUSY; 3892 } else { 3893 ram_block_coordinated_discard_required_cnt++; 3894 } 3895 ram_block_discard_disable_mutex_unlock(); 3896 return ret; 3897 } 3898 3899 bool ram_block_discard_is_disabled(void) 3900 { 3901 return qatomic_read(&ram_block_discard_disabled_cnt) || 3902 qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt); 3903 } 3904 3905 bool ram_block_discard_is_required(void) 3906 { 3907 return qatomic_read(&ram_block_discard_required_cnt) || 3908 qatomic_read(&ram_block_coordinated_discard_required_cnt); 3909 } 3910