1 /* 2 * Physical memory management 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu/osdep.h" 17 #include "qemu/log.h" 18 #include "qapi/error.h" 19 #include "system/memory.h" 20 #include "qapi/visitor.h" 21 #include "qemu/bitops.h" 22 #include "qemu/error-report.h" 23 #include "qemu/main-loop.h" 24 #include "qemu/qemu-print.h" 25 #include "qemu/target-info.h" 26 #include "qom/object.h" 27 #include "trace.h" 28 #include "system/ram_addr.h" 29 #include "system/kvm.h" 30 #include "system/runstate.h" 31 #include "system/tcg.h" 32 #include "qemu/accel.h" 33 #include "accel/accel-ops.h" 34 #include "hw/boards.h" 35 #include "migration/vmstate.h" 36 #include "system/address-spaces.h" 37 38 #include "memory-internal.h" 39 40 //#define DEBUG_UNASSIGNED 41 42 static unsigned memory_region_transaction_depth; 43 static bool memory_region_update_pending; 44 static bool ioeventfd_update_pending; 45 unsigned int global_dirty_tracking; 46 47 static QTAILQ_HEAD(, MemoryListener) memory_listeners 48 = QTAILQ_HEAD_INITIALIZER(memory_listeners); 49 50 static QTAILQ_HEAD(, AddressSpace) address_spaces 51 = QTAILQ_HEAD_INITIALIZER(address_spaces); 52 53 static GHashTable *flat_views; 54 55 typedef struct AddrRange AddrRange; 56 57 /* 58 * Note that signed integers are needed for negative offsetting in aliases 59 * (large MemoryRegion::alias_offset). 60 */ 61 struct AddrRange { 62 Int128 start; 63 Int128 size; 64 }; 65 66 static AddrRange addrrange_make(Int128 start, Int128 size) 67 { 68 return (AddrRange) { start, size }; 69 } 70 71 static bool addrrange_equal(AddrRange r1, AddrRange r2) 72 { 73 return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size); 74 } 75 76 static Int128 addrrange_end(AddrRange r) 77 { 78 return int128_add(r.start, r.size); 79 } 80 81 static AddrRange addrrange_shift(AddrRange range, Int128 delta) 82 { 83 int128_addto(&range.start, delta); 84 return range; 85 } 86 87 static bool addrrange_contains(AddrRange range, Int128 addr) 88 { 89 return int128_ge(addr, range.start) 90 && int128_lt(addr, addrrange_end(range)); 91 } 92 93 static bool addrrange_intersects(AddrRange r1, AddrRange r2) 94 { 95 return addrrange_contains(r1, r2.start) 96 || addrrange_contains(r2, r1.start); 97 } 98 99 static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) 100 { 101 Int128 start = int128_max(r1.start, r2.start); 102 Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2)); 103 return addrrange_make(start, int128_sub(end, start)); 104 } 105 106 enum ListenerDirection { Forward, Reverse }; 107 108 #define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ 109 do { \ 110 MemoryListener *_listener; \ 111 \ 112 switch (_direction) { \ 113 case Forward: \ 114 QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ 115 if (_listener->_callback) { \ 116 _listener->_callback(_listener, ##_args); \ 117 } \ 118 } \ 119 break; \ 120 case Reverse: \ 121 QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, link) { \ 122 if (_listener->_callback) { \ 123 _listener->_callback(_listener, ##_args); \ 124 } \ 125 } \ 126 break; \ 127 default: \ 128 abort(); \ 129 } \ 130 } while (0) 131 132 #define MEMORY_LISTENER_CALL(_as, _callback, _direction, _section, _args...) \ 133 do { \ 134 MemoryListener *_listener; \ 135 \ 136 switch (_direction) { \ 137 case Forward: \ 138 QTAILQ_FOREACH(_listener, &(_as)->listeners, link_as) { \ 139 if (_listener->_callback) { \ 140 _listener->_callback(_listener, _section, ##_args); \ 141 } \ 142 } \ 143 break; \ 144 case Reverse: \ 145 QTAILQ_FOREACH_REVERSE(_listener, &(_as)->listeners, link_as) { \ 146 if (_listener->_callback) { \ 147 _listener->_callback(_listener, _section, ##_args); \ 148 } \ 149 } \ 150 break; \ 151 default: \ 152 abort(); \ 153 } \ 154 } while (0) 155 156 /* No need to ref/unref .mr, the FlatRange keeps it alive. */ 157 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \ 158 do { \ 159 MemoryRegionSection mrs = section_from_flat_range(fr, \ 160 address_space_to_flatview(as)); \ 161 MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \ 162 } while(0) 163 164 struct CoalescedMemoryRange { 165 AddrRange addr; 166 QTAILQ_ENTRY(CoalescedMemoryRange) link; 167 }; 168 169 struct MemoryRegionIoeventfd { 170 AddrRange addr; 171 bool match_data; 172 uint64_t data; 173 EventNotifier *e; 174 }; 175 176 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd *a, 177 MemoryRegionIoeventfd *b) 178 { 179 if (int128_lt(a->addr.start, b->addr.start)) { 180 return true; 181 } else if (int128_gt(a->addr.start, b->addr.start)) { 182 return false; 183 } else if (int128_lt(a->addr.size, b->addr.size)) { 184 return true; 185 } else if (int128_gt(a->addr.size, b->addr.size)) { 186 return false; 187 } else if (a->match_data < b->match_data) { 188 return true; 189 } else if (a->match_data > b->match_data) { 190 return false; 191 } else if (a->match_data) { 192 if (a->data < b->data) { 193 return true; 194 } else if (a->data > b->data) { 195 return false; 196 } 197 } 198 if (a->e < b->e) { 199 return true; 200 } else if (a->e > b->e) { 201 return false; 202 } 203 return false; 204 } 205 206 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd *a, 207 MemoryRegionIoeventfd *b) 208 { 209 if (int128_eq(a->addr.start, b->addr.start) && 210 (!int128_nz(a->addr.size) || !int128_nz(b->addr.size) || 211 (int128_eq(a->addr.size, b->addr.size) && 212 (a->match_data == b->match_data) && 213 ((a->match_data && (a->data == b->data)) || !a->match_data) && 214 (a->e == b->e)))) 215 return true; 216 217 return false; 218 } 219 220 /* Range of memory in the global map. Addresses are absolute. */ 221 struct FlatRange { 222 MemoryRegion *mr; 223 hwaddr offset_in_region; 224 AddrRange addr; 225 uint8_t dirty_log_mask; 226 bool romd_mode; 227 bool readonly; 228 bool nonvolatile; 229 bool unmergeable; 230 }; 231 232 #define FOR_EACH_FLAT_RANGE(var, view) \ 233 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) 234 235 static inline MemoryRegionSection 236 section_from_flat_range(FlatRange *fr, FlatView *fv) 237 { 238 return (MemoryRegionSection) { 239 .mr = fr->mr, 240 .fv = fv, 241 .offset_within_region = fr->offset_in_region, 242 .size = fr->addr.size, 243 .offset_within_address_space = int128_get64(fr->addr.start), 244 .readonly = fr->readonly, 245 .nonvolatile = fr->nonvolatile, 246 .unmergeable = fr->unmergeable, 247 }; 248 } 249 250 static bool flatrange_equal(FlatRange *a, FlatRange *b) 251 { 252 return a->mr == b->mr 253 && addrrange_equal(a->addr, b->addr) 254 && a->offset_in_region == b->offset_in_region 255 && a->romd_mode == b->romd_mode 256 && a->readonly == b->readonly 257 && a->nonvolatile == b->nonvolatile 258 && a->unmergeable == b->unmergeable; 259 } 260 261 static FlatView *flatview_new(MemoryRegion *mr_root) 262 { 263 FlatView *view; 264 265 view = g_new0(FlatView, 1); 266 view->ref = 1; 267 view->root = mr_root; 268 memory_region_ref(mr_root); 269 trace_flatview_new(view, mr_root); 270 271 return view; 272 } 273 274 /* Insert a range into a given position. Caller is responsible for maintaining 275 * sorting order. 276 */ 277 static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range) 278 { 279 if (view->nr == view->nr_allocated) { 280 view->nr_allocated = MAX(2 * view->nr, 10); 281 view->ranges = g_realloc(view->ranges, 282 view->nr_allocated * sizeof(*view->ranges)); 283 } 284 memmove(view->ranges + pos + 1, view->ranges + pos, 285 (view->nr - pos) * sizeof(FlatRange)); 286 view->ranges[pos] = *range; 287 memory_region_ref(range->mr); 288 ++view->nr; 289 } 290 291 static void flatview_destroy(FlatView *view) 292 { 293 int i; 294 295 trace_flatview_destroy(view, view->root); 296 if (view->dispatch) { 297 address_space_dispatch_free(view->dispatch); 298 } 299 for (i = 0; i < view->nr; i++) { 300 memory_region_unref(view->ranges[i].mr); 301 } 302 g_free(view->ranges); 303 memory_region_unref(view->root); 304 g_free(view); 305 } 306 307 static bool flatview_ref(FlatView *view) 308 { 309 return qatomic_fetch_inc_nonzero(&view->ref) > 0; 310 } 311 312 void flatview_unref(FlatView *view) 313 { 314 if (qatomic_fetch_dec(&view->ref) == 1) { 315 trace_flatview_destroy_rcu(view, view->root); 316 assert(view->root); 317 call_rcu(view, flatview_destroy, rcu); 318 } 319 } 320 321 static bool can_merge(FlatRange *r1, FlatRange *r2) 322 { 323 return int128_eq(addrrange_end(r1->addr), r2->addr.start) 324 && r1->mr == r2->mr 325 && int128_eq(int128_add(int128_make64(r1->offset_in_region), 326 r1->addr.size), 327 int128_make64(r2->offset_in_region)) 328 && r1->dirty_log_mask == r2->dirty_log_mask 329 && r1->romd_mode == r2->romd_mode 330 && r1->readonly == r2->readonly 331 && r1->nonvolatile == r2->nonvolatile 332 && !r1->unmergeable && !r2->unmergeable; 333 } 334 335 /* Attempt to simplify a view by merging adjacent ranges */ 336 static void flatview_simplify(FlatView *view) 337 { 338 unsigned i, j, k; 339 340 i = 0; 341 while (i < view->nr) { 342 j = i + 1; 343 while (j < view->nr 344 && can_merge(&view->ranges[j-1], &view->ranges[j])) { 345 int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size); 346 ++j; 347 } 348 ++i; 349 for (k = i; k < j; k++) { 350 memory_region_unref(view->ranges[k].mr); 351 } 352 memmove(&view->ranges[i], &view->ranges[j], 353 (view->nr - j) * sizeof(view->ranges[j])); 354 view->nr -= j - i; 355 } 356 } 357 358 static void adjust_endianness(MemoryRegion *mr, uint64_t *data, MemOp op) 359 { 360 if ((op & MO_BSWAP) != devend_memop(mr->ops->endianness)) { 361 switch (op & MO_SIZE) { 362 case MO_8: 363 break; 364 case MO_16: 365 *data = bswap16(*data); 366 break; 367 case MO_32: 368 *data = bswap32(*data); 369 break; 370 case MO_64: 371 *data = bswap64(*data); 372 break; 373 default: 374 g_assert_not_reached(); 375 } 376 } 377 } 378 379 static inline void memory_region_shift_read_access(uint64_t *value, 380 signed shift, 381 uint64_t mask, 382 uint64_t tmp) 383 { 384 if (shift >= 0) { 385 *value |= (tmp & mask) << shift; 386 } else { 387 *value |= (tmp & mask) >> -shift; 388 } 389 } 390 391 static inline uint64_t memory_region_shift_write_access(uint64_t *value, 392 signed shift, 393 uint64_t mask) 394 { 395 uint64_t tmp; 396 397 if (shift >= 0) { 398 tmp = (*value >> shift) & mask; 399 } else { 400 tmp = (*value << -shift) & mask; 401 } 402 403 return tmp; 404 } 405 406 static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset) 407 { 408 MemoryRegion *root; 409 hwaddr abs_addr = offset; 410 411 abs_addr += mr->addr; 412 for (root = mr; root->container; ) { 413 root = root->container; 414 abs_addr += root->addr; 415 } 416 417 return abs_addr; 418 } 419 420 static int get_cpu_index(void) 421 { 422 if (current_cpu) { 423 return current_cpu->cpu_index; 424 } 425 return -1; 426 } 427 428 static MemTxResult memory_region_read_accessor(MemoryRegion *mr, 429 hwaddr addr, 430 uint64_t *value, 431 unsigned size, 432 signed shift, 433 uint64_t mask, 434 MemTxAttrs attrs) 435 { 436 uint64_t tmp; 437 438 tmp = mr->ops->read(mr->opaque, addr, size); 439 if (mr->subpage) { 440 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size); 441 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) { 442 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 443 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size, 444 memory_region_name(mr)); 445 } 446 memory_region_shift_read_access(value, shift, mask, tmp); 447 return MEMTX_OK; 448 } 449 450 static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr, 451 hwaddr addr, 452 uint64_t *value, 453 unsigned size, 454 signed shift, 455 uint64_t mask, 456 MemTxAttrs attrs) 457 { 458 uint64_t tmp = 0; 459 MemTxResult r; 460 461 r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs); 462 if (mr->subpage) { 463 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size); 464 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) { 465 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 466 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size, 467 memory_region_name(mr)); 468 } 469 memory_region_shift_read_access(value, shift, mask, tmp); 470 return r; 471 } 472 473 static MemTxResult memory_region_write_accessor(MemoryRegion *mr, 474 hwaddr addr, 475 uint64_t *value, 476 unsigned size, 477 signed shift, 478 uint64_t mask, 479 MemTxAttrs attrs) 480 { 481 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); 482 483 if (mr->subpage) { 484 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); 485 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) { 486 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 487 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size, 488 memory_region_name(mr)); 489 } 490 mr->ops->write(mr->opaque, addr, tmp, size); 491 return MEMTX_OK; 492 } 493 494 static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr, 495 hwaddr addr, 496 uint64_t *value, 497 unsigned size, 498 signed shift, 499 uint64_t mask, 500 MemTxAttrs attrs) 501 { 502 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); 503 504 if (mr->subpage) { 505 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); 506 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) { 507 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 508 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size, 509 memory_region_name(mr)); 510 } 511 return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs); 512 } 513 514 static MemTxResult access_with_adjusted_size(hwaddr addr, 515 uint64_t *value, 516 unsigned size, 517 unsigned access_size_min, 518 unsigned access_size_max, 519 MemTxResult (*access_fn) 520 (MemoryRegion *mr, 521 hwaddr addr, 522 uint64_t *value, 523 unsigned size, 524 signed shift, 525 uint64_t mask, 526 MemTxAttrs attrs), 527 MemoryRegion *mr, 528 MemTxAttrs attrs) 529 { 530 uint64_t access_mask; 531 unsigned access_size; 532 unsigned i; 533 MemTxResult r = MEMTX_OK; 534 bool reentrancy_guard_applied = false; 535 536 if (!access_size_min) { 537 access_size_min = 1; 538 } 539 if (!access_size_max) { 540 access_size_max = 4; 541 } 542 543 /* Do not allow more than one simultaneous access to a device's IO Regions */ 544 if (mr->dev && !mr->disable_reentrancy_guard && 545 !mr->ram_device && !mr->ram && !mr->rom_device && !mr->readonly) { 546 if (mr->dev->mem_reentrancy_guard.engaged_in_io) { 547 warn_report_once("Blocked re-entrant IO on MemoryRegion: " 548 "%s at addr: 0x%" HWADDR_PRIX, 549 memory_region_name(mr), addr); 550 return MEMTX_ACCESS_ERROR; 551 } 552 mr->dev->mem_reentrancy_guard.engaged_in_io = true; 553 reentrancy_guard_applied = true; 554 } 555 556 /* FIXME: support unaligned access? */ 557 access_size = MAX(MIN(size, access_size_max), access_size_min); 558 access_mask = MAKE_64BIT_MASK(0, access_size * 8); 559 if (devend_big_endian(mr->ops->endianness)) { 560 for (i = 0; i < size; i += access_size) { 561 r |= access_fn(mr, addr + i, value, access_size, 562 (size - access_size - i) * 8, access_mask, attrs); 563 } 564 } else { 565 for (i = 0; i < size; i += access_size) { 566 r |= access_fn(mr, addr + i, value, access_size, i * 8, 567 access_mask, attrs); 568 } 569 } 570 if (mr->dev && reentrancy_guard_applied) { 571 mr->dev->mem_reentrancy_guard.engaged_in_io = false; 572 } 573 return r; 574 } 575 576 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) 577 { 578 AddressSpace *as; 579 580 while (mr->container) { 581 mr = mr->container; 582 } 583 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 584 if (mr == as->root) { 585 return as; 586 } 587 } 588 return NULL; 589 } 590 591 /* Render a memory region into the global view. Ranges in @view obscure 592 * ranges in @mr. 593 */ 594 static void render_memory_region(FlatView *view, 595 MemoryRegion *mr, 596 Int128 base, 597 AddrRange clip, 598 bool readonly, 599 bool nonvolatile, 600 bool unmergeable) 601 { 602 MemoryRegion *subregion; 603 unsigned i; 604 hwaddr offset_in_region; 605 Int128 remain; 606 Int128 now; 607 FlatRange fr; 608 AddrRange tmp; 609 610 if (!mr->enabled) { 611 return; 612 } 613 614 int128_addto(&base, int128_make64(mr->addr)); 615 readonly |= mr->readonly; 616 nonvolatile |= mr->nonvolatile; 617 unmergeable |= mr->unmergeable; 618 619 tmp = addrrange_make(base, mr->size); 620 621 if (!addrrange_intersects(tmp, clip)) { 622 return; 623 } 624 625 clip = addrrange_intersection(tmp, clip); 626 627 if (mr->alias) { 628 int128_subfrom(&base, int128_make64(mr->alias->addr)); 629 int128_subfrom(&base, int128_make64(mr->alias_offset)); 630 render_memory_region(view, mr->alias, base, clip, 631 readonly, nonvolatile, unmergeable); 632 return; 633 } 634 635 /* Render subregions in priority order. */ 636 QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) { 637 render_memory_region(view, subregion, base, clip, 638 readonly, nonvolatile, unmergeable); 639 } 640 641 if (!mr->terminates) { 642 return; 643 } 644 645 offset_in_region = int128_get64(int128_sub(clip.start, base)); 646 base = clip.start; 647 remain = clip.size; 648 649 fr.mr = mr; 650 fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr); 651 fr.romd_mode = mr->romd_mode; 652 fr.readonly = readonly; 653 fr.nonvolatile = nonvolatile; 654 fr.unmergeable = unmergeable; 655 656 /* Render the region itself into any gaps left by the current view. */ 657 for (i = 0; i < view->nr && int128_nz(remain); ++i) { 658 if (int128_ge(base, addrrange_end(view->ranges[i].addr))) { 659 continue; 660 } 661 if (int128_lt(base, view->ranges[i].addr.start)) { 662 now = int128_min(remain, 663 int128_sub(view->ranges[i].addr.start, base)); 664 fr.offset_in_region = offset_in_region; 665 fr.addr = addrrange_make(base, now); 666 flatview_insert(view, i, &fr); 667 ++i; 668 int128_addto(&base, now); 669 offset_in_region += int128_get64(now); 670 int128_subfrom(&remain, now); 671 } 672 now = int128_sub(int128_min(int128_add(base, remain), 673 addrrange_end(view->ranges[i].addr)), 674 base); 675 int128_addto(&base, now); 676 offset_in_region += int128_get64(now); 677 int128_subfrom(&remain, now); 678 } 679 if (int128_nz(remain)) { 680 fr.offset_in_region = offset_in_region; 681 fr.addr = addrrange_make(base, remain); 682 flatview_insert(view, i, &fr); 683 } 684 } 685 686 void flatview_for_each_range(FlatView *fv, flatview_cb cb , void *opaque) 687 { 688 FlatRange *fr; 689 690 assert(fv); 691 assert(cb); 692 693 FOR_EACH_FLAT_RANGE(fr, fv) { 694 if (cb(fr->addr.start, fr->addr.size, fr->mr, 695 fr->offset_in_region, opaque)) { 696 break; 697 } 698 } 699 } 700 701 static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) 702 { 703 while (mr->enabled) { 704 if (mr->alias) { 705 if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) { 706 /* The alias is included in its entirety. Use it as 707 * the "real" root, so that we can share more FlatViews. 708 */ 709 mr = mr->alias; 710 continue; 711 } 712 } else if (!mr->terminates) { 713 unsigned int found = 0; 714 MemoryRegion *child, *next = NULL; 715 QTAILQ_FOREACH(child, &mr->subregions, subregions_link) { 716 if (child->enabled) { 717 if (++found > 1) { 718 next = NULL; 719 break; 720 } 721 if (!child->addr && int128_ge(mr->size, child->size)) { 722 /* A child is included in its entirety. If it's the only 723 * enabled one, use it in the hope of finding an alias down the 724 * way. This will also let us share FlatViews. 725 */ 726 next = child; 727 } 728 } 729 } 730 if (found == 0) { 731 return NULL; 732 } 733 if (next) { 734 mr = next; 735 continue; 736 } 737 } 738 739 return mr; 740 } 741 742 return NULL; 743 } 744 745 /* Render a memory topology into a list of disjoint absolute ranges. */ 746 static FlatView *generate_memory_topology(MemoryRegion *mr) 747 { 748 int i; 749 FlatView *view; 750 751 view = flatview_new(mr); 752 753 if (mr) { 754 render_memory_region(view, mr, int128_zero(), 755 addrrange_make(int128_zero(), int128_2_64()), 756 false, false, false); 757 } 758 flatview_simplify(view); 759 760 view->dispatch = address_space_dispatch_new(view); 761 for (i = 0; i < view->nr; i++) { 762 MemoryRegionSection mrs = 763 section_from_flat_range(&view->ranges[i], view); 764 flatview_add_to_dispatch(view, &mrs); 765 } 766 address_space_dispatch_compact(view->dispatch); 767 g_hash_table_replace(flat_views, mr, view); 768 769 return view; 770 } 771 772 static void address_space_add_del_ioeventfds(AddressSpace *as, 773 MemoryRegionIoeventfd *fds_new, 774 unsigned fds_new_nb, 775 MemoryRegionIoeventfd *fds_old, 776 unsigned fds_old_nb) 777 { 778 unsigned iold, inew; 779 MemoryRegionIoeventfd *fd; 780 MemoryRegionSection section; 781 782 /* Generate a symmetric difference of the old and new fd sets, adding 783 * and deleting as necessary. 784 */ 785 786 iold = inew = 0; 787 while (iold < fds_old_nb || inew < fds_new_nb) { 788 if (iold < fds_old_nb 789 && (inew == fds_new_nb 790 || memory_region_ioeventfd_before(&fds_old[iold], 791 &fds_new[inew]))) { 792 fd = &fds_old[iold]; 793 section = (MemoryRegionSection) { 794 .fv = address_space_to_flatview(as), 795 .offset_within_address_space = int128_get64(fd->addr.start), 796 .size = fd->addr.size, 797 }; 798 MEMORY_LISTENER_CALL(as, eventfd_del, Forward, §ion, 799 fd->match_data, fd->data, fd->e); 800 ++iold; 801 } else if (inew < fds_new_nb 802 && (iold == fds_old_nb 803 || memory_region_ioeventfd_before(&fds_new[inew], 804 &fds_old[iold]))) { 805 fd = &fds_new[inew]; 806 section = (MemoryRegionSection) { 807 .fv = address_space_to_flatview(as), 808 .offset_within_address_space = int128_get64(fd->addr.start), 809 .size = fd->addr.size, 810 }; 811 MEMORY_LISTENER_CALL(as, eventfd_add, Reverse, §ion, 812 fd->match_data, fd->data, fd->e); 813 ++inew; 814 } else { 815 ++iold; 816 ++inew; 817 } 818 } 819 } 820 821 FlatView *address_space_get_flatview(AddressSpace *as) 822 { 823 FlatView *view; 824 825 RCU_READ_LOCK_GUARD(); 826 do { 827 view = address_space_to_flatview(as); 828 /* If somebody has replaced as->current_map concurrently, 829 * flatview_ref returns false. 830 */ 831 } while (!flatview_ref(view)); 832 return view; 833 } 834 835 static void address_space_update_ioeventfds(AddressSpace *as) 836 { 837 FlatView *view; 838 FlatRange *fr; 839 unsigned ioeventfd_nb = 0; 840 unsigned ioeventfd_max; 841 MemoryRegionIoeventfd *ioeventfds; 842 AddrRange tmp; 843 unsigned i; 844 845 if (!as->ioeventfd_notifiers) { 846 return; 847 } 848 849 /* 850 * It is likely that the number of ioeventfds hasn't changed much, so use 851 * the previous size as the starting value, with some headroom to avoid 852 * gratuitous reallocations. 853 */ 854 ioeventfd_max = QEMU_ALIGN_UP(as->ioeventfd_nb, 4); 855 ioeventfds = g_new(MemoryRegionIoeventfd, ioeventfd_max); 856 857 view = address_space_get_flatview(as); 858 FOR_EACH_FLAT_RANGE(fr, view) { 859 for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { 860 tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, 861 int128_sub(fr->addr.start, 862 int128_make64(fr->offset_in_region))); 863 if (addrrange_intersects(fr->addr, tmp)) { 864 ++ioeventfd_nb; 865 if (ioeventfd_nb > ioeventfd_max) { 866 ioeventfd_max = MAX(ioeventfd_max * 2, 4); 867 ioeventfds = g_realloc(ioeventfds, 868 ioeventfd_max * sizeof(*ioeventfds)); 869 } 870 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i]; 871 ioeventfds[ioeventfd_nb-1].addr = tmp; 872 } 873 } 874 } 875 876 address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb, 877 as->ioeventfds, as->ioeventfd_nb); 878 879 g_free(as->ioeventfds); 880 as->ioeventfds = ioeventfds; 881 as->ioeventfd_nb = ioeventfd_nb; 882 flatview_unref(view); 883 } 884 885 /* 886 * Notify the memory listeners about the coalesced IO change events of 887 * range `cmr'. Only the part that has intersection of the specified 888 * FlatRange will be sent. 889 */ 890 static void flat_range_coalesced_io_notify(FlatRange *fr, AddressSpace *as, 891 CoalescedMemoryRange *cmr, bool add) 892 { 893 AddrRange tmp; 894 895 tmp = addrrange_shift(cmr->addr, 896 int128_sub(fr->addr.start, 897 int128_make64(fr->offset_in_region))); 898 if (!addrrange_intersects(tmp, fr->addr)) { 899 return; 900 } 901 tmp = addrrange_intersection(tmp, fr->addr); 902 903 if (add) { 904 MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, coalesced_io_add, 905 int128_get64(tmp.start), 906 int128_get64(tmp.size)); 907 } else { 908 MEMORY_LISTENER_UPDATE_REGION(fr, as, Reverse, coalesced_io_del, 909 int128_get64(tmp.start), 910 int128_get64(tmp.size)); 911 } 912 } 913 914 static void flat_range_coalesced_io_del(FlatRange *fr, AddressSpace *as) 915 { 916 CoalescedMemoryRange *cmr; 917 918 QTAILQ_FOREACH(cmr, &fr->mr->coalesced, link) { 919 flat_range_coalesced_io_notify(fr, as, cmr, false); 920 } 921 } 922 923 static void flat_range_coalesced_io_add(FlatRange *fr, AddressSpace *as) 924 { 925 MemoryRegion *mr = fr->mr; 926 CoalescedMemoryRange *cmr; 927 928 if (QTAILQ_EMPTY(&mr->coalesced)) { 929 return; 930 } 931 932 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { 933 flat_range_coalesced_io_notify(fr, as, cmr, true); 934 } 935 } 936 937 static void 938 flat_range_coalesced_io_notify_listener_add_del(FlatRange *fr, 939 MemoryRegionSection *mrs, 940 MemoryListener *listener, 941 AddressSpace *as, bool add) 942 { 943 CoalescedMemoryRange *cmr; 944 MemoryRegion *mr = fr->mr; 945 AddrRange tmp; 946 947 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { 948 tmp = addrrange_shift(cmr->addr, 949 int128_sub(fr->addr.start, 950 int128_make64(fr->offset_in_region))); 951 952 if (!addrrange_intersects(tmp, fr->addr)) { 953 return; 954 } 955 tmp = addrrange_intersection(tmp, fr->addr); 956 957 if (add && listener->coalesced_io_add) { 958 listener->coalesced_io_add(listener, mrs, 959 int128_get64(tmp.start), 960 int128_get64(tmp.size)); 961 } else if (!add && listener->coalesced_io_del) { 962 listener->coalesced_io_del(listener, mrs, 963 int128_get64(tmp.start), 964 int128_get64(tmp.size)); 965 } 966 } 967 } 968 969 static void address_space_update_topology_pass(AddressSpace *as, 970 const FlatView *old_view, 971 const FlatView *new_view, 972 bool adding) 973 { 974 unsigned iold, inew; 975 FlatRange *frold, *frnew; 976 977 /* Generate a symmetric difference of the old and new memory maps. 978 * Kill ranges in the old map, and instantiate ranges in the new map. 979 */ 980 iold = inew = 0; 981 while (iold < old_view->nr || inew < new_view->nr) { 982 if (iold < old_view->nr) { 983 frold = &old_view->ranges[iold]; 984 } else { 985 frold = NULL; 986 } 987 if (inew < new_view->nr) { 988 frnew = &new_view->ranges[inew]; 989 } else { 990 frnew = NULL; 991 } 992 993 if (frold 994 && (!frnew 995 || int128_lt(frold->addr.start, frnew->addr.start) 996 || (int128_eq(frold->addr.start, frnew->addr.start) 997 && !flatrange_equal(frold, frnew)))) { 998 /* In old but not in new, or in both but attributes changed. */ 999 1000 if (!adding) { 1001 flat_range_coalesced_io_del(frold, as); 1002 MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); 1003 } 1004 1005 ++iold; 1006 } else if (frold && frnew && flatrange_equal(frold, frnew)) { 1007 /* In both and unchanged (except logging may have changed) */ 1008 1009 if (adding) { 1010 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); 1011 if (frnew->dirty_log_mask & ~frold->dirty_log_mask) { 1012 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start, 1013 frold->dirty_log_mask, 1014 frnew->dirty_log_mask); 1015 } 1016 if (frold->dirty_log_mask & ~frnew->dirty_log_mask) { 1017 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop, 1018 frold->dirty_log_mask, 1019 frnew->dirty_log_mask); 1020 } 1021 } 1022 1023 ++iold; 1024 ++inew; 1025 } else { 1026 /* In new */ 1027 1028 if (adding) { 1029 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); 1030 flat_range_coalesced_io_add(frnew, as); 1031 } 1032 1033 ++inew; 1034 } 1035 } 1036 } 1037 1038 static void flatviews_init(void) 1039 { 1040 static FlatView *empty_view; 1041 1042 if (flat_views) { 1043 return; 1044 } 1045 1046 flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, 1047 (GDestroyNotify) flatview_unref); 1048 if (!empty_view) { 1049 empty_view = generate_memory_topology(NULL); 1050 /* We keep it alive forever in the global variable. */ 1051 flatview_ref(empty_view); 1052 } else { 1053 g_hash_table_replace(flat_views, NULL, empty_view); 1054 flatview_ref(empty_view); 1055 } 1056 } 1057 1058 static void flatviews_reset(void) 1059 { 1060 AddressSpace *as; 1061 1062 if (flat_views) { 1063 g_hash_table_unref(flat_views); 1064 flat_views = NULL; 1065 } 1066 flatviews_init(); 1067 1068 /* Render unique FVs */ 1069 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1070 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1071 1072 if (g_hash_table_lookup(flat_views, physmr)) { 1073 continue; 1074 } 1075 1076 generate_memory_topology(physmr); 1077 } 1078 } 1079 1080 static void address_space_set_flatview(AddressSpace *as) 1081 { 1082 FlatView *old_view = address_space_to_flatview(as); 1083 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1084 FlatView *new_view = g_hash_table_lookup(flat_views, physmr); 1085 1086 assert(new_view); 1087 1088 if (old_view == new_view) { 1089 return; 1090 } 1091 1092 if (old_view) { 1093 flatview_ref(old_view); 1094 } 1095 1096 flatview_ref(new_view); 1097 1098 if (!QTAILQ_EMPTY(&as->listeners)) { 1099 FlatView tmpview = { .nr = 0 }, *old_view2 = old_view; 1100 1101 if (!old_view2) { 1102 old_view2 = &tmpview; 1103 } 1104 address_space_update_topology_pass(as, old_view2, new_view, false); 1105 address_space_update_topology_pass(as, old_view2, new_view, true); 1106 } 1107 1108 /* Writes are protected by the BQL. */ 1109 qatomic_rcu_set(&as->current_map, new_view); 1110 if (old_view) { 1111 flatview_unref(old_view); 1112 } 1113 1114 /* Note that all the old MemoryRegions are still alive up to this 1115 * point. This relieves most MemoryListeners from the need to 1116 * ref/unref the MemoryRegions they get---unless they use them 1117 * outside the iothread mutex, in which case precise reference 1118 * counting is necessary. 1119 */ 1120 if (old_view) { 1121 flatview_unref(old_view); 1122 } 1123 } 1124 1125 static void address_space_update_topology(AddressSpace *as) 1126 { 1127 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1128 1129 flatviews_init(); 1130 if (!g_hash_table_lookup(flat_views, physmr)) { 1131 generate_memory_topology(physmr); 1132 } 1133 address_space_set_flatview(as); 1134 } 1135 1136 void memory_region_transaction_begin(void) 1137 { 1138 qemu_flush_coalesced_mmio_buffer(); 1139 ++memory_region_transaction_depth; 1140 } 1141 1142 void memory_region_transaction_commit(void) 1143 { 1144 AddressSpace *as; 1145 1146 assert(memory_region_transaction_depth); 1147 assert(bql_locked()); 1148 1149 --memory_region_transaction_depth; 1150 if (!memory_region_transaction_depth) { 1151 if (memory_region_update_pending) { 1152 flatviews_reset(); 1153 1154 MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); 1155 1156 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1157 address_space_set_flatview(as); 1158 address_space_update_ioeventfds(as); 1159 } 1160 memory_region_update_pending = false; 1161 ioeventfd_update_pending = false; 1162 MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); 1163 } else if (ioeventfd_update_pending) { 1164 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1165 address_space_update_ioeventfds(as); 1166 } 1167 ioeventfd_update_pending = false; 1168 } 1169 } 1170 } 1171 1172 static void memory_region_destructor_none(MemoryRegion *mr) 1173 { 1174 } 1175 1176 static void memory_region_destructor_ram(MemoryRegion *mr) 1177 { 1178 qemu_ram_free(mr->ram_block); 1179 } 1180 1181 static bool memory_region_need_escape(char c) 1182 { 1183 return c == '/' || c == '[' || c == '\\' || c == ']'; 1184 } 1185 1186 static char *memory_region_escape_name(const char *name) 1187 { 1188 const char *p; 1189 char *escaped, *q; 1190 uint8_t c; 1191 size_t bytes = 0; 1192 1193 for (p = name; *p; p++) { 1194 bytes += memory_region_need_escape(*p) ? 4 : 1; 1195 } 1196 if (bytes == p - name) { 1197 return g_memdup(name, bytes + 1); 1198 } 1199 1200 escaped = g_malloc(bytes + 1); 1201 for (p = name, q = escaped; *p; p++) { 1202 c = *p; 1203 if (unlikely(memory_region_need_escape(c))) { 1204 *q++ = '\\'; 1205 *q++ = 'x'; 1206 *q++ = "0123456789abcdef"[c >> 4]; 1207 c = "0123456789abcdef"[c & 15]; 1208 } 1209 *q++ = c; 1210 } 1211 *q = 0; 1212 return escaped; 1213 } 1214 1215 static void memory_region_do_init(MemoryRegion *mr, 1216 Object *owner, 1217 const char *name, 1218 uint64_t size) 1219 { 1220 mr->size = int128_make64(size); 1221 if (size == UINT64_MAX) { 1222 mr->size = int128_2_64(); 1223 } 1224 mr->name = g_strdup(name); 1225 mr->owner = owner; 1226 mr->dev = (DeviceState *) object_dynamic_cast(mr->owner, TYPE_DEVICE); 1227 mr->ram_block = NULL; 1228 1229 if (name) { 1230 char *escaped_name = memory_region_escape_name(name); 1231 char *name_array = g_strdup_printf("%s[*]", escaped_name); 1232 1233 if (!owner) { 1234 owner = machine_get_container("unattached"); 1235 } 1236 1237 object_property_add_child(owner, name_array, OBJECT(mr)); 1238 object_unref(OBJECT(mr)); 1239 g_free(name_array); 1240 g_free(escaped_name); 1241 } 1242 } 1243 1244 void memory_region_init(MemoryRegion *mr, 1245 Object *owner, 1246 const char *name, 1247 uint64_t size) 1248 { 1249 object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION); 1250 memory_region_do_init(mr, owner, name, size); 1251 } 1252 1253 static void memory_region_get_container(Object *obj, Visitor *v, 1254 const char *name, void *opaque, 1255 Error **errp) 1256 { 1257 MemoryRegion *mr = MEMORY_REGION(obj); 1258 char *path = (char *)""; 1259 1260 if (mr->container) { 1261 path = object_get_canonical_path(OBJECT(mr->container)); 1262 } 1263 visit_type_str(v, name, &path, errp); 1264 if (mr->container) { 1265 g_free(path); 1266 } 1267 } 1268 1269 static Object *memory_region_resolve_container(Object *obj, void *opaque, 1270 const char *part) 1271 { 1272 MemoryRegion *mr = MEMORY_REGION(obj); 1273 1274 return OBJECT(mr->container); 1275 } 1276 1277 static void memory_region_get_priority(Object *obj, Visitor *v, 1278 const char *name, void *opaque, 1279 Error **errp) 1280 { 1281 MemoryRegion *mr = MEMORY_REGION(obj); 1282 int32_t value = mr->priority; 1283 1284 visit_type_int32(v, name, &value, errp); 1285 } 1286 1287 static void memory_region_get_size(Object *obj, Visitor *v, const char *name, 1288 void *opaque, Error **errp) 1289 { 1290 MemoryRegion *mr = MEMORY_REGION(obj); 1291 uint64_t value = memory_region_size(mr); 1292 1293 visit_type_uint64(v, name, &value, errp); 1294 } 1295 1296 static void memory_region_initfn(Object *obj) 1297 { 1298 MemoryRegion *mr = MEMORY_REGION(obj); 1299 ObjectProperty *op; 1300 1301 mr->ops = &unassigned_mem_ops; 1302 mr->enabled = true; 1303 mr->romd_mode = true; 1304 mr->destructor = memory_region_destructor_none; 1305 QTAILQ_INIT(&mr->subregions); 1306 QTAILQ_INIT(&mr->coalesced); 1307 1308 op = object_property_add(OBJECT(mr), "container", 1309 "link<" TYPE_MEMORY_REGION ">", 1310 memory_region_get_container, 1311 NULL, /* memory_region_set_container */ 1312 NULL, NULL); 1313 op->resolve = memory_region_resolve_container; 1314 1315 object_property_add_uint64_ptr(OBJECT(mr), "addr", 1316 &mr->addr, OBJ_PROP_FLAG_READ); 1317 object_property_add(OBJECT(mr), "priority", "uint32", 1318 memory_region_get_priority, 1319 NULL, /* memory_region_set_priority */ 1320 NULL, NULL); 1321 object_property_add(OBJECT(mr), "size", "uint64", 1322 memory_region_get_size, 1323 NULL, /* memory_region_set_size, */ 1324 NULL, NULL); 1325 } 1326 1327 static void iommu_memory_region_initfn(Object *obj) 1328 { 1329 MemoryRegion *mr = MEMORY_REGION(obj); 1330 1331 mr->is_iommu = true; 1332 } 1333 1334 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr, 1335 unsigned size) 1336 { 1337 #ifdef DEBUG_UNASSIGNED 1338 printf("Unassigned mem read " HWADDR_FMT_plx "\n", addr); 1339 #endif 1340 return 0; 1341 } 1342 1343 static void unassigned_mem_write(void *opaque, hwaddr addr, 1344 uint64_t val, unsigned size) 1345 { 1346 #ifdef DEBUG_UNASSIGNED 1347 printf("Unassigned mem write " HWADDR_FMT_plx " = 0x%"PRIx64"\n", addr, val); 1348 #endif 1349 } 1350 1351 static bool unassigned_mem_accepts(void *opaque, hwaddr addr, 1352 unsigned size, bool is_write, 1353 MemTxAttrs attrs) 1354 { 1355 return false; 1356 } 1357 1358 const MemoryRegionOps unassigned_mem_ops = { 1359 .valid.accepts = unassigned_mem_accepts, 1360 .endianness = DEVICE_NATIVE_ENDIAN, 1361 }; 1362 1363 static uint64_t memory_region_ram_device_read(void *opaque, 1364 hwaddr addr, unsigned size) 1365 { 1366 MemoryRegion *mr = opaque; 1367 uint64_t data = ldn_he_p(mr->ram_block->host + addr, size); 1368 1369 trace_memory_region_ram_device_read(get_cpu_index(), mr, addr, data, size); 1370 1371 return data; 1372 } 1373 1374 static void memory_region_ram_device_write(void *opaque, hwaddr addr, 1375 uint64_t data, unsigned size) 1376 { 1377 MemoryRegion *mr = opaque; 1378 1379 trace_memory_region_ram_device_write(get_cpu_index(), mr, addr, data, size); 1380 1381 stn_he_p(mr->ram_block->host + addr, size, data); 1382 } 1383 1384 static const MemoryRegionOps ram_device_mem_ops = { 1385 .read = memory_region_ram_device_read, 1386 .write = memory_region_ram_device_write, 1387 .endianness = HOST_BIG_ENDIAN ? DEVICE_BIG_ENDIAN : DEVICE_LITTLE_ENDIAN, 1388 .valid = { 1389 .min_access_size = 1, 1390 .max_access_size = 8, 1391 .unaligned = true, 1392 }, 1393 .impl = { 1394 .min_access_size = 1, 1395 .max_access_size = 8, 1396 .unaligned = true, 1397 }, 1398 }; 1399 1400 bool memory_region_access_valid(MemoryRegion *mr, 1401 hwaddr addr, 1402 unsigned size, 1403 bool is_write, 1404 MemTxAttrs attrs) 1405 { 1406 if (mr->ops->valid.accepts 1407 && !mr->ops->valid.accepts(mr->opaque, addr, size, is_write, attrs)) { 1408 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1409 ", size %u, region '%s', reason: rejected\n", 1410 is_write ? "write" : "read", 1411 addr, size, memory_region_name(mr)); 1412 return false; 1413 } 1414 1415 if (!mr->ops->valid.unaligned && (addr & (size - 1))) { 1416 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1417 ", size %u, region '%s', reason: unaligned\n", 1418 is_write ? "write" : "read", 1419 addr, size, memory_region_name(mr)); 1420 return false; 1421 } 1422 1423 /* Treat zero as compatibility all valid */ 1424 if (!mr->ops->valid.max_access_size) { 1425 return true; 1426 } 1427 1428 if (size > mr->ops->valid.max_access_size 1429 || size < mr->ops->valid.min_access_size) { 1430 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1431 ", size %u, region '%s', reason: invalid size " 1432 "(min:%u max:%u)\n", 1433 is_write ? "write" : "read", 1434 addr, size, memory_region_name(mr), 1435 mr->ops->valid.min_access_size, 1436 mr->ops->valid.max_access_size); 1437 return false; 1438 } 1439 return true; 1440 } 1441 1442 static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr, 1443 hwaddr addr, 1444 uint64_t *pval, 1445 unsigned size, 1446 MemTxAttrs attrs) 1447 { 1448 *pval = 0; 1449 1450 if (mr->ops->read) { 1451 return access_with_adjusted_size(addr, pval, size, 1452 mr->ops->impl.min_access_size, 1453 mr->ops->impl.max_access_size, 1454 memory_region_read_accessor, 1455 mr, attrs); 1456 } else { 1457 return access_with_adjusted_size(addr, pval, size, 1458 mr->ops->impl.min_access_size, 1459 mr->ops->impl.max_access_size, 1460 memory_region_read_with_attrs_accessor, 1461 mr, attrs); 1462 } 1463 } 1464 1465 MemTxResult memory_region_dispatch_read(MemoryRegion *mr, 1466 hwaddr addr, 1467 uint64_t *pval, 1468 MemOp op, 1469 MemTxAttrs attrs) 1470 { 1471 unsigned size = memop_size(op); 1472 MemTxResult r; 1473 1474 if (mr->alias) { 1475 return memory_region_dispatch_read(mr->alias, 1476 mr->alias_offset + addr, 1477 pval, op, attrs); 1478 } 1479 if (!memory_region_access_valid(mr, addr, size, false, attrs)) { 1480 *pval = unassigned_mem_read(mr, addr, size); 1481 return MEMTX_DECODE_ERROR; 1482 } 1483 1484 r = memory_region_dispatch_read1(mr, addr, pval, size, attrs); 1485 adjust_endianness(mr, pval, op); 1486 return r; 1487 } 1488 1489 /* Return true if an eventfd was signalled */ 1490 static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr, 1491 hwaddr addr, 1492 uint64_t data, 1493 unsigned size, 1494 MemTxAttrs attrs) 1495 { 1496 MemoryRegionIoeventfd ioeventfd = { 1497 .addr = addrrange_make(int128_make64(addr), int128_make64(size)), 1498 .data = data, 1499 }; 1500 unsigned i; 1501 1502 for (i = 0; i < mr->ioeventfd_nb; i++) { 1503 ioeventfd.match_data = mr->ioeventfds[i].match_data; 1504 ioeventfd.e = mr->ioeventfds[i].e; 1505 1506 if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) { 1507 event_notifier_set(ioeventfd.e); 1508 return true; 1509 } 1510 } 1511 1512 return false; 1513 } 1514 1515 MemTxResult memory_region_dispatch_write(MemoryRegion *mr, 1516 hwaddr addr, 1517 uint64_t data, 1518 MemOp op, 1519 MemTxAttrs attrs) 1520 { 1521 unsigned size = memop_size(op); 1522 1523 if (mr->alias) { 1524 return memory_region_dispatch_write(mr->alias, 1525 mr->alias_offset + addr, 1526 data, op, attrs); 1527 } 1528 if (!memory_region_access_valid(mr, addr, size, true, attrs)) { 1529 unassigned_mem_write(mr, addr, data, size); 1530 return MEMTX_DECODE_ERROR; 1531 } 1532 1533 adjust_endianness(mr, &data, op); 1534 1535 /* 1536 * FIXME: it's not clear why under KVM the write would be processed 1537 * directly, instead of going through eventfd. This probably should 1538 * test "tcg_enabled() || qtest_enabled()", or should just go away. 1539 */ 1540 if (!kvm_enabled() && 1541 memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) { 1542 return MEMTX_OK; 1543 } 1544 1545 if (mr->ops->write) { 1546 return access_with_adjusted_size(addr, &data, size, 1547 mr->ops->impl.min_access_size, 1548 mr->ops->impl.max_access_size, 1549 memory_region_write_accessor, mr, 1550 attrs); 1551 } else { 1552 return 1553 access_with_adjusted_size(addr, &data, size, 1554 mr->ops->impl.min_access_size, 1555 mr->ops->impl.max_access_size, 1556 memory_region_write_with_attrs_accessor, 1557 mr, attrs); 1558 } 1559 } 1560 1561 void memory_region_init_io(MemoryRegion *mr, 1562 Object *owner, 1563 const MemoryRegionOps *ops, 1564 void *opaque, 1565 const char *name, 1566 uint64_t size) 1567 { 1568 memory_region_init(mr, owner, name, size); 1569 mr->ops = ops ? ops : &unassigned_mem_ops; 1570 mr->opaque = opaque; 1571 mr->terminates = true; 1572 } 1573 1574 bool memory_region_init_ram_nomigrate(MemoryRegion *mr, 1575 Object *owner, 1576 const char *name, 1577 uint64_t size, 1578 Error **errp) 1579 { 1580 return memory_region_init_ram_flags_nomigrate(mr, owner, name, 1581 size, 0, errp); 1582 } 1583 1584 bool memory_region_init_ram_flags_nomigrate(MemoryRegion *mr, 1585 Object *owner, 1586 const char *name, 1587 uint64_t size, 1588 uint32_t ram_flags, 1589 Error **errp) 1590 { 1591 Error *err = NULL; 1592 memory_region_init(mr, owner, name, size); 1593 mr->ram = true; 1594 mr->terminates = true; 1595 mr->destructor = memory_region_destructor_ram; 1596 mr->ram_block = qemu_ram_alloc(size, ram_flags, mr, &err); 1597 if (err) { 1598 mr->size = int128_zero(); 1599 object_unparent(OBJECT(mr)); 1600 error_propagate(errp, err); 1601 return false; 1602 } 1603 return true; 1604 } 1605 1606 bool memory_region_init_resizeable_ram(MemoryRegion *mr, 1607 Object *owner, 1608 const char *name, 1609 uint64_t size, 1610 uint64_t max_size, 1611 void (*resized)(const char*, 1612 uint64_t length, 1613 void *host), 1614 Error **errp) 1615 { 1616 Error *err = NULL; 1617 memory_region_init(mr, owner, name, size); 1618 mr->ram = true; 1619 mr->terminates = true; 1620 mr->destructor = memory_region_destructor_ram; 1621 mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized, 1622 mr, &err); 1623 if (err) { 1624 mr->size = int128_zero(); 1625 object_unparent(OBJECT(mr)); 1626 error_propagate(errp, err); 1627 return false; 1628 } 1629 return true; 1630 } 1631 1632 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) 1633 bool memory_region_init_ram_from_file(MemoryRegion *mr, 1634 Object *owner, 1635 const char *name, 1636 uint64_t size, 1637 uint64_t align, 1638 uint32_t ram_flags, 1639 const char *path, 1640 ram_addr_t offset, 1641 Error **errp) 1642 { 1643 Error *err = NULL; 1644 memory_region_init(mr, owner, name, size); 1645 mr->ram = true; 1646 mr->readonly = !!(ram_flags & RAM_READONLY); 1647 mr->terminates = true; 1648 mr->destructor = memory_region_destructor_ram; 1649 mr->align = align; 1650 mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, 1651 offset, &err); 1652 if (err) { 1653 mr->size = int128_zero(); 1654 object_unparent(OBJECT(mr)); 1655 error_propagate(errp, err); 1656 return false; 1657 } 1658 return true; 1659 } 1660 1661 bool memory_region_init_ram_from_fd(MemoryRegion *mr, 1662 Object *owner, 1663 const char *name, 1664 uint64_t size, 1665 uint32_t ram_flags, 1666 int fd, 1667 ram_addr_t offset, 1668 Error **errp) 1669 { 1670 Error *err = NULL; 1671 memory_region_init(mr, owner, name, size); 1672 mr->ram = true; 1673 mr->readonly = !!(ram_flags & RAM_READONLY); 1674 mr->terminates = true; 1675 mr->destructor = memory_region_destructor_ram; 1676 mr->ram_block = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, 1677 offset, false, &err); 1678 if (err) { 1679 mr->size = int128_zero(); 1680 object_unparent(OBJECT(mr)); 1681 error_propagate(errp, err); 1682 return false; 1683 } 1684 return true; 1685 } 1686 #endif 1687 1688 void memory_region_init_ram_ptr(MemoryRegion *mr, 1689 Object *owner, 1690 const char *name, 1691 uint64_t size, 1692 void *ptr) 1693 { 1694 memory_region_init(mr, owner, name, size); 1695 mr->ram = true; 1696 mr->terminates = true; 1697 mr->destructor = memory_region_destructor_ram; 1698 1699 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ 1700 assert(ptr != NULL); 1701 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); 1702 } 1703 1704 void memory_region_init_ram_device_ptr(MemoryRegion *mr, 1705 Object *owner, 1706 const char *name, 1707 uint64_t size, 1708 void *ptr) 1709 { 1710 memory_region_init(mr, owner, name, size); 1711 mr->ram = true; 1712 mr->terminates = true; 1713 mr->ram_device = true; 1714 mr->ops = &ram_device_mem_ops; 1715 mr->opaque = mr; 1716 mr->destructor = memory_region_destructor_ram; 1717 1718 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ 1719 assert(ptr != NULL); 1720 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); 1721 } 1722 1723 void memory_region_init_alias(MemoryRegion *mr, 1724 Object *owner, 1725 const char *name, 1726 MemoryRegion *orig, 1727 hwaddr offset, 1728 uint64_t size) 1729 { 1730 memory_region_init(mr, owner, name, size); 1731 mr->alias = orig; 1732 mr->alias_offset = offset; 1733 } 1734 1735 bool memory_region_init_rom_nomigrate(MemoryRegion *mr, 1736 Object *owner, 1737 const char *name, 1738 uint64_t size, 1739 Error **errp) 1740 { 1741 if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, 1742 size, 0, errp)) { 1743 return false; 1744 } 1745 mr->readonly = true; 1746 1747 return true; 1748 } 1749 1750 bool memory_region_init_rom_device_nomigrate(MemoryRegion *mr, 1751 Object *owner, 1752 const MemoryRegionOps *ops, 1753 void *opaque, 1754 const char *name, 1755 uint64_t size, 1756 Error **errp) 1757 { 1758 Error *err = NULL; 1759 assert(ops); 1760 memory_region_init(mr, owner, name, size); 1761 mr->ops = ops; 1762 mr->opaque = opaque; 1763 mr->terminates = true; 1764 mr->rom_device = true; 1765 mr->destructor = memory_region_destructor_ram; 1766 mr->ram_block = qemu_ram_alloc(size, 0, mr, &err); 1767 if (err) { 1768 mr->size = int128_zero(); 1769 object_unparent(OBJECT(mr)); 1770 error_propagate(errp, err); 1771 return false; 1772 } 1773 return true; 1774 } 1775 1776 void memory_region_init_iommu(void *_iommu_mr, 1777 size_t instance_size, 1778 const char *mrtypename, 1779 Object *owner, 1780 const char *name, 1781 uint64_t size) 1782 { 1783 struct IOMMUMemoryRegion *iommu_mr; 1784 struct MemoryRegion *mr; 1785 1786 object_initialize(_iommu_mr, instance_size, mrtypename); 1787 mr = MEMORY_REGION(_iommu_mr); 1788 memory_region_do_init(mr, owner, name, size); 1789 iommu_mr = IOMMU_MEMORY_REGION(mr); 1790 mr->terminates = true; /* then re-forwards */ 1791 QLIST_INIT(&iommu_mr->iommu_notify); 1792 iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE; 1793 } 1794 1795 static void memory_region_finalize(Object *obj) 1796 { 1797 MemoryRegion *mr = MEMORY_REGION(obj); 1798 1799 assert(!mr->container); 1800 1801 /* We know the region is not visible in any address space (it 1802 * does not have a container and cannot be a root either because 1803 * it has no references, so we can blindly clear mr->enabled. 1804 * memory_region_set_enabled instead could trigger a transaction 1805 * and cause an infinite loop. 1806 */ 1807 mr->enabled = false; 1808 memory_region_transaction_begin(); 1809 while (!QTAILQ_EMPTY(&mr->subregions)) { 1810 MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions); 1811 memory_region_del_subregion(mr, subregion); 1812 } 1813 memory_region_transaction_commit(); 1814 1815 mr->destructor(mr); 1816 memory_region_clear_coalescing(mr); 1817 g_free((char *)mr->name); 1818 g_free(mr->ioeventfds); 1819 } 1820 1821 Object *memory_region_owner(MemoryRegion *mr) 1822 { 1823 Object *obj = OBJECT(mr); 1824 return obj->parent; 1825 } 1826 1827 void memory_region_ref(MemoryRegion *mr) 1828 { 1829 /* MMIO callbacks most likely will access data that belongs 1830 * to the owner, hence the need to ref/unref the owner whenever 1831 * the memory region is in use. 1832 * 1833 * The memory region is a child of its owner. As long as the 1834 * owner doesn't call unparent itself on the memory region, 1835 * ref-ing the owner will also keep the memory region alive. 1836 * Memory regions without an owner are supposed to never go away; 1837 * we do not ref/unref them because it slows down DMA sensibly. 1838 */ 1839 if (mr && mr->owner) { 1840 object_ref(mr->owner); 1841 } 1842 } 1843 1844 void memory_region_unref(MemoryRegion *mr) 1845 { 1846 if (mr && mr->owner) { 1847 object_unref(mr->owner); 1848 } 1849 } 1850 1851 uint64_t memory_region_size(MemoryRegion *mr) 1852 { 1853 if (int128_eq(mr->size, int128_2_64())) { 1854 return UINT64_MAX; 1855 } 1856 return int128_get64(mr->size); 1857 } 1858 1859 const char *memory_region_name(const MemoryRegion *mr) 1860 { 1861 if (!mr->name) { 1862 ((MemoryRegion *)mr)->name = 1863 g_strdup(object_get_canonical_path_component(OBJECT(mr))); 1864 } 1865 return mr->name; 1866 } 1867 1868 bool memory_region_is_ram_device(MemoryRegion *mr) 1869 { 1870 return mr->ram_device; 1871 } 1872 1873 bool memory_region_is_protected(MemoryRegion *mr) 1874 { 1875 return mr->ram && (mr->ram_block->flags & RAM_PROTECTED); 1876 } 1877 1878 bool memory_region_has_guest_memfd(MemoryRegion *mr) 1879 { 1880 return mr->ram_block && mr->ram_block->guest_memfd >= 0; 1881 } 1882 1883 uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr) 1884 { 1885 uint8_t mask = mr->dirty_log_mask; 1886 RAMBlock *rb = mr->ram_block; 1887 1888 if (global_dirty_tracking && ((rb && qemu_ram_is_migratable(rb)) || 1889 memory_region_is_iommu(mr))) { 1890 mask |= (1 << DIRTY_MEMORY_MIGRATION); 1891 } 1892 1893 if (tcg_enabled() && rb) { 1894 /* TCG only cares about dirty memory logging for RAM, not IOMMU. */ 1895 mask |= (1 << DIRTY_MEMORY_CODE); 1896 } 1897 return mask; 1898 } 1899 1900 bool memory_region_is_logging(MemoryRegion *mr, uint8_t client) 1901 { 1902 return memory_region_get_dirty_log_mask(mr) & (1 << client); 1903 } 1904 1905 static int memory_region_update_iommu_notify_flags(IOMMUMemoryRegion *iommu_mr, 1906 Error **errp) 1907 { 1908 IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE; 1909 IOMMUNotifier *iommu_notifier; 1910 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 1911 int ret = 0; 1912 1913 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) { 1914 flags |= iommu_notifier->notifier_flags; 1915 } 1916 1917 if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) { 1918 ret = imrc->notify_flag_changed(iommu_mr, 1919 iommu_mr->iommu_notify_flags, 1920 flags, errp); 1921 } 1922 1923 if (!ret) { 1924 iommu_mr->iommu_notify_flags = flags; 1925 } 1926 return ret; 1927 } 1928 1929 int memory_region_register_iommu_notifier(MemoryRegion *mr, 1930 IOMMUNotifier *n, Error **errp) 1931 { 1932 IOMMUMemoryRegion *iommu_mr; 1933 int ret; 1934 1935 if (mr->alias) { 1936 return memory_region_register_iommu_notifier(mr->alias, n, errp); 1937 } 1938 1939 /* We need to register for at least one bitfield */ 1940 iommu_mr = IOMMU_MEMORY_REGION(mr); 1941 assert(n->notifier_flags != IOMMU_NOTIFIER_NONE); 1942 assert(n->start <= n->end); 1943 assert(n->iommu_idx >= 0 && 1944 n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr)); 1945 1946 QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node); 1947 ret = memory_region_update_iommu_notify_flags(iommu_mr, errp); 1948 if (ret) { 1949 QLIST_REMOVE(n, node); 1950 } 1951 return ret; 1952 } 1953 1954 uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr) 1955 { 1956 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 1957 1958 if (imrc->get_min_page_size) { 1959 return imrc->get_min_page_size(iommu_mr); 1960 } 1961 return TARGET_PAGE_SIZE; 1962 } 1963 1964 void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 1965 { 1966 MemoryRegion *mr = MEMORY_REGION(iommu_mr); 1967 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 1968 hwaddr addr, granularity; 1969 IOMMUTLBEntry iotlb; 1970 1971 /* If the IOMMU has its own replay callback, override */ 1972 if (imrc->replay) { 1973 imrc->replay(iommu_mr, n); 1974 return; 1975 } 1976 1977 granularity = memory_region_iommu_get_min_page_size(iommu_mr); 1978 1979 for (addr = 0; addr < memory_region_size(mr); addr += granularity) { 1980 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx); 1981 if (iotlb.perm != IOMMU_NONE) { 1982 n->notify(n, &iotlb); 1983 } 1984 1985 /* if (2^64 - MR size) < granularity, it's possible to get an 1986 * infinite loop here. This should catch such a wraparound */ 1987 if ((addr + granularity) < addr) { 1988 break; 1989 } 1990 } 1991 } 1992 1993 void memory_region_unregister_iommu_notifier(MemoryRegion *mr, 1994 IOMMUNotifier *n) 1995 { 1996 IOMMUMemoryRegion *iommu_mr; 1997 1998 if (mr->alias) { 1999 memory_region_unregister_iommu_notifier(mr->alias, n); 2000 return; 2001 } 2002 QLIST_REMOVE(n, node); 2003 iommu_mr = IOMMU_MEMORY_REGION(mr); 2004 memory_region_update_iommu_notify_flags(iommu_mr, NULL); 2005 } 2006 2007 void memory_region_notify_iommu_one(IOMMUNotifier *notifier, 2008 const IOMMUTLBEvent *event) 2009 { 2010 const IOMMUTLBEntry *entry = &event->entry; 2011 hwaddr entry_end = entry->iova + entry->addr_mask; 2012 IOMMUTLBEntry tmp = *entry; 2013 2014 if (event->type == IOMMU_NOTIFIER_UNMAP) { 2015 assert(entry->perm == IOMMU_NONE); 2016 } 2017 2018 /* 2019 * Skip the notification if the notification does not overlap 2020 * with registered range. 2021 */ 2022 if (notifier->start > entry_end || notifier->end < entry->iova) { 2023 return; 2024 } 2025 2026 if (notifier->notifier_flags & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 2027 /* Crop (iova, addr_mask) to range */ 2028 tmp.iova = MAX(tmp.iova, notifier->start); 2029 tmp.addr_mask = MIN(entry_end, notifier->end) - tmp.iova; 2030 } else { 2031 assert(entry->iova >= notifier->start && entry_end <= notifier->end); 2032 } 2033 2034 if (event->type & notifier->notifier_flags) { 2035 notifier->notify(notifier, &tmp); 2036 } 2037 } 2038 2039 void memory_region_unmap_iommu_notifier_range(IOMMUNotifier *notifier) 2040 { 2041 IOMMUTLBEvent event; 2042 2043 event.type = IOMMU_NOTIFIER_UNMAP; 2044 event.entry.target_as = &address_space_memory; 2045 event.entry.iova = notifier->start; 2046 event.entry.perm = IOMMU_NONE; 2047 event.entry.addr_mask = notifier->end - notifier->start; 2048 2049 memory_region_notify_iommu_one(notifier, &event); 2050 } 2051 2052 void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr, 2053 int iommu_idx, 2054 const IOMMUTLBEvent event) 2055 { 2056 IOMMUNotifier *iommu_notifier; 2057 2058 assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr))); 2059 2060 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) { 2061 if (iommu_notifier->iommu_idx == iommu_idx) { 2062 memory_region_notify_iommu_one(iommu_notifier, &event); 2063 } 2064 } 2065 } 2066 2067 int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr, 2068 enum IOMMUMemoryRegionAttr attr, 2069 void *data) 2070 { 2071 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2072 2073 if (!imrc->get_attr) { 2074 return -EINVAL; 2075 } 2076 2077 return imrc->get_attr(iommu_mr, attr, data); 2078 } 2079 2080 int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr, 2081 MemTxAttrs attrs) 2082 { 2083 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2084 2085 if (!imrc->attrs_to_index) { 2086 return 0; 2087 } 2088 2089 return imrc->attrs_to_index(iommu_mr, attrs); 2090 } 2091 2092 int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr) 2093 { 2094 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2095 2096 if (!imrc->num_indexes) { 2097 return 1; 2098 } 2099 2100 return imrc->num_indexes(iommu_mr); 2101 } 2102 2103 RamDiscardManager *memory_region_get_ram_discard_manager(MemoryRegion *mr) 2104 { 2105 if (!memory_region_is_ram(mr)) { 2106 return NULL; 2107 } 2108 return mr->rdm; 2109 } 2110 2111 int memory_region_set_ram_discard_manager(MemoryRegion *mr, 2112 RamDiscardManager *rdm) 2113 { 2114 g_assert(memory_region_is_ram(mr)); 2115 if (mr->rdm && rdm) { 2116 return -EBUSY; 2117 } 2118 2119 mr->rdm = rdm; 2120 return 0; 2121 } 2122 2123 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm, 2124 const MemoryRegion *mr) 2125 { 2126 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2127 2128 g_assert(rdmc->get_min_granularity); 2129 return rdmc->get_min_granularity(rdm, mr); 2130 } 2131 2132 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm, 2133 const MemoryRegionSection *section) 2134 { 2135 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2136 2137 g_assert(rdmc->is_populated); 2138 return rdmc->is_populated(rdm, section); 2139 } 2140 2141 int ram_discard_manager_replay_populated(const RamDiscardManager *rdm, 2142 MemoryRegionSection *section, 2143 ReplayRamDiscardState replay_fn, 2144 void *opaque) 2145 { 2146 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2147 2148 g_assert(rdmc->replay_populated); 2149 return rdmc->replay_populated(rdm, section, replay_fn, opaque); 2150 } 2151 2152 int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm, 2153 MemoryRegionSection *section, 2154 ReplayRamDiscardState replay_fn, 2155 void *opaque) 2156 { 2157 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2158 2159 g_assert(rdmc->replay_discarded); 2160 return rdmc->replay_discarded(rdm, section, replay_fn, opaque); 2161 } 2162 2163 void ram_discard_manager_register_listener(RamDiscardManager *rdm, 2164 RamDiscardListener *rdl, 2165 MemoryRegionSection *section) 2166 { 2167 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2168 2169 g_assert(rdmc->register_listener); 2170 rdmc->register_listener(rdm, rdl, section); 2171 } 2172 2173 void ram_discard_manager_unregister_listener(RamDiscardManager *rdm, 2174 RamDiscardListener *rdl) 2175 { 2176 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2177 2178 g_assert(rdmc->unregister_listener); 2179 rdmc->unregister_listener(rdm, rdl); 2180 } 2181 2182 /* Called with rcu_read_lock held. */ 2183 MemoryRegion *memory_translate_iotlb(IOMMUTLBEntry *iotlb, hwaddr *xlat_p, 2184 Error **errp) 2185 { 2186 MemoryRegion *mr; 2187 hwaddr xlat; 2188 hwaddr len = iotlb->addr_mask + 1; 2189 bool writable = iotlb->perm & IOMMU_WO; 2190 2191 /* 2192 * The IOMMU TLB entry we have just covers translation through 2193 * this IOMMU to its immediate target. We need to translate 2194 * it the rest of the way through to memory. 2195 */ 2196 mr = address_space_translate(&address_space_memory, iotlb->translated_addr, 2197 &xlat, &len, writable, MEMTXATTRS_UNSPECIFIED); 2198 if (!memory_region_is_ram(mr)) { 2199 error_setg(errp, "iommu map to non memory area %" HWADDR_PRIx "", xlat); 2200 return NULL; 2201 } else if (memory_region_has_ram_discard_manager(mr)) { 2202 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr); 2203 MemoryRegionSection tmp = { 2204 .mr = mr, 2205 .offset_within_region = xlat, 2206 .size = int128_make64(len), 2207 }; 2208 /* 2209 * Malicious VMs can map memory into the IOMMU, which is expected 2210 * to remain discarded. vfio will pin all pages, populating memory. 2211 * Disallow that. vmstate priorities make sure any RamDiscardManager 2212 * were already restored before IOMMUs are restored. 2213 */ 2214 if (!ram_discard_manager_is_populated(rdm, &tmp)) { 2215 error_setg(errp, "iommu map to discarded memory (e.g., unplugged" 2216 " via virtio-mem): %" HWADDR_PRIx "", 2217 iotlb->translated_addr); 2218 return NULL; 2219 } 2220 } 2221 2222 /* 2223 * Translation truncates length to the IOMMU page size, 2224 * check that it did not truncate too much. 2225 */ 2226 if (len & iotlb->addr_mask) { 2227 error_setg(errp, "iommu has granularity incompatible with target AS"); 2228 return NULL; 2229 } 2230 2231 *xlat_p = xlat; 2232 return mr; 2233 } 2234 2235 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) 2236 { 2237 uint8_t mask = 1 << client; 2238 uint8_t old_logging; 2239 2240 assert(client == DIRTY_MEMORY_VGA); 2241 old_logging = mr->vga_logging_count; 2242 mr->vga_logging_count += log ? 1 : -1; 2243 if (!!old_logging == !!mr->vga_logging_count) { 2244 return; 2245 } 2246 2247 memory_region_transaction_begin(); 2248 mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); 2249 memory_region_update_pending |= mr->enabled; 2250 memory_region_transaction_commit(); 2251 } 2252 2253 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, 2254 hwaddr size) 2255 { 2256 assert(mr->ram_block); 2257 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, 2258 size, 2259 memory_region_get_dirty_log_mask(mr)); 2260 } 2261 2262 /* 2263 * If memory region `mr' is NULL, do global sync. Otherwise, sync 2264 * dirty bitmap for the specified memory region. 2265 */ 2266 static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage) 2267 { 2268 MemoryListener *listener; 2269 AddressSpace *as; 2270 FlatView *view; 2271 FlatRange *fr; 2272 2273 /* If the same address space has multiple log_sync listeners, we 2274 * visit that address space's FlatView multiple times. But because 2275 * log_sync listeners are rare, it's still cheaper than walking each 2276 * address space once. 2277 */ 2278 QTAILQ_FOREACH(listener, &memory_listeners, link) { 2279 if (listener->log_sync) { 2280 as = listener->address_space; 2281 view = address_space_get_flatview(as); 2282 FOR_EACH_FLAT_RANGE(fr, view) { 2283 if (fr->dirty_log_mask && (!mr || fr->mr == mr)) { 2284 MemoryRegionSection mrs = section_from_flat_range(fr, view); 2285 listener->log_sync(listener, &mrs); 2286 } 2287 } 2288 flatview_unref(view); 2289 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 0); 2290 } else if (listener->log_sync_global) { 2291 /* 2292 * No matter whether MR is specified, what we can do here 2293 * is to do a global sync, because we are not capable to 2294 * sync in a finer granularity. 2295 */ 2296 listener->log_sync_global(listener, last_stage); 2297 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1); 2298 } 2299 } 2300 } 2301 2302 void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start, 2303 hwaddr len) 2304 { 2305 MemoryRegionSection mrs; 2306 MemoryListener *listener; 2307 AddressSpace *as; 2308 FlatView *view; 2309 FlatRange *fr; 2310 hwaddr sec_start, sec_end, sec_size; 2311 2312 QTAILQ_FOREACH(listener, &memory_listeners, link) { 2313 if (!listener->log_clear) { 2314 continue; 2315 } 2316 as = listener->address_space; 2317 view = address_space_get_flatview(as); 2318 FOR_EACH_FLAT_RANGE(fr, view) { 2319 if (!fr->dirty_log_mask || fr->mr != mr) { 2320 /* 2321 * Clear dirty bitmap operation only applies to those 2322 * regions whose dirty logging is at least enabled 2323 */ 2324 continue; 2325 } 2326 2327 mrs = section_from_flat_range(fr, view); 2328 2329 sec_start = MAX(mrs.offset_within_region, start); 2330 sec_end = mrs.offset_within_region + int128_get64(mrs.size); 2331 sec_end = MIN(sec_end, start + len); 2332 2333 if (sec_start >= sec_end) { 2334 /* 2335 * If this memory region section has no intersection 2336 * with the requested range, skip. 2337 */ 2338 continue; 2339 } 2340 2341 /* Valid case; shrink the section if needed */ 2342 mrs.offset_within_address_space += 2343 sec_start - mrs.offset_within_region; 2344 mrs.offset_within_region = sec_start; 2345 sec_size = sec_end - sec_start; 2346 mrs.size = int128_make64(sec_size); 2347 listener->log_clear(listener, &mrs); 2348 } 2349 flatview_unref(view); 2350 } 2351 } 2352 2353 DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, 2354 hwaddr addr, 2355 hwaddr size, 2356 unsigned client) 2357 { 2358 DirtyBitmapSnapshot *snapshot; 2359 assert(mr->ram_block); 2360 memory_region_sync_dirty_bitmap(mr, false); 2361 snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); 2362 memory_global_after_dirty_log_sync(); 2363 return snapshot; 2364 } 2365 2366 bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap, 2367 hwaddr addr, hwaddr size) 2368 { 2369 assert(mr->ram_block); 2370 return cpu_physical_memory_snapshot_get_dirty(snap, 2371 memory_region_get_ram_addr(mr) + addr, size); 2372 } 2373 2374 void memory_region_set_readonly(MemoryRegion *mr, bool readonly) 2375 { 2376 if (mr->readonly != readonly) { 2377 memory_region_transaction_begin(); 2378 mr->readonly = readonly; 2379 memory_region_update_pending |= mr->enabled; 2380 memory_region_transaction_commit(); 2381 } 2382 } 2383 2384 void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile) 2385 { 2386 if (mr->nonvolatile != nonvolatile) { 2387 memory_region_transaction_begin(); 2388 mr->nonvolatile = nonvolatile; 2389 memory_region_update_pending |= mr->enabled; 2390 memory_region_transaction_commit(); 2391 } 2392 } 2393 2394 void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode) 2395 { 2396 if (mr->romd_mode != romd_mode) { 2397 memory_region_transaction_begin(); 2398 mr->romd_mode = romd_mode; 2399 memory_region_update_pending |= mr->enabled; 2400 memory_region_transaction_commit(); 2401 } 2402 } 2403 2404 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, 2405 hwaddr size, unsigned client) 2406 { 2407 assert(mr->ram_block); 2408 cpu_physical_memory_test_and_clear_dirty( 2409 memory_region_get_ram_addr(mr) + addr, size, client); 2410 } 2411 2412 int memory_region_get_fd(MemoryRegion *mr) 2413 { 2414 RCU_READ_LOCK_GUARD(); 2415 while (mr->alias) { 2416 mr = mr->alias; 2417 } 2418 return mr->ram_block->fd; 2419 } 2420 2421 void *memory_region_get_ram_ptr(MemoryRegion *mr) 2422 { 2423 uint64_t offset = 0; 2424 2425 RCU_READ_LOCK_GUARD(); 2426 while (mr->alias) { 2427 offset += mr->alias_offset; 2428 mr = mr->alias; 2429 } 2430 assert(mr->ram_block); 2431 return qemu_map_ram_ptr(mr->ram_block, offset); 2432 } 2433 2434 MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset) 2435 { 2436 RAMBlock *block; 2437 2438 block = qemu_ram_block_from_host(ptr, false, offset); 2439 if (!block) { 2440 return NULL; 2441 } 2442 2443 return block->mr; 2444 } 2445 2446 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr) 2447 { 2448 return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID; 2449 } 2450 2451 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp) 2452 { 2453 assert(mr->ram_block); 2454 2455 qemu_ram_resize(mr->ram_block, newsize, errp); 2456 } 2457 2458 void memory_region_msync(MemoryRegion *mr, hwaddr addr, hwaddr size) 2459 { 2460 if (mr->ram_block) { 2461 qemu_ram_msync(mr->ram_block, addr, size); 2462 } 2463 } 2464 2465 void memory_region_writeback(MemoryRegion *mr, hwaddr addr, hwaddr size) 2466 { 2467 /* 2468 * Might be extended case needed to cover 2469 * different types of memory regions 2470 */ 2471 if (mr->dirty_log_mask) { 2472 memory_region_msync(mr, addr, size); 2473 } 2474 } 2475 2476 /* 2477 * Call proper memory listeners about the change on the newly 2478 * added/removed CoalescedMemoryRange. 2479 */ 2480 static void memory_region_update_coalesced_range(MemoryRegion *mr, 2481 CoalescedMemoryRange *cmr, 2482 bool add) 2483 { 2484 AddressSpace *as; 2485 FlatView *view; 2486 FlatRange *fr; 2487 2488 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 2489 view = address_space_get_flatview(as); 2490 FOR_EACH_FLAT_RANGE(fr, view) { 2491 if (fr->mr == mr) { 2492 flat_range_coalesced_io_notify(fr, as, cmr, add); 2493 } 2494 } 2495 flatview_unref(view); 2496 } 2497 } 2498 2499 void memory_region_set_coalescing(MemoryRegion *mr) 2500 { 2501 memory_region_clear_coalescing(mr); 2502 memory_region_add_coalescing(mr, 0, int128_get64(mr->size)); 2503 } 2504 2505 void memory_region_add_coalescing(MemoryRegion *mr, 2506 hwaddr offset, 2507 uint64_t size) 2508 { 2509 CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr)); 2510 2511 cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size)); 2512 QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link); 2513 memory_region_update_coalesced_range(mr, cmr, true); 2514 memory_region_set_flush_coalesced(mr); 2515 } 2516 2517 void memory_region_clear_coalescing(MemoryRegion *mr) 2518 { 2519 CoalescedMemoryRange *cmr; 2520 2521 if (QTAILQ_EMPTY(&mr->coalesced)) { 2522 return; 2523 } 2524 2525 qemu_flush_coalesced_mmio_buffer(); 2526 mr->flush_coalesced_mmio = false; 2527 2528 while (!QTAILQ_EMPTY(&mr->coalesced)) { 2529 cmr = QTAILQ_FIRST(&mr->coalesced); 2530 QTAILQ_REMOVE(&mr->coalesced, cmr, link); 2531 memory_region_update_coalesced_range(mr, cmr, false); 2532 g_free(cmr); 2533 } 2534 } 2535 2536 void memory_region_set_flush_coalesced(MemoryRegion *mr) 2537 { 2538 mr->flush_coalesced_mmio = true; 2539 } 2540 2541 void memory_region_clear_flush_coalesced(MemoryRegion *mr) 2542 { 2543 qemu_flush_coalesced_mmio_buffer(); 2544 if (QTAILQ_EMPTY(&mr->coalesced)) { 2545 mr->flush_coalesced_mmio = false; 2546 } 2547 } 2548 2549 void memory_region_add_eventfd(MemoryRegion *mr, 2550 hwaddr addr, 2551 unsigned size, 2552 bool match_data, 2553 uint64_t data, 2554 EventNotifier *e) 2555 { 2556 MemoryRegionIoeventfd mrfd = { 2557 .addr.start = int128_make64(addr), 2558 .addr.size = int128_make64(size), 2559 .match_data = match_data, 2560 .data = data, 2561 .e = e, 2562 }; 2563 unsigned i; 2564 2565 if (size) { 2566 MemOp mop = (target_big_endian() ? MO_BE : MO_LE) | size_memop(size); 2567 adjust_endianness(mr, &mrfd.data, mop); 2568 } 2569 memory_region_transaction_begin(); 2570 for (i = 0; i < mr->ioeventfd_nb; ++i) { 2571 if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) { 2572 break; 2573 } 2574 } 2575 ++mr->ioeventfd_nb; 2576 mr->ioeventfds = g_realloc(mr->ioeventfds, 2577 sizeof(*mr->ioeventfds) * mr->ioeventfd_nb); 2578 memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], 2579 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); 2580 mr->ioeventfds[i] = mrfd; 2581 ioeventfd_update_pending |= mr->enabled; 2582 memory_region_transaction_commit(); 2583 } 2584 2585 void memory_region_del_eventfd(MemoryRegion *mr, 2586 hwaddr addr, 2587 unsigned size, 2588 bool match_data, 2589 uint64_t data, 2590 EventNotifier *e) 2591 { 2592 MemoryRegionIoeventfd mrfd = { 2593 .addr.start = int128_make64(addr), 2594 .addr.size = int128_make64(size), 2595 .match_data = match_data, 2596 .data = data, 2597 .e = e, 2598 }; 2599 unsigned i; 2600 2601 if (size) { 2602 MemOp mop = (target_big_endian() ? MO_BE : MO_LE) | size_memop(size); 2603 adjust_endianness(mr, &mrfd.data, mop); 2604 } 2605 memory_region_transaction_begin(); 2606 for (i = 0; i < mr->ioeventfd_nb; ++i) { 2607 if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) { 2608 break; 2609 } 2610 } 2611 assert(i != mr->ioeventfd_nb); 2612 memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1], 2613 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1))); 2614 --mr->ioeventfd_nb; 2615 mr->ioeventfds = g_realloc(mr->ioeventfds, 2616 sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); 2617 ioeventfd_update_pending |= mr->enabled; 2618 memory_region_transaction_commit(); 2619 } 2620 2621 static void memory_region_update_container_subregions(MemoryRegion *subregion) 2622 { 2623 MemoryRegion *mr = subregion->container; 2624 MemoryRegion *other; 2625 2626 memory_region_transaction_begin(); 2627 2628 memory_region_ref(subregion); 2629 QTAILQ_FOREACH(other, &mr->subregions, subregions_link) { 2630 if (subregion->priority >= other->priority) { 2631 QTAILQ_INSERT_BEFORE(other, subregion, subregions_link); 2632 goto done; 2633 } 2634 } 2635 QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); 2636 done: 2637 memory_region_update_pending |= mr->enabled && subregion->enabled; 2638 memory_region_transaction_commit(); 2639 } 2640 2641 static void memory_region_add_subregion_common(MemoryRegion *mr, 2642 hwaddr offset, 2643 MemoryRegion *subregion) 2644 { 2645 MemoryRegion *alias; 2646 2647 assert(!subregion->container); 2648 subregion->container = mr; 2649 for (alias = subregion->alias; alias; alias = alias->alias) { 2650 alias->mapped_via_alias++; 2651 } 2652 subregion->addr = offset; 2653 memory_region_update_container_subregions(subregion); 2654 } 2655 2656 void memory_region_add_subregion(MemoryRegion *mr, 2657 hwaddr offset, 2658 MemoryRegion *subregion) 2659 { 2660 subregion->priority = 0; 2661 memory_region_add_subregion_common(mr, offset, subregion); 2662 } 2663 2664 void memory_region_add_subregion_overlap(MemoryRegion *mr, 2665 hwaddr offset, 2666 MemoryRegion *subregion, 2667 int priority) 2668 { 2669 subregion->priority = priority; 2670 memory_region_add_subregion_common(mr, offset, subregion); 2671 } 2672 2673 void memory_region_del_subregion(MemoryRegion *mr, 2674 MemoryRegion *subregion) 2675 { 2676 MemoryRegion *alias; 2677 2678 memory_region_transaction_begin(); 2679 assert(subregion->container == mr); 2680 subregion->container = NULL; 2681 for (alias = subregion->alias; alias; alias = alias->alias) { 2682 alias->mapped_via_alias--; 2683 assert(alias->mapped_via_alias >= 0); 2684 } 2685 QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); 2686 memory_region_unref(subregion); 2687 memory_region_update_pending |= mr->enabled && subregion->enabled; 2688 memory_region_transaction_commit(); 2689 } 2690 2691 void memory_region_set_enabled(MemoryRegion *mr, bool enabled) 2692 { 2693 if (enabled == mr->enabled) { 2694 return; 2695 } 2696 memory_region_transaction_begin(); 2697 mr->enabled = enabled; 2698 memory_region_update_pending = true; 2699 memory_region_transaction_commit(); 2700 } 2701 2702 void memory_region_set_size(MemoryRegion *mr, uint64_t size) 2703 { 2704 Int128 s = int128_make64(size); 2705 2706 if (size == UINT64_MAX) { 2707 s = int128_2_64(); 2708 } 2709 if (int128_eq(s, mr->size)) { 2710 return; 2711 } 2712 memory_region_transaction_begin(); 2713 mr->size = s; 2714 memory_region_update_pending = true; 2715 memory_region_transaction_commit(); 2716 } 2717 2718 static void memory_region_readd_subregion(MemoryRegion *mr) 2719 { 2720 MemoryRegion *container = mr->container; 2721 2722 if (container) { 2723 memory_region_transaction_begin(); 2724 memory_region_ref(mr); 2725 memory_region_del_subregion(container, mr); 2726 memory_region_add_subregion_common(container, mr->addr, mr); 2727 memory_region_unref(mr); 2728 memory_region_transaction_commit(); 2729 } 2730 } 2731 2732 void memory_region_set_address(MemoryRegion *mr, hwaddr addr) 2733 { 2734 if (addr != mr->addr) { 2735 mr->addr = addr; 2736 memory_region_readd_subregion(mr); 2737 } 2738 } 2739 2740 void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset) 2741 { 2742 assert(mr->alias); 2743 2744 if (offset == mr->alias_offset) { 2745 return; 2746 } 2747 2748 memory_region_transaction_begin(); 2749 mr->alias_offset = offset; 2750 memory_region_update_pending |= mr->enabled; 2751 memory_region_transaction_commit(); 2752 } 2753 2754 void memory_region_set_unmergeable(MemoryRegion *mr, bool unmergeable) 2755 { 2756 if (unmergeable == mr->unmergeable) { 2757 return; 2758 } 2759 2760 memory_region_transaction_begin(); 2761 mr->unmergeable = unmergeable; 2762 memory_region_update_pending |= mr->enabled; 2763 memory_region_transaction_commit(); 2764 } 2765 2766 uint64_t memory_region_get_alignment(const MemoryRegion *mr) 2767 { 2768 return mr->align; 2769 } 2770 2771 static int cmp_flatrange_addr(const void *addr_, const void *fr_) 2772 { 2773 const AddrRange *addr = addr_; 2774 const FlatRange *fr = fr_; 2775 2776 if (int128_le(addrrange_end(*addr), fr->addr.start)) { 2777 return -1; 2778 } else if (int128_ge(addr->start, addrrange_end(fr->addr))) { 2779 return 1; 2780 } 2781 return 0; 2782 } 2783 2784 static FlatRange *flatview_lookup(FlatView *view, AddrRange addr) 2785 { 2786 return bsearch(&addr, view->ranges, view->nr, 2787 sizeof(FlatRange), cmp_flatrange_addr); 2788 } 2789 2790 bool memory_region_is_mapped(MemoryRegion *mr) 2791 { 2792 return !!mr->container || mr->mapped_via_alias; 2793 } 2794 2795 /* Same as memory_region_find, but it does not add a reference to the 2796 * returned region. It must be called from an RCU critical section. 2797 */ 2798 static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, 2799 hwaddr addr, uint64_t size) 2800 { 2801 MemoryRegionSection ret = { .mr = NULL }; 2802 MemoryRegion *root; 2803 AddressSpace *as; 2804 AddrRange range; 2805 FlatView *view; 2806 FlatRange *fr; 2807 2808 addr += mr->addr; 2809 for (root = mr; root->container; ) { 2810 root = root->container; 2811 addr += root->addr; 2812 } 2813 2814 as = memory_region_to_address_space(root); 2815 if (!as) { 2816 return ret; 2817 } 2818 range = addrrange_make(int128_make64(addr), int128_make64(size)); 2819 2820 view = address_space_to_flatview(as); 2821 fr = flatview_lookup(view, range); 2822 if (!fr) { 2823 return ret; 2824 } 2825 2826 while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) { 2827 --fr; 2828 } 2829 2830 ret.mr = fr->mr; 2831 ret.fv = view; 2832 range = addrrange_intersection(range, fr->addr); 2833 ret.offset_within_region = fr->offset_in_region; 2834 ret.offset_within_region += int128_get64(int128_sub(range.start, 2835 fr->addr.start)); 2836 ret.size = range.size; 2837 ret.offset_within_address_space = int128_get64(range.start); 2838 ret.readonly = fr->readonly; 2839 ret.nonvolatile = fr->nonvolatile; 2840 return ret; 2841 } 2842 2843 MemoryRegionSection memory_region_find(MemoryRegion *mr, 2844 hwaddr addr, uint64_t size) 2845 { 2846 MemoryRegionSection ret; 2847 RCU_READ_LOCK_GUARD(); 2848 ret = memory_region_find_rcu(mr, addr, size); 2849 if (ret.mr) { 2850 memory_region_ref(ret.mr); 2851 } 2852 return ret; 2853 } 2854 2855 MemoryRegionSection *memory_region_section_new_copy(MemoryRegionSection *s) 2856 { 2857 MemoryRegionSection *tmp = g_new(MemoryRegionSection, 1); 2858 2859 *tmp = *s; 2860 if (tmp->mr) { 2861 memory_region_ref(tmp->mr); 2862 } 2863 if (tmp->fv) { 2864 bool ret = flatview_ref(tmp->fv); 2865 2866 g_assert(ret); 2867 } 2868 return tmp; 2869 } 2870 2871 void memory_region_section_free_copy(MemoryRegionSection *s) 2872 { 2873 if (s->fv) { 2874 flatview_unref(s->fv); 2875 } 2876 if (s->mr) { 2877 memory_region_unref(s->mr); 2878 } 2879 g_free(s); 2880 } 2881 2882 bool memory_region_present(MemoryRegion *container, hwaddr addr) 2883 { 2884 MemoryRegion *mr; 2885 2886 RCU_READ_LOCK_GUARD(); 2887 mr = memory_region_find_rcu(container, addr, 1).mr; 2888 return mr && mr != container; 2889 } 2890 2891 void memory_global_dirty_log_sync(bool last_stage) 2892 { 2893 memory_region_sync_dirty_bitmap(NULL, last_stage); 2894 } 2895 2896 void memory_global_after_dirty_log_sync(void) 2897 { 2898 MEMORY_LISTENER_CALL_GLOBAL(log_global_after_sync, Forward); 2899 } 2900 2901 /* 2902 * Dirty track stop flags that are postponed due to VM being stopped. Should 2903 * only be used within vmstate_change hook. 2904 */ 2905 static unsigned int postponed_stop_flags; 2906 static VMChangeStateEntry *vmstate_change; 2907 static void memory_global_dirty_log_stop_postponed_run(void); 2908 2909 static bool memory_global_dirty_log_do_start(Error **errp) 2910 { 2911 MemoryListener *listener; 2912 2913 QTAILQ_FOREACH(listener, &memory_listeners, link) { 2914 if (listener->log_global_start) { 2915 if (!listener->log_global_start(listener, errp)) { 2916 goto err; 2917 } 2918 } 2919 } 2920 return true; 2921 2922 err: 2923 while ((listener = QTAILQ_PREV(listener, link)) != NULL) { 2924 if (listener->log_global_stop) { 2925 listener->log_global_stop(listener); 2926 } 2927 } 2928 2929 return false; 2930 } 2931 2932 bool memory_global_dirty_log_start(unsigned int flags, Error **errp) 2933 { 2934 unsigned int old_flags; 2935 2936 assert(flags && !(flags & (~GLOBAL_DIRTY_MASK))); 2937 2938 if (vmstate_change) { 2939 /* If there is postponed stop(), operate on it first */ 2940 postponed_stop_flags &= ~flags; 2941 memory_global_dirty_log_stop_postponed_run(); 2942 } 2943 2944 flags &= ~global_dirty_tracking; 2945 if (!flags) { 2946 return true; 2947 } 2948 2949 old_flags = global_dirty_tracking; 2950 global_dirty_tracking |= flags; 2951 trace_global_dirty_changed(global_dirty_tracking); 2952 2953 if (!old_flags) { 2954 if (!memory_global_dirty_log_do_start(errp)) { 2955 global_dirty_tracking &= ~flags; 2956 trace_global_dirty_changed(global_dirty_tracking); 2957 return false; 2958 } 2959 2960 memory_region_transaction_begin(); 2961 memory_region_update_pending = true; 2962 memory_region_transaction_commit(); 2963 } 2964 return true; 2965 } 2966 2967 static void memory_global_dirty_log_do_stop(unsigned int flags) 2968 { 2969 assert(flags && !(flags & (~GLOBAL_DIRTY_MASK))); 2970 assert((global_dirty_tracking & flags) == flags); 2971 global_dirty_tracking &= ~flags; 2972 2973 trace_global_dirty_changed(global_dirty_tracking); 2974 2975 if (!global_dirty_tracking) { 2976 memory_region_transaction_begin(); 2977 memory_region_update_pending = true; 2978 memory_region_transaction_commit(); 2979 MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); 2980 } 2981 } 2982 2983 /* 2984 * Execute the postponed dirty log stop operations if there is, then reset 2985 * everything (including the flags and the vmstate change hook). 2986 */ 2987 static void memory_global_dirty_log_stop_postponed_run(void) 2988 { 2989 /* This must be called with the vmstate handler registered */ 2990 assert(vmstate_change); 2991 2992 /* Note: postponed_stop_flags can be cleared in log start routine */ 2993 if (postponed_stop_flags) { 2994 memory_global_dirty_log_do_stop(postponed_stop_flags); 2995 postponed_stop_flags = 0; 2996 } 2997 2998 qemu_del_vm_change_state_handler(vmstate_change); 2999 vmstate_change = NULL; 3000 } 3001 3002 static void memory_vm_change_state_handler(void *opaque, bool running, 3003 RunState state) 3004 { 3005 if (running) { 3006 memory_global_dirty_log_stop_postponed_run(); 3007 } 3008 } 3009 3010 void memory_global_dirty_log_stop(unsigned int flags) 3011 { 3012 if (!runstate_is_running()) { 3013 /* Postpone the dirty log stop, e.g., to when VM starts again */ 3014 if (vmstate_change) { 3015 /* Batch with previous postponed flags */ 3016 postponed_stop_flags |= flags; 3017 } else { 3018 postponed_stop_flags = flags; 3019 vmstate_change = qemu_add_vm_change_state_handler( 3020 memory_vm_change_state_handler, NULL); 3021 } 3022 return; 3023 } 3024 3025 memory_global_dirty_log_do_stop(flags); 3026 } 3027 3028 static void listener_add_address_space(MemoryListener *listener, 3029 AddressSpace *as) 3030 { 3031 unsigned i; 3032 FlatView *view; 3033 FlatRange *fr; 3034 MemoryRegionIoeventfd *fd; 3035 3036 if (listener->begin) { 3037 listener->begin(listener); 3038 } 3039 if (global_dirty_tracking) { 3040 /* 3041 * Currently only VFIO can fail log_global_start(), and it's not 3042 * yet allowed to hotplug any PCI device during migration. So this 3043 * should never fail when invoked, guard it with error_abort. If 3044 * it can start to fail in the future, we need to be able to fail 3045 * the whole listener_add_address_space() and its callers. 3046 */ 3047 if (listener->log_global_start) { 3048 listener->log_global_start(listener, &error_abort); 3049 } 3050 } 3051 3052 view = address_space_get_flatview(as); 3053 FOR_EACH_FLAT_RANGE(fr, view) { 3054 MemoryRegionSection section = section_from_flat_range(fr, view); 3055 3056 if (listener->region_add) { 3057 listener->region_add(listener, §ion); 3058 } 3059 3060 /* send coalesced io add notifications */ 3061 flat_range_coalesced_io_notify_listener_add_del(fr, §ion, 3062 listener, as, true); 3063 3064 if (fr->dirty_log_mask && listener->log_start) { 3065 listener->log_start(listener, §ion, 0, fr->dirty_log_mask); 3066 } 3067 } 3068 3069 /* 3070 * register all eventfds for this address space for the newly registered 3071 * listener. 3072 */ 3073 for (i = 0; i < as->ioeventfd_nb; i++) { 3074 fd = &as->ioeventfds[i]; 3075 MemoryRegionSection section = (MemoryRegionSection) { 3076 .fv = view, 3077 .offset_within_address_space = int128_get64(fd->addr.start), 3078 .size = fd->addr.size, 3079 }; 3080 3081 if (listener->eventfd_add) { 3082 listener->eventfd_add(listener, §ion, 3083 fd->match_data, fd->data, fd->e); 3084 } 3085 } 3086 3087 if (listener->commit) { 3088 listener->commit(listener); 3089 } 3090 flatview_unref(view); 3091 } 3092 3093 static void listener_del_address_space(MemoryListener *listener, 3094 AddressSpace *as) 3095 { 3096 unsigned i; 3097 FlatView *view; 3098 FlatRange *fr; 3099 MemoryRegionIoeventfd *fd; 3100 3101 if (listener->begin) { 3102 listener->begin(listener); 3103 } 3104 view = address_space_get_flatview(as); 3105 FOR_EACH_FLAT_RANGE(fr, view) { 3106 MemoryRegionSection section = section_from_flat_range(fr, view); 3107 3108 if (fr->dirty_log_mask && listener->log_stop) { 3109 listener->log_stop(listener, §ion, fr->dirty_log_mask, 0); 3110 } 3111 3112 /* send coalesced io del notifications */ 3113 flat_range_coalesced_io_notify_listener_add_del(fr, §ion, 3114 listener, as, false); 3115 if (listener->region_del) { 3116 listener->region_del(listener, §ion); 3117 } 3118 } 3119 3120 /* 3121 * de-register all eventfds for this address space for the current 3122 * listener. 3123 */ 3124 for (i = 0; i < as->ioeventfd_nb; i++) { 3125 fd = &as->ioeventfds[i]; 3126 MemoryRegionSection section = (MemoryRegionSection) { 3127 .fv = view, 3128 .offset_within_address_space = int128_get64(fd->addr.start), 3129 .size = fd->addr.size, 3130 }; 3131 3132 if (listener->eventfd_del) { 3133 listener->eventfd_del(listener, §ion, 3134 fd->match_data, fd->data, fd->e); 3135 } 3136 } 3137 3138 if (listener->commit) { 3139 listener->commit(listener); 3140 } 3141 flatview_unref(view); 3142 } 3143 3144 void memory_listener_register(MemoryListener *listener, AddressSpace *as) 3145 { 3146 MemoryListener *other = NULL; 3147 3148 /* Only one of them can be defined for a listener */ 3149 assert(!(listener->log_sync && listener->log_sync_global)); 3150 3151 listener->address_space = as; 3152 if (QTAILQ_EMPTY(&memory_listeners) 3153 || listener->priority >= QTAILQ_LAST(&memory_listeners)->priority) { 3154 QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); 3155 } else { 3156 QTAILQ_FOREACH(other, &memory_listeners, link) { 3157 if (listener->priority < other->priority) { 3158 break; 3159 } 3160 } 3161 QTAILQ_INSERT_BEFORE(other, listener, link); 3162 } 3163 3164 if (QTAILQ_EMPTY(&as->listeners) 3165 || listener->priority >= QTAILQ_LAST(&as->listeners)->priority) { 3166 QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as); 3167 } else { 3168 QTAILQ_FOREACH(other, &as->listeners, link_as) { 3169 if (listener->priority < other->priority) { 3170 break; 3171 } 3172 } 3173 QTAILQ_INSERT_BEFORE(other, listener, link_as); 3174 } 3175 3176 listener_add_address_space(listener, as); 3177 3178 if (listener->eventfd_add || listener->eventfd_del) { 3179 as->ioeventfd_notifiers++; 3180 } 3181 } 3182 3183 void memory_listener_unregister(MemoryListener *listener) 3184 { 3185 if (!listener->address_space) { 3186 return; 3187 } 3188 3189 if (listener->eventfd_add || listener->eventfd_del) { 3190 listener->address_space->ioeventfd_notifiers--; 3191 } 3192 3193 listener_del_address_space(listener, listener->address_space); 3194 QTAILQ_REMOVE(&memory_listeners, listener, link); 3195 QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as); 3196 listener->address_space = NULL; 3197 } 3198 3199 void address_space_remove_listeners(AddressSpace *as) 3200 { 3201 while (!QTAILQ_EMPTY(&as->listeners)) { 3202 memory_listener_unregister(QTAILQ_FIRST(&as->listeners)); 3203 } 3204 } 3205 3206 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) 3207 { 3208 memory_region_ref(root); 3209 as->root = root; 3210 as->current_map = NULL; 3211 as->ioeventfd_nb = 0; 3212 as->ioeventfds = NULL; 3213 QTAILQ_INIT(&as->listeners); 3214 QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); 3215 as->max_bounce_buffer_size = DEFAULT_MAX_BOUNCE_BUFFER_SIZE; 3216 as->bounce_buffer_size = 0; 3217 qemu_mutex_init(&as->map_client_list_lock); 3218 QLIST_INIT(&as->map_client_list); 3219 as->name = g_strdup(name ? name : "anonymous"); 3220 address_space_update_topology(as); 3221 address_space_update_ioeventfds(as); 3222 } 3223 3224 static void do_address_space_destroy(AddressSpace *as) 3225 { 3226 assert(qatomic_read(&as->bounce_buffer_size) == 0); 3227 assert(QLIST_EMPTY(&as->map_client_list)); 3228 qemu_mutex_destroy(&as->map_client_list_lock); 3229 3230 assert(QTAILQ_EMPTY(&as->listeners)); 3231 3232 flatview_unref(as->current_map); 3233 g_free(as->name); 3234 g_free(as->ioeventfds); 3235 memory_region_unref(as->root); 3236 } 3237 3238 void address_space_destroy(AddressSpace *as) 3239 { 3240 MemoryRegion *root = as->root; 3241 3242 /* Flush out anything from MemoryListeners listening in on this */ 3243 memory_region_transaction_begin(); 3244 as->root = NULL; 3245 memory_region_transaction_commit(); 3246 QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); 3247 3248 /* At this point, as->dispatch and as->current_map are dummy 3249 * entries that the guest should never use. Wait for the old 3250 * values to expire before freeing the data. 3251 */ 3252 as->root = root; 3253 call_rcu(as, do_address_space_destroy, rcu); 3254 } 3255 3256 static const char *memory_region_type(MemoryRegion *mr) 3257 { 3258 if (mr->alias) { 3259 return memory_region_type(mr->alias); 3260 } 3261 if (memory_region_is_ram_device(mr)) { 3262 return "ramd"; 3263 } else if (memory_region_is_romd(mr)) { 3264 return "romd"; 3265 } else if (memory_region_is_rom(mr)) { 3266 return "rom"; 3267 } else if (memory_region_is_ram(mr)) { 3268 return "ram"; 3269 } else { 3270 return "i/o"; 3271 } 3272 } 3273 3274 typedef struct MemoryRegionList MemoryRegionList; 3275 3276 struct MemoryRegionList { 3277 const MemoryRegion *mr; 3278 QTAILQ_ENTRY(MemoryRegionList) mrqueue; 3279 }; 3280 3281 typedef QTAILQ_HEAD(, MemoryRegionList) MemoryRegionListHead; 3282 3283 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \ 3284 int128_sub((size), int128_one())) : 0) 3285 #define MTREE_INDENT " " 3286 3287 static void mtree_expand_owner(const char *label, Object *obj) 3288 { 3289 DeviceState *dev = (DeviceState *) object_dynamic_cast(obj, TYPE_DEVICE); 3290 3291 qemu_printf(" %s:{%s", label, dev ? "dev" : "obj"); 3292 if (dev && dev->id) { 3293 qemu_printf(" id=%s", dev->id); 3294 } else { 3295 char *canonical_path = object_get_canonical_path(obj); 3296 if (canonical_path) { 3297 qemu_printf(" path=%s", canonical_path); 3298 g_free(canonical_path); 3299 } else { 3300 qemu_printf(" type=%s", object_get_typename(obj)); 3301 } 3302 } 3303 qemu_printf("}"); 3304 } 3305 3306 static void mtree_print_mr_owner(const MemoryRegion *mr) 3307 { 3308 Object *owner = mr->owner; 3309 Object *parent = memory_region_owner((MemoryRegion *)mr); 3310 3311 if (!owner && !parent) { 3312 qemu_printf(" orphan"); 3313 return; 3314 } 3315 if (owner) { 3316 mtree_expand_owner("owner", owner); 3317 } 3318 if (parent && parent != owner) { 3319 mtree_expand_owner("parent", parent); 3320 } 3321 } 3322 3323 static void mtree_print_mr(const MemoryRegion *mr, unsigned int level, 3324 hwaddr base, 3325 MemoryRegionListHead *alias_print_queue, 3326 bool owner, bool display_disabled) 3327 { 3328 MemoryRegionList *new_ml, *ml, *next_ml; 3329 MemoryRegionListHead submr_print_queue; 3330 const MemoryRegion *submr; 3331 unsigned int i; 3332 hwaddr cur_start, cur_end; 3333 3334 if (!mr) { 3335 return; 3336 } 3337 3338 cur_start = base + mr->addr; 3339 cur_end = cur_start + MR_SIZE(mr->size); 3340 3341 /* 3342 * Try to detect overflow of memory region. This should never 3343 * happen normally. When it happens, we dump something to warn the 3344 * user who is observing this. 3345 */ 3346 if (cur_start < base || cur_end < cur_start) { 3347 qemu_printf("[DETECTED OVERFLOW!] "); 3348 } 3349 3350 if (mr->alias) { 3351 bool found = false; 3352 3353 /* check if the alias is already in the queue */ 3354 QTAILQ_FOREACH(ml, alias_print_queue, mrqueue) { 3355 if (ml->mr == mr->alias) { 3356 found = true; 3357 } 3358 } 3359 3360 if (!found) { 3361 ml = g_new(MemoryRegionList, 1); 3362 ml->mr = mr->alias; 3363 QTAILQ_INSERT_TAIL(alias_print_queue, ml, mrqueue); 3364 } 3365 if (mr->enabled || display_disabled) { 3366 for (i = 0; i < level; i++) { 3367 qemu_printf(MTREE_INDENT); 3368 } 3369 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx 3370 " (prio %d, %s%s): alias %s @%s " HWADDR_FMT_plx 3371 "-" HWADDR_FMT_plx "%s", 3372 cur_start, cur_end, 3373 mr->priority, 3374 mr->nonvolatile ? "nv-" : "", 3375 memory_region_type((MemoryRegion *)mr), 3376 memory_region_name(mr), 3377 memory_region_name(mr->alias), 3378 mr->alias_offset, 3379 mr->alias_offset + MR_SIZE(mr->size), 3380 mr->enabled ? "" : " [disabled]"); 3381 if (owner) { 3382 mtree_print_mr_owner(mr); 3383 } 3384 qemu_printf("\n"); 3385 } 3386 } else { 3387 if (mr->enabled || display_disabled) { 3388 for (i = 0; i < level; i++) { 3389 qemu_printf(MTREE_INDENT); 3390 } 3391 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx 3392 " (prio %d, %s%s): %s%s", 3393 cur_start, cur_end, 3394 mr->priority, 3395 mr->nonvolatile ? "nv-" : "", 3396 memory_region_type((MemoryRegion *)mr), 3397 memory_region_name(mr), 3398 mr->enabled ? "" : " [disabled]"); 3399 if (owner) { 3400 mtree_print_mr_owner(mr); 3401 } 3402 qemu_printf("\n"); 3403 } 3404 } 3405 3406 QTAILQ_INIT(&submr_print_queue); 3407 3408 QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) { 3409 new_ml = g_new(MemoryRegionList, 1); 3410 new_ml->mr = submr; 3411 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) { 3412 if (new_ml->mr->addr < ml->mr->addr || 3413 (new_ml->mr->addr == ml->mr->addr && 3414 new_ml->mr->priority > ml->mr->priority)) { 3415 QTAILQ_INSERT_BEFORE(ml, new_ml, mrqueue); 3416 new_ml = NULL; 3417 break; 3418 } 3419 } 3420 if (new_ml) { 3421 QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, mrqueue); 3422 } 3423 } 3424 3425 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) { 3426 mtree_print_mr(ml->mr, level + 1, cur_start, 3427 alias_print_queue, owner, display_disabled); 3428 } 3429 3430 QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) { 3431 g_free(ml); 3432 } 3433 } 3434 3435 struct FlatViewInfo { 3436 int counter; 3437 bool dispatch_tree; 3438 bool owner; 3439 AccelClass *ac; 3440 }; 3441 3442 static void mtree_print_flatview(gpointer key, gpointer value, 3443 gpointer user_data) 3444 { 3445 FlatView *view = key; 3446 GArray *fv_address_spaces = value; 3447 struct FlatViewInfo *fvi = user_data; 3448 FlatRange *range = &view->ranges[0]; 3449 MemoryRegion *mr; 3450 int n = view->nr; 3451 int i; 3452 AddressSpace *as; 3453 3454 qemu_printf("FlatView #%d\n", fvi->counter); 3455 ++fvi->counter; 3456 3457 for (i = 0; i < fv_address_spaces->len; ++i) { 3458 as = g_array_index(fv_address_spaces, AddressSpace*, i); 3459 qemu_printf(" AS \"%s\", root: %s", 3460 as->name, memory_region_name(as->root)); 3461 if (as->root->alias) { 3462 qemu_printf(", alias %s", memory_region_name(as->root->alias)); 3463 } 3464 qemu_printf("\n"); 3465 } 3466 3467 qemu_printf(" Root memory region: %s\n", 3468 view->root ? memory_region_name(view->root) : "(none)"); 3469 3470 if (n <= 0) { 3471 qemu_printf(MTREE_INDENT "No rendered FlatView\n\n"); 3472 return; 3473 } 3474 3475 while (n--) { 3476 mr = range->mr; 3477 if (range->offset_in_region) { 3478 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx 3479 " (prio %d, %s%s): %s @" HWADDR_FMT_plx, 3480 int128_get64(range->addr.start), 3481 int128_get64(range->addr.start) 3482 + MR_SIZE(range->addr.size), 3483 mr->priority, 3484 range->nonvolatile ? "nv-" : "", 3485 range->readonly ? "rom" : memory_region_type(mr), 3486 memory_region_name(mr), 3487 range->offset_in_region); 3488 } else { 3489 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx 3490 " (prio %d, %s%s): %s", 3491 int128_get64(range->addr.start), 3492 int128_get64(range->addr.start) 3493 + MR_SIZE(range->addr.size), 3494 mr->priority, 3495 range->nonvolatile ? "nv-" : "", 3496 range->readonly ? "rom" : memory_region_type(mr), 3497 memory_region_name(mr)); 3498 } 3499 if (fvi->owner) { 3500 mtree_print_mr_owner(mr); 3501 } 3502 3503 if (fvi->ac) { 3504 for (i = 0; i < fv_address_spaces->len; ++i) { 3505 as = g_array_index(fv_address_spaces, AddressSpace*, i); 3506 if (fvi->ac->has_memory(current_machine->accelerator, as, 3507 int128_get64(range->addr.start), 3508 MR_SIZE(range->addr.size) + 1)) { 3509 qemu_printf(" %s", fvi->ac->name); 3510 } 3511 } 3512 } 3513 qemu_printf("\n"); 3514 range++; 3515 } 3516 3517 #if !defined(CONFIG_USER_ONLY) 3518 if (fvi->dispatch_tree && view->root) { 3519 mtree_print_dispatch(view->dispatch, view->root); 3520 } 3521 #endif 3522 3523 qemu_printf("\n"); 3524 } 3525 3526 static gboolean mtree_info_flatview_free(gpointer key, gpointer value, 3527 gpointer user_data) 3528 { 3529 FlatView *view = key; 3530 GArray *fv_address_spaces = value; 3531 3532 g_array_unref(fv_address_spaces); 3533 flatview_unref(view); 3534 3535 return true; 3536 } 3537 3538 static void mtree_info_flatview(bool dispatch_tree, bool owner) 3539 { 3540 struct FlatViewInfo fvi = { 3541 .counter = 0, 3542 .dispatch_tree = dispatch_tree, 3543 .owner = owner, 3544 }; 3545 AddressSpace *as; 3546 FlatView *view; 3547 GArray *fv_address_spaces; 3548 GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); 3549 AccelClass *ac = ACCEL_GET_CLASS(current_accel()); 3550 3551 if (ac->has_memory) { 3552 fvi.ac = ac; 3553 } 3554 3555 /* Gather all FVs in one table */ 3556 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 3557 view = address_space_get_flatview(as); 3558 3559 fv_address_spaces = g_hash_table_lookup(views, view); 3560 if (!fv_address_spaces) { 3561 fv_address_spaces = g_array_new(false, false, sizeof(as)); 3562 g_hash_table_insert(views, view, fv_address_spaces); 3563 } 3564 3565 g_array_append_val(fv_address_spaces, as); 3566 } 3567 3568 /* Print */ 3569 g_hash_table_foreach(views, mtree_print_flatview, &fvi); 3570 3571 /* Free */ 3572 g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0); 3573 g_hash_table_unref(views); 3574 } 3575 3576 struct AddressSpaceInfo { 3577 MemoryRegionListHead *ml_head; 3578 bool owner; 3579 bool disabled; 3580 }; 3581 3582 /* Returns negative value if a < b; zero if a = b; positive value if a > b. */ 3583 static gint address_space_compare_name(gconstpointer a, gconstpointer b) 3584 { 3585 const AddressSpace *as_a = a; 3586 const AddressSpace *as_b = b; 3587 3588 return g_strcmp0(as_a->name, as_b->name); 3589 } 3590 3591 static void mtree_print_as_name(gpointer data, gpointer user_data) 3592 { 3593 AddressSpace *as = data; 3594 3595 qemu_printf("address-space: %s\n", as->name); 3596 } 3597 3598 static void mtree_print_as(gpointer key, gpointer value, gpointer user_data) 3599 { 3600 MemoryRegion *mr = key; 3601 GSList *as_same_root_mr_list = value; 3602 struct AddressSpaceInfo *asi = user_data; 3603 3604 g_slist_foreach(as_same_root_mr_list, mtree_print_as_name, NULL); 3605 mtree_print_mr(mr, 1, 0, asi->ml_head, asi->owner, asi->disabled); 3606 qemu_printf("\n"); 3607 } 3608 3609 static gboolean mtree_info_as_free(gpointer key, gpointer value, 3610 gpointer user_data) 3611 { 3612 GSList *as_same_root_mr_list = value; 3613 3614 g_slist_free(as_same_root_mr_list); 3615 3616 return true; 3617 } 3618 3619 static void mtree_info_as(bool dispatch_tree, bool owner, bool disabled) 3620 { 3621 MemoryRegionListHead ml_head; 3622 MemoryRegionList *ml, *ml2; 3623 AddressSpace *as; 3624 GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); 3625 GSList *as_same_root_mr_list; 3626 struct AddressSpaceInfo asi = { 3627 .ml_head = &ml_head, 3628 .owner = owner, 3629 .disabled = disabled, 3630 }; 3631 3632 QTAILQ_INIT(&ml_head); 3633 3634 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 3635 /* Create hashtable, key=AS root MR, value = list of AS */ 3636 as_same_root_mr_list = g_hash_table_lookup(views, as->root); 3637 as_same_root_mr_list = g_slist_insert_sorted(as_same_root_mr_list, as, 3638 address_space_compare_name); 3639 g_hash_table_insert(views, as->root, as_same_root_mr_list); 3640 } 3641 3642 /* print address spaces */ 3643 g_hash_table_foreach(views, mtree_print_as, &asi); 3644 g_hash_table_foreach_remove(views, mtree_info_as_free, 0); 3645 g_hash_table_unref(views); 3646 3647 /* print aliased regions */ 3648 QTAILQ_FOREACH(ml, &ml_head, mrqueue) { 3649 qemu_printf("memory-region: %s\n", memory_region_name(ml->mr)); 3650 mtree_print_mr(ml->mr, 1, 0, &ml_head, owner, disabled); 3651 qemu_printf("\n"); 3652 } 3653 3654 QTAILQ_FOREACH_SAFE(ml, &ml_head, mrqueue, ml2) { 3655 g_free(ml); 3656 } 3657 } 3658 3659 void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled) 3660 { 3661 if (flatview) { 3662 mtree_info_flatview(dispatch_tree, owner); 3663 } else { 3664 mtree_info_as(dispatch_tree, owner, disabled); 3665 } 3666 } 3667 3668 bool memory_region_init_ram(MemoryRegion *mr, 3669 Object *owner, 3670 const char *name, 3671 uint64_t size, 3672 Error **errp) 3673 { 3674 DeviceState *owner_dev; 3675 3676 if (!memory_region_init_ram_nomigrate(mr, owner, name, size, errp)) { 3677 return false; 3678 } 3679 /* This will assert if owner is neither NULL nor a DeviceState. 3680 * We only want the owner here for the purposes of defining a 3681 * unique name for migration. TODO: Ideally we should implement 3682 * a naming scheme for Objects which are not DeviceStates, in 3683 * which case we can relax this restriction. 3684 */ 3685 owner_dev = DEVICE(owner); 3686 vmstate_register_ram(mr, owner_dev); 3687 3688 return true; 3689 } 3690 3691 bool memory_region_init_ram_guest_memfd(MemoryRegion *mr, 3692 Object *owner, 3693 const char *name, 3694 uint64_t size, 3695 Error **errp) 3696 { 3697 DeviceState *owner_dev; 3698 3699 if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, size, 3700 RAM_GUEST_MEMFD, errp)) { 3701 return false; 3702 } 3703 /* This will assert if owner is neither NULL nor a DeviceState. 3704 * We only want the owner here for the purposes of defining a 3705 * unique name for migration. TODO: Ideally we should implement 3706 * a naming scheme for Objects which are not DeviceStates, in 3707 * which case we can relax this restriction. 3708 */ 3709 owner_dev = DEVICE(owner); 3710 vmstate_register_ram(mr, owner_dev); 3711 3712 return true; 3713 } 3714 3715 bool memory_region_init_rom(MemoryRegion *mr, 3716 Object *owner, 3717 const char *name, 3718 uint64_t size, 3719 Error **errp) 3720 { 3721 DeviceState *owner_dev; 3722 3723 if (!memory_region_init_rom_nomigrate(mr, owner, name, size, errp)) { 3724 return false; 3725 } 3726 /* This will assert if owner is neither NULL nor a DeviceState. 3727 * We only want the owner here for the purposes of defining a 3728 * unique name for migration. TODO: Ideally we should implement 3729 * a naming scheme for Objects which are not DeviceStates, in 3730 * which case we can relax this restriction. 3731 */ 3732 owner_dev = DEVICE(owner); 3733 vmstate_register_ram(mr, owner_dev); 3734 3735 return true; 3736 } 3737 3738 bool memory_region_init_rom_device(MemoryRegion *mr, 3739 Object *owner, 3740 const MemoryRegionOps *ops, 3741 void *opaque, 3742 const char *name, 3743 uint64_t size, 3744 Error **errp) 3745 { 3746 DeviceState *owner_dev; 3747 3748 if (!memory_region_init_rom_device_nomigrate(mr, owner, ops, opaque, 3749 name, size, errp)) { 3750 return false; 3751 } 3752 /* This will assert if owner is neither NULL nor a DeviceState. 3753 * We only want the owner here for the purposes of defining a 3754 * unique name for migration. TODO: Ideally we should implement 3755 * a naming scheme for Objects which are not DeviceStates, in 3756 * which case we can relax this restriction. 3757 */ 3758 owner_dev = DEVICE(owner); 3759 vmstate_register_ram(mr, owner_dev); 3760 3761 return true; 3762 } 3763 3764 /* 3765 * Support system builds with CONFIG_FUZZ using a weak symbol and a stub for 3766 * the fuzz_dma_read_cb callback 3767 */ 3768 #ifdef CONFIG_FUZZ 3769 void __attribute__((weak)) fuzz_dma_read_cb(size_t addr, 3770 size_t len, 3771 MemoryRegion *mr) 3772 { 3773 } 3774 #endif 3775 3776 static const TypeInfo memory_region_info = { 3777 .parent = TYPE_OBJECT, 3778 .name = TYPE_MEMORY_REGION, 3779 .class_size = sizeof(MemoryRegionClass), 3780 .instance_size = sizeof(MemoryRegion), 3781 .instance_init = memory_region_initfn, 3782 .instance_finalize = memory_region_finalize, 3783 }; 3784 3785 static const TypeInfo iommu_memory_region_info = { 3786 .parent = TYPE_MEMORY_REGION, 3787 .name = TYPE_IOMMU_MEMORY_REGION, 3788 .class_size = sizeof(IOMMUMemoryRegionClass), 3789 .instance_size = sizeof(IOMMUMemoryRegion), 3790 .instance_init = iommu_memory_region_initfn, 3791 .abstract = true, 3792 }; 3793 3794 static const TypeInfo ram_discard_manager_info = { 3795 .parent = TYPE_INTERFACE, 3796 .name = TYPE_RAM_DISCARD_MANAGER, 3797 .class_size = sizeof(RamDiscardManagerClass), 3798 }; 3799 3800 static void memory_register_types(void) 3801 { 3802 type_register_static(&memory_region_info); 3803 type_register_static(&iommu_memory_region_info); 3804 type_register_static(&ram_discard_manager_info); 3805 } 3806 3807 type_init(memory_register_types) 3808