1 /* 2 * Physical memory management 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Contributions after 2012-01-13 are licensed under the terms of the 13 * GNU GPL, version 2 or (at your option) any later version. 14 */ 15 16 #include "qemu/osdep.h" 17 #include "qemu/log.h" 18 #include "qapi/error.h" 19 #include "system/memory.h" 20 #include "qapi/visitor.h" 21 #include "qemu/bitops.h" 22 #include "qemu/error-report.h" 23 #include "qemu/main-loop.h" 24 #include "qemu/qemu-print.h" 25 #include "qemu/target-info.h" 26 #include "qom/object.h" 27 #include "trace.h" 28 #include "system/ram_addr.h" 29 #include "system/kvm.h" 30 #include "system/runstate.h" 31 #include "system/tcg.h" 32 #include "qemu/accel.h" 33 #include "accel/accel-ops.h" 34 #include "hw/boards.h" 35 #include "migration/vmstate.h" 36 #include "system/address-spaces.h" 37 38 #include "memory-internal.h" 39 40 //#define DEBUG_UNASSIGNED 41 42 static unsigned memory_region_transaction_depth; 43 static bool memory_region_update_pending; 44 static bool ioeventfd_update_pending; 45 unsigned int global_dirty_tracking; 46 47 static QTAILQ_HEAD(, MemoryListener) memory_listeners 48 = QTAILQ_HEAD_INITIALIZER(memory_listeners); 49 50 static QTAILQ_HEAD(, AddressSpace) address_spaces 51 = QTAILQ_HEAD_INITIALIZER(address_spaces); 52 53 static GHashTable *flat_views; 54 55 typedef struct AddrRange AddrRange; 56 57 /* 58 * Note that signed integers are needed for negative offsetting in aliases 59 * (large MemoryRegion::alias_offset). 60 */ 61 struct AddrRange { 62 Int128 start; 63 Int128 size; 64 }; 65 66 static AddrRange addrrange_make(Int128 start, Int128 size) 67 { 68 return (AddrRange) { start, size }; 69 } 70 71 static bool addrrange_equal(AddrRange r1, AddrRange r2) 72 { 73 return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size); 74 } 75 76 static Int128 addrrange_end(AddrRange r) 77 { 78 return int128_add(r.start, r.size); 79 } 80 81 static AddrRange addrrange_shift(AddrRange range, Int128 delta) 82 { 83 int128_addto(&range.start, delta); 84 return range; 85 } 86 87 static bool addrrange_contains(AddrRange range, Int128 addr) 88 { 89 return int128_ge(addr, range.start) 90 && int128_lt(addr, addrrange_end(range)); 91 } 92 93 static bool addrrange_intersects(AddrRange r1, AddrRange r2) 94 { 95 return addrrange_contains(r1, r2.start) 96 || addrrange_contains(r2, r1.start); 97 } 98 99 static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) 100 { 101 Int128 start = int128_max(r1.start, r2.start); 102 Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2)); 103 return addrrange_make(start, int128_sub(end, start)); 104 } 105 106 enum ListenerDirection { Forward, Reverse }; 107 108 #define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ 109 do { \ 110 MemoryListener *_listener; \ 111 \ 112 switch (_direction) { \ 113 case Forward: \ 114 QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ 115 if (_listener->_callback) { \ 116 _listener->_callback(_listener, ##_args); \ 117 } \ 118 } \ 119 break; \ 120 case Reverse: \ 121 QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, link) { \ 122 if (_listener->_callback) { \ 123 _listener->_callback(_listener, ##_args); \ 124 } \ 125 } \ 126 break; \ 127 default: \ 128 abort(); \ 129 } \ 130 } while (0) 131 132 #define MEMORY_LISTENER_CALL(_as, _callback, _direction, _section, _args...) \ 133 do { \ 134 MemoryListener *_listener; \ 135 \ 136 switch (_direction) { \ 137 case Forward: \ 138 QTAILQ_FOREACH(_listener, &(_as)->listeners, link_as) { \ 139 if (_listener->_callback) { \ 140 _listener->_callback(_listener, _section, ##_args); \ 141 } \ 142 } \ 143 break; \ 144 case Reverse: \ 145 QTAILQ_FOREACH_REVERSE(_listener, &(_as)->listeners, link_as) { \ 146 if (_listener->_callback) { \ 147 _listener->_callback(_listener, _section, ##_args); \ 148 } \ 149 } \ 150 break; \ 151 default: \ 152 abort(); \ 153 } \ 154 } while (0) 155 156 /* No need to ref/unref .mr, the FlatRange keeps it alive. */ 157 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \ 158 do { \ 159 MemoryRegionSection mrs = section_from_flat_range(fr, \ 160 address_space_to_flatview(as)); \ 161 MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \ 162 } while(0) 163 164 struct CoalescedMemoryRange { 165 AddrRange addr; 166 QTAILQ_ENTRY(CoalescedMemoryRange) link; 167 }; 168 169 struct MemoryRegionIoeventfd { 170 AddrRange addr; 171 bool match_data; 172 uint64_t data; 173 EventNotifier *e; 174 }; 175 176 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd *a, 177 MemoryRegionIoeventfd *b) 178 { 179 if (int128_lt(a->addr.start, b->addr.start)) { 180 return true; 181 } else if (int128_gt(a->addr.start, b->addr.start)) { 182 return false; 183 } else if (int128_lt(a->addr.size, b->addr.size)) { 184 return true; 185 } else if (int128_gt(a->addr.size, b->addr.size)) { 186 return false; 187 } else if (a->match_data < b->match_data) { 188 return true; 189 } else if (a->match_data > b->match_data) { 190 return false; 191 } else if (a->match_data) { 192 if (a->data < b->data) { 193 return true; 194 } else if (a->data > b->data) { 195 return false; 196 } 197 } 198 if (a->e < b->e) { 199 return true; 200 } else if (a->e > b->e) { 201 return false; 202 } 203 return false; 204 } 205 206 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd *a, 207 MemoryRegionIoeventfd *b) 208 { 209 if (int128_eq(a->addr.start, b->addr.start) && 210 (!int128_nz(a->addr.size) || !int128_nz(b->addr.size) || 211 (int128_eq(a->addr.size, b->addr.size) && 212 (a->match_data == b->match_data) && 213 ((a->match_data && (a->data == b->data)) || !a->match_data) && 214 (a->e == b->e)))) 215 return true; 216 217 return false; 218 } 219 220 /* Range of memory in the global map. Addresses are absolute. */ 221 struct FlatRange { 222 MemoryRegion *mr; 223 hwaddr offset_in_region; 224 AddrRange addr; 225 uint8_t dirty_log_mask; 226 bool romd_mode; 227 bool readonly; 228 bool nonvolatile; 229 bool unmergeable; 230 }; 231 232 #define FOR_EACH_FLAT_RANGE(var, view) \ 233 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) 234 235 static inline MemoryRegionSection 236 section_from_flat_range(FlatRange *fr, FlatView *fv) 237 { 238 return (MemoryRegionSection) { 239 .mr = fr->mr, 240 .fv = fv, 241 .offset_within_region = fr->offset_in_region, 242 .size = fr->addr.size, 243 .offset_within_address_space = int128_get64(fr->addr.start), 244 .readonly = fr->readonly, 245 .nonvolatile = fr->nonvolatile, 246 .unmergeable = fr->unmergeable, 247 }; 248 } 249 250 static bool flatrange_equal(FlatRange *a, FlatRange *b) 251 { 252 return a->mr == b->mr 253 && addrrange_equal(a->addr, b->addr) 254 && a->offset_in_region == b->offset_in_region 255 && a->romd_mode == b->romd_mode 256 && a->readonly == b->readonly 257 && a->nonvolatile == b->nonvolatile 258 && a->unmergeable == b->unmergeable; 259 } 260 261 static FlatView *flatview_new(MemoryRegion *mr_root) 262 { 263 FlatView *view; 264 265 view = g_new0(FlatView, 1); 266 view->ref = 1; 267 view->root = mr_root; 268 memory_region_ref(mr_root); 269 trace_flatview_new(view, mr_root); 270 271 return view; 272 } 273 274 /* Insert a range into a given position. Caller is responsible for maintaining 275 * sorting order. 276 */ 277 static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range) 278 { 279 if (view->nr == view->nr_allocated) { 280 view->nr_allocated = MAX(2 * view->nr, 10); 281 view->ranges = g_realloc(view->ranges, 282 view->nr_allocated * sizeof(*view->ranges)); 283 } 284 memmove(view->ranges + pos + 1, view->ranges + pos, 285 (view->nr - pos) * sizeof(FlatRange)); 286 view->ranges[pos] = *range; 287 memory_region_ref(range->mr); 288 ++view->nr; 289 } 290 291 static void flatview_destroy(FlatView *view) 292 { 293 int i; 294 295 trace_flatview_destroy(view, view->root); 296 if (view->dispatch) { 297 address_space_dispatch_free(view->dispatch); 298 } 299 for (i = 0; i < view->nr; i++) { 300 memory_region_unref(view->ranges[i].mr); 301 } 302 g_free(view->ranges); 303 memory_region_unref(view->root); 304 g_free(view); 305 } 306 307 static bool flatview_ref(FlatView *view) 308 { 309 return qatomic_fetch_inc_nonzero(&view->ref) > 0; 310 } 311 312 void flatview_unref(FlatView *view) 313 { 314 if (qatomic_fetch_dec(&view->ref) == 1) { 315 trace_flatview_destroy_rcu(view, view->root); 316 assert(view->root); 317 call_rcu(view, flatview_destroy, rcu); 318 } 319 } 320 321 static bool can_merge(FlatRange *r1, FlatRange *r2) 322 { 323 return int128_eq(addrrange_end(r1->addr), r2->addr.start) 324 && r1->mr == r2->mr 325 && int128_eq(int128_add(int128_make64(r1->offset_in_region), 326 r1->addr.size), 327 int128_make64(r2->offset_in_region)) 328 && r1->dirty_log_mask == r2->dirty_log_mask 329 && r1->romd_mode == r2->romd_mode 330 && r1->readonly == r2->readonly 331 && r1->nonvolatile == r2->nonvolatile 332 && !r1->unmergeable && !r2->unmergeable; 333 } 334 335 /* Attempt to simplify a view by merging adjacent ranges */ 336 static void flatview_simplify(FlatView *view) 337 { 338 unsigned i, j, k; 339 340 i = 0; 341 while (i < view->nr) { 342 j = i + 1; 343 while (j < view->nr 344 && can_merge(&view->ranges[j-1], &view->ranges[j])) { 345 int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size); 346 ++j; 347 } 348 ++i; 349 for (k = i; k < j; k++) { 350 memory_region_unref(view->ranges[k].mr); 351 } 352 memmove(&view->ranges[i], &view->ranges[j], 353 (view->nr - j) * sizeof(view->ranges[j])); 354 view->nr -= j - i; 355 } 356 } 357 358 static void adjust_endianness(MemoryRegion *mr, uint64_t *data, MemOp op) 359 { 360 if ((op & MO_BSWAP) != devend_memop(mr->ops->endianness)) { 361 switch (op & MO_SIZE) { 362 case MO_8: 363 break; 364 case MO_16: 365 *data = bswap16(*data); 366 break; 367 case MO_32: 368 *data = bswap32(*data); 369 break; 370 case MO_64: 371 *data = bswap64(*data); 372 break; 373 default: 374 g_assert_not_reached(); 375 } 376 } 377 } 378 379 static inline void memory_region_shift_read_access(uint64_t *value, 380 signed shift, 381 uint64_t mask, 382 uint64_t tmp) 383 { 384 if (shift >= 0) { 385 *value |= (tmp & mask) << shift; 386 } else { 387 *value |= (tmp & mask) >> -shift; 388 } 389 } 390 391 static inline uint64_t memory_region_shift_write_access(uint64_t *value, 392 signed shift, 393 uint64_t mask) 394 { 395 uint64_t tmp; 396 397 if (shift >= 0) { 398 tmp = (*value >> shift) & mask; 399 } else { 400 tmp = (*value << -shift) & mask; 401 } 402 403 return tmp; 404 } 405 406 static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset) 407 { 408 MemoryRegion *root; 409 hwaddr abs_addr = offset; 410 411 abs_addr += mr->addr; 412 for (root = mr; root->container; ) { 413 root = root->container; 414 abs_addr += root->addr; 415 } 416 417 return abs_addr; 418 } 419 420 static int get_cpu_index(void) 421 { 422 if (current_cpu) { 423 return current_cpu->cpu_index; 424 } 425 return -1; 426 } 427 428 static MemTxResult memory_region_read_accessor(MemoryRegion *mr, 429 hwaddr addr, 430 uint64_t *value, 431 unsigned size, 432 signed shift, 433 uint64_t mask, 434 MemTxAttrs attrs) 435 { 436 uint64_t tmp; 437 438 tmp = mr->ops->read(mr->opaque, addr, size); 439 if (mr->subpage) { 440 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size); 441 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) { 442 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 443 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size, 444 memory_region_name(mr)); 445 } 446 memory_region_shift_read_access(value, shift, mask, tmp); 447 return MEMTX_OK; 448 } 449 450 static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr, 451 hwaddr addr, 452 uint64_t *value, 453 unsigned size, 454 signed shift, 455 uint64_t mask, 456 MemTxAttrs attrs) 457 { 458 uint64_t tmp = 0; 459 MemTxResult r; 460 461 r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs); 462 if (mr->subpage) { 463 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size); 464 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_READ)) { 465 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 466 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size, 467 memory_region_name(mr)); 468 } 469 memory_region_shift_read_access(value, shift, mask, tmp); 470 return r; 471 } 472 473 static MemTxResult memory_region_write_accessor(MemoryRegion *mr, 474 hwaddr addr, 475 uint64_t *value, 476 unsigned size, 477 signed shift, 478 uint64_t mask, 479 MemTxAttrs attrs) 480 { 481 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); 482 483 if (mr->subpage) { 484 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); 485 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) { 486 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 487 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size, 488 memory_region_name(mr)); 489 } 490 mr->ops->write(mr->opaque, addr, tmp, size); 491 return MEMTX_OK; 492 } 493 494 static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr, 495 hwaddr addr, 496 uint64_t *value, 497 unsigned size, 498 signed shift, 499 uint64_t mask, 500 MemTxAttrs attrs) 501 { 502 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); 503 504 if (mr->subpage) { 505 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); 506 } else if (trace_event_get_state_backends(TRACE_MEMORY_REGION_OPS_WRITE)) { 507 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); 508 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size, 509 memory_region_name(mr)); 510 } 511 return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs); 512 } 513 514 static MemTxResult access_with_adjusted_size_aligned(hwaddr addr, 515 uint64_t *value, 516 unsigned size, 517 unsigned access_size_min, 518 unsigned access_size_max, 519 MemTxResult (*access_fn) 520 (MemoryRegion *mr, 521 hwaddr addr, 522 uint64_t *value, 523 unsigned size, 524 signed shift, 525 uint64_t mask, 526 MemTxAttrs attrs), 527 MemoryRegion *mr, 528 MemTxAttrs attrs) 529 { 530 uint64_t access_mask; 531 unsigned access_size; 532 unsigned i; 533 MemTxResult r = MEMTX_OK; 534 bool reentrancy_guard_applied = false; 535 536 if (!access_size_min) { 537 access_size_min = 1; 538 } 539 if (!access_size_max) { 540 access_size_max = 4; 541 } 542 543 /* Do not allow more than one simultaneous access to a device's IO Regions */ 544 if (mr->dev && !mr->disable_reentrancy_guard && 545 !mr->ram_device && !mr->ram && !mr->rom_device && !mr->readonly) { 546 if (mr->dev->mem_reentrancy_guard.engaged_in_io) { 547 warn_report_once("Blocked re-entrant IO on MemoryRegion: " 548 "%s at addr: 0x%" HWADDR_PRIX, 549 memory_region_name(mr), addr); 550 return MEMTX_ACCESS_ERROR; 551 } 552 mr->dev->mem_reentrancy_guard.engaged_in_io = true; 553 reentrancy_guard_applied = true; 554 } 555 556 access_size = MAX(MIN(size, access_size_max), access_size_min); 557 access_mask = MAKE_64BIT_MASK(0, access_size * 8); 558 if (devend_big_endian(mr->ops->endianness)) { 559 for (i = 0; i < size; i += access_size) { 560 r |= access_fn(mr, addr + i, value, access_size, 561 (size - access_size - i) * 8, access_mask, attrs); 562 } 563 } else { 564 for (i = 0; i < size; i += access_size) { 565 r |= access_fn(mr, addr + i, value, access_size, i * 8, 566 access_mask, attrs); 567 } 568 } 569 if (mr->dev && reentrancy_guard_applied) { 570 mr->dev->mem_reentrancy_guard.engaged_in_io = false; 571 } 572 return r; 573 } 574 575 /* Assume power-of-two size */ 576 #define align_down(addr, size) ((addr) & ~((size) - 1)) 577 #define align_up(addr, size) \ 578 ({ typeof(size) __size = size; \ 579 align_down((addr) + (__size) - 1, (__size)); }) 580 581 static MemTxResult access_with_adjusted_size_unaligned(hwaddr addr, 582 uint64_t *value, 583 unsigned size, 584 unsigned access_size_min, 585 unsigned access_size_max, 586 bool unaligned, 587 MemTxResult (*access)(MemoryRegion *mr, 588 hwaddr addr, 589 uint64_t *value, 590 unsigned size, 591 signed shift, 592 uint64_t mask, 593 MemTxAttrs attrs), 594 MemoryRegion *mr, 595 MemTxAttrs attrs) 596 { 597 uint64_t access_value = 0; 598 MemTxResult r = MEMTX_OK; 599 hwaddr access_addr[2]; 600 uint64_t access_mask; 601 unsigned access_size; 602 603 if (unlikely(!access_size_min)) { 604 access_size_min = 1; 605 } 606 if (unlikely(!access_size_max)) { 607 access_size_max = 4; 608 } 609 610 access_size = MAX(MIN(size, access_size_max), access_size_min); 611 access_addr[0] = align_down(addr, access_size); 612 access_addr[1] = align_up(addr + size, access_size); 613 614 if (devend_big_endian(mr->ops->endianness)) { 615 hwaddr cur; 616 617 /* XXX: Big-endian path is untested... */ 618 619 for (cur = access_addr[0]; cur < access_addr[1]; cur += access_size) { 620 uint64_t mask_bounds[2]; 621 622 mask_bounds[0] = MAX(addr, cur) - cur; 623 mask_bounds[1] = 624 MIN(addr + size, align_up(cur + 1, access_size)) - cur; 625 626 access_mask = (-1ULL << mask_bounds[0] * 8) & 627 (-1ULL >> (64 - mask_bounds[1] * 8)); 628 629 r |= access(mr, cur, &access_value, access_size, 630 (size - access_size - (MAX(addr, cur) - addr)), 631 access_mask, attrs); 632 633 /* XXX: Can't do this hack for writes */ 634 access_value >>= mask_bounds[0] * 8; 635 } 636 } else { 637 hwaddr cur; 638 639 for (cur = access_addr[0]; cur < access_addr[1]; cur += access_size) { 640 uint64_t mask_bounds[2]; 641 642 mask_bounds[0] = MAX(addr, cur) - cur; 643 mask_bounds[1] = 644 MIN(addr + size, align_up(cur + 1, access_size)) - cur; 645 646 access_mask = (-1ULL << mask_bounds[0] * 8) & 647 (-1ULL >> (64 - mask_bounds[1] * 8)); 648 649 r |= access(mr, cur, &access_value, access_size, 650 (MAX(addr, cur) - addr), access_mask, attrs); 651 652 /* XXX: Can't do this hack for writes */ 653 access_value >>= mask_bounds[0] * 8; 654 } 655 } 656 657 *value = access_value; 658 659 return r; 660 } 661 662 static inline MemTxResult access_with_adjusted_size(hwaddr addr, 663 uint64_t *value, 664 unsigned size, 665 unsigned access_size_min, 666 unsigned access_size_max, 667 bool unaligned, 668 MemTxResult (*access)(MemoryRegion *mr, 669 hwaddr addr, 670 uint64_t *value, 671 unsigned size, 672 signed shift, 673 uint64_t mask, 674 MemTxAttrs attrs), 675 MemoryRegion *mr, 676 MemTxAttrs attrs) 677 { 678 unsigned access_size; 679 680 if (!access_size_min) { 681 access_size_min = 1; 682 } 683 if (!access_size_max) { 684 access_size_max = 4; 685 } 686 687 access_size = MAX(MIN(size, access_size_max), access_size_min); 688 689 /* Handle unaligned accesses if the model only supports natural alignment */ 690 if (unlikely((addr & (access_size - 1)) && !unaligned)) { 691 return access_with_adjusted_size_unaligned(addr, value, size, 692 access_size_min, access_size_max, unaligned, access, mr, attrs); 693 } 694 695 /* 696 * Otherwise, if the access is aligned or the model specifies it can handle 697 * unaligned accesses, use the 'aligned' handler 698 */ 699 return access_with_adjusted_size_aligned(addr, value, size, 700 access_size_min, access_size_max, access, mr, attrs); 701 } 702 703 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) 704 { 705 AddressSpace *as; 706 707 while (mr->container) { 708 mr = mr->container; 709 } 710 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 711 if (mr == as->root) { 712 return as; 713 } 714 } 715 return NULL; 716 } 717 718 /* Render a memory region into the global view. Ranges in @view obscure 719 * ranges in @mr. 720 */ 721 static void render_memory_region(FlatView *view, 722 MemoryRegion *mr, 723 Int128 base, 724 AddrRange clip, 725 bool readonly, 726 bool nonvolatile, 727 bool unmergeable) 728 { 729 MemoryRegion *subregion; 730 unsigned i; 731 hwaddr offset_in_region; 732 Int128 remain; 733 Int128 now; 734 FlatRange fr; 735 AddrRange tmp; 736 737 if (!mr->enabled) { 738 return; 739 } 740 741 int128_addto(&base, int128_make64(mr->addr)); 742 readonly |= mr->readonly; 743 nonvolatile |= mr->nonvolatile; 744 unmergeable |= mr->unmergeable; 745 746 tmp = addrrange_make(base, mr->size); 747 748 if (!addrrange_intersects(tmp, clip)) { 749 return; 750 } 751 752 clip = addrrange_intersection(tmp, clip); 753 754 if (mr->alias) { 755 int128_subfrom(&base, int128_make64(mr->alias->addr)); 756 int128_subfrom(&base, int128_make64(mr->alias_offset)); 757 render_memory_region(view, mr->alias, base, clip, 758 readonly, nonvolatile, unmergeable); 759 return; 760 } 761 762 /* Render subregions in priority order. */ 763 QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) { 764 render_memory_region(view, subregion, base, clip, 765 readonly, nonvolatile, unmergeable); 766 } 767 768 if (!mr->terminates) { 769 return; 770 } 771 772 offset_in_region = int128_get64(int128_sub(clip.start, base)); 773 base = clip.start; 774 remain = clip.size; 775 776 fr.mr = mr; 777 fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr); 778 fr.romd_mode = mr->romd_mode; 779 fr.readonly = readonly; 780 fr.nonvolatile = nonvolatile; 781 fr.unmergeable = unmergeable; 782 783 /* Render the region itself into any gaps left by the current view. */ 784 for (i = 0; i < view->nr && int128_nz(remain); ++i) { 785 if (int128_ge(base, addrrange_end(view->ranges[i].addr))) { 786 continue; 787 } 788 if (int128_lt(base, view->ranges[i].addr.start)) { 789 now = int128_min(remain, 790 int128_sub(view->ranges[i].addr.start, base)); 791 fr.offset_in_region = offset_in_region; 792 fr.addr = addrrange_make(base, now); 793 flatview_insert(view, i, &fr); 794 ++i; 795 int128_addto(&base, now); 796 offset_in_region += int128_get64(now); 797 int128_subfrom(&remain, now); 798 } 799 now = int128_sub(int128_min(int128_add(base, remain), 800 addrrange_end(view->ranges[i].addr)), 801 base); 802 int128_addto(&base, now); 803 offset_in_region += int128_get64(now); 804 int128_subfrom(&remain, now); 805 } 806 if (int128_nz(remain)) { 807 fr.offset_in_region = offset_in_region; 808 fr.addr = addrrange_make(base, remain); 809 flatview_insert(view, i, &fr); 810 } 811 } 812 813 void flatview_for_each_range(FlatView *fv, flatview_cb cb , void *opaque) 814 { 815 FlatRange *fr; 816 817 assert(fv); 818 assert(cb); 819 820 FOR_EACH_FLAT_RANGE(fr, fv) { 821 if (cb(fr->addr.start, fr->addr.size, fr->mr, 822 fr->offset_in_region, opaque)) { 823 break; 824 } 825 } 826 } 827 828 static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr) 829 { 830 while (mr->enabled) { 831 if (mr->alias) { 832 if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) { 833 /* The alias is included in its entirety. Use it as 834 * the "real" root, so that we can share more FlatViews. 835 */ 836 mr = mr->alias; 837 continue; 838 } 839 } else if (!mr->terminates) { 840 unsigned int found = 0; 841 MemoryRegion *child, *next = NULL; 842 QTAILQ_FOREACH(child, &mr->subregions, subregions_link) { 843 if (child->enabled) { 844 if (++found > 1) { 845 next = NULL; 846 break; 847 } 848 if (!child->addr && int128_ge(mr->size, child->size)) { 849 /* A child is included in its entirety. If it's the only 850 * enabled one, use it in the hope of finding an alias down the 851 * way. This will also let us share FlatViews. 852 */ 853 next = child; 854 } 855 } 856 } 857 if (found == 0) { 858 return NULL; 859 } 860 if (next) { 861 mr = next; 862 continue; 863 } 864 } 865 866 return mr; 867 } 868 869 return NULL; 870 } 871 872 /* Render a memory topology into a list of disjoint absolute ranges. */ 873 static FlatView *generate_memory_topology(MemoryRegion *mr) 874 { 875 int i; 876 FlatView *view; 877 878 view = flatview_new(mr); 879 880 if (mr) { 881 render_memory_region(view, mr, int128_zero(), 882 addrrange_make(int128_zero(), int128_2_64()), 883 false, false, false); 884 } 885 flatview_simplify(view); 886 887 view->dispatch = address_space_dispatch_new(view); 888 for (i = 0; i < view->nr; i++) { 889 MemoryRegionSection mrs = 890 section_from_flat_range(&view->ranges[i], view); 891 flatview_add_to_dispatch(view, &mrs); 892 } 893 address_space_dispatch_compact(view->dispatch); 894 g_hash_table_replace(flat_views, mr, view); 895 896 return view; 897 } 898 899 static void address_space_add_del_ioeventfds(AddressSpace *as, 900 MemoryRegionIoeventfd *fds_new, 901 unsigned fds_new_nb, 902 MemoryRegionIoeventfd *fds_old, 903 unsigned fds_old_nb) 904 { 905 unsigned iold, inew; 906 MemoryRegionIoeventfd *fd; 907 MemoryRegionSection section; 908 909 /* Generate a symmetric difference of the old and new fd sets, adding 910 * and deleting as necessary. 911 */ 912 913 iold = inew = 0; 914 while (iold < fds_old_nb || inew < fds_new_nb) { 915 if (iold < fds_old_nb 916 && (inew == fds_new_nb 917 || memory_region_ioeventfd_before(&fds_old[iold], 918 &fds_new[inew]))) { 919 fd = &fds_old[iold]; 920 section = (MemoryRegionSection) { 921 .fv = address_space_to_flatview(as), 922 .offset_within_address_space = int128_get64(fd->addr.start), 923 .size = fd->addr.size, 924 }; 925 MEMORY_LISTENER_CALL(as, eventfd_del, Forward, §ion, 926 fd->match_data, fd->data, fd->e); 927 ++iold; 928 } else if (inew < fds_new_nb 929 && (iold == fds_old_nb 930 || memory_region_ioeventfd_before(&fds_new[inew], 931 &fds_old[iold]))) { 932 fd = &fds_new[inew]; 933 section = (MemoryRegionSection) { 934 .fv = address_space_to_flatview(as), 935 .offset_within_address_space = int128_get64(fd->addr.start), 936 .size = fd->addr.size, 937 }; 938 MEMORY_LISTENER_CALL(as, eventfd_add, Reverse, §ion, 939 fd->match_data, fd->data, fd->e); 940 ++inew; 941 } else { 942 ++iold; 943 ++inew; 944 } 945 } 946 } 947 948 FlatView *address_space_get_flatview(AddressSpace *as) 949 { 950 FlatView *view; 951 952 RCU_READ_LOCK_GUARD(); 953 do { 954 view = address_space_to_flatview(as); 955 /* If somebody has replaced as->current_map concurrently, 956 * flatview_ref returns false. 957 */ 958 } while (!flatview_ref(view)); 959 return view; 960 } 961 962 static void address_space_update_ioeventfds(AddressSpace *as) 963 { 964 FlatView *view; 965 FlatRange *fr; 966 unsigned ioeventfd_nb = 0; 967 unsigned ioeventfd_max; 968 MemoryRegionIoeventfd *ioeventfds; 969 AddrRange tmp; 970 unsigned i; 971 972 if (!as->ioeventfd_notifiers) { 973 return; 974 } 975 976 /* 977 * It is likely that the number of ioeventfds hasn't changed much, so use 978 * the previous size as the starting value, with some headroom to avoid 979 * gratuitous reallocations. 980 */ 981 ioeventfd_max = QEMU_ALIGN_UP(as->ioeventfd_nb, 4); 982 ioeventfds = g_new(MemoryRegionIoeventfd, ioeventfd_max); 983 984 view = address_space_get_flatview(as); 985 FOR_EACH_FLAT_RANGE(fr, view) { 986 for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { 987 tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, 988 int128_sub(fr->addr.start, 989 int128_make64(fr->offset_in_region))); 990 if (addrrange_intersects(fr->addr, tmp)) { 991 ++ioeventfd_nb; 992 if (ioeventfd_nb > ioeventfd_max) { 993 ioeventfd_max = MAX(ioeventfd_max * 2, 4); 994 ioeventfds = g_realloc(ioeventfds, 995 ioeventfd_max * sizeof(*ioeventfds)); 996 } 997 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i]; 998 ioeventfds[ioeventfd_nb-1].addr = tmp; 999 } 1000 } 1001 } 1002 1003 address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb, 1004 as->ioeventfds, as->ioeventfd_nb); 1005 1006 g_free(as->ioeventfds); 1007 as->ioeventfds = ioeventfds; 1008 as->ioeventfd_nb = ioeventfd_nb; 1009 flatview_unref(view); 1010 } 1011 1012 /* 1013 * Notify the memory listeners about the coalesced IO change events of 1014 * range `cmr'. Only the part that has intersection of the specified 1015 * FlatRange will be sent. 1016 */ 1017 static void flat_range_coalesced_io_notify(FlatRange *fr, AddressSpace *as, 1018 CoalescedMemoryRange *cmr, bool add) 1019 { 1020 AddrRange tmp; 1021 1022 tmp = addrrange_shift(cmr->addr, 1023 int128_sub(fr->addr.start, 1024 int128_make64(fr->offset_in_region))); 1025 if (!addrrange_intersects(tmp, fr->addr)) { 1026 return; 1027 } 1028 tmp = addrrange_intersection(tmp, fr->addr); 1029 1030 if (add) { 1031 MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, coalesced_io_add, 1032 int128_get64(tmp.start), 1033 int128_get64(tmp.size)); 1034 } else { 1035 MEMORY_LISTENER_UPDATE_REGION(fr, as, Reverse, coalesced_io_del, 1036 int128_get64(tmp.start), 1037 int128_get64(tmp.size)); 1038 } 1039 } 1040 1041 static void flat_range_coalesced_io_del(FlatRange *fr, AddressSpace *as) 1042 { 1043 CoalescedMemoryRange *cmr; 1044 1045 QTAILQ_FOREACH(cmr, &fr->mr->coalesced, link) { 1046 flat_range_coalesced_io_notify(fr, as, cmr, false); 1047 } 1048 } 1049 1050 static void flat_range_coalesced_io_add(FlatRange *fr, AddressSpace *as) 1051 { 1052 MemoryRegion *mr = fr->mr; 1053 CoalescedMemoryRange *cmr; 1054 1055 if (QTAILQ_EMPTY(&mr->coalesced)) { 1056 return; 1057 } 1058 1059 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { 1060 flat_range_coalesced_io_notify(fr, as, cmr, true); 1061 } 1062 } 1063 1064 static void 1065 flat_range_coalesced_io_notify_listener_add_del(FlatRange *fr, 1066 MemoryRegionSection *mrs, 1067 MemoryListener *listener, 1068 AddressSpace *as, bool add) 1069 { 1070 CoalescedMemoryRange *cmr; 1071 MemoryRegion *mr = fr->mr; 1072 AddrRange tmp; 1073 1074 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { 1075 tmp = addrrange_shift(cmr->addr, 1076 int128_sub(fr->addr.start, 1077 int128_make64(fr->offset_in_region))); 1078 1079 if (!addrrange_intersects(tmp, fr->addr)) { 1080 return; 1081 } 1082 tmp = addrrange_intersection(tmp, fr->addr); 1083 1084 if (add && listener->coalesced_io_add) { 1085 listener->coalesced_io_add(listener, mrs, 1086 int128_get64(tmp.start), 1087 int128_get64(tmp.size)); 1088 } else if (!add && listener->coalesced_io_del) { 1089 listener->coalesced_io_del(listener, mrs, 1090 int128_get64(tmp.start), 1091 int128_get64(tmp.size)); 1092 } 1093 } 1094 } 1095 1096 static void address_space_update_topology_pass(AddressSpace *as, 1097 const FlatView *old_view, 1098 const FlatView *new_view, 1099 bool adding) 1100 { 1101 unsigned iold, inew; 1102 FlatRange *frold, *frnew; 1103 1104 /* Generate a symmetric difference of the old and new memory maps. 1105 * Kill ranges in the old map, and instantiate ranges in the new map. 1106 */ 1107 iold = inew = 0; 1108 while (iold < old_view->nr || inew < new_view->nr) { 1109 if (iold < old_view->nr) { 1110 frold = &old_view->ranges[iold]; 1111 } else { 1112 frold = NULL; 1113 } 1114 if (inew < new_view->nr) { 1115 frnew = &new_view->ranges[inew]; 1116 } else { 1117 frnew = NULL; 1118 } 1119 1120 if (frold 1121 && (!frnew 1122 || int128_lt(frold->addr.start, frnew->addr.start) 1123 || (int128_eq(frold->addr.start, frnew->addr.start) 1124 && !flatrange_equal(frold, frnew)))) { 1125 /* In old but not in new, or in both but attributes changed. */ 1126 1127 if (!adding) { 1128 flat_range_coalesced_io_del(frold, as); 1129 MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); 1130 } 1131 1132 ++iold; 1133 } else if (frold && frnew && flatrange_equal(frold, frnew)) { 1134 /* In both and unchanged (except logging may have changed) */ 1135 1136 if (adding) { 1137 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); 1138 if (frnew->dirty_log_mask & ~frold->dirty_log_mask) { 1139 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start, 1140 frold->dirty_log_mask, 1141 frnew->dirty_log_mask); 1142 } 1143 if (frold->dirty_log_mask & ~frnew->dirty_log_mask) { 1144 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop, 1145 frold->dirty_log_mask, 1146 frnew->dirty_log_mask); 1147 } 1148 } 1149 1150 ++iold; 1151 ++inew; 1152 } else { 1153 /* In new */ 1154 1155 if (adding) { 1156 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); 1157 flat_range_coalesced_io_add(frnew, as); 1158 } 1159 1160 ++inew; 1161 } 1162 } 1163 } 1164 1165 static void flatviews_init(void) 1166 { 1167 static FlatView *empty_view; 1168 1169 if (flat_views) { 1170 return; 1171 } 1172 1173 flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, 1174 (GDestroyNotify) flatview_unref); 1175 if (!empty_view) { 1176 empty_view = generate_memory_topology(NULL); 1177 /* We keep it alive forever in the global variable. */ 1178 flatview_ref(empty_view); 1179 } else { 1180 g_hash_table_replace(flat_views, NULL, empty_view); 1181 flatview_ref(empty_view); 1182 } 1183 } 1184 1185 static void flatviews_reset(void) 1186 { 1187 AddressSpace *as; 1188 1189 if (flat_views) { 1190 g_hash_table_unref(flat_views); 1191 flat_views = NULL; 1192 } 1193 flatviews_init(); 1194 1195 /* Render unique FVs */ 1196 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1197 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1198 1199 if (g_hash_table_lookup(flat_views, physmr)) { 1200 continue; 1201 } 1202 1203 generate_memory_topology(physmr); 1204 } 1205 } 1206 1207 static void address_space_set_flatview(AddressSpace *as) 1208 { 1209 FlatView *old_view = address_space_to_flatview(as); 1210 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1211 FlatView *new_view = g_hash_table_lookup(flat_views, physmr); 1212 1213 assert(new_view); 1214 1215 if (old_view == new_view) { 1216 return; 1217 } 1218 1219 if (old_view) { 1220 flatview_ref(old_view); 1221 } 1222 1223 flatview_ref(new_view); 1224 1225 if (!QTAILQ_EMPTY(&as->listeners)) { 1226 FlatView tmpview = { .nr = 0 }, *old_view2 = old_view; 1227 1228 if (!old_view2) { 1229 old_view2 = &tmpview; 1230 } 1231 address_space_update_topology_pass(as, old_view2, new_view, false); 1232 address_space_update_topology_pass(as, old_view2, new_view, true); 1233 } 1234 1235 /* Writes are protected by the BQL. */ 1236 qatomic_rcu_set(&as->current_map, new_view); 1237 if (old_view) { 1238 flatview_unref(old_view); 1239 } 1240 1241 /* Note that all the old MemoryRegions are still alive up to this 1242 * point. This relieves most MemoryListeners from the need to 1243 * ref/unref the MemoryRegions they get---unless they use them 1244 * outside the iothread mutex, in which case precise reference 1245 * counting is necessary. 1246 */ 1247 if (old_view) { 1248 flatview_unref(old_view); 1249 } 1250 } 1251 1252 static void address_space_update_topology(AddressSpace *as) 1253 { 1254 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); 1255 1256 flatviews_init(); 1257 if (!g_hash_table_lookup(flat_views, physmr)) { 1258 generate_memory_topology(physmr); 1259 } 1260 address_space_set_flatview(as); 1261 } 1262 1263 void memory_region_transaction_begin(void) 1264 { 1265 qemu_flush_coalesced_mmio_buffer(); 1266 ++memory_region_transaction_depth; 1267 } 1268 1269 void memory_region_transaction_commit(void) 1270 { 1271 AddressSpace *as; 1272 1273 assert(memory_region_transaction_depth); 1274 assert(bql_locked()); 1275 1276 --memory_region_transaction_depth; 1277 if (!memory_region_transaction_depth) { 1278 if (memory_region_update_pending) { 1279 flatviews_reset(); 1280 1281 MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); 1282 1283 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1284 address_space_set_flatview(as); 1285 address_space_update_ioeventfds(as); 1286 } 1287 memory_region_update_pending = false; 1288 ioeventfd_update_pending = false; 1289 MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); 1290 } else if (ioeventfd_update_pending) { 1291 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 1292 address_space_update_ioeventfds(as); 1293 } 1294 ioeventfd_update_pending = false; 1295 } 1296 } 1297 } 1298 1299 static void memory_region_destructor_none(MemoryRegion *mr) 1300 { 1301 } 1302 1303 static void memory_region_destructor_ram(MemoryRegion *mr) 1304 { 1305 qemu_ram_free(mr->ram_block); 1306 } 1307 1308 static bool memory_region_need_escape(char c) 1309 { 1310 return c == '/' || c == '[' || c == '\\' || c == ']'; 1311 } 1312 1313 static char *memory_region_escape_name(const char *name) 1314 { 1315 const char *p; 1316 char *escaped, *q; 1317 uint8_t c; 1318 size_t bytes = 0; 1319 1320 for (p = name; *p; p++) { 1321 bytes += memory_region_need_escape(*p) ? 4 : 1; 1322 } 1323 if (bytes == p - name) { 1324 return g_memdup(name, bytes + 1); 1325 } 1326 1327 escaped = g_malloc(bytes + 1); 1328 for (p = name, q = escaped; *p; p++) { 1329 c = *p; 1330 if (unlikely(memory_region_need_escape(c))) { 1331 *q++ = '\\'; 1332 *q++ = 'x'; 1333 *q++ = "0123456789abcdef"[c >> 4]; 1334 c = "0123456789abcdef"[c & 15]; 1335 } 1336 *q++ = c; 1337 } 1338 *q = 0; 1339 return escaped; 1340 } 1341 1342 static void memory_region_do_init(MemoryRegion *mr, 1343 Object *owner, 1344 const char *name, 1345 uint64_t size) 1346 { 1347 mr->size = int128_make64(size); 1348 if (size == UINT64_MAX) { 1349 mr->size = int128_2_64(); 1350 } 1351 mr->name = g_strdup(name); 1352 mr->owner = owner; 1353 mr->dev = (DeviceState *) object_dynamic_cast(mr->owner, TYPE_DEVICE); 1354 mr->ram_block = NULL; 1355 1356 if (name) { 1357 char *escaped_name = memory_region_escape_name(name); 1358 char *name_array = g_strdup_printf("%s[*]", escaped_name); 1359 1360 if (!owner) { 1361 owner = machine_get_container("unattached"); 1362 } 1363 1364 object_property_add_child(owner, name_array, OBJECT(mr)); 1365 object_unref(OBJECT(mr)); 1366 g_free(name_array); 1367 g_free(escaped_name); 1368 } 1369 } 1370 1371 void memory_region_init(MemoryRegion *mr, 1372 Object *owner, 1373 const char *name, 1374 uint64_t size) 1375 { 1376 object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION); 1377 memory_region_do_init(mr, owner, name, size); 1378 } 1379 1380 static void memory_region_get_container(Object *obj, Visitor *v, 1381 const char *name, void *opaque, 1382 Error **errp) 1383 { 1384 MemoryRegion *mr = MEMORY_REGION(obj); 1385 char *path = (char *)""; 1386 1387 if (mr->container) { 1388 path = object_get_canonical_path(OBJECT(mr->container)); 1389 } 1390 visit_type_str(v, name, &path, errp); 1391 if (mr->container) { 1392 g_free(path); 1393 } 1394 } 1395 1396 static Object *memory_region_resolve_container(Object *obj, void *opaque, 1397 const char *part) 1398 { 1399 MemoryRegion *mr = MEMORY_REGION(obj); 1400 1401 return OBJECT(mr->container); 1402 } 1403 1404 static void memory_region_get_priority(Object *obj, Visitor *v, 1405 const char *name, void *opaque, 1406 Error **errp) 1407 { 1408 MemoryRegion *mr = MEMORY_REGION(obj); 1409 int32_t value = mr->priority; 1410 1411 visit_type_int32(v, name, &value, errp); 1412 } 1413 1414 static void memory_region_get_size(Object *obj, Visitor *v, const char *name, 1415 void *opaque, Error **errp) 1416 { 1417 MemoryRegion *mr = MEMORY_REGION(obj); 1418 uint64_t value = memory_region_size(mr); 1419 1420 visit_type_uint64(v, name, &value, errp); 1421 } 1422 1423 static void memory_region_initfn(Object *obj) 1424 { 1425 MemoryRegion *mr = MEMORY_REGION(obj); 1426 ObjectProperty *op; 1427 1428 mr->ops = &unassigned_mem_ops; 1429 mr->enabled = true; 1430 mr->romd_mode = true; 1431 mr->destructor = memory_region_destructor_none; 1432 QTAILQ_INIT(&mr->subregions); 1433 QTAILQ_INIT(&mr->coalesced); 1434 1435 op = object_property_add(OBJECT(mr), "container", 1436 "link<" TYPE_MEMORY_REGION ">", 1437 memory_region_get_container, 1438 NULL, /* memory_region_set_container */ 1439 NULL, NULL); 1440 op->resolve = memory_region_resolve_container; 1441 1442 object_property_add_uint64_ptr(OBJECT(mr), "addr", 1443 &mr->addr, OBJ_PROP_FLAG_READ); 1444 object_property_add(OBJECT(mr), "priority", "uint32", 1445 memory_region_get_priority, 1446 NULL, /* memory_region_set_priority */ 1447 NULL, NULL); 1448 object_property_add(OBJECT(mr), "size", "uint64", 1449 memory_region_get_size, 1450 NULL, /* memory_region_set_size, */ 1451 NULL, NULL); 1452 } 1453 1454 static void iommu_memory_region_initfn(Object *obj) 1455 { 1456 MemoryRegion *mr = MEMORY_REGION(obj); 1457 1458 mr->is_iommu = true; 1459 } 1460 1461 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr, 1462 unsigned size) 1463 { 1464 #ifdef DEBUG_UNASSIGNED 1465 printf("Unassigned mem read " HWADDR_FMT_plx "\n", addr); 1466 #endif 1467 return 0; 1468 } 1469 1470 static void unassigned_mem_write(void *opaque, hwaddr addr, 1471 uint64_t val, unsigned size) 1472 { 1473 #ifdef DEBUG_UNASSIGNED 1474 printf("Unassigned mem write " HWADDR_FMT_plx " = 0x%"PRIx64"\n", addr, val); 1475 #endif 1476 } 1477 1478 static bool unassigned_mem_accepts(void *opaque, hwaddr addr, 1479 unsigned size, bool is_write, 1480 MemTxAttrs attrs) 1481 { 1482 return false; 1483 } 1484 1485 const MemoryRegionOps unassigned_mem_ops = { 1486 .valid.accepts = unassigned_mem_accepts, 1487 .endianness = DEVICE_NATIVE_ENDIAN, 1488 }; 1489 1490 static uint64_t memory_region_ram_device_read(void *opaque, 1491 hwaddr addr, unsigned size) 1492 { 1493 MemoryRegion *mr = opaque; 1494 uint64_t data = ldn_he_p(mr->ram_block->host + addr, size); 1495 1496 trace_memory_region_ram_device_read(get_cpu_index(), mr, addr, data, size); 1497 1498 return data; 1499 } 1500 1501 static void memory_region_ram_device_write(void *opaque, hwaddr addr, 1502 uint64_t data, unsigned size) 1503 { 1504 MemoryRegion *mr = opaque; 1505 1506 trace_memory_region_ram_device_write(get_cpu_index(), mr, addr, data, size); 1507 1508 stn_he_p(mr->ram_block->host + addr, size, data); 1509 } 1510 1511 static const MemoryRegionOps ram_device_mem_ops = { 1512 .read = memory_region_ram_device_read, 1513 .write = memory_region_ram_device_write, 1514 .endianness = HOST_BIG_ENDIAN ? DEVICE_BIG_ENDIAN : DEVICE_LITTLE_ENDIAN, 1515 .valid = { 1516 .min_access_size = 1, 1517 .max_access_size = 8, 1518 .unaligned = true, 1519 }, 1520 .impl = { 1521 .min_access_size = 1, 1522 .max_access_size = 8, 1523 .unaligned = true, 1524 }, 1525 }; 1526 1527 bool memory_region_access_valid(MemoryRegion *mr, 1528 hwaddr addr, 1529 unsigned size, 1530 bool is_write, 1531 MemTxAttrs attrs) 1532 { 1533 if (mr->ops->valid.accepts 1534 && !mr->ops->valid.accepts(mr->opaque, addr, size, is_write, attrs)) { 1535 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1536 ", size %u, region '%s', reason: rejected\n", 1537 is_write ? "write" : "read", 1538 addr, size, memory_region_name(mr)); 1539 return false; 1540 } 1541 1542 if (!mr->ops->valid.unaligned && (addr & (size - 1))) { 1543 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1544 ", size %u, region '%s', reason: unaligned\n", 1545 is_write ? "write" : "read", 1546 addr, size, memory_region_name(mr)); 1547 return false; 1548 } 1549 1550 /* Treat zero as compatibility all valid */ 1551 if (!mr->ops->valid.max_access_size) { 1552 return true; 1553 } 1554 1555 if (size > mr->ops->valid.max_access_size 1556 || size < mr->ops->valid.min_access_size) { 1557 qemu_log_mask(LOG_INVALID_MEM, "Invalid %s at addr 0x%" HWADDR_PRIX 1558 ", size %u, region '%s', reason: invalid size " 1559 "(min:%u max:%u)\n", 1560 is_write ? "write" : "read", 1561 addr, size, memory_region_name(mr), 1562 mr->ops->valid.min_access_size, 1563 mr->ops->valid.max_access_size); 1564 return false; 1565 } 1566 return true; 1567 } 1568 1569 static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr, 1570 hwaddr addr, 1571 uint64_t *pval, 1572 unsigned size, 1573 MemTxAttrs attrs) 1574 { 1575 *pval = 0; 1576 1577 if (mr->ops->read) { 1578 return access_with_adjusted_size(addr, pval, size, 1579 mr->ops->impl.min_access_size, 1580 mr->ops->impl.max_access_size, 1581 mr->ops->impl.unaligned, 1582 memory_region_read_accessor, 1583 mr, attrs); 1584 } else { 1585 return access_with_adjusted_size(addr, pval, size, 1586 mr->ops->impl.min_access_size, 1587 mr->ops->impl.max_access_size, 1588 mr->ops->impl.unaligned, 1589 memory_region_read_with_attrs_accessor, 1590 mr, attrs); 1591 } 1592 } 1593 1594 MemTxResult memory_region_dispatch_read(MemoryRegion *mr, 1595 hwaddr addr, 1596 uint64_t *pval, 1597 MemOp op, 1598 MemTxAttrs attrs) 1599 { 1600 unsigned size = memop_size(op); 1601 MemTxResult r; 1602 1603 if (mr->alias) { 1604 return memory_region_dispatch_read(mr->alias, 1605 mr->alias_offset + addr, 1606 pval, op, attrs); 1607 } 1608 if (!memory_region_access_valid(mr, addr, size, false, attrs)) { 1609 *pval = unassigned_mem_read(mr, addr, size); 1610 return MEMTX_DECODE_ERROR; 1611 } 1612 1613 r = memory_region_dispatch_read1(mr, addr, pval, size, attrs); 1614 adjust_endianness(mr, pval, op); 1615 return r; 1616 } 1617 1618 /* Return true if an eventfd was signalled */ 1619 static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr, 1620 hwaddr addr, 1621 uint64_t data, 1622 unsigned size, 1623 MemTxAttrs attrs) 1624 { 1625 MemoryRegionIoeventfd ioeventfd = { 1626 .addr = addrrange_make(int128_make64(addr), int128_make64(size)), 1627 .data = data, 1628 }; 1629 unsigned i; 1630 1631 for (i = 0; i < mr->ioeventfd_nb; i++) { 1632 ioeventfd.match_data = mr->ioeventfds[i].match_data; 1633 ioeventfd.e = mr->ioeventfds[i].e; 1634 1635 if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) { 1636 event_notifier_set(ioeventfd.e); 1637 return true; 1638 } 1639 } 1640 1641 return false; 1642 } 1643 1644 MemTxResult memory_region_dispatch_write(MemoryRegion *mr, 1645 hwaddr addr, 1646 uint64_t data, 1647 MemOp op, 1648 MemTxAttrs attrs) 1649 { 1650 unsigned size = memop_size(op); 1651 1652 if (mr->alias) { 1653 return memory_region_dispatch_write(mr->alias, 1654 mr->alias_offset + addr, 1655 data, op, attrs); 1656 } 1657 if (!memory_region_access_valid(mr, addr, size, true, attrs)) { 1658 unassigned_mem_write(mr, addr, data, size); 1659 return MEMTX_DECODE_ERROR; 1660 } 1661 1662 adjust_endianness(mr, &data, op); 1663 1664 /* 1665 * FIXME: it's not clear why under KVM the write would be processed 1666 * directly, instead of going through eventfd. This probably should 1667 * test "tcg_enabled() || qtest_enabled()", or should just go away. 1668 */ 1669 if (!kvm_enabled() && 1670 memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) { 1671 return MEMTX_OK; 1672 } 1673 1674 if (mr->ops->write) { 1675 return access_with_adjusted_size_aligned(addr, &data, size, 1676 mr->ops->impl.min_access_size, 1677 mr->ops->impl.max_access_size, 1678 memory_region_write_accessor, mr, 1679 attrs); 1680 } else { 1681 return 1682 access_with_adjusted_size_aligned(addr, &data, size, 1683 mr->ops->impl.min_access_size, 1684 mr->ops->impl.max_access_size, 1685 memory_region_write_with_attrs_accessor, 1686 mr, attrs); 1687 } 1688 } 1689 1690 void memory_region_init_io(MemoryRegion *mr, 1691 Object *owner, 1692 const MemoryRegionOps *ops, 1693 void *opaque, 1694 const char *name, 1695 uint64_t size) 1696 { 1697 memory_region_init(mr, owner, name, size); 1698 mr->ops = ops ? ops : &unassigned_mem_ops; 1699 mr->opaque = opaque; 1700 mr->terminates = true; 1701 } 1702 1703 bool memory_region_init_ram_nomigrate(MemoryRegion *mr, 1704 Object *owner, 1705 const char *name, 1706 uint64_t size, 1707 Error **errp) 1708 { 1709 return memory_region_init_ram_flags_nomigrate(mr, owner, name, 1710 size, 0, errp); 1711 } 1712 1713 bool memory_region_init_ram_flags_nomigrate(MemoryRegion *mr, 1714 Object *owner, 1715 const char *name, 1716 uint64_t size, 1717 uint32_t ram_flags, 1718 Error **errp) 1719 { 1720 Error *err = NULL; 1721 memory_region_init(mr, owner, name, size); 1722 mr->ram = true; 1723 mr->terminates = true; 1724 mr->destructor = memory_region_destructor_ram; 1725 mr->ram_block = qemu_ram_alloc(size, ram_flags, mr, &err); 1726 if (err) { 1727 mr->size = int128_zero(); 1728 object_unparent(OBJECT(mr)); 1729 error_propagate(errp, err); 1730 return false; 1731 } 1732 return true; 1733 } 1734 1735 bool memory_region_init_resizeable_ram(MemoryRegion *mr, 1736 Object *owner, 1737 const char *name, 1738 uint64_t size, 1739 uint64_t max_size, 1740 void (*resized)(const char*, 1741 uint64_t length, 1742 void *host), 1743 Error **errp) 1744 { 1745 Error *err = NULL; 1746 memory_region_init(mr, owner, name, size); 1747 mr->ram = true; 1748 mr->terminates = true; 1749 mr->destructor = memory_region_destructor_ram; 1750 mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized, 1751 mr, &err); 1752 if (err) { 1753 mr->size = int128_zero(); 1754 object_unparent(OBJECT(mr)); 1755 error_propagate(errp, err); 1756 return false; 1757 } 1758 return true; 1759 } 1760 1761 #if defined(CONFIG_POSIX) && !defined(EMSCRIPTEN) 1762 bool memory_region_init_ram_from_file(MemoryRegion *mr, 1763 Object *owner, 1764 const char *name, 1765 uint64_t size, 1766 uint64_t align, 1767 uint32_t ram_flags, 1768 const char *path, 1769 ram_addr_t offset, 1770 Error **errp) 1771 { 1772 Error *err = NULL; 1773 memory_region_init(mr, owner, name, size); 1774 mr->ram = true; 1775 mr->readonly = !!(ram_flags & RAM_READONLY); 1776 mr->terminates = true; 1777 mr->destructor = memory_region_destructor_ram; 1778 mr->align = align; 1779 mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, 1780 offset, &err); 1781 if (err) { 1782 mr->size = int128_zero(); 1783 object_unparent(OBJECT(mr)); 1784 error_propagate(errp, err); 1785 return false; 1786 } 1787 return true; 1788 } 1789 1790 bool memory_region_init_ram_from_fd(MemoryRegion *mr, 1791 Object *owner, 1792 const char *name, 1793 uint64_t size, 1794 uint32_t ram_flags, 1795 int fd, 1796 ram_addr_t offset, 1797 Error **errp) 1798 { 1799 Error *err = NULL; 1800 memory_region_init(mr, owner, name, size); 1801 mr->ram = true; 1802 mr->readonly = !!(ram_flags & RAM_READONLY); 1803 mr->terminates = true; 1804 mr->destructor = memory_region_destructor_ram; 1805 mr->ram_block = qemu_ram_alloc_from_fd(size, size, NULL, mr, ram_flags, fd, 1806 offset, false, &err); 1807 if (err) { 1808 mr->size = int128_zero(); 1809 object_unparent(OBJECT(mr)); 1810 error_propagate(errp, err); 1811 return false; 1812 } 1813 return true; 1814 } 1815 #endif 1816 1817 void memory_region_init_ram_ptr(MemoryRegion *mr, 1818 Object *owner, 1819 const char *name, 1820 uint64_t size, 1821 void *ptr) 1822 { 1823 memory_region_init(mr, owner, name, size); 1824 mr->ram = true; 1825 mr->terminates = true; 1826 mr->destructor = memory_region_destructor_ram; 1827 1828 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ 1829 assert(ptr != NULL); 1830 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); 1831 } 1832 1833 void memory_region_init_ram_device_ptr(MemoryRegion *mr, 1834 Object *owner, 1835 const char *name, 1836 uint64_t size, 1837 void *ptr) 1838 { 1839 memory_region_init(mr, owner, name, size); 1840 mr->ram = true; 1841 mr->terminates = true; 1842 mr->ram_device = true; 1843 mr->ops = &ram_device_mem_ops; 1844 mr->opaque = mr; 1845 mr->destructor = memory_region_destructor_ram; 1846 1847 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ 1848 assert(ptr != NULL); 1849 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); 1850 } 1851 1852 void memory_region_init_alias(MemoryRegion *mr, 1853 Object *owner, 1854 const char *name, 1855 MemoryRegion *orig, 1856 hwaddr offset, 1857 uint64_t size) 1858 { 1859 memory_region_init(mr, owner, name, size); 1860 mr->alias = orig; 1861 mr->alias_offset = offset; 1862 } 1863 1864 bool memory_region_init_rom_nomigrate(MemoryRegion *mr, 1865 Object *owner, 1866 const char *name, 1867 uint64_t size, 1868 Error **errp) 1869 { 1870 if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, 1871 size, 0, errp)) { 1872 return false; 1873 } 1874 mr->readonly = true; 1875 1876 return true; 1877 } 1878 1879 bool memory_region_init_rom_device_nomigrate(MemoryRegion *mr, 1880 Object *owner, 1881 const MemoryRegionOps *ops, 1882 void *opaque, 1883 const char *name, 1884 uint64_t size, 1885 Error **errp) 1886 { 1887 Error *err = NULL; 1888 assert(ops); 1889 memory_region_init(mr, owner, name, size); 1890 mr->ops = ops; 1891 mr->opaque = opaque; 1892 mr->terminates = true; 1893 mr->rom_device = true; 1894 mr->destructor = memory_region_destructor_ram; 1895 mr->ram_block = qemu_ram_alloc(size, 0, mr, &err); 1896 if (err) { 1897 mr->size = int128_zero(); 1898 object_unparent(OBJECT(mr)); 1899 error_propagate(errp, err); 1900 return false; 1901 } 1902 return true; 1903 } 1904 1905 void memory_region_init_iommu(void *_iommu_mr, 1906 size_t instance_size, 1907 const char *mrtypename, 1908 Object *owner, 1909 const char *name, 1910 uint64_t size) 1911 { 1912 struct IOMMUMemoryRegion *iommu_mr; 1913 struct MemoryRegion *mr; 1914 1915 object_initialize(_iommu_mr, instance_size, mrtypename); 1916 mr = MEMORY_REGION(_iommu_mr); 1917 memory_region_do_init(mr, owner, name, size); 1918 iommu_mr = IOMMU_MEMORY_REGION(mr); 1919 mr->terminates = true; /* then re-forwards */ 1920 QLIST_INIT(&iommu_mr->iommu_notify); 1921 iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE; 1922 } 1923 1924 static void memory_region_finalize(Object *obj) 1925 { 1926 MemoryRegion *mr = MEMORY_REGION(obj); 1927 1928 assert(!mr->container); 1929 1930 /* We know the region is not visible in any address space (it 1931 * does not have a container and cannot be a root either because 1932 * it has no references, so we can blindly clear mr->enabled. 1933 * memory_region_set_enabled instead could trigger a transaction 1934 * and cause an infinite loop. 1935 */ 1936 mr->enabled = false; 1937 memory_region_transaction_begin(); 1938 while (!QTAILQ_EMPTY(&mr->subregions)) { 1939 MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions); 1940 memory_region_del_subregion(mr, subregion); 1941 } 1942 memory_region_transaction_commit(); 1943 1944 mr->destructor(mr); 1945 memory_region_clear_coalescing(mr); 1946 g_free((char *)mr->name); 1947 g_free(mr->ioeventfds); 1948 } 1949 1950 Object *memory_region_owner(MemoryRegion *mr) 1951 { 1952 Object *obj = OBJECT(mr); 1953 return obj->parent; 1954 } 1955 1956 void memory_region_ref(MemoryRegion *mr) 1957 { 1958 /* MMIO callbacks most likely will access data that belongs 1959 * to the owner, hence the need to ref/unref the owner whenever 1960 * the memory region is in use. 1961 * 1962 * The memory region is a child of its owner. As long as the 1963 * owner doesn't call unparent itself on the memory region, 1964 * ref-ing the owner will also keep the memory region alive. 1965 * Memory regions without an owner are supposed to never go away; 1966 * we do not ref/unref them because it slows down DMA sensibly. 1967 */ 1968 if (mr && mr->owner) { 1969 object_ref(mr->owner); 1970 } 1971 } 1972 1973 void memory_region_unref(MemoryRegion *mr) 1974 { 1975 if (mr && mr->owner) { 1976 object_unref(mr->owner); 1977 } 1978 } 1979 1980 uint64_t memory_region_size(MemoryRegion *mr) 1981 { 1982 if (int128_eq(mr->size, int128_2_64())) { 1983 return UINT64_MAX; 1984 } 1985 return int128_get64(mr->size); 1986 } 1987 1988 const char *memory_region_name(const MemoryRegion *mr) 1989 { 1990 if (!mr->name) { 1991 ((MemoryRegion *)mr)->name = 1992 g_strdup(object_get_canonical_path_component(OBJECT(mr))); 1993 } 1994 return mr->name; 1995 } 1996 1997 bool memory_region_is_ram_device(MemoryRegion *mr) 1998 { 1999 return mr->ram_device; 2000 } 2001 2002 bool memory_region_is_protected(MemoryRegion *mr) 2003 { 2004 return mr->ram && (mr->ram_block->flags & RAM_PROTECTED); 2005 } 2006 2007 bool memory_region_has_guest_memfd(MemoryRegion *mr) 2008 { 2009 return mr->ram_block && mr->ram_block->guest_memfd >= 0; 2010 } 2011 2012 uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr) 2013 { 2014 uint8_t mask = mr->dirty_log_mask; 2015 RAMBlock *rb = mr->ram_block; 2016 2017 if (global_dirty_tracking && ((rb && qemu_ram_is_migratable(rb)) || 2018 memory_region_is_iommu(mr))) { 2019 mask |= (1 << DIRTY_MEMORY_MIGRATION); 2020 } 2021 2022 if (tcg_enabled() && rb) { 2023 /* TCG only cares about dirty memory logging for RAM, not IOMMU. */ 2024 mask |= (1 << DIRTY_MEMORY_CODE); 2025 } 2026 return mask; 2027 } 2028 2029 bool memory_region_is_logging(MemoryRegion *mr, uint8_t client) 2030 { 2031 return memory_region_get_dirty_log_mask(mr) & (1 << client); 2032 } 2033 2034 static int memory_region_update_iommu_notify_flags(IOMMUMemoryRegion *iommu_mr, 2035 Error **errp) 2036 { 2037 IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE; 2038 IOMMUNotifier *iommu_notifier; 2039 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2040 int ret = 0; 2041 2042 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) { 2043 flags |= iommu_notifier->notifier_flags; 2044 } 2045 2046 if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) { 2047 ret = imrc->notify_flag_changed(iommu_mr, 2048 iommu_mr->iommu_notify_flags, 2049 flags, errp); 2050 } 2051 2052 if (!ret) { 2053 iommu_mr->iommu_notify_flags = flags; 2054 } 2055 return ret; 2056 } 2057 2058 int memory_region_register_iommu_notifier(MemoryRegion *mr, 2059 IOMMUNotifier *n, Error **errp) 2060 { 2061 IOMMUMemoryRegion *iommu_mr; 2062 int ret; 2063 2064 if (mr->alias) { 2065 return memory_region_register_iommu_notifier(mr->alias, n, errp); 2066 } 2067 2068 /* We need to register for at least one bitfield */ 2069 iommu_mr = IOMMU_MEMORY_REGION(mr); 2070 assert(n->notifier_flags != IOMMU_NOTIFIER_NONE); 2071 assert(n->start <= n->end); 2072 assert(n->iommu_idx >= 0 && 2073 n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr)); 2074 2075 QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node); 2076 ret = memory_region_update_iommu_notify_flags(iommu_mr, errp); 2077 if (ret) { 2078 QLIST_REMOVE(n, node); 2079 } 2080 return ret; 2081 } 2082 2083 uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr) 2084 { 2085 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2086 2087 if (imrc->get_min_page_size) { 2088 return imrc->get_min_page_size(iommu_mr); 2089 } 2090 return TARGET_PAGE_SIZE; 2091 } 2092 2093 void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 2094 { 2095 MemoryRegion *mr = MEMORY_REGION(iommu_mr); 2096 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2097 hwaddr addr, granularity; 2098 IOMMUTLBEntry iotlb; 2099 2100 /* If the IOMMU has its own replay callback, override */ 2101 if (imrc->replay) { 2102 imrc->replay(iommu_mr, n); 2103 return; 2104 } 2105 2106 granularity = memory_region_iommu_get_min_page_size(iommu_mr); 2107 2108 for (addr = 0; addr < memory_region_size(mr); addr += granularity) { 2109 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx); 2110 if (iotlb.perm != IOMMU_NONE) { 2111 n->notify(n, &iotlb); 2112 } 2113 2114 /* if (2^64 - MR size) < granularity, it's possible to get an 2115 * infinite loop here. This should catch such a wraparound */ 2116 if ((addr + granularity) < addr) { 2117 break; 2118 } 2119 } 2120 } 2121 2122 void memory_region_unregister_iommu_notifier(MemoryRegion *mr, 2123 IOMMUNotifier *n) 2124 { 2125 IOMMUMemoryRegion *iommu_mr; 2126 2127 if (mr->alias) { 2128 memory_region_unregister_iommu_notifier(mr->alias, n); 2129 return; 2130 } 2131 QLIST_REMOVE(n, node); 2132 iommu_mr = IOMMU_MEMORY_REGION(mr); 2133 memory_region_update_iommu_notify_flags(iommu_mr, NULL); 2134 } 2135 2136 void memory_region_notify_iommu_one(IOMMUNotifier *notifier, 2137 const IOMMUTLBEvent *event) 2138 { 2139 const IOMMUTLBEntry *entry = &event->entry; 2140 hwaddr entry_end = entry->iova + entry->addr_mask; 2141 IOMMUTLBEntry tmp = *entry; 2142 2143 if (event->type == IOMMU_NOTIFIER_UNMAP) { 2144 assert(entry->perm == IOMMU_NONE); 2145 } 2146 2147 /* 2148 * Skip the notification if the notification does not overlap 2149 * with registered range. 2150 */ 2151 if (notifier->start > entry_end || notifier->end < entry->iova) { 2152 return; 2153 } 2154 2155 if (notifier->notifier_flags & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 2156 /* Crop (iova, addr_mask) to range */ 2157 tmp.iova = MAX(tmp.iova, notifier->start); 2158 tmp.addr_mask = MIN(entry_end, notifier->end) - tmp.iova; 2159 } else { 2160 assert(entry->iova >= notifier->start && entry_end <= notifier->end); 2161 } 2162 2163 if (event->type & notifier->notifier_flags) { 2164 notifier->notify(notifier, &tmp); 2165 } 2166 } 2167 2168 void memory_region_unmap_iommu_notifier_range(IOMMUNotifier *notifier) 2169 { 2170 IOMMUTLBEvent event; 2171 2172 event.type = IOMMU_NOTIFIER_UNMAP; 2173 event.entry.target_as = &address_space_memory; 2174 event.entry.iova = notifier->start; 2175 event.entry.perm = IOMMU_NONE; 2176 event.entry.addr_mask = notifier->end - notifier->start; 2177 2178 memory_region_notify_iommu_one(notifier, &event); 2179 } 2180 2181 void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr, 2182 int iommu_idx, 2183 const IOMMUTLBEvent event) 2184 { 2185 IOMMUNotifier *iommu_notifier; 2186 2187 assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr))); 2188 2189 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) { 2190 if (iommu_notifier->iommu_idx == iommu_idx) { 2191 memory_region_notify_iommu_one(iommu_notifier, &event); 2192 } 2193 } 2194 } 2195 2196 int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr, 2197 enum IOMMUMemoryRegionAttr attr, 2198 void *data) 2199 { 2200 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2201 2202 if (!imrc->get_attr) { 2203 return -EINVAL; 2204 } 2205 2206 return imrc->get_attr(iommu_mr, attr, data); 2207 } 2208 2209 int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr, 2210 MemTxAttrs attrs) 2211 { 2212 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2213 2214 if (!imrc->attrs_to_index) { 2215 return 0; 2216 } 2217 2218 return imrc->attrs_to_index(iommu_mr, attrs); 2219 } 2220 2221 int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr) 2222 { 2223 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); 2224 2225 if (!imrc->num_indexes) { 2226 return 1; 2227 } 2228 2229 return imrc->num_indexes(iommu_mr); 2230 } 2231 2232 RamDiscardManager *memory_region_get_ram_discard_manager(MemoryRegion *mr) 2233 { 2234 if (!memory_region_is_ram(mr)) { 2235 return NULL; 2236 } 2237 return mr->rdm; 2238 } 2239 2240 int memory_region_set_ram_discard_manager(MemoryRegion *mr, 2241 RamDiscardManager *rdm) 2242 { 2243 g_assert(memory_region_is_ram(mr)); 2244 if (mr->rdm && rdm) { 2245 return -EBUSY; 2246 } 2247 2248 mr->rdm = rdm; 2249 return 0; 2250 } 2251 2252 uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm, 2253 const MemoryRegion *mr) 2254 { 2255 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2256 2257 g_assert(rdmc->get_min_granularity); 2258 return rdmc->get_min_granularity(rdm, mr); 2259 } 2260 2261 bool ram_discard_manager_is_populated(const RamDiscardManager *rdm, 2262 const MemoryRegionSection *section) 2263 { 2264 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2265 2266 g_assert(rdmc->is_populated); 2267 return rdmc->is_populated(rdm, section); 2268 } 2269 2270 int ram_discard_manager_replay_populated(const RamDiscardManager *rdm, 2271 MemoryRegionSection *section, 2272 ReplayRamDiscardState replay_fn, 2273 void *opaque) 2274 { 2275 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2276 2277 g_assert(rdmc->replay_populated); 2278 return rdmc->replay_populated(rdm, section, replay_fn, opaque); 2279 } 2280 2281 int ram_discard_manager_replay_discarded(const RamDiscardManager *rdm, 2282 MemoryRegionSection *section, 2283 ReplayRamDiscardState replay_fn, 2284 void *opaque) 2285 { 2286 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2287 2288 g_assert(rdmc->replay_discarded); 2289 return rdmc->replay_discarded(rdm, section, replay_fn, opaque); 2290 } 2291 2292 void ram_discard_manager_register_listener(RamDiscardManager *rdm, 2293 RamDiscardListener *rdl, 2294 MemoryRegionSection *section) 2295 { 2296 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2297 2298 g_assert(rdmc->register_listener); 2299 rdmc->register_listener(rdm, rdl, section); 2300 } 2301 2302 void ram_discard_manager_unregister_listener(RamDiscardManager *rdm, 2303 RamDiscardListener *rdl) 2304 { 2305 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm); 2306 2307 g_assert(rdmc->unregister_listener); 2308 rdmc->unregister_listener(rdm, rdl); 2309 } 2310 2311 /* Called with rcu_read_lock held. */ 2312 MemoryRegion *memory_translate_iotlb(IOMMUTLBEntry *iotlb, hwaddr *xlat_p, 2313 Error **errp) 2314 { 2315 MemoryRegion *mr; 2316 hwaddr xlat; 2317 hwaddr len = iotlb->addr_mask + 1; 2318 bool writable = iotlb->perm & IOMMU_WO; 2319 2320 /* 2321 * The IOMMU TLB entry we have just covers translation through 2322 * this IOMMU to its immediate target. We need to translate 2323 * it the rest of the way through to memory. 2324 */ 2325 mr = address_space_translate(&address_space_memory, iotlb->translated_addr, 2326 &xlat, &len, writable, MEMTXATTRS_UNSPECIFIED); 2327 if (!memory_region_is_ram(mr)) { 2328 error_setg(errp, "iommu map to non memory area %" HWADDR_PRIx "", xlat); 2329 return NULL; 2330 } else if (memory_region_has_ram_discard_manager(mr)) { 2331 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr); 2332 MemoryRegionSection tmp = { 2333 .mr = mr, 2334 .offset_within_region = xlat, 2335 .size = int128_make64(len), 2336 }; 2337 /* 2338 * Malicious VMs can map memory into the IOMMU, which is expected 2339 * to remain discarded. vfio will pin all pages, populating memory. 2340 * Disallow that. vmstate priorities make sure any RamDiscardManager 2341 * were already restored before IOMMUs are restored. 2342 */ 2343 if (!ram_discard_manager_is_populated(rdm, &tmp)) { 2344 error_setg(errp, "iommu map to discarded memory (e.g., unplugged" 2345 " via virtio-mem): %" HWADDR_PRIx "", 2346 iotlb->translated_addr); 2347 return NULL; 2348 } 2349 } 2350 2351 /* 2352 * Translation truncates length to the IOMMU page size, 2353 * check that it did not truncate too much. 2354 */ 2355 if (len & iotlb->addr_mask) { 2356 error_setg(errp, "iommu has granularity incompatible with target AS"); 2357 return NULL; 2358 } 2359 2360 *xlat_p = xlat; 2361 return mr; 2362 } 2363 2364 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) 2365 { 2366 uint8_t mask = 1 << client; 2367 uint8_t old_logging; 2368 2369 assert(client == DIRTY_MEMORY_VGA); 2370 old_logging = mr->vga_logging_count; 2371 mr->vga_logging_count += log ? 1 : -1; 2372 if (!!old_logging == !!mr->vga_logging_count) { 2373 return; 2374 } 2375 2376 memory_region_transaction_begin(); 2377 mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); 2378 memory_region_update_pending |= mr->enabled; 2379 memory_region_transaction_commit(); 2380 } 2381 2382 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, 2383 hwaddr size) 2384 { 2385 assert(mr->ram_block); 2386 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, 2387 size, 2388 memory_region_get_dirty_log_mask(mr)); 2389 } 2390 2391 /* 2392 * If memory region `mr' is NULL, do global sync. Otherwise, sync 2393 * dirty bitmap for the specified memory region. 2394 */ 2395 static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage) 2396 { 2397 MemoryListener *listener; 2398 AddressSpace *as; 2399 FlatView *view; 2400 FlatRange *fr; 2401 2402 /* If the same address space has multiple log_sync listeners, we 2403 * visit that address space's FlatView multiple times. But because 2404 * log_sync listeners are rare, it's still cheaper than walking each 2405 * address space once. 2406 */ 2407 QTAILQ_FOREACH(listener, &memory_listeners, link) { 2408 if (listener->log_sync) { 2409 as = listener->address_space; 2410 view = address_space_get_flatview(as); 2411 FOR_EACH_FLAT_RANGE(fr, view) { 2412 if (fr->dirty_log_mask && (!mr || fr->mr == mr)) { 2413 MemoryRegionSection mrs = section_from_flat_range(fr, view); 2414 listener->log_sync(listener, &mrs); 2415 } 2416 } 2417 flatview_unref(view); 2418 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 0); 2419 } else if (listener->log_sync_global) { 2420 /* 2421 * No matter whether MR is specified, what we can do here 2422 * is to do a global sync, because we are not capable to 2423 * sync in a finer granularity. 2424 */ 2425 listener->log_sync_global(listener, last_stage); 2426 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1); 2427 } 2428 } 2429 } 2430 2431 void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start, 2432 hwaddr len) 2433 { 2434 MemoryRegionSection mrs; 2435 MemoryListener *listener; 2436 AddressSpace *as; 2437 FlatView *view; 2438 FlatRange *fr; 2439 hwaddr sec_start, sec_end, sec_size; 2440 2441 QTAILQ_FOREACH(listener, &memory_listeners, link) { 2442 if (!listener->log_clear) { 2443 continue; 2444 } 2445 as = listener->address_space; 2446 view = address_space_get_flatview(as); 2447 FOR_EACH_FLAT_RANGE(fr, view) { 2448 if (!fr->dirty_log_mask || fr->mr != mr) { 2449 /* 2450 * Clear dirty bitmap operation only applies to those 2451 * regions whose dirty logging is at least enabled 2452 */ 2453 continue; 2454 } 2455 2456 mrs = section_from_flat_range(fr, view); 2457 2458 sec_start = MAX(mrs.offset_within_region, start); 2459 sec_end = mrs.offset_within_region + int128_get64(mrs.size); 2460 sec_end = MIN(sec_end, start + len); 2461 2462 if (sec_start >= sec_end) { 2463 /* 2464 * If this memory region section has no intersection 2465 * with the requested range, skip. 2466 */ 2467 continue; 2468 } 2469 2470 /* Valid case; shrink the section if needed */ 2471 mrs.offset_within_address_space += 2472 sec_start - mrs.offset_within_region; 2473 mrs.offset_within_region = sec_start; 2474 sec_size = sec_end - sec_start; 2475 mrs.size = int128_make64(sec_size); 2476 listener->log_clear(listener, &mrs); 2477 } 2478 flatview_unref(view); 2479 } 2480 } 2481 2482 DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, 2483 hwaddr addr, 2484 hwaddr size, 2485 unsigned client) 2486 { 2487 DirtyBitmapSnapshot *snapshot; 2488 assert(mr->ram_block); 2489 memory_region_sync_dirty_bitmap(mr, false); 2490 snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); 2491 memory_global_after_dirty_log_sync(); 2492 return snapshot; 2493 } 2494 2495 bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap, 2496 hwaddr addr, hwaddr size) 2497 { 2498 assert(mr->ram_block); 2499 return cpu_physical_memory_snapshot_get_dirty(snap, 2500 memory_region_get_ram_addr(mr) + addr, size); 2501 } 2502 2503 void memory_region_set_readonly(MemoryRegion *mr, bool readonly) 2504 { 2505 if (mr->readonly != readonly) { 2506 memory_region_transaction_begin(); 2507 mr->readonly = readonly; 2508 memory_region_update_pending |= mr->enabled; 2509 memory_region_transaction_commit(); 2510 } 2511 } 2512 2513 void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile) 2514 { 2515 if (mr->nonvolatile != nonvolatile) { 2516 memory_region_transaction_begin(); 2517 mr->nonvolatile = nonvolatile; 2518 memory_region_update_pending |= mr->enabled; 2519 memory_region_transaction_commit(); 2520 } 2521 } 2522 2523 void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode) 2524 { 2525 if (mr->romd_mode != romd_mode) { 2526 memory_region_transaction_begin(); 2527 mr->romd_mode = romd_mode; 2528 memory_region_update_pending |= mr->enabled; 2529 memory_region_transaction_commit(); 2530 } 2531 } 2532 2533 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, 2534 hwaddr size, unsigned client) 2535 { 2536 assert(mr->ram_block); 2537 cpu_physical_memory_test_and_clear_dirty( 2538 memory_region_get_ram_addr(mr) + addr, size, client); 2539 } 2540 2541 int memory_region_get_fd(MemoryRegion *mr) 2542 { 2543 RCU_READ_LOCK_GUARD(); 2544 while (mr->alias) { 2545 mr = mr->alias; 2546 } 2547 return mr->ram_block->fd; 2548 } 2549 2550 void *memory_region_get_ram_ptr(MemoryRegion *mr) 2551 { 2552 uint64_t offset = 0; 2553 2554 RCU_READ_LOCK_GUARD(); 2555 while (mr->alias) { 2556 offset += mr->alias_offset; 2557 mr = mr->alias; 2558 } 2559 assert(mr->ram_block); 2560 return qemu_map_ram_ptr(mr->ram_block, offset); 2561 } 2562 2563 MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset) 2564 { 2565 RAMBlock *block; 2566 2567 block = qemu_ram_block_from_host(ptr, false, offset); 2568 if (!block) { 2569 return NULL; 2570 } 2571 2572 return block->mr; 2573 } 2574 2575 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr) 2576 { 2577 return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID; 2578 } 2579 2580 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp) 2581 { 2582 assert(mr->ram_block); 2583 2584 qemu_ram_resize(mr->ram_block, newsize, errp); 2585 } 2586 2587 void memory_region_msync(MemoryRegion *mr, hwaddr addr, hwaddr size) 2588 { 2589 if (mr->ram_block) { 2590 qemu_ram_msync(mr->ram_block, addr, size); 2591 } 2592 } 2593 2594 void memory_region_writeback(MemoryRegion *mr, hwaddr addr, hwaddr size) 2595 { 2596 /* 2597 * Might be extended case needed to cover 2598 * different types of memory regions 2599 */ 2600 if (mr->dirty_log_mask) { 2601 memory_region_msync(mr, addr, size); 2602 } 2603 } 2604 2605 /* 2606 * Call proper memory listeners about the change on the newly 2607 * added/removed CoalescedMemoryRange. 2608 */ 2609 static void memory_region_update_coalesced_range(MemoryRegion *mr, 2610 CoalescedMemoryRange *cmr, 2611 bool add) 2612 { 2613 AddressSpace *as; 2614 FlatView *view; 2615 FlatRange *fr; 2616 2617 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 2618 view = address_space_get_flatview(as); 2619 FOR_EACH_FLAT_RANGE(fr, view) { 2620 if (fr->mr == mr) { 2621 flat_range_coalesced_io_notify(fr, as, cmr, add); 2622 } 2623 } 2624 flatview_unref(view); 2625 } 2626 } 2627 2628 void memory_region_set_coalescing(MemoryRegion *mr) 2629 { 2630 memory_region_clear_coalescing(mr); 2631 memory_region_add_coalescing(mr, 0, int128_get64(mr->size)); 2632 } 2633 2634 void memory_region_add_coalescing(MemoryRegion *mr, 2635 hwaddr offset, 2636 uint64_t size) 2637 { 2638 CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr)); 2639 2640 cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size)); 2641 QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link); 2642 memory_region_update_coalesced_range(mr, cmr, true); 2643 memory_region_set_flush_coalesced(mr); 2644 } 2645 2646 void memory_region_clear_coalescing(MemoryRegion *mr) 2647 { 2648 CoalescedMemoryRange *cmr; 2649 2650 if (QTAILQ_EMPTY(&mr->coalesced)) { 2651 return; 2652 } 2653 2654 qemu_flush_coalesced_mmio_buffer(); 2655 mr->flush_coalesced_mmio = false; 2656 2657 while (!QTAILQ_EMPTY(&mr->coalesced)) { 2658 cmr = QTAILQ_FIRST(&mr->coalesced); 2659 QTAILQ_REMOVE(&mr->coalesced, cmr, link); 2660 memory_region_update_coalesced_range(mr, cmr, false); 2661 g_free(cmr); 2662 } 2663 } 2664 2665 void memory_region_set_flush_coalesced(MemoryRegion *mr) 2666 { 2667 mr->flush_coalesced_mmio = true; 2668 } 2669 2670 void memory_region_clear_flush_coalesced(MemoryRegion *mr) 2671 { 2672 qemu_flush_coalesced_mmio_buffer(); 2673 if (QTAILQ_EMPTY(&mr->coalesced)) { 2674 mr->flush_coalesced_mmio = false; 2675 } 2676 } 2677 2678 void memory_region_add_eventfd(MemoryRegion *mr, 2679 hwaddr addr, 2680 unsigned size, 2681 bool match_data, 2682 uint64_t data, 2683 EventNotifier *e) 2684 { 2685 MemoryRegionIoeventfd mrfd = { 2686 .addr.start = int128_make64(addr), 2687 .addr.size = int128_make64(size), 2688 .match_data = match_data, 2689 .data = data, 2690 .e = e, 2691 }; 2692 unsigned i; 2693 2694 if (size) { 2695 MemOp mop = (target_big_endian() ? MO_BE : MO_LE) | size_memop(size); 2696 adjust_endianness(mr, &mrfd.data, mop); 2697 } 2698 memory_region_transaction_begin(); 2699 for (i = 0; i < mr->ioeventfd_nb; ++i) { 2700 if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) { 2701 break; 2702 } 2703 } 2704 ++mr->ioeventfd_nb; 2705 mr->ioeventfds = g_realloc(mr->ioeventfds, 2706 sizeof(*mr->ioeventfds) * mr->ioeventfd_nb); 2707 memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], 2708 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); 2709 mr->ioeventfds[i] = mrfd; 2710 ioeventfd_update_pending |= mr->enabled; 2711 memory_region_transaction_commit(); 2712 } 2713 2714 void memory_region_del_eventfd(MemoryRegion *mr, 2715 hwaddr addr, 2716 unsigned size, 2717 bool match_data, 2718 uint64_t data, 2719 EventNotifier *e) 2720 { 2721 MemoryRegionIoeventfd mrfd = { 2722 .addr.start = int128_make64(addr), 2723 .addr.size = int128_make64(size), 2724 .match_data = match_data, 2725 .data = data, 2726 .e = e, 2727 }; 2728 unsigned i; 2729 2730 if (size) { 2731 MemOp mop = (target_big_endian() ? MO_BE : MO_LE) | size_memop(size); 2732 adjust_endianness(mr, &mrfd.data, mop); 2733 } 2734 memory_region_transaction_begin(); 2735 for (i = 0; i < mr->ioeventfd_nb; ++i) { 2736 if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) { 2737 break; 2738 } 2739 } 2740 assert(i != mr->ioeventfd_nb); 2741 memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1], 2742 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1))); 2743 --mr->ioeventfd_nb; 2744 mr->ioeventfds = g_realloc(mr->ioeventfds, 2745 sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); 2746 ioeventfd_update_pending |= mr->enabled; 2747 memory_region_transaction_commit(); 2748 } 2749 2750 static void memory_region_update_container_subregions(MemoryRegion *subregion) 2751 { 2752 MemoryRegion *mr = subregion->container; 2753 MemoryRegion *other; 2754 2755 memory_region_transaction_begin(); 2756 2757 memory_region_ref(subregion); 2758 QTAILQ_FOREACH(other, &mr->subregions, subregions_link) { 2759 if (subregion->priority >= other->priority) { 2760 QTAILQ_INSERT_BEFORE(other, subregion, subregions_link); 2761 goto done; 2762 } 2763 } 2764 QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); 2765 done: 2766 memory_region_update_pending |= mr->enabled && subregion->enabled; 2767 memory_region_transaction_commit(); 2768 } 2769 2770 static void memory_region_add_subregion_common(MemoryRegion *mr, 2771 hwaddr offset, 2772 MemoryRegion *subregion) 2773 { 2774 MemoryRegion *alias; 2775 2776 assert(!subregion->container); 2777 subregion->container = mr; 2778 for (alias = subregion->alias; alias; alias = alias->alias) { 2779 alias->mapped_via_alias++; 2780 } 2781 subregion->addr = offset; 2782 memory_region_update_container_subregions(subregion); 2783 } 2784 2785 void memory_region_add_subregion(MemoryRegion *mr, 2786 hwaddr offset, 2787 MemoryRegion *subregion) 2788 { 2789 subregion->priority = 0; 2790 memory_region_add_subregion_common(mr, offset, subregion); 2791 } 2792 2793 void memory_region_add_subregion_overlap(MemoryRegion *mr, 2794 hwaddr offset, 2795 MemoryRegion *subregion, 2796 int priority) 2797 { 2798 subregion->priority = priority; 2799 memory_region_add_subregion_common(mr, offset, subregion); 2800 } 2801 2802 void memory_region_del_subregion(MemoryRegion *mr, 2803 MemoryRegion *subregion) 2804 { 2805 MemoryRegion *alias; 2806 2807 memory_region_transaction_begin(); 2808 assert(subregion->container == mr); 2809 subregion->container = NULL; 2810 for (alias = subregion->alias; alias; alias = alias->alias) { 2811 alias->mapped_via_alias--; 2812 assert(alias->mapped_via_alias >= 0); 2813 } 2814 QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); 2815 memory_region_unref(subregion); 2816 memory_region_update_pending |= mr->enabled && subregion->enabled; 2817 memory_region_transaction_commit(); 2818 } 2819 2820 void memory_region_set_enabled(MemoryRegion *mr, bool enabled) 2821 { 2822 if (enabled == mr->enabled) { 2823 return; 2824 } 2825 memory_region_transaction_begin(); 2826 mr->enabled = enabled; 2827 memory_region_update_pending = true; 2828 memory_region_transaction_commit(); 2829 } 2830 2831 void memory_region_set_size(MemoryRegion *mr, uint64_t size) 2832 { 2833 Int128 s = int128_make64(size); 2834 2835 if (size == UINT64_MAX) { 2836 s = int128_2_64(); 2837 } 2838 if (int128_eq(s, mr->size)) { 2839 return; 2840 } 2841 memory_region_transaction_begin(); 2842 mr->size = s; 2843 memory_region_update_pending = true; 2844 memory_region_transaction_commit(); 2845 } 2846 2847 static void memory_region_readd_subregion(MemoryRegion *mr) 2848 { 2849 MemoryRegion *container = mr->container; 2850 2851 if (container) { 2852 memory_region_transaction_begin(); 2853 memory_region_ref(mr); 2854 memory_region_del_subregion(container, mr); 2855 memory_region_add_subregion_common(container, mr->addr, mr); 2856 memory_region_unref(mr); 2857 memory_region_transaction_commit(); 2858 } 2859 } 2860 2861 void memory_region_set_address(MemoryRegion *mr, hwaddr addr) 2862 { 2863 if (addr != mr->addr) { 2864 mr->addr = addr; 2865 memory_region_readd_subregion(mr); 2866 } 2867 } 2868 2869 void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset) 2870 { 2871 assert(mr->alias); 2872 2873 if (offset == mr->alias_offset) { 2874 return; 2875 } 2876 2877 memory_region_transaction_begin(); 2878 mr->alias_offset = offset; 2879 memory_region_update_pending |= mr->enabled; 2880 memory_region_transaction_commit(); 2881 } 2882 2883 void memory_region_set_unmergeable(MemoryRegion *mr, bool unmergeable) 2884 { 2885 if (unmergeable == mr->unmergeable) { 2886 return; 2887 } 2888 2889 memory_region_transaction_begin(); 2890 mr->unmergeable = unmergeable; 2891 memory_region_update_pending |= mr->enabled; 2892 memory_region_transaction_commit(); 2893 } 2894 2895 uint64_t memory_region_get_alignment(const MemoryRegion *mr) 2896 { 2897 return mr->align; 2898 } 2899 2900 static int cmp_flatrange_addr(const void *addr_, const void *fr_) 2901 { 2902 const AddrRange *addr = addr_; 2903 const FlatRange *fr = fr_; 2904 2905 if (int128_le(addrrange_end(*addr), fr->addr.start)) { 2906 return -1; 2907 } else if (int128_ge(addr->start, addrrange_end(fr->addr))) { 2908 return 1; 2909 } 2910 return 0; 2911 } 2912 2913 static FlatRange *flatview_lookup(FlatView *view, AddrRange addr) 2914 { 2915 return bsearch(&addr, view->ranges, view->nr, 2916 sizeof(FlatRange), cmp_flatrange_addr); 2917 } 2918 2919 bool memory_region_is_mapped(MemoryRegion *mr) 2920 { 2921 return !!mr->container || mr->mapped_via_alias; 2922 } 2923 2924 /* Same as memory_region_find, but it does not add a reference to the 2925 * returned region. It must be called from an RCU critical section. 2926 */ 2927 static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr, 2928 hwaddr addr, uint64_t size) 2929 { 2930 MemoryRegionSection ret = { .mr = NULL }; 2931 MemoryRegion *root; 2932 AddressSpace *as; 2933 AddrRange range; 2934 FlatView *view; 2935 FlatRange *fr; 2936 2937 addr += mr->addr; 2938 for (root = mr; root->container; ) { 2939 root = root->container; 2940 addr += root->addr; 2941 } 2942 2943 as = memory_region_to_address_space(root); 2944 if (!as) { 2945 return ret; 2946 } 2947 range = addrrange_make(int128_make64(addr), int128_make64(size)); 2948 2949 view = address_space_to_flatview(as); 2950 fr = flatview_lookup(view, range); 2951 if (!fr) { 2952 return ret; 2953 } 2954 2955 while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) { 2956 --fr; 2957 } 2958 2959 ret.mr = fr->mr; 2960 ret.fv = view; 2961 range = addrrange_intersection(range, fr->addr); 2962 ret.offset_within_region = fr->offset_in_region; 2963 ret.offset_within_region += int128_get64(int128_sub(range.start, 2964 fr->addr.start)); 2965 ret.size = range.size; 2966 ret.offset_within_address_space = int128_get64(range.start); 2967 ret.readonly = fr->readonly; 2968 ret.nonvolatile = fr->nonvolatile; 2969 return ret; 2970 } 2971 2972 MemoryRegionSection memory_region_find(MemoryRegion *mr, 2973 hwaddr addr, uint64_t size) 2974 { 2975 MemoryRegionSection ret; 2976 RCU_READ_LOCK_GUARD(); 2977 ret = memory_region_find_rcu(mr, addr, size); 2978 if (ret.mr) { 2979 memory_region_ref(ret.mr); 2980 } 2981 return ret; 2982 } 2983 2984 MemoryRegionSection *memory_region_section_new_copy(MemoryRegionSection *s) 2985 { 2986 MemoryRegionSection *tmp = g_new(MemoryRegionSection, 1); 2987 2988 *tmp = *s; 2989 if (tmp->mr) { 2990 memory_region_ref(tmp->mr); 2991 } 2992 if (tmp->fv) { 2993 bool ret = flatview_ref(tmp->fv); 2994 2995 g_assert(ret); 2996 } 2997 return tmp; 2998 } 2999 3000 void memory_region_section_free_copy(MemoryRegionSection *s) 3001 { 3002 if (s->fv) { 3003 flatview_unref(s->fv); 3004 } 3005 if (s->mr) { 3006 memory_region_unref(s->mr); 3007 } 3008 g_free(s); 3009 } 3010 3011 bool memory_region_present(MemoryRegion *container, hwaddr addr) 3012 { 3013 MemoryRegion *mr; 3014 3015 RCU_READ_LOCK_GUARD(); 3016 mr = memory_region_find_rcu(container, addr, 1).mr; 3017 return mr && mr != container; 3018 } 3019 3020 void memory_global_dirty_log_sync(bool last_stage) 3021 { 3022 memory_region_sync_dirty_bitmap(NULL, last_stage); 3023 } 3024 3025 void memory_global_after_dirty_log_sync(void) 3026 { 3027 MEMORY_LISTENER_CALL_GLOBAL(log_global_after_sync, Forward); 3028 } 3029 3030 /* 3031 * Dirty track stop flags that are postponed due to VM being stopped. Should 3032 * only be used within vmstate_change hook. 3033 */ 3034 static unsigned int postponed_stop_flags; 3035 static VMChangeStateEntry *vmstate_change; 3036 static void memory_global_dirty_log_stop_postponed_run(void); 3037 3038 static bool memory_global_dirty_log_do_start(Error **errp) 3039 { 3040 MemoryListener *listener; 3041 3042 QTAILQ_FOREACH(listener, &memory_listeners, link) { 3043 if (listener->log_global_start) { 3044 if (!listener->log_global_start(listener, errp)) { 3045 goto err; 3046 } 3047 } 3048 } 3049 return true; 3050 3051 err: 3052 while ((listener = QTAILQ_PREV(listener, link)) != NULL) { 3053 if (listener->log_global_stop) { 3054 listener->log_global_stop(listener); 3055 } 3056 } 3057 3058 return false; 3059 } 3060 3061 bool memory_global_dirty_log_start(unsigned int flags, Error **errp) 3062 { 3063 unsigned int old_flags; 3064 3065 assert(flags && !(flags & (~GLOBAL_DIRTY_MASK))); 3066 3067 if (vmstate_change) { 3068 /* If there is postponed stop(), operate on it first */ 3069 postponed_stop_flags &= ~flags; 3070 memory_global_dirty_log_stop_postponed_run(); 3071 } 3072 3073 flags &= ~global_dirty_tracking; 3074 if (!flags) { 3075 return true; 3076 } 3077 3078 old_flags = global_dirty_tracking; 3079 global_dirty_tracking |= flags; 3080 trace_global_dirty_changed(global_dirty_tracking); 3081 3082 if (!old_flags) { 3083 if (!memory_global_dirty_log_do_start(errp)) { 3084 global_dirty_tracking &= ~flags; 3085 trace_global_dirty_changed(global_dirty_tracking); 3086 return false; 3087 } 3088 3089 memory_region_transaction_begin(); 3090 memory_region_update_pending = true; 3091 memory_region_transaction_commit(); 3092 } 3093 return true; 3094 } 3095 3096 static void memory_global_dirty_log_do_stop(unsigned int flags) 3097 { 3098 assert(flags && !(flags & (~GLOBAL_DIRTY_MASK))); 3099 assert((global_dirty_tracking & flags) == flags); 3100 global_dirty_tracking &= ~flags; 3101 3102 trace_global_dirty_changed(global_dirty_tracking); 3103 3104 if (!global_dirty_tracking) { 3105 memory_region_transaction_begin(); 3106 memory_region_update_pending = true; 3107 memory_region_transaction_commit(); 3108 MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); 3109 } 3110 } 3111 3112 /* 3113 * Execute the postponed dirty log stop operations if there is, then reset 3114 * everything (including the flags and the vmstate change hook). 3115 */ 3116 static void memory_global_dirty_log_stop_postponed_run(void) 3117 { 3118 /* This must be called with the vmstate handler registered */ 3119 assert(vmstate_change); 3120 3121 /* Note: postponed_stop_flags can be cleared in log start routine */ 3122 if (postponed_stop_flags) { 3123 memory_global_dirty_log_do_stop(postponed_stop_flags); 3124 postponed_stop_flags = 0; 3125 } 3126 3127 qemu_del_vm_change_state_handler(vmstate_change); 3128 vmstate_change = NULL; 3129 } 3130 3131 static void memory_vm_change_state_handler(void *opaque, bool running, 3132 RunState state) 3133 { 3134 if (running) { 3135 memory_global_dirty_log_stop_postponed_run(); 3136 } 3137 } 3138 3139 void memory_global_dirty_log_stop(unsigned int flags) 3140 { 3141 if (!runstate_is_running()) { 3142 /* Postpone the dirty log stop, e.g., to when VM starts again */ 3143 if (vmstate_change) { 3144 /* Batch with previous postponed flags */ 3145 postponed_stop_flags |= flags; 3146 } else { 3147 postponed_stop_flags = flags; 3148 vmstate_change = qemu_add_vm_change_state_handler( 3149 memory_vm_change_state_handler, NULL); 3150 } 3151 return; 3152 } 3153 3154 memory_global_dirty_log_do_stop(flags); 3155 } 3156 3157 static void listener_add_address_space(MemoryListener *listener, 3158 AddressSpace *as) 3159 { 3160 unsigned i; 3161 FlatView *view; 3162 FlatRange *fr; 3163 MemoryRegionIoeventfd *fd; 3164 3165 if (listener->begin) { 3166 listener->begin(listener); 3167 } 3168 if (global_dirty_tracking) { 3169 /* 3170 * Currently only VFIO can fail log_global_start(), and it's not 3171 * yet allowed to hotplug any PCI device during migration. So this 3172 * should never fail when invoked, guard it with error_abort. If 3173 * it can start to fail in the future, we need to be able to fail 3174 * the whole listener_add_address_space() and its callers. 3175 */ 3176 if (listener->log_global_start) { 3177 listener->log_global_start(listener, &error_abort); 3178 } 3179 } 3180 3181 view = address_space_get_flatview(as); 3182 FOR_EACH_FLAT_RANGE(fr, view) { 3183 MemoryRegionSection section = section_from_flat_range(fr, view); 3184 3185 if (listener->region_add) { 3186 listener->region_add(listener, §ion); 3187 } 3188 3189 /* send coalesced io add notifications */ 3190 flat_range_coalesced_io_notify_listener_add_del(fr, §ion, 3191 listener, as, true); 3192 3193 if (fr->dirty_log_mask && listener->log_start) { 3194 listener->log_start(listener, §ion, 0, fr->dirty_log_mask); 3195 } 3196 } 3197 3198 /* 3199 * register all eventfds for this address space for the newly registered 3200 * listener. 3201 */ 3202 for (i = 0; i < as->ioeventfd_nb; i++) { 3203 fd = &as->ioeventfds[i]; 3204 MemoryRegionSection section = (MemoryRegionSection) { 3205 .fv = view, 3206 .offset_within_address_space = int128_get64(fd->addr.start), 3207 .size = fd->addr.size, 3208 }; 3209 3210 if (listener->eventfd_add) { 3211 listener->eventfd_add(listener, §ion, 3212 fd->match_data, fd->data, fd->e); 3213 } 3214 } 3215 3216 if (listener->commit) { 3217 listener->commit(listener); 3218 } 3219 flatview_unref(view); 3220 } 3221 3222 static void listener_del_address_space(MemoryListener *listener, 3223 AddressSpace *as) 3224 { 3225 unsigned i; 3226 FlatView *view; 3227 FlatRange *fr; 3228 MemoryRegionIoeventfd *fd; 3229 3230 if (listener->begin) { 3231 listener->begin(listener); 3232 } 3233 view = address_space_get_flatview(as); 3234 FOR_EACH_FLAT_RANGE(fr, view) { 3235 MemoryRegionSection section = section_from_flat_range(fr, view); 3236 3237 if (fr->dirty_log_mask && listener->log_stop) { 3238 listener->log_stop(listener, §ion, fr->dirty_log_mask, 0); 3239 } 3240 3241 /* send coalesced io del notifications */ 3242 flat_range_coalesced_io_notify_listener_add_del(fr, §ion, 3243 listener, as, false); 3244 if (listener->region_del) { 3245 listener->region_del(listener, §ion); 3246 } 3247 } 3248 3249 /* 3250 * de-register all eventfds for this address space for the current 3251 * listener. 3252 */ 3253 for (i = 0; i < as->ioeventfd_nb; i++) { 3254 fd = &as->ioeventfds[i]; 3255 MemoryRegionSection section = (MemoryRegionSection) { 3256 .fv = view, 3257 .offset_within_address_space = int128_get64(fd->addr.start), 3258 .size = fd->addr.size, 3259 }; 3260 3261 if (listener->eventfd_del) { 3262 listener->eventfd_del(listener, §ion, 3263 fd->match_data, fd->data, fd->e); 3264 } 3265 } 3266 3267 if (listener->commit) { 3268 listener->commit(listener); 3269 } 3270 flatview_unref(view); 3271 } 3272 3273 void memory_listener_register(MemoryListener *listener, AddressSpace *as) 3274 { 3275 MemoryListener *other = NULL; 3276 3277 /* Only one of them can be defined for a listener */ 3278 assert(!(listener->log_sync && listener->log_sync_global)); 3279 3280 listener->address_space = as; 3281 if (QTAILQ_EMPTY(&memory_listeners) 3282 || listener->priority >= QTAILQ_LAST(&memory_listeners)->priority) { 3283 QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); 3284 } else { 3285 QTAILQ_FOREACH(other, &memory_listeners, link) { 3286 if (listener->priority < other->priority) { 3287 break; 3288 } 3289 } 3290 QTAILQ_INSERT_BEFORE(other, listener, link); 3291 } 3292 3293 if (QTAILQ_EMPTY(&as->listeners) 3294 || listener->priority >= QTAILQ_LAST(&as->listeners)->priority) { 3295 QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as); 3296 } else { 3297 QTAILQ_FOREACH(other, &as->listeners, link_as) { 3298 if (listener->priority < other->priority) { 3299 break; 3300 } 3301 } 3302 QTAILQ_INSERT_BEFORE(other, listener, link_as); 3303 } 3304 3305 listener_add_address_space(listener, as); 3306 3307 if (listener->eventfd_add || listener->eventfd_del) { 3308 as->ioeventfd_notifiers++; 3309 } 3310 } 3311 3312 void memory_listener_unregister(MemoryListener *listener) 3313 { 3314 if (!listener->address_space) { 3315 return; 3316 } 3317 3318 if (listener->eventfd_add || listener->eventfd_del) { 3319 listener->address_space->ioeventfd_notifiers--; 3320 } 3321 3322 listener_del_address_space(listener, listener->address_space); 3323 QTAILQ_REMOVE(&memory_listeners, listener, link); 3324 QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as); 3325 listener->address_space = NULL; 3326 } 3327 3328 void address_space_remove_listeners(AddressSpace *as) 3329 { 3330 while (!QTAILQ_EMPTY(&as->listeners)) { 3331 memory_listener_unregister(QTAILQ_FIRST(&as->listeners)); 3332 } 3333 } 3334 3335 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) 3336 { 3337 memory_region_ref(root); 3338 as->root = root; 3339 as->current_map = NULL; 3340 as->ioeventfd_nb = 0; 3341 as->ioeventfds = NULL; 3342 QTAILQ_INIT(&as->listeners); 3343 QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); 3344 as->max_bounce_buffer_size = DEFAULT_MAX_BOUNCE_BUFFER_SIZE; 3345 as->bounce_buffer_size = 0; 3346 qemu_mutex_init(&as->map_client_list_lock); 3347 QLIST_INIT(&as->map_client_list); 3348 as->name = g_strdup(name ? name : "anonymous"); 3349 address_space_update_topology(as); 3350 address_space_update_ioeventfds(as); 3351 } 3352 3353 static void do_address_space_destroy(AddressSpace *as) 3354 { 3355 assert(qatomic_read(&as->bounce_buffer_size) == 0); 3356 assert(QLIST_EMPTY(&as->map_client_list)); 3357 qemu_mutex_destroy(&as->map_client_list_lock); 3358 3359 assert(QTAILQ_EMPTY(&as->listeners)); 3360 3361 flatview_unref(as->current_map); 3362 g_free(as->name); 3363 g_free(as->ioeventfds); 3364 memory_region_unref(as->root); 3365 } 3366 3367 void address_space_destroy(AddressSpace *as) 3368 { 3369 MemoryRegion *root = as->root; 3370 3371 /* Flush out anything from MemoryListeners listening in on this */ 3372 memory_region_transaction_begin(); 3373 as->root = NULL; 3374 memory_region_transaction_commit(); 3375 QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); 3376 3377 /* At this point, as->dispatch and as->current_map are dummy 3378 * entries that the guest should never use. Wait for the old 3379 * values to expire before freeing the data. 3380 */ 3381 as->root = root; 3382 call_rcu(as, do_address_space_destroy, rcu); 3383 } 3384 3385 static const char *memory_region_type(MemoryRegion *mr) 3386 { 3387 if (mr->alias) { 3388 return memory_region_type(mr->alias); 3389 } 3390 if (memory_region_is_ram_device(mr)) { 3391 return "ramd"; 3392 } else if (memory_region_is_romd(mr)) { 3393 return "romd"; 3394 } else if (memory_region_is_rom(mr)) { 3395 return "rom"; 3396 } else if (memory_region_is_ram(mr)) { 3397 return "ram"; 3398 } else { 3399 return "i/o"; 3400 } 3401 } 3402 3403 typedef struct MemoryRegionList MemoryRegionList; 3404 3405 struct MemoryRegionList { 3406 const MemoryRegion *mr; 3407 QTAILQ_ENTRY(MemoryRegionList) mrqueue; 3408 }; 3409 3410 typedef QTAILQ_HEAD(, MemoryRegionList) MemoryRegionListHead; 3411 3412 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \ 3413 int128_sub((size), int128_one())) : 0) 3414 #define MTREE_INDENT " " 3415 3416 static void mtree_expand_owner(const char *label, Object *obj) 3417 { 3418 DeviceState *dev = (DeviceState *) object_dynamic_cast(obj, TYPE_DEVICE); 3419 3420 qemu_printf(" %s:{%s", label, dev ? "dev" : "obj"); 3421 if (dev && dev->id) { 3422 qemu_printf(" id=%s", dev->id); 3423 } else { 3424 char *canonical_path = object_get_canonical_path(obj); 3425 if (canonical_path) { 3426 qemu_printf(" path=%s", canonical_path); 3427 g_free(canonical_path); 3428 } else { 3429 qemu_printf(" type=%s", object_get_typename(obj)); 3430 } 3431 } 3432 qemu_printf("}"); 3433 } 3434 3435 static void mtree_print_mr_owner(const MemoryRegion *mr) 3436 { 3437 Object *owner = mr->owner; 3438 Object *parent = memory_region_owner((MemoryRegion *)mr); 3439 3440 if (!owner && !parent) { 3441 qemu_printf(" orphan"); 3442 return; 3443 } 3444 if (owner) { 3445 mtree_expand_owner("owner", owner); 3446 } 3447 if (parent && parent != owner) { 3448 mtree_expand_owner("parent", parent); 3449 } 3450 } 3451 3452 static void mtree_print_mr(const MemoryRegion *mr, unsigned int level, 3453 hwaddr base, 3454 MemoryRegionListHead *alias_print_queue, 3455 bool owner, bool display_disabled) 3456 { 3457 MemoryRegionList *new_ml, *ml, *next_ml; 3458 MemoryRegionListHead submr_print_queue; 3459 const MemoryRegion *submr; 3460 unsigned int i; 3461 hwaddr cur_start, cur_end; 3462 3463 if (!mr) { 3464 return; 3465 } 3466 3467 cur_start = base + mr->addr; 3468 cur_end = cur_start + MR_SIZE(mr->size); 3469 3470 /* 3471 * Try to detect overflow of memory region. This should never 3472 * happen normally. When it happens, we dump something to warn the 3473 * user who is observing this. 3474 */ 3475 if (cur_start < base || cur_end < cur_start) { 3476 qemu_printf("[DETECTED OVERFLOW!] "); 3477 } 3478 3479 if (mr->alias) { 3480 bool found = false; 3481 3482 /* check if the alias is already in the queue */ 3483 QTAILQ_FOREACH(ml, alias_print_queue, mrqueue) { 3484 if (ml->mr == mr->alias) { 3485 found = true; 3486 } 3487 } 3488 3489 if (!found) { 3490 ml = g_new(MemoryRegionList, 1); 3491 ml->mr = mr->alias; 3492 QTAILQ_INSERT_TAIL(alias_print_queue, ml, mrqueue); 3493 } 3494 if (mr->enabled || display_disabled) { 3495 for (i = 0; i < level; i++) { 3496 qemu_printf(MTREE_INDENT); 3497 } 3498 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx 3499 " (prio %d, %s%s): alias %s @%s " HWADDR_FMT_plx 3500 "-" HWADDR_FMT_plx "%s", 3501 cur_start, cur_end, 3502 mr->priority, 3503 mr->nonvolatile ? "nv-" : "", 3504 memory_region_type((MemoryRegion *)mr), 3505 memory_region_name(mr), 3506 memory_region_name(mr->alias), 3507 mr->alias_offset, 3508 mr->alias_offset + MR_SIZE(mr->size), 3509 mr->enabled ? "" : " [disabled]"); 3510 if (owner) { 3511 mtree_print_mr_owner(mr); 3512 } 3513 qemu_printf("\n"); 3514 } 3515 } else { 3516 if (mr->enabled || display_disabled) { 3517 for (i = 0; i < level; i++) { 3518 qemu_printf(MTREE_INDENT); 3519 } 3520 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx 3521 " (prio %d, %s%s): %s%s", 3522 cur_start, cur_end, 3523 mr->priority, 3524 mr->nonvolatile ? "nv-" : "", 3525 memory_region_type((MemoryRegion *)mr), 3526 memory_region_name(mr), 3527 mr->enabled ? "" : " [disabled]"); 3528 if (owner) { 3529 mtree_print_mr_owner(mr); 3530 } 3531 qemu_printf("\n"); 3532 } 3533 } 3534 3535 QTAILQ_INIT(&submr_print_queue); 3536 3537 QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) { 3538 new_ml = g_new(MemoryRegionList, 1); 3539 new_ml->mr = submr; 3540 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) { 3541 if (new_ml->mr->addr < ml->mr->addr || 3542 (new_ml->mr->addr == ml->mr->addr && 3543 new_ml->mr->priority > ml->mr->priority)) { 3544 QTAILQ_INSERT_BEFORE(ml, new_ml, mrqueue); 3545 new_ml = NULL; 3546 break; 3547 } 3548 } 3549 if (new_ml) { 3550 QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, mrqueue); 3551 } 3552 } 3553 3554 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) { 3555 mtree_print_mr(ml->mr, level + 1, cur_start, 3556 alias_print_queue, owner, display_disabled); 3557 } 3558 3559 QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) { 3560 g_free(ml); 3561 } 3562 } 3563 3564 struct FlatViewInfo { 3565 int counter; 3566 bool dispatch_tree; 3567 bool owner; 3568 AccelClass *ac; 3569 }; 3570 3571 static void mtree_print_flatview(gpointer key, gpointer value, 3572 gpointer user_data) 3573 { 3574 FlatView *view = key; 3575 GArray *fv_address_spaces = value; 3576 struct FlatViewInfo *fvi = user_data; 3577 FlatRange *range = &view->ranges[0]; 3578 MemoryRegion *mr; 3579 int n = view->nr; 3580 int i; 3581 AddressSpace *as; 3582 3583 qemu_printf("FlatView #%d\n", fvi->counter); 3584 ++fvi->counter; 3585 3586 for (i = 0; i < fv_address_spaces->len; ++i) { 3587 as = g_array_index(fv_address_spaces, AddressSpace*, i); 3588 qemu_printf(" AS \"%s\", root: %s", 3589 as->name, memory_region_name(as->root)); 3590 if (as->root->alias) { 3591 qemu_printf(", alias %s", memory_region_name(as->root->alias)); 3592 } 3593 qemu_printf("\n"); 3594 } 3595 3596 qemu_printf(" Root memory region: %s\n", 3597 view->root ? memory_region_name(view->root) : "(none)"); 3598 3599 if (n <= 0) { 3600 qemu_printf(MTREE_INDENT "No rendered FlatView\n\n"); 3601 return; 3602 } 3603 3604 while (n--) { 3605 mr = range->mr; 3606 if (range->offset_in_region) { 3607 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx 3608 " (prio %d, %s%s): %s @" HWADDR_FMT_plx, 3609 int128_get64(range->addr.start), 3610 int128_get64(range->addr.start) 3611 + MR_SIZE(range->addr.size), 3612 mr->priority, 3613 range->nonvolatile ? "nv-" : "", 3614 range->readonly ? "rom" : memory_region_type(mr), 3615 memory_region_name(mr), 3616 range->offset_in_region); 3617 } else { 3618 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx 3619 " (prio %d, %s%s): %s", 3620 int128_get64(range->addr.start), 3621 int128_get64(range->addr.start) 3622 + MR_SIZE(range->addr.size), 3623 mr->priority, 3624 range->nonvolatile ? "nv-" : "", 3625 range->readonly ? "rom" : memory_region_type(mr), 3626 memory_region_name(mr)); 3627 } 3628 if (fvi->owner) { 3629 mtree_print_mr_owner(mr); 3630 } 3631 3632 if (fvi->ac) { 3633 for (i = 0; i < fv_address_spaces->len; ++i) { 3634 as = g_array_index(fv_address_spaces, AddressSpace*, i); 3635 if (fvi->ac->has_memory(current_machine->accelerator, as, 3636 int128_get64(range->addr.start), 3637 MR_SIZE(range->addr.size) + 1)) { 3638 qemu_printf(" %s", fvi->ac->name); 3639 } 3640 } 3641 } 3642 qemu_printf("\n"); 3643 range++; 3644 } 3645 3646 #if !defined(CONFIG_USER_ONLY) 3647 if (fvi->dispatch_tree && view->root) { 3648 mtree_print_dispatch(view->dispatch, view->root); 3649 } 3650 #endif 3651 3652 qemu_printf("\n"); 3653 } 3654 3655 static gboolean mtree_info_flatview_free(gpointer key, gpointer value, 3656 gpointer user_data) 3657 { 3658 FlatView *view = key; 3659 GArray *fv_address_spaces = value; 3660 3661 g_array_unref(fv_address_spaces); 3662 flatview_unref(view); 3663 3664 return true; 3665 } 3666 3667 static void mtree_info_flatview(bool dispatch_tree, bool owner) 3668 { 3669 struct FlatViewInfo fvi = { 3670 .counter = 0, 3671 .dispatch_tree = dispatch_tree, 3672 .owner = owner, 3673 }; 3674 AddressSpace *as; 3675 FlatView *view; 3676 GArray *fv_address_spaces; 3677 GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); 3678 AccelClass *ac = ACCEL_GET_CLASS(current_accel()); 3679 3680 if (ac->has_memory) { 3681 fvi.ac = ac; 3682 } 3683 3684 /* Gather all FVs in one table */ 3685 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 3686 view = address_space_get_flatview(as); 3687 3688 fv_address_spaces = g_hash_table_lookup(views, view); 3689 if (!fv_address_spaces) { 3690 fv_address_spaces = g_array_new(false, false, sizeof(as)); 3691 g_hash_table_insert(views, view, fv_address_spaces); 3692 } 3693 3694 g_array_append_val(fv_address_spaces, as); 3695 } 3696 3697 /* Print */ 3698 g_hash_table_foreach(views, mtree_print_flatview, &fvi); 3699 3700 /* Free */ 3701 g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0); 3702 g_hash_table_unref(views); 3703 } 3704 3705 struct AddressSpaceInfo { 3706 MemoryRegionListHead *ml_head; 3707 bool owner; 3708 bool disabled; 3709 }; 3710 3711 /* Returns negative value if a < b; zero if a = b; positive value if a > b. */ 3712 static gint address_space_compare_name(gconstpointer a, gconstpointer b) 3713 { 3714 const AddressSpace *as_a = a; 3715 const AddressSpace *as_b = b; 3716 3717 return g_strcmp0(as_a->name, as_b->name); 3718 } 3719 3720 static void mtree_print_as_name(gpointer data, gpointer user_data) 3721 { 3722 AddressSpace *as = data; 3723 3724 qemu_printf("address-space: %s\n", as->name); 3725 } 3726 3727 static void mtree_print_as(gpointer key, gpointer value, gpointer user_data) 3728 { 3729 MemoryRegion *mr = key; 3730 GSList *as_same_root_mr_list = value; 3731 struct AddressSpaceInfo *asi = user_data; 3732 3733 g_slist_foreach(as_same_root_mr_list, mtree_print_as_name, NULL); 3734 mtree_print_mr(mr, 1, 0, asi->ml_head, asi->owner, asi->disabled); 3735 qemu_printf("\n"); 3736 } 3737 3738 static gboolean mtree_info_as_free(gpointer key, gpointer value, 3739 gpointer user_data) 3740 { 3741 GSList *as_same_root_mr_list = value; 3742 3743 g_slist_free(as_same_root_mr_list); 3744 3745 return true; 3746 } 3747 3748 static void mtree_info_as(bool dispatch_tree, bool owner, bool disabled) 3749 { 3750 MemoryRegionListHead ml_head; 3751 MemoryRegionList *ml, *ml2; 3752 AddressSpace *as; 3753 GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); 3754 GSList *as_same_root_mr_list; 3755 struct AddressSpaceInfo asi = { 3756 .ml_head = &ml_head, 3757 .owner = owner, 3758 .disabled = disabled, 3759 }; 3760 3761 QTAILQ_INIT(&ml_head); 3762 3763 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { 3764 /* Create hashtable, key=AS root MR, value = list of AS */ 3765 as_same_root_mr_list = g_hash_table_lookup(views, as->root); 3766 as_same_root_mr_list = g_slist_insert_sorted(as_same_root_mr_list, as, 3767 address_space_compare_name); 3768 g_hash_table_insert(views, as->root, as_same_root_mr_list); 3769 } 3770 3771 /* print address spaces */ 3772 g_hash_table_foreach(views, mtree_print_as, &asi); 3773 g_hash_table_foreach_remove(views, mtree_info_as_free, 0); 3774 g_hash_table_unref(views); 3775 3776 /* print aliased regions */ 3777 QTAILQ_FOREACH(ml, &ml_head, mrqueue) { 3778 qemu_printf("memory-region: %s\n", memory_region_name(ml->mr)); 3779 mtree_print_mr(ml->mr, 1, 0, &ml_head, owner, disabled); 3780 qemu_printf("\n"); 3781 } 3782 3783 QTAILQ_FOREACH_SAFE(ml, &ml_head, mrqueue, ml2) { 3784 g_free(ml); 3785 } 3786 } 3787 3788 void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled) 3789 { 3790 if (flatview) { 3791 mtree_info_flatview(dispatch_tree, owner); 3792 } else { 3793 mtree_info_as(dispatch_tree, owner, disabled); 3794 } 3795 } 3796 3797 bool memory_region_init_ram(MemoryRegion *mr, 3798 Object *owner, 3799 const char *name, 3800 uint64_t size, 3801 Error **errp) 3802 { 3803 DeviceState *owner_dev; 3804 3805 if (!memory_region_init_ram_nomigrate(mr, owner, name, size, errp)) { 3806 return false; 3807 } 3808 /* This will assert if owner is neither NULL nor a DeviceState. 3809 * We only want the owner here for the purposes of defining a 3810 * unique name for migration. TODO: Ideally we should implement 3811 * a naming scheme for Objects which are not DeviceStates, in 3812 * which case we can relax this restriction. 3813 */ 3814 owner_dev = DEVICE(owner); 3815 vmstate_register_ram(mr, owner_dev); 3816 3817 return true; 3818 } 3819 3820 bool memory_region_init_ram_guest_memfd(MemoryRegion *mr, 3821 Object *owner, 3822 const char *name, 3823 uint64_t size, 3824 Error **errp) 3825 { 3826 DeviceState *owner_dev; 3827 3828 if (!memory_region_init_ram_flags_nomigrate(mr, owner, name, size, 3829 RAM_GUEST_MEMFD, errp)) { 3830 return false; 3831 } 3832 /* This will assert if owner is neither NULL nor a DeviceState. 3833 * We only want the owner here for the purposes of defining a 3834 * unique name for migration. TODO: Ideally we should implement 3835 * a naming scheme for Objects which are not DeviceStates, in 3836 * which case we can relax this restriction. 3837 */ 3838 owner_dev = DEVICE(owner); 3839 vmstate_register_ram(mr, owner_dev); 3840 3841 return true; 3842 } 3843 3844 bool memory_region_init_rom(MemoryRegion *mr, 3845 Object *owner, 3846 const char *name, 3847 uint64_t size, 3848 Error **errp) 3849 { 3850 DeviceState *owner_dev; 3851 3852 if (!memory_region_init_rom_nomigrate(mr, owner, name, size, errp)) { 3853 return false; 3854 } 3855 /* This will assert if owner is neither NULL nor a DeviceState. 3856 * We only want the owner here for the purposes of defining a 3857 * unique name for migration. TODO: Ideally we should implement 3858 * a naming scheme for Objects which are not DeviceStates, in 3859 * which case we can relax this restriction. 3860 */ 3861 owner_dev = DEVICE(owner); 3862 vmstate_register_ram(mr, owner_dev); 3863 3864 return true; 3865 } 3866 3867 bool memory_region_init_rom_device(MemoryRegion *mr, 3868 Object *owner, 3869 const MemoryRegionOps *ops, 3870 void *opaque, 3871 const char *name, 3872 uint64_t size, 3873 Error **errp) 3874 { 3875 DeviceState *owner_dev; 3876 3877 if (!memory_region_init_rom_device_nomigrate(mr, owner, ops, opaque, 3878 name, size, errp)) { 3879 return false; 3880 } 3881 /* This will assert if owner is neither NULL nor a DeviceState. 3882 * We only want the owner here for the purposes of defining a 3883 * unique name for migration. TODO: Ideally we should implement 3884 * a naming scheme for Objects which are not DeviceStates, in 3885 * which case we can relax this restriction. 3886 */ 3887 owner_dev = DEVICE(owner); 3888 vmstate_register_ram(mr, owner_dev); 3889 3890 return true; 3891 } 3892 3893 /* 3894 * Support system builds with CONFIG_FUZZ using a weak symbol and a stub for 3895 * the fuzz_dma_read_cb callback 3896 */ 3897 #ifdef CONFIG_FUZZ 3898 void __attribute__((weak)) fuzz_dma_read_cb(size_t addr, 3899 size_t len, 3900 MemoryRegion *mr) 3901 { 3902 } 3903 #endif 3904 3905 static const TypeInfo memory_region_info = { 3906 .parent = TYPE_OBJECT, 3907 .name = TYPE_MEMORY_REGION, 3908 .class_size = sizeof(MemoryRegionClass), 3909 .instance_size = sizeof(MemoryRegion), 3910 .instance_init = memory_region_initfn, 3911 .instance_finalize = memory_region_finalize, 3912 }; 3913 3914 static const TypeInfo iommu_memory_region_info = { 3915 .parent = TYPE_MEMORY_REGION, 3916 .name = TYPE_IOMMU_MEMORY_REGION, 3917 .class_size = sizeof(IOMMUMemoryRegionClass), 3918 .instance_size = sizeof(IOMMUMemoryRegion), 3919 .instance_init = iommu_memory_region_initfn, 3920 .abstract = true, 3921 }; 3922 3923 static const TypeInfo ram_discard_manager_info = { 3924 .parent = TYPE_INTERFACE, 3925 .name = TYPE_RAM_DISCARD_MANAGER, 3926 .class_size = sizeof(RamDiscardManagerClass), 3927 }; 3928 3929 static void memory_register_types(void) 3930 { 3931 type_register_static(&memory_region_info); 3932 type_register_static(&iommu_memory_region_info); 3933 type_register_static(&ram_discard_manager_info); 3934 } 3935 3936 type_init(memory_register_types) 3937