Lines Matching +full:rcu +full:- +full:big +full:- +full:endian +full:- +full:mask
10 * the COPYING file in the top-level directory.
12 * Contributions after 2012-01-13 are licensed under the terms of the
22 #include "qemu/error-report.h"
23 #include "qemu/main-loop.h"
24 #include "qemu/qemu-print.h"
28 #include "exec/memory-internal.h"
36 #include "exec/address-spaces.h"
113 if (_listener->_callback) { \
114 _listener->_callback(_listener, ##_args); \
120 if (_listener->_callback) { \
121 _listener->_callback(_listener, ##_args); \
136 QTAILQ_FOREACH(_listener, &(_as)->listeners, link_as) { \
137 if (_listener->_callback) { \
138 _listener->_callback(_listener, _section, ##_args); \
143 QTAILQ_FOREACH_REVERSE(_listener, &(_as)->listeners, link_as) { \
144 if (_listener->_callback) { \
145 _listener->_callback(_listener, _section, ##_args); \
177 if (int128_lt(a->addr.start, b->addr.start)) { in memory_region_ioeventfd_before()
179 } else if (int128_gt(a->addr.start, b->addr.start)) { in memory_region_ioeventfd_before()
181 } else if (int128_lt(a->addr.size, b->addr.size)) { in memory_region_ioeventfd_before()
183 } else if (int128_gt(a->addr.size, b->addr.size)) { in memory_region_ioeventfd_before()
185 } else if (a->match_data < b->match_data) { in memory_region_ioeventfd_before()
187 } else if (a->match_data > b->match_data) { in memory_region_ioeventfd_before()
189 } else if (a->match_data) { in memory_region_ioeventfd_before()
190 if (a->data < b->data) { in memory_region_ioeventfd_before()
192 } else if (a->data > b->data) { in memory_region_ioeventfd_before()
196 if (a->e < b->e) { in memory_region_ioeventfd_before()
198 } else if (a->e > b->e) { in memory_region_ioeventfd_before()
207 if (int128_eq(a->addr.start, b->addr.start) && in memory_region_ioeventfd_equal()
208 (!int128_nz(a->addr.size) || !int128_nz(b->addr.size) || in memory_region_ioeventfd_equal()
209 (int128_eq(a->addr.size, b->addr.size) && in memory_region_ioeventfd_equal()
210 (a->match_data == b->match_data) && in memory_region_ioeventfd_equal()
211 ((a->match_data && (a->data == b->data)) || !a->match_data) && in memory_region_ioeventfd_equal()
212 (a->e == b->e)))) in memory_region_ioeventfd_equal()
231 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
237 .mr = fr->mr, in section_from_flat_range()
239 .offset_within_region = fr->offset_in_region, in section_from_flat_range()
240 .size = fr->addr.size, in section_from_flat_range()
241 .offset_within_address_space = int128_get64(fr->addr.start), in section_from_flat_range()
242 .readonly = fr->readonly, in section_from_flat_range()
243 .nonvolatile = fr->nonvolatile, in section_from_flat_range()
244 .unmergeable = fr->unmergeable, in section_from_flat_range()
250 return a->mr == b->mr in flatrange_equal()
251 && addrrange_equal(a->addr, b->addr) in flatrange_equal()
252 && a->offset_in_region == b->offset_in_region in flatrange_equal()
253 && a->romd_mode == b->romd_mode in flatrange_equal()
254 && a->readonly == b->readonly in flatrange_equal()
255 && a->nonvolatile == b->nonvolatile in flatrange_equal()
256 && a->unmergeable == b->unmergeable; in flatrange_equal()
264 view->ref = 1; in flatview_new()
265 view->root = mr_root; in flatview_new()
277 if (view->nr == view->nr_allocated) { in flatview_insert()
278 view->nr_allocated = MAX(2 * view->nr, 10); in flatview_insert()
279 view->ranges = g_realloc(view->ranges, in flatview_insert()
280 view->nr_allocated * sizeof(*view->ranges)); in flatview_insert()
282 memmove(view->ranges + pos + 1, view->ranges + pos, in flatview_insert()
283 (view->nr - pos) * sizeof(FlatRange)); in flatview_insert()
284 view->ranges[pos] = *range; in flatview_insert()
285 memory_region_ref(range->mr); in flatview_insert()
286 ++view->nr; in flatview_insert()
293 trace_flatview_destroy(view, view->root); in flatview_destroy()
294 if (view->dispatch) { in flatview_destroy()
295 address_space_dispatch_free(view->dispatch); in flatview_destroy()
297 for (i = 0; i < view->nr; i++) { in flatview_destroy()
298 memory_region_unref(view->ranges[i].mr); in flatview_destroy()
300 g_free(view->ranges); in flatview_destroy()
301 memory_region_unref(view->root); in flatview_destroy()
307 return qatomic_fetch_inc_nonzero(&view->ref) > 0; in flatview_ref()
312 if (qatomic_fetch_dec(&view->ref) == 1) { in flatview_unref()
313 trace_flatview_destroy_rcu(view, view->root); in flatview_unref()
314 assert(view->root); in flatview_unref()
315 call_rcu(view, flatview_destroy, rcu); in flatview_unref()
321 return int128_eq(addrrange_end(r1->addr), r2->addr.start) in can_merge()
322 && r1->mr == r2->mr in can_merge()
323 && int128_eq(int128_add(int128_make64(r1->offset_in_region), in can_merge()
324 r1->addr.size), in can_merge()
325 int128_make64(r2->offset_in_region)) in can_merge()
326 && r1->dirty_log_mask == r2->dirty_log_mask in can_merge()
327 && r1->romd_mode == r2->romd_mode in can_merge()
328 && r1->readonly == r2->readonly in can_merge()
329 && r1->nonvolatile == r2->nonvolatile in can_merge()
330 && !r1->unmergeable && !r2->unmergeable; in can_merge()
339 while (i < view->nr) { in flatview_simplify()
341 while (j < view->nr in flatview_simplify()
342 && can_merge(&view->ranges[j-1], &view->ranges[j])) { in flatview_simplify()
343 int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size); in flatview_simplify()
348 memory_region_unref(view->ranges[k].mr); in flatview_simplify()
350 memmove(&view->ranges[i], &view->ranges[j], in flatview_simplify()
351 (view->nr - j) * sizeof(view->ranges[j])); in flatview_simplify()
352 view->nr -= j - i; in flatview_simplify()
359 return mr->ops->endianness != DEVICE_LITTLE_ENDIAN; in memory_region_big_endian()
361 return mr->ops->endianness == DEVICE_BIG_ENDIAN; in memory_region_big_endian()
367 if ((op & MO_BSWAP) != devend_memop(mr->ops->endianness)) { in adjust_endianness()
388 uint64_t mask, in memory_region_shift_read_access() argument
392 *value |= (tmp & mask) << shift; in memory_region_shift_read_access()
394 *value |= (tmp & mask) >> -shift; in memory_region_shift_read_access()
400 uint64_t mask) in memory_region_shift_write_access() argument
405 tmp = (*value >> shift) & mask; in memory_region_shift_write_access()
407 tmp = (*value << -shift) & mask; in memory_region_shift_write_access()
418 abs_addr += mr->addr; in memory_region_to_absolute_addr()
419 for (root = mr; root->container; ) { in memory_region_to_absolute_addr()
420 root = root->container; in memory_region_to_absolute_addr()
421 abs_addr += root->addr; in memory_region_to_absolute_addr()
430 return current_cpu->cpu_index; in get_cpu_index()
432 return -1; in get_cpu_index()
440 uint64_t mask, in memory_region_read_accessor() argument
445 tmp = mr->ops->read(mr->opaque, addr, size); in memory_region_read_accessor()
446 if (mr->subpage) { in memory_region_read_accessor()
453 memory_region_shift_read_access(value, shift, mask, tmp); in memory_region_read_accessor()
462 uint64_t mask, in memory_region_read_with_attrs_accessor() argument
468 r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs); in memory_region_read_with_attrs_accessor()
469 if (mr->subpage) { in memory_region_read_with_attrs_accessor()
476 memory_region_shift_read_access(value, shift, mask, tmp); in memory_region_read_with_attrs_accessor()
485 uint64_t mask, in memory_region_write_accessor() argument
488 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); in memory_region_write_accessor()
490 if (mr->subpage) { in memory_region_write_accessor()
497 mr->ops->write(mr->opaque, addr, tmp, size); in memory_region_write_accessor()
506 uint64_t mask, in memory_region_write_with_attrs_accessor() argument
509 uint64_t tmp = memory_region_shift_write_access(value, shift, mask); in memory_region_write_with_attrs_accessor()
511 if (mr->subpage) { in memory_region_write_with_attrs_accessor()
518 return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs); in memory_region_write_with_attrs_accessor()
532 uint64_t mask, in access_with_adjusted_size_aligned() argument
551 if (mr->dev && !mr->disable_reentrancy_guard && in access_with_adjusted_size_aligned()
552 !mr->ram_device && !mr->ram && !mr->rom_device && !mr->readonly) { in access_with_adjusted_size_aligned()
553 if (mr->dev->mem_reentrancy_guard.engaged_in_io) { in access_with_adjusted_size_aligned()
554 warn_report_once("Blocked re-entrant IO on MemoryRegion: " in access_with_adjusted_size_aligned()
559 mr->dev->mem_reentrancy_guard.engaged_in_io = true; in access_with_adjusted_size_aligned()
568 (size - access_size - i) * 8, access_mask, attrs); in access_with_adjusted_size_aligned()
576 if (mr->dev && reentrancy_guard_applied) { in access_with_adjusted_size_aligned()
577 mr->dev->mem_reentrancy_guard.engaged_in_io = false; in access_with_adjusted_size_aligned()
582 /* Assume power-of-two size */
583 #define align_down(addr, size) ((addr) & ~((size) - 1))
586 align_down((addr) + (__size) - 1, (__size)); })
599 uint64_t mask, in access_with_adjusted_size_unaligned() argument
624 /* XXX: Big-endian path is untested... */ in access_with_adjusted_size_unaligned()
629 mask_bounds[0] = MAX(addr, cur) - cur; in access_with_adjusted_size_unaligned()
631 MIN(addr + size, align_up(cur + 1, access_size)) - cur; in access_with_adjusted_size_unaligned()
633 access_mask = (-1ULL << mask_bounds[0] * 8) & in access_with_adjusted_size_unaligned()
634 (-1ULL >> (64 - mask_bounds[1] * 8)); in access_with_adjusted_size_unaligned()
637 (size - access_size - (MAX(addr, cur) - addr)), in access_with_adjusted_size_unaligned()
649 mask_bounds[0] = MAX(addr, cur) - cur; in access_with_adjusted_size_unaligned()
651 MIN(addr + size, align_up(cur + 1, access_size)) - cur; in access_with_adjusted_size_unaligned()
653 access_mask = (-1ULL << mask_bounds[0] * 8) & in access_with_adjusted_size_unaligned()
654 (-1ULL >> (64 - mask_bounds[1] * 8)); in access_with_adjusted_size_unaligned()
657 (MAX(addr, cur) - addr), access_mask, attrs); in access_with_adjusted_size_unaligned()
680 uint64_t mask, in access_with_adjusted_size() argument
697 if (unlikely((addr & (access_size - 1)) && !unaligned)) { in access_with_adjusted_size()
714 while (mr->container) { in memory_region_to_address_space()
715 mr = mr->container; in memory_region_to_address_space()
718 if (mr == as->root) { in memory_region_to_address_space()
744 if (!mr->enabled) { in render_memory_region()
748 int128_addto(&base, int128_make64(mr->addr)); in render_memory_region()
749 readonly |= mr->readonly; in render_memory_region()
750 nonvolatile |= mr->nonvolatile; in render_memory_region()
751 unmergeable |= mr->unmergeable; in render_memory_region()
753 tmp = addrrange_make(base, mr->size); in render_memory_region()
761 if (mr->alias) { in render_memory_region()
762 int128_subfrom(&base, int128_make64(mr->alias->addr)); in render_memory_region()
763 int128_subfrom(&base, int128_make64(mr->alias_offset)); in render_memory_region()
764 render_memory_region(view, mr->alias, base, clip, in render_memory_region()
770 QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) { in render_memory_region()
775 if (!mr->terminates) { in render_memory_region()
785 fr.romd_mode = mr->romd_mode; in render_memory_region()
791 for (i = 0; i < view->nr && int128_nz(remain); ++i) { in render_memory_region()
792 if (int128_ge(base, addrrange_end(view->ranges[i].addr))) { in render_memory_region()
795 if (int128_lt(base, view->ranges[i].addr.start)) { in render_memory_region()
797 int128_sub(view->ranges[i].addr.start, base)); in render_memory_region()
807 addrrange_end(view->ranges[i].addr)), in render_memory_region()
828 if (cb(fr->addr.start, fr->addr.size, fr->mr, in flatview_for_each_range()
829 fr->offset_in_region, opaque)) { in flatview_for_each_range()
837 while (mr->enabled) { in memory_region_get_flatview_root()
838 if (mr->alias) { in memory_region_get_flatview_root()
839 if (!mr->alias_offset && int128_ge(mr->size, mr->alias->size)) { in memory_region_get_flatview_root()
843 mr = mr->alias; in memory_region_get_flatview_root()
846 } else if (!mr->terminates) { in memory_region_get_flatview_root()
849 QTAILQ_FOREACH(child, &mr->subregions, subregions_link) { in memory_region_get_flatview_root()
850 if (child->enabled) { in memory_region_get_flatview_root()
855 if (!child->addr && int128_ge(mr->size, child->size)) { in memory_region_get_flatview_root()
894 view->dispatch = address_space_dispatch_new(view); in generate_memory_topology()
895 for (i = 0; i < view->nr; i++) { in generate_memory_topology()
897 section_from_flat_range(&view->ranges[i], view); in generate_memory_topology()
900 address_space_dispatch_compact(view->dispatch); in generate_memory_topology()
929 .offset_within_address_space = int128_get64(fd->addr.start), in address_space_add_del_ioeventfds()
930 .size = fd->addr.size, in address_space_add_del_ioeventfds()
933 fd->match_data, fd->data, fd->e); in address_space_add_del_ioeventfds()
942 .offset_within_address_space = int128_get64(fd->addr.start), in address_space_add_del_ioeventfds()
943 .size = fd->addr.size, in address_space_add_del_ioeventfds()
946 fd->match_data, fd->data, fd->e); in address_space_add_del_ioeventfds()
962 /* If somebody has replaced as->current_map concurrently, in address_space_get_flatview()
979 if (!as->ioeventfd_notifiers) { in address_space_update_ioeventfds()
988 ioeventfd_max = QEMU_ALIGN_UP(as->ioeventfd_nb, 4); in address_space_update_ioeventfds()
993 for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { in address_space_update_ioeventfds()
994 tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, in address_space_update_ioeventfds()
995 int128_sub(fr->addr.start, in address_space_update_ioeventfds()
996 int128_make64(fr->offset_in_region))); in address_space_update_ioeventfds()
997 if (addrrange_intersects(fr->addr, tmp)) { in address_space_update_ioeventfds()
1004 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i]; in address_space_update_ioeventfds()
1005 ioeventfds[ioeventfd_nb-1].addr = tmp; in address_space_update_ioeventfds()
1011 as->ioeventfds, as->ioeventfd_nb); in address_space_update_ioeventfds()
1013 g_free(as->ioeventfds); in address_space_update_ioeventfds()
1014 as->ioeventfds = ioeventfds; in address_space_update_ioeventfds()
1015 as->ioeventfd_nb = ioeventfd_nb; in address_space_update_ioeventfds()
1029 tmp = addrrange_shift(cmr->addr, in flat_range_coalesced_io_notify()
1030 int128_sub(fr->addr.start, in flat_range_coalesced_io_notify()
1031 int128_make64(fr->offset_in_region))); in flat_range_coalesced_io_notify()
1032 if (!addrrange_intersects(tmp, fr->addr)) { in flat_range_coalesced_io_notify()
1035 tmp = addrrange_intersection(tmp, fr->addr); in flat_range_coalesced_io_notify()
1052 QTAILQ_FOREACH(cmr, &fr->mr->coalesced, link) { in flat_range_coalesced_io_del()
1059 MemoryRegion *mr = fr->mr; in flat_range_coalesced_io_add()
1062 if (QTAILQ_EMPTY(&mr->coalesced)) { in flat_range_coalesced_io_add()
1066 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { in flat_range_coalesced_io_add()
1078 MemoryRegion *mr = fr->mr; in flat_range_coalesced_io_notify_listener_add_del()
1081 QTAILQ_FOREACH(cmr, &mr->coalesced, link) { in flat_range_coalesced_io_notify_listener_add_del()
1082 tmp = addrrange_shift(cmr->addr, in flat_range_coalesced_io_notify_listener_add_del()
1083 int128_sub(fr->addr.start, in flat_range_coalesced_io_notify_listener_add_del()
1084 int128_make64(fr->offset_in_region))); in flat_range_coalesced_io_notify_listener_add_del()
1086 if (!addrrange_intersects(tmp, fr->addr)) { in flat_range_coalesced_io_notify_listener_add_del()
1089 tmp = addrrange_intersection(tmp, fr->addr); in flat_range_coalesced_io_notify_listener_add_del()
1091 if (add && listener->coalesced_io_add) { in flat_range_coalesced_io_notify_listener_add_del()
1092 listener->coalesced_io_add(listener, mrs, in flat_range_coalesced_io_notify_listener_add_del()
1095 } else if (!add && listener->coalesced_io_del) { in flat_range_coalesced_io_notify_listener_add_del()
1096 listener->coalesced_io_del(listener, mrs, in flat_range_coalesced_io_notify_listener_add_del()
1115 while (iold < old_view->nr || inew < new_view->nr) { in address_space_update_topology_pass()
1116 if (iold < old_view->nr) { in address_space_update_topology_pass()
1117 frold = &old_view->ranges[iold]; in address_space_update_topology_pass()
1121 if (inew < new_view->nr) { in address_space_update_topology_pass()
1122 frnew = &new_view->ranges[inew]; in address_space_update_topology_pass()
1129 || int128_lt(frold->addr.start, frnew->addr.start) in address_space_update_topology_pass()
1130 || (int128_eq(frold->addr.start, frnew->addr.start) in address_space_update_topology_pass()
1145 if (frnew->dirty_log_mask & ~frold->dirty_log_mask) { in address_space_update_topology_pass()
1147 frold->dirty_log_mask, in address_space_update_topology_pass()
1148 frnew->dirty_log_mask); in address_space_update_topology_pass()
1150 if (frold->dirty_log_mask & ~frnew->dirty_log_mask) { in address_space_update_topology_pass()
1152 frold->dirty_log_mask, in address_space_update_topology_pass()
1153 frnew->dirty_log_mask); in address_space_update_topology_pass()
1204 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); in flatviews_reset()
1217 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); in address_space_set_flatview()
1232 if (!QTAILQ_EMPTY(&as->listeners)) { in address_space_set_flatview()
1243 qatomic_rcu_set(&as->current_map, new_view); in address_space_set_flatview()
1250 * ref/unref the MemoryRegions they get---unless they use them in address_space_set_flatview()
1261 MemoryRegion *physmr = memory_region_get_flatview_root(as->root); in address_space_update_topology()
1283 --memory_region_transaction_depth; in memory_region_transaction_commit()
1312 qemu_ram_free(mr->ram_block); in memory_region_destructor_ram()
1330 if (bytes == p - name) { in memory_region_escape_name()
1354 mr->size = int128_make64(size); in memory_region_do_init()
1356 mr->size = int128_2_64(); in memory_region_do_init()
1358 mr->name = g_strdup(name); in memory_region_do_init()
1359 mr->owner = owner; in memory_region_do_init()
1360 mr->dev = (DeviceState *) object_dynamic_cast(mr->owner, TYPE_DEVICE); in memory_region_do_init()
1361 mr->ram_block = NULL; in memory_region_do_init()
1394 if (mr->container) { in memory_region_get_container()
1395 path = object_get_canonical_path(OBJECT(mr->container)); in memory_region_get_container()
1398 if (mr->container) { in memory_region_get_container()
1408 return OBJECT(mr->container); in memory_region_resolve_container()
1416 int32_t value = mr->priority; in memory_region_get_priority()
1435 mr->ops = &unassigned_mem_ops; in memory_region_initfn()
1436 mr->enabled = true; in memory_region_initfn()
1437 mr->romd_mode = true; in memory_region_initfn()
1438 mr->destructor = memory_region_destructor_none; in memory_region_initfn()
1439 QTAILQ_INIT(&mr->subregions); in memory_region_initfn()
1440 QTAILQ_INIT(&mr->coalesced); in memory_region_initfn()
1447 op->resolve = memory_region_resolve_container; in memory_region_initfn()
1450 &mr->addr, OBJ_PROP_FLAG_READ); in memory_region_initfn()
1465 mr->is_iommu = true; in iommu_memory_region_initfn()
1501 uint64_t data = ldn_he_p(mr->ram_block->host + addr, size); in memory_region_ram_device_read()
1515 stn_he_p(mr->ram_block->host + addr, size, data); in memory_region_ram_device_write()
1540 if (mr->ops->valid.accepts in memory_region_access_valid()
1541 && !mr->ops->valid.accepts(mr->opaque, addr, size, is_write, attrs)) { in memory_region_access_valid()
1549 if (!mr->ops->valid.unaligned && (addr & (size - 1))) { in memory_region_access_valid()
1558 if (!mr->ops->valid.max_access_size) { in memory_region_access_valid()
1562 if (size > mr->ops->valid.max_access_size in memory_region_access_valid()
1563 || size < mr->ops->valid.min_access_size) { in memory_region_access_valid()
1569 mr->ops->valid.min_access_size, in memory_region_access_valid()
1570 mr->ops->valid.max_access_size); in memory_region_access_valid()
1584 if (mr->ops->read) { in memory_region_dispatch_read1()
1586 mr->ops->impl.min_access_size, in memory_region_dispatch_read1()
1587 mr->ops->impl.max_access_size, in memory_region_dispatch_read1()
1588 mr->ops->impl.unaligned, in memory_region_dispatch_read1()
1593 mr->ops->impl.min_access_size, in memory_region_dispatch_read1()
1594 mr->ops->impl.max_access_size, in memory_region_dispatch_read1()
1595 mr->ops->impl.unaligned, in memory_region_dispatch_read1()
1610 if (mr->alias) { in memory_region_dispatch_read()
1611 return memory_region_dispatch_read(mr->alias, in memory_region_dispatch_read()
1612 mr->alias_offset + addr, in memory_region_dispatch_read()
1638 for (i = 0; i < mr->ioeventfd_nb; i++) { in memory_region_dispatch_write_eventfds()
1639 ioeventfd.match_data = mr->ioeventfds[i].match_data; in memory_region_dispatch_write_eventfds()
1640 ioeventfd.e = mr->ioeventfds[i].e; in memory_region_dispatch_write_eventfds()
1642 if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) { in memory_region_dispatch_write_eventfds()
1659 if (mr->alias) { in memory_region_dispatch_write()
1660 return memory_region_dispatch_write(mr->alias, in memory_region_dispatch_write()
1661 mr->alias_offset + addr, in memory_region_dispatch_write()
1681 if (mr->ops->write) { in memory_region_dispatch_write()
1683 mr->ops->impl.min_access_size, in memory_region_dispatch_write()
1684 mr->ops->impl.max_access_size, in memory_region_dispatch_write()
1690 mr->ops->impl.min_access_size, in memory_region_dispatch_write()
1691 mr->ops->impl.max_access_size, in memory_region_dispatch_write()
1705 mr->ops = ops ? ops : &unassigned_mem_ops; in memory_region_init_io()
1706 mr->opaque = opaque; in memory_region_init_io()
1707 mr->terminates = true; in memory_region_init_io()
1729 mr->ram = true; in memory_region_init_ram_flags_nomigrate()
1730 mr->terminates = true; in memory_region_init_ram_flags_nomigrate()
1731 mr->destructor = memory_region_destructor_ram; in memory_region_init_ram_flags_nomigrate()
1732 mr->ram_block = qemu_ram_alloc(size, ram_flags, mr, &err); in memory_region_init_ram_flags_nomigrate()
1734 mr->size = int128_zero(); in memory_region_init_ram_flags_nomigrate()
1754 mr->ram = true; in memory_region_init_resizeable_ram()
1755 mr->terminates = true; in memory_region_init_resizeable_ram()
1756 mr->destructor = memory_region_destructor_ram; in memory_region_init_resizeable_ram()
1757 mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized, in memory_region_init_resizeable_ram()
1760 mr->size = int128_zero(); in memory_region_init_resizeable_ram()
1781 mr->ram = true; in memory_region_init_ram_from_file()
1782 mr->readonly = !!(ram_flags & RAM_READONLY); in memory_region_init_ram_from_file()
1783 mr->terminates = true; in memory_region_init_ram_from_file()
1784 mr->destructor = memory_region_destructor_ram; in memory_region_init_ram_from_file()
1785 mr->align = align; in memory_region_init_ram_from_file()
1786 mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, in memory_region_init_ram_from_file()
1789 mr->size = int128_zero(); in memory_region_init_ram_from_file()
1808 mr->ram = true; in memory_region_init_ram_from_fd()
1809 mr->readonly = !!(ram_flags & RAM_READONLY); in memory_region_init_ram_from_fd()
1810 mr->terminates = true; in memory_region_init_ram_from_fd()
1811 mr->destructor = memory_region_destructor_ram; in memory_region_init_ram_from_fd()
1812 mr->ram_block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, in memory_region_init_ram_from_fd()
1815 mr->size = int128_zero(); in memory_region_init_ram_from_fd()
1831 mr->ram = true; in memory_region_init_ram_ptr()
1832 mr->terminates = true; in memory_region_init_ram_ptr()
1833 mr->destructor = memory_region_destructor_ram; in memory_region_init_ram_ptr()
1837 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); in memory_region_init_ram_ptr()
1847 mr->ram = true; in memory_region_init_ram_device_ptr()
1848 mr->terminates = true; in memory_region_init_ram_device_ptr()
1849 mr->ram_device = true; in memory_region_init_ram_device_ptr()
1850 mr->ops = &ram_device_mem_ops; in memory_region_init_ram_device_ptr()
1851 mr->opaque = mr; in memory_region_init_ram_device_ptr()
1852 mr->destructor = memory_region_destructor_ram; in memory_region_init_ram_device_ptr()
1856 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); in memory_region_init_ram_device_ptr()
1867 mr->alias = orig; in memory_region_init_alias()
1868 mr->alias_offset = offset; in memory_region_init_alias()
1881 mr->readonly = true; in memory_region_init_rom_nomigrate()
1897 mr->ops = ops; in memory_region_init_rom_device_nomigrate()
1898 mr->opaque = opaque; in memory_region_init_rom_device_nomigrate()
1899 mr->terminates = true; in memory_region_init_rom_device_nomigrate()
1900 mr->rom_device = true; in memory_region_init_rom_device_nomigrate()
1901 mr->destructor = memory_region_destructor_ram; in memory_region_init_rom_device_nomigrate()
1902 mr->ram_block = qemu_ram_alloc(size, 0, mr, &err); in memory_region_init_rom_device_nomigrate()
1904 mr->size = int128_zero(); in memory_region_init_rom_device_nomigrate()
1926 mr->terminates = true; /* then re-forwards */ in memory_region_init_iommu()
1927 QLIST_INIT(&iommu_mr->iommu_notify); in memory_region_init_iommu()
1928 iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE; in memory_region_init_iommu()
1935 assert(!mr->container); in memory_region_finalize()
1939 * it has no references, so we can blindly clear mr->enabled. in memory_region_finalize()
1943 mr->enabled = false; in memory_region_finalize()
1945 while (!QTAILQ_EMPTY(&mr->subregions)) { in memory_region_finalize()
1946 MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions); in memory_region_finalize()
1951 mr->destructor(mr); in memory_region_finalize()
1953 g_free((char *)mr->name); in memory_region_finalize()
1954 g_free(mr->ioeventfds); in memory_region_finalize()
1960 return obj->parent; in memory_region_owner()
1971 * ref-ing the owner will also keep the memory region alive. in memory_region_ref()
1975 if (mr && mr->owner) { in memory_region_ref()
1976 object_ref(mr->owner); in memory_region_ref()
1982 if (mr && mr->owner) { in memory_region_unref()
1983 object_unref(mr->owner); in memory_region_unref()
1989 if (int128_eq(mr->size, int128_2_64())) { in memory_region_size()
1992 return int128_get64(mr->size); in memory_region_size()
1997 if (!mr->name) { in memory_region_name()
1998 ((MemoryRegion *)mr)->name = in memory_region_name()
2001 return mr->name; in memory_region_name()
2006 return mr->ram_device; in memory_region_is_ram_device()
2011 return mr->ram && (mr->ram_block->flags & RAM_PROTECTED); in memory_region_is_protected()
2016 return mr->ram_block && mr->ram_block->guest_memfd >= 0; in memory_region_has_guest_memfd()
2021 uint8_t mask = mr->dirty_log_mask; in memory_region_get_dirty_log_mask() local
2022 RAMBlock *rb = mr->ram_block; in memory_region_get_dirty_log_mask()
2026 mask |= (1 << DIRTY_MEMORY_MIGRATION); in memory_region_get_dirty_log_mask()
2031 mask |= (1 << DIRTY_MEMORY_CODE); in memory_region_get_dirty_log_mask()
2033 return mask; in memory_region_get_dirty_log_mask()
2050 flags |= iommu_notifier->notifier_flags; in memory_region_update_iommu_notify_flags()
2053 if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) { in memory_region_update_iommu_notify_flags()
2054 ret = imrc->notify_flag_changed(iommu_mr, in memory_region_update_iommu_notify_flags()
2055 iommu_mr->iommu_notify_flags, in memory_region_update_iommu_notify_flags()
2060 iommu_mr->iommu_notify_flags = flags; in memory_region_update_iommu_notify_flags()
2071 if (mr->alias) { in memory_region_register_iommu_notifier()
2072 return memory_region_register_iommu_notifier(mr->alias, n, errp); in memory_region_register_iommu_notifier()
2077 assert(n->notifier_flags != IOMMU_NOTIFIER_NONE); in memory_region_register_iommu_notifier()
2078 assert(n->start <= n->end); in memory_region_register_iommu_notifier()
2079 assert(n->iommu_idx >= 0 && in memory_region_register_iommu_notifier()
2080 n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr)); in memory_region_register_iommu_notifier()
2082 QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node); in memory_region_register_iommu_notifier()
2094 if (imrc->get_min_page_size) { in memory_region_iommu_get_min_page_size()
2095 return imrc->get_min_page_size(iommu_mr); in memory_region_iommu_get_min_page_size()
2108 if (imrc->replay) { in memory_region_iommu_replay()
2109 imrc->replay(iommu_mr, n); in memory_region_iommu_replay()
2116 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx); in memory_region_iommu_replay()
2118 n->notify(n, &iotlb); in memory_region_iommu_replay()
2121 /* if (2^64 - MR size) < granularity, it's possible to get an in memory_region_iommu_replay()
2134 if (mr->alias) { in memory_region_unregister_iommu_notifier()
2135 memory_region_unregister_iommu_notifier(mr->alias, n); in memory_region_unregister_iommu_notifier()
2146 const IOMMUTLBEntry *entry = &event->entry; in memory_region_notify_iommu_one()
2147 hwaddr entry_end = entry->iova + entry->addr_mask; in memory_region_notify_iommu_one()
2150 if (event->type == IOMMU_NOTIFIER_UNMAP) { in memory_region_notify_iommu_one()
2151 assert(entry->perm == IOMMU_NONE); in memory_region_notify_iommu_one()
2158 if (notifier->start > entry_end || notifier->end < entry->iova) { in memory_region_notify_iommu_one()
2162 if (notifier->notifier_flags & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { in memory_region_notify_iommu_one()
2164 tmp.iova = MAX(tmp.iova, notifier->start); in memory_region_notify_iommu_one()
2165 tmp.addr_mask = MIN(entry_end, notifier->end) - tmp.iova; in memory_region_notify_iommu_one()
2167 assert(entry->iova >= notifier->start && entry_end <= notifier->end); in memory_region_notify_iommu_one()
2170 if (event->type & notifier->notifier_flags) { in memory_region_notify_iommu_one()
2171 notifier->notify(notifier, &tmp); in memory_region_notify_iommu_one()
2181 event.entry.iova = notifier->start; in memory_region_unmap_iommu_notifier_range()
2183 event.entry.addr_mask = notifier->end - notifier->start; in memory_region_unmap_iommu_notifier_range()
2197 if (iommu_notifier->iommu_idx == iommu_idx) { in memory_region_notify_iommu()
2209 if (!imrc->get_attr) { in memory_region_iommu_get_attr()
2210 return -EINVAL; in memory_region_iommu_get_attr()
2213 return imrc->get_attr(iommu_mr, attr, data); in memory_region_iommu_get_attr()
2221 if (!imrc->attrs_to_index) { in memory_region_iommu_attrs_to_index()
2225 return imrc->attrs_to_index(iommu_mr, attrs); in memory_region_iommu_attrs_to_index()
2232 if (!imrc->num_indexes) { in memory_region_iommu_num_indexes()
2236 return imrc->num_indexes(iommu_mr); in memory_region_iommu_num_indexes()
2244 return mr->rdm; in memory_region_get_ram_discard_manager()
2251 g_assert(!rdm || !mr->rdm); in memory_region_set_ram_discard_manager()
2252 mr->rdm = rdm; in memory_region_set_ram_discard_manager()
2260 g_assert(rdmc->get_min_granularity); in ram_discard_manager_get_min_granularity()
2261 return rdmc->get_min_granularity(rdm, mr); in ram_discard_manager_get_min_granularity()
2269 g_assert(rdmc->is_populated); in ram_discard_manager_is_populated()
2270 return rdmc->is_populated(rdm, section); in ram_discard_manager_is_populated()
2280 g_assert(rdmc->replay_populated); in ram_discard_manager_replay_populated()
2281 return rdmc->replay_populated(rdm, section, replay_fn, opaque); in ram_discard_manager_replay_populated()
2291 g_assert(rdmc->replay_discarded); in ram_discard_manager_replay_discarded()
2292 rdmc->replay_discarded(rdm, section, replay_fn, opaque); in ram_discard_manager_replay_discarded()
2301 g_assert(rdmc->register_listener); in ram_discard_manager_register_listener()
2302 rdmc->register_listener(rdm, rdl, section); in ram_discard_manager_register_listener()
2310 g_assert(rdmc->unregister_listener); in ram_discard_manager_unregister_listener()
2311 rdmc->unregister_listener(rdm, rdl); in ram_discard_manager_unregister_listener()
2321 hwaddr len = iotlb->addr_mask + 1; in memory_get_xlat_addr()
2322 bool writable = iotlb->perm & IOMMU_WO; in memory_get_xlat_addr()
2332 mr = address_space_translate(&address_space_memory, iotlb->translated_addr, in memory_get_xlat_addr()
2355 " via virtio-mem): %" HWADDR_PRIx "", in memory_get_xlat_addr()
2356 iotlb->translated_addr); in memory_get_xlat_addr()
2365 if (len & iotlb->addr_mask) { in memory_get_xlat_addr()
2379 *read_only = !writable || mr->readonly; in memory_get_xlat_addr()
2387 uint8_t mask = 1 << client; in memory_region_set_log() local
2391 old_logging = mr->vga_logging_count; in memory_region_set_log()
2392 mr->vga_logging_count += log ? 1 : -1; in memory_region_set_log()
2393 if (!!old_logging == !!mr->vga_logging_count) { in memory_region_set_log()
2398 mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); in memory_region_set_log()
2399 memory_region_update_pending |= mr->enabled; in memory_region_set_log()
2406 assert(mr->ram_block); in memory_region_set_dirty()
2429 if (listener->log_sync) { in memory_region_sync_dirty_bitmap()
2430 as = listener->address_space; in memory_region_sync_dirty_bitmap()
2433 if (fr->dirty_log_mask && (!mr || fr->mr == mr)) { in memory_region_sync_dirty_bitmap()
2435 listener->log_sync(listener, &mrs); in memory_region_sync_dirty_bitmap()
2439 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 0); in memory_region_sync_dirty_bitmap()
2440 } else if (listener->log_sync_global) { in memory_region_sync_dirty_bitmap()
2446 listener->log_sync_global(listener, last_stage); in memory_region_sync_dirty_bitmap()
2447 trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1); in memory_region_sync_dirty_bitmap()
2463 if (!listener->log_clear) { in memory_region_clear_dirty_bitmap()
2466 as = listener->address_space; in memory_region_clear_dirty_bitmap()
2469 if (!fr->dirty_log_mask || fr->mr != mr) { in memory_region_clear_dirty_bitmap()
2493 sec_start - mrs.offset_within_region; in memory_region_clear_dirty_bitmap()
2495 sec_size = sec_end - sec_start; in memory_region_clear_dirty_bitmap()
2497 listener->log_clear(listener, &mrs); in memory_region_clear_dirty_bitmap()
2509 assert(mr->ram_block); in memory_region_snapshot_and_clear_dirty()
2519 assert(mr->ram_block); in memory_region_snapshot_get_dirty()
2526 if (mr->readonly != readonly) { in memory_region_set_readonly()
2528 mr->readonly = readonly; in memory_region_set_readonly()
2529 memory_region_update_pending |= mr->enabled; in memory_region_set_readonly()
2536 if (mr->nonvolatile != nonvolatile) { in memory_region_set_nonvolatile()
2538 mr->nonvolatile = nonvolatile; in memory_region_set_nonvolatile()
2539 memory_region_update_pending |= mr->enabled; in memory_region_set_nonvolatile()
2546 if (mr->romd_mode != romd_mode) { in memory_region_rom_device_set_romd()
2548 mr->romd_mode = romd_mode; in memory_region_rom_device_set_romd()
2549 memory_region_update_pending |= mr->enabled; in memory_region_rom_device_set_romd()
2557 assert(mr->ram_block); in memory_region_reset_dirty()
2565 while (mr->alias) { in memory_region_get_fd()
2566 mr = mr->alias; in memory_region_get_fd()
2568 return mr->ram_block->fd; in memory_region_get_fd()
2576 while (mr->alias) { in memory_region_get_ram_ptr()
2577 offset += mr->alias_offset; in memory_region_get_ram_ptr()
2578 mr = mr->alias; in memory_region_get_ram_ptr()
2580 assert(mr->ram_block); in memory_region_get_ram_ptr()
2581 return qemu_map_ram_ptr(mr->ram_block, offset); in memory_region_get_ram_ptr()
2593 return block->mr; in memory_region_from_host()
2598 return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID; in memory_region_get_ram_addr()
2603 assert(mr->ram_block); in memory_region_ram_resize()
2605 qemu_ram_resize(mr->ram_block, newsize, errp); in memory_region_ram_resize()
2610 if (mr->ram_block) { in memory_region_msync()
2611 qemu_ram_msync(mr->ram_block, addr, size); in memory_region_msync()
2621 if (mr->dirty_log_mask) { in memory_region_writeback()
2641 if (fr->mr == mr) { in memory_region_update_coalesced_range()
2652 memory_region_add_coalescing(mr, 0, int128_get64(mr->size)); in memory_region_set_coalescing()
2661 cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size)); in memory_region_add_coalescing()
2662 QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link); in memory_region_add_coalescing()
2671 if (QTAILQ_EMPTY(&mr->coalesced)) { in memory_region_clear_coalescing()
2676 mr->flush_coalesced_mmio = false; in memory_region_clear_coalescing()
2678 while (!QTAILQ_EMPTY(&mr->coalesced)) { in memory_region_clear_coalescing()
2679 cmr = QTAILQ_FIRST(&mr->coalesced); in memory_region_clear_coalescing()
2680 QTAILQ_REMOVE(&mr->coalesced, cmr, link); in memory_region_clear_coalescing()
2688 mr->flush_coalesced_mmio = true; in memory_region_set_flush_coalesced()
2694 if (QTAILQ_EMPTY(&mr->coalesced)) { in memory_region_clear_flush_coalesced()
2695 mr->flush_coalesced_mmio = false; in memory_region_clear_flush_coalesced()
2719 for (i = 0; i < mr->ioeventfd_nb; ++i) { in memory_region_add_eventfd()
2720 if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) { in memory_region_add_eventfd()
2724 ++mr->ioeventfd_nb; in memory_region_add_eventfd()
2725 mr->ioeventfds = g_realloc(mr->ioeventfds, in memory_region_add_eventfd()
2726 sizeof(*mr->ioeventfds) * mr->ioeventfd_nb); in memory_region_add_eventfd()
2727 memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], in memory_region_add_eventfd()
2728 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); in memory_region_add_eventfd()
2729 mr->ioeventfds[i] = mrfd; in memory_region_add_eventfd()
2730 ioeventfd_update_pending |= mr->enabled; in memory_region_add_eventfd()
2754 for (i = 0; i < mr->ioeventfd_nb; ++i) { in memory_region_del_eventfd()
2755 if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) { in memory_region_del_eventfd()
2759 assert(i != mr->ioeventfd_nb); in memory_region_del_eventfd()
2760 memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1], in memory_region_del_eventfd()
2761 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1))); in memory_region_del_eventfd()
2762 --mr->ioeventfd_nb; in memory_region_del_eventfd()
2763 mr->ioeventfds = g_realloc(mr->ioeventfds, in memory_region_del_eventfd()
2764 sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); in memory_region_del_eventfd()
2765 ioeventfd_update_pending |= mr->enabled; in memory_region_del_eventfd()
2771 MemoryRegion *mr = subregion->container; in memory_region_update_container_subregions()
2777 QTAILQ_FOREACH(other, &mr->subregions, subregions_link) { in memory_region_update_container_subregions()
2778 if (subregion->priority >= other->priority) { in memory_region_update_container_subregions()
2783 QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); in memory_region_update_container_subregions()
2785 memory_region_update_pending |= mr->enabled && subregion->enabled; in memory_region_update_container_subregions()
2795 assert(!subregion->container); in memory_region_add_subregion_common()
2796 subregion->container = mr; in memory_region_add_subregion_common()
2797 for (alias = subregion->alias; alias; alias = alias->alias) { in memory_region_add_subregion_common()
2798 alias->mapped_via_alias++; in memory_region_add_subregion_common()
2800 subregion->addr = offset; in memory_region_add_subregion_common()
2808 subregion->priority = 0; in memory_region_add_subregion()
2817 subregion->priority = priority; in memory_region_add_subregion_overlap()
2827 assert(subregion->container == mr); in memory_region_del_subregion()
2828 subregion->container = NULL; in memory_region_del_subregion()
2829 for (alias = subregion->alias; alias; alias = alias->alias) { in memory_region_del_subregion()
2830 alias->mapped_via_alias--; in memory_region_del_subregion()
2831 assert(alias->mapped_via_alias >= 0); in memory_region_del_subregion()
2833 QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); in memory_region_del_subregion()
2835 memory_region_update_pending |= mr->enabled && subregion->enabled; in memory_region_del_subregion()
2841 if (enabled == mr->enabled) { in memory_region_set_enabled()
2845 mr->enabled = enabled; in memory_region_set_enabled()
2857 if (int128_eq(s, mr->size)) { in memory_region_set_size()
2861 mr->size = s; in memory_region_set_size()
2868 MemoryRegion *container = mr->container; in memory_region_readd_subregion()
2874 memory_region_add_subregion_common(container, mr->addr, mr); in memory_region_readd_subregion()
2882 if (addr != mr->addr) { in memory_region_set_address()
2883 mr->addr = addr; in memory_region_set_address()
2890 assert(mr->alias); in memory_region_set_alias_offset()
2892 if (offset == mr->alias_offset) { in memory_region_set_alias_offset()
2897 mr->alias_offset = offset; in memory_region_set_alias_offset()
2898 memory_region_update_pending |= mr->enabled; in memory_region_set_alias_offset()
2904 if (unmergeable == mr->unmergeable) { in memory_region_set_unmergeable()
2909 mr->unmergeable = unmergeable; in memory_region_set_unmergeable()
2910 memory_region_update_pending |= mr->enabled; in memory_region_set_unmergeable()
2916 return mr->align; in memory_region_get_alignment()
2924 if (int128_le(addrrange_end(*addr), fr->addr.start)) { in cmp_flatrange_addr()
2925 return -1; in cmp_flatrange_addr()
2926 } else if (int128_ge(addr->start, addrrange_end(fr->addr))) { in cmp_flatrange_addr()
2934 return bsearch(&addr, view->ranges, view->nr, in flatview_lookup()
2940 return !!mr->container || mr->mapped_via_alias; in memory_region_is_mapped()
2944 * returned region. It must be called from an RCU critical section.
2956 addr += mr->addr; in memory_region_find_rcu()
2957 for (root = mr; root->container; ) { in memory_region_find_rcu()
2958 root = root->container; in memory_region_find_rcu()
2959 addr += root->addr; in memory_region_find_rcu()
2974 while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) { in memory_region_find_rcu()
2975 --fr; in memory_region_find_rcu()
2978 ret.mr = fr->mr; in memory_region_find_rcu()
2980 range = addrrange_intersection(range, fr->addr); in memory_region_find_rcu()
2981 ret.offset_within_region = fr->offset_in_region; in memory_region_find_rcu()
2983 fr->addr.start)); in memory_region_find_rcu()
2986 ret.readonly = fr->readonly; in memory_region_find_rcu()
2987 ret.nonvolatile = fr->nonvolatile; in memory_region_find_rcu()
3008 if (tmp->mr) { in memory_region_section_new_copy()
3009 memory_region_ref(tmp->mr); in memory_region_section_new_copy()
3011 if (tmp->fv) { in memory_region_section_new_copy()
3012 bool ret = flatview_ref(tmp->fv); in memory_region_section_new_copy()
3021 if (s->fv) { in memory_region_section_free_copy()
3022 flatview_unref(s->fv); in memory_region_section_free_copy()
3024 if (s->mr) { in memory_region_section_free_copy()
3025 memory_region_unref(s->mr); in memory_region_section_free_copy()
3062 if (listener->log_global_start) { in memory_global_dirty_log_do_start()
3063 if (!listener->log_global_start(listener, errp)) { in memory_global_dirty_log_do_start()
3072 if (listener->log_global_stop) { in memory_global_dirty_log_do_start()
3073 listener->log_global_stop(listener); in memory_global_dirty_log_do_start()
3184 if (listener->begin) { in listener_add_address_space()
3185 listener->begin(listener); in listener_add_address_space()
3195 if (listener->log_global_start) { in listener_add_address_space()
3196 listener->log_global_start(listener, &error_abort); in listener_add_address_space()
3204 if (listener->region_add) { in listener_add_address_space()
3205 listener->region_add(listener, §ion); in listener_add_address_space()
3212 if (fr->dirty_log_mask && listener->log_start) { in listener_add_address_space()
3213 listener->log_start(listener, §ion, 0, fr->dirty_log_mask); in listener_add_address_space()
3221 for (i = 0; i < as->ioeventfd_nb; i++) { in listener_add_address_space()
3222 fd = &as->ioeventfds[i]; in listener_add_address_space()
3225 .offset_within_address_space = int128_get64(fd->addr.start), in listener_add_address_space()
3226 .size = fd->addr.size, in listener_add_address_space()
3229 if (listener->eventfd_add) { in listener_add_address_space()
3230 listener->eventfd_add(listener, §ion, in listener_add_address_space()
3231 fd->match_data, fd->data, fd->e); in listener_add_address_space()
3235 if (listener->commit) { in listener_add_address_space()
3236 listener->commit(listener); in listener_add_address_space()
3249 if (listener->begin) { in listener_del_address_space()
3250 listener->begin(listener); in listener_del_address_space()
3256 if (fr->dirty_log_mask && listener->log_stop) { in listener_del_address_space()
3257 listener->log_stop(listener, §ion, fr->dirty_log_mask, 0); in listener_del_address_space()
3263 if (listener->region_del) { in listener_del_address_space()
3264 listener->region_del(listener, §ion); in listener_del_address_space()
3269 * de-register all eventfds for this address space for the current in listener_del_address_space()
3272 for (i = 0; i < as->ioeventfd_nb; i++) { in listener_del_address_space()
3273 fd = &as->ioeventfds[i]; in listener_del_address_space()
3276 .offset_within_address_space = int128_get64(fd->addr.start), in listener_del_address_space()
3277 .size = fd->addr.size, in listener_del_address_space()
3280 if (listener->eventfd_del) { in listener_del_address_space()
3281 listener->eventfd_del(listener, §ion, in listener_del_address_space()
3282 fd->match_data, fd->data, fd->e); in listener_del_address_space()
3286 if (listener->commit) { in listener_del_address_space()
3287 listener->commit(listener); in listener_del_address_space()
3297 assert(!(listener->log_sync && listener->log_sync_global)); in memory_listener_register()
3299 listener->address_space = as; in memory_listener_register()
3301 || listener->priority >= QTAILQ_LAST(&memory_listeners)->priority) { in memory_listener_register()
3305 if (listener->priority < other->priority) { in memory_listener_register()
3312 if (QTAILQ_EMPTY(&as->listeners) in memory_listener_register()
3313 || listener->priority >= QTAILQ_LAST(&as->listeners)->priority) { in memory_listener_register()
3314 QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as); in memory_listener_register()
3316 QTAILQ_FOREACH(other, &as->listeners, link_as) { in memory_listener_register()
3317 if (listener->priority < other->priority) { in memory_listener_register()
3326 if (listener->eventfd_add || listener->eventfd_del) { in memory_listener_register()
3327 as->ioeventfd_notifiers++; in memory_listener_register()
3333 if (!listener->address_space) { in memory_listener_unregister()
3337 if (listener->eventfd_add || listener->eventfd_del) { in memory_listener_unregister()
3338 listener->address_space->ioeventfd_notifiers--; in memory_listener_unregister()
3341 listener_del_address_space(listener, listener->address_space); in memory_listener_unregister()
3343 QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as); in memory_listener_unregister()
3344 listener->address_space = NULL; in memory_listener_unregister()
3349 while (!QTAILQ_EMPTY(&as->listeners)) { in address_space_remove_listeners()
3350 memory_listener_unregister(QTAILQ_FIRST(&as->listeners)); in address_space_remove_listeners()
3357 as->root = root; in address_space_init()
3358 as->current_map = NULL; in address_space_init()
3359 as->ioeventfd_nb = 0; in address_space_init()
3360 as->ioeventfds = NULL; in address_space_init()
3361 QTAILQ_INIT(&as->listeners); in address_space_init()
3363 as->max_bounce_buffer_size = DEFAULT_MAX_BOUNCE_BUFFER_SIZE; in address_space_init()
3364 as->bounce_buffer_size = 0; in address_space_init()
3365 qemu_mutex_init(&as->map_client_list_lock); in address_space_init()
3366 QLIST_INIT(&as->map_client_list); in address_space_init()
3367 as->name = g_strdup(name ? name : "anonymous"); in address_space_init()
3374 assert(qatomic_read(&as->bounce_buffer_size) == 0); in do_address_space_destroy()
3375 assert(QLIST_EMPTY(&as->map_client_list)); in do_address_space_destroy()
3376 qemu_mutex_destroy(&as->map_client_list_lock); in do_address_space_destroy()
3378 assert(QTAILQ_EMPTY(&as->listeners)); in do_address_space_destroy()
3380 flatview_unref(as->current_map); in do_address_space_destroy()
3381 g_free(as->name); in do_address_space_destroy()
3382 g_free(as->ioeventfds); in do_address_space_destroy()
3383 memory_region_unref(as->root); in do_address_space_destroy()
3388 MemoryRegion *root = as->root; in address_space_destroy()
3392 as->root = NULL; in address_space_destroy()
3396 /* At this point, as->dispatch and as->current_map are dummy in address_space_destroy()
3400 as->root = root; in address_space_destroy()
3401 call_rcu(as, do_address_space_destroy, rcu); in address_space_destroy()
3406 if (mr->alias) { in memory_region_type()
3407 return memory_region_type(mr->alias); in memory_region_type()
3440 if (dev && dev->id) { in mtree_expand_owner()
3441 qemu_printf(" id=%s", dev->id); in mtree_expand_owner()
3456 Object *owner = mr->owner; in mtree_print_mr_owner()
3486 cur_start = base + mr->addr; in mtree_print_mr()
3487 cur_end = cur_start + MR_SIZE(mr->size); in mtree_print_mr()
3498 if (mr->alias) { in mtree_print_mr()
3503 if (ml->mr == mr->alias) { in mtree_print_mr()
3510 ml->mr = mr->alias; in mtree_print_mr()
3513 if (mr->enabled || display_disabled) { in mtree_print_mr()
3517 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx in mtree_print_mr()
3519 "-" HWADDR_FMT_plx "%s", in mtree_print_mr()
3521 mr->priority, in mtree_print_mr()
3522 mr->nonvolatile ? "nv-" : "", in mtree_print_mr()
3525 memory_region_name(mr->alias), in mtree_print_mr()
3526 mr->alias_offset, in mtree_print_mr()
3527 mr->alias_offset + MR_SIZE(mr->size), in mtree_print_mr()
3528 mr->enabled ? "" : " [disabled]"); in mtree_print_mr()
3535 if (mr->enabled || display_disabled) { in mtree_print_mr()
3539 qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx in mtree_print_mr()
3542 mr->priority, in mtree_print_mr()
3543 mr->nonvolatile ? "nv-" : "", in mtree_print_mr()
3546 mr->enabled ? "" : " [disabled]"); in mtree_print_mr()
3556 QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) { in mtree_print_mr()
3558 new_ml->mr = submr; in mtree_print_mr()
3560 if (new_ml->mr->addr < ml->mr->addr || in mtree_print_mr()
3561 (new_ml->mr->addr == ml->mr->addr && in mtree_print_mr()
3562 new_ml->mr->priority > ml->mr->priority)) { in mtree_print_mr()
3574 mtree_print_mr(ml->mr, level + 1, cur_start, in mtree_print_mr()
3596 FlatRange *range = &view->ranges[0]; in mtree_print_flatview()
3598 int n = view->nr; in mtree_print_flatview()
3602 qemu_printf("FlatView #%d\n", fvi->counter); in mtree_print_flatview()
3603 ++fvi->counter; in mtree_print_flatview()
3605 for (i = 0; i < fv_address_spaces->len; ++i) { in mtree_print_flatview()
3608 as->name, memory_region_name(as->root)); in mtree_print_flatview()
3609 if (as->root->alias) { in mtree_print_flatview()
3610 qemu_printf(", alias %s", memory_region_name(as->root->alias)); in mtree_print_flatview()
3616 view->root ? memory_region_name(view->root) : "(none)"); in mtree_print_flatview()
3623 while (n--) { in mtree_print_flatview()
3624 mr = range->mr; in mtree_print_flatview()
3625 if (range->offset_in_region) { in mtree_print_flatview()
3626 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx in mtree_print_flatview()
3628 int128_get64(range->addr.start), in mtree_print_flatview()
3629 int128_get64(range->addr.start) in mtree_print_flatview()
3630 + MR_SIZE(range->addr.size), in mtree_print_flatview()
3631 mr->priority, in mtree_print_flatview()
3632 range->nonvolatile ? "nv-" : "", in mtree_print_flatview()
3633 range->readonly ? "rom" : memory_region_type(mr), in mtree_print_flatview()
3635 range->offset_in_region); in mtree_print_flatview()
3637 qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx in mtree_print_flatview()
3639 int128_get64(range->addr.start), in mtree_print_flatview()
3640 int128_get64(range->addr.start) in mtree_print_flatview()
3641 + MR_SIZE(range->addr.size), in mtree_print_flatview()
3642 mr->priority, in mtree_print_flatview()
3643 range->nonvolatile ? "nv-" : "", in mtree_print_flatview()
3644 range->readonly ? "rom" : memory_region_type(mr), in mtree_print_flatview()
3647 if (fvi->owner) { in mtree_print_flatview()
3651 if (fvi->ac) { in mtree_print_flatview()
3652 for (i = 0; i < fv_address_spaces->len; ++i) { in mtree_print_flatview()
3654 if (fvi->ac->has_memory(current_machine, as, in mtree_print_flatview()
3655 int128_get64(range->addr.start), in mtree_print_flatview()
3656 MR_SIZE(range->addr.size) + 1)) { in mtree_print_flatview()
3657 qemu_printf(" %s", fvi->ac->name); in mtree_print_flatview()
3666 if (fvi->dispatch_tree && view->root) { in mtree_print_flatview()
3667 mtree_print_dispatch(view->dispatch, view->root); in mtree_print_flatview()
3699 if (ac->has_memory) { in mtree_info_flatview()
3736 return g_strcmp0(as_a->name, as_b->name); in address_space_compare_name()
3743 qemu_printf("address-space: %s\n", as->name); in mtree_print_as_name()
3753 mtree_print_mr(mr, 1, 0, asi->ml_head, asi->owner, asi->disabled); in mtree_print_as()
3784 as_same_root_mr_list = g_hash_table_lookup(views, as->root); in mtree_info_as()
3787 g_hash_table_insert(views, as->root, as_same_root_mr_list); in mtree_info_as()
3797 qemu_printf("memory-region: %s\n", memory_region_name(ml->mr)); in mtree_info_as()
3798 mtree_print_mr(ml->mr, 1, 0, &ml_head, owner, disabled); in mtree_info_as()