1*8d7f2e76SPhilippe Mathieu-Daudé /* 2*8d7f2e76SPhilippe Mathieu-Daudé * RAM allocation and memory access 3*8d7f2e76SPhilippe Mathieu-Daudé * 4*8d7f2e76SPhilippe Mathieu-Daudé * Copyright (c) 2003 Fabrice Bellard 5*8d7f2e76SPhilippe Mathieu-Daudé * 6*8d7f2e76SPhilippe Mathieu-Daudé * This library is free software; you can redistribute it and/or 7*8d7f2e76SPhilippe Mathieu-Daudé * modify it under the terms of the GNU Lesser General Public 8*8d7f2e76SPhilippe Mathieu-Daudé * License as published by the Free Software Foundation; either 9*8d7f2e76SPhilippe Mathieu-Daudé * version 2.1 of the License, or (at your option) any later version. 10*8d7f2e76SPhilippe Mathieu-Daudé * 11*8d7f2e76SPhilippe Mathieu-Daudé * This library is distributed in the hope that it will be useful, 12*8d7f2e76SPhilippe Mathieu-Daudé * but WITHOUT ANY WARRANTY; without even the implied warranty of 13*8d7f2e76SPhilippe Mathieu-Daudé * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14*8d7f2e76SPhilippe Mathieu-Daudé * Lesser General Public License for more details. 15*8d7f2e76SPhilippe Mathieu-Daudé * 16*8d7f2e76SPhilippe Mathieu-Daudé * You should have received a copy of the GNU Lesser General Public 17*8d7f2e76SPhilippe Mathieu-Daudé * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18*8d7f2e76SPhilippe Mathieu-Daudé */ 19*8d7f2e76SPhilippe Mathieu-Daudé 20*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/osdep.h" 21*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/page-vary.h" 22*8d7f2e76SPhilippe Mathieu-Daudé #include "qapi/error.h" 23*8d7f2e76SPhilippe Mathieu-Daudé 24*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/cutils.h" 25*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/cacheflush.h" 26*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/hbitmap.h" 27*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/madvise.h" 28*8d7f2e76SPhilippe Mathieu-Daudé 29*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_TCG 30*8d7f2e76SPhilippe Mathieu-Daudé #include "hw/core/tcg-cpu-ops.h" 31*8d7f2e76SPhilippe Mathieu-Daudé #endif /* CONFIG_TCG */ 32*8d7f2e76SPhilippe Mathieu-Daudé 33*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/exec-all.h" 34*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/target_page.h" 35*8d7f2e76SPhilippe Mathieu-Daudé #include "hw/qdev-core.h" 36*8d7f2e76SPhilippe Mathieu-Daudé #include "hw/qdev-properties.h" 37*8d7f2e76SPhilippe Mathieu-Daudé #include "hw/boards.h" 38*8d7f2e76SPhilippe Mathieu-Daudé #include "hw/xen/xen.h" 39*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/kvm.h" 40*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/tcg.h" 41*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/qtest.h" 42*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/timer.h" 43*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/config-file.h" 44*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/error-report.h" 45*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/qemu-print.h" 46*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/log.h" 47*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/memalign.h" 48*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/memory.h" 49*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/ioport.h" 50*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/dma.h" 51*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/hostmem.h" 52*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/hw_accel.h" 53*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/xen-mapcache.h" 54*8d7f2e76SPhilippe Mathieu-Daudé #include "trace/trace-root.h" 55*8d7f2e76SPhilippe Mathieu-Daudé 56*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_FALLOCATE_PUNCH_HOLE 57*8d7f2e76SPhilippe Mathieu-Daudé #include <linux/falloc.h> 58*8d7f2e76SPhilippe Mathieu-Daudé #endif 59*8d7f2e76SPhilippe Mathieu-Daudé 60*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/rcu_queue.h" 61*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/main-loop.h" 62*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/translate-all.h" 63*8d7f2e76SPhilippe Mathieu-Daudé #include "sysemu/replay.h" 64*8d7f2e76SPhilippe Mathieu-Daudé 65*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/memory-internal.h" 66*8d7f2e76SPhilippe Mathieu-Daudé #include "exec/ram_addr.h" 67*8d7f2e76SPhilippe Mathieu-Daudé 68*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/pmem.h" 69*8d7f2e76SPhilippe Mathieu-Daudé 70*8d7f2e76SPhilippe Mathieu-Daudé #include "migration/vmstate.h" 71*8d7f2e76SPhilippe Mathieu-Daudé 72*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/range.h" 73*8d7f2e76SPhilippe Mathieu-Daudé #ifndef _WIN32 74*8d7f2e76SPhilippe Mathieu-Daudé #include "qemu/mmap-alloc.h" 75*8d7f2e76SPhilippe Mathieu-Daudé #endif 76*8d7f2e76SPhilippe Mathieu-Daudé 77*8d7f2e76SPhilippe Mathieu-Daudé #include "monitor/monitor.h" 78*8d7f2e76SPhilippe Mathieu-Daudé 79*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_LIBDAXCTL 80*8d7f2e76SPhilippe Mathieu-Daudé #include <daxctl/libdaxctl.h> 81*8d7f2e76SPhilippe Mathieu-Daudé #endif 82*8d7f2e76SPhilippe Mathieu-Daudé 83*8d7f2e76SPhilippe Mathieu-Daudé //#define DEBUG_SUBPAGE 84*8d7f2e76SPhilippe Mathieu-Daudé 85*8d7f2e76SPhilippe Mathieu-Daudé /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes 86*8d7f2e76SPhilippe Mathieu-Daudé * are protected by the ramlist lock. 87*8d7f2e76SPhilippe Mathieu-Daudé */ 88*8d7f2e76SPhilippe Mathieu-Daudé RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) }; 89*8d7f2e76SPhilippe Mathieu-Daudé 90*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegion *system_memory; 91*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegion *system_io; 92*8d7f2e76SPhilippe Mathieu-Daudé 93*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace address_space_io; 94*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace address_space_memory; 95*8d7f2e76SPhilippe Mathieu-Daudé 96*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegion io_mem_unassigned; 97*8d7f2e76SPhilippe Mathieu-Daudé 98*8d7f2e76SPhilippe Mathieu-Daudé typedef struct PhysPageEntry PhysPageEntry; 99*8d7f2e76SPhilippe Mathieu-Daudé 100*8d7f2e76SPhilippe Mathieu-Daudé struct PhysPageEntry { 101*8d7f2e76SPhilippe Mathieu-Daudé /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */ 102*8d7f2e76SPhilippe Mathieu-Daudé uint32_t skip : 6; 103*8d7f2e76SPhilippe Mathieu-Daudé /* index into phys_sections (!skip) or phys_map_nodes (skip) */ 104*8d7f2e76SPhilippe Mathieu-Daudé uint32_t ptr : 26; 105*8d7f2e76SPhilippe Mathieu-Daudé }; 106*8d7f2e76SPhilippe Mathieu-Daudé 107*8d7f2e76SPhilippe Mathieu-Daudé #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6) 108*8d7f2e76SPhilippe Mathieu-Daudé 109*8d7f2e76SPhilippe Mathieu-Daudé /* Size of the L2 (and L3, etc) page tables. */ 110*8d7f2e76SPhilippe Mathieu-Daudé #define ADDR_SPACE_BITS 64 111*8d7f2e76SPhilippe Mathieu-Daudé 112*8d7f2e76SPhilippe Mathieu-Daudé #define P_L2_BITS 9 113*8d7f2e76SPhilippe Mathieu-Daudé #define P_L2_SIZE (1 << P_L2_BITS) 114*8d7f2e76SPhilippe Mathieu-Daudé 115*8d7f2e76SPhilippe Mathieu-Daudé #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1) 116*8d7f2e76SPhilippe Mathieu-Daudé 117*8d7f2e76SPhilippe Mathieu-Daudé typedef PhysPageEntry Node[P_L2_SIZE]; 118*8d7f2e76SPhilippe Mathieu-Daudé 119*8d7f2e76SPhilippe Mathieu-Daudé typedef struct PhysPageMap { 120*8d7f2e76SPhilippe Mathieu-Daudé struct rcu_head rcu; 121*8d7f2e76SPhilippe Mathieu-Daudé 122*8d7f2e76SPhilippe Mathieu-Daudé unsigned sections_nb; 123*8d7f2e76SPhilippe Mathieu-Daudé unsigned sections_nb_alloc; 124*8d7f2e76SPhilippe Mathieu-Daudé unsigned nodes_nb; 125*8d7f2e76SPhilippe Mathieu-Daudé unsigned nodes_nb_alloc; 126*8d7f2e76SPhilippe Mathieu-Daudé Node *nodes; 127*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *sections; 128*8d7f2e76SPhilippe Mathieu-Daudé } PhysPageMap; 129*8d7f2e76SPhilippe Mathieu-Daudé 130*8d7f2e76SPhilippe Mathieu-Daudé struct AddressSpaceDispatch { 131*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *mru_section; 132*8d7f2e76SPhilippe Mathieu-Daudé /* This is a multi-level map on the physical address space. 133*8d7f2e76SPhilippe Mathieu-Daudé * The bottom level has pointers to MemoryRegionSections. 134*8d7f2e76SPhilippe Mathieu-Daudé */ 135*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry phys_map; 136*8d7f2e76SPhilippe Mathieu-Daudé PhysPageMap map; 137*8d7f2e76SPhilippe Mathieu-Daudé }; 138*8d7f2e76SPhilippe Mathieu-Daudé 139*8d7f2e76SPhilippe Mathieu-Daudé #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK) 140*8d7f2e76SPhilippe Mathieu-Daudé typedef struct subpage_t { 141*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion iomem; 142*8d7f2e76SPhilippe Mathieu-Daudé FlatView *fv; 143*8d7f2e76SPhilippe Mathieu-Daudé hwaddr base; 144*8d7f2e76SPhilippe Mathieu-Daudé uint16_t sub_section[]; 145*8d7f2e76SPhilippe Mathieu-Daudé } subpage_t; 146*8d7f2e76SPhilippe Mathieu-Daudé 147*8d7f2e76SPhilippe Mathieu-Daudé #define PHYS_SECTION_UNASSIGNED 0 148*8d7f2e76SPhilippe Mathieu-Daudé 149*8d7f2e76SPhilippe Mathieu-Daudé static void io_mem_init(void); 150*8d7f2e76SPhilippe Mathieu-Daudé static void memory_map_init(void); 151*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_log_global_after_sync(MemoryListener *listener); 152*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_commit(MemoryListener *listener); 153*8d7f2e76SPhilippe Mathieu-Daudé 154*8d7f2e76SPhilippe Mathieu-Daudé /** 155*8d7f2e76SPhilippe Mathieu-Daudé * CPUAddressSpace: all the information a CPU needs about an AddressSpace 156*8d7f2e76SPhilippe Mathieu-Daudé * @cpu: the CPU whose AddressSpace this is 157*8d7f2e76SPhilippe Mathieu-Daudé * @as: the AddressSpace itself 158*8d7f2e76SPhilippe Mathieu-Daudé * @memory_dispatch: its dispatch pointer (cached, RCU protected) 159*8d7f2e76SPhilippe Mathieu-Daudé * @tcg_as_listener: listener for tracking changes to the AddressSpace 160*8d7f2e76SPhilippe Mathieu-Daudé */ 161*8d7f2e76SPhilippe Mathieu-Daudé struct CPUAddressSpace { 162*8d7f2e76SPhilippe Mathieu-Daudé CPUState *cpu; 163*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *as; 164*8d7f2e76SPhilippe Mathieu-Daudé struct AddressSpaceDispatch *memory_dispatch; 165*8d7f2e76SPhilippe Mathieu-Daudé MemoryListener tcg_as_listener; 166*8d7f2e76SPhilippe Mathieu-Daudé }; 167*8d7f2e76SPhilippe Mathieu-Daudé 168*8d7f2e76SPhilippe Mathieu-Daudé struct DirtyBitmapSnapshot { 169*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t start; 170*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t end; 171*8d7f2e76SPhilippe Mathieu-Daudé unsigned long dirty[]; 172*8d7f2e76SPhilippe Mathieu-Daudé }; 173*8d7f2e76SPhilippe Mathieu-Daudé 174*8d7f2e76SPhilippe Mathieu-Daudé static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes) 175*8d7f2e76SPhilippe Mathieu-Daudé { 176*8d7f2e76SPhilippe Mathieu-Daudé static unsigned alloc_hint = 16; 177*8d7f2e76SPhilippe Mathieu-Daudé if (map->nodes_nb + nodes > map->nodes_nb_alloc) { 178*8d7f2e76SPhilippe Mathieu-Daudé map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes); 179*8d7f2e76SPhilippe Mathieu-Daudé map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc); 180*8d7f2e76SPhilippe Mathieu-Daudé alloc_hint = map->nodes_nb_alloc; 181*8d7f2e76SPhilippe Mathieu-Daudé } 182*8d7f2e76SPhilippe Mathieu-Daudé } 183*8d7f2e76SPhilippe Mathieu-Daudé 184*8d7f2e76SPhilippe Mathieu-Daudé static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf) 185*8d7f2e76SPhilippe Mathieu-Daudé { 186*8d7f2e76SPhilippe Mathieu-Daudé unsigned i; 187*8d7f2e76SPhilippe Mathieu-Daudé uint32_t ret; 188*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry e; 189*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry *p; 190*8d7f2e76SPhilippe Mathieu-Daudé 191*8d7f2e76SPhilippe Mathieu-Daudé ret = map->nodes_nb++; 192*8d7f2e76SPhilippe Mathieu-Daudé p = map->nodes[ret]; 193*8d7f2e76SPhilippe Mathieu-Daudé assert(ret != PHYS_MAP_NODE_NIL); 194*8d7f2e76SPhilippe Mathieu-Daudé assert(ret != map->nodes_nb_alloc); 195*8d7f2e76SPhilippe Mathieu-Daudé 196*8d7f2e76SPhilippe Mathieu-Daudé e.skip = leaf ? 0 : 1; 197*8d7f2e76SPhilippe Mathieu-Daudé e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL; 198*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < P_L2_SIZE; ++i) { 199*8d7f2e76SPhilippe Mathieu-Daudé memcpy(&p[i], &e, sizeof(e)); 200*8d7f2e76SPhilippe Mathieu-Daudé } 201*8d7f2e76SPhilippe Mathieu-Daudé return ret; 202*8d7f2e76SPhilippe Mathieu-Daudé } 203*8d7f2e76SPhilippe Mathieu-Daudé 204*8d7f2e76SPhilippe Mathieu-Daudé static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp, 205*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *index, uint64_t *nb, uint16_t leaf, 206*8d7f2e76SPhilippe Mathieu-Daudé int level) 207*8d7f2e76SPhilippe Mathieu-Daudé { 208*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry *p; 209*8d7f2e76SPhilippe Mathieu-Daudé hwaddr step = (hwaddr)1 << (level * P_L2_BITS); 210*8d7f2e76SPhilippe Mathieu-Daudé 211*8d7f2e76SPhilippe Mathieu-Daudé if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) { 212*8d7f2e76SPhilippe Mathieu-Daudé lp->ptr = phys_map_node_alloc(map, level == 0); 213*8d7f2e76SPhilippe Mathieu-Daudé } 214*8d7f2e76SPhilippe Mathieu-Daudé p = map->nodes[lp->ptr]; 215*8d7f2e76SPhilippe Mathieu-Daudé lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)]; 216*8d7f2e76SPhilippe Mathieu-Daudé 217*8d7f2e76SPhilippe Mathieu-Daudé while (*nb && lp < &p[P_L2_SIZE]) { 218*8d7f2e76SPhilippe Mathieu-Daudé if ((*index & (step - 1)) == 0 && *nb >= step) { 219*8d7f2e76SPhilippe Mathieu-Daudé lp->skip = 0; 220*8d7f2e76SPhilippe Mathieu-Daudé lp->ptr = leaf; 221*8d7f2e76SPhilippe Mathieu-Daudé *index += step; 222*8d7f2e76SPhilippe Mathieu-Daudé *nb -= step; 223*8d7f2e76SPhilippe Mathieu-Daudé } else { 224*8d7f2e76SPhilippe Mathieu-Daudé phys_page_set_level(map, lp, index, nb, leaf, level - 1); 225*8d7f2e76SPhilippe Mathieu-Daudé } 226*8d7f2e76SPhilippe Mathieu-Daudé ++lp; 227*8d7f2e76SPhilippe Mathieu-Daudé } 228*8d7f2e76SPhilippe Mathieu-Daudé } 229*8d7f2e76SPhilippe Mathieu-Daudé 230*8d7f2e76SPhilippe Mathieu-Daudé static void phys_page_set(AddressSpaceDispatch *d, 231*8d7f2e76SPhilippe Mathieu-Daudé hwaddr index, uint64_t nb, 232*8d7f2e76SPhilippe Mathieu-Daudé uint16_t leaf) 233*8d7f2e76SPhilippe Mathieu-Daudé { 234*8d7f2e76SPhilippe Mathieu-Daudé /* Wildly overreserve - it doesn't matter much. */ 235*8d7f2e76SPhilippe Mathieu-Daudé phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS); 236*8d7f2e76SPhilippe Mathieu-Daudé 237*8d7f2e76SPhilippe Mathieu-Daudé phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); 238*8d7f2e76SPhilippe Mathieu-Daudé } 239*8d7f2e76SPhilippe Mathieu-Daudé 240*8d7f2e76SPhilippe Mathieu-Daudé /* Compact a non leaf page entry. Simply detect that the entry has a single child, 241*8d7f2e76SPhilippe Mathieu-Daudé * and update our entry so we can skip it and go directly to the destination. 242*8d7f2e76SPhilippe Mathieu-Daudé */ 243*8d7f2e76SPhilippe Mathieu-Daudé static void phys_page_compact(PhysPageEntry *lp, Node *nodes) 244*8d7f2e76SPhilippe Mathieu-Daudé { 245*8d7f2e76SPhilippe Mathieu-Daudé unsigned valid_ptr = P_L2_SIZE; 246*8d7f2e76SPhilippe Mathieu-Daudé int valid = 0; 247*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry *p; 248*8d7f2e76SPhilippe Mathieu-Daudé int i; 249*8d7f2e76SPhilippe Mathieu-Daudé 250*8d7f2e76SPhilippe Mathieu-Daudé if (lp->ptr == PHYS_MAP_NODE_NIL) { 251*8d7f2e76SPhilippe Mathieu-Daudé return; 252*8d7f2e76SPhilippe Mathieu-Daudé } 253*8d7f2e76SPhilippe Mathieu-Daudé 254*8d7f2e76SPhilippe Mathieu-Daudé p = nodes[lp->ptr]; 255*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < P_L2_SIZE; i++) { 256*8d7f2e76SPhilippe Mathieu-Daudé if (p[i].ptr == PHYS_MAP_NODE_NIL) { 257*8d7f2e76SPhilippe Mathieu-Daudé continue; 258*8d7f2e76SPhilippe Mathieu-Daudé } 259*8d7f2e76SPhilippe Mathieu-Daudé 260*8d7f2e76SPhilippe Mathieu-Daudé valid_ptr = i; 261*8d7f2e76SPhilippe Mathieu-Daudé valid++; 262*8d7f2e76SPhilippe Mathieu-Daudé if (p[i].skip) { 263*8d7f2e76SPhilippe Mathieu-Daudé phys_page_compact(&p[i], nodes); 264*8d7f2e76SPhilippe Mathieu-Daudé } 265*8d7f2e76SPhilippe Mathieu-Daudé } 266*8d7f2e76SPhilippe Mathieu-Daudé 267*8d7f2e76SPhilippe Mathieu-Daudé /* We can only compress if there's only one child. */ 268*8d7f2e76SPhilippe Mathieu-Daudé if (valid != 1) { 269*8d7f2e76SPhilippe Mathieu-Daudé return; 270*8d7f2e76SPhilippe Mathieu-Daudé } 271*8d7f2e76SPhilippe Mathieu-Daudé 272*8d7f2e76SPhilippe Mathieu-Daudé assert(valid_ptr < P_L2_SIZE); 273*8d7f2e76SPhilippe Mathieu-Daudé 274*8d7f2e76SPhilippe Mathieu-Daudé /* Don't compress if it won't fit in the # of bits we have. */ 275*8d7f2e76SPhilippe Mathieu-Daudé if (P_L2_LEVELS >= (1 << 6) && 276*8d7f2e76SPhilippe Mathieu-Daudé lp->skip + p[valid_ptr].skip >= (1 << 6)) { 277*8d7f2e76SPhilippe Mathieu-Daudé return; 278*8d7f2e76SPhilippe Mathieu-Daudé } 279*8d7f2e76SPhilippe Mathieu-Daudé 280*8d7f2e76SPhilippe Mathieu-Daudé lp->ptr = p[valid_ptr].ptr; 281*8d7f2e76SPhilippe Mathieu-Daudé if (!p[valid_ptr].skip) { 282*8d7f2e76SPhilippe Mathieu-Daudé /* If our only child is a leaf, make this a leaf. */ 283*8d7f2e76SPhilippe Mathieu-Daudé /* By design, we should have made this node a leaf to begin with so we 284*8d7f2e76SPhilippe Mathieu-Daudé * should never reach here. 285*8d7f2e76SPhilippe Mathieu-Daudé * But since it's so simple to handle this, let's do it just in case we 286*8d7f2e76SPhilippe Mathieu-Daudé * change this rule. 287*8d7f2e76SPhilippe Mathieu-Daudé */ 288*8d7f2e76SPhilippe Mathieu-Daudé lp->skip = 0; 289*8d7f2e76SPhilippe Mathieu-Daudé } else { 290*8d7f2e76SPhilippe Mathieu-Daudé lp->skip += p[valid_ptr].skip; 291*8d7f2e76SPhilippe Mathieu-Daudé } 292*8d7f2e76SPhilippe Mathieu-Daudé } 293*8d7f2e76SPhilippe Mathieu-Daudé 294*8d7f2e76SPhilippe Mathieu-Daudé void address_space_dispatch_compact(AddressSpaceDispatch *d) 295*8d7f2e76SPhilippe Mathieu-Daudé { 296*8d7f2e76SPhilippe Mathieu-Daudé if (d->phys_map.skip) { 297*8d7f2e76SPhilippe Mathieu-Daudé phys_page_compact(&d->phys_map, d->map.nodes); 298*8d7f2e76SPhilippe Mathieu-Daudé } 299*8d7f2e76SPhilippe Mathieu-Daudé } 300*8d7f2e76SPhilippe Mathieu-Daudé 301*8d7f2e76SPhilippe Mathieu-Daudé static inline bool section_covers_addr(const MemoryRegionSection *section, 302*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr) 303*8d7f2e76SPhilippe Mathieu-Daudé { 304*8d7f2e76SPhilippe Mathieu-Daudé /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means 305*8d7f2e76SPhilippe Mathieu-Daudé * the section must cover the entire address space. 306*8d7f2e76SPhilippe Mathieu-Daudé */ 307*8d7f2e76SPhilippe Mathieu-Daudé return int128_gethi(section->size) || 308*8d7f2e76SPhilippe Mathieu-Daudé range_covers_byte(section->offset_within_address_space, 309*8d7f2e76SPhilippe Mathieu-Daudé int128_getlo(section->size), addr); 310*8d7f2e76SPhilippe Mathieu-Daudé } 311*8d7f2e76SPhilippe Mathieu-Daudé 312*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr) 313*8d7f2e76SPhilippe Mathieu-Daudé { 314*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry lp = d->phys_map, *p; 315*8d7f2e76SPhilippe Mathieu-Daudé Node *nodes = d->map.nodes; 316*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *sections = d->map.sections; 317*8d7f2e76SPhilippe Mathieu-Daudé hwaddr index = addr >> TARGET_PAGE_BITS; 318*8d7f2e76SPhilippe Mathieu-Daudé int i; 319*8d7f2e76SPhilippe Mathieu-Daudé 320*8d7f2e76SPhilippe Mathieu-Daudé for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) { 321*8d7f2e76SPhilippe Mathieu-Daudé if (lp.ptr == PHYS_MAP_NODE_NIL) { 322*8d7f2e76SPhilippe Mathieu-Daudé return §ions[PHYS_SECTION_UNASSIGNED]; 323*8d7f2e76SPhilippe Mathieu-Daudé } 324*8d7f2e76SPhilippe Mathieu-Daudé p = nodes[lp.ptr]; 325*8d7f2e76SPhilippe Mathieu-Daudé lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)]; 326*8d7f2e76SPhilippe Mathieu-Daudé } 327*8d7f2e76SPhilippe Mathieu-Daudé 328*8d7f2e76SPhilippe Mathieu-Daudé if (section_covers_addr(§ions[lp.ptr], addr)) { 329*8d7f2e76SPhilippe Mathieu-Daudé return §ions[lp.ptr]; 330*8d7f2e76SPhilippe Mathieu-Daudé } else { 331*8d7f2e76SPhilippe Mathieu-Daudé return §ions[PHYS_SECTION_UNASSIGNED]; 332*8d7f2e76SPhilippe Mathieu-Daudé } 333*8d7f2e76SPhilippe Mathieu-Daudé } 334*8d7f2e76SPhilippe Mathieu-Daudé 335*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 336*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d, 337*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 338*8d7f2e76SPhilippe Mathieu-Daudé bool resolve_subpage) 339*8d7f2e76SPhilippe Mathieu-Daudé { 340*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section = qatomic_read(&d->mru_section); 341*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage; 342*8d7f2e76SPhilippe Mathieu-Daudé 343*8d7f2e76SPhilippe Mathieu-Daudé if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] || 344*8d7f2e76SPhilippe Mathieu-Daudé !section_covers_addr(section, addr)) { 345*8d7f2e76SPhilippe Mathieu-Daudé section = phys_page_find(d, addr); 346*8d7f2e76SPhilippe Mathieu-Daudé qatomic_set(&d->mru_section, section); 347*8d7f2e76SPhilippe Mathieu-Daudé } 348*8d7f2e76SPhilippe Mathieu-Daudé if (resolve_subpage && section->mr->subpage) { 349*8d7f2e76SPhilippe Mathieu-Daudé subpage = container_of(section->mr, subpage_t, iomem); 350*8d7f2e76SPhilippe Mathieu-Daudé section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]]; 351*8d7f2e76SPhilippe Mathieu-Daudé } 352*8d7f2e76SPhilippe Mathieu-Daudé return section; 353*8d7f2e76SPhilippe Mathieu-Daudé } 354*8d7f2e76SPhilippe Mathieu-Daudé 355*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 356*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegionSection * 357*8d7f2e76SPhilippe Mathieu-Daudé address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat, 358*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen, bool resolve_subpage) 359*8d7f2e76SPhilippe Mathieu-Daudé { 360*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section; 361*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 362*8d7f2e76SPhilippe Mathieu-Daudé Int128 diff; 363*8d7f2e76SPhilippe Mathieu-Daudé 364*8d7f2e76SPhilippe Mathieu-Daudé section = address_space_lookup_region(d, addr, resolve_subpage); 365*8d7f2e76SPhilippe Mathieu-Daudé /* Compute offset within MemoryRegionSection */ 366*8d7f2e76SPhilippe Mathieu-Daudé addr -= section->offset_within_address_space; 367*8d7f2e76SPhilippe Mathieu-Daudé 368*8d7f2e76SPhilippe Mathieu-Daudé /* Compute offset within MemoryRegion */ 369*8d7f2e76SPhilippe Mathieu-Daudé *xlat = addr + section->offset_within_region; 370*8d7f2e76SPhilippe Mathieu-Daudé 371*8d7f2e76SPhilippe Mathieu-Daudé mr = section->mr; 372*8d7f2e76SPhilippe Mathieu-Daudé 373*8d7f2e76SPhilippe Mathieu-Daudé /* MMIO registers can be expected to perform full-width accesses based only 374*8d7f2e76SPhilippe Mathieu-Daudé * on their address, without considering adjacent registers that could 375*8d7f2e76SPhilippe Mathieu-Daudé * decode to completely different MemoryRegions. When such registers 376*8d7f2e76SPhilippe Mathieu-Daudé * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO 377*8d7f2e76SPhilippe Mathieu-Daudé * regions overlap wildly. For this reason we cannot clamp the accesses 378*8d7f2e76SPhilippe Mathieu-Daudé * here. 379*8d7f2e76SPhilippe Mathieu-Daudé * 380*8d7f2e76SPhilippe Mathieu-Daudé * If the length is small (as is the case for address_space_ldl/stl), 381*8d7f2e76SPhilippe Mathieu-Daudé * everything works fine. If the incoming length is large, however, 382*8d7f2e76SPhilippe Mathieu-Daudé * the caller really has to do the clamping through memory_access_size. 383*8d7f2e76SPhilippe Mathieu-Daudé */ 384*8d7f2e76SPhilippe Mathieu-Daudé if (memory_region_is_ram(mr)) { 385*8d7f2e76SPhilippe Mathieu-Daudé diff = int128_sub(section->size, int128_make64(addr)); 386*8d7f2e76SPhilippe Mathieu-Daudé *plen = int128_get64(int128_min(diff, int128_make64(*plen))); 387*8d7f2e76SPhilippe Mathieu-Daudé } 388*8d7f2e76SPhilippe Mathieu-Daudé return section; 389*8d7f2e76SPhilippe Mathieu-Daudé } 390*8d7f2e76SPhilippe Mathieu-Daudé 391*8d7f2e76SPhilippe Mathieu-Daudé /** 392*8d7f2e76SPhilippe Mathieu-Daudé * address_space_translate_iommu - translate an address through an IOMMU 393*8d7f2e76SPhilippe Mathieu-Daudé * memory region and then through the target address space. 394*8d7f2e76SPhilippe Mathieu-Daudé * 395*8d7f2e76SPhilippe Mathieu-Daudé * @iommu_mr: the IOMMU memory region that we start the translation from 396*8d7f2e76SPhilippe Mathieu-Daudé * @addr: the address to be translated through the MMU 397*8d7f2e76SPhilippe Mathieu-Daudé * @xlat: the translated address offset within the destination memory region. 398*8d7f2e76SPhilippe Mathieu-Daudé * It cannot be %NULL. 399*8d7f2e76SPhilippe Mathieu-Daudé * @plen_out: valid read/write length of the translated address. It 400*8d7f2e76SPhilippe Mathieu-Daudé * cannot be %NULL. 401*8d7f2e76SPhilippe Mathieu-Daudé * @page_mask_out: page mask for the translated address. This 402*8d7f2e76SPhilippe Mathieu-Daudé * should only be meaningful for IOMMU translated 403*8d7f2e76SPhilippe Mathieu-Daudé * addresses, since there may be huge pages that this bit 404*8d7f2e76SPhilippe Mathieu-Daudé * would tell. It can be %NULL if we don't care about it. 405*8d7f2e76SPhilippe Mathieu-Daudé * @is_write: whether the translation operation is for write 406*8d7f2e76SPhilippe Mathieu-Daudé * @is_mmio: whether this can be MMIO, set true if it can 407*8d7f2e76SPhilippe Mathieu-Daudé * @target_as: the address space targeted by the IOMMU 408*8d7f2e76SPhilippe Mathieu-Daudé * @attrs: transaction attributes 409*8d7f2e76SPhilippe Mathieu-Daudé * 410*8d7f2e76SPhilippe Mathieu-Daudé * This function is called from RCU critical section. It is the common 411*8d7f2e76SPhilippe Mathieu-Daudé * part of flatview_do_translate and address_space_translate_cached. 412*8d7f2e76SPhilippe Mathieu-Daudé */ 413*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr, 414*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *xlat, 415*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen_out, 416*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *page_mask_out, 417*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, 418*8d7f2e76SPhilippe Mathieu-Daudé bool is_mmio, 419*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace **target_as, 420*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 421*8d7f2e76SPhilippe Mathieu-Daudé { 422*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section; 423*8d7f2e76SPhilippe Mathieu-Daudé hwaddr page_mask = (hwaddr)-1; 424*8d7f2e76SPhilippe Mathieu-Daudé 425*8d7f2e76SPhilippe Mathieu-Daudé do { 426*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr = *xlat; 427*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr); 428*8d7f2e76SPhilippe Mathieu-Daudé int iommu_idx = 0; 429*8d7f2e76SPhilippe Mathieu-Daudé IOMMUTLBEntry iotlb; 430*8d7f2e76SPhilippe Mathieu-Daudé 431*8d7f2e76SPhilippe Mathieu-Daudé if (imrc->attrs_to_index) { 432*8d7f2e76SPhilippe Mathieu-Daudé iommu_idx = imrc->attrs_to_index(iommu_mr, attrs); 433*8d7f2e76SPhilippe Mathieu-Daudé } 434*8d7f2e76SPhilippe Mathieu-Daudé 435*8d7f2e76SPhilippe Mathieu-Daudé iotlb = imrc->translate(iommu_mr, addr, is_write ? 436*8d7f2e76SPhilippe Mathieu-Daudé IOMMU_WO : IOMMU_RO, iommu_idx); 437*8d7f2e76SPhilippe Mathieu-Daudé 438*8d7f2e76SPhilippe Mathieu-Daudé if (!(iotlb.perm & (1 << is_write))) { 439*8d7f2e76SPhilippe Mathieu-Daudé goto unassigned; 440*8d7f2e76SPhilippe Mathieu-Daudé } 441*8d7f2e76SPhilippe Mathieu-Daudé 442*8d7f2e76SPhilippe Mathieu-Daudé addr = ((iotlb.translated_addr & ~iotlb.addr_mask) 443*8d7f2e76SPhilippe Mathieu-Daudé | (addr & iotlb.addr_mask)); 444*8d7f2e76SPhilippe Mathieu-Daudé page_mask &= iotlb.addr_mask; 445*8d7f2e76SPhilippe Mathieu-Daudé *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1); 446*8d7f2e76SPhilippe Mathieu-Daudé *target_as = iotlb.target_as; 447*8d7f2e76SPhilippe Mathieu-Daudé 448*8d7f2e76SPhilippe Mathieu-Daudé section = address_space_translate_internal( 449*8d7f2e76SPhilippe Mathieu-Daudé address_space_to_dispatch(iotlb.target_as), addr, xlat, 450*8d7f2e76SPhilippe Mathieu-Daudé plen_out, is_mmio); 451*8d7f2e76SPhilippe Mathieu-Daudé 452*8d7f2e76SPhilippe Mathieu-Daudé iommu_mr = memory_region_get_iommu(section->mr); 453*8d7f2e76SPhilippe Mathieu-Daudé } while (unlikely(iommu_mr)); 454*8d7f2e76SPhilippe Mathieu-Daudé 455*8d7f2e76SPhilippe Mathieu-Daudé if (page_mask_out) { 456*8d7f2e76SPhilippe Mathieu-Daudé *page_mask_out = page_mask; 457*8d7f2e76SPhilippe Mathieu-Daudé } 458*8d7f2e76SPhilippe Mathieu-Daudé return *section; 459*8d7f2e76SPhilippe Mathieu-Daudé 460*8d7f2e76SPhilippe Mathieu-Daudé unassigned: 461*8d7f2e76SPhilippe Mathieu-Daudé return (MemoryRegionSection) { .mr = &io_mem_unassigned }; 462*8d7f2e76SPhilippe Mathieu-Daudé } 463*8d7f2e76SPhilippe Mathieu-Daudé 464*8d7f2e76SPhilippe Mathieu-Daudé /** 465*8d7f2e76SPhilippe Mathieu-Daudé * flatview_do_translate - translate an address in FlatView 466*8d7f2e76SPhilippe Mathieu-Daudé * 467*8d7f2e76SPhilippe Mathieu-Daudé * @fv: the flat view that we want to translate on 468*8d7f2e76SPhilippe Mathieu-Daudé * @addr: the address to be translated in above address space 469*8d7f2e76SPhilippe Mathieu-Daudé * @xlat: the translated address offset within memory region. It 470*8d7f2e76SPhilippe Mathieu-Daudé * cannot be @NULL. 471*8d7f2e76SPhilippe Mathieu-Daudé * @plen_out: valid read/write length of the translated address. It 472*8d7f2e76SPhilippe Mathieu-Daudé * can be @NULL when we don't care about it. 473*8d7f2e76SPhilippe Mathieu-Daudé * @page_mask_out: page mask for the translated address. This 474*8d7f2e76SPhilippe Mathieu-Daudé * should only be meaningful for IOMMU translated 475*8d7f2e76SPhilippe Mathieu-Daudé * addresses, since there may be huge pages that this bit 476*8d7f2e76SPhilippe Mathieu-Daudé * would tell. It can be @NULL if we don't care about it. 477*8d7f2e76SPhilippe Mathieu-Daudé * @is_write: whether the translation operation is for write 478*8d7f2e76SPhilippe Mathieu-Daudé * @is_mmio: whether this can be MMIO, set true if it can 479*8d7f2e76SPhilippe Mathieu-Daudé * @target_as: the address space targeted by the IOMMU 480*8d7f2e76SPhilippe Mathieu-Daudé * @attrs: memory transaction attributes 481*8d7f2e76SPhilippe Mathieu-Daudé * 482*8d7f2e76SPhilippe Mathieu-Daudé * This function is called from RCU critical section 483*8d7f2e76SPhilippe Mathieu-Daudé */ 484*8d7f2e76SPhilippe Mathieu-Daudé static MemoryRegionSection flatview_do_translate(FlatView *fv, 485*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 486*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *xlat, 487*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen_out, 488*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *page_mask_out, 489*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, 490*8d7f2e76SPhilippe Mathieu-Daudé bool is_mmio, 491*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace **target_as, 492*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 493*8d7f2e76SPhilippe Mathieu-Daudé { 494*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section; 495*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegion *iommu_mr; 496*8d7f2e76SPhilippe Mathieu-Daudé hwaddr plen = (hwaddr)(-1); 497*8d7f2e76SPhilippe Mathieu-Daudé 498*8d7f2e76SPhilippe Mathieu-Daudé if (!plen_out) { 499*8d7f2e76SPhilippe Mathieu-Daudé plen_out = &plen; 500*8d7f2e76SPhilippe Mathieu-Daudé } 501*8d7f2e76SPhilippe Mathieu-Daudé 502*8d7f2e76SPhilippe Mathieu-Daudé section = address_space_translate_internal( 503*8d7f2e76SPhilippe Mathieu-Daudé flatview_to_dispatch(fv), addr, xlat, 504*8d7f2e76SPhilippe Mathieu-Daudé plen_out, is_mmio); 505*8d7f2e76SPhilippe Mathieu-Daudé 506*8d7f2e76SPhilippe Mathieu-Daudé iommu_mr = memory_region_get_iommu(section->mr); 507*8d7f2e76SPhilippe Mathieu-Daudé if (unlikely(iommu_mr)) { 508*8d7f2e76SPhilippe Mathieu-Daudé return address_space_translate_iommu(iommu_mr, xlat, 509*8d7f2e76SPhilippe Mathieu-Daudé plen_out, page_mask_out, 510*8d7f2e76SPhilippe Mathieu-Daudé is_write, is_mmio, 511*8d7f2e76SPhilippe Mathieu-Daudé target_as, attrs); 512*8d7f2e76SPhilippe Mathieu-Daudé } 513*8d7f2e76SPhilippe Mathieu-Daudé if (page_mask_out) { 514*8d7f2e76SPhilippe Mathieu-Daudé /* Not behind an IOMMU, use default page size. */ 515*8d7f2e76SPhilippe Mathieu-Daudé *page_mask_out = ~TARGET_PAGE_MASK; 516*8d7f2e76SPhilippe Mathieu-Daudé } 517*8d7f2e76SPhilippe Mathieu-Daudé 518*8d7f2e76SPhilippe Mathieu-Daudé return *section; 519*8d7f2e76SPhilippe Mathieu-Daudé } 520*8d7f2e76SPhilippe Mathieu-Daudé 521*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 522*8d7f2e76SPhilippe Mathieu-Daudé IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr, 523*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, MemTxAttrs attrs) 524*8d7f2e76SPhilippe Mathieu-Daudé { 525*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection section; 526*8d7f2e76SPhilippe Mathieu-Daudé hwaddr xlat, page_mask; 527*8d7f2e76SPhilippe Mathieu-Daudé 528*8d7f2e76SPhilippe Mathieu-Daudé /* 529*8d7f2e76SPhilippe Mathieu-Daudé * This can never be MMIO, and we don't really care about plen, 530*8d7f2e76SPhilippe Mathieu-Daudé * but page mask. 531*8d7f2e76SPhilippe Mathieu-Daudé */ 532*8d7f2e76SPhilippe Mathieu-Daudé section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat, 533*8d7f2e76SPhilippe Mathieu-Daudé NULL, &page_mask, is_write, false, &as, 534*8d7f2e76SPhilippe Mathieu-Daudé attrs); 535*8d7f2e76SPhilippe Mathieu-Daudé 536*8d7f2e76SPhilippe Mathieu-Daudé /* Illegal translation */ 537*8d7f2e76SPhilippe Mathieu-Daudé if (section.mr == &io_mem_unassigned) { 538*8d7f2e76SPhilippe Mathieu-Daudé goto iotlb_fail; 539*8d7f2e76SPhilippe Mathieu-Daudé } 540*8d7f2e76SPhilippe Mathieu-Daudé 541*8d7f2e76SPhilippe Mathieu-Daudé /* Convert memory region offset into address space offset */ 542*8d7f2e76SPhilippe Mathieu-Daudé xlat += section.offset_within_address_space - 543*8d7f2e76SPhilippe Mathieu-Daudé section.offset_within_region; 544*8d7f2e76SPhilippe Mathieu-Daudé 545*8d7f2e76SPhilippe Mathieu-Daudé return (IOMMUTLBEntry) { 546*8d7f2e76SPhilippe Mathieu-Daudé .target_as = as, 547*8d7f2e76SPhilippe Mathieu-Daudé .iova = addr & ~page_mask, 548*8d7f2e76SPhilippe Mathieu-Daudé .translated_addr = xlat & ~page_mask, 549*8d7f2e76SPhilippe Mathieu-Daudé .addr_mask = page_mask, 550*8d7f2e76SPhilippe Mathieu-Daudé /* IOTLBs are for DMAs, and DMA only allows on RAMs. */ 551*8d7f2e76SPhilippe Mathieu-Daudé .perm = IOMMU_RW, 552*8d7f2e76SPhilippe Mathieu-Daudé }; 553*8d7f2e76SPhilippe Mathieu-Daudé 554*8d7f2e76SPhilippe Mathieu-Daudé iotlb_fail: 555*8d7f2e76SPhilippe Mathieu-Daudé return (IOMMUTLBEntry) {0}; 556*8d7f2e76SPhilippe Mathieu-Daudé } 557*8d7f2e76SPhilippe Mathieu-Daudé 558*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 559*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat, 560*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen, bool is_write, 561*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 562*8d7f2e76SPhilippe Mathieu-Daudé { 563*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 564*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection section; 565*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *as = NULL; 566*8d7f2e76SPhilippe Mathieu-Daudé 567*8d7f2e76SPhilippe Mathieu-Daudé /* This can be MMIO, so setup MMIO bit. */ 568*8d7f2e76SPhilippe Mathieu-Daudé section = flatview_do_translate(fv, addr, xlat, plen, NULL, 569*8d7f2e76SPhilippe Mathieu-Daudé is_write, true, &as, attrs); 570*8d7f2e76SPhilippe Mathieu-Daudé mr = section.mr; 571*8d7f2e76SPhilippe Mathieu-Daudé 572*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled() && memory_access_is_direct(mr, is_write)) { 573*8d7f2e76SPhilippe Mathieu-Daudé hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr; 574*8d7f2e76SPhilippe Mathieu-Daudé *plen = MIN(page, *plen); 575*8d7f2e76SPhilippe Mathieu-Daudé } 576*8d7f2e76SPhilippe Mathieu-Daudé 577*8d7f2e76SPhilippe Mathieu-Daudé return mr; 578*8d7f2e76SPhilippe Mathieu-Daudé } 579*8d7f2e76SPhilippe Mathieu-Daudé 580*8d7f2e76SPhilippe Mathieu-Daudé typedef struct TCGIOMMUNotifier { 581*8d7f2e76SPhilippe Mathieu-Daudé IOMMUNotifier n; 582*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 583*8d7f2e76SPhilippe Mathieu-Daudé CPUState *cpu; 584*8d7f2e76SPhilippe Mathieu-Daudé int iommu_idx; 585*8d7f2e76SPhilippe Mathieu-Daudé bool active; 586*8d7f2e76SPhilippe Mathieu-Daudé } TCGIOMMUNotifier; 587*8d7f2e76SPhilippe Mathieu-Daudé 588*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 589*8d7f2e76SPhilippe Mathieu-Daudé { 590*8d7f2e76SPhilippe Mathieu-Daudé TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n); 591*8d7f2e76SPhilippe Mathieu-Daudé 592*8d7f2e76SPhilippe Mathieu-Daudé if (!notifier->active) { 593*8d7f2e76SPhilippe Mathieu-Daudé return; 594*8d7f2e76SPhilippe Mathieu-Daudé } 595*8d7f2e76SPhilippe Mathieu-Daudé tlb_flush(notifier->cpu); 596*8d7f2e76SPhilippe Mathieu-Daudé notifier->active = false; 597*8d7f2e76SPhilippe Mathieu-Daudé /* We leave the notifier struct on the list to avoid reallocating it later. 598*8d7f2e76SPhilippe Mathieu-Daudé * Generally the number of IOMMUs a CPU deals with will be small. 599*8d7f2e76SPhilippe Mathieu-Daudé * In any case we can't unregister the iommu notifier from a notify 600*8d7f2e76SPhilippe Mathieu-Daudé * callback. 601*8d7f2e76SPhilippe Mathieu-Daudé */ 602*8d7f2e76SPhilippe Mathieu-Daudé } 603*8d7f2e76SPhilippe Mathieu-Daudé 604*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_register_iommu_notifier(CPUState *cpu, 605*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegion *iommu_mr, 606*8d7f2e76SPhilippe Mathieu-Daudé int iommu_idx) 607*8d7f2e76SPhilippe Mathieu-Daudé { 608*8d7f2e76SPhilippe Mathieu-Daudé /* Make sure this CPU has an IOMMU notifier registered for this 609*8d7f2e76SPhilippe Mathieu-Daudé * IOMMU/IOMMU index combination, so that we can flush its TLB 610*8d7f2e76SPhilippe Mathieu-Daudé * when the IOMMU tells us the mappings we've cached have changed. 611*8d7f2e76SPhilippe Mathieu-Daudé */ 612*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr = MEMORY_REGION(iommu_mr); 613*8d7f2e76SPhilippe Mathieu-Daudé TCGIOMMUNotifier *notifier = NULL; 614*8d7f2e76SPhilippe Mathieu-Daudé int i; 615*8d7f2e76SPhilippe Mathieu-Daudé 616*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < cpu->iommu_notifiers->len; i++) { 617*8d7f2e76SPhilippe Mathieu-Daudé notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); 618*8d7f2e76SPhilippe Mathieu-Daudé if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) { 619*8d7f2e76SPhilippe Mathieu-Daudé break; 620*8d7f2e76SPhilippe Mathieu-Daudé } 621*8d7f2e76SPhilippe Mathieu-Daudé } 622*8d7f2e76SPhilippe Mathieu-Daudé if (i == cpu->iommu_notifiers->len) { 623*8d7f2e76SPhilippe Mathieu-Daudé /* Not found, add a new entry at the end of the array */ 624*8d7f2e76SPhilippe Mathieu-Daudé cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1); 625*8d7f2e76SPhilippe Mathieu-Daudé notifier = g_new0(TCGIOMMUNotifier, 1); 626*8d7f2e76SPhilippe Mathieu-Daudé g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier; 627*8d7f2e76SPhilippe Mathieu-Daudé 628*8d7f2e76SPhilippe Mathieu-Daudé notifier->mr = mr; 629*8d7f2e76SPhilippe Mathieu-Daudé notifier->iommu_idx = iommu_idx; 630*8d7f2e76SPhilippe Mathieu-Daudé notifier->cpu = cpu; 631*8d7f2e76SPhilippe Mathieu-Daudé /* Rather than trying to register interest in the specific part 632*8d7f2e76SPhilippe Mathieu-Daudé * of the iommu's address space that we've accessed and then 633*8d7f2e76SPhilippe Mathieu-Daudé * expand it later as subsequent accesses touch more of it, we 634*8d7f2e76SPhilippe Mathieu-Daudé * just register interest in the whole thing, on the assumption 635*8d7f2e76SPhilippe Mathieu-Daudé * that iommu reconfiguration will be rare. 636*8d7f2e76SPhilippe Mathieu-Daudé */ 637*8d7f2e76SPhilippe Mathieu-Daudé iommu_notifier_init(¬ifier->n, 638*8d7f2e76SPhilippe Mathieu-Daudé tcg_iommu_unmap_notify, 639*8d7f2e76SPhilippe Mathieu-Daudé IOMMU_NOTIFIER_UNMAP, 640*8d7f2e76SPhilippe Mathieu-Daudé 0, 641*8d7f2e76SPhilippe Mathieu-Daudé HWADDR_MAX, 642*8d7f2e76SPhilippe Mathieu-Daudé iommu_idx); 643*8d7f2e76SPhilippe Mathieu-Daudé memory_region_register_iommu_notifier(notifier->mr, ¬ifier->n, 644*8d7f2e76SPhilippe Mathieu-Daudé &error_fatal); 645*8d7f2e76SPhilippe Mathieu-Daudé } 646*8d7f2e76SPhilippe Mathieu-Daudé 647*8d7f2e76SPhilippe Mathieu-Daudé if (!notifier->active) { 648*8d7f2e76SPhilippe Mathieu-Daudé notifier->active = true; 649*8d7f2e76SPhilippe Mathieu-Daudé } 650*8d7f2e76SPhilippe Mathieu-Daudé } 651*8d7f2e76SPhilippe Mathieu-Daudé 652*8d7f2e76SPhilippe Mathieu-Daudé void tcg_iommu_free_notifier_list(CPUState *cpu) 653*8d7f2e76SPhilippe Mathieu-Daudé { 654*8d7f2e76SPhilippe Mathieu-Daudé /* Destroy the CPU's notifier list */ 655*8d7f2e76SPhilippe Mathieu-Daudé int i; 656*8d7f2e76SPhilippe Mathieu-Daudé TCGIOMMUNotifier *notifier; 657*8d7f2e76SPhilippe Mathieu-Daudé 658*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < cpu->iommu_notifiers->len; i++) { 659*8d7f2e76SPhilippe Mathieu-Daudé notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i); 660*8d7f2e76SPhilippe Mathieu-Daudé memory_region_unregister_iommu_notifier(notifier->mr, ¬ifier->n); 661*8d7f2e76SPhilippe Mathieu-Daudé g_free(notifier); 662*8d7f2e76SPhilippe Mathieu-Daudé } 663*8d7f2e76SPhilippe Mathieu-Daudé g_array_free(cpu->iommu_notifiers, true); 664*8d7f2e76SPhilippe Mathieu-Daudé } 665*8d7f2e76SPhilippe Mathieu-Daudé 666*8d7f2e76SPhilippe Mathieu-Daudé void tcg_iommu_init_notifier_list(CPUState *cpu) 667*8d7f2e76SPhilippe Mathieu-Daudé { 668*8d7f2e76SPhilippe Mathieu-Daudé cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *)); 669*8d7f2e76SPhilippe Mathieu-Daudé } 670*8d7f2e76SPhilippe Mathieu-Daudé 671*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 672*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection * 673*8d7f2e76SPhilippe Mathieu-Daudé address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr, 674*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *xlat, hwaddr *plen, 675*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, int *prot) 676*8d7f2e76SPhilippe Mathieu-Daudé { 677*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section; 678*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegion *iommu_mr; 679*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegionClass *imrc; 680*8d7f2e76SPhilippe Mathieu-Daudé IOMMUTLBEntry iotlb; 681*8d7f2e76SPhilippe Mathieu-Daudé int iommu_idx; 682*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr = orig_addr; 683*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch; 684*8d7f2e76SPhilippe Mathieu-Daudé 685*8d7f2e76SPhilippe Mathieu-Daudé for (;;) { 686*8d7f2e76SPhilippe Mathieu-Daudé section = address_space_translate_internal(d, addr, &addr, plen, false); 687*8d7f2e76SPhilippe Mathieu-Daudé 688*8d7f2e76SPhilippe Mathieu-Daudé iommu_mr = memory_region_get_iommu(section->mr); 689*8d7f2e76SPhilippe Mathieu-Daudé if (!iommu_mr) { 690*8d7f2e76SPhilippe Mathieu-Daudé break; 691*8d7f2e76SPhilippe Mathieu-Daudé } 692*8d7f2e76SPhilippe Mathieu-Daudé 693*8d7f2e76SPhilippe Mathieu-Daudé imrc = memory_region_get_iommu_class_nocheck(iommu_mr); 694*8d7f2e76SPhilippe Mathieu-Daudé 695*8d7f2e76SPhilippe Mathieu-Daudé iommu_idx = imrc->attrs_to_index(iommu_mr, attrs); 696*8d7f2e76SPhilippe Mathieu-Daudé tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx); 697*8d7f2e76SPhilippe Mathieu-Daudé /* We need all the permissions, so pass IOMMU_NONE so the IOMMU 698*8d7f2e76SPhilippe Mathieu-Daudé * doesn't short-cut its translation table walk. 699*8d7f2e76SPhilippe Mathieu-Daudé */ 700*8d7f2e76SPhilippe Mathieu-Daudé iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx); 701*8d7f2e76SPhilippe Mathieu-Daudé addr = ((iotlb.translated_addr & ~iotlb.addr_mask) 702*8d7f2e76SPhilippe Mathieu-Daudé | (addr & iotlb.addr_mask)); 703*8d7f2e76SPhilippe Mathieu-Daudé /* Update the caller's prot bits to remove permissions the IOMMU 704*8d7f2e76SPhilippe Mathieu-Daudé * is giving us a failure response for. If we get down to no 705*8d7f2e76SPhilippe Mathieu-Daudé * permissions left at all we can give up now. 706*8d7f2e76SPhilippe Mathieu-Daudé */ 707*8d7f2e76SPhilippe Mathieu-Daudé if (!(iotlb.perm & IOMMU_RO)) { 708*8d7f2e76SPhilippe Mathieu-Daudé *prot &= ~(PAGE_READ | PAGE_EXEC); 709*8d7f2e76SPhilippe Mathieu-Daudé } 710*8d7f2e76SPhilippe Mathieu-Daudé if (!(iotlb.perm & IOMMU_WO)) { 711*8d7f2e76SPhilippe Mathieu-Daudé *prot &= ~PAGE_WRITE; 712*8d7f2e76SPhilippe Mathieu-Daudé } 713*8d7f2e76SPhilippe Mathieu-Daudé 714*8d7f2e76SPhilippe Mathieu-Daudé if (!*prot) { 715*8d7f2e76SPhilippe Mathieu-Daudé goto translate_fail; 716*8d7f2e76SPhilippe Mathieu-Daudé } 717*8d7f2e76SPhilippe Mathieu-Daudé 718*8d7f2e76SPhilippe Mathieu-Daudé d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as)); 719*8d7f2e76SPhilippe Mathieu-Daudé } 720*8d7f2e76SPhilippe Mathieu-Daudé 721*8d7f2e76SPhilippe Mathieu-Daudé assert(!memory_region_is_iommu(section->mr)); 722*8d7f2e76SPhilippe Mathieu-Daudé *xlat = addr; 723*8d7f2e76SPhilippe Mathieu-Daudé return section; 724*8d7f2e76SPhilippe Mathieu-Daudé 725*8d7f2e76SPhilippe Mathieu-Daudé translate_fail: 726*8d7f2e76SPhilippe Mathieu-Daudé /* 727*8d7f2e76SPhilippe Mathieu-Daudé * We should be given a page-aligned address -- certainly 728*8d7f2e76SPhilippe Mathieu-Daudé * tlb_set_page_with_attrs() does so. The page offset of xlat 729*8d7f2e76SPhilippe Mathieu-Daudé * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0. 730*8d7f2e76SPhilippe Mathieu-Daudé * The page portion of xlat will be logged by memory_region_access_valid() 731*8d7f2e76SPhilippe Mathieu-Daudé * when this memory access is rejected, so use the original untranslated 732*8d7f2e76SPhilippe Mathieu-Daudé * physical address. 733*8d7f2e76SPhilippe Mathieu-Daudé */ 734*8d7f2e76SPhilippe Mathieu-Daudé assert((orig_addr & ~TARGET_PAGE_MASK) == 0); 735*8d7f2e76SPhilippe Mathieu-Daudé *xlat = orig_addr; 736*8d7f2e76SPhilippe Mathieu-Daudé return &d->map.sections[PHYS_SECTION_UNASSIGNED]; 737*8d7f2e76SPhilippe Mathieu-Daudé } 738*8d7f2e76SPhilippe Mathieu-Daudé 739*8d7f2e76SPhilippe Mathieu-Daudé void cpu_address_space_init(CPUState *cpu, int asidx, 740*8d7f2e76SPhilippe Mathieu-Daudé const char *prefix, MemoryRegion *mr) 741*8d7f2e76SPhilippe Mathieu-Daudé { 742*8d7f2e76SPhilippe Mathieu-Daudé CPUAddressSpace *newas; 743*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *as = g_new0(AddressSpace, 1); 744*8d7f2e76SPhilippe Mathieu-Daudé char *as_name; 745*8d7f2e76SPhilippe Mathieu-Daudé 746*8d7f2e76SPhilippe Mathieu-Daudé assert(mr); 747*8d7f2e76SPhilippe Mathieu-Daudé as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index); 748*8d7f2e76SPhilippe Mathieu-Daudé address_space_init(as, mr, as_name); 749*8d7f2e76SPhilippe Mathieu-Daudé g_free(as_name); 750*8d7f2e76SPhilippe Mathieu-Daudé 751*8d7f2e76SPhilippe Mathieu-Daudé /* Target code should have set num_ases before calling us */ 752*8d7f2e76SPhilippe Mathieu-Daudé assert(asidx < cpu->num_ases); 753*8d7f2e76SPhilippe Mathieu-Daudé 754*8d7f2e76SPhilippe Mathieu-Daudé if (asidx == 0) { 755*8d7f2e76SPhilippe Mathieu-Daudé /* address space 0 gets the convenience alias */ 756*8d7f2e76SPhilippe Mathieu-Daudé cpu->as = as; 757*8d7f2e76SPhilippe Mathieu-Daudé } 758*8d7f2e76SPhilippe Mathieu-Daudé 759*8d7f2e76SPhilippe Mathieu-Daudé /* KVM cannot currently support multiple address spaces. */ 760*8d7f2e76SPhilippe Mathieu-Daudé assert(asidx == 0 || !kvm_enabled()); 761*8d7f2e76SPhilippe Mathieu-Daudé 762*8d7f2e76SPhilippe Mathieu-Daudé if (!cpu->cpu_ases) { 763*8d7f2e76SPhilippe Mathieu-Daudé cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases); 764*8d7f2e76SPhilippe Mathieu-Daudé } 765*8d7f2e76SPhilippe Mathieu-Daudé 766*8d7f2e76SPhilippe Mathieu-Daudé newas = &cpu->cpu_ases[asidx]; 767*8d7f2e76SPhilippe Mathieu-Daudé newas->cpu = cpu; 768*8d7f2e76SPhilippe Mathieu-Daudé newas->as = as; 769*8d7f2e76SPhilippe Mathieu-Daudé if (tcg_enabled()) { 770*8d7f2e76SPhilippe Mathieu-Daudé newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync; 771*8d7f2e76SPhilippe Mathieu-Daudé newas->tcg_as_listener.commit = tcg_commit; 772*8d7f2e76SPhilippe Mathieu-Daudé newas->tcg_as_listener.name = "tcg"; 773*8d7f2e76SPhilippe Mathieu-Daudé memory_listener_register(&newas->tcg_as_listener, as); 774*8d7f2e76SPhilippe Mathieu-Daudé } 775*8d7f2e76SPhilippe Mathieu-Daudé } 776*8d7f2e76SPhilippe Mathieu-Daudé 777*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx) 778*8d7f2e76SPhilippe Mathieu-Daudé { 779*8d7f2e76SPhilippe Mathieu-Daudé /* Return the AddressSpace corresponding to the specified index */ 780*8d7f2e76SPhilippe Mathieu-Daudé return cpu->cpu_ases[asidx].as; 781*8d7f2e76SPhilippe Mathieu-Daudé } 782*8d7f2e76SPhilippe Mathieu-Daudé 783*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 784*8d7f2e76SPhilippe Mathieu-Daudé static RAMBlock *qemu_get_ram_block(ram_addr_t addr) 785*8d7f2e76SPhilippe Mathieu-Daudé { 786*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 787*8d7f2e76SPhilippe Mathieu-Daudé 788*8d7f2e76SPhilippe Mathieu-Daudé block = qatomic_rcu_read(&ram_list.mru_block); 789*8d7f2e76SPhilippe Mathieu-Daudé if (block && addr - block->offset < block->max_length) { 790*8d7f2e76SPhilippe Mathieu-Daudé return block; 791*8d7f2e76SPhilippe Mathieu-Daudé } 792*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 793*8d7f2e76SPhilippe Mathieu-Daudé if (addr - block->offset < block->max_length) { 794*8d7f2e76SPhilippe Mathieu-Daudé goto found; 795*8d7f2e76SPhilippe Mathieu-Daudé } 796*8d7f2e76SPhilippe Mathieu-Daudé } 797*8d7f2e76SPhilippe Mathieu-Daudé 798*8d7f2e76SPhilippe Mathieu-Daudé fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr); 799*8d7f2e76SPhilippe Mathieu-Daudé abort(); 800*8d7f2e76SPhilippe Mathieu-Daudé 801*8d7f2e76SPhilippe Mathieu-Daudé found: 802*8d7f2e76SPhilippe Mathieu-Daudé /* It is safe to write mru_block outside the iothread lock. This 803*8d7f2e76SPhilippe Mathieu-Daudé * is what happens: 804*8d7f2e76SPhilippe Mathieu-Daudé * 805*8d7f2e76SPhilippe Mathieu-Daudé * mru_block = xxx 806*8d7f2e76SPhilippe Mathieu-Daudé * rcu_read_unlock() 807*8d7f2e76SPhilippe Mathieu-Daudé * xxx removed from list 808*8d7f2e76SPhilippe Mathieu-Daudé * rcu_read_lock() 809*8d7f2e76SPhilippe Mathieu-Daudé * read mru_block 810*8d7f2e76SPhilippe Mathieu-Daudé * mru_block = NULL; 811*8d7f2e76SPhilippe Mathieu-Daudé * call_rcu(reclaim_ramblock, xxx); 812*8d7f2e76SPhilippe Mathieu-Daudé * rcu_read_unlock() 813*8d7f2e76SPhilippe Mathieu-Daudé * 814*8d7f2e76SPhilippe Mathieu-Daudé * qatomic_rcu_set is not needed here. The block was already published 815*8d7f2e76SPhilippe Mathieu-Daudé * when it was placed into the list. Here we're just making an extra 816*8d7f2e76SPhilippe Mathieu-Daudé * copy of the pointer. 817*8d7f2e76SPhilippe Mathieu-Daudé */ 818*8d7f2e76SPhilippe Mathieu-Daudé ram_list.mru_block = block; 819*8d7f2e76SPhilippe Mathieu-Daudé return block; 820*8d7f2e76SPhilippe Mathieu-Daudé } 821*8d7f2e76SPhilippe Mathieu-Daudé 822*8d7f2e76SPhilippe Mathieu-Daudé static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) 823*8d7f2e76SPhilippe Mathieu-Daudé { 824*8d7f2e76SPhilippe Mathieu-Daudé CPUState *cpu; 825*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t start1; 826*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 827*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t end; 828*8d7f2e76SPhilippe Mathieu-Daudé 829*8d7f2e76SPhilippe Mathieu-Daudé assert(tcg_enabled()); 830*8d7f2e76SPhilippe Mathieu-Daudé end = TARGET_PAGE_ALIGN(start + length); 831*8d7f2e76SPhilippe Mathieu-Daudé start &= TARGET_PAGE_MASK; 832*8d7f2e76SPhilippe Mathieu-Daudé 833*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 834*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_get_ram_block(start); 835*8d7f2e76SPhilippe Mathieu-Daudé assert(block == qemu_get_ram_block(end - 1)); 836*8d7f2e76SPhilippe Mathieu-Daudé start1 = (uintptr_t)ramblock_ptr(block, start - block->offset); 837*8d7f2e76SPhilippe Mathieu-Daudé CPU_FOREACH(cpu) { 838*8d7f2e76SPhilippe Mathieu-Daudé tlb_reset_dirty(cpu, start1, length); 839*8d7f2e76SPhilippe Mathieu-Daudé } 840*8d7f2e76SPhilippe Mathieu-Daudé } 841*8d7f2e76SPhilippe Mathieu-Daudé 842*8d7f2e76SPhilippe Mathieu-Daudé /* Note: start and end must be within the same ram block. */ 843*8d7f2e76SPhilippe Mathieu-Daudé bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 844*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t length, 845*8d7f2e76SPhilippe Mathieu-Daudé unsigned client) 846*8d7f2e76SPhilippe Mathieu-Daudé { 847*8d7f2e76SPhilippe Mathieu-Daudé DirtyMemoryBlocks *blocks; 848*8d7f2e76SPhilippe Mathieu-Daudé unsigned long end, page, start_page; 849*8d7f2e76SPhilippe Mathieu-Daudé bool dirty = false; 850*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *ramblock; 851*8d7f2e76SPhilippe Mathieu-Daudé uint64_t mr_offset, mr_size; 852*8d7f2e76SPhilippe Mathieu-Daudé 853*8d7f2e76SPhilippe Mathieu-Daudé if (length == 0) { 854*8d7f2e76SPhilippe Mathieu-Daudé return false; 855*8d7f2e76SPhilippe Mathieu-Daudé } 856*8d7f2e76SPhilippe Mathieu-Daudé 857*8d7f2e76SPhilippe Mathieu-Daudé end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 858*8d7f2e76SPhilippe Mathieu-Daudé start_page = start >> TARGET_PAGE_BITS; 859*8d7f2e76SPhilippe Mathieu-Daudé page = start_page; 860*8d7f2e76SPhilippe Mathieu-Daudé 861*8d7f2e76SPhilippe Mathieu-Daudé WITH_RCU_READ_LOCK_GUARD() { 862*8d7f2e76SPhilippe Mathieu-Daudé blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 863*8d7f2e76SPhilippe Mathieu-Daudé ramblock = qemu_get_ram_block(start); 864*8d7f2e76SPhilippe Mathieu-Daudé /* Range sanity check on the ramblock */ 865*8d7f2e76SPhilippe Mathieu-Daudé assert(start >= ramblock->offset && 866*8d7f2e76SPhilippe Mathieu-Daudé start + length <= ramblock->offset + ramblock->used_length); 867*8d7f2e76SPhilippe Mathieu-Daudé 868*8d7f2e76SPhilippe Mathieu-Daudé while (page < end) { 869*8d7f2e76SPhilippe Mathieu-Daudé unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE; 870*8d7f2e76SPhilippe Mathieu-Daudé unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE; 871*8d7f2e76SPhilippe Mathieu-Daudé unsigned long num = MIN(end - page, 872*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_MEMORY_BLOCK_SIZE - offset); 873*8d7f2e76SPhilippe Mathieu-Daudé 874*8d7f2e76SPhilippe Mathieu-Daudé dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx], 875*8d7f2e76SPhilippe Mathieu-Daudé offset, num); 876*8d7f2e76SPhilippe Mathieu-Daudé page += num; 877*8d7f2e76SPhilippe Mathieu-Daudé } 878*8d7f2e76SPhilippe Mathieu-Daudé 879*8d7f2e76SPhilippe Mathieu-Daudé mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset; 880*8d7f2e76SPhilippe Mathieu-Daudé mr_size = (end - start_page) << TARGET_PAGE_BITS; 881*8d7f2e76SPhilippe Mathieu-Daudé memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size); 882*8d7f2e76SPhilippe Mathieu-Daudé } 883*8d7f2e76SPhilippe Mathieu-Daudé 884*8d7f2e76SPhilippe Mathieu-Daudé if (dirty && tcg_enabled()) { 885*8d7f2e76SPhilippe Mathieu-Daudé tlb_reset_dirty_range_all(start, length); 886*8d7f2e76SPhilippe Mathieu-Daudé } 887*8d7f2e76SPhilippe Mathieu-Daudé 888*8d7f2e76SPhilippe Mathieu-Daudé return dirty; 889*8d7f2e76SPhilippe Mathieu-Daudé } 890*8d7f2e76SPhilippe Mathieu-Daudé 891*8d7f2e76SPhilippe Mathieu-Daudé DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty 892*8d7f2e76SPhilippe Mathieu-Daudé (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) 893*8d7f2e76SPhilippe Mathieu-Daudé { 894*8d7f2e76SPhilippe Mathieu-Daudé DirtyMemoryBlocks *blocks; 895*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t start = memory_region_get_ram_addr(mr) + offset; 896*8d7f2e76SPhilippe Mathieu-Daudé unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL); 897*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t first = QEMU_ALIGN_DOWN(start, align); 898*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t last = QEMU_ALIGN_UP(start + length, align); 899*8d7f2e76SPhilippe Mathieu-Daudé DirtyBitmapSnapshot *snap; 900*8d7f2e76SPhilippe Mathieu-Daudé unsigned long page, end, dest; 901*8d7f2e76SPhilippe Mathieu-Daudé 902*8d7f2e76SPhilippe Mathieu-Daudé snap = g_malloc0(sizeof(*snap) + 903*8d7f2e76SPhilippe Mathieu-Daudé ((last - first) >> (TARGET_PAGE_BITS + 3))); 904*8d7f2e76SPhilippe Mathieu-Daudé snap->start = first; 905*8d7f2e76SPhilippe Mathieu-Daudé snap->end = last; 906*8d7f2e76SPhilippe Mathieu-Daudé 907*8d7f2e76SPhilippe Mathieu-Daudé page = first >> TARGET_PAGE_BITS; 908*8d7f2e76SPhilippe Mathieu-Daudé end = last >> TARGET_PAGE_BITS; 909*8d7f2e76SPhilippe Mathieu-Daudé dest = 0; 910*8d7f2e76SPhilippe Mathieu-Daudé 911*8d7f2e76SPhilippe Mathieu-Daudé WITH_RCU_READ_LOCK_GUARD() { 912*8d7f2e76SPhilippe Mathieu-Daudé blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 913*8d7f2e76SPhilippe Mathieu-Daudé 914*8d7f2e76SPhilippe Mathieu-Daudé while (page < end) { 915*8d7f2e76SPhilippe Mathieu-Daudé unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE; 916*8d7f2e76SPhilippe Mathieu-Daudé unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE; 917*8d7f2e76SPhilippe Mathieu-Daudé unsigned long num = MIN(end - page, 918*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_MEMORY_BLOCK_SIZE - ofs); 919*8d7f2e76SPhilippe Mathieu-Daudé 920*8d7f2e76SPhilippe Mathieu-Daudé assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL))); 921*8d7f2e76SPhilippe Mathieu-Daudé assert(QEMU_IS_ALIGNED(num, (1 << BITS_PER_LEVEL))); 922*8d7f2e76SPhilippe Mathieu-Daudé ofs >>= BITS_PER_LEVEL; 923*8d7f2e76SPhilippe Mathieu-Daudé 924*8d7f2e76SPhilippe Mathieu-Daudé bitmap_copy_and_clear_atomic(snap->dirty + dest, 925*8d7f2e76SPhilippe Mathieu-Daudé blocks->blocks[idx] + ofs, 926*8d7f2e76SPhilippe Mathieu-Daudé num); 927*8d7f2e76SPhilippe Mathieu-Daudé page += num; 928*8d7f2e76SPhilippe Mathieu-Daudé dest += num >> BITS_PER_LEVEL; 929*8d7f2e76SPhilippe Mathieu-Daudé } 930*8d7f2e76SPhilippe Mathieu-Daudé } 931*8d7f2e76SPhilippe Mathieu-Daudé 932*8d7f2e76SPhilippe Mathieu-Daudé if (tcg_enabled()) { 933*8d7f2e76SPhilippe Mathieu-Daudé tlb_reset_dirty_range_all(start, length); 934*8d7f2e76SPhilippe Mathieu-Daudé } 935*8d7f2e76SPhilippe Mathieu-Daudé 936*8d7f2e76SPhilippe Mathieu-Daudé memory_region_clear_dirty_bitmap(mr, offset, length); 937*8d7f2e76SPhilippe Mathieu-Daudé 938*8d7f2e76SPhilippe Mathieu-Daudé return snap; 939*8d7f2e76SPhilippe Mathieu-Daudé } 940*8d7f2e76SPhilippe Mathieu-Daudé 941*8d7f2e76SPhilippe Mathieu-Daudé bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, 942*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t start, 943*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t length) 944*8d7f2e76SPhilippe Mathieu-Daudé { 945*8d7f2e76SPhilippe Mathieu-Daudé unsigned long page, end; 946*8d7f2e76SPhilippe Mathieu-Daudé 947*8d7f2e76SPhilippe Mathieu-Daudé assert(start >= snap->start); 948*8d7f2e76SPhilippe Mathieu-Daudé assert(start + length <= snap->end); 949*8d7f2e76SPhilippe Mathieu-Daudé 950*8d7f2e76SPhilippe Mathieu-Daudé end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS; 951*8d7f2e76SPhilippe Mathieu-Daudé page = (start - snap->start) >> TARGET_PAGE_BITS; 952*8d7f2e76SPhilippe Mathieu-Daudé 953*8d7f2e76SPhilippe Mathieu-Daudé while (page < end) { 954*8d7f2e76SPhilippe Mathieu-Daudé if (test_bit(page, snap->dirty)) { 955*8d7f2e76SPhilippe Mathieu-Daudé return true; 956*8d7f2e76SPhilippe Mathieu-Daudé } 957*8d7f2e76SPhilippe Mathieu-Daudé page++; 958*8d7f2e76SPhilippe Mathieu-Daudé } 959*8d7f2e76SPhilippe Mathieu-Daudé return false; 960*8d7f2e76SPhilippe Mathieu-Daudé } 961*8d7f2e76SPhilippe Mathieu-Daudé 962*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section */ 963*8d7f2e76SPhilippe Mathieu-Daudé hwaddr memory_region_section_get_iotlb(CPUState *cpu, 964*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section) 965*8d7f2e76SPhilippe Mathieu-Daudé { 966*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = flatview_to_dispatch(section->fv); 967*8d7f2e76SPhilippe Mathieu-Daudé return section - d->map.sections; 968*8d7f2e76SPhilippe Mathieu-Daudé } 969*8d7f2e76SPhilippe Mathieu-Daudé 970*8d7f2e76SPhilippe Mathieu-Daudé static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, 971*8d7f2e76SPhilippe Mathieu-Daudé uint16_t section); 972*8d7f2e76SPhilippe Mathieu-Daudé static subpage_t *subpage_init(FlatView *fv, hwaddr base); 973*8d7f2e76SPhilippe Mathieu-Daudé 974*8d7f2e76SPhilippe Mathieu-Daudé static uint16_t phys_section_add(PhysPageMap *map, 975*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section) 976*8d7f2e76SPhilippe Mathieu-Daudé { 977*8d7f2e76SPhilippe Mathieu-Daudé /* The physical section number is ORed with a page-aligned 978*8d7f2e76SPhilippe Mathieu-Daudé * pointer to produce the iotlb entries. Thus it should 979*8d7f2e76SPhilippe Mathieu-Daudé * never overflow into the page-aligned value. 980*8d7f2e76SPhilippe Mathieu-Daudé */ 981*8d7f2e76SPhilippe Mathieu-Daudé assert(map->sections_nb < TARGET_PAGE_SIZE); 982*8d7f2e76SPhilippe Mathieu-Daudé 983*8d7f2e76SPhilippe Mathieu-Daudé if (map->sections_nb == map->sections_nb_alloc) { 984*8d7f2e76SPhilippe Mathieu-Daudé map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16); 985*8d7f2e76SPhilippe Mathieu-Daudé map->sections = g_renew(MemoryRegionSection, map->sections, 986*8d7f2e76SPhilippe Mathieu-Daudé map->sections_nb_alloc); 987*8d7f2e76SPhilippe Mathieu-Daudé } 988*8d7f2e76SPhilippe Mathieu-Daudé map->sections[map->sections_nb] = *section; 989*8d7f2e76SPhilippe Mathieu-Daudé memory_region_ref(section->mr); 990*8d7f2e76SPhilippe Mathieu-Daudé return map->sections_nb++; 991*8d7f2e76SPhilippe Mathieu-Daudé } 992*8d7f2e76SPhilippe Mathieu-Daudé 993*8d7f2e76SPhilippe Mathieu-Daudé static void phys_section_destroy(MemoryRegion *mr) 994*8d7f2e76SPhilippe Mathieu-Daudé { 995*8d7f2e76SPhilippe Mathieu-Daudé bool have_sub_page = mr->subpage; 996*8d7f2e76SPhilippe Mathieu-Daudé 997*8d7f2e76SPhilippe Mathieu-Daudé memory_region_unref(mr); 998*8d7f2e76SPhilippe Mathieu-Daudé 999*8d7f2e76SPhilippe Mathieu-Daudé if (have_sub_page) { 1000*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage = container_of(mr, subpage_t, iomem); 1001*8d7f2e76SPhilippe Mathieu-Daudé object_unref(OBJECT(&subpage->iomem)); 1002*8d7f2e76SPhilippe Mathieu-Daudé g_free(subpage); 1003*8d7f2e76SPhilippe Mathieu-Daudé } 1004*8d7f2e76SPhilippe Mathieu-Daudé } 1005*8d7f2e76SPhilippe Mathieu-Daudé 1006*8d7f2e76SPhilippe Mathieu-Daudé static void phys_sections_free(PhysPageMap *map) 1007*8d7f2e76SPhilippe Mathieu-Daudé { 1008*8d7f2e76SPhilippe Mathieu-Daudé while (map->sections_nb > 0) { 1009*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section = &map->sections[--map->sections_nb]; 1010*8d7f2e76SPhilippe Mathieu-Daudé phys_section_destroy(section->mr); 1011*8d7f2e76SPhilippe Mathieu-Daudé } 1012*8d7f2e76SPhilippe Mathieu-Daudé g_free(map->sections); 1013*8d7f2e76SPhilippe Mathieu-Daudé g_free(map->nodes); 1014*8d7f2e76SPhilippe Mathieu-Daudé } 1015*8d7f2e76SPhilippe Mathieu-Daudé 1016*8d7f2e76SPhilippe Mathieu-Daudé static void register_subpage(FlatView *fv, MemoryRegionSection *section) 1017*8d7f2e76SPhilippe Mathieu-Daudé { 1018*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = flatview_to_dispatch(fv); 1019*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage; 1020*8d7f2e76SPhilippe Mathieu-Daudé hwaddr base = section->offset_within_address_space 1021*8d7f2e76SPhilippe Mathieu-Daudé & TARGET_PAGE_MASK; 1022*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *existing = phys_page_find(d, base); 1023*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection subsection = { 1024*8d7f2e76SPhilippe Mathieu-Daudé .offset_within_address_space = base, 1025*8d7f2e76SPhilippe Mathieu-Daudé .size = int128_make64(TARGET_PAGE_SIZE), 1026*8d7f2e76SPhilippe Mathieu-Daudé }; 1027*8d7f2e76SPhilippe Mathieu-Daudé hwaddr start, end; 1028*8d7f2e76SPhilippe Mathieu-Daudé 1029*8d7f2e76SPhilippe Mathieu-Daudé assert(existing->mr->subpage || existing->mr == &io_mem_unassigned); 1030*8d7f2e76SPhilippe Mathieu-Daudé 1031*8d7f2e76SPhilippe Mathieu-Daudé if (!(existing->mr->subpage)) { 1032*8d7f2e76SPhilippe Mathieu-Daudé subpage = subpage_init(fv, base); 1033*8d7f2e76SPhilippe Mathieu-Daudé subsection.fv = fv; 1034*8d7f2e76SPhilippe Mathieu-Daudé subsection.mr = &subpage->iomem; 1035*8d7f2e76SPhilippe Mathieu-Daudé phys_page_set(d, base >> TARGET_PAGE_BITS, 1, 1036*8d7f2e76SPhilippe Mathieu-Daudé phys_section_add(&d->map, &subsection)); 1037*8d7f2e76SPhilippe Mathieu-Daudé } else { 1038*8d7f2e76SPhilippe Mathieu-Daudé subpage = container_of(existing->mr, subpage_t, iomem); 1039*8d7f2e76SPhilippe Mathieu-Daudé } 1040*8d7f2e76SPhilippe Mathieu-Daudé start = section->offset_within_address_space & ~TARGET_PAGE_MASK; 1041*8d7f2e76SPhilippe Mathieu-Daudé end = start + int128_get64(section->size) - 1; 1042*8d7f2e76SPhilippe Mathieu-Daudé subpage_register(subpage, start, end, 1043*8d7f2e76SPhilippe Mathieu-Daudé phys_section_add(&d->map, section)); 1044*8d7f2e76SPhilippe Mathieu-Daudé } 1045*8d7f2e76SPhilippe Mathieu-Daudé 1046*8d7f2e76SPhilippe Mathieu-Daudé 1047*8d7f2e76SPhilippe Mathieu-Daudé static void register_multipage(FlatView *fv, 1048*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *section) 1049*8d7f2e76SPhilippe Mathieu-Daudé { 1050*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = flatview_to_dispatch(fv); 1051*8d7f2e76SPhilippe Mathieu-Daudé hwaddr start_addr = section->offset_within_address_space; 1052*8d7f2e76SPhilippe Mathieu-Daudé uint16_t section_index = phys_section_add(&d->map, section); 1053*8d7f2e76SPhilippe Mathieu-Daudé uint64_t num_pages = int128_get64(int128_rshift(section->size, 1054*8d7f2e76SPhilippe Mathieu-Daudé TARGET_PAGE_BITS)); 1055*8d7f2e76SPhilippe Mathieu-Daudé 1056*8d7f2e76SPhilippe Mathieu-Daudé assert(num_pages); 1057*8d7f2e76SPhilippe Mathieu-Daudé phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index); 1058*8d7f2e76SPhilippe Mathieu-Daudé } 1059*8d7f2e76SPhilippe Mathieu-Daudé 1060*8d7f2e76SPhilippe Mathieu-Daudé /* 1061*8d7f2e76SPhilippe Mathieu-Daudé * The range in *section* may look like this: 1062*8d7f2e76SPhilippe Mathieu-Daudé * 1063*8d7f2e76SPhilippe Mathieu-Daudé * |s|PPPPPPP|s| 1064*8d7f2e76SPhilippe Mathieu-Daudé * 1065*8d7f2e76SPhilippe Mathieu-Daudé * where s stands for subpage and P for page. 1066*8d7f2e76SPhilippe Mathieu-Daudé */ 1067*8d7f2e76SPhilippe Mathieu-Daudé void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section) 1068*8d7f2e76SPhilippe Mathieu-Daudé { 1069*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection remain = *section; 1070*8d7f2e76SPhilippe Mathieu-Daudé Int128 page_size = int128_make64(TARGET_PAGE_SIZE); 1071*8d7f2e76SPhilippe Mathieu-Daudé 1072*8d7f2e76SPhilippe Mathieu-Daudé /* register first subpage */ 1073*8d7f2e76SPhilippe Mathieu-Daudé if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) { 1074*8d7f2e76SPhilippe Mathieu-Daudé uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space) 1075*8d7f2e76SPhilippe Mathieu-Daudé - remain.offset_within_address_space; 1076*8d7f2e76SPhilippe Mathieu-Daudé 1077*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection now = remain; 1078*8d7f2e76SPhilippe Mathieu-Daudé now.size = int128_min(int128_make64(left), now.size); 1079*8d7f2e76SPhilippe Mathieu-Daudé register_subpage(fv, &now); 1080*8d7f2e76SPhilippe Mathieu-Daudé if (int128_eq(remain.size, now.size)) { 1081*8d7f2e76SPhilippe Mathieu-Daudé return; 1082*8d7f2e76SPhilippe Mathieu-Daudé } 1083*8d7f2e76SPhilippe Mathieu-Daudé remain.size = int128_sub(remain.size, now.size); 1084*8d7f2e76SPhilippe Mathieu-Daudé remain.offset_within_address_space += int128_get64(now.size); 1085*8d7f2e76SPhilippe Mathieu-Daudé remain.offset_within_region += int128_get64(now.size); 1086*8d7f2e76SPhilippe Mathieu-Daudé } 1087*8d7f2e76SPhilippe Mathieu-Daudé 1088*8d7f2e76SPhilippe Mathieu-Daudé /* register whole pages */ 1089*8d7f2e76SPhilippe Mathieu-Daudé if (int128_ge(remain.size, page_size)) { 1090*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection now = remain; 1091*8d7f2e76SPhilippe Mathieu-Daudé now.size = int128_and(now.size, int128_neg(page_size)); 1092*8d7f2e76SPhilippe Mathieu-Daudé register_multipage(fv, &now); 1093*8d7f2e76SPhilippe Mathieu-Daudé if (int128_eq(remain.size, now.size)) { 1094*8d7f2e76SPhilippe Mathieu-Daudé return; 1095*8d7f2e76SPhilippe Mathieu-Daudé } 1096*8d7f2e76SPhilippe Mathieu-Daudé remain.size = int128_sub(remain.size, now.size); 1097*8d7f2e76SPhilippe Mathieu-Daudé remain.offset_within_address_space += int128_get64(now.size); 1098*8d7f2e76SPhilippe Mathieu-Daudé remain.offset_within_region += int128_get64(now.size); 1099*8d7f2e76SPhilippe Mathieu-Daudé } 1100*8d7f2e76SPhilippe Mathieu-Daudé 1101*8d7f2e76SPhilippe Mathieu-Daudé /* register last subpage */ 1102*8d7f2e76SPhilippe Mathieu-Daudé register_subpage(fv, &remain); 1103*8d7f2e76SPhilippe Mathieu-Daudé } 1104*8d7f2e76SPhilippe Mathieu-Daudé 1105*8d7f2e76SPhilippe Mathieu-Daudé void qemu_flush_coalesced_mmio_buffer(void) 1106*8d7f2e76SPhilippe Mathieu-Daudé { 1107*8d7f2e76SPhilippe Mathieu-Daudé if (kvm_enabled()) 1108*8d7f2e76SPhilippe Mathieu-Daudé kvm_flush_coalesced_mmio_buffer(); 1109*8d7f2e76SPhilippe Mathieu-Daudé } 1110*8d7f2e76SPhilippe Mathieu-Daudé 1111*8d7f2e76SPhilippe Mathieu-Daudé void qemu_mutex_lock_ramlist(void) 1112*8d7f2e76SPhilippe Mathieu-Daudé { 1113*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock(&ram_list.mutex); 1114*8d7f2e76SPhilippe Mathieu-Daudé } 1115*8d7f2e76SPhilippe Mathieu-Daudé 1116*8d7f2e76SPhilippe Mathieu-Daudé void qemu_mutex_unlock_ramlist(void) 1117*8d7f2e76SPhilippe Mathieu-Daudé { 1118*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock(&ram_list.mutex); 1119*8d7f2e76SPhilippe Mathieu-Daudé } 1120*8d7f2e76SPhilippe Mathieu-Daudé 1121*8d7f2e76SPhilippe Mathieu-Daudé GString *ram_block_format(void) 1122*8d7f2e76SPhilippe Mathieu-Daudé { 1123*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1124*8d7f2e76SPhilippe Mathieu-Daudé char *psize; 1125*8d7f2e76SPhilippe Mathieu-Daudé GString *buf = g_string_new(""); 1126*8d7f2e76SPhilippe Mathieu-Daudé 1127*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 1128*8d7f2e76SPhilippe Mathieu-Daudé g_string_append_printf(buf, "%24s %8s %18s %18s %18s %18s %3s\n", 1129*8d7f2e76SPhilippe Mathieu-Daudé "Block Name", "PSize", "Offset", "Used", "Total", 1130*8d7f2e76SPhilippe Mathieu-Daudé "HVA", "RO"); 1131*8d7f2e76SPhilippe Mathieu-Daudé 1132*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1133*8d7f2e76SPhilippe Mathieu-Daudé psize = size_to_str(block->page_size); 1134*8d7f2e76SPhilippe Mathieu-Daudé g_string_append_printf(buf, "%24s %8s 0x%016" PRIx64 " 0x%016" PRIx64 1135*8d7f2e76SPhilippe Mathieu-Daudé " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n", 1136*8d7f2e76SPhilippe Mathieu-Daudé block->idstr, psize, 1137*8d7f2e76SPhilippe Mathieu-Daudé (uint64_t)block->offset, 1138*8d7f2e76SPhilippe Mathieu-Daudé (uint64_t)block->used_length, 1139*8d7f2e76SPhilippe Mathieu-Daudé (uint64_t)block->max_length, 1140*8d7f2e76SPhilippe Mathieu-Daudé (uint64_t)(uintptr_t)block->host, 1141*8d7f2e76SPhilippe Mathieu-Daudé block->mr->readonly ? "ro" : "rw"); 1142*8d7f2e76SPhilippe Mathieu-Daudé 1143*8d7f2e76SPhilippe Mathieu-Daudé g_free(psize); 1144*8d7f2e76SPhilippe Mathieu-Daudé } 1145*8d7f2e76SPhilippe Mathieu-Daudé 1146*8d7f2e76SPhilippe Mathieu-Daudé return buf; 1147*8d7f2e76SPhilippe Mathieu-Daudé } 1148*8d7f2e76SPhilippe Mathieu-Daudé 1149*8d7f2e76SPhilippe Mathieu-Daudé static int find_min_backend_pagesize(Object *obj, void *opaque) 1150*8d7f2e76SPhilippe Mathieu-Daudé { 1151*8d7f2e76SPhilippe Mathieu-Daudé long *hpsize_min = opaque; 1152*8d7f2e76SPhilippe Mathieu-Daudé 1153*8d7f2e76SPhilippe Mathieu-Daudé if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) { 1154*8d7f2e76SPhilippe Mathieu-Daudé HostMemoryBackend *backend = MEMORY_BACKEND(obj); 1155*8d7f2e76SPhilippe Mathieu-Daudé long hpsize = host_memory_backend_pagesize(backend); 1156*8d7f2e76SPhilippe Mathieu-Daudé 1157*8d7f2e76SPhilippe Mathieu-Daudé if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) { 1158*8d7f2e76SPhilippe Mathieu-Daudé *hpsize_min = hpsize; 1159*8d7f2e76SPhilippe Mathieu-Daudé } 1160*8d7f2e76SPhilippe Mathieu-Daudé } 1161*8d7f2e76SPhilippe Mathieu-Daudé 1162*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1163*8d7f2e76SPhilippe Mathieu-Daudé } 1164*8d7f2e76SPhilippe Mathieu-Daudé 1165*8d7f2e76SPhilippe Mathieu-Daudé static int find_max_backend_pagesize(Object *obj, void *opaque) 1166*8d7f2e76SPhilippe Mathieu-Daudé { 1167*8d7f2e76SPhilippe Mathieu-Daudé long *hpsize_max = opaque; 1168*8d7f2e76SPhilippe Mathieu-Daudé 1169*8d7f2e76SPhilippe Mathieu-Daudé if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) { 1170*8d7f2e76SPhilippe Mathieu-Daudé HostMemoryBackend *backend = MEMORY_BACKEND(obj); 1171*8d7f2e76SPhilippe Mathieu-Daudé long hpsize = host_memory_backend_pagesize(backend); 1172*8d7f2e76SPhilippe Mathieu-Daudé 1173*8d7f2e76SPhilippe Mathieu-Daudé if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) { 1174*8d7f2e76SPhilippe Mathieu-Daudé *hpsize_max = hpsize; 1175*8d7f2e76SPhilippe Mathieu-Daudé } 1176*8d7f2e76SPhilippe Mathieu-Daudé } 1177*8d7f2e76SPhilippe Mathieu-Daudé 1178*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1179*8d7f2e76SPhilippe Mathieu-Daudé } 1180*8d7f2e76SPhilippe Mathieu-Daudé 1181*8d7f2e76SPhilippe Mathieu-Daudé /* 1182*8d7f2e76SPhilippe Mathieu-Daudé * TODO: We assume right now that all mapped host memory backends are 1183*8d7f2e76SPhilippe Mathieu-Daudé * used as RAM, however some might be used for different purposes. 1184*8d7f2e76SPhilippe Mathieu-Daudé */ 1185*8d7f2e76SPhilippe Mathieu-Daudé long qemu_minrampagesize(void) 1186*8d7f2e76SPhilippe Mathieu-Daudé { 1187*8d7f2e76SPhilippe Mathieu-Daudé long hpsize = LONG_MAX; 1188*8d7f2e76SPhilippe Mathieu-Daudé Object *memdev_root = object_resolve_path("/objects", NULL); 1189*8d7f2e76SPhilippe Mathieu-Daudé 1190*8d7f2e76SPhilippe Mathieu-Daudé object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize); 1191*8d7f2e76SPhilippe Mathieu-Daudé return hpsize; 1192*8d7f2e76SPhilippe Mathieu-Daudé } 1193*8d7f2e76SPhilippe Mathieu-Daudé 1194*8d7f2e76SPhilippe Mathieu-Daudé long qemu_maxrampagesize(void) 1195*8d7f2e76SPhilippe Mathieu-Daudé { 1196*8d7f2e76SPhilippe Mathieu-Daudé long pagesize = 0; 1197*8d7f2e76SPhilippe Mathieu-Daudé Object *memdev_root = object_resolve_path("/objects", NULL); 1198*8d7f2e76SPhilippe Mathieu-Daudé 1199*8d7f2e76SPhilippe Mathieu-Daudé object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize); 1200*8d7f2e76SPhilippe Mathieu-Daudé return pagesize; 1201*8d7f2e76SPhilippe Mathieu-Daudé } 1202*8d7f2e76SPhilippe Mathieu-Daudé 1203*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_POSIX 1204*8d7f2e76SPhilippe Mathieu-Daudé static int64_t get_file_size(int fd) 1205*8d7f2e76SPhilippe Mathieu-Daudé { 1206*8d7f2e76SPhilippe Mathieu-Daudé int64_t size; 1207*8d7f2e76SPhilippe Mathieu-Daudé #if defined(__linux__) 1208*8d7f2e76SPhilippe Mathieu-Daudé struct stat st; 1209*8d7f2e76SPhilippe Mathieu-Daudé 1210*8d7f2e76SPhilippe Mathieu-Daudé if (fstat(fd, &st) < 0) { 1211*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1212*8d7f2e76SPhilippe Mathieu-Daudé } 1213*8d7f2e76SPhilippe Mathieu-Daudé 1214*8d7f2e76SPhilippe Mathieu-Daudé /* Special handling for devdax character devices */ 1215*8d7f2e76SPhilippe Mathieu-Daudé if (S_ISCHR(st.st_mode)) { 1216*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *subsystem_path = NULL; 1217*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *subsystem = NULL; 1218*8d7f2e76SPhilippe Mathieu-Daudé 1219*8d7f2e76SPhilippe Mathieu-Daudé subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem", 1220*8d7f2e76SPhilippe Mathieu-Daudé major(st.st_rdev), minor(st.st_rdev)); 1221*8d7f2e76SPhilippe Mathieu-Daudé subsystem = g_file_read_link(subsystem_path, NULL); 1222*8d7f2e76SPhilippe Mathieu-Daudé 1223*8d7f2e76SPhilippe Mathieu-Daudé if (subsystem && g_str_has_suffix(subsystem, "/dax")) { 1224*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *size_path = NULL; 1225*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *size_str = NULL; 1226*8d7f2e76SPhilippe Mathieu-Daudé 1227*8d7f2e76SPhilippe Mathieu-Daudé size_path = g_strdup_printf("/sys/dev/char/%d:%d/size", 1228*8d7f2e76SPhilippe Mathieu-Daudé major(st.st_rdev), minor(st.st_rdev)); 1229*8d7f2e76SPhilippe Mathieu-Daudé 1230*8d7f2e76SPhilippe Mathieu-Daudé if (g_file_get_contents(size_path, &size_str, NULL, NULL)) { 1231*8d7f2e76SPhilippe Mathieu-Daudé return g_ascii_strtoll(size_str, NULL, 0); 1232*8d7f2e76SPhilippe Mathieu-Daudé } 1233*8d7f2e76SPhilippe Mathieu-Daudé } 1234*8d7f2e76SPhilippe Mathieu-Daudé } 1235*8d7f2e76SPhilippe Mathieu-Daudé #endif /* defined(__linux__) */ 1236*8d7f2e76SPhilippe Mathieu-Daudé 1237*8d7f2e76SPhilippe Mathieu-Daudé /* st.st_size may be zero for special files yet lseek(2) works */ 1238*8d7f2e76SPhilippe Mathieu-Daudé size = lseek(fd, 0, SEEK_END); 1239*8d7f2e76SPhilippe Mathieu-Daudé if (size < 0) { 1240*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1241*8d7f2e76SPhilippe Mathieu-Daudé } 1242*8d7f2e76SPhilippe Mathieu-Daudé return size; 1243*8d7f2e76SPhilippe Mathieu-Daudé } 1244*8d7f2e76SPhilippe Mathieu-Daudé 1245*8d7f2e76SPhilippe Mathieu-Daudé static int64_t get_file_align(int fd) 1246*8d7f2e76SPhilippe Mathieu-Daudé { 1247*8d7f2e76SPhilippe Mathieu-Daudé int64_t align = -1; 1248*8d7f2e76SPhilippe Mathieu-Daudé #if defined(__linux__) && defined(CONFIG_LIBDAXCTL) 1249*8d7f2e76SPhilippe Mathieu-Daudé struct stat st; 1250*8d7f2e76SPhilippe Mathieu-Daudé 1251*8d7f2e76SPhilippe Mathieu-Daudé if (fstat(fd, &st) < 0) { 1252*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1253*8d7f2e76SPhilippe Mathieu-Daudé } 1254*8d7f2e76SPhilippe Mathieu-Daudé 1255*8d7f2e76SPhilippe Mathieu-Daudé /* Special handling for devdax character devices */ 1256*8d7f2e76SPhilippe Mathieu-Daudé if (S_ISCHR(st.st_mode)) { 1257*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *path = NULL; 1258*8d7f2e76SPhilippe Mathieu-Daudé g_autofree char *rpath = NULL; 1259*8d7f2e76SPhilippe Mathieu-Daudé struct daxctl_ctx *ctx; 1260*8d7f2e76SPhilippe Mathieu-Daudé struct daxctl_region *region; 1261*8d7f2e76SPhilippe Mathieu-Daudé int rc = 0; 1262*8d7f2e76SPhilippe Mathieu-Daudé 1263*8d7f2e76SPhilippe Mathieu-Daudé path = g_strdup_printf("/sys/dev/char/%d:%d", 1264*8d7f2e76SPhilippe Mathieu-Daudé major(st.st_rdev), minor(st.st_rdev)); 1265*8d7f2e76SPhilippe Mathieu-Daudé rpath = realpath(path, NULL); 1266*8d7f2e76SPhilippe Mathieu-Daudé if (!rpath) { 1267*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1268*8d7f2e76SPhilippe Mathieu-Daudé } 1269*8d7f2e76SPhilippe Mathieu-Daudé 1270*8d7f2e76SPhilippe Mathieu-Daudé rc = daxctl_new(&ctx); 1271*8d7f2e76SPhilippe Mathieu-Daudé if (rc) { 1272*8d7f2e76SPhilippe Mathieu-Daudé return -1; 1273*8d7f2e76SPhilippe Mathieu-Daudé } 1274*8d7f2e76SPhilippe Mathieu-Daudé 1275*8d7f2e76SPhilippe Mathieu-Daudé daxctl_region_foreach(ctx, region) { 1276*8d7f2e76SPhilippe Mathieu-Daudé if (strstr(rpath, daxctl_region_get_path(region))) { 1277*8d7f2e76SPhilippe Mathieu-Daudé align = daxctl_region_get_align(region); 1278*8d7f2e76SPhilippe Mathieu-Daudé break; 1279*8d7f2e76SPhilippe Mathieu-Daudé } 1280*8d7f2e76SPhilippe Mathieu-Daudé } 1281*8d7f2e76SPhilippe Mathieu-Daudé daxctl_unref(ctx); 1282*8d7f2e76SPhilippe Mathieu-Daudé } 1283*8d7f2e76SPhilippe Mathieu-Daudé #endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */ 1284*8d7f2e76SPhilippe Mathieu-Daudé 1285*8d7f2e76SPhilippe Mathieu-Daudé return align; 1286*8d7f2e76SPhilippe Mathieu-Daudé } 1287*8d7f2e76SPhilippe Mathieu-Daudé 1288*8d7f2e76SPhilippe Mathieu-Daudé static int file_ram_open(const char *path, 1289*8d7f2e76SPhilippe Mathieu-Daudé const char *region_name, 1290*8d7f2e76SPhilippe Mathieu-Daudé bool readonly, 1291*8d7f2e76SPhilippe Mathieu-Daudé bool *created) 1292*8d7f2e76SPhilippe Mathieu-Daudé { 1293*8d7f2e76SPhilippe Mathieu-Daudé char *filename; 1294*8d7f2e76SPhilippe Mathieu-Daudé char *sanitized_name; 1295*8d7f2e76SPhilippe Mathieu-Daudé char *c; 1296*8d7f2e76SPhilippe Mathieu-Daudé int fd = -1; 1297*8d7f2e76SPhilippe Mathieu-Daudé 1298*8d7f2e76SPhilippe Mathieu-Daudé *created = false; 1299*8d7f2e76SPhilippe Mathieu-Daudé for (;;) { 1300*8d7f2e76SPhilippe Mathieu-Daudé fd = open(path, readonly ? O_RDONLY : O_RDWR); 1301*8d7f2e76SPhilippe Mathieu-Daudé if (fd >= 0) { 1302*8d7f2e76SPhilippe Mathieu-Daudé /* 1303*8d7f2e76SPhilippe Mathieu-Daudé * open(O_RDONLY) won't fail with EISDIR. Check manually if we 1304*8d7f2e76SPhilippe Mathieu-Daudé * opened a directory and fail similarly to how we fail ENOENT 1305*8d7f2e76SPhilippe Mathieu-Daudé * in readonly mode. Note that mkstemp() would imply O_RDWR. 1306*8d7f2e76SPhilippe Mathieu-Daudé */ 1307*8d7f2e76SPhilippe Mathieu-Daudé if (readonly) { 1308*8d7f2e76SPhilippe Mathieu-Daudé struct stat file_stat; 1309*8d7f2e76SPhilippe Mathieu-Daudé 1310*8d7f2e76SPhilippe Mathieu-Daudé if (fstat(fd, &file_stat)) { 1311*8d7f2e76SPhilippe Mathieu-Daudé close(fd); 1312*8d7f2e76SPhilippe Mathieu-Daudé if (errno == EINTR) { 1313*8d7f2e76SPhilippe Mathieu-Daudé continue; 1314*8d7f2e76SPhilippe Mathieu-Daudé } 1315*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1316*8d7f2e76SPhilippe Mathieu-Daudé } else if (S_ISDIR(file_stat.st_mode)) { 1317*8d7f2e76SPhilippe Mathieu-Daudé close(fd); 1318*8d7f2e76SPhilippe Mathieu-Daudé return -EISDIR; 1319*8d7f2e76SPhilippe Mathieu-Daudé } 1320*8d7f2e76SPhilippe Mathieu-Daudé } 1321*8d7f2e76SPhilippe Mathieu-Daudé /* @path names an existing file, use it */ 1322*8d7f2e76SPhilippe Mathieu-Daudé break; 1323*8d7f2e76SPhilippe Mathieu-Daudé } 1324*8d7f2e76SPhilippe Mathieu-Daudé if (errno == ENOENT) { 1325*8d7f2e76SPhilippe Mathieu-Daudé if (readonly) { 1326*8d7f2e76SPhilippe Mathieu-Daudé /* Refuse to create new, readonly files. */ 1327*8d7f2e76SPhilippe Mathieu-Daudé return -ENOENT; 1328*8d7f2e76SPhilippe Mathieu-Daudé } 1329*8d7f2e76SPhilippe Mathieu-Daudé /* @path names a file that doesn't exist, create it */ 1330*8d7f2e76SPhilippe Mathieu-Daudé fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644); 1331*8d7f2e76SPhilippe Mathieu-Daudé if (fd >= 0) { 1332*8d7f2e76SPhilippe Mathieu-Daudé *created = true; 1333*8d7f2e76SPhilippe Mathieu-Daudé break; 1334*8d7f2e76SPhilippe Mathieu-Daudé } 1335*8d7f2e76SPhilippe Mathieu-Daudé } else if (errno == EISDIR) { 1336*8d7f2e76SPhilippe Mathieu-Daudé /* @path names a directory, create a file there */ 1337*8d7f2e76SPhilippe Mathieu-Daudé /* Make name safe to use with mkstemp by replacing '/' with '_'. */ 1338*8d7f2e76SPhilippe Mathieu-Daudé sanitized_name = g_strdup(region_name); 1339*8d7f2e76SPhilippe Mathieu-Daudé for (c = sanitized_name; *c != '\0'; c++) { 1340*8d7f2e76SPhilippe Mathieu-Daudé if (*c == '/') { 1341*8d7f2e76SPhilippe Mathieu-Daudé *c = '_'; 1342*8d7f2e76SPhilippe Mathieu-Daudé } 1343*8d7f2e76SPhilippe Mathieu-Daudé } 1344*8d7f2e76SPhilippe Mathieu-Daudé 1345*8d7f2e76SPhilippe Mathieu-Daudé filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path, 1346*8d7f2e76SPhilippe Mathieu-Daudé sanitized_name); 1347*8d7f2e76SPhilippe Mathieu-Daudé g_free(sanitized_name); 1348*8d7f2e76SPhilippe Mathieu-Daudé 1349*8d7f2e76SPhilippe Mathieu-Daudé fd = mkstemp(filename); 1350*8d7f2e76SPhilippe Mathieu-Daudé if (fd >= 0) { 1351*8d7f2e76SPhilippe Mathieu-Daudé unlink(filename); 1352*8d7f2e76SPhilippe Mathieu-Daudé g_free(filename); 1353*8d7f2e76SPhilippe Mathieu-Daudé break; 1354*8d7f2e76SPhilippe Mathieu-Daudé } 1355*8d7f2e76SPhilippe Mathieu-Daudé g_free(filename); 1356*8d7f2e76SPhilippe Mathieu-Daudé } 1357*8d7f2e76SPhilippe Mathieu-Daudé if (errno != EEXIST && errno != EINTR) { 1358*8d7f2e76SPhilippe Mathieu-Daudé return -errno; 1359*8d7f2e76SPhilippe Mathieu-Daudé } 1360*8d7f2e76SPhilippe Mathieu-Daudé /* 1361*8d7f2e76SPhilippe Mathieu-Daudé * Try again on EINTR and EEXIST. The latter happens when 1362*8d7f2e76SPhilippe Mathieu-Daudé * something else creates the file between our two open(). 1363*8d7f2e76SPhilippe Mathieu-Daudé */ 1364*8d7f2e76SPhilippe Mathieu-Daudé } 1365*8d7f2e76SPhilippe Mathieu-Daudé 1366*8d7f2e76SPhilippe Mathieu-Daudé return fd; 1367*8d7f2e76SPhilippe Mathieu-Daudé } 1368*8d7f2e76SPhilippe Mathieu-Daudé 1369*8d7f2e76SPhilippe Mathieu-Daudé static void *file_ram_alloc(RAMBlock *block, 1370*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t memory, 1371*8d7f2e76SPhilippe Mathieu-Daudé int fd, 1372*8d7f2e76SPhilippe Mathieu-Daudé bool truncate, 1373*8d7f2e76SPhilippe Mathieu-Daudé off_t offset, 1374*8d7f2e76SPhilippe Mathieu-Daudé Error **errp) 1375*8d7f2e76SPhilippe Mathieu-Daudé { 1376*8d7f2e76SPhilippe Mathieu-Daudé uint32_t qemu_map_flags; 1377*8d7f2e76SPhilippe Mathieu-Daudé void *area; 1378*8d7f2e76SPhilippe Mathieu-Daudé 1379*8d7f2e76SPhilippe Mathieu-Daudé block->page_size = qemu_fd_getpagesize(fd); 1380*8d7f2e76SPhilippe Mathieu-Daudé if (block->mr->align % block->page_size) { 1381*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "alignment 0x%" PRIx64 1382*8d7f2e76SPhilippe Mathieu-Daudé " must be multiples of page size 0x%zx", 1383*8d7f2e76SPhilippe Mathieu-Daudé block->mr->align, block->page_size); 1384*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1385*8d7f2e76SPhilippe Mathieu-Daudé } else if (block->mr->align && !is_power_of_2(block->mr->align)) { 1386*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "alignment 0x%" PRIx64 1387*8d7f2e76SPhilippe Mathieu-Daudé " must be a power of two", block->mr->align); 1388*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1389*8d7f2e76SPhilippe Mathieu-Daudé } else if (offset % block->page_size) { 1390*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "offset 0x%" PRIx64 1391*8d7f2e76SPhilippe Mathieu-Daudé " must be multiples of page size 0x%zx", 1392*8d7f2e76SPhilippe Mathieu-Daudé offset, block->page_size); 1393*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1394*8d7f2e76SPhilippe Mathieu-Daudé } 1395*8d7f2e76SPhilippe Mathieu-Daudé block->mr->align = MAX(block->page_size, block->mr->align); 1396*8d7f2e76SPhilippe Mathieu-Daudé #if defined(__s390x__) 1397*8d7f2e76SPhilippe Mathieu-Daudé if (kvm_enabled()) { 1398*8d7f2e76SPhilippe Mathieu-Daudé block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN); 1399*8d7f2e76SPhilippe Mathieu-Daudé } 1400*8d7f2e76SPhilippe Mathieu-Daudé #endif 1401*8d7f2e76SPhilippe Mathieu-Daudé 1402*8d7f2e76SPhilippe Mathieu-Daudé if (memory < block->page_size) { 1403*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to " 1404*8d7f2e76SPhilippe Mathieu-Daudé "or larger than page size 0x%zx", 1405*8d7f2e76SPhilippe Mathieu-Daudé memory, block->page_size); 1406*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1407*8d7f2e76SPhilippe Mathieu-Daudé } 1408*8d7f2e76SPhilippe Mathieu-Daudé 1409*8d7f2e76SPhilippe Mathieu-Daudé memory = ROUND_UP(memory, block->page_size); 1410*8d7f2e76SPhilippe Mathieu-Daudé 1411*8d7f2e76SPhilippe Mathieu-Daudé /* 1412*8d7f2e76SPhilippe Mathieu-Daudé * ftruncate is not supported by hugetlbfs in older 1413*8d7f2e76SPhilippe Mathieu-Daudé * hosts, so don't bother bailing out on errors. 1414*8d7f2e76SPhilippe Mathieu-Daudé * If anything goes wrong with it under other filesystems, 1415*8d7f2e76SPhilippe Mathieu-Daudé * mmap will fail. 1416*8d7f2e76SPhilippe Mathieu-Daudé * 1417*8d7f2e76SPhilippe Mathieu-Daudé * Do not truncate the non-empty backend file to avoid corrupting 1418*8d7f2e76SPhilippe Mathieu-Daudé * the existing data in the file. Disabling shrinking is not 1419*8d7f2e76SPhilippe Mathieu-Daudé * enough. For example, the current vNVDIMM implementation stores 1420*8d7f2e76SPhilippe Mathieu-Daudé * the guest NVDIMM labels at the end of the backend file. If the 1421*8d7f2e76SPhilippe Mathieu-Daudé * backend file is later extended, QEMU will not be able to find 1422*8d7f2e76SPhilippe Mathieu-Daudé * those labels. Therefore, extending the non-empty backend file 1423*8d7f2e76SPhilippe Mathieu-Daudé * is disabled as well. 1424*8d7f2e76SPhilippe Mathieu-Daudé */ 1425*8d7f2e76SPhilippe Mathieu-Daudé if (truncate && ftruncate(fd, offset + memory)) { 1426*8d7f2e76SPhilippe Mathieu-Daudé perror("ftruncate"); 1427*8d7f2e76SPhilippe Mathieu-Daudé } 1428*8d7f2e76SPhilippe Mathieu-Daudé 1429*8d7f2e76SPhilippe Mathieu-Daudé qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0; 1430*8d7f2e76SPhilippe Mathieu-Daudé qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0; 1431*8d7f2e76SPhilippe Mathieu-Daudé qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0; 1432*8d7f2e76SPhilippe Mathieu-Daudé qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0; 1433*8d7f2e76SPhilippe Mathieu-Daudé area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset); 1434*8d7f2e76SPhilippe Mathieu-Daudé if (area == MAP_FAILED) { 1435*8d7f2e76SPhilippe Mathieu-Daudé error_setg_errno(errp, errno, 1436*8d7f2e76SPhilippe Mathieu-Daudé "unable to map backing store for guest RAM"); 1437*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1438*8d7f2e76SPhilippe Mathieu-Daudé } 1439*8d7f2e76SPhilippe Mathieu-Daudé 1440*8d7f2e76SPhilippe Mathieu-Daudé block->fd = fd; 1441*8d7f2e76SPhilippe Mathieu-Daudé block->fd_offset = offset; 1442*8d7f2e76SPhilippe Mathieu-Daudé return area; 1443*8d7f2e76SPhilippe Mathieu-Daudé } 1444*8d7f2e76SPhilippe Mathieu-Daudé #endif 1445*8d7f2e76SPhilippe Mathieu-Daudé 1446*8d7f2e76SPhilippe Mathieu-Daudé /* Allocate space within the ram_addr_t space that governs the 1447*8d7f2e76SPhilippe Mathieu-Daudé * dirty bitmaps. 1448*8d7f2e76SPhilippe Mathieu-Daudé * Called with the ramlist lock held. 1449*8d7f2e76SPhilippe Mathieu-Daudé */ 1450*8d7f2e76SPhilippe Mathieu-Daudé static ram_addr_t find_ram_offset(ram_addr_t size) 1451*8d7f2e76SPhilippe Mathieu-Daudé { 1452*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block, *next_block; 1453*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX; 1454*8d7f2e76SPhilippe Mathieu-Daudé 1455*8d7f2e76SPhilippe Mathieu-Daudé assert(size != 0); /* it would hand out same offset multiple times */ 1456*8d7f2e76SPhilippe Mathieu-Daudé 1457*8d7f2e76SPhilippe Mathieu-Daudé if (QLIST_EMPTY_RCU(&ram_list.blocks)) { 1458*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1459*8d7f2e76SPhilippe Mathieu-Daudé } 1460*8d7f2e76SPhilippe Mathieu-Daudé 1461*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1462*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t candidate, next = RAM_ADDR_MAX; 1463*8d7f2e76SPhilippe Mathieu-Daudé 1464*8d7f2e76SPhilippe Mathieu-Daudé /* Align blocks to start on a 'long' in the bitmap 1465*8d7f2e76SPhilippe Mathieu-Daudé * which makes the bitmap sync'ing take the fast path. 1466*8d7f2e76SPhilippe Mathieu-Daudé */ 1467*8d7f2e76SPhilippe Mathieu-Daudé candidate = block->offset + block->max_length; 1468*8d7f2e76SPhilippe Mathieu-Daudé candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS); 1469*8d7f2e76SPhilippe Mathieu-Daudé 1470*8d7f2e76SPhilippe Mathieu-Daudé /* Search for the closest following block 1471*8d7f2e76SPhilippe Mathieu-Daudé * and find the gap. 1472*8d7f2e76SPhilippe Mathieu-Daudé */ 1473*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(next_block) { 1474*8d7f2e76SPhilippe Mathieu-Daudé if (next_block->offset >= candidate) { 1475*8d7f2e76SPhilippe Mathieu-Daudé next = MIN(next, next_block->offset); 1476*8d7f2e76SPhilippe Mathieu-Daudé } 1477*8d7f2e76SPhilippe Mathieu-Daudé } 1478*8d7f2e76SPhilippe Mathieu-Daudé 1479*8d7f2e76SPhilippe Mathieu-Daudé /* If it fits remember our place and remember the size 1480*8d7f2e76SPhilippe Mathieu-Daudé * of gap, but keep going so that we might find a smaller 1481*8d7f2e76SPhilippe Mathieu-Daudé * gap to fill so avoiding fragmentation. 1482*8d7f2e76SPhilippe Mathieu-Daudé */ 1483*8d7f2e76SPhilippe Mathieu-Daudé if (next - candidate >= size && next - candidate < mingap) { 1484*8d7f2e76SPhilippe Mathieu-Daudé offset = candidate; 1485*8d7f2e76SPhilippe Mathieu-Daudé mingap = next - candidate; 1486*8d7f2e76SPhilippe Mathieu-Daudé } 1487*8d7f2e76SPhilippe Mathieu-Daudé 1488*8d7f2e76SPhilippe Mathieu-Daudé trace_find_ram_offset_loop(size, candidate, offset, next, mingap); 1489*8d7f2e76SPhilippe Mathieu-Daudé } 1490*8d7f2e76SPhilippe Mathieu-Daudé 1491*8d7f2e76SPhilippe Mathieu-Daudé if (offset == RAM_ADDR_MAX) { 1492*8d7f2e76SPhilippe Mathieu-Daudé fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n", 1493*8d7f2e76SPhilippe Mathieu-Daudé (uint64_t)size); 1494*8d7f2e76SPhilippe Mathieu-Daudé abort(); 1495*8d7f2e76SPhilippe Mathieu-Daudé } 1496*8d7f2e76SPhilippe Mathieu-Daudé 1497*8d7f2e76SPhilippe Mathieu-Daudé trace_find_ram_offset(size, offset); 1498*8d7f2e76SPhilippe Mathieu-Daudé 1499*8d7f2e76SPhilippe Mathieu-Daudé return offset; 1500*8d7f2e76SPhilippe Mathieu-Daudé } 1501*8d7f2e76SPhilippe Mathieu-Daudé 1502*8d7f2e76SPhilippe Mathieu-Daudé static unsigned long last_ram_page(void) 1503*8d7f2e76SPhilippe Mathieu-Daudé { 1504*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1505*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t last = 0; 1506*8d7f2e76SPhilippe Mathieu-Daudé 1507*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 1508*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1509*8d7f2e76SPhilippe Mathieu-Daudé last = MAX(last, block->offset + block->max_length); 1510*8d7f2e76SPhilippe Mathieu-Daudé } 1511*8d7f2e76SPhilippe Mathieu-Daudé return last >> TARGET_PAGE_BITS; 1512*8d7f2e76SPhilippe Mathieu-Daudé } 1513*8d7f2e76SPhilippe Mathieu-Daudé 1514*8d7f2e76SPhilippe Mathieu-Daudé static void qemu_ram_setup_dump(void *addr, ram_addr_t size) 1515*8d7f2e76SPhilippe Mathieu-Daudé { 1516*8d7f2e76SPhilippe Mathieu-Daudé int ret; 1517*8d7f2e76SPhilippe Mathieu-Daudé 1518*8d7f2e76SPhilippe Mathieu-Daudé /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */ 1519*8d7f2e76SPhilippe Mathieu-Daudé if (!machine_dump_guest_core(current_machine)) { 1520*8d7f2e76SPhilippe Mathieu-Daudé ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP); 1521*8d7f2e76SPhilippe Mathieu-Daudé if (ret) { 1522*8d7f2e76SPhilippe Mathieu-Daudé perror("qemu_madvise"); 1523*8d7f2e76SPhilippe Mathieu-Daudé fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, " 1524*8d7f2e76SPhilippe Mathieu-Daudé "but dump_guest_core=off specified\n"); 1525*8d7f2e76SPhilippe Mathieu-Daudé } 1526*8d7f2e76SPhilippe Mathieu-Daudé } 1527*8d7f2e76SPhilippe Mathieu-Daudé } 1528*8d7f2e76SPhilippe Mathieu-Daudé 1529*8d7f2e76SPhilippe Mathieu-Daudé const char *qemu_ram_get_idstr(RAMBlock *rb) 1530*8d7f2e76SPhilippe Mathieu-Daudé { 1531*8d7f2e76SPhilippe Mathieu-Daudé return rb->idstr; 1532*8d7f2e76SPhilippe Mathieu-Daudé } 1533*8d7f2e76SPhilippe Mathieu-Daudé 1534*8d7f2e76SPhilippe Mathieu-Daudé void *qemu_ram_get_host_addr(RAMBlock *rb) 1535*8d7f2e76SPhilippe Mathieu-Daudé { 1536*8d7f2e76SPhilippe Mathieu-Daudé return rb->host; 1537*8d7f2e76SPhilippe Mathieu-Daudé } 1538*8d7f2e76SPhilippe Mathieu-Daudé 1539*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_get_offset(RAMBlock *rb) 1540*8d7f2e76SPhilippe Mathieu-Daudé { 1541*8d7f2e76SPhilippe Mathieu-Daudé return rb->offset; 1542*8d7f2e76SPhilippe Mathieu-Daudé } 1543*8d7f2e76SPhilippe Mathieu-Daudé 1544*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_get_used_length(RAMBlock *rb) 1545*8d7f2e76SPhilippe Mathieu-Daudé { 1546*8d7f2e76SPhilippe Mathieu-Daudé return rb->used_length; 1547*8d7f2e76SPhilippe Mathieu-Daudé } 1548*8d7f2e76SPhilippe Mathieu-Daudé 1549*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_get_max_length(RAMBlock *rb) 1550*8d7f2e76SPhilippe Mathieu-Daudé { 1551*8d7f2e76SPhilippe Mathieu-Daudé return rb->max_length; 1552*8d7f2e76SPhilippe Mathieu-Daudé } 1553*8d7f2e76SPhilippe Mathieu-Daudé 1554*8d7f2e76SPhilippe Mathieu-Daudé bool qemu_ram_is_shared(RAMBlock *rb) 1555*8d7f2e76SPhilippe Mathieu-Daudé { 1556*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_SHARED; 1557*8d7f2e76SPhilippe Mathieu-Daudé } 1558*8d7f2e76SPhilippe Mathieu-Daudé 1559*8d7f2e76SPhilippe Mathieu-Daudé bool qemu_ram_is_noreserve(RAMBlock *rb) 1560*8d7f2e76SPhilippe Mathieu-Daudé { 1561*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_NORESERVE; 1562*8d7f2e76SPhilippe Mathieu-Daudé } 1563*8d7f2e76SPhilippe Mathieu-Daudé 1564*8d7f2e76SPhilippe Mathieu-Daudé /* Note: Only set at the start of postcopy */ 1565*8d7f2e76SPhilippe Mathieu-Daudé bool qemu_ram_is_uf_zeroable(RAMBlock *rb) 1566*8d7f2e76SPhilippe Mathieu-Daudé { 1567*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_UF_ZEROPAGE; 1568*8d7f2e76SPhilippe Mathieu-Daudé } 1569*8d7f2e76SPhilippe Mathieu-Daudé 1570*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_set_uf_zeroable(RAMBlock *rb) 1571*8d7f2e76SPhilippe Mathieu-Daudé { 1572*8d7f2e76SPhilippe Mathieu-Daudé rb->flags |= RAM_UF_ZEROPAGE; 1573*8d7f2e76SPhilippe Mathieu-Daudé } 1574*8d7f2e76SPhilippe Mathieu-Daudé 1575*8d7f2e76SPhilippe Mathieu-Daudé bool qemu_ram_is_migratable(RAMBlock *rb) 1576*8d7f2e76SPhilippe Mathieu-Daudé { 1577*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_MIGRATABLE; 1578*8d7f2e76SPhilippe Mathieu-Daudé } 1579*8d7f2e76SPhilippe Mathieu-Daudé 1580*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_set_migratable(RAMBlock *rb) 1581*8d7f2e76SPhilippe Mathieu-Daudé { 1582*8d7f2e76SPhilippe Mathieu-Daudé rb->flags |= RAM_MIGRATABLE; 1583*8d7f2e76SPhilippe Mathieu-Daudé } 1584*8d7f2e76SPhilippe Mathieu-Daudé 1585*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_unset_migratable(RAMBlock *rb) 1586*8d7f2e76SPhilippe Mathieu-Daudé { 1587*8d7f2e76SPhilippe Mathieu-Daudé rb->flags &= ~RAM_MIGRATABLE; 1588*8d7f2e76SPhilippe Mathieu-Daudé } 1589*8d7f2e76SPhilippe Mathieu-Daudé 1590*8d7f2e76SPhilippe Mathieu-Daudé bool qemu_ram_is_named_file(RAMBlock *rb) 1591*8d7f2e76SPhilippe Mathieu-Daudé { 1592*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_NAMED_FILE; 1593*8d7f2e76SPhilippe Mathieu-Daudé } 1594*8d7f2e76SPhilippe Mathieu-Daudé 1595*8d7f2e76SPhilippe Mathieu-Daudé int qemu_ram_get_fd(RAMBlock *rb) 1596*8d7f2e76SPhilippe Mathieu-Daudé { 1597*8d7f2e76SPhilippe Mathieu-Daudé return rb->fd; 1598*8d7f2e76SPhilippe Mathieu-Daudé } 1599*8d7f2e76SPhilippe Mathieu-Daudé 1600*8d7f2e76SPhilippe Mathieu-Daudé /* Called with iothread lock held. */ 1601*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev) 1602*8d7f2e76SPhilippe Mathieu-Daudé { 1603*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1604*8d7f2e76SPhilippe Mathieu-Daudé 1605*8d7f2e76SPhilippe Mathieu-Daudé assert(new_block); 1606*8d7f2e76SPhilippe Mathieu-Daudé assert(!new_block->idstr[0]); 1607*8d7f2e76SPhilippe Mathieu-Daudé 1608*8d7f2e76SPhilippe Mathieu-Daudé if (dev) { 1609*8d7f2e76SPhilippe Mathieu-Daudé char *id = qdev_get_dev_path(dev); 1610*8d7f2e76SPhilippe Mathieu-Daudé if (id) { 1611*8d7f2e76SPhilippe Mathieu-Daudé snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id); 1612*8d7f2e76SPhilippe Mathieu-Daudé g_free(id); 1613*8d7f2e76SPhilippe Mathieu-Daudé } 1614*8d7f2e76SPhilippe Mathieu-Daudé } 1615*8d7f2e76SPhilippe Mathieu-Daudé pstrcat(new_block->idstr, sizeof(new_block->idstr), name); 1616*8d7f2e76SPhilippe Mathieu-Daudé 1617*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 1618*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1619*8d7f2e76SPhilippe Mathieu-Daudé if (block != new_block && 1620*8d7f2e76SPhilippe Mathieu-Daudé !strcmp(block->idstr, new_block->idstr)) { 1621*8d7f2e76SPhilippe Mathieu-Daudé fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n", 1622*8d7f2e76SPhilippe Mathieu-Daudé new_block->idstr); 1623*8d7f2e76SPhilippe Mathieu-Daudé abort(); 1624*8d7f2e76SPhilippe Mathieu-Daudé } 1625*8d7f2e76SPhilippe Mathieu-Daudé } 1626*8d7f2e76SPhilippe Mathieu-Daudé } 1627*8d7f2e76SPhilippe Mathieu-Daudé 1628*8d7f2e76SPhilippe Mathieu-Daudé /* Called with iothread lock held. */ 1629*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_unset_idstr(RAMBlock *block) 1630*8d7f2e76SPhilippe Mathieu-Daudé { 1631*8d7f2e76SPhilippe Mathieu-Daudé /* FIXME: arch_init.c assumes that this is not called throughout 1632*8d7f2e76SPhilippe Mathieu-Daudé * migration. Ignore the problem since hot-unplug during migration 1633*8d7f2e76SPhilippe Mathieu-Daudé * does not work anyway. 1634*8d7f2e76SPhilippe Mathieu-Daudé */ 1635*8d7f2e76SPhilippe Mathieu-Daudé if (block) { 1636*8d7f2e76SPhilippe Mathieu-Daudé memset(block->idstr, 0, sizeof(block->idstr)); 1637*8d7f2e76SPhilippe Mathieu-Daudé } 1638*8d7f2e76SPhilippe Mathieu-Daudé } 1639*8d7f2e76SPhilippe Mathieu-Daudé 1640*8d7f2e76SPhilippe Mathieu-Daudé size_t qemu_ram_pagesize(RAMBlock *rb) 1641*8d7f2e76SPhilippe Mathieu-Daudé { 1642*8d7f2e76SPhilippe Mathieu-Daudé return rb->page_size; 1643*8d7f2e76SPhilippe Mathieu-Daudé } 1644*8d7f2e76SPhilippe Mathieu-Daudé 1645*8d7f2e76SPhilippe Mathieu-Daudé /* Returns the largest size of page in use */ 1646*8d7f2e76SPhilippe Mathieu-Daudé size_t qemu_ram_pagesize_largest(void) 1647*8d7f2e76SPhilippe Mathieu-Daudé { 1648*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1649*8d7f2e76SPhilippe Mathieu-Daudé size_t largest = 0; 1650*8d7f2e76SPhilippe Mathieu-Daudé 1651*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1652*8d7f2e76SPhilippe Mathieu-Daudé largest = MAX(largest, qemu_ram_pagesize(block)); 1653*8d7f2e76SPhilippe Mathieu-Daudé } 1654*8d7f2e76SPhilippe Mathieu-Daudé 1655*8d7f2e76SPhilippe Mathieu-Daudé return largest; 1656*8d7f2e76SPhilippe Mathieu-Daudé } 1657*8d7f2e76SPhilippe Mathieu-Daudé 1658*8d7f2e76SPhilippe Mathieu-Daudé static int memory_try_enable_merging(void *addr, size_t len) 1659*8d7f2e76SPhilippe Mathieu-Daudé { 1660*8d7f2e76SPhilippe Mathieu-Daudé if (!machine_mem_merge(current_machine)) { 1661*8d7f2e76SPhilippe Mathieu-Daudé /* disabled by the user */ 1662*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1663*8d7f2e76SPhilippe Mathieu-Daudé } 1664*8d7f2e76SPhilippe Mathieu-Daudé 1665*8d7f2e76SPhilippe Mathieu-Daudé return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE); 1666*8d7f2e76SPhilippe Mathieu-Daudé } 1667*8d7f2e76SPhilippe Mathieu-Daudé 1668*8d7f2e76SPhilippe Mathieu-Daudé /* 1669*8d7f2e76SPhilippe Mathieu-Daudé * Resizing RAM while migrating can result in the migration being canceled. 1670*8d7f2e76SPhilippe Mathieu-Daudé * Care has to be taken if the guest might have already detected the memory. 1671*8d7f2e76SPhilippe Mathieu-Daudé * 1672*8d7f2e76SPhilippe Mathieu-Daudé * As memory core doesn't know how is memory accessed, it is up to 1673*8d7f2e76SPhilippe Mathieu-Daudé * resize callback to update device state and/or add assertions to detect 1674*8d7f2e76SPhilippe Mathieu-Daudé * misuse, if necessary. 1675*8d7f2e76SPhilippe Mathieu-Daudé */ 1676*8d7f2e76SPhilippe Mathieu-Daudé int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) 1677*8d7f2e76SPhilippe Mathieu-Daudé { 1678*8d7f2e76SPhilippe Mathieu-Daudé const ram_addr_t oldsize = block->used_length; 1679*8d7f2e76SPhilippe Mathieu-Daudé const ram_addr_t unaligned_size = newsize; 1680*8d7f2e76SPhilippe Mathieu-Daudé 1681*8d7f2e76SPhilippe Mathieu-Daudé assert(block); 1682*8d7f2e76SPhilippe Mathieu-Daudé 1683*8d7f2e76SPhilippe Mathieu-Daudé newsize = HOST_PAGE_ALIGN(newsize); 1684*8d7f2e76SPhilippe Mathieu-Daudé 1685*8d7f2e76SPhilippe Mathieu-Daudé if (block->used_length == newsize) { 1686*8d7f2e76SPhilippe Mathieu-Daudé /* 1687*8d7f2e76SPhilippe Mathieu-Daudé * We don't have to resize the ram block (which only knows aligned 1688*8d7f2e76SPhilippe Mathieu-Daudé * sizes), however, we have to notify if the unaligned size changed. 1689*8d7f2e76SPhilippe Mathieu-Daudé */ 1690*8d7f2e76SPhilippe Mathieu-Daudé if (unaligned_size != memory_region_size(block->mr)) { 1691*8d7f2e76SPhilippe Mathieu-Daudé memory_region_set_size(block->mr, unaligned_size); 1692*8d7f2e76SPhilippe Mathieu-Daudé if (block->resized) { 1693*8d7f2e76SPhilippe Mathieu-Daudé block->resized(block->idstr, unaligned_size, block->host); 1694*8d7f2e76SPhilippe Mathieu-Daudé } 1695*8d7f2e76SPhilippe Mathieu-Daudé } 1696*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1697*8d7f2e76SPhilippe Mathieu-Daudé } 1698*8d7f2e76SPhilippe Mathieu-Daudé 1699*8d7f2e76SPhilippe Mathieu-Daudé if (!(block->flags & RAM_RESIZEABLE)) { 1700*8d7f2e76SPhilippe Mathieu-Daudé error_setg_errno(errp, EINVAL, 1701*8d7f2e76SPhilippe Mathieu-Daudé "Size mismatch: %s: 0x" RAM_ADDR_FMT 1702*8d7f2e76SPhilippe Mathieu-Daudé " != 0x" RAM_ADDR_FMT, block->idstr, 1703*8d7f2e76SPhilippe Mathieu-Daudé newsize, block->used_length); 1704*8d7f2e76SPhilippe Mathieu-Daudé return -EINVAL; 1705*8d7f2e76SPhilippe Mathieu-Daudé } 1706*8d7f2e76SPhilippe Mathieu-Daudé 1707*8d7f2e76SPhilippe Mathieu-Daudé if (block->max_length < newsize) { 1708*8d7f2e76SPhilippe Mathieu-Daudé error_setg_errno(errp, EINVAL, 1709*8d7f2e76SPhilippe Mathieu-Daudé "Size too large: %s: 0x" RAM_ADDR_FMT 1710*8d7f2e76SPhilippe Mathieu-Daudé " > 0x" RAM_ADDR_FMT, block->idstr, 1711*8d7f2e76SPhilippe Mathieu-Daudé newsize, block->max_length); 1712*8d7f2e76SPhilippe Mathieu-Daudé return -EINVAL; 1713*8d7f2e76SPhilippe Mathieu-Daudé } 1714*8d7f2e76SPhilippe Mathieu-Daudé 1715*8d7f2e76SPhilippe Mathieu-Daudé /* Notify before modifying the ram block and touching the bitmaps. */ 1716*8d7f2e76SPhilippe Mathieu-Daudé if (block->host) { 1717*8d7f2e76SPhilippe Mathieu-Daudé ram_block_notify_resize(block->host, oldsize, newsize); 1718*8d7f2e76SPhilippe Mathieu-Daudé } 1719*8d7f2e76SPhilippe Mathieu-Daudé 1720*8d7f2e76SPhilippe Mathieu-Daudé cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); 1721*8d7f2e76SPhilippe Mathieu-Daudé block->used_length = newsize; 1722*8d7f2e76SPhilippe Mathieu-Daudé cpu_physical_memory_set_dirty_range(block->offset, block->used_length, 1723*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_CLIENTS_ALL); 1724*8d7f2e76SPhilippe Mathieu-Daudé memory_region_set_size(block->mr, unaligned_size); 1725*8d7f2e76SPhilippe Mathieu-Daudé if (block->resized) { 1726*8d7f2e76SPhilippe Mathieu-Daudé block->resized(block->idstr, unaligned_size, block->host); 1727*8d7f2e76SPhilippe Mathieu-Daudé } 1728*8d7f2e76SPhilippe Mathieu-Daudé return 0; 1729*8d7f2e76SPhilippe Mathieu-Daudé } 1730*8d7f2e76SPhilippe Mathieu-Daudé 1731*8d7f2e76SPhilippe Mathieu-Daudé /* 1732*8d7f2e76SPhilippe Mathieu-Daudé * Trigger sync on the given ram block for range [start, start + length] 1733*8d7f2e76SPhilippe Mathieu-Daudé * with the backing store if one is available. 1734*8d7f2e76SPhilippe Mathieu-Daudé * Otherwise no-op. 1735*8d7f2e76SPhilippe Mathieu-Daudé * @Note: this is supposed to be a synchronous op. 1736*8d7f2e76SPhilippe Mathieu-Daudé */ 1737*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length) 1738*8d7f2e76SPhilippe Mathieu-Daudé { 1739*8d7f2e76SPhilippe Mathieu-Daudé /* The requested range should fit in within the block range */ 1740*8d7f2e76SPhilippe Mathieu-Daudé g_assert((start + length) <= block->used_length); 1741*8d7f2e76SPhilippe Mathieu-Daudé 1742*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_LIBPMEM 1743*8d7f2e76SPhilippe Mathieu-Daudé /* The lack of support for pmem should not block the sync */ 1744*8d7f2e76SPhilippe Mathieu-Daudé if (ramblock_is_pmem(block)) { 1745*8d7f2e76SPhilippe Mathieu-Daudé void *addr = ramblock_ptr(block, start); 1746*8d7f2e76SPhilippe Mathieu-Daudé pmem_persist(addr, length); 1747*8d7f2e76SPhilippe Mathieu-Daudé return; 1748*8d7f2e76SPhilippe Mathieu-Daudé } 1749*8d7f2e76SPhilippe Mathieu-Daudé #endif 1750*8d7f2e76SPhilippe Mathieu-Daudé if (block->fd >= 0) { 1751*8d7f2e76SPhilippe Mathieu-Daudé /** 1752*8d7f2e76SPhilippe Mathieu-Daudé * Case there is no support for PMEM or the memory has not been 1753*8d7f2e76SPhilippe Mathieu-Daudé * specified as persistent (or is not one) - use the msync. 1754*8d7f2e76SPhilippe Mathieu-Daudé * Less optimal but still achieves the same goal 1755*8d7f2e76SPhilippe Mathieu-Daudé */ 1756*8d7f2e76SPhilippe Mathieu-Daudé void *addr = ramblock_ptr(block, start); 1757*8d7f2e76SPhilippe Mathieu-Daudé if (qemu_msync(addr, length, block->fd)) { 1758*8d7f2e76SPhilippe Mathieu-Daudé warn_report("%s: failed to sync memory range: start: " 1759*8d7f2e76SPhilippe Mathieu-Daudé RAM_ADDR_FMT " length: " RAM_ADDR_FMT, 1760*8d7f2e76SPhilippe Mathieu-Daudé __func__, start, length); 1761*8d7f2e76SPhilippe Mathieu-Daudé } 1762*8d7f2e76SPhilippe Mathieu-Daudé } 1763*8d7f2e76SPhilippe Mathieu-Daudé } 1764*8d7f2e76SPhilippe Mathieu-Daudé 1765*8d7f2e76SPhilippe Mathieu-Daudé /* Called with ram_list.mutex held */ 1766*8d7f2e76SPhilippe Mathieu-Daudé static void dirty_memory_extend(ram_addr_t old_ram_size, 1767*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t new_ram_size) 1768*8d7f2e76SPhilippe Mathieu-Daudé { 1769*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size, 1770*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_MEMORY_BLOCK_SIZE); 1771*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size, 1772*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_MEMORY_BLOCK_SIZE); 1773*8d7f2e76SPhilippe Mathieu-Daudé int i; 1774*8d7f2e76SPhilippe Mathieu-Daudé 1775*8d7f2e76SPhilippe Mathieu-Daudé /* Only need to extend if block count increased */ 1776*8d7f2e76SPhilippe Mathieu-Daudé if (new_num_blocks <= old_num_blocks) { 1777*8d7f2e76SPhilippe Mathieu-Daudé return; 1778*8d7f2e76SPhilippe Mathieu-Daudé } 1779*8d7f2e76SPhilippe Mathieu-Daudé 1780*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 1781*8d7f2e76SPhilippe Mathieu-Daudé DirtyMemoryBlocks *old_blocks; 1782*8d7f2e76SPhilippe Mathieu-Daudé DirtyMemoryBlocks *new_blocks; 1783*8d7f2e76SPhilippe Mathieu-Daudé int j; 1784*8d7f2e76SPhilippe Mathieu-Daudé 1785*8d7f2e76SPhilippe Mathieu-Daudé old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]); 1786*8d7f2e76SPhilippe Mathieu-Daudé new_blocks = g_malloc(sizeof(*new_blocks) + 1787*8d7f2e76SPhilippe Mathieu-Daudé sizeof(new_blocks->blocks[0]) * new_num_blocks); 1788*8d7f2e76SPhilippe Mathieu-Daudé 1789*8d7f2e76SPhilippe Mathieu-Daudé if (old_num_blocks) { 1790*8d7f2e76SPhilippe Mathieu-Daudé memcpy(new_blocks->blocks, old_blocks->blocks, 1791*8d7f2e76SPhilippe Mathieu-Daudé old_num_blocks * sizeof(old_blocks->blocks[0])); 1792*8d7f2e76SPhilippe Mathieu-Daudé } 1793*8d7f2e76SPhilippe Mathieu-Daudé 1794*8d7f2e76SPhilippe Mathieu-Daudé for (j = old_num_blocks; j < new_num_blocks; j++) { 1795*8d7f2e76SPhilippe Mathieu-Daudé new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE); 1796*8d7f2e76SPhilippe Mathieu-Daudé } 1797*8d7f2e76SPhilippe Mathieu-Daudé 1798*8d7f2e76SPhilippe Mathieu-Daudé qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks); 1799*8d7f2e76SPhilippe Mathieu-Daudé 1800*8d7f2e76SPhilippe Mathieu-Daudé if (old_blocks) { 1801*8d7f2e76SPhilippe Mathieu-Daudé g_free_rcu(old_blocks, rcu); 1802*8d7f2e76SPhilippe Mathieu-Daudé } 1803*8d7f2e76SPhilippe Mathieu-Daudé } 1804*8d7f2e76SPhilippe Mathieu-Daudé } 1805*8d7f2e76SPhilippe Mathieu-Daudé 1806*8d7f2e76SPhilippe Mathieu-Daudé static void ram_block_add(RAMBlock *new_block, Error **errp) 1807*8d7f2e76SPhilippe Mathieu-Daudé { 1808*8d7f2e76SPhilippe Mathieu-Daudé const bool noreserve = qemu_ram_is_noreserve(new_block); 1809*8d7f2e76SPhilippe Mathieu-Daudé const bool shared = qemu_ram_is_shared(new_block); 1810*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1811*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *last_block = NULL; 1812*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t old_ram_size, new_ram_size; 1813*8d7f2e76SPhilippe Mathieu-Daudé Error *err = NULL; 1814*8d7f2e76SPhilippe Mathieu-Daudé 1815*8d7f2e76SPhilippe Mathieu-Daudé old_ram_size = last_ram_page(); 1816*8d7f2e76SPhilippe Mathieu-Daudé 1817*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock_ramlist(); 1818*8d7f2e76SPhilippe Mathieu-Daudé new_block->offset = find_ram_offset(new_block->max_length); 1819*8d7f2e76SPhilippe Mathieu-Daudé 1820*8d7f2e76SPhilippe Mathieu-Daudé if (!new_block->host) { 1821*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled()) { 1822*8d7f2e76SPhilippe Mathieu-Daudé xen_ram_alloc(new_block->offset, new_block->max_length, 1823*8d7f2e76SPhilippe Mathieu-Daudé new_block->mr, &err); 1824*8d7f2e76SPhilippe Mathieu-Daudé if (err) { 1825*8d7f2e76SPhilippe Mathieu-Daudé error_propagate(errp, err); 1826*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_ramlist(); 1827*8d7f2e76SPhilippe Mathieu-Daudé return; 1828*8d7f2e76SPhilippe Mathieu-Daudé } 1829*8d7f2e76SPhilippe Mathieu-Daudé } else { 1830*8d7f2e76SPhilippe Mathieu-Daudé new_block->host = qemu_anon_ram_alloc(new_block->max_length, 1831*8d7f2e76SPhilippe Mathieu-Daudé &new_block->mr->align, 1832*8d7f2e76SPhilippe Mathieu-Daudé shared, noreserve); 1833*8d7f2e76SPhilippe Mathieu-Daudé if (!new_block->host) { 1834*8d7f2e76SPhilippe Mathieu-Daudé error_setg_errno(errp, errno, 1835*8d7f2e76SPhilippe Mathieu-Daudé "cannot set up guest memory '%s'", 1836*8d7f2e76SPhilippe Mathieu-Daudé memory_region_name(new_block->mr)); 1837*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_ramlist(); 1838*8d7f2e76SPhilippe Mathieu-Daudé return; 1839*8d7f2e76SPhilippe Mathieu-Daudé } 1840*8d7f2e76SPhilippe Mathieu-Daudé memory_try_enable_merging(new_block->host, new_block->max_length); 1841*8d7f2e76SPhilippe Mathieu-Daudé } 1842*8d7f2e76SPhilippe Mathieu-Daudé } 1843*8d7f2e76SPhilippe Mathieu-Daudé 1844*8d7f2e76SPhilippe Mathieu-Daudé new_ram_size = MAX(old_ram_size, 1845*8d7f2e76SPhilippe Mathieu-Daudé (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS); 1846*8d7f2e76SPhilippe Mathieu-Daudé if (new_ram_size > old_ram_size) { 1847*8d7f2e76SPhilippe Mathieu-Daudé dirty_memory_extend(old_ram_size, new_ram_size); 1848*8d7f2e76SPhilippe Mathieu-Daudé } 1849*8d7f2e76SPhilippe Mathieu-Daudé /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ, 1850*8d7f2e76SPhilippe Mathieu-Daudé * QLIST (which has an RCU-friendly variant) does not have insertion at 1851*8d7f2e76SPhilippe Mathieu-Daudé * tail, so save the last element in last_block. 1852*8d7f2e76SPhilippe Mathieu-Daudé */ 1853*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 1854*8d7f2e76SPhilippe Mathieu-Daudé last_block = block; 1855*8d7f2e76SPhilippe Mathieu-Daudé if (block->max_length < new_block->max_length) { 1856*8d7f2e76SPhilippe Mathieu-Daudé break; 1857*8d7f2e76SPhilippe Mathieu-Daudé } 1858*8d7f2e76SPhilippe Mathieu-Daudé } 1859*8d7f2e76SPhilippe Mathieu-Daudé if (block) { 1860*8d7f2e76SPhilippe Mathieu-Daudé QLIST_INSERT_BEFORE_RCU(block, new_block, next); 1861*8d7f2e76SPhilippe Mathieu-Daudé } else if (last_block) { 1862*8d7f2e76SPhilippe Mathieu-Daudé QLIST_INSERT_AFTER_RCU(last_block, new_block, next); 1863*8d7f2e76SPhilippe Mathieu-Daudé } else { /* list is empty */ 1864*8d7f2e76SPhilippe Mathieu-Daudé QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next); 1865*8d7f2e76SPhilippe Mathieu-Daudé } 1866*8d7f2e76SPhilippe Mathieu-Daudé ram_list.mru_block = NULL; 1867*8d7f2e76SPhilippe Mathieu-Daudé 1868*8d7f2e76SPhilippe Mathieu-Daudé /* Write list before version */ 1869*8d7f2e76SPhilippe Mathieu-Daudé smp_wmb(); 1870*8d7f2e76SPhilippe Mathieu-Daudé ram_list.version++; 1871*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_ramlist(); 1872*8d7f2e76SPhilippe Mathieu-Daudé 1873*8d7f2e76SPhilippe Mathieu-Daudé cpu_physical_memory_set_dirty_range(new_block->offset, 1874*8d7f2e76SPhilippe Mathieu-Daudé new_block->used_length, 1875*8d7f2e76SPhilippe Mathieu-Daudé DIRTY_CLIENTS_ALL); 1876*8d7f2e76SPhilippe Mathieu-Daudé 1877*8d7f2e76SPhilippe Mathieu-Daudé if (new_block->host) { 1878*8d7f2e76SPhilippe Mathieu-Daudé qemu_ram_setup_dump(new_block->host, new_block->max_length); 1879*8d7f2e76SPhilippe Mathieu-Daudé qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE); 1880*8d7f2e76SPhilippe Mathieu-Daudé /* 1881*8d7f2e76SPhilippe Mathieu-Daudé * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU 1882*8d7f2e76SPhilippe Mathieu-Daudé * Configure it unless the machine is a qtest server, in which case 1883*8d7f2e76SPhilippe Mathieu-Daudé * KVM is not used and it may be forked (eg for fuzzing purposes). 1884*8d7f2e76SPhilippe Mathieu-Daudé */ 1885*8d7f2e76SPhilippe Mathieu-Daudé if (!qtest_enabled()) { 1886*8d7f2e76SPhilippe Mathieu-Daudé qemu_madvise(new_block->host, new_block->max_length, 1887*8d7f2e76SPhilippe Mathieu-Daudé QEMU_MADV_DONTFORK); 1888*8d7f2e76SPhilippe Mathieu-Daudé } 1889*8d7f2e76SPhilippe Mathieu-Daudé ram_block_notify_add(new_block->host, new_block->used_length, 1890*8d7f2e76SPhilippe Mathieu-Daudé new_block->max_length); 1891*8d7f2e76SPhilippe Mathieu-Daudé } 1892*8d7f2e76SPhilippe Mathieu-Daudé } 1893*8d7f2e76SPhilippe Mathieu-Daudé 1894*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_POSIX 1895*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, 1896*8d7f2e76SPhilippe Mathieu-Daudé uint32_t ram_flags, int fd, off_t offset, 1897*8d7f2e76SPhilippe Mathieu-Daudé Error **errp) 1898*8d7f2e76SPhilippe Mathieu-Daudé { 1899*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *new_block; 1900*8d7f2e76SPhilippe Mathieu-Daudé Error *local_err = NULL; 1901*8d7f2e76SPhilippe Mathieu-Daudé int64_t file_size, file_align; 1902*8d7f2e76SPhilippe Mathieu-Daudé 1903*8d7f2e76SPhilippe Mathieu-Daudé /* Just support these ram flags by now. */ 1904*8d7f2e76SPhilippe Mathieu-Daudé assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE | 1905*8d7f2e76SPhilippe Mathieu-Daudé RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY | 1906*8d7f2e76SPhilippe Mathieu-Daudé RAM_READONLY_FD)) == 0); 1907*8d7f2e76SPhilippe Mathieu-Daudé 1908*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled()) { 1909*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "-mem-path not supported with Xen"); 1910*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1911*8d7f2e76SPhilippe Mathieu-Daudé } 1912*8d7f2e76SPhilippe Mathieu-Daudé 1913*8d7f2e76SPhilippe Mathieu-Daudé if (kvm_enabled() && !kvm_has_sync_mmu()) { 1914*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, 1915*8d7f2e76SPhilippe Mathieu-Daudé "host lacks kvm mmu notifiers, -mem-path unsupported"); 1916*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1917*8d7f2e76SPhilippe Mathieu-Daudé } 1918*8d7f2e76SPhilippe Mathieu-Daudé 1919*8d7f2e76SPhilippe Mathieu-Daudé size = HOST_PAGE_ALIGN(size); 1920*8d7f2e76SPhilippe Mathieu-Daudé file_size = get_file_size(fd); 1921*8d7f2e76SPhilippe Mathieu-Daudé if (file_size > offset && file_size < (offset + size)) { 1922*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "backing store size 0x%" PRIx64 1923*8d7f2e76SPhilippe Mathieu-Daudé " does not match 'size' option 0x" RAM_ADDR_FMT, 1924*8d7f2e76SPhilippe Mathieu-Daudé file_size, size); 1925*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1926*8d7f2e76SPhilippe Mathieu-Daudé } 1927*8d7f2e76SPhilippe Mathieu-Daudé 1928*8d7f2e76SPhilippe Mathieu-Daudé file_align = get_file_align(fd); 1929*8d7f2e76SPhilippe Mathieu-Daudé if (file_align > 0 && file_align > mr->align) { 1930*8d7f2e76SPhilippe Mathieu-Daudé error_setg(errp, "backing store align 0x%" PRIx64 1931*8d7f2e76SPhilippe Mathieu-Daudé " is larger than 'align' option 0x%" PRIx64, 1932*8d7f2e76SPhilippe Mathieu-Daudé file_align, mr->align); 1933*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1934*8d7f2e76SPhilippe Mathieu-Daudé } 1935*8d7f2e76SPhilippe Mathieu-Daudé 1936*8d7f2e76SPhilippe Mathieu-Daudé new_block = g_malloc0(sizeof(*new_block)); 1937*8d7f2e76SPhilippe Mathieu-Daudé new_block->mr = mr; 1938*8d7f2e76SPhilippe Mathieu-Daudé new_block->used_length = size; 1939*8d7f2e76SPhilippe Mathieu-Daudé new_block->max_length = size; 1940*8d7f2e76SPhilippe Mathieu-Daudé new_block->flags = ram_flags; 1941*8d7f2e76SPhilippe Mathieu-Daudé new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset, 1942*8d7f2e76SPhilippe Mathieu-Daudé errp); 1943*8d7f2e76SPhilippe Mathieu-Daudé if (!new_block->host) { 1944*8d7f2e76SPhilippe Mathieu-Daudé g_free(new_block); 1945*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1946*8d7f2e76SPhilippe Mathieu-Daudé } 1947*8d7f2e76SPhilippe Mathieu-Daudé 1948*8d7f2e76SPhilippe Mathieu-Daudé ram_block_add(new_block, &local_err); 1949*8d7f2e76SPhilippe Mathieu-Daudé if (local_err) { 1950*8d7f2e76SPhilippe Mathieu-Daudé g_free(new_block); 1951*8d7f2e76SPhilippe Mathieu-Daudé error_propagate(errp, local_err); 1952*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1953*8d7f2e76SPhilippe Mathieu-Daudé } 1954*8d7f2e76SPhilippe Mathieu-Daudé return new_block; 1955*8d7f2e76SPhilippe Mathieu-Daudé 1956*8d7f2e76SPhilippe Mathieu-Daudé } 1957*8d7f2e76SPhilippe Mathieu-Daudé 1958*8d7f2e76SPhilippe Mathieu-Daudé 1959*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 1960*8d7f2e76SPhilippe Mathieu-Daudé uint32_t ram_flags, const char *mem_path, 1961*8d7f2e76SPhilippe Mathieu-Daudé off_t offset, Error **errp) 1962*8d7f2e76SPhilippe Mathieu-Daudé { 1963*8d7f2e76SPhilippe Mathieu-Daudé int fd; 1964*8d7f2e76SPhilippe Mathieu-Daudé bool created; 1965*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 1966*8d7f2e76SPhilippe Mathieu-Daudé 1967*8d7f2e76SPhilippe Mathieu-Daudé fd = file_ram_open(mem_path, memory_region_name(mr), 1968*8d7f2e76SPhilippe Mathieu-Daudé !!(ram_flags & RAM_READONLY_FD), &created); 1969*8d7f2e76SPhilippe Mathieu-Daudé if (fd < 0) { 1970*8d7f2e76SPhilippe Mathieu-Daudé error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM", 1971*8d7f2e76SPhilippe Mathieu-Daudé mem_path); 1972*8d7f2e76SPhilippe Mathieu-Daudé if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) && 1973*8d7f2e76SPhilippe Mathieu-Daudé fd == -EACCES) { 1974*8d7f2e76SPhilippe Mathieu-Daudé /* 1975*8d7f2e76SPhilippe Mathieu-Daudé * If we can open the file R/O (note: will never create a new file) 1976*8d7f2e76SPhilippe Mathieu-Daudé * and we are dealing with a private mapping, there are still ways 1977*8d7f2e76SPhilippe Mathieu-Daudé * to consume such files and get RAM instead of ROM. 1978*8d7f2e76SPhilippe Mathieu-Daudé */ 1979*8d7f2e76SPhilippe Mathieu-Daudé fd = file_ram_open(mem_path, memory_region_name(mr), true, 1980*8d7f2e76SPhilippe Mathieu-Daudé &created); 1981*8d7f2e76SPhilippe Mathieu-Daudé if (fd < 0) { 1982*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1983*8d7f2e76SPhilippe Mathieu-Daudé } 1984*8d7f2e76SPhilippe Mathieu-Daudé assert(!created); 1985*8d7f2e76SPhilippe Mathieu-Daudé close(fd); 1986*8d7f2e76SPhilippe Mathieu-Daudé error_append_hint(errp, "Consider opening the backing store" 1987*8d7f2e76SPhilippe Mathieu-Daudé " read-only but still creating writable RAM using" 1988*8d7f2e76SPhilippe Mathieu-Daudé " '-object memory-backend-file,readonly=on,rom=off...'" 1989*8d7f2e76SPhilippe Mathieu-Daudé " (see \"VM templating\" documentation)\n"); 1990*8d7f2e76SPhilippe Mathieu-Daudé } 1991*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 1992*8d7f2e76SPhilippe Mathieu-Daudé } 1993*8d7f2e76SPhilippe Mathieu-Daudé 1994*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp); 1995*8d7f2e76SPhilippe Mathieu-Daudé if (!block) { 1996*8d7f2e76SPhilippe Mathieu-Daudé if (created) { 1997*8d7f2e76SPhilippe Mathieu-Daudé unlink(mem_path); 1998*8d7f2e76SPhilippe Mathieu-Daudé } 1999*8d7f2e76SPhilippe Mathieu-Daudé close(fd); 2000*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 2001*8d7f2e76SPhilippe Mathieu-Daudé } 2002*8d7f2e76SPhilippe Mathieu-Daudé 2003*8d7f2e76SPhilippe Mathieu-Daudé return block; 2004*8d7f2e76SPhilippe Mathieu-Daudé } 2005*8d7f2e76SPhilippe Mathieu-Daudé #endif 2006*8d7f2e76SPhilippe Mathieu-Daudé 2007*8d7f2e76SPhilippe Mathieu-Daudé static 2008*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size, 2009*8d7f2e76SPhilippe Mathieu-Daudé void (*resized)(const char*, 2010*8d7f2e76SPhilippe Mathieu-Daudé uint64_t length, 2011*8d7f2e76SPhilippe Mathieu-Daudé void *host), 2012*8d7f2e76SPhilippe Mathieu-Daudé void *host, uint32_t ram_flags, 2013*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr, Error **errp) 2014*8d7f2e76SPhilippe Mathieu-Daudé { 2015*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *new_block; 2016*8d7f2e76SPhilippe Mathieu-Daudé Error *local_err = NULL; 2017*8d7f2e76SPhilippe Mathieu-Daudé 2018*8d7f2e76SPhilippe Mathieu-Daudé assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC | 2019*8d7f2e76SPhilippe Mathieu-Daudé RAM_NORESERVE)) == 0); 2020*8d7f2e76SPhilippe Mathieu-Daudé assert(!host ^ (ram_flags & RAM_PREALLOC)); 2021*8d7f2e76SPhilippe Mathieu-Daudé 2022*8d7f2e76SPhilippe Mathieu-Daudé size = HOST_PAGE_ALIGN(size); 2023*8d7f2e76SPhilippe Mathieu-Daudé max_size = HOST_PAGE_ALIGN(max_size); 2024*8d7f2e76SPhilippe Mathieu-Daudé new_block = g_malloc0(sizeof(*new_block)); 2025*8d7f2e76SPhilippe Mathieu-Daudé new_block->mr = mr; 2026*8d7f2e76SPhilippe Mathieu-Daudé new_block->resized = resized; 2027*8d7f2e76SPhilippe Mathieu-Daudé new_block->used_length = size; 2028*8d7f2e76SPhilippe Mathieu-Daudé new_block->max_length = max_size; 2029*8d7f2e76SPhilippe Mathieu-Daudé assert(max_size >= size); 2030*8d7f2e76SPhilippe Mathieu-Daudé new_block->fd = -1; 2031*8d7f2e76SPhilippe Mathieu-Daudé new_block->page_size = qemu_real_host_page_size(); 2032*8d7f2e76SPhilippe Mathieu-Daudé new_block->host = host; 2033*8d7f2e76SPhilippe Mathieu-Daudé new_block->flags = ram_flags; 2034*8d7f2e76SPhilippe Mathieu-Daudé ram_block_add(new_block, &local_err); 2035*8d7f2e76SPhilippe Mathieu-Daudé if (local_err) { 2036*8d7f2e76SPhilippe Mathieu-Daudé g_free(new_block); 2037*8d7f2e76SPhilippe Mathieu-Daudé error_propagate(errp, local_err); 2038*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 2039*8d7f2e76SPhilippe Mathieu-Daudé } 2040*8d7f2e76SPhilippe Mathieu-Daudé return new_block; 2041*8d7f2e76SPhilippe Mathieu-Daudé } 2042*8d7f2e76SPhilippe Mathieu-Daudé 2043*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 2044*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr, Error **errp) 2045*8d7f2e76SPhilippe Mathieu-Daudé { 2046*8d7f2e76SPhilippe Mathieu-Daudé return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr, 2047*8d7f2e76SPhilippe Mathieu-Daudé errp); 2048*8d7f2e76SPhilippe Mathieu-Daudé } 2049*8d7f2e76SPhilippe Mathieu-Daudé 2050*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, 2051*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr, Error **errp) 2052*8d7f2e76SPhilippe Mathieu-Daudé { 2053*8d7f2e76SPhilippe Mathieu-Daudé assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE)) == 0); 2054*8d7f2e76SPhilippe Mathieu-Daudé return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp); 2055*8d7f2e76SPhilippe Mathieu-Daudé } 2056*8d7f2e76SPhilippe Mathieu-Daudé 2057*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz, 2058*8d7f2e76SPhilippe Mathieu-Daudé void (*resized)(const char*, 2059*8d7f2e76SPhilippe Mathieu-Daudé uint64_t length, 2060*8d7f2e76SPhilippe Mathieu-Daudé void *host), 2061*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr, Error **errp) 2062*8d7f2e76SPhilippe Mathieu-Daudé { 2063*8d7f2e76SPhilippe Mathieu-Daudé return qemu_ram_alloc_internal(size, maxsz, resized, NULL, 2064*8d7f2e76SPhilippe Mathieu-Daudé RAM_RESIZEABLE, mr, errp); 2065*8d7f2e76SPhilippe Mathieu-Daudé } 2066*8d7f2e76SPhilippe Mathieu-Daudé 2067*8d7f2e76SPhilippe Mathieu-Daudé static void reclaim_ramblock(RAMBlock *block) 2068*8d7f2e76SPhilippe Mathieu-Daudé { 2069*8d7f2e76SPhilippe Mathieu-Daudé if (block->flags & RAM_PREALLOC) { 2070*8d7f2e76SPhilippe Mathieu-Daudé ; 2071*8d7f2e76SPhilippe Mathieu-Daudé } else if (xen_enabled()) { 2072*8d7f2e76SPhilippe Mathieu-Daudé xen_invalidate_map_cache_entry(block->host); 2073*8d7f2e76SPhilippe Mathieu-Daudé #ifndef _WIN32 2074*8d7f2e76SPhilippe Mathieu-Daudé } else if (block->fd >= 0) { 2075*8d7f2e76SPhilippe Mathieu-Daudé qemu_ram_munmap(block->fd, block->host, block->max_length); 2076*8d7f2e76SPhilippe Mathieu-Daudé close(block->fd); 2077*8d7f2e76SPhilippe Mathieu-Daudé #endif 2078*8d7f2e76SPhilippe Mathieu-Daudé } else { 2079*8d7f2e76SPhilippe Mathieu-Daudé qemu_anon_ram_free(block->host, block->max_length); 2080*8d7f2e76SPhilippe Mathieu-Daudé } 2081*8d7f2e76SPhilippe Mathieu-Daudé g_free(block); 2082*8d7f2e76SPhilippe Mathieu-Daudé } 2083*8d7f2e76SPhilippe Mathieu-Daudé 2084*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_free(RAMBlock *block) 2085*8d7f2e76SPhilippe Mathieu-Daudé { 2086*8d7f2e76SPhilippe Mathieu-Daudé if (!block) { 2087*8d7f2e76SPhilippe Mathieu-Daudé return; 2088*8d7f2e76SPhilippe Mathieu-Daudé } 2089*8d7f2e76SPhilippe Mathieu-Daudé 2090*8d7f2e76SPhilippe Mathieu-Daudé if (block->host) { 2091*8d7f2e76SPhilippe Mathieu-Daudé ram_block_notify_remove(block->host, block->used_length, 2092*8d7f2e76SPhilippe Mathieu-Daudé block->max_length); 2093*8d7f2e76SPhilippe Mathieu-Daudé } 2094*8d7f2e76SPhilippe Mathieu-Daudé 2095*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock_ramlist(); 2096*8d7f2e76SPhilippe Mathieu-Daudé QLIST_REMOVE_RCU(block, next); 2097*8d7f2e76SPhilippe Mathieu-Daudé ram_list.mru_block = NULL; 2098*8d7f2e76SPhilippe Mathieu-Daudé /* Write list before version */ 2099*8d7f2e76SPhilippe Mathieu-Daudé smp_wmb(); 2100*8d7f2e76SPhilippe Mathieu-Daudé ram_list.version++; 2101*8d7f2e76SPhilippe Mathieu-Daudé call_rcu(block, reclaim_ramblock, rcu); 2102*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_ramlist(); 2103*8d7f2e76SPhilippe Mathieu-Daudé } 2104*8d7f2e76SPhilippe Mathieu-Daudé 2105*8d7f2e76SPhilippe Mathieu-Daudé #ifndef _WIN32 2106*8d7f2e76SPhilippe Mathieu-Daudé void qemu_ram_remap(ram_addr_t addr, ram_addr_t length) 2107*8d7f2e76SPhilippe Mathieu-Daudé { 2108*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 2109*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t offset; 2110*8d7f2e76SPhilippe Mathieu-Daudé int flags; 2111*8d7f2e76SPhilippe Mathieu-Daudé void *area, *vaddr; 2112*8d7f2e76SPhilippe Mathieu-Daudé int prot; 2113*8d7f2e76SPhilippe Mathieu-Daudé 2114*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 2115*8d7f2e76SPhilippe Mathieu-Daudé offset = addr - block->offset; 2116*8d7f2e76SPhilippe Mathieu-Daudé if (offset < block->max_length) { 2117*8d7f2e76SPhilippe Mathieu-Daudé vaddr = ramblock_ptr(block, offset); 2118*8d7f2e76SPhilippe Mathieu-Daudé if (block->flags & RAM_PREALLOC) { 2119*8d7f2e76SPhilippe Mathieu-Daudé ; 2120*8d7f2e76SPhilippe Mathieu-Daudé } else if (xen_enabled()) { 2121*8d7f2e76SPhilippe Mathieu-Daudé abort(); 2122*8d7f2e76SPhilippe Mathieu-Daudé } else { 2123*8d7f2e76SPhilippe Mathieu-Daudé flags = MAP_FIXED; 2124*8d7f2e76SPhilippe Mathieu-Daudé flags |= block->flags & RAM_SHARED ? 2125*8d7f2e76SPhilippe Mathieu-Daudé MAP_SHARED : MAP_PRIVATE; 2126*8d7f2e76SPhilippe Mathieu-Daudé flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0; 2127*8d7f2e76SPhilippe Mathieu-Daudé prot = PROT_READ; 2128*8d7f2e76SPhilippe Mathieu-Daudé prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE; 2129*8d7f2e76SPhilippe Mathieu-Daudé if (block->fd >= 0) { 2130*8d7f2e76SPhilippe Mathieu-Daudé area = mmap(vaddr, length, prot, flags, block->fd, 2131*8d7f2e76SPhilippe Mathieu-Daudé offset + block->fd_offset); 2132*8d7f2e76SPhilippe Mathieu-Daudé } else { 2133*8d7f2e76SPhilippe Mathieu-Daudé flags |= MAP_ANONYMOUS; 2134*8d7f2e76SPhilippe Mathieu-Daudé area = mmap(vaddr, length, prot, flags, -1, 0); 2135*8d7f2e76SPhilippe Mathieu-Daudé } 2136*8d7f2e76SPhilippe Mathieu-Daudé if (area != vaddr) { 2137*8d7f2e76SPhilippe Mathieu-Daudé error_report("Could not remap addr: " 2138*8d7f2e76SPhilippe Mathieu-Daudé RAM_ADDR_FMT "@" RAM_ADDR_FMT "", 2139*8d7f2e76SPhilippe Mathieu-Daudé length, addr); 2140*8d7f2e76SPhilippe Mathieu-Daudé exit(1); 2141*8d7f2e76SPhilippe Mathieu-Daudé } 2142*8d7f2e76SPhilippe Mathieu-Daudé memory_try_enable_merging(vaddr, length); 2143*8d7f2e76SPhilippe Mathieu-Daudé qemu_ram_setup_dump(vaddr, length); 2144*8d7f2e76SPhilippe Mathieu-Daudé } 2145*8d7f2e76SPhilippe Mathieu-Daudé } 2146*8d7f2e76SPhilippe Mathieu-Daudé } 2147*8d7f2e76SPhilippe Mathieu-Daudé } 2148*8d7f2e76SPhilippe Mathieu-Daudé #endif /* !_WIN32 */ 2149*8d7f2e76SPhilippe Mathieu-Daudé 2150*8d7f2e76SPhilippe Mathieu-Daudé /* Return a host pointer to ram allocated with qemu_ram_alloc. 2151*8d7f2e76SPhilippe Mathieu-Daudé * This should not be used for general purpose DMA. Use address_space_map 2152*8d7f2e76SPhilippe Mathieu-Daudé * or address_space_rw instead. For local memory (e.g. video ram) that the 2153*8d7f2e76SPhilippe Mathieu-Daudé * device owns, use memory_region_get_ram_ptr. 2154*8d7f2e76SPhilippe Mathieu-Daudé * 2155*8d7f2e76SPhilippe Mathieu-Daudé * Called within RCU critical section. 2156*8d7f2e76SPhilippe Mathieu-Daudé */ 2157*8d7f2e76SPhilippe Mathieu-Daudé void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr) 2158*8d7f2e76SPhilippe Mathieu-Daudé { 2159*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block = ram_block; 2160*8d7f2e76SPhilippe Mathieu-Daudé 2161*8d7f2e76SPhilippe Mathieu-Daudé if (block == NULL) { 2162*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_get_ram_block(addr); 2163*8d7f2e76SPhilippe Mathieu-Daudé addr -= block->offset; 2164*8d7f2e76SPhilippe Mathieu-Daudé } 2165*8d7f2e76SPhilippe Mathieu-Daudé 2166*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled() && block->host == NULL) { 2167*8d7f2e76SPhilippe Mathieu-Daudé /* We need to check if the requested address is in the RAM 2168*8d7f2e76SPhilippe Mathieu-Daudé * because we don't want to map the entire memory in QEMU. 2169*8d7f2e76SPhilippe Mathieu-Daudé * In that case just map until the end of the page. 2170*8d7f2e76SPhilippe Mathieu-Daudé */ 2171*8d7f2e76SPhilippe Mathieu-Daudé if (block->offset == 0) { 2172*8d7f2e76SPhilippe Mathieu-Daudé return xen_map_cache(addr, 0, 0, false); 2173*8d7f2e76SPhilippe Mathieu-Daudé } 2174*8d7f2e76SPhilippe Mathieu-Daudé 2175*8d7f2e76SPhilippe Mathieu-Daudé block->host = xen_map_cache(block->offset, block->max_length, 1, false); 2176*8d7f2e76SPhilippe Mathieu-Daudé } 2177*8d7f2e76SPhilippe Mathieu-Daudé return ramblock_ptr(block, addr); 2178*8d7f2e76SPhilippe Mathieu-Daudé } 2179*8d7f2e76SPhilippe Mathieu-Daudé 2180*8d7f2e76SPhilippe Mathieu-Daudé /* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr 2181*8d7f2e76SPhilippe Mathieu-Daudé * but takes a size argument. 2182*8d7f2e76SPhilippe Mathieu-Daudé * 2183*8d7f2e76SPhilippe Mathieu-Daudé * Called within RCU critical section. 2184*8d7f2e76SPhilippe Mathieu-Daudé */ 2185*8d7f2e76SPhilippe Mathieu-Daudé static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr, 2186*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *size, bool lock) 2187*8d7f2e76SPhilippe Mathieu-Daudé { 2188*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block = ram_block; 2189*8d7f2e76SPhilippe Mathieu-Daudé if (*size == 0) { 2190*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 2191*8d7f2e76SPhilippe Mathieu-Daudé } 2192*8d7f2e76SPhilippe Mathieu-Daudé 2193*8d7f2e76SPhilippe Mathieu-Daudé if (block == NULL) { 2194*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_get_ram_block(addr); 2195*8d7f2e76SPhilippe Mathieu-Daudé addr -= block->offset; 2196*8d7f2e76SPhilippe Mathieu-Daudé } 2197*8d7f2e76SPhilippe Mathieu-Daudé *size = MIN(*size, block->max_length - addr); 2198*8d7f2e76SPhilippe Mathieu-Daudé 2199*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled() && block->host == NULL) { 2200*8d7f2e76SPhilippe Mathieu-Daudé /* We need to check if the requested address is in the RAM 2201*8d7f2e76SPhilippe Mathieu-Daudé * because we don't want to map the entire memory in QEMU. 2202*8d7f2e76SPhilippe Mathieu-Daudé * In that case just map the requested area. 2203*8d7f2e76SPhilippe Mathieu-Daudé */ 2204*8d7f2e76SPhilippe Mathieu-Daudé if (block->offset == 0) { 2205*8d7f2e76SPhilippe Mathieu-Daudé return xen_map_cache(addr, *size, lock, lock); 2206*8d7f2e76SPhilippe Mathieu-Daudé } 2207*8d7f2e76SPhilippe Mathieu-Daudé 2208*8d7f2e76SPhilippe Mathieu-Daudé block->host = xen_map_cache(block->offset, block->max_length, 1, lock); 2209*8d7f2e76SPhilippe Mathieu-Daudé } 2210*8d7f2e76SPhilippe Mathieu-Daudé 2211*8d7f2e76SPhilippe Mathieu-Daudé return ramblock_ptr(block, addr); 2212*8d7f2e76SPhilippe Mathieu-Daudé } 2213*8d7f2e76SPhilippe Mathieu-Daudé 2214*8d7f2e76SPhilippe Mathieu-Daudé /* Return the offset of a hostpointer within a ramblock */ 2215*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host) 2216*8d7f2e76SPhilippe Mathieu-Daudé { 2217*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host; 2218*8d7f2e76SPhilippe Mathieu-Daudé assert((uintptr_t)host >= (uintptr_t)rb->host); 2219*8d7f2e76SPhilippe Mathieu-Daudé assert(res < rb->max_length); 2220*8d7f2e76SPhilippe Mathieu-Daudé 2221*8d7f2e76SPhilippe Mathieu-Daudé return res; 2222*8d7f2e76SPhilippe Mathieu-Daudé } 2223*8d7f2e76SPhilippe Mathieu-Daudé 2224*8d7f2e76SPhilippe Mathieu-Daudé /* 2225*8d7f2e76SPhilippe Mathieu-Daudé * Translates a host ptr back to a RAMBlock, a ram_addr and an offset 2226*8d7f2e76SPhilippe Mathieu-Daudé * in that RAMBlock. 2227*8d7f2e76SPhilippe Mathieu-Daudé * 2228*8d7f2e76SPhilippe Mathieu-Daudé * ptr: Host pointer to look up 2229*8d7f2e76SPhilippe Mathieu-Daudé * round_offset: If true round the result offset down to a page boundary 2230*8d7f2e76SPhilippe Mathieu-Daudé * *ram_addr: set to result ram_addr 2231*8d7f2e76SPhilippe Mathieu-Daudé * *offset: set to result offset within the RAMBlock 2232*8d7f2e76SPhilippe Mathieu-Daudé * 2233*8d7f2e76SPhilippe Mathieu-Daudé * Returns: RAMBlock (or NULL if not found) 2234*8d7f2e76SPhilippe Mathieu-Daudé * 2235*8d7f2e76SPhilippe Mathieu-Daudé * By the time this function returns, the returned pointer is not protected 2236*8d7f2e76SPhilippe Mathieu-Daudé * by RCU anymore. If the caller is not within an RCU critical section and 2237*8d7f2e76SPhilippe Mathieu-Daudé * does not hold the iothread lock, it must have other means of protecting the 2238*8d7f2e76SPhilippe Mathieu-Daudé * pointer, such as a reference to the region that includes the incoming 2239*8d7f2e76SPhilippe Mathieu-Daudé * ram_addr_t. 2240*8d7f2e76SPhilippe Mathieu-Daudé */ 2241*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset, 2242*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t *offset) 2243*8d7f2e76SPhilippe Mathieu-Daudé { 2244*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 2245*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *host = ptr; 2246*8d7f2e76SPhilippe Mathieu-Daudé 2247*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled()) { 2248*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t ram_addr; 2249*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 2250*8d7f2e76SPhilippe Mathieu-Daudé ram_addr = xen_ram_addr_from_mapcache(ptr); 2251*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_get_ram_block(ram_addr); 2252*8d7f2e76SPhilippe Mathieu-Daudé if (block) { 2253*8d7f2e76SPhilippe Mathieu-Daudé *offset = ram_addr - block->offset; 2254*8d7f2e76SPhilippe Mathieu-Daudé } 2255*8d7f2e76SPhilippe Mathieu-Daudé return block; 2256*8d7f2e76SPhilippe Mathieu-Daudé } 2257*8d7f2e76SPhilippe Mathieu-Daudé 2258*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 2259*8d7f2e76SPhilippe Mathieu-Daudé block = qatomic_rcu_read(&ram_list.mru_block); 2260*8d7f2e76SPhilippe Mathieu-Daudé if (block && block->host && host - block->host < block->max_length) { 2261*8d7f2e76SPhilippe Mathieu-Daudé goto found; 2262*8d7f2e76SPhilippe Mathieu-Daudé } 2263*8d7f2e76SPhilippe Mathieu-Daudé 2264*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 2265*8d7f2e76SPhilippe Mathieu-Daudé /* This case append when the block is not mapped. */ 2266*8d7f2e76SPhilippe Mathieu-Daudé if (block->host == NULL) { 2267*8d7f2e76SPhilippe Mathieu-Daudé continue; 2268*8d7f2e76SPhilippe Mathieu-Daudé } 2269*8d7f2e76SPhilippe Mathieu-Daudé if (host - block->host < block->max_length) { 2270*8d7f2e76SPhilippe Mathieu-Daudé goto found; 2271*8d7f2e76SPhilippe Mathieu-Daudé } 2272*8d7f2e76SPhilippe Mathieu-Daudé } 2273*8d7f2e76SPhilippe Mathieu-Daudé 2274*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 2275*8d7f2e76SPhilippe Mathieu-Daudé 2276*8d7f2e76SPhilippe Mathieu-Daudé found: 2277*8d7f2e76SPhilippe Mathieu-Daudé *offset = (host - block->host); 2278*8d7f2e76SPhilippe Mathieu-Daudé if (round_offset) { 2279*8d7f2e76SPhilippe Mathieu-Daudé *offset &= TARGET_PAGE_MASK; 2280*8d7f2e76SPhilippe Mathieu-Daudé } 2281*8d7f2e76SPhilippe Mathieu-Daudé return block; 2282*8d7f2e76SPhilippe Mathieu-Daudé } 2283*8d7f2e76SPhilippe Mathieu-Daudé 2284*8d7f2e76SPhilippe Mathieu-Daudé /* 2285*8d7f2e76SPhilippe Mathieu-Daudé * Finds the named RAMBlock 2286*8d7f2e76SPhilippe Mathieu-Daudé * 2287*8d7f2e76SPhilippe Mathieu-Daudé * name: The name of RAMBlock to find 2288*8d7f2e76SPhilippe Mathieu-Daudé * 2289*8d7f2e76SPhilippe Mathieu-Daudé * Returns: RAMBlock (or NULL if not found) 2290*8d7f2e76SPhilippe Mathieu-Daudé */ 2291*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *qemu_ram_block_by_name(const char *name) 2292*8d7f2e76SPhilippe Mathieu-Daudé { 2293*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 2294*8d7f2e76SPhilippe Mathieu-Daudé 2295*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 2296*8d7f2e76SPhilippe Mathieu-Daudé if (!strcmp(name, block->idstr)) { 2297*8d7f2e76SPhilippe Mathieu-Daudé return block; 2298*8d7f2e76SPhilippe Mathieu-Daudé } 2299*8d7f2e76SPhilippe Mathieu-Daudé } 2300*8d7f2e76SPhilippe Mathieu-Daudé 2301*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 2302*8d7f2e76SPhilippe Mathieu-Daudé } 2303*8d7f2e76SPhilippe Mathieu-Daudé 2304*8d7f2e76SPhilippe Mathieu-Daudé /* 2305*8d7f2e76SPhilippe Mathieu-Daudé * Some of the system routines need to translate from a host pointer 2306*8d7f2e76SPhilippe Mathieu-Daudé * (typically a TLB entry) back to a ram offset. 2307*8d7f2e76SPhilippe Mathieu-Daudé */ 2308*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_addr_from_host(void *ptr) 2309*8d7f2e76SPhilippe Mathieu-Daudé { 2310*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 2311*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t offset; 2312*8d7f2e76SPhilippe Mathieu-Daudé 2313*8d7f2e76SPhilippe Mathieu-Daudé block = qemu_ram_block_from_host(ptr, false, &offset); 2314*8d7f2e76SPhilippe Mathieu-Daudé if (!block) { 2315*8d7f2e76SPhilippe Mathieu-Daudé return RAM_ADDR_INVALID; 2316*8d7f2e76SPhilippe Mathieu-Daudé } 2317*8d7f2e76SPhilippe Mathieu-Daudé 2318*8d7f2e76SPhilippe Mathieu-Daudé return block->offset + offset; 2319*8d7f2e76SPhilippe Mathieu-Daudé } 2320*8d7f2e76SPhilippe Mathieu-Daudé 2321*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) 2322*8d7f2e76SPhilippe Mathieu-Daudé { 2323*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t ram_addr; 2324*8d7f2e76SPhilippe Mathieu-Daudé 2325*8d7f2e76SPhilippe Mathieu-Daudé ram_addr = qemu_ram_addr_from_host(ptr); 2326*8d7f2e76SPhilippe Mathieu-Daudé if (ram_addr == RAM_ADDR_INVALID) { 2327*8d7f2e76SPhilippe Mathieu-Daudé error_report("Bad ram pointer %p", ptr); 2328*8d7f2e76SPhilippe Mathieu-Daudé abort(); 2329*8d7f2e76SPhilippe Mathieu-Daudé } 2330*8d7f2e76SPhilippe Mathieu-Daudé return ram_addr; 2331*8d7f2e76SPhilippe Mathieu-Daudé } 2332*8d7f2e76SPhilippe Mathieu-Daudé 2333*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult flatview_read(FlatView *fv, hwaddr addr, 2334*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, void *buf, hwaddr len); 2335*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, 2336*8d7f2e76SPhilippe Mathieu-Daudé const void *buf, hwaddr len); 2337*8d7f2e76SPhilippe Mathieu-Daudé static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len, 2338*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, MemTxAttrs attrs); 2339*8d7f2e76SPhilippe Mathieu-Daudé 2340*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data, 2341*8d7f2e76SPhilippe Mathieu-Daudé unsigned len, MemTxAttrs attrs) 2342*8d7f2e76SPhilippe Mathieu-Daudé { 2343*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage = opaque; 2344*8d7f2e76SPhilippe Mathieu-Daudé uint8_t buf[8]; 2345*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult res; 2346*8d7f2e76SPhilippe Mathieu-Daudé 2347*8d7f2e76SPhilippe Mathieu-Daudé #if defined(DEBUG_SUBPAGE) 2348*8d7f2e76SPhilippe Mathieu-Daudé printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__, 2349*8d7f2e76SPhilippe Mathieu-Daudé subpage, len, addr); 2350*8d7f2e76SPhilippe Mathieu-Daudé #endif 2351*8d7f2e76SPhilippe Mathieu-Daudé res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len); 2352*8d7f2e76SPhilippe Mathieu-Daudé if (res) { 2353*8d7f2e76SPhilippe Mathieu-Daudé return res; 2354*8d7f2e76SPhilippe Mathieu-Daudé } 2355*8d7f2e76SPhilippe Mathieu-Daudé *data = ldn_p(buf, len); 2356*8d7f2e76SPhilippe Mathieu-Daudé return MEMTX_OK; 2357*8d7f2e76SPhilippe Mathieu-Daudé } 2358*8d7f2e76SPhilippe Mathieu-Daudé 2359*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult subpage_write(void *opaque, hwaddr addr, 2360*8d7f2e76SPhilippe Mathieu-Daudé uint64_t value, unsigned len, MemTxAttrs attrs) 2361*8d7f2e76SPhilippe Mathieu-Daudé { 2362*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage = opaque; 2363*8d7f2e76SPhilippe Mathieu-Daudé uint8_t buf[8]; 2364*8d7f2e76SPhilippe Mathieu-Daudé 2365*8d7f2e76SPhilippe Mathieu-Daudé #if defined(DEBUG_SUBPAGE) 2366*8d7f2e76SPhilippe Mathieu-Daudé printf("%s: subpage %p len %u addr " HWADDR_FMT_plx 2367*8d7f2e76SPhilippe Mathieu-Daudé " value %"PRIx64"\n", 2368*8d7f2e76SPhilippe Mathieu-Daudé __func__, subpage, len, addr, value); 2369*8d7f2e76SPhilippe Mathieu-Daudé #endif 2370*8d7f2e76SPhilippe Mathieu-Daudé stn_p(buf, len, value); 2371*8d7f2e76SPhilippe Mathieu-Daudé return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len); 2372*8d7f2e76SPhilippe Mathieu-Daudé } 2373*8d7f2e76SPhilippe Mathieu-Daudé 2374*8d7f2e76SPhilippe Mathieu-Daudé static bool subpage_accepts(void *opaque, hwaddr addr, 2375*8d7f2e76SPhilippe Mathieu-Daudé unsigned len, bool is_write, 2376*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 2377*8d7f2e76SPhilippe Mathieu-Daudé { 2378*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *subpage = opaque; 2379*8d7f2e76SPhilippe Mathieu-Daudé #if defined(DEBUG_SUBPAGE) 2380*8d7f2e76SPhilippe Mathieu-Daudé printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n", 2381*8d7f2e76SPhilippe Mathieu-Daudé __func__, subpage, is_write ? 'w' : 'r', len, addr); 2382*8d7f2e76SPhilippe Mathieu-Daudé #endif 2383*8d7f2e76SPhilippe Mathieu-Daudé 2384*8d7f2e76SPhilippe Mathieu-Daudé return flatview_access_valid(subpage->fv, addr + subpage->base, 2385*8d7f2e76SPhilippe Mathieu-Daudé len, is_write, attrs); 2386*8d7f2e76SPhilippe Mathieu-Daudé } 2387*8d7f2e76SPhilippe Mathieu-Daudé 2388*8d7f2e76SPhilippe Mathieu-Daudé static const MemoryRegionOps subpage_ops = { 2389*8d7f2e76SPhilippe Mathieu-Daudé .read_with_attrs = subpage_read, 2390*8d7f2e76SPhilippe Mathieu-Daudé .write_with_attrs = subpage_write, 2391*8d7f2e76SPhilippe Mathieu-Daudé .impl.min_access_size = 1, 2392*8d7f2e76SPhilippe Mathieu-Daudé .impl.max_access_size = 8, 2393*8d7f2e76SPhilippe Mathieu-Daudé .valid.min_access_size = 1, 2394*8d7f2e76SPhilippe Mathieu-Daudé .valid.max_access_size = 8, 2395*8d7f2e76SPhilippe Mathieu-Daudé .valid.accepts = subpage_accepts, 2396*8d7f2e76SPhilippe Mathieu-Daudé .endianness = DEVICE_NATIVE_ENDIAN, 2397*8d7f2e76SPhilippe Mathieu-Daudé }; 2398*8d7f2e76SPhilippe Mathieu-Daudé 2399*8d7f2e76SPhilippe Mathieu-Daudé static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, 2400*8d7f2e76SPhilippe Mathieu-Daudé uint16_t section) 2401*8d7f2e76SPhilippe Mathieu-Daudé { 2402*8d7f2e76SPhilippe Mathieu-Daudé int idx, eidx; 2403*8d7f2e76SPhilippe Mathieu-Daudé 2404*8d7f2e76SPhilippe Mathieu-Daudé if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE) 2405*8d7f2e76SPhilippe Mathieu-Daudé return -1; 2406*8d7f2e76SPhilippe Mathieu-Daudé idx = SUBPAGE_IDX(start); 2407*8d7f2e76SPhilippe Mathieu-Daudé eidx = SUBPAGE_IDX(end); 2408*8d7f2e76SPhilippe Mathieu-Daudé #if defined(DEBUG_SUBPAGE) 2409*8d7f2e76SPhilippe Mathieu-Daudé printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n", 2410*8d7f2e76SPhilippe Mathieu-Daudé __func__, mmio, start, end, idx, eidx, section); 2411*8d7f2e76SPhilippe Mathieu-Daudé #endif 2412*8d7f2e76SPhilippe Mathieu-Daudé for (; idx <= eidx; idx++) { 2413*8d7f2e76SPhilippe Mathieu-Daudé mmio->sub_section[idx] = section; 2414*8d7f2e76SPhilippe Mathieu-Daudé } 2415*8d7f2e76SPhilippe Mathieu-Daudé 2416*8d7f2e76SPhilippe Mathieu-Daudé return 0; 2417*8d7f2e76SPhilippe Mathieu-Daudé } 2418*8d7f2e76SPhilippe Mathieu-Daudé 2419*8d7f2e76SPhilippe Mathieu-Daudé static subpage_t *subpage_init(FlatView *fv, hwaddr base) 2420*8d7f2e76SPhilippe Mathieu-Daudé { 2421*8d7f2e76SPhilippe Mathieu-Daudé subpage_t *mmio; 2422*8d7f2e76SPhilippe Mathieu-Daudé 2423*8d7f2e76SPhilippe Mathieu-Daudé /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */ 2424*8d7f2e76SPhilippe Mathieu-Daudé mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t)); 2425*8d7f2e76SPhilippe Mathieu-Daudé mmio->fv = fv; 2426*8d7f2e76SPhilippe Mathieu-Daudé mmio->base = base; 2427*8d7f2e76SPhilippe Mathieu-Daudé memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio, 2428*8d7f2e76SPhilippe Mathieu-Daudé NULL, TARGET_PAGE_SIZE); 2429*8d7f2e76SPhilippe Mathieu-Daudé mmio->iomem.subpage = true; 2430*8d7f2e76SPhilippe Mathieu-Daudé #if defined(DEBUG_SUBPAGE) 2431*8d7f2e76SPhilippe Mathieu-Daudé printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__, 2432*8d7f2e76SPhilippe Mathieu-Daudé mmio, base, TARGET_PAGE_SIZE); 2433*8d7f2e76SPhilippe Mathieu-Daudé #endif 2434*8d7f2e76SPhilippe Mathieu-Daudé 2435*8d7f2e76SPhilippe Mathieu-Daudé return mmio; 2436*8d7f2e76SPhilippe Mathieu-Daudé } 2437*8d7f2e76SPhilippe Mathieu-Daudé 2438*8d7f2e76SPhilippe Mathieu-Daudé static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr) 2439*8d7f2e76SPhilippe Mathieu-Daudé { 2440*8d7f2e76SPhilippe Mathieu-Daudé assert(fv); 2441*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection section = { 2442*8d7f2e76SPhilippe Mathieu-Daudé .fv = fv, 2443*8d7f2e76SPhilippe Mathieu-Daudé .mr = mr, 2444*8d7f2e76SPhilippe Mathieu-Daudé .offset_within_address_space = 0, 2445*8d7f2e76SPhilippe Mathieu-Daudé .offset_within_region = 0, 2446*8d7f2e76SPhilippe Mathieu-Daudé .size = int128_2_64(), 2447*8d7f2e76SPhilippe Mathieu-Daudé }; 2448*8d7f2e76SPhilippe Mathieu-Daudé 2449*8d7f2e76SPhilippe Mathieu-Daudé return phys_section_add(map, §ion); 2450*8d7f2e76SPhilippe Mathieu-Daudé } 2451*8d7f2e76SPhilippe Mathieu-Daudé 2452*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *iotlb_to_section(CPUState *cpu, 2453*8d7f2e76SPhilippe Mathieu-Daudé hwaddr index, MemTxAttrs attrs) 2454*8d7f2e76SPhilippe Mathieu-Daudé { 2455*8d7f2e76SPhilippe Mathieu-Daudé int asidx = cpu_asidx_from_attrs(cpu, attrs); 2456*8d7f2e76SPhilippe Mathieu-Daudé CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx]; 2457*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = cpuas->memory_dispatch; 2458*8d7f2e76SPhilippe Mathieu-Daudé int section_index = index & ~TARGET_PAGE_MASK; 2459*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *ret; 2460*8d7f2e76SPhilippe Mathieu-Daudé 2461*8d7f2e76SPhilippe Mathieu-Daudé assert(section_index < d->map.sections_nb); 2462*8d7f2e76SPhilippe Mathieu-Daudé ret = d->map.sections + section_index; 2463*8d7f2e76SPhilippe Mathieu-Daudé assert(ret->mr); 2464*8d7f2e76SPhilippe Mathieu-Daudé assert(ret->mr->ops); 2465*8d7f2e76SPhilippe Mathieu-Daudé 2466*8d7f2e76SPhilippe Mathieu-Daudé return ret; 2467*8d7f2e76SPhilippe Mathieu-Daudé } 2468*8d7f2e76SPhilippe Mathieu-Daudé 2469*8d7f2e76SPhilippe Mathieu-Daudé static void io_mem_init(void) 2470*8d7f2e76SPhilippe Mathieu-Daudé { 2471*8d7f2e76SPhilippe Mathieu-Daudé memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL, 2472*8d7f2e76SPhilippe Mathieu-Daudé NULL, UINT64_MAX); 2473*8d7f2e76SPhilippe Mathieu-Daudé } 2474*8d7f2e76SPhilippe Mathieu-Daudé 2475*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv) 2476*8d7f2e76SPhilippe Mathieu-Daudé { 2477*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1); 2478*8d7f2e76SPhilippe Mathieu-Daudé uint16_t n; 2479*8d7f2e76SPhilippe Mathieu-Daudé 2480*8d7f2e76SPhilippe Mathieu-Daudé n = dummy_section(&d->map, fv, &io_mem_unassigned); 2481*8d7f2e76SPhilippe Mathieu-Daudé assert(n == PHYS_SECTION_UNASSIGNED); 2482*8d7f2e76SPhilippe Mathieu-Daudé 2483*8d7f2e76SPhilippe Mathieu-Daudé d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 }; 2484*8d7f2e76SPhilippe Mathieu-Daudé 2485*8d7f2e76SPhilippe Mathieu-Daudé return d; 2486*8d7f2e76SPhilippe Mathieu-Daudé } 2487*8d7f2e76SPhilippe Mathieu-Daudé 2488*8d7f2e76SPhilippe Mathieu-Daudé void address_space_dispatch_free(AddressSpaceDispatch *d) 2489*8d7f2e76SPhilippe Mathieu-Daudé { 2490*8d7f2e76SPhilippe Mathieu-Daudé phys_sections_free(&d->map); 2491*8d7f2e76SPhilippe Mathieu-Daudé g_free(d); 2492*8d7f2e76SPhilippe Mathieu-Daudé } 2493*8d7f2e76SPhilippe Mathieu-Daudé 2494*8d7f2e76SPhilippe Mathieu-Daudé static void do_nothing(CPUState *cpu, run_on_cpu_data d) 2495*8d7f2e76SPhilippe Mathieu-Daudé { 2496*8d7f2e76SPhilippe Mathieu-Daudé } 2497*8d7f2e76SPhilippe Mathieu-Daudé 2498*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_log_global_after_sync(MemoryListener *listener) 2499*8d7f2e76SPhilippe Mathieu-Daudé { 2500*8d7f2e76SPhilippe Mathieu-Daudé CPUAddressSpace *cpuas; 2501*8d7f2e76SPhilippe Mathieu-Daudé 2502*8d7f2e76SPhilippe Mathieu-Daudé /* Wait for the CPU to end the current TB. This avoids the following 2503*8d7f2e76SPhilippe Mathieu-Daudé * incorrect race: 2504*8d7f2e76SPhilippe Mathieu-Daudé * 2505*8d7f2e76SPhilippe Mathieu-Daudé * vCPU migration 2506*8d7f2e76SPhilippe Mathieu-Daudé * ---------------------- ------------------------- 2507*8d7f2e76SPhilippe Mathieu-Daudé * TLB check -> slow path 2508*8d7f2e76SPhilippe Mathieu-Daudé * notdirty_mem_write 2509*8d7f2e76SPhilippe Mathieu-Daudé * write to RAM 2510*8d7f2e76SPhilippe Mathieu-Daudé * mark dirty 2511*8d7f2e76SPhilippe Mathieu-Daudé * clear dirty flag 2512*8d7f2e76SPhilippe Mathieu-Daudé * TLB check -> fast path 2513*8d7f2e76SPhilippe Mathieu-Daudé * read memory 2514*8d7f2e76SPhilippe Mathieu-Daudé * write to RAM 2515*8d7f2e76SPhilippe Mathieu-Daudé * 2516*8d7f2e76SPhilippe Mathieu-Daudé * by pushing the migration thread's memory read after the vCPU thread has 2517*8d7f2e76SPhilippe Mathieu-Daudé * written the memory. 2518*8d7f2e76SPhilippe Mathieu-Daudé */ 2519*8d7f2e76SPhilippe Mathieu-Daudé if (replay_mode == REPLAY_MODE_NONE) { 2520*8d7f2e76SPhilippe Mathieu-Daudé /* 2521*8d7f2e76SPhilippe Mathieu-Daudé * VGA can make calls to this function while updating the screen. 2522*8d7f2e76SPhilippe Mathieu-Daudé * In record/replay mode this causes a deadlock, because 2523*8d7f2e76SPhilippe Mathieu-Daudé * run_on_cpu waits for rr mutex. Therefore no races are possible 2524*8d7f2e76SPhilippe Mathieu-Daudé * in this case and no need for making run_on_cpu when 2525*8d7f2e76SPhilippe Mathieu-Daudé * record/replay is enabled. 2526*8d7f2e76SPhilippe Mathieu-Daudé */ 2527*8d7f2e76SPhilippe Mathieu-Daudé cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener); 2528*8d7f2e76SPhilippe Mathieu-Daudé run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL); 2529*8d7f2e76SPhilippe Mathieu-Daudé } 2530*8d7f2e76SPhilippe Mathieu-Daudé } 2531*8d7f2e76SPhilippe Mathieu-Daudé 2532*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data) 2533*8d7f2e76SPhilippe Mathieu-Daudé { 2534*8d7f2e76SPhilippe Mathieu-Daudé CPUAddressSpace *cpuas = data.host_ptr; 2535*8d7f2e76SPhilippe Mathieu-Daudé 2536*8d7f2e76SPhilippe Mathieu-Daudé cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as); 2537*8d7f2e76SPhilippe Mathieu-Daudé tlb_flush(cpu); 2538*8d7f2e76SPhilippe Mathieu-Daudé } 2539*8d7f2e76SPhilippe Mathieu-Daudé 2540*8d7f2e76SPhilippe Mathieu-Daudé static void tcg_commit(MemoryListener *listener) 2541*8d7f2e76SPhilippe Mathieu-Daudé { 2542*8d7f2e76SPhilippe Mathieu-Daudé CPUAddressSpace *cpuas; 2543*8d7f2e76SPhilippe Mathieu-Daudé CPUState *cpu; 2544*8d7f2e76SPhilippe Mathieu-Daudé 2545*8d7f2e76SPhilippe Mathieu-Daudé assert(tcg_enabled()); 2546*8d7f2e76SPhilippe Mathieu-Daudé /* since each CPU stores ram addresses in its TLB cache, we must 2547*8d7f2e76SPhilippe Mathieu-Daudé reset the modified entries */ 2548*8d7f2e76SPhilippe Mathieu-Daudé cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener); 2549*8d7f2e76SPhilippe Mathieu-Daudé cpu = cpuas->cpu; 2550*8d7f2e76SPhilippe Mathieu-Daudé 2551*8d7f2e76SPhilippe Mathieu-Daudé /* 2552*8d7f2e76SPhilippe Mathieu-Daudé * Defer changes to as->memory_dispatch until the cpu is quiescent. 2553*8d7f2e76SPhilippe Mathieu-Daudé * Otherwise we race between (1) other cpu threads and (2) ongoing 2554*8d7f2e76SPhilippe Mathieu-Daudé * i/o for the current cpu thread, with data cached by mmu_lookup(). 2555*8d7f2e76SPhilippe Mathieu-Daudé * 2556*8d7f2e76SPhilippe Mathieu-Daudé * In addition, queueing the work function will kick the cpu back to 2557*8d7f2e76SPhilippe Mathieu-Daudé * the main loop, which will end the RCU critical section and reclaim 2558*8d7f2e76SPhilippe Mathieu-Daudé * the memory data structures. 2559*8d7f2e76SPhilippe Mathieu-Daudé * 2560*8d7f2e76SPhilippe Mathieu-Daudé * That said, the listener is also called during realize, before 2561*8d7f2e76SPhilippe Mathieu-Daudé * all of the tcg machinery for run-on is initialized: thus halt_cond. 2562*8d7f2e76SPhilippe Mathieu-Daudé */ 2563*8d7f2e76SPhilippe Mathieu-Daudé if (cpu->halt_cond) { 2564*8d7f2e76SPhilippe Mathieu-Daudé async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas)); 2565*8d7f2e76SPhilippe Mathieu-Daudé } else { 2566*8d7f2e76SPhilippe Mathieu-Daudé tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas)); 2567*8d7f2e76SPhilippe Mathieu-Daudé } 2568*8d7f2e76SPhilippe Mathieu-Daudé } 2569*8d7f2e76SPhilippe Mathieu-Daudé 2570*8d7f2e76SPhilippe Mathieu-Daudé static void memory_map_init(void) 2571*8d7f2e76SPhilippe Mathieu-Daudé { 2572*8d7f2e76SPhilippe Mathieu-Daudé system_memory = g_malloc(sizeof(*system_memory)); 2573*8d7f2e76SPhilippe Mathieu-Daudé 2574*8d7f2e76SPhilippe Mathieu-Daudé memory_region_init(system_memory, NULL, "system", UINT64_MAX); 2575*8d7f2e76SPhilippe Mathieu-Daudé address_space_init(&address_space_memory, system_memory, "memory"); 2576*8d7f2e76SPhilippe Mathieu-Daudé 2577*8d7f2e76SPhilippe Mathieu-Daudé system_io = g_malloc(sizeof(*system_io)); 2578*8d7f2e76SPhilippe Mathieu-Daudé memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io", 2579*8d7f2e76SPhilippe Mathieu-Daudé 65536); 2580*8d7f2e76SPhilippe Mathieu-Daudé address_space_init(&address_space_io, system_io, "I/O"); 2581*8d7f2e76SPhilippe Mathieu-Daudé } 2582*8d7f2e76SPhilippe Mathieu-Daudé 2583*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *get_system_memory(void) 2584*8d7f2e76SPhilippe Mathieu-Daudé { 2585*8d7f2e76SPhilippe Mathieu-Daudé return system_memory; 2586*8d7f2e76SPhilippe Mathieu-Daudé } 2587*8d7f2e76SPhilippe Mathieu-Daudé 2588*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *get_system_io(void) 2589*8d7f2e76SPhilippe Mathieu-Daudé { 2590*8d7f2e76SPhilippe Mathieu-Daudé return system_io; 2591*8d7f2e76SPhilippe Mathieu-Daudé } 2592*8d7f2e76SPhilippe Mathieu-Daudé 2593*8d7f2e76SPhilippe Mathieu-Daudé static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, 2594*8d7f2e76SPhilippe Mathieu-Daudé hwaddr length) 2595*8d7f2e76SPhilippe Mathieu-Daudé { 2596*8d7f2e76SPhilippe Mathieu-Daudé uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr); 2597*8d7f2e76SPhilippe Mathieu-Daudé addr += memory_region_get_ram_addr(mr); 2598*8d7f2e76SPhilippe Mathieu-Daudé 2599*8d7f2e76SPhilippe Mathieu-Daudé /* No early return if dirty_log_mask is or becomes 0, because 2600*8d7f2e76SPhilippe Mathieu-Daudé * cpu_physical_memory_set_dirty_range will still call 2601*8d7f2e76SPhilippe Mathieu-Daudé * xen_modified_memory. 2602*8d7f2e76SPhilippe Mathieu-Daudé */ 2603*8d7f2e76SPhilippe Mathieu-Daudé if (dirty_log_mask) { 2604*8d7f2e76SPhilippe Mathieu-Daudé dirty_log_mask = 2605*8d7f2e76SPhilippe Mathieu-Daudé cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); 2606*8d7f2e76SPhilippe Mathieu-Daudé } 2607*8d7f2e76SPhilippe Mathieu-Daudé if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { 2608*8d7f2e76SPhilippe Mathieu-Daudé assert(tcg_enabled()); 2609*8d7f2e76SPhilippe Mathieu-Daudé tb_invalidate_phys_range(addr, addr + length - 1); 2610*8d7f2e76SPhilippe Mathieu-Daudé dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); 2611*8d7f2e76SPhilippe Mathieu-Daudé } 2612*8d7f2e76SPhilippe Mathieu-Daudé cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); 2613*8d7f2e76SPhilippe Mathieu-Daudé } 2614*8d7f2e76SPhilippe Mathieu-Daudé 2615*8d7f2e76SPhilippe Mathieu-Daudé void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size) 2616*8d7f2e76SPhilippe Mathieu-Daudé { 2617*8d7f2e76SPhilippe Mathieu-Daudé /* 2618*8d7f2e76SPhilippe Mathieu-Daudé * In principle this function would work on other memory region types too, 2619*8d7f2e76SPhilippe Mathieu-Daudé * but the ROM device use case is the only one where this operation is 2620*8d7f2e76SPhilippe Mathieu-Daudé * necessary. Other memory regions should use the 2621*8d7f2e76SPhilippe Mathieu-Daudé * address_space_read/write() APIs. 2622*8d7f2e76SPhilippe Mathieu-Daudé */ 2623*8d7f2e76SPhilippe Mathieu-Daudé assert(memory_region_is_romd(mr)); 2624*8d7f2e76SPhilippe Mathieu-Daudé 2625*8d7f2e76SPhilippe Mathieu-Daudé invalidate_and_set_dirty(mr, addr, size); 2626*8d7f2e76SPhilippe Mathieu-Daudé } 2627*8d7f2e76SPhilippe Mathieu-Daudé 2628*8d7f2e76SPhilippe Mathieu-Daudé int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) 2629*8d7f2e76SPhilippe Mathieu-Daudé { 2630*8d7f2e76SPhilippe Mathieu-Daudé unsigned access_size_max = mr->ops->valid.max_access_size; 2631*8d7f2e76SPhilippe Mathieu-Daudé 2632*8d7f2e76SPhilippe Mathieu-Daudé /* Regions are assumed to support 1-4 byte accesses unless 2633*8d7f2e76SPhilippe Mathieu-Daudé otherwise specified. */ 2634*8d7f2e76SPhilippe Mathieu-Daudé if (access_size_max == 0) { 2635*8d7f2e76SPhilippe Mathieu-Daudé access_size_max = 4; 2636*8d7f2e76SPhilippe Mathieu-Daudé } 2637*8d7f2e76SPhilippe Mathieu-Daudé 2638*8d7f2e76SPhilippe Mathieu-Daudé /* Bound the maximum access by the alignment of the address. */ 2639*8d7f2e76SPhilippe Mathieu-Daudé if (!mr->ops->impl.unaligned) { 2640*8d7f2e76SPhilippe Mathieu-Daudé unsigned align_size_max = addr & -addr; 2641*8d7f2e76SPhilippe Mathieu-Daudé if (align_size_max != 0 && align_size_max < access_size_max) { 2642*8d7f2e76SPhilippe Mathieu-Daudé access_size_max = align_size_max; 2643*8d7f2e76SPhilippe Mathieu-Daudé } 2644*8d7f2e76SPhilippe Mathieu-Daudé } 2645*8d7f2e76SPhilippe Mathieu-Daudé 2646*8d7f2e76SPhilippe Mathieu-Daudé /* Don't attempt accesses larger than the maximum. */ 2647*8d7f2e76SPhilippe Mathieu-Daudé if (l > access_size_max) { 2648*8d7f2e76SPhilippe Mathieu-Daudé l = access_size_max; 2649*8d7f2e76SPhilippe Mathieu-Daudé } 2650*8d7f2e76SPhilippe Mathieu-Daudé l = pow2floor(l); 2651*8d7f2e76SPhilippe Mathieu-Daudé 2652*8d7f2e76SPhilippe Mathieu-Daudé return l; 2653*8d7f2e76SPhilippe Mathieu-Daudé } 2654*8d7f2e76SPhilippe Mathieu-Daudé 2655*8d7f2e76SPhilippe Mathieu-Daudé bool prepare_mmio_access(MemoryRegion *mr) 2656*8d7f2e76SPhilippe Mathieu-Daudé { 2657*8d7f2e76SPhilippe Mathieu-Daudé bool release_lock = false; 2658*8d7f2e76SPhilippe Mathieu-Daudé 2659*8d7f2e76SPhilippe Mathieu-Daudé if (!qemu_mutex_iothread_locked()) { 2660*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock_iothread(); 2661*8d7f2e76SPhilippe Mathieu-Daudé release_lock = true; 2662*8d7f2e76SPhilippe Mathieu-Daudé } 2663*8d7f2e76SPhilippe Mathieu-Daudé if (mr->flush_coalesced_mmio) { 2664*8d7f2e76SPhilippe Mathieu-Daudé qemu_flush_coalesced_mmio_buffer(); 2665*8d7f2e76SPhilippe Mathieu-Daudé } 2666*8d7f2e76SPhilippe Mathieu-Daudé 2667*8d7f2e76SPhilippe Mathieu-Daudé return release_lock; 2668*8d7f2e76SPhilippe Mathieu-Daudé } 2669*8d7f2e76SPhilippe Mathieu-Daudé 2670*8d7f2e76SPhilippe Mathieu-Daudé /** 2671*8d7f2e76SPhilippe Mathieu-Daudé * flatview_access_allowed 2672*8d7f2e76SPhilippe Mathieu-Daudé * @mr: #MemoryRegion to be accessed 2673*8d7f2e76SPhilippe Mathieu-Daudé * @attrs: memory transaction attributes 2674*8d7f2e76SPhilippe Mathieu-Daudé * @addr: address within that memory region 2675*8d7f2e76SPhilippe Mathieu-Daudé * @len: the number of bytes to access 2676*8d7f2e76SPhilippe Mathieu-Daudé * 2677*8d7f2e76SPhilippe Mathieu-Daudé * Check if a memory transaction is allowed. 2678*8d7f2e76SPhilippe Mathieu-Daudé * 2679*8d7f2e76SPhilippe Mathieu-Daudé * Returns: true if transaction is allowed, false if denied. 2680*8d7f2e76SPhilippe Mathieu-Daudé */ 2681*8d7f2e76SPhilippe Mathieu-Daudé static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs, 2682*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, hwaddr len) 2683*8d7f2e76SPhilippe Mathieu-Daudé { 2684*8d7f2e76SPhilippe Mathieu-Daudé if (likely(!attrs.memory)) { 2685*8d7f2e76SPhilippe Mathieu-Daudé return true; 2686*8d7f2e76SPhilippe Mathieu-Daudé } 2687*8d7f2e76SPhilippe Mathieu-Daudé if (memory_region_is_ram(mr)) { 2688*8d7f2e76SPhilippe Mathieu-Daudé return true; 2689*8d7f2e76SPhilippe Mathieu-Daudé } 2690*8d7f2e76SPhilippe Mathieu-Daudé qemu_log_mask(LOG_GUEST_ERROR, 2691*8d7f2e76SPhilippe Mathieu-Daudé "Invalid access to non-RAM device at " 2692*8d7f2e76SPhilippe Mathieu-Daudé "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", " 2693*8d7f2e76SPhilippe Mathieu-Daudé "region '%s'\n", addr, len, memory_region_name(mr)); 2694*8d7f2e76SPhilippe Mathieu-Daudé return false; 2695*8d7f2e76SPhilippe Mathieu-Daudé } 2696*8d7f2e76SPhilippe Mathieu-Daudé 2697*8d7f2e76SPhilippe Mathieu-Daudé /* Called within RCU critical section. */ 2698*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr, 2699*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, 2700*8d7f2e76SPhilippe Mathieu-Daudé const void *ptr, 2701*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, hwaddr addr1, 2702*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l, MemoryRegion *mr) 2703*8d7f2e76SPhilippe Mathieu-Daudé { 2704*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *ram_ptr; 2705*8d7f2e76SPhilippe Mathieu-Daudé uint64_t val; 2706*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult result = MEMTX_OK; 2707*8d7f2e76SPhilippe Mathieu-Daudé bool release_lock = false; 2708*8d7f2e76SPhilippe Mathieu-Daudé const uint8_t *buf = ptr; 2709*8d7f2e76SPhilippe Mathieu-Daudé 2710*8d7f2e76SPhilippe Mathieu-Daudé for (;;) { 2711*8d7f2e76SPhilippe Mathieu-Daudé if (!flatview_access_allowed(mr, attrs, addr1, l)) { 2712*8d7f2e76SPhilippe Mathieu-Daudé result |= MEMTX_ACCESS_ERROR; 2713*8d7f2e76SPhilippe Mathieu-Daudé /* Keep going. */ 2714*8d7f2e76SPhilippe Mathieu-Daudé } else if (!memory_access_is_direct(mr, true)) { 2715*8d7f2e76SPhilippe Mathieu-Daudé release_lock |= prepare_mmio_access(mr); 2716*8d7f2e76SPhilippe Mathieu-Daudé l = memory_access_size(mr, l, addr1); 2717*8d7f2e76SPhilippe Mathieu-Daudé /* XXX: could force current_cpu to NULL to avoid 2718*8d7f2e76SPhilippe Mathieu-Daudé potential bugs */ 2719*8d7f2e76SPhilippe Mathieu-Daudé val = ldn_he_p(buf, l); 2720*8d7f2e76SPhilippe Mathieu-Daudé result |= memory_region_dispatch_write(mr, addr1, val, 2721*8d7f2e76SPhilippe Mathieu-Daudé size_memop(l), attrs); 2722*8d7f2e76SPhilippe Mathieu-Daudé } else { 2723*8d7f2e76SPhilippe Mathieu-Daudé /* RAM case */ 2724*8d7f2e76SPhilippe Mathieu-Daudé ram_ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false); 2725*8d7f2e76SPhilippe Mathieu-Daudé memmove(ram_ptr, buf, l); 2726*8d7f2e76SPhilippe Mathieu-Daudé invalidate_and_set_dirty(mr, addr1, l); 2727*8d7f2e76SPhilippe Mathieu-Daudé } 2728*8d7f2e76SPhilippe Mathieu-Daudé 2729*8d7f2e76SPhilippe Mathieu-Daudé if (release_lock) { 2730*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_iothread(); 2731*8d7f2e76SPhilippe Mathieu-Daudé release_lock = false; 2732*8d7f2e76SPhilippe Mathieu-Daudé } 2733*8d7f2e76SPhilippe Mathieu-Daudé 2734*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 2735*8d7f2e76SPhilippe Mathieu-Daudé buf += l; 2736*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 2737*8d7f2e76SPhilippe Mathieu-Daudé 2738*8d7f2e76SPhilippe Mathieu-Daudé if (!len) { 2739*8d7f2e76SPhilippe Mathieu-Daudé break; 2740*8d7f2e76SPhilippe Mathieu-Daudé } 2741*8d7f2e76SPhilippe Mathieu-Daudé 2742*8d7f2e76SPhilippe Mathieu-Daudé l = len; 2743*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &addr1, &l, true, attrs); 2744*8d7f2e76SPhilippe Mathieu-Daudé } 2745*8d7f2e76SPhilippe Mathieu-Daudé 2746*8d7f2e76SPhilippe Mathieu-Daudé return result; 2747*8d7f2e76SPhilippe Mathieu-Daudé } 2748*8d7f2e76SPhilippe Mathieu-Daudé 2749*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section. */ 2750*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs, 2751*8d7f2e76SPhilippe Mathieu-Daudé const void *buf, hwaddr len) 2752*8d7f2e76SPhilippe Mathieu-Daudé { 2753*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l; 2754*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr1; 2755*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 2756*8d7f2e76SPhilippe Mathieu-Daudé 2757*8d7f2e76SPhilippe Mathieu-Daudé l = len; 2758*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &addr1, &l, true, attrs); 2759*8d7f2e76SPhilippe Mathieu-Daudé if (!flatview_access_allowed(mr, attrs, addr, len)) { 2760*8d7f2e76SPhilippe Mathieu-Daudé return MEMTX_ACCESS_ERROR; 2761*8d7f2e76SPhilippe Mathieu-Daudé } 2762*8d7f2e76SPhilippe Mathieu-Daudé return flatview_write_continue(fv, addr, attrs, buf, len, 2763*8d7f2e76SPhilippe Mathieu-Daudé addr1, l, mr); 2764*8d7f2e76SPhilippe Mathieu-Daudé } 2765*8d7f2e76SPhilippe Mathieu-Daudé 2766*8d7f2e76SPhilippe Mathieu-Daudé /* Called within RCU critical section. */ 2767*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, 2768*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, void *ptr, 2769*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, hwaddr addr1, hwaddr l, 2770*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr) 2771*8d7f2e76SPhilippe Mathieu-Daudé { 2772*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *ram_ptr; 2773*8d7f2e76SPhilippe Mathieu-Daudé uint64_t val; 2774*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult result = MEMTX_OK; 2775*8d7f2e76SPhilippe Mathieu-Daudé bool release_lock = false; 2776*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *buf = ptr; 2777*8d7f2e76SPhilippe Mathieu-Daudé 2778*8d7f2e76SPhilippe Mathieu-Daudé fuzz_dma_read_cb(addr, len, mr); 2779*8d7f2e76SPhilippe Mathieu-Daudé for (;;) { 2780*8d7f2e76SPhilippe Mathieu-Daudé if (!flatview_access_allowed(mr, attrs, addr1, l)) { 2781*8d7f2e76SPhilippe Mathieu-Daudé result |= MEMTX_ACCESS_ERROR; 2782*8d7f2e76SPhilippe Mathieu-Daudé /* Keep going. */ 2783*8d7f2e76SPhilippe Mathieu-Daudé } else if (!memory_access_is_direct(mr, false)) { 2784*8d7f2e76SPhilippe Mathieu-Daudé /* I/O case */ 2785*8d7f2e76SPhilippe Mathieu-Daudé release_lock |= prepare_mmio_access(mr); 2786*8d7f2e76SPhilippe Mathieu-Daudé l = memory_access_size(mr, l, addr1); 2787*8d7f2e76SPhilippe Mathieu-Daudé result |= memory_region_dispatch_read(mr, addr1, &val, 2788*8d7f2e76SPhilippe Mathieu-Daudé size_memop(l), attrs); 2789*8d7f2e76SPhilippe Mathieu-Daudé stn_he_p(buf, l, val); 2790*8d7f2e76SPhilippe Mathieu-Daudé } else { 2791*8d7f2e76SPhilippe Mathieu-Daudé /* RAM case */ 2792*8d7f2e76SPhilippe Mathieu-Daudé ram_ptr = qemu_ram_ptr_length(mr->ram_block, addr1, &l, false); 2793*8d7f2e76SPhilippe Mathieu-Daudé memcpy(buf, ram_ptr, l); 2794*8d7f2e76SPhilippe Mathieu-Daudé } 2795*8d7f2e76SPhilippe Mathieu-Daudé 2796*8d7f2e76SPhilippe Mathieu-Daudé if (release_lock) { 2797*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock_iothread(); 2798*8d7f2e76SPhilippe Mathieu-Daudé release_lock = false; 2799*8d7f2e76SPhilippe Mathieu-Daudé } 2800*8d7f2e76SPhilippe Mathieu-Daudé 2801*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 2802*8d7f2e76SPhilippe Mathieu-Daudé buf += l; 2803*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 2804*8d7f2e76SPhilippe Mathieu-Daudé 2805*8d7f2e76SPhilippe Mathieu-Daudé if (!len) { 2806*8d7f2e76SPhilippe Mathieu-Daudé break; 2807*8d7f2e76SPhilippe Mathieu-Daudé } 2808*8d7f2e76SPhilippe Mathieu-Daudé 2809*8d7f2e76SPhilippe Mathieu-Daudé l = len; 2810*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &addr1, &l, false, attrs); 2811*8d7f2e76SPhilippe Mathieu-Daudé } 2812*8d7f2e76SPhilippe Mathieu-Daudé 2813*8d7f2e76SPhilippe Mathieu-Daudé return result; 2814*8d7f2e76SPhilippe Mathieu-Daudé } 2815*8d7f2e76SPhilippe Mathieu-Daudé 2816*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section. */ 2817*8d7f2e76SPhilippe Mathieu-Daudé static MemTxResult flatview_read(FlatView *fv, hwaddr addr, 2818*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, void *buf, hwaddr len) 2819*8d7f2e76SPhilippe Mathieu-Daudé { 2820*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l; 2821*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr1; 2822*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 2823*8d7f2e76SPhilippe Mathieu-Daudé 2824*8d7f2e76SPhilippe Mathieu-Daudé l = len; 2825*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &addr1, &l, false, attrs); 2826*8d7f2e76SPhilippe Mathieu-Daudé if (!flatview_access_allowed(mr, attrs, addr, len)) { 2827*8d7f2e76SPhilippe Mathieu-Daudé return MEMTX_ACCESS_ERROR; 2828*8d7f2e76SPhilippe Mathieu-Daudé } 2829*8d7f2e76SPhilippe Mathieu-Daudé return flatview_read_continue(fv, addr, attrs, buf, len, 2830*8d7f2e76SPhilippe Mathieu-Daudé addr1, l, mr); 2831*8d7f2e76SPhilippe Mathieu-Daudé } 2832*8d7f2e76SPhilippe Mathieu-Daudé 2833*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr, 2834*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, void *buf, hwaddr len) 2835*8d7f2e76SPhilippe Mathieu-Daudé { 2836*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult result = MEMTX_OK; 2837*8d7f2e76SPhilippe Mathieu-Daudé FlatView *fv; 2838*8d7f2e76SPhilippe Mathieu-Daudé 2839*8d7f2e76SPhilippe Mathieu-Daudé if (len > 0) { 2840*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 2841*8d7f2e76SPhilippe Mathieu-Daudé fv = address_space_to_flatview(as); 2842*8d7f2e76SPhilippe Mathieu-Daudé result = flatview_read(fv, addr, attrs, buf, len); 2843*8d7f2e76SPhilippe Mathieu-Daudé } 2844*8d7f2e76SPhilippe Mathieu-Daudé 2845*8d7f2e76SPhilippe Mathieu-Daudé return result; 2846*8d7f2e76SPhilippe Mathieu-Daudé } 2847*8d7f2e76SPhilippe Mathieu-Daudé 2848*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult address_space_write(AddressSpace *as, hwaddr addr, 2849*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, 2850*8d7f2e76SPhilippe Mathieu-Daudé const void *buf, hwaddr len) 2851*8d7f2e76SPhilippe Mathieu-Daudé { 2852*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult result = MEMTX_OK; 2853*8d7f2e76SPhilippe Mathieu-Daudé FlatView *fv; 2854*8d7f2e76SPhilippe Mathieu-Daudé 2855*8d7f2e76SPhilippe Mathieu-Daudé if (len > 0) { 2856*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 2857*8d7f2e76SPhilippe Mathieu-Daudé fv = address_space_to_flatview(as); 2858*8d7f2e76SPhilippe Mathieu-Daudé result = flatview_write(fv, addr, attrs, buf, len); 2859*8d7f2e76SPhilippe Mathieu-Daudé } 2860*8d7f2e76SPhilippe Mathieu-Daudé 2861*8d7f2e76SPhilippe Mathieu-Daudé return result; 2862*8d7f2e76SPhilippe Mathieu-Daudé } 2863*8d7f2e76SPhilippe Mathieu-Daudé 2864*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, 2865*8d7f2e76SPhilippe Mathieu-Daudé void *buf, hwaddr len, bool is_write) 2866*8d7f2e76SPhilippe Mathieu-Daudé { 2867*8d7f2e76SPhilippe Mathieu-Daudé if (is_write) { 2868*8d7f2e76SPhilippe Mathieu-Daudé return address_space_write(as, addr, attrs, buf, len); 2869*8d7f2e76SPhilippe Mathieu-Daudé } else { 2870*8d7f2e76SPhilippe Mathieu-Daudé return address_space_read_full(as, addr, attrs, buf, len); 2871*8d7f2e76SPhilippe Mathieu-Daudé } 2872*8d7f2e76SPhilippe Mathieu-Daudé } 2873*8d7f2e76SPhilippe Mathieu-Daudé 2874*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult address_space_set(AddressSpace *as, hwaddr addr, 2875*8d7f2e76SPhilippe Mathieu-Daudé uint8_t c, hwaddr len, MemTxAttrs attrs) 2876*8d7f2e76SPhilippe Mathieu-Daudé { 2877*8d7f2e76SPhilippe Mathieu-Daudé #define FILLBUF_SIZE 512 2878*8d7f2e76SPhilippe Mathieu-Daudé uint8_t fillbuf[FILLBUF_SIZE]; 2879*8d7f2e76SPhilippe Mathieu-Daudé int l; 2880*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult error = MEMTX_OK; 2881*8d7f2e76SPhilippe Mathieu-Daudé 2882*8d7f2e76SPhilippe Mathieu-Daudé memset(fillbuf, c, FILLBUF_SIZE); 2883*8d7f2e76SPhilippe Mathieu-Daudé while (len > 0) { 2884*8d7f2e76SPhilippe Mathieu-Daudé l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE; 2885*8d7f2e76SPhilippe Mathieu-Daudé error |= address_space_write(as, addr, attrs, fillbuf, l); 2886*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 2887*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 2888*8d7f2e76SPhilippe Mathieu-Daudé } 2889*8d7f2e76SPhilippe Mathieu-Daudé 2890*8d7f2e76SPhilippe Mathieu-Daudé return error; 2891*8d7f2e76SPhilippe Mathieu-Daudé } 2892*8d7f2e76SPhilippe Mathieu-Daudé 2893*8d7f2e76SPhilippe Mathieu-Daudé void cpu_physical_memory_rw(hwaddr addr, void *buf, 2894*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, bool is_write) 2895*8d7f2e76SPhilippe Mathieu-Daudé { 2896*8d7f2e76SPhilippe Mathieu-Daudé address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED, 2897*8d7f2e76SPhilippe Mathieu-Daudé buf, len, is_write); 2898*8d7f2e76SPhilippe Mathieu-Daudé } 2899*8d7f2e76SPhilippe Mathieu-Daudé 2900*8d7f2e76SPhilippe Mathieu-Daudé enum write_rom_type { 2901*8d7f2e76SPhilippe Mathieu-Daudé WRITE_DATA, 2902*8d7f2e76SPhilippe Mathieu-Daudé FLUSH_CACHE, 2903*8d7f2e76SPhilippe Mathieu-Daudé }; 2904*8d7f2e76SPhilippe Mathieu-Daudé 2905*8d7f2e76SPhilippe Mathieu-Daudé static inline MemTxResult address_space_write_rom_internal(AddressSpace *as, 2906*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 2907*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, 2908*8d7f2e76SPhilippe Mathieu-Daudé const void *ptr, 2909*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, 2910*8d7f2e76SPhilippe Mathieu-Daudé enum write_rom_type type) 2911*8d7f2e76SPhilippe Mathieu-Daudé { 2912*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l; 2913*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *ram_ptr; 2914*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr1; 2915*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 2916*8d7f2e76SPhilippe Mathieu-Daudé const uint8_t *buf = ptr; 2917*8d7f2e76SPhilippe Mathieu-Daudé 2918*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 2919*8d7f2e76SPhilippe Mathieu-Daudé while (len > 0) { 2920*8d7f2e76SPhilippe Mathieu-Daudé l = len; 2921*8d7f2e76SPhilippe Mathieu-Daudé mr = address_space_translate(as, addr, &addr1, &l, true, attrs); 2922*8d7f2e76SPhilippe Mathieu-Daudé 2923*8d7f2e76SPhilippe Mathieu-Daudé if (!(memory_region_is_ram(mr) || 2924*8d7f2e76SPhilippe Mathieu-Daudé memory_region_is_romd(mr))) { 2925*8d7f2e76SPhilippe Mathieu-Daudé l = memory_access_size(mr, l, addr1); 2926*8d7f2e76SPhilippe Mathieu-Daudé } else { 2927*8d7f2e76SPhilippe Mathieu-Daudé /* ROM/RAM case */ 2928*8d7f2e76SPhilippe Mathieu-Daudé ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); 2929*8d7f2e76SPhilippe Mathieu-Daudé switch (type) { 2930*8d7f2e76SPhilippe Mathieu-Daudé case WRITE_DATA: 2931*8d7f2e76SPhilippe Mathieu-Daudé memcpy(ram_ptr, buf, l); 2932*8d7f2e76SPhilippe Mathieu-Daudé invalidate_and_set_dirty(mr, addr1, l); 2933*8d7f2e76SPhilippe Mathieu-Daudé break; 2934*8d7f2e76SPhilippe Mathieu-Daudé case FLUSH_CACHE: 2935*8d7f2e76SPhilippe Mathieu-Daudé flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l); 2936*8d7f2e76SPhilippe Mathieu-Daudé break; 2937*8d7f2e76SPhilippe Mathieu-Daudé } 2938*8d7f2e76SPhilippe Mathieu-Daudé } 2939*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 2940*8d7f2e76SPhilippe Mathieu-Daudé buf += l; 2941*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 2942*8d7f2e76SPhilippe Mathieu-Daudé } 2943*8d7f2e76SPhilippe Mathieu-Daudé return MEMTX_OK; 2944*8d7f2e76SPhilippe Mathieu-Daudé } 2945*8d7f2e76SPhilippe Mathieu-Daudé 2946*8d7f2e76SPhilippe Mathieu-Daudé /* used for ROM loading : can write in RAM and ROM */ 2947*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr, 2948*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs, 2949*8d7f2e76SPhilippe Mathieu-Daudé const void *buf, hwaddr len) 2950*8d7f2e76SPhilippe Mathieu-Daudé { 2951*8d7f2e76SPhilippe Mathieu-Daudé return address_space_write_rom_internal(as, addr, attrs, 2952*8d7f2e76SPhilippe Mathieu-Daudé buf, len, WRITE_DATA); 2953*8d7f2e76SPhilippe Mathieu-Daudé } 2954*8d7f2e76SPhilippe Mathieu-Daudé 2955*8d7f2e76SPhilippe Mathieu-Daudé void cpu_flush_icache_range(hwaddr start, hwaddr len) 2956*8d7f2e76SPhilippe Mathieu-Daudé { 2957*8d7f2e76SPhilippe Mathieu-Daudé /* 2958*8d7f2e76SPhilippe Mathieu-Daudé * This function should do the same thing as an icache flush that was 2959*8d7f2e76SPhilippe Mathieu-Daudé * triggered from within the guest. For TCG we are always cache coherent, 2960*8d7f2e76SPhilippe Mathieu-Daudé * so there is no need to flush anything. For KVM / Xen we need to flush 2961*8d7f2e76SPhilippe Mathieu-Daudé * the host's instruction cache at least. 2962*8d7f2e76SPhilippe Mathieu-Daudé */ 2963*8d7f2e76SPhilippe Mathieu-Daudé if (tcg_enabled()) { 2964*8d7f2e76SPhilippe Mathieu-Daudé return; 2965*8d7f2e76SPhilippe Mathieu-Daudé } 2966*8d7f2e76SPhilippe Mathieu-Daudé 2967*8d7f2e76SPhilippe Mathieu-Daudé address_space_write_rom_internal(&address_space_memory, 2968*8d7f2e76SPhilippe Mathieu-Daudé start, MEMTXATTRS_UNSPECIFIED, 2969*8d7f2e76SPhilippe Mathieu-Daudé NULL, len, FLUSH_CACHE); 2970*8d7f2e76SPhilippe Mathieu-Daudé } 2971*8d7f2e76SPhilippe Mathieu-Daudé 2972*8d7f2e76SPhilippe Mathieu-Daudé typedef struct { 2973*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 2974*8d7f2e76SPhilippe Mathieu-Daudé void *buffer; 2975*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr; 2976*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len; 2977*8d7f2e76SPhilippe Mathieu-Daudé bool in_use; 2978*8d7f2e76SPhilippe Mathieu-Daudé } BounceBuffer; 2979*8d7f2e76SPhilippe Mathieu-Daudé 2980*8d7f2e76SPhilippe Mathieu-Daudé static BounceBuffer bounce; 2981*8d7f2e76SPhilippe Mathieu-Daudé 2982*8d7f2e76SPhilippe Mathieu-Daudé typedef struct MapClient { 2983*8d7f2e76SPhilippe Mathieu-Daudé QEMUBH *bh; 2984*8d7f2e76SPhilippe Mathieu-Daudé QLIST_ENTRY(MapClient) link; 2985*8d7f2e76SPhilippe Mathieu-Daudé } MapClient; 2986*8d7f2e76SPhilippe Mathieu-Daudé 2987*8d7f2e76SPhilippe Mathieu-Daudé QemuMutex map_client_list_lock; 2988*8d7f2e76SPhilippe Mathieu-Daudé static QLIST_HEAD(, MapClient) map_client_list 2989*8d7f2e76SPhilippe Mathieu-Daudé = QLIST_HEAD_INITIALIZER(map_client_list); 2990*8d7f2e76SPhilippe Mathieu-Daudé 2991*8d7f2e76SPhilippe Mathieu-Daudé static void cpu_unregister_map_client_do(MapClient *client) 2992*8d7f2e76SPhilippe Mathieu-Daudé { 2993*8d7f2e76SPhilippe Mathieu-Daudé QLIST_REMOVE(client, link); 2994*8d7f2e76SPhilippe Mathieu-Daudé g_free(client); 2995*8d7f2e76SPhilippe Mathieu-Daudé } 2996*8d7f2e76SPhilippe Mathieu-Daudé 2997*8d7f2e76SPhilippe Mathieu-Daudé static void cpu_notify_map_clients_locked(void) 2998*8d7f2e76SPhilippe Mathieu-Daudé { 2999*8d7f2e76SPhilippe Mathieu-Daudé MapClient *client; 3000*8d7f2e76SPhilippe Mathieu-Daudé 3001*8d7f2e76SPhilippe Mathieu-Daudé while (!QLIST_EMPTY(&map_client_list)) { 3002*8d7f2e76SPhilippe Mathieu-Daudé client = QLIST_FIRST(&map_client_list); 3003*8d7f2e76SPhilippe Mathieu-Daudé qemu_bh_schedule(client->bh); 3004*8d7f2e76SPhilippe Mathieu-Daudé cpu_unregister_map_client_do(client); 3005*8d7f2e76SPhilippe Mathieu-Daudé } 3006*8d7f2e76SPhilippe Mathieu-Daudé } 3007*8d7f2e76SPhilippe Mathieu-Daudé 3008*8d7f2e76SPhilippe Mathieu-Daudé void cpu_register_map_client(QEMUBH *bh) 3009*8d7f2e76SPhilippe Mathieu-Daudé { 3010*8d7f2e76SPhilippe Mathieu-Daudé MapClient *client = g_malloc(sizeof(*client)); 3011*8d7f2e76SPhilippe Mathieu-Daudé 3012*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock(&map_client_list_lock); 3013*8d7f2e76SPhilippe Mathieu-Daudé client->bh = bh; 3014*8d7f2e76SPhilippe Mathieu-Daudé QLIST_INSERT_HEAD(&map_client_list, client, link); 3015*8d7f2e76SPhilippe Mathieu-Daudé /* Write map_client_list before reading in_use. */ 3016*8d7f2e76SPhilippe Mathieu-Daudé smp_mb(); 3017*8d7f2e76SPhilippe Mathieu-Daudé if (!qatomic_read(&bounce.in_use)) { 3018*8d7f2e76SPhilippe Mathieu-Daudé cpu_notify_map_clients_locked(); 3019*8d7f2e76SPhilippe Mathieu-Daudé } 3020*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock(&map_client_list_lock); 3021*8d7f2e76SPhilippe Mathieu-Daudé } 3022*8d7f2e76SPhilippe Mathieu-Daudé 3023*8d7f2e76SPhilippe Mathieu-Daudé void cpu_exec_init_all(void) 3024*8d7f2e76SPhilippe Mathieu-Daudé { 3025*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_init(&ram_list.mutex); 3026*8d7f2e76SPhilippe Mathieu-Daudé /* The data structures we set up here depend on knowing the page size, 3027*8d7f2e76SPhilippe Mathieu-Daudé * so no more changes can be made after this point. 3028*8d7f2e76SPhilippe Mathieu-Daudé * In an ideal world, nothing we did before we had finished the 3029*8d7f2e76SPhilippe Mathieu-Daudé * machine setup would care about the target page size, and we could 3030*8d7f2e76SPhilippe Mathieu-Daudé * do this much later, rather than requiring board models to state 3031*8d7f2e76SPhilippe Mathieu-Daudé * up front what their requirements are. 3032*8d7f2e76SPhilippe Mathieu-Daudé */ 3033*8d7f2e76SPhilippe Mathieu-Daudé finalize_target_page_bits(); 3034*8d7f2e76SPhilippe Mathieu-Daudé io_mem_init(); 3035*8d7f2e76SPhilippe Mathieu-Daudé memory_map_init(); 3036*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_init(&map_client_list_lock); 3037*8d7f2e76SPhilippe Mathieu-Daudé } 3038*8d7f2e76SPhilippe Mathieu-Daudé 3039*8d7f2e76SPhilippe Mathieu-Daudé void cpu_unregister_map_client(QEMUBH *bh) 3040*8d7f2e76SPhilippe Mathieu-Daudé { 3041*8d7f2e76SPhilippe Mathieu-Daudé MapClient *client; 3042*8d7f2e76SPhilippe Mathieu-Daudé 3043*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock(&map_client_list_lock); 3044*8d7f2e76SPhilippe Mathieu-Daudé QLIST_FOREACH(client, &map_client_list, link) { 3045*8d7f2e76SPhilippe Mathieu-Daudé if (client->bh == bh) { 3046*8d7f2e76SPhilippe Mathieu-Daudé cpu_unregister_map_client_do(client); 3047*8d7f2e76SPhilippe Mathieu-Daudé break; 3048*8d7f2e76SPhilippe Mathieu-Daudé } 3049*8d7f2e76SPhilippe Mathieu-Daudé } 3050*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock(&map_client_list_lock); 3051*8d7f2e76SPhilippe Mathieu-Daudé } 3052*8d7f2e76SPhilippe Mathieu-Daudé 3053*8d7f2e76SPhilippe Mathieu-Daudé static void cpu_notify_map_clients(void) 3054*8d7f2e76SPhilippe Mathieu-Daudé { 3055*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock(&map_client_list_lock); 3056*8d7f2e76SPhilippe Mathieu-Daudé cpu_notify_map_clients_locked(); 3057*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock(&map_client_list_lock); 3058*8d7f2e76SPhilippe Mathieu-Daudé } 3059*8d7f2e76SPhilippe Mathieu-Daudé 3060*8d7f2e76SPhilippe Mathieu-Daudé static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len, 3061*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, MemTxAttrs attrs) 3062*8d7f2e76SPhilippe Mathieu-Daudé { 3063*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3064*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l, xlat; 3065*8d7f2e76SPhilippe Mathieu-Daudé 3066*8d7f2e76SPhilippe Mathieu-Daudé while (len > 0) { 3067*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3068*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs); 3069*8d7f2e76SPhilippe Mathieu-Daudé if (!memory_access_is_direct(mr, is_write)) { 3070*8d7f2e76SPhilippe Mathieu-Daudé l = memory_access_size(mr, l, addr); 3071*8d7f2e76SPhilippe Mathieu-Daudé if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) { 3072*8d7f2e76SPhilippe Mathieu-Daudé return false; 3073*8d7f2e76SPhilippe Mathieu-Daudé } 3074*8d7f2e76SPhilippe Mathieu-Daudé } 3075*8d7f2e76SPhilippe Mathieu-Daudé 3076*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 3077*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 3078*8d7f2e76SPhilippe Mathieu-Daudé } 3079*8d7f2e76SPhilippe Mathieu-Daudé return true; 3080*8d7f2e76SPhilippe Mathieu-Daudé } 3081*8d7f2e76SPhilippe Mathieu-Daudé 3082*8d7f2e76SPhilippe Mathieu-Daudé bool address_space_access_valid(AddressSpace *as, hwaddr addr, 3083*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, bool is_write, 3084*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 3085*8d7f2e76SPhilippe Mathieu-Daudé { 3086*8d7f2e76SPhilippe Mathieu-Daudé FlatView *fv; 3087*8d7f2e76SPhilippe Mathieu-Daudé 3088*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 3089*8d7f2e76SPhilippe Mathieu-Daudé fv = address_space_to_flatview(as); 3090*8d7f2e76SPhilippe Mathieu-Daudé return flatview_access_valid(fv, addr, len, is_write, attrs); 3091*8d7f2e76SPhilippe Mathieu-Daudé } 3092*8d7f2e76SPhilippe Mathieu-Daudé 3093*8d7f2e76SPhilippe Mathieu-Daudé static hwaddr 3094*8d7f2e76SPhilippe Mathieu-Daudé flatview_extend_translation(FlatView *fv, hwaddr addr, 3095*8d7f2e76SPhilippe Mathieu-Daudé hwaddr target_len, 3096*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr, hwaddr base, hwaddr len, 3097*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, MemTxAttrs attrs) 3098*8d7f2e76SPhilippe Mathieu-Daudé { 3099*8d7f2e76SPhilippe Mathieu-Daudé hwaddr done = 0; 3100*8d7f2e76SPhilippe Mathieu-Daudé hwaddr xlat; 3101*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *this_mr; 3102*8d7f2e76SPhilippe Mathieu-Daudé 3103*8d7f2e76SPhilippe Mathieu-Daudé for (;;) { 3104*8d7f2e76SPhilippe Mathieu-Daudé target_len -= len; 3105*8d7f2e76SPhilippe Mathieu-Daudé addr += len; 3106*8d7f2e76SPhilippe Mathieu-Daudé done += len; 3107*8d7f2e76SPhilippe Mathieu-Daudé if (target_len == 0) { 3108*8d7f2e76SPhilippe Mathieu-Daudé return done; 3109*8d7f2e76SPhilippe Mathieu-Daudé } 3110*8d7f2e76SPhilippe Mathieu-Daudé 3111*8d7f2e76SPhilippe Mathieu-Daudé len = target_len; 3112*8d7f2e76SPhilippe Mathieu-Daudé this_mr = flatview_translate(fv, addr, &xlat, 3113*8d7f2e76SPhilippe Mathieu-Daudé &len, is_write, attrs); 3114*8d7f2e76SPhilippe Mathieu-Daudé if (this_mr != mr || xlat != base + done) { 3115*8d7f2e76SPhilippe Mathieu-Daudé return done; 3116*8d7f2e76SPhilippe Mathieu-Daudé } 3117*8d7f2e76SPhilippe Mathieu-Daudé } 3118*8d7f2e76SPhilippe Mathieu-Daudé } 3119*8d7f2e76SPhilippe Mathieu-Daudé 3120*8d7f2e76SPhilippe Mathieu-Daudé /* Map a physical memory region into a host virtual address. 3121*8d7f2e76SPhilippe Mathieu-Daudé * May map a subset of the requested range, given by and returned in *plen. 3122*8d7f2e76SPhilippe Mathieu-Daudé * May return NULL if resources needed to perform the mapping are exhausted. 3123*8d7f2e76SPhilippe Mathieu-Daudé * Use only for reads OR writes - not for read-modify-write operations. 3124*8d7f2e76SPhilippe Mathieu-Daudé * Use cpu_register_map_client() to know when retrying the map operation is 3125*8d7f2e76SPhilippe Mathieu-Daudé * likely to succeed. 3126*8d7f2e76SPhilippe Mathieu-Daudé */ 3127*8d7f2e76SPhilippe Mathieu-Daudé void *address_space_map(AddressSpace *as, 3128*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 3129*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen, 3130*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, 3131*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs) 3132*8d7f2e76SPhilippe Mathieu-Daudé { 3133*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len = *plen; 3134*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l, xlat; 3135*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3136*8d7f2e76SPhilippe Mathieu-Daudé FlatView *fv; 3137*8d7f2e76SPhilippe Mathieu-Daudé 3138*8d7f2e76SPhilippe Mathieu-Daudé if (len == 0) { 3139*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 3140*8d7f2e76SPhilippe Mathieu-Daudé } 3141*8d7f2e76SPhilippe Mathieu-Daudé 3142*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3143*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 3144*8d7f2e76SPhilippe Mathieu-Daudé fv = address_space_to_flatview(as); 3145*8d7f2e76SPhilippe Mathieu-Daudé mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs); 3146*8d7f2e76SPhilippe Mathieu-Daudé 3147*8d7f2e76SPhilippe Mathieu-Daudé if (!memory_access_is_direct(mr, is_write)) { 3148*8d7f2e76SPhilippe Mathieu-Daudé if (qatomic_xchg(&bounce.in_use, true)) { 3149*8d7f2e76SPhilippe Mathieu-Daudé *plen = 0; 3150*8d7f2e76SPhilippe Mathieu-Daudé return NULL; 3151*8d7f2e76SPhilippe Mathieu-Daudé } 3152*8d7f2e76SPhilippe Mathieu-Daudé /* Avoid unbounded allocations */ 3153*8d7f2e76SPhilippe Mathieu-Daudé l = MIN(l, TARGET_PAGE_SIZE); 3154*8d7f2e76SPhilippe Mathieu-Daudé bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l); 3155*8d7f2e76SPhilippe Mathieu-Daudé bounce.addr = addr; 3156*8d7f2e76SPhilippe Mathieu-Daudé bounce.len = l; 3157*8d7f2e76SPhilippe Mathieu-Daudé 3158*8d7f2e76SPhilippe Mathieu-Daudé memory_region_ref(mr); 3159*8d7f2e76SPhilippe Mathieu-Daudé bounce.mr = mr; 3160*8d7f2e76SPhilippe Mathieu-Daudé if (!is_write) { 3161*8d7f2e76SPhilippe Mathieu-Daudé flatview_read(fv, addr, MEMTXATTRS_UNSPECIFIED, 3162*8d7f2e76SPhilippe Mathieu-Daudé bounce.buffer, l); 3163*8d7f2e76SPhilippe Mathieu-Daudé } 3164*8d7f2e76SPhilippe Mathieu-Daudé 3165*8d7f2e76SPhilippe Mathieu-Daudé *plen = l; 3166*8d7f2e76SPhilippe Mathieu-Daudé return bounce.buffer; 3167*8d7f2e76SPhilippe Mathieu-Daudé } 3168*8d7f2e76SPhilippe Mathieu-Daudé 3169*8d7f2e76SPhilippe Mathieu-Daudé 3170*8d7f2e76SPhilippe Mathieu-Daudé memory_region_ref(mr); 3171*8d7f2e76SPhilippe Mathieu-Daudé *plen = flatview_extend_translation(fv, addr, len, mr, xlat, 3172*8d7f2e76SPhilippe Mathieu-Daudé l, is_write, attrs); 3173*8d7f2e76SPhilippe Mathieu-Daudé fuzz_dma_read_cb(addr, *plen, mr); 3174*8d7f2e76SPhilippe Mathieu-Daudé return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true); 3175*8d7f2e76SPhilippe Mathieu-Daudé } 3176*8d7f2e76SPhilippe Mathieu-Daudé 3177*8d7f2e76SPhilippe Mathieu-Daudé /* Unmaps a memory region previously mapped by address_space_map(). 3178*8d7f2e76SPhilippe Mathieu-Daudé * Will also mark the memory as dirty if is_write is true. access_len gives 3179*8d7f2e76SPhilippe Mathieu-Daudé * the amount of memory that was actually read or written by the caller. 3180*8d7f2e76SPhilippe Mathieu-Daudé */ 3181*8d7f2e76SPhilippe Mathieu-Daudé void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len, 3182*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, hwaddr access_len) 3183*8d7f2e76SPhilippe Mathieu-Daudé { 3184*8d7f2e76SPhilippe Mathieu-Daudé if (buffer != bounce.buffer) { 3185*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3186*8d7f2e76SPhilippe Mathieu-Daudé ram_addr_t addr1; 3187*8d7f2e76SPhilippe Mathieu-Daudé 3188*8d7f2e76SPhilippe Mathieu-Daudé mr = memory_region_from_host(buffer, &addr1); 3189*8d7f2e76SPhilippe Mathieu-Daudé assert(mr != NULL); 3190*8d7f2e76SPhilippe Mathieu-Daudé if (is_write) { 3191*8d7f2e76SPhilippe Mathieu-Daudé invalidate_and_set_dirty(mr, addr1, access_len); 3192*8d7f2e76SPhilippe Mathieu-Daudé } 3193*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled()) { 3194*8d7f2e76SPhilippe Mathieu-Daudé xen_invalidate_map_cache_entry(buffer); 3195*8d7f2e76SPhilippe Mathieu-Daudé } 3196*8d7f2e76SPhilippe Mathieu-Daudé memory_region_unref(mr); 3197*8d7f2e76SPhilippe Mathieu-Daudé return; 3198*8d7f2e76SPhilippe Mathieu-Daudé } 3199*8d7f2e76SPhilippe Mathieu-Daudé if (is_write) { 3200*8d7f2e76SPhilippe Mathieu-Daudé address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED, 3201*8d7f2e76SPhilippe Mathieu-Daudé bounce.buffer, access_len); 3202*8d7f2e76SPhilippe Mathieu-Daudé } 3203*8d7f2e76SPhilippe Mathieu-Daudé qemu_vfree(bounce.buffer); 3204*8d7f2e76SPhilippe Mathieu-Daudé bounce.buffer = NULL; 3205*8d7f2e76SPhilippe Mathieu-Daudé memory_region_unref(bounce.mr); 3206*8d7f2e76SPhilippe Mathieu-Daudé /* Clear in_use before reading map_client_list. */ 3207*8d7f2e76SPhilippe Mathieu-Daudé qatomic_set_mb(&bounce.in_use, false); 3208*8d7f2e76SPhilippe Mathieu-Daudé cpu_notify_map_clients(); 3209*8d7f2e76SPhilippe Mathieu-Daudé } 3210*8d7f2e76SPhilippe Mathieu-Daudé 3211*8d7f2e76SPhilippe Mathieu-Daudé void *cpu_physical_memory_map(hwaddr addr, 3212*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen, 3213*8d7f2e76SPhilippe Mathieu-Daudé bool is_write) 3214*8d7f2e76SPhilippe Mathieu-Daudé { 3215*8d7f2e76SPhilippe Mathieu-Daudé return address_space_map(&address_space_memory, addr, plen, is_write, 3216*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 3217*8d7f2e76SPhilippe Mathieu-Daudé } 3218*8d7f2e76SPhilippe Mathieu-Daudé 3219*8d7f2e76SPhilippe Mathieu-Daudé void cpu_physical_memory_unmap(void *buffer, hwaddr len, 3220*8d7f2e76SPhilippe Mathieu-Daudé bool is_write, hwaddr access_len) 3221*8d7f2e76SPhilippe Mathieu-Daudé { 3222*8d7f2e76SPhilippe Mathieu-Daudé return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len); 3223*8d7f2e76SPhilippe Mathieu-Daudé } 3224*8d7f2e76SPhilippe Mathieu-Daudé 3225*8d7f2e76SPhilippe Mathieu-Daudé #define ARG1_DECL AddressSpace *as 3226*8d7f2e76SPhilippe Mathieu-Daudé #define ARG1 as 3227*8d7f2e76SPhilippe Mathieu-Daudé #define SUFFIX 3228*8d7f2e76SPhilippe Mathieu-Daudé #define TRANSLATE(...) address_space_translate(as, __VA_ARGS__) 3229*8d7f2e76SPhilippe Mathieu-Daudé #define RCU_READ_LOCK(...) rcu_read_lock() 3230*8d7f2e76SPhilippe Mathieu-Daudé #define RCU_READ_UNLOCK(...) rcu_read_unlock() 3231*8d7f2e76SPhilippe Mathieu-Daudé #include "memory_ldst.c.inc" 3232*8d7f2e76SPhilippe Mathieu-Daudé 3233*8d7f2e76SPhilippe Mathieu-Daudé int64_t address_space_cache_init(MemoryRegionCache *cache, 3234*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *as, 3235*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 3236*8d7f2e76SPhilippe Mathieu-Daudé hwaddr len, 3237*8d7f2e76SPhilippe Mathieu-Daudé bool is_write) 3238*8d7f2e76SPhilippe Mathieu-Daudé { 3239*8d7f2e76SPhilippe Mathieu-Daudé AddressSpaceDispatch *d; 3240*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l; 3241*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3242*8d7f2e76SPhilippe Mathieu-Daudé Int128 diff; 3243*8d7f2e76SPhilippe Mathieu-Daudé 3244*8d7f2e76SPhilippe Mathieu-Daudé assert(len > 0); 3245*8d7f2e76SPhilippe Mathieu-Daudé 3246*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3247*8d7f2e76SPhilippe Mathieu-Daudé cache->fv = address_space_get_flatview(as); 3248*8d7f2e76SPhilippe Mathieu-Daudé d = flatview_to_dispatch(cache->fv); 3249*8d7f2e76SPhilippe Mathieu-Daudé cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true); 3250*8d7f2e76SPhilippe Mathieu-Daudé 3251*8d7f2e76SPhilippe Mathieu-Daudé /* 3252*8d7f2e76SPhilippe Mathieu-Daudé * cache->xlat is now relative to cache->mrs.mr, not to the section itself. 3253*8d7f2e76SPhilippe Mathieu-Daudé * Take that into account to compute how many bytes are there between 3254*8d7f2e76SPhilippe Mathieu-Daudé * cache->xlat and the end of the section. 3255*8d7f2e76SPhilippe Mathieu-Daudé */ 3256*8d7f2e76SPhilippe Mathieu-Daudé diff = int128_sub(cache->mrs.size, 3257*8d7f2e76SPhilippe Mathieu-Daudé int128_make64(cache->xlat - cache->mrs.offset_within_region)); 3258*8d7f2e76SPhilippe Mathieu-Daudé l = int128_get64(int128_min(diff, int128_make64(l))); 3259*8d7f2e76SPhilippe Mathieu-Daudé 3260*8d7f2e76SPhilippe Mathieu-Daudé mr = cache->mrs.mr; 3261*8d7f2e76SPhilippe Mathieu-Daudé memory_region_ref(mr); 3262*8d7f2e76SPhilippe Mathieu-Daudé if (memory_access_is_direct(mr, is_write)) { 3263*8d7f2e76SPhilippe Mathieu-Daudé /* We don't care about the memory attributes here as we're only 3264*8d7f2e76SPhilippe Mathieu-Daudé * doing this if we found actual RAM, which behaves the same 3265*8d7f2e76SPhilippe Mathieu-Daudé * regardless of attributes; so UNSPECIFIED is fine. 3266*8d7f2e76SPhilippe Mathieu-Daudé */ 3267*8d7f2e76SPhilippe Mathieu-Daudé l = flatview_extend_translation(cache->fv, addr, len, mr, 3268*8d7f2e76SPhilippe Mathieu-Daudé cache->xlat, l, is_write, 3269*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 3270*8d7f2e76SPhilippe Mathieu-Daudé cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true); 3271*8d7f2e76SPhilippe Mathieu-Daudé } else { 3272*8d7f2e76SPhilippe Mathieu-Daudé cache->ptr = NULL; 3273*8d7f2e76SPhilippe Mathieu-Daudé } 3274*8d7f2e76SPhilippe Mathieu-Daudé 3275*8d7f2e76SPhilippe Mathieu-Daudé cache->len = l; 3276*8d7f2e76SPhilippe Mathieu-Daudé cache->is_write = is_write; 3277*8d7f2e76SPhilippe Mathieu-Daudé return l; 3278*8d7f2e76SPhilippe Mathieu-Daudé } 3279*8d7f2e76SPhilippe Mathieu-Daudé 3280*8d7f2e76SPhilippe Mathieu-Daudé void address_space_cache_invalidate(MemoryRegionCache *cache, 3281*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr, 3282*8d7f2e76SPhilippe Mathieu-Daudé hwaddr access_len) 3283*8d7f2e76SPhilippe Mathieu-Daudé { 3284*8d7f2e76SPhilippe Mathieu-Daudé assert(cache->is_write); 3285*8d7f2e76SPhilippe Mathieu-Daudé if (likely(cache->ptr)) { 3286*8d7f2e76SPhilippe Mathieu-Daudé invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len); 3287*8d7f2e76SPhilippe Mathieu-Daudé } 3288*8d7f2e76SPhilippe Mathieu-Daudé } 3289*8d7f2e76SPhilippe Mathieu-Daudé 3290*8d7f2e76SPhilippe Mathieu-Daudé void address_space_cache_destroy(MemoryRegionCache *cache) 3291*8d7f2e76SPhilippe Mathieu-Daudé { 3292*8d7f2e76SPhilippe Mathieu-Daudé if (!cache->mrs.mr) { 3293*8d7f2e76SPhilippe Mathieu-Daudé return; 3294*8d7f2e76SPhilippe Mathieu-Daudé } 3295*8d7f2e76SPhilippe Mathieu-Daudé 3296*8d7f2e76SPhilippe Mathieu-Daudé if (xen_enabled()) { 3297*8d7f2e76SPhilippe Mathieu-Daudé xen_invalidate_map_cache_entry(cache->ptr); 3298*8d7f2e76SPhilippe Mathieu-Daudé } 3299*8d7f2e76SPhilippe Mathieu-Daudé memory_region_unref(cache->mrs.mr); 3300*8d7f2e76SPhilippe Mathieu-Daudé flatview_unref(cache->fv); 3301*8d7f2e76SPhilippe Mathieu-Daudé cache->mrs.mr = NULL; 3302*8d7f2e76SPhilippe Mathieu-Daudé cache->fv = NULL; 3303*8d7f2e76SPhilippe Mathieu-Daudé } 3304*8d7f2e76SPhilippe Mathieu-Daudé 3305*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section. This function has the same 3306*8d7f2e76SPhilippe Mathieu-Daudé * semantics as address_space_translate, but it only works on a 3307*8d7f2e76SPhilippe Mathieu-Daudé * predefined range of a MemoryRegion that was mapped with 3308*8d7f2e76SPhilippe Mathieu-Daudé * address_space_cache_init. 3309*8d7f2e76SPhilippe Mathieu-Daudé */ 3310*8d7f2e76SPhilippe Mathieu-Daudé static inline MemoryRegion *address_space_translate_cached( 3311*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat, 3312*8d7f2e76SPhilippe Mathieu-Daudé hwaddr *plen, bool is_write, MemTxAttrs attrs) 3313*8d7f2e76SPhilippe Mathieu-Daudé { 3314*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection section; 3315*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3316*8d7f2e76SPhilippe Mathieu-Daudé IOMMUMemoryRegion *iommu_mr; 3317*8d7f2e76SPhilippe Mathieu-Daudé AddressSpace *target_as; 3318*8d7f2e76SPhilippe Mathieu-Daudé 3319*8d7f2e76SPhilippe Mathieu-Daudé assert(!cache->ptr); 3320*8d7f2e76SPhilippe Mathieu-Daudé *xlat = addr + cache->xlat; 3321*8d7f2e76SPhilippe Mathieu-Daudé 3322*8d7f2e76SPhilippe Mathieu-Daudé mr = cache->mrs.mr; 3323*8d7f2e76SPhilippe Mathieu-Daudé iommu_mr = memory_region_get_iommu(mr); 3324*8d7f2e76SPhilippe Mathieu-Daudé if (!iommu_mr) { 3325*8d7f2e76SPhilippe Mathieu-Daudé /* MMIO region. */ 3326*8d7f2e76SPhilippe Mathieu-Daudé return mr; 3327*8d7f2e76SPhilippe Mathieu-Daudé } 3328*8d7f2e76SPhilippe Mathieu-Daudé 3329*8d7f2e76SPhilippe Mathieu-Daudé section = address_space_translate_iommu(iommu_mr, xlat, plen, 3330*8d7f2e76SPhilippe Mathieu-Daudé NULL, is_write, true, 3331*8d7f2e76SPhilippe Mathieu-Daudé &target_as, attrs); 3332*8d7f2e76SPhilippe Mathieu-Daudé return section.mr; 3333*8d7f2e76SPhilippe Mathieu-Daudé } 3334*8d7f2e76SPhilippe Mathieu-Daudé 3335*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section. address_space_read_cached uses this 3336*8d7f2e76SPhilippe Mathieu-Daudé * out of line function when the target is an MMIO or IOMMU region. 3337*8d7f2e76SPhilippe Mathieu-Daudé */ 3338*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult 3339*8d7f2e76SPhilippe Mathieu-Daudé address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr, 3340*8d7f2e76SPhilippe Mathieu-Daudé void *buf, hwaddr len) 3341*8d7f2e76SPhilippe Mathieu-Daudé { 3342*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr1, l; 3343*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3344*8d7f2e76SPhilippe Mathieu-Daudé 3345*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3346*8d7f2e76SPhilippe Mathieu-Daudé mr = address_space_translate_cached(cache, addr, &addr1, &l, false, 3347*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 3348*8d7f2e76SPhilippe Mathieu-Daudé return flatview_read_continue(cache->fv, 3349*8d7f2e76SPhilippe Mathieu-Daudé addr, MEMTXATTRS_UNSPECIFIED, buf, len, 3350*8d7f2e76SPhilippe Mathieu-Daudé addr1, l, mr); 3351*8d7f2e76SPhilippe Mathieu-Daudé } 3352*8d7f2e76SPhilippe Mathieu-Daudé 3353*8d7f2e76SPhilippe Mathieu-Daudé /* Called from RCU critical section. address_space_write_cached uses this 3354*8d7f2e76SPhilippe Mathieu-Daudé * out of line function when the target is an MMIO or IOMMU region. 3355*8d7f2e76SPhilippe Mathieu-Daudé */ 3356*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult 3357*8d7f2e76SPhilippe Mathieu-Daudé address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr, 3358*8d7f2e76SPhilippe Mathieu-Daudé const void *buf, hwaddr len) 3359*8d7f2e76SPhilippe Mathieu-Daudé { 3360*8d7f2e76SPhilippe Mathieu-Daudé hwaddr addr1, l; 3361*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion *mr; 3362*8d7f2e76SPhilippe Mathieu-Daudé 3363*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3364*8d7f2e76SPhilippe Mathieu-Daudé mr = address_space_translate_cached(cache, addr, &addr1, &l, true, 3365*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 3366*8d7f2e76SPhilippe Mathieu-Daudé return flatview_write_continue(cache->fv, 3367*8d7f2e76SPhilippe Mathieu-Daudé addr, MEMTXATTRS_UNSPECIFIED, buf, len, 3368*8d7f2e76SPhilippe Mathieu-Daudé addr1, l, mr); 3369*8d7f2e76SPhilippe Mathieu-Daudé } 3370*8d7f2e76SPhilippe Mathieu-Daudé 3371*8d7f2e76SPhilippe Mathieu-Daudé #define ARG1_DECL MemoryRegionCache *cache 3372*8d7f2e76SPhilippe Mathieu-Daudé #define ARG1 cache 3373*8d7f2e76SPhilippe Mathieu-Daudé #define SUFFIX _cached_slow 3374*8d7f2e76SPhilippe Mathieu-Daudé #define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__) 3375*8d7f2e76SPhilippe Mathieu-Daudé #define RCU_READ_LOCK() ((void)0) 3376*8d7f2e76SPhilippe Mathieu-Daudé #define RCU_READ_UNLOCK() ((void)0) 3377*8d7f2e76SPhilippe Mathieu-Daudé #include "memory_ldst.c.inc" 3378*8d7f2e76SPhilippe Mathieu-Daudé 3379*8d7f2e76SPhilippe Mathieu-Daudé /* virtual memory access for debug (includes writing to ROM) */ 3380*8d7f2e76SPhilippe Mathieu-Daudé int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, 3381*8d7f2e76SPhilippe Mathieu-Daudé void *ptr, size_t len, bool is_write) 3382*8d7f2e76SPhilippe Mathieu-Daudé { 3383*8d7f2e76SPhilippe Mathieu-Daudé hwaddr phys_addr; 3384*8d7f2e76SPhilippe Mathieu-Daudé vaddr l, page; 3385*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *buf = ptr; 3386*8d7f2e76SPhilippe Mathieu-Daudé 3387*8d7f2e76SPhilippe Mathieu-Daudé cpu_synchronize_state(cpu); 3388*8d7f2e76SPhilippe Mathieu-Daudé while (len > 0) { 3389*8d7f2e76SPhilippe Mathieu-Daudé int asidx; 3390*8d7f2e76SPhilippe Mathieu-Daudé MemTxAttrs attrs; 3391*8d7f2e76SPhilippe Mathieu-Daudé MemTxResult res; 3392*8d7f2e76SPhilippe Mathieu-Daudé 3393*8d7f2e76SPhilippe Mathieu-Daudé page = addr & TARGET_PAGE_MASK; 3394*8d7f2e76SPhilippe Mathieu-Daudé phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs); 3395*8d7f2e76SPhilippe Mathieu-Daudé asidx = cpu_asidx_from_attrs(cpu, attrs); 3396*8d7f2e76SPhilippe Mathieu-Daudé /* if no physical page mapped, return an error */ 3397*8d7f2e76SPhilippe Mathieu-Daudé if (phys_addr == -1) 3398*8d7f2e76SPhilippe Mathieu-Daudé return -1; 3399*8d7f2e76SPhilippe Mathieu-Daudé l = (page + TARGET_PAGE_SIZE) - addr; 3400*8d7f2e76SPhilippe Mathieu-Daudé if (l > len) 3401*8d7f2e76SPhilippe Mathieu-Daudé l = len; 3402*8d7f2e76SPhilippe Mathieu-Daudé phys_addr += (addr & ~TARGET_PAGE_MASK); 3403*8d7f2e76SPhilippe Mathieu-Daudé if (is_write) { 3404*8d7f2e76SPhilippe Mathieu-Daudé res = address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr, 3405*8d7f2e76SPhilippe Mathieu-Daudé attrs, buf, l); 3406*8d7f2e76SPhilippe Mathieu-Daudé } else { 3407*8d7f2e76SPhilippe Mathieu-Daudé res = address_space_read(cpu->cpu_ases[asidx].as, phys_addr, 3408*8d7f2e76SPhilippe Mathieu-Daudé attrs, buf, l); 3409*8d7f2e76SPhilippe Mathieu-Daudé } 3410*8d7f2e76SPhilippe Mathieu-Daudé if (res != MEMTX_OK) { 3411*8d7f2e76SPhilippe Mathieu-Daudé return -1; 3412*8d7f2e76SPhilippe Mathieu-Daudé } 3413*8d7f2e76SPhilippe Mathieu-Daudé len -= l; 3414*8d7f2e76SPhilippe Mathieu-Daudé buf += l; 3415*8d7f2e76SPhilippe Mathieu-Daudé addr += l; 3416*8d7f2e76SPhilippe Mathieu-Daudé } 3417*8d7f2e76SPhilippe Mathieu-Daudé return 0; 3418*8d7f2e76SPhilippe Mathieu-Daudé } 3419*8d7f2e76SPhilippe Mathieu-Daudé 3420*8d7f2e76SPhilippe Mathieu-Daudé /* 3421*8d7f2e76SPhilippe Mathieu-Daudé * Allows code that needs to deal with migration bitmaps etc to still be built 3422*8d7f2e76SPhilippe Mathieu-Daudé * target independent. 3423*8d7f2e76SPhilippe Mathieu-Daudé */ 3424*8d7f2e76SPhilippe Mathieu-Daudé size_t qemu_target_page_size(void) 3425*8d7f2e76SPhilippe Mathieu-Daudé { 3426*8d7f2e76SPhilippe Mathieu-Daudé return TARGET_PAGE_SIZE; 3427*8d7f2e76SPhilippe Mathieu-Daudé } 3428*8d7f2e76SPhilippe Mathieu-Daudé 3429*8d7f2e76SPhilippe Mathieu-Daudé int qemu_target_page_mask(void) 3430*8d7f2e76SPhilippe Mathieu-Daudé { 3431*8d7f2e76SPhilippe Mathieu-Daudé return TARGET_PAGE_MASK; 3432*8d7f2e76SPhilippe Mathieu-Daudé } 3433*8d7f2e76SPhilippe Mathieu-Daudé 3434*8d7f2e76SPhilippe Mathieu-Daudé int qemu_target_page_bits(void) 3435*8d7f2e76SPhilippe Mathieu-Daudé { 3436*8d7f2e76SPhilippe Mathieu-Daudé return TARGET_PAGE_BITS; 3437*8d7f2e76SPhilippe Mathieu-Daudé } 3438*8d7f2e76SPhilippe Mathieu-Daudé 3439*8d7f2e76SPhilippe Mathieu-Daudé int qemu_target_page_bits_min(void) 3440*8d7f2e76SPhilippe Mathieu-Daudé { 3441*8d7f2e76SPhilippe Mathieu-Daudé return TARGET_PAGE_BITS_MIN; 3442*8d7f2e76SPhilippe Mathieu-Daudé } 3443*8d7f2e76SPhilippe Mathieu-Daudé 3444*8d7f2e76SPhilippe Mathieu-Daudé /* Convert target pages to MiB (2**20). */ 3445*8d7f2e76SPhilippe Mathieu-Daudé size_t qemu_target_pages_to_MiB(size_t pages) 3446*8d7f2e76SPhilippe Mathieu-Daudé { 3447*8d7f2e76SPhilippe Mathieu-Daudé int page_bits = TARGET_PAGE_BITS; 3448*8d7f2e76SPhilippe Mathieu-Daudé 3449*8d7f2e76SPhilippe Mathieu-Daudé /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */ 3450*8d7f2e76SPhilippe Mathieu-Daudé g_assert(page_bits < 20); 3451*8d7f2e76SPhilippe Mathieu-Daudé 3452*8d7f2e76SPhilippe Mathieu-Daudé return pages >> (20 - page_bits); 3453*8d7f2e76SPhilippe Mathieu-Daudé } 3454*8d7f2e76SPhilippe Mathieu-Daudé 3455*8d7f2e76SPhilippe Mathieu-Daudé bool cpu_physical_memory_is_io(hwaddr phys_addr) 3456*8d7f2e76SPhilippe Mathieu-Daudé { 3457*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegion*mr; 3458*8d7f2e76SPhilippe Mathieu-Daudé hwaddr l = 1; 3459*8d7f2e76SPhilippe Mathieu-Daudé 3460*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 3461*8d7f2e76SPhilippe Mathieu-Daudé mr = address_space_translate(&address_space_memory, 3462*8d7f2e76SPhilippe Mathieu-Daudé phys_addr, &phys_addr, &l, false, 3463*8d7f2e76SPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 3464*8d7f2e76SPhilippe Mathieu-Daudé 3465*8d7f2e76SPhilippe Mathieu-Daudé return !(memory_region_is_ram(mr) || memory_region_is_romd(mr)); 3466*8d7f2e76SPhilippe Mathieu-Daudé } 3467*8d7f2e76SPhilippe Mathieu-Daudé 3468*8d7f2e76SPhilippe Mathieu-Daudé int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque) 3469*8d7f2e76SPhilippe Mathieu-Daudé { 3470*8d7f2e76SPhilippe Mathieu-Daudé RAMBlock *block; 3471*8d7f2e76SPhilippe Mathieu-Daudé int ret = 0; 3472*8d7f2e76SPhilippe Mathieu-Daudé 3473*8d7f2e76SPhilippe Mathieu-Daudé RCU_READ_LOCK_GUARD(); 3474*8d7f2e76SPhilippe Mathieu-Daudé RAMBLOCK_FOREACH(block) { 3475*8d7f2e76SPhilippe Mathieu-Daudé ret = func(block, opaque); 3476*8d7f2e76SPhilippe Mathieu-Daudé if (ret) { 3477*8d7f2e76SPhilippe Mathieu-Daudé break; 3478*8d7f2e76SPhilippe Mathieu-Daudé } 3479*8d7f2e76SPhilippe Mathieu-Daudé } 3480*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3481*8d7f2e76SPhilippe Mathieu-Daudé } 3482*8d7f2e76SPhilippe Mathieu-Daudé 3483*8d7f2e76SPhilippe Mathieu-Daudé /* 3484*8d7f2e76SPhilippe Mathieu-Daudé * Unmap pages of memory from start to start+length such that 3485*8d7f2e76SPhilippe Mathieu-Daudé * they a) read as 0, b) Trigger whatever fault mechanism 3486*8d7f2e76SPhilippe Mathieu-Daudé * the OS provides for postcopy. 3487*8d7f2e76SPhilippe Mathieu-Daudé * The pages must be unmapped by the end of the function. 3488*8d7f2e76SPhilippe Mathieu-Daudé * Returns: 0 on success, none-0 on failure 3489*8d7f2e76SPhilippe Mathieu-Daudé * 3490*8d7f2e76SPhilippe Mathieu-Daudé */ 3491*8d7f2e76SPhilippe Mathieu-Daudé int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) 3492*8d7f2e76SPhilippe Mathieu-Daudé { 3493*8d7f2e76SPhilippe Mathieu-Daudé int ret = -1; 3494*8d7f2e76SPhilippe Mathieu-Daudé 3495*8d7f2e76SPhilippe Mathieu-Daudé uint8_t *host_startaddr = rb->host + start; 3496*8d7f2e76SPhilippe Mathieu-Daudé 3497*8d7f2e76SPhilippe Mathieu-Daudé if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) { 3498*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Unaligned start address: %p", 3499*8d7f2e76SPhilippe Mathieu-Daudé host_startaddr); 3500*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3501*8d7f2e76SPhilippe Mathieu-Daudé } 3502*8d7f2e76SPhilippe Mathieu-Daudé 3503*8d7f2e76SPhilippe Mathieu-Daudé if ((start + length) <= rb->max_length) { 3504*8d7f2e76SPhilippe Mathieu-Daudé bool need_madvise, need_fallocate; 3505*8d7f2e76SPhilippe Mathieu-Daudé if (!QEMU_IS_ALIGNED(length, rb->page_size)) { 3506*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Unaligned length: %zx", 3507*8d7f2e76SPhilippe Mathieu-Daudé length); 3508*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3509*8d7f2e76SPhilippe Mathieu-Daudé } 3510*8d7f2e76SPhilippe Mathieu-Daudé 3511*8d7f2e76SPhilippe Mathieu-Daudé errno = ENOTSUP; /* If we are missing MADVISE etc */ 3512*8d7f2e76SPhilippe Mathieu-Daudé 3513*8d7f2e76SPhilippe Mathieu-Daudé /* The logic here is messy; 3514*8d7f2e76SPhilippe Mathieu-Daudé * madvise DONTNEED fails for hugepages 3515*8d7f2e76SPhilippe Mathieu-Daudé * fallocate works on hugepages and shmem 3516*8d7f2e76SPhilippe Mathieu-Daudé * shared anonymous memory requires madvise REMOVE 3517*8d7f2e76SPhilippe Mathieu-Daudé */ 3518*8d7f2e76SPhilippe Mathieu-Daudé need_madvise = (rb->page_size == qemu_host_page_size); 3519*8d7f2e76SPhilippe Mathieu-Daudé need_fallocate = rb->fd != -1; 3520*8d7f2e76SPhilippe Mathieu-Daudé if (need_fallocate) { 3521*8d7f2e76SPhilippe Mathieu-Daudé /* For a file, this causes the area of the file to be zero'd 3522*8d7f2e76SPhilippe Mathieu-Daudé * if read, and for hugetlbfs also causes it to be unmapped 3523*8d7f2e76SPhilippe Mathieu-Daudé * so a userfault will trigger. 3524*8d7f2e76SPhilippe Mathieu-Daudé */ 3525*8d7f2e76SPhilippe Mathieu-Daudé #ifdef CONFIG_FALLOCATE_PUNCH_HOLE 3526*8d7f2e76SPhilippe Mathieu-Daudé /* 3527*8d7f2e76SPhilippe Mathieu-Daudé * fallocate() will fail with readonly files. Let's print a 3528*8d7f2e76SPhilippe Mathieu-Daudé * proper error message. 3529*8d7f2e76SPhilippe Mathieu-Daudé */ 3530*8d7f2e76SPhilippe Mathieu-Daudé if (rb->flags & RAM_READONLY_FD) { 3531*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Discarding RAM" 3532*8d7f2e76SPhilippe Mathieu-Daudé " with readonly files is not supported"); 3533*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3534*8d7f2e76SPhilippe Mathieu-Daudé 3535*8d7f2e76SPhilippe Mathieu-Daudé } 3536*8d7f2e76SPhilippe Mathieu-Daudé /* 3537*8d7f2e76SPhilippe Mathieu-Daudé * We'll discard data from the actual file, even though we only 3538*8d7f2e76SPhilippe Mathieu-Daudé * have a MAP_PRIVATE mapping, possibly messing with other 3539*8d7f2e76SPhilippe Mathieu-Daudé * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to 3540*8d7f2e76SPhilippe Mathieu-Daudé * change that behavior whithout violating the promised 3541*8d7f2e76SPhilippe Mathieu-Daudé * semantics of ram_block_discard_range(). 3542*8d7f2e76SPhilippe Mathieu-Daudé * 3543*8d7f2e76SPhilippe Mathieu-Daudé * Only warn, because it works as long as nobody else uses that 3544*8d7f2e76SPhilippe Mathieu-Daudé * file. 3545*8d7f2e76SPhilippe Mathieu-Daudé */ 3546*8d7f2e76SPhilippe Mathieu-Daudé if (!qemu_ram_is_shared(rb)) { 3547*8d7f2e76SPhilippe Mathieu-Daudé warn_report_once("ram_block_discard_range: Discarding RAM" 3548*8d7f2e76SPhilippe Mathieu-Daudé " in private file mappings is possibly" 3549*8d7f2e76SPhilippe Mathieu-Daudé " dangerous, because it will modify the" 3550*8d7f2e76SPhilippe Mathieu-Daudé " underlying file and will affect other" 3551*8d7f2e76SPhilippe Mathieu-Daudé " users of the file"); 3552*8d7f2e76SPhilippe Mathieu-Daudé } 3553*8d7f2e76SPhilippe Mathieu-Daudé 3554*8d7f2e76SPhilippe Mathieu-Daudé ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 3555*8d7f2e76SPhilippe Mathieu-Daudé start, length); 3556*8d7f2e76SPhilippe Mathieu-Daudé if (ret) { 3557*8d7f2e76SPhilippe Mathieu-Daudé ret = -errno; 3558*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Failed to fallocate " 3559*8d7f2e76SPhilippe Mathieu-Daudé "%s:%" PRIx64 " +%zx (%d)", 3560*8d7f2e76SPhilippe Mathieu-Daudé rb->idstr, start, length, ret); 3561*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3562*8d7f2e76SPhilippe Mathieu-Daudé } 3563*8d7f2e76SPhilippe Mathieu-Daudé #else 3564*8d7f2e76SPhilippe Mathieu-Daudé ret = -ENOSYS; 3565*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: fallocate not available/file" 3566*8d7f2e76SPhilippe Mathieu-Daudé "%s:%" PRIx64 " +%zx (%d)", 3567*8d7f2e76SPhilippe Mathieu-Daudé rb->idstr, start, length, ret); 3568*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3569*8d7f2e76SPhilippe Mathieu-Daudé #endif 3570*8d7f2e76SPhilippe Mathieu-Daudé } 3571*8d7f2e76SPhilippe Mathieu-Daudé if (need_madvise) { 3572*8d7f2e76SPhilippe Mathieu-Daudé /* For normal RAM this causes it to be unmapped, 3573*8d7f2e76SPhilippe Mathieu-Daudé * for shared memory it causes the local mapping to disappear 3574*8d7f2e76SPhilippe Mathieu-Daudé * and to fall back on the file contents (which we just 3575*8d7f2e76SPhilippe Mathieu-Daudé * fallocate'd away). 3576*8d7f2e76SPhilippe Mathieu-Daudé */ 3577*8d7f2e76SPhilippe Mathieu-Daudé #if defined(CONFIG_MADVISE) 3578*8d7f2e76SPhilippe Mathieu-Daudé if (qemu_ram_is_shared(rb) && rb->fd < 0) { 3579*8d7f2e76SPhilippe Mathieu-Daudé ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE); 3580*8d7f2e76SPhilippe Mathieu-Daudé } else { 3581*8d7f2e76SPhilippe Mathieu-Daudé ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED); 3582*8d7f2e76SPhilippe Mathieu-Daudé } 3583*8d7f2e76SPhilippe Mathieu-Daudé if (ret) { 3584*8d7f2e76SPhilippe Mathieu-Daudé ret = -errno; 3585*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Failed to discard range " 3586*8d7f2e76SPhilippe Mathieu-Daudé "%s:%" PRIx64 " +%zx (%d)", 3587*8d7f2e76SPhilippe Mathieu-Daudé rb->idstr, start, length, ret); 3588*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3589*8d7f2e76SPhilippe Mathieu-Daudé } 3590*8d7f2e76SPhilippe Mathieu-Daudé #else 3591*8d7f2e76SPhilippe Mathieu-Daudé ret = -ENOSYS; 3592*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: MADVISE not available" 3593*8d7f2e76SPhilippe Mathieu-Daudé "%s:%" PRIx64 " +%zx (%d)", 3594*8d7f2e76SPhilippe Mathieu-Daudé rb->idstr, start, length, ret); 3595*8d7f2e76SPhilippe Mathieu-Daudé goto err; 3596*8d7f2e76SPhilippe Mathieu-Daudé #endif 3597*8d7f2e76SPhilippe Mathieu-Daudé } 3598*8d7f2e76SPhilippe Mathieu-Daudé trace_ram_block_discard_range(rb->idstr, host_startaddr, length, 3599*8d7f2e76SPhilippe Mathieu-Daudé need_madvise, need_fallocate, ret); 3600*8d7f2e76SPhilippe Mathieu-Daudé } else { 3601*8d7f2e76SPhilippe Mathieu-Daudé error_report("ram_block_discard_range: Overrun block '%s' (%" PRIu64 3602*8d7f2e76SPhilippe Mathieu-Daudé "/%zx/" RAM_ADDR_FMT")", 3603*8d7f2e76SPhilippe Mathieu-Daudé rb->idstr, start, length, rb->max_length); 3604*8d7f2e76SPhilippe Mathieu-Daudé } 3605*8d7f2e76SPhilippe Mathieu-Daudé 3606*8d7f2e76SPhilippe Mathieu-Daudé err: 3607*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3608*8d7f2e76SPhilippe Mathieu-Daudé } 3609*8d7f2e76SPhilippe Mathieu-Daudé 3610*8d7f2e76SPhilippe Mathieu-Daudé bool ramblock_is_pmem(RAMBlock *rb) 3611*8d7f2e76SPhilippe Mathieu-Daudé { 3612*8d7f2e76SPhilippe Mathieu-Daudé return rb->flags & RAM_PMEM; 3613*8d7f2e76SPhilippe Mathieu-Daudé } 3614*8d7f2e76SPhilippe Mathieu-Daudé 3615*8d7f2e76SPhilippe Mathieu-Daudé static void mtree_print_phys_entries(int start, int end, int skip, int ptr) 3616*8d7f2e76SPhilippe Mathieu-Daudé { 3617*8d7f2e76SPhilippe Mathieu-Daudé if (start == end - 1) { 3618*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf("\t%3d ", start); 3619*8d7f2e76SPhilippe Mathieu-Daudé } else { 3620*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf("\t%3d..%-3d ", start, end - 1); 3621*8d7f2e76SPhilippe Mathieu-Daudé } 3622*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" skip=%d ", skip); 3623*8d7f2e76SPhilippe Mathieu-Daudé if (ptr == PHYS_MAP_NODE_NIL) { 3624*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" ptr=NIL"); 3625*8d7f2e76SPhilippe Mathieu-Daudé } else if (!skip) { 3626*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" ptr=#%d", ptr); 3627*8d7f2e76SPhilippe Mathieu-Daudé } else { 3628*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" ptr=[%d]", ptr); 3629*8d7f2e76SPhilippe Mathieu-Daudé } 3630*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf("\n"); 3631*8d7f2e76SPhilippe Mathieu-Daudé } 3632*8d7f2e76SPhilippe Mathieu-Daudé 3633*8d7f2e76SPhilippe Mathieu-Daudé #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \ 3634*8d7f2e76SPhilippe Mathieu-Daudé int128_sub((size), int128_one())) : 0) 3635*8d7f2e76SPhilippe Mathieu-Daudé 3636*8d7f2e76SPhilippe Mathieu-Daudé void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root) 3637*8d7f2e76SPhilippe Mathieu-Daudé { 3638*8d7f2e76SPhilippe Mathieu-Daudé int i; 3639*8d7f2e76SPhilippe Mathieu-Daudé 3640*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" Dispatch\n"); 3641*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" Physical sections\n"); 3642*8d7f2e76SPhilippe Mathieu-Daudé 3643*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < d->map.sections_nb; ++i) { 3644*8d7f2e76SPhilippe Mathieu-Daudé MemoryRegionSection *s = d->map.sections + i; 3645*8d7f2e76SPhilippe Mathieu-Daudé const char *names[] = { " [unassigned]", " [not dirty]", 3646*8d7f2e76SPhilippe Mathieu-Daudé " [ROM]", " [watch]" }; 3647*8d7f2e76SPhilippe Mathieu-Daudé 3648*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx 3649*8d7f2e76SPhilippe Mathieu-Daudé " %s%s%s%s%s", 3650*8d7f2e76SPhilippe Mathieu-Daudé i, 3651*8d7f2e76SPhilippe Mathieu-Daudé s->offset_within_address_space, 3652*8d7f2e76SPhilippe Mathieu-Daudé s->offset_within_address_space + MR_SIZE(s->size), 3653*8d7f2e76SPhilippe Mathieu-Daudé s->mr->name ? s->mr->name : "(noname)", 3654*8d7f2e76SPhilippe Mathieu-Daudé i < ARRAY_SIZE(names) ? names[i] : "", 3655*8d7f2e76SPhilippe Mathieu-Daudé s->mr == root ? " [ROOT]" : "", 3656*8d7f2e76SPhilippe Mathieu-Daudé s == d->mru_section ? " [MRU]" : "", 3657*8d7f2e76SPhilippe Mathieu-Daudé s->mr->is_iommu ? " [iommu]" : ""); 3658*8d7f2e76SPhilippe Mathieu-Daudé 3659*8d7f2e76SPhilippe Mathieu-Daudé if (s->mr->alias) { 3660*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" alias=%s", s->mr->alias->name ? 3661*8d7f2e76SPhilippe Mathieu-Daudé s->mr->alias->name : "noname"); 3662*8d7f2e76SPhilippe Mathieu-Daudé } 3663*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf("\n"); 3664*8d7f2e76SPhilippe Mathieu-Daudé } 3665*8d7f2e76SPhilippe Mathieu-Daudé 3666*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n", 3667*8d7f2e76SPhilippe Mathieu-Daudé P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip); 3668*8d7f2e76SPhilippe Mathieu-Daudé for (i = 0; i < d->map.nodes_nb; ++i) { 3669*8d7f2e76SPhilippe Mathieu-Daudé int j, jprev; 3670*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry prev; 3671*8d7f2e76SPhilippe Mathieu-Daudé Node *n = d->map.nodes + i; 3672*8d7f2e76SPhilippe Mathieu-Daudé 3673*8d7f2e76SPhilippe Mathieu-Daudé qemu_printf(" [%d]\n", i); 3674*8d7f2e76SPhilippe Mathieu-Daudé 3675*8d7f2e76SPhilippe Mathieu-Daudé for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) { 3676*8d7f2e76SPhilippe Mathieu-Daudé PhysPageEntry *pe = *n + j; 3677*8d7f2e76SPhilippe Mathieu-Daudé 3678*8d7f2e76SPhilippe Mathieu-Daudé if (pe->ptr == prev.ptr && pe->skip == prev.skip) { 3679*8d7f2e76SPhilippe Mathieu-Daudé continue; 3680*8d7f2e76SPhilippe Mathieu-Daudé } 3681*8d7f2e76SPhilippe Mathieu-Daudé 3682*8d7f2e76SPhilippe Mathieu-Daudé mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr); 3683*8d7f2e76SPhilippe Mathieu-Daudé 3684*8d7f2e76SPhilippe Mathieu-Daudé jprev = j; 3685*8d7f2e76SPhilippe Mathieu-Daudé prev = *pe; 3686*8d7f2e76SPhilippe Mathieu-Daudé } 3687*8d7f2e76SPhilippe Mathieu-Daudé 3688*8d7f2e76SPhilippe Mathieu-Daudé if (jprev != ARRAY_SIZE(*n)) { 3689*8d7f2e76SPhilippe Mathieu-Daudé mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr); 3690*8d7f2e76SPhilippe Mathieu-Daudé } 3691*8d7f2e76SPhilippe Mathieu-Daudé } 3692*8d7f2e76SPhilippe Mathieu-Daudé } 3693*8d7f2e76SPhilippe Mathieu-Daudé 3694*8d7f2e76SPhilippe Mathieu-Daudé /* Require any discards to work. */ 3695*8d7f2e76SPhilippe Mathieu-Daudé static unsigned int ram_block_discard_required_cnt; 3696*8d7f2e76SPhilippe Mathieu-Daudé /* Require only coordinated discards to work. */ 3697*8d7f2e76SPhilippe Mathieu-Daudé static unsigned int ram_block_coordinated_discard_required_cnt; 3698*8d7f2e76SPhilippe Mathieu-Daudé /* Disable any discards. */ 3699*8d7f2e76SPhilippe Mathieu-Daudé static unsigned int ram_block_discard_disabled_cnt; 3700*8d7f2e76SPhilippe Mathieu-Daudé /* Disable only uncoordinated discards. */ 3701*8d7f2e76SPhilippe Mathieu-Daudé static unsigned int ram_block_uncoordinated_discard_disabled_cnt; 3702*8d7f2e76SPhilippe Mathieu-Daudé static QemuMutex ram_block_discard_disable_mutex; 3703*8d7f2e76SPhilippe Mathieu-Daudé 3704*8d7f2e76SPhilippe Mathieu-Daudé static void ram_block_discard_disable_mutex_lock(void) 3705*8d7f2e76SPhilippe Mathieu-Daudé { 3706*8d7f2e76SPhilippe Mathieu-Daudé static gsize initialized; 3707*8d7f2e76SPhilippe Mathieu-Daudé 3708*8d7f2e76SPhilippe Mathieu-Daudé if (g_once_init_enter(&initialized)) { 3709*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_init(&ram_block_discard_disable_mutex); 3710*8d7f2e76SPhilippe Mathieu-Daudé g_once_init_leave(&initialized, 1); 3711*8d7f2e76SPhilippe Mathieu-Daudé } 3712*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_lock(&ram_block_discard_disable_mutex); 3713*8d7f2e76SPhilippe Mathieu-Daudé } 3714*8d7f2e76SPhilippe Mathieu-Daudé 3715*8d7f2e76SPhilippe Mathieu-Daudé static void ram_block_discard_disable_mutex_unlock(void) 3716*8d7f2e76SPhilippe Mathieu-Daudé { 3717*8d7f2e76SPhilippe Mathieu-Daudé qemu_mutex_unlock(&ram_block_discard_disable_mutex); 3718*8d7f2e76SPhilippe Mathieu-Daudé } 3719*8d7f2e76SPhilippe Mathieu-Daudé 3720*8d7f2e76SPhilippe Mathieu-Daudé int ram_block_discard_disable(bool state) 3721*8d7f2e76SPhilippe Mathieu-Daudé { 3722*8d7f2e76SPhilippe Mathieu-Daudé int ret = 0; 3723*8d7f2e76SPhilippe Mathieu-Daudé 3724*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_lock(); 3725*8d7f2e76SPhilippe Mathieu-Daudé if (!state) { 3726*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disabled_cnt--; 3727*8d7f2e76SPhilippe Mathieu-Daudé } else if (ram_block_discard_required_cnt || 3728*8d7f2e76SPhilippe Mathieu-Daudé ram_block_coordinated_discard_required_cnt) { 3729*8d7f2e76SPhilippe Mathieu-Daudé ret = -EBUSY; 3730*8d7f2e76SPhilippe Mathieu-Daudé } else { 3731*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disabled_cnt++; 3732*8d7f2e76SPhilippe Mathieu-Daudé } 3733*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_unlock(); 3734*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3735*8d7f2e76SPhilippe Mathieu-Daudé } 3736*8d7f2e76SPhilippe Mathieu-Daudé 3737*8d7f2e76SPhilippe Mathieu-Daudé int ram_block_uncoordinated_discard_disable(bool state) 3738*8d7f2e76SPhilippe Mathieu-Daudé { 3739*8d7f2e76SPhilippe Mathieu-Daudé int ret = 0; 3740*8d7f2e76SPhilippe Mathieu-Daudé 3741*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_lock(); 3742*8d7f2e76SPhilippe Mathieu-Daudé if (!state) { 3743*8d7f2e76SPhilippe Mathieu-Daudé ram_block_uncoordinated_discard_disabled_cnt--; 3744*8d7f2e76SPhilippe Mathieu-Daudé } else if (ram_block_discard_required_cnt) { 3745*8d7f2e76SPhilippe Mathieu-Daudé ret = -EBUSY; 3746*8d7f2e76SPhilippe Mathieu-Daudé } else { 3747*8d7f2e76SPhilippe Mathieu-Daudé ram_block_uncoordinated_discard_disabled_cnt++; 3748*8d7f2e76SPhilippe Mathieu-Daudé } 3749*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_unlock(); 3750*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3751*8d7f2e76SPhilippe Mathieu-Daudé } 3752*8d7f2e76SPhilippe Mathieu-Daudé 3753*8d7f2e76SPhilippe Mathieu-Daudé int ram_block_discard_require(bool state) 3754*8d7f2e76SPhilippe Mathieu-Daudé { 3755*8d7f2e76SPhilippe Mathieu-Daudé int ret = 0; 3756*8d7f2e76SPhilippe Mathieu-Daudé 3757*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_lock(); 3758*8d7f2e76SPhilippe Mathieu-Daudé if (!state) { 3759*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_required_cnt--; 3760*8d7f2e76SPhilippe Mathieu-Daudé } else if (ram_block_discard_disabled_cnt || 3761*8d7f2e76SPhilippe Mathieu-Daudé ram_block_uncoordinated_discard_disabled_cnt) { 3762*8d7f2e76SPhilippe Mathieu-Daudé ret = -EBUSY; 3763*8d7f2e76SPhilippe Mathieu-Daudé } else { 3764*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_required_cnt++; 3765*8d7f2e76SPhilippe Mathieu-Daudé } 3766*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_unlock(); 3767*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3768*8d7f2e76SPhilippe Mathieu-Daudé } 3769*8d7f2e76SPhilippe Mathieu-Daudé 3770*8d7f2e76SPhilippe Mathieu-Daudé int ram_block_coordinated_discard_require(bool state) 3771*8d7f2e76SPhilippe Mathieu-Daudé { 3772*8d7f2e76SPhilippe Mathieu-Daudé int ret = 0; 3773*8d7f2e76SPhilippe Mathieu-Daudé 3774*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_lock(); 3775*8d7f2e76SPhilippe Mathieu-Daudé if (!state) { 3776*8d7f2e76SPhilippe Mathieu-Daudé ram_block_coordinated_discard_required_cnt--; 3777*8d7f2e76SPhilippe Mathieu-Daudé } else if (ram_block_discard_disabled_cnt) { 3778*8d7f2e76SPhilippe Mathieu-Daudé ret = -EBUSY; 3779*8d7f2e76SPhilippe Mathieu-Daudé } else { 3780*8d7f2e76SPhilippe Mathieu-Daudé ram_block_coordinated_discard_required_cnt++; 3781*8d7f2e76SPhilippe Mathieu-Daudé } 3782*8d7f2e76SPhilippe Mathieu-Daudé ram_block_discard_disable_mutex_unlock(); 3783*8d7f2e76SPhilippe Mathieu-Daudé return ret; 3784*8d7f2e76SPhilippe Mathieu-Daudé } 3785*8d7f2e76SPhilippe Mathieu-Daudé 3786*8d7f2e76SPhilippe Mathieu-Daudé bool ram_block_discard_is_disabled(void) 3787*8d7f2e76SPhilippe Mathieu-Daudé { 3788*8d7f2e76SPhilippe Mathieu-Daudé return qatomic_read(&ram_block_discard_disabled_cnt) || 3789*8d7f2e76SPhilippe Mathieu-Daudé qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt); 3790*8d7f2e76SPhilippe Mathieu-Daudé } 3791*8d7f2e76SPhilippe Mathieu-Daudé 3792*8d7f2e76SPhilippe Mathieu-Daudé bool ram_block_discard_is_required(void) 3793*8d7f2e76SPhilippe Mathieu-Daudé { 3794*8d7f2e76SPhilippe Mathieu-Daudé return qatomic_read(&ram_block_discard_required_cnt) || 3795*8d7f2e76SPhilippe Mathieu-Daudé qatomic_read(&ram_block_coordinated_discard_required_cnt); 3796*8d7f2e76SPhilippe Mathieu-Daudé } 3797