1 /* 2 * Declarations for cpu physical memory functions 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * later. See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * This header is for use by exec.c and memory.c ONLY. Do not include it. 16 * The functions declared here will be removed soon. 17 */ 18 19 #ifndef RAM_ADDR_H 20 #define RAM_ADDR_H 21 22 #ifndef CONFIG_USER_ONLY 23 #include "hw/xen/xen.h" 24 25 typedef struct RAMBlock RAMBlock; 26 27 struct RAMBlock { 28 struct rcu_head rcu; 29 struct MemoryRegion *mr; 30 uint8_t *host; 31 ram_addr_t offset; 32 ram_addr_t used_length; 33 ram_addr_t max_length; 34 void (*resized)(const char*, uint64_t length, void *host); 35 uint32_t flags; 36 /* Protected by iothread lock. */ 37 char idstr[256]; 38 /* RCU-enabled, writes protected by the ramlist lock */ 39 QLIST_ENTRY(RAMBlock) next; 40 int fd; 41 }; 42 43 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) 44 { 45 assert(offset < block->used_length); 46 assert(block->host); 47 return (char *)block->host + offset; 48 } 49 50 typedef struct RAMList { 51 QemuMutex mutex; 52 /* Protected by the iothread lock. */ 53 unsigned long *dirty_memory[DIRTY_MEMORY_NUM]; 54 RAMBlock *mru_block; 55 /* RCU-enabled, writes protected by the ramlist lock. */ 56 QLIST_HEAD(, RAMBlock) blocks; 57 uint32_t version; 58 } RAMList; 59 extern RAMList ram_list; 60 61 ram_addr_t last_ram_offset(void); 62 void qemu_mutex_lock_ramlist(void); 63 void qemu_mutex_unlock_ramlist(void); 64 65 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 66 bool share, const char *mem_path, 67 Error **errp); 68 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 69 MemoryRegion *mr, Error **errp); 70 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp); 71 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 72 void (*resized)(const char*, 73 uint64_t length, 74 void *host), 75 MemoryRegion *mr, Error **errp); 76 int qemu_get_ram_fd(ram_addr_t addr); 77 void *qemu_get_ram_block_host_ptr(ram_addr_t addr); 78 void *qemu_get_ram_ptr(ram_addr_t addr); 79 void qemu_ram_free(ram_addr_t addr); 80 void qemu_ram_free_from_ptr(ram_addr_t addr); 81 82 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp); 83 84 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) 85 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) 86 87 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, 88 ram_addr_t length, 89 unsigned client) 90 { 91 unsigned long end, page, next; 92 93 assert(client < DIRTY_MEMORY_NUM); 94 95 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 96 page = start >> TARGET_PAGE_BITS; 97 next = find_next_bit(ram_list.dirty_memory[client], end, page); 98 99 return next < end; 100 } 101 102 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, 103 ram_addr_t length, 104 unsigned client) 105 { 106 unsigned long end, page, next; 107 108 assert(client < DIRTY_MEMORY_NUM); 109 110 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 111 page = start >> TARGET_PAGE_BITS; 112 next = find_next_zero_bit(ram_list.dirty_memory[client], end, page); 113 114 return next >= end; 115 } 116 117 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, 118 unsigned client) 119 { 120 return cpu_physical_memory_get_dirty(addr, 1, client); 121 } 122 123 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) 124 { 125 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); 126 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); 127 bool migration = 128 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); 129 return !(vga && code && migration); 130 } 131 132 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, 133 ram_addr_t length, 134 uint8_t mask) 135 { 136 uint8_t ret = 0; 137 138 if (mask & (1 << DIRTY_MEMORY_VGA) && 139 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { 140 ret |= (1 << DIRTY_MEMORY_VGA); 141 } 142 if (mask & (1 << DIRTY_MEMORY_CODE) && 143 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { 144 ret |= (1 << DIRTY_MEMORY_CODE); 145 } 146 if (mask & (1 << DIRTY_MEMORY_MIGRATION) && 147 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { 148 ret |= (1 << DIRTY_MEMORY_MIGRATION); 149 } 150 return ret; 151 } 152 153 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, 154 unsigned client) 155 { 156 assert(client < DIRTY_MEMORY_NUM); 157 set_bit_atomic(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]); 158 } 159 160 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, 161 ram_addr_t length, 162 uint8_t mask) 163 { 164 unsigned long end, page; 165 unsigned long **d = ram_list.dirty_memory; 166 167 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 168 page = start >> TARGET_PAGE_BITS; 169 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { 170 bitmap_set_atomic(d[DIRTY_MEMORY_MIGRATION], page, end - page); 171 } 172 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { 173 bitmap_set_atomic(d[DIRTY_MEMORY_VGA], page, end - page); 174 } 175 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { 176 bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page); 177 } 178 xen_modified_memory(start, length); 179 } 180 181 #if !defined(_WIN32) 182 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, 183 ram_addr_t start, 184 ram_addr_t pages) 185 { 186 unsigned long i, j; 187 unsigned long page_number, c; 188 hwaddr addr; 189 ram_addr_t ram_addr; 190 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; 191 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE; 192 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 193 194 /* start address is aligned at the start of a word? */ 195 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && 196 (hpratio == 1)) { 197 long k; 198 long nr = BITS_TO_LONGS(pages); 199 200 for (k = 0; k < nr; k++) { 201 if (bitmap[k]) { 202 unsigned long temp = leul_to_cpu(bitmap[k]); 203 unsigned long **d = ram_list.dirty_memory; 204 205 atomic_or(&d[DIRTY_MEMORY_MIGRATION][page + k], temp); 206 atomic_or(&d[DIRTY_MEMORY_VGA][page + k], temp); 207 if (tcg_enabled()) { 208 atomic_or(&d[DIRTY_MEMORY_CODE][page + k], temp); 209 } 210 } 211 } 212 xen_modified_memory(start, pages << TARGET_PAGE_BITS); 213 } else { 214 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; 215 /* 216 * bitmap-traveling is faster than memory-traveling (for addr...) 217 * especially when most of the memory is not dirty. 218 */ 219 for (i = 0; i < len; i++) { 220 if (bitmap[i] != 0) { 221 c = leul_to_cpu(bitmap[i]); 222 do { 223 j = ctzl(c); 224 c &= ~(1ul << j); 225 page_number = (i * HOST_LONG_BITS + j) * hpratio; 226 addr = page_number * TARGET_PAGE_SIZE; 227 ram_addr = start + addr; 228 cpu_physical_memory_set_dirty_range(ram_addr, 229 TARGET_PAGE_SIZE * hpratio, clients); 230 } while (c != 0); 231 } 232 } 233 } 234 } 235 #endif /* not _WIN32 */ 236 237 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 238 ram_addr_t length, 239 unsigned client); 240 241 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, 242 ram_addr_t length) 243 { 244 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); 245 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); 246 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); 247 } 248 249 250 static inline 251 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, 252 ram_addr_t start, 253 ram_addr_t length) 254 { 255 ram_addr_t addr; 256 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 257 uint64_t num_dirty = 0; 258 259 /* start address is aligned at the start of a word? */ 260 if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) { 261 int k; 262 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); 263 unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]; 264 265 for (k = page; k < page + nr; k++) { 266 if (src[k]) { 267 unsigned long bits = atomic_xchg(&src[k], 0); 268 unsigned long new_dirty; 269 new_dirty = ~dest[k]; 270 dest[k] |= bits; 271 new_dirty &= bits; 272 num_dirty += ctpopl(new_dirty); 273 } 274 } 275 } else { 276 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { 277 if (cpu_physical_memory_test_and_clear_dirty( 278 start + addr, 279 TARGET_PAGE_SIZE, 280 DIRTY_MEMORY_MIGRATION)) { 281 long k = (start + addr) >> TARGET_PAGE_BITS; 282 if (!test_and_set_bit(k, dest)) { 283 num_dirty++; 284 } 285 } 286 } 287 } 288 289 return num_dirty; 290 } 291 292 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new); 293 #endif 294 #endif 295