1 /* 2 * Declarations for cpu physical memory functions 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * later. See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * This header is for use by exec.c and memory.c ONLY. Do not include it. 16 * The functions declared here will be removed soon. 17 */ 18 19 #ifndef RAM_ADDR_H 20 #define RAM_ADDR_H 21 22 #ifndef CONFIG_USER_ONLY 23 #include "hw/xen/xen.h" 24 #include "exec/ramlist.h" 25 26 struct RAMBlock { 27 struct rcu_head rcu; 28 struct MemoryRegion *mr; 29 uint8_t *host; 30 ram_addr_t offset; 31 ram_addr_t used_length; 32 ram_addr_t max_length; 33 void (*resized)(const char*, uint64_t length, void *host); 34 uint32_t flags; 35 /* Protected by iothread lock. */ 36 char idstr[256]; 37 /* RCU-enabled, writes protected by the ramlist lock */ 38 QLIST_ENTRY(RAMBlock) next; 39 QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers; 40 int fd; 41 size_t page_size; 42 }; 43 44 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) 45 { 46 return (b && b->host && offset < b->used_length) ? true : false; 47 } 48 49 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) 50 { 51 assert(offset_in_ramblock(block, offset)); 52 return (char *)block->host + offset; 53 } 54 55 long qemu_getrampagesize(void); 56 unsigned long last_ram_page(void); 57 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 58 bool share, const char *mem_path, 59 Error **errp); 60 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 61 MemoryRegion *mr, Error **errp); 62 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp); 63 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 64 void (*resized)(const char*, 65 uint64_t length, 66 void *host), 67 MemoryRegion *mr, Error **errp); 68 void qemu_ram_free(RAMBlock *block); 69 70 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); 71 72 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) 73 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) 74 75 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, 76 ram_addr_t length, 77 unsigned client) 78 { 79 DirtyMemoryBlocks *blocks; 80 unsigned long end, page; 81 unsigned long idx, offset, base; 82 bool dirty = false; 83 84 assert(client < DIRTY_MEMORY_NUM); 85 86 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 87 page = start >> TARGET_PAGE_BITS; 88 89 rcu_read_lock(); 90 91 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 92 93 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 94 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 95 base = page - offset; 96 while (page < end) { 97 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 98 unsigned long num = next - base; 99 unsigned long found = find_next_bit(blocks->blocks[idx], num, offset); 100 if (found < num) { 101 dirty = true; 102 break; 103 } 104 105 page = next; 106 idx++; 107 offset = 0; 108 base += DIRTY_MEMORY_BLOCK_SIZE; 109 } 110 111 rcu_read_unlock(); 112 113 return dirty; 114 } 115 116 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, 117 ram_addr_t length, 118 unsigned client) 119 { 120 DirtyMemoryBlocks *blocks; 121 unsigned long end, page; 122 unsigned long idx, offset, base; 123 bool dirty = true; 124 125 assert(client < DIRTY_MEMORY_NUM); 126 127 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 128 page = start >> TARGET_PAGE_BITS; 129 130 rcu_read_lock(); 131 132 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 133 134 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 135 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 136 base = page - offset; 137 while (page < end) { 138 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 139 unsigned long num = next - base; 140 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); 141 if (found < num) { 142 dirty = false; 143 break; 144 } 145 146 page = next; 147 idx++; 148 offset = 0; 149 base += DIRTY_MEMORY_BLOCK_SIZE; 150 } 151 152 rcu_read_unlock(); 153 154 return dirty; 155 } 156 157 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, 158 unsigned client) 159 { 160 return cpu_physical_memory_get_dirty(addr, 1, client); 161 } 162 163 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) 164 { 165 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); 166 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); 167 bool migration = 168 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); 169 return !(vga && code && migration); 170 } 171 172 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, 173 ram_addr_t length, 174 uint8_t mask) 175 { 176 uint8_t ret = 0; 177 178 if (mask & (1 << DIRTY_MEMORY_VGA) && 179 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { 180 ret |= (1 << DIRTY_MEMORY_VGA); 181 } 182 if (mask & (1 << DIRTY_MEMORY_CODE) && 183 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { 184 ret |= (1 << DIRTY_MEMORY_CODE); 185 } 186 if (mask & (1 << DIRTY_MEMORY_MIGRATION) && 187 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { 188 ret |= (1 << DIRTY_MEMORY_MIGRATION); 189 } 190 return ret; 191 } 192 193 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, 194 unsigned client) 195 { 196 unsigned long page, idx, offset; 197 DirtyMemoryBlocks *blocks; 198 199 assert(client < DIRTY_MEMORY_NUM); 200 201 page = addr >> TARGET_PAGE_BITS; 202 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 203 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 204 205 rcu_read_lock(); 206 207 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 208 209 set_bit_atomic(offset, blocks->blocks[idx]); 210 211 rcu_read_unlock(); 212 } 213 214 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, 215 ram_addr_t length, 216 uint8_t mask) 217 { 218 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; 219 unsigned long end, page; 220 unsigned long idx, offset, base; 221 int i; 222 223 if (!mask && !xen_enabled()) { 224 return; 225 } 226 227 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 228 page = start >> TARGET_PAGE_BITS; 229 230 rcu_read_lock(); 231 232 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 233 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]); 234 } 235 236 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 237 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 238 base = page - offset; 239 while (page < end) { 240 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 241 242 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { 243 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], 244 offset, next - page); 245 } 246 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { 247 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], 248 offset, next - page); 249 } 250 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { 251 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], 252 offset, next - page); 253 } 254 255 page = next; 256 idx++; 257 offset = 0; 258 base += DIRTY_MEMORY_BLOCK_SIZE; 259 } 260 261 rcu_read_unlock(); 262 263 xen_hvm_modified_memory(start, length); 264 } 265 266 #if !defined(_WIN32) 267 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, 268 ram_addr_t start, 269 ram_addr_t pages) 270 { 271 unsigned long i, j; 272 unsigned long page_number, c; 273 hwaddr addr; 274 ram_addr_t ram_addr; 275 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; 276 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE; 277 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 278 279 /* start address is aligned at the start of a word? */ 280 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && 281 (hpratio == 1)) { 282 unsigned long **blocks[DIRTY_MEMORY_NUM]; 283 unsigned long idx; 284 unsigned long offset; 285 long k; 286 long nr = BITS_TO_LONGS(pages); 287 288 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; 289 offset = BIT_WORD((start >> TARGET_PAGE_BITS) % 290 DIRTY_MEMORY_BLOCK_SIZE); 291 292 rcu_read_lock(); 293 294 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 295 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks; 296 } 297 298 for (k = 0; k < nr; k++) { 299 if (bitmap[k]) { 300 unsigned long temp = leul_to_cpu(bitmap[k]); 301 302 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp); 303 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); 304 if (tcg_enabled()) { 305 atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp); 306 } 307 } 308 309 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 310 offset = 0; 311 idx++; 312 } 313 } 314 315 rcu_read_unlock(); 316 317 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); 318 } else { 319 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; 320 /* 321 * bitmap-traveling is faster than memory-traveling (for addr...) 322 * especially when most of the memory is not dirty. 323 */ 324 for (i = 0; i < len; i++) { 325 if (bitmap[i] != 0) { 326 c = leul_to_cpu(bitmap[i]); 327 do { 328 j = ctzl(c); 329 c &= ~(1ul << j); 330 page_number = (i * HOST_LONG_BITS + j) * hpratio; 331 addr = page_number * TARGET_PAGE_SIZE; 332 ram_addr = start + addr; 333 cpu_physical_memory_set_dirty_range(ram_addr, 334 TARGET_PAGE_SIZE * hpratio, clients); 335 } while (c != 0); 336 } 337 } 338 } 339 } 340 #endif /* not _WIN32 */ 341 342 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 343 ram_addr_t length, 344 unsigned client); 345 346 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty 347 (ram_addr_t start, ram_addr_t length, unsigned client); 348 349 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, 350 ram_addr_t start, 351 ram_addr_t length); 352 353 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, 354 ram_addr_t length) 355 { 356 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); 357 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); 358 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); 359 } 360 361 362 static inline 363 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, 364 RAMBlock *rb, 365 ram_addr_t start, 366 ram_addr_t length, 367 uint64_t *real_dirty_pages) 368 { 369 ram_addr_t addr; 370 start = rb->offset + start; 371 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 372 uint64_t num_dirty = 0; 373 374 /* start address is aligned at the start of a word? */ 375 if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) { 376 int k; 377 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); 378 unsigned long * const *src; 379 unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; 380 unsigned long offset = BIT_WORD((page * BITS_PER_LONG) % 381 DIRTY_MEMORY_BLOCK_SIZE); 382 383 rcu_read_lock(); 384 385 src = atomic_rcu_read( 386 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; 387 388 for (k = page; k < page + nr; k++) { 389 if (src[idx][offset]) { 390 unsigned long bits = atomic_xchg(&src[idx][offset], 0); 391 unsigned long new_dirty; 392 *real_dirty_pages += ctpopl(bits); 393 new_dirty = ~dest[k]; 394 dest[k] |= bits; 395 new_dirty &= bits; 396 num_dirty += ctpopl(new_dirty); 397 } 398 399 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 400 offset = 0; 401 idx++; 402 } 403 } 404 405 rcu_read_unlock(); 406 } else { 407 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { 408 if (cpu_physical_memory_test_and_clear_dirty( 409 start + addr, 410 TARGET_PAGE_SIZE, 411 DIRTY_MEMORY_MIGRATION)) { 412 *real_dirty_pages += 1; 413 long k = (start + addr) >> TARGET_PAGE_BITS; 414 if (!test_and_set_bit(k, dest)) { 415 num_dirty++; 416 } 417 } 418 } 419 } 420 421 return num_dirty; 422 } 423 #endif 424 #endif 425