1 /* 2 * Declarations for cpu physical memory functions 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * later. See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * This header is for use by exec.c and memory.c ONLY. Do not include it. 16 * The functions declared here will be removed soon. 17 */ 18 19 #ifndef RAM_ADDR_H 20 #define RAM_ADDR_H 21 22 #ifndef CONFIG_USER_ONLY 23 #include "cpu.h" 24 #include "sysemu/xen.h" 25 #include "sysemu/tcg.h" 26 #include "exec/ramlist.h" 27 #include "exec/ramblock.h" 28 29 extern uint64_t total_dirty_pages; 30 31 /** 32 * clear_bmap_size: calculate clear bitmap size 33 * 34 * @pages: number of guest pages 35 * @shift: guest page number shift 36 * 37 * Returns: number of bits for the clear bitmap 38 */ 39 static inline long clear_bmap_size(uint64_t pages, uint8_t shift) 40 { 41 return DIV_ROUND_UP(pages, 1UL << shift); 42 } 43 44 /** 45 * clear_bmap_set: set clear bitmap for the page range. Must be with 46 * bitmap_mutex held. 47 * 48 * @rb: the ramblock to operate on 49 * @start: the start page number 50 * @size: number of pages to set in the bitmap 51 * 52 * Returns: None 53 */ 54 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start, 55 uint64_t npages) 56 { 57 uint8_t shift = rb->clear_bmap_shift; 58 59 bitmap_set(rb->clear_bmap, start >> shift, clear_bmap_size(npages, shift)); 60 } 61 62 /** 63 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set. 64 * Must be with bitmap_mutex held. 65 * 66 * @rb: the ramblock to operate on 67 * @page: the page number to check 68 * 69 * Returns: true if the bit was set, false otherwise 70 */ 71 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) 72 { 73 uint8_t shift = rb->clear_bmap_shift; 74 75 return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1); 76 } 77 78 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) 79 { 80 return (b && b->host && offset < b->used_length) ? true : false; 81 } 82 83 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) 84 { 85 assert(offset_in_ramblock(block, offset)); 86 return (char *)block->host + offset; 87 } 88 89 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, 90 RAMBlock *rb) 91 { 92 uint64_t host_addr_offset = 93 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host); 94 return host_addr_offset >> TARGET_PAGE_BITS; 95 } 96 97 bool ramblock_is_pmem(RAMBlock *rb); 98 99 long qemu_minrampagesize(void); 100 long qemu_maxrampagesize(void); 101 102 /** 103 * qemu_ram_alloc_from_file, 104 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing 105 * file or device 106 * 107 * Parameters: 108 * @size: the size in bytes of the ram block 109 * @mr: the memory region where the ram block is 110 * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, 111 * RAM_NORESERVE. 112 * @mem_path or @fd: specify the backing file or device 113 * @readonly: true to open @path for reading, false for read/write. 114 * @errp: pointer to Error*, to store an error if it happens 115 * 116 * Return: 117 * On success, return a pointer to the ram block. 118 * On failure, return NULL. 119 */ 120 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 121 uint32_t ram_flags, const char *mem_path, 122 bool readonly, Error **errp); 123 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, 124 uint32_t ram_flags, int fd, off_t offset, 125 bool readonly, Error **errp); 126 127 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 128 MemoryRegion *mr, Error **errp); 129 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr, 130 Error **errp); 131 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 132 void (*resized)(const char*, 133 uint64_t length, 134 void *host), 135 MemoryRegion *mr, Error **errp); 136 void qemu_ram_free(RAMBlock *block); 137 138 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); 139 140 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length); 141 142 /* Clear whole block of mem */ 143 static inline void qemu_ram_block_writeback(RAMBlock *block) 144 { 145 qemu_ram_msync(block, 0, block->used_length); 146 } 147 148 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) 149 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) 150 151 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, 152 ram_addr_t length, 153 unsigned client) 154 { 155 DirtyMemoryBlocks *blocks; 156 unsigned long end, page; 157 unsigned long idx, offset, base; 158 bool dirty = false; 159 160 assert(client < DIRTY_MEMORY_NUM); 161 162 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 163 page = start >> TARGET_PAGE_BITS; 164 165 WITH_RCU_READ_LOCK_GUARD() { 166 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 167 168 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 169 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 170 base = page - offset; 171 while (page < end) { 172 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 173 unsigned long num = next - base; 174 unsigned long found = find_next_bit(blocks->blocks[idx], 175 num, offset); 176 if (found < num) { 177 dirty = true; 178 break; 179 } 180 181 page = next; 182 idx++; 183 offset = 0; 184 base += DIRTY_MEMORY_BLOCK_SIZE; 185 } 186 } 187 188 return dirty; 189 } 190 191 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, 192 ram_addr_t length, 193 unsigned client) 194 { 195 DirtyMemoryBlocks *blocks; 196 unsigned long end, page; 197 unsigned long idx, offset, base; 198 bool dirty = true; 199 200 assert(client < DIRTY_MEMORY_NUM); 201 202 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 203 page = start >> TARGET_PAGE_BITS; 204 205 RCU_READ_LOCK_GUARD(); 206 207 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 208 209 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 210 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 211 base = page - offset; 212 while (page < end) { 213 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 214 unsigned long num = next - base; 215 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); 216 if (found < num) { 217 dirty = false; 218 break; 219 } 220 221 page = next; 222 idx++; 223 offset = 0; 224 base += DIRTY_MEMORY_BLOCK_SIZE; 225 } 226 227 return dirty; 228 } 229 230 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, 231 unsigned client) 232 { 233 return cpu_physical_memory_get_dirty(addr, 1, client); 234 } 235 236 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) 237 { 238 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); 239 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); 240 bool migration = 241 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); 242 return !(vga && code && migration); 243 } 244 245 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, 246 ram_addr_t length, 247 uint8_t mask) 248 { 249 uint8_t ret = 0; 250 251 if (mask & (1 << DIRTY_MEMORY_VGA) && 252 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { 253 ret |= (1 << DIRTY_MEMORY_VGA); 254 } 255 if (mask & (1 << DIRTY_MEMORY_CODE) && 256 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { 257 ret |= (1 << DIRTY_MEMORY_CODE); 258 } 259 if (mask & (1 << DIRTY_MEMORY_MIGRATION) && 260 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { 261 ret |= (1 << DIRTY_MEMORY_MIGRATION); 262 } 263 return ret; 264 } 265 266 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, 267 unsigned client) 268 { 269 unsigned long page, idx, offset; 270 DirtyMemoryBlocks *blocks; 271 272 assert(client < DIRTY_MEMORY_NUM); 273 274 page = addr >> TARGET_PAGE_BITS; 275 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 276 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 277 278 RCU_READ_LOCK_GUARD(); 279 280 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 281 282 set_bit_atomic(offset, blocks->blocks[idx]); 283 } 284 285 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, 286 ram_addr_t length, 287 uint8_t mask) 288 { 289 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; 290 unsigned long end, page; 291 unsigned long idx, offset, base; 292 int i; 293 294 if (!mask && !xen_enabled()) { 295 return; 296 } 297 298 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 299 page = start >> TARGET_PAGE_BITS; 300 301 WITH_RCU_READ_LOCK_GUARD() { 302 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 303 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); 304 } 305 306 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 307 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 308 base = page - offset; 309 while (page < end) { 310 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 311 312 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { 313 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], 314 offset, next - page); 315 } 316 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { 317 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], 318 offset, next - page); 319 } 320 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { 321 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], 322 offset, next - page); 323 } 324 325 page = next; 326 idx++; 327 offset = 0; 328 base += DIRTY_MEMORY_BLOCK_SIZE; 329 } 330 } 331 332 xen_hvm_modified_memory(start, length); 333 } 334 335 #if !defined(_WIN32) 336 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, 337 ram_addr_t start, 338 ram_addr_t pages) 339 { 340 unsigned long i, j; 341 unsigned long page_number, c; 342 hwaddr addr; 343 ram_addr_t ram_addr; 344 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; 345 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; 346 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 347 348 /* start address is aligned at the start of a word? */ 349 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && 350 (hpratio == 1)) { 351 unsigned long **blocks[DIRTY_MEMORY_NUM]; 352 unsigned long idx; 353 unsigned long offset; 354 long k; 355 long nr = BITS_TO_LONGS(pages); 356 357 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; 358 offset = BIT_WORD((start >> TARGET_PAGE_BITS) % 359 DIRTY_MEMORY_BLOCK_SIZE); 360 361 WITH_RCU_READ_LOCK_GUARD() { 362 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 363 blocks[i] = 364 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; 365 } 366 367 for (k = 0; k < nr; k++) { 368 if (bitmap[k]) { 369 unsigned long temp = leul_to_cpu(bitmap[k]); 370 371 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); 372 373 if (global_dirty_tracking) { 374 qatomic_or( 375 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], 376 temp); 377 if (unlikely( 378 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { 379 total_dirty_pages += ctpopl(temp); 380 } 381 } 382 383 if (tcg_enabled()) { 384 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], 385 temp); 386 } 387 } 388 389 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 390 offset = 0; 391 idx++; 392 } 393 } 394 } 395 396 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); 397 } else { 398 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; 399 400 if (!global_dirty_tracking) { 401 clients &= ~(1 << DIRTY_MEMORY_MIGRATION); 402 } 403 404 /* 405 * bitmap-traveling is faster than memory-traveling (for addr...) 406 * especially when most of the memory is not dirty. 407 */ 408 for (i = 0; i < len; i++) { 409 if (bitmap[i] != 0) { 410 c = leul_to_cpu(bitmap[i]); 411 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { 412 total_dirty_pages += ctpopl(c); 413 } 414 do { 415 j = ctzl(c); 416 c &= ~(1ul << j); 417 page_number = (i * HOST_LONG_BITS + j) * hpratio; 418 addr = page_number * TARGET_PAGE_SIZE; 419 ram_addr = start + addr; 420 cpu_physical_memory_set_dirty_range(ram_addr, 421 TARGET_PAGE_SIZE * hpratio, clients); 422 } while (c != 0); 423 } 424 } 425 } 426 } 427 #endif /* not _WIN32 */ 428 429 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 430 ram_addr_t length, 431 unsigned client); 432 433 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty 434 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); 435 436 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, 437 ram_addr_t start, 438 ram_addr_t length); 439 440 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, 441 ram_addr_t length) 442 { 443 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); 444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); 445 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); 446 } 447 448 449 /* Called with RCU critical section */ 450 static inline 451 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, 452 ram_addr_t start, 453 ram_addr_t length) 454 { 455 ram_addr_t addr; 456 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); 457 uint64_t num_dirty = 0; 458 unsigned long *dest = rb->bmap; 459 460 /* start address and length is aligned at the start of a word? */ 461 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == 462 (start + rb->offset) && 463 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { 464 int k; 465 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); 466 unsigned long * const *src; 467 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; 468 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % 469 DIRTY_MEMORY_BLOCK_SIZE); 470 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 471 472 src = qatomic_rcu_read( 473 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; 474 475 for (k = page; k < page + nr; k++) { 476 if (src[idx][offset]) { 477 unsigned long bits = qatomic_xchg(&src[idx][offset], 0); 478 unsigned long new_dirty; 479 new_dirty = ~dest[k]; 480 dest[k] |= bits; 481 new_dirty &= bits; 482 num_dirty += ctpopl(new_dirty); 483 } 484 485 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 486 offset = 0; 487 idx++; 488 } 489 } 490 491 if (rb->clear_bmap) { 492 /* 493 * Postpone the dirty bitmap clear to the point before we 494 * really send the pages, also we will split the clear 495 * dirty procedure into smaller chunks. 496 */ 497 clear_bmap_set(rb, start >> TARGET_PAGE_BITS, 498 length >> TARGET_PAGE_BITS); 499 } else { 500 /* Slow path - still do that in a huge chunk */ 501 memory_region_clear_dirty_bitmap(rb->mr, start, length); 502 } 503 } else { 504 ram_addr_t offset = rb->offset; 505 506 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { 507 if (cpu_physical_memory_test_and_clear_dirty( 508 start + addr + offset, 509 TARGET_PAGE_SIZE, 510 DIRTY_MEMORY_MIGRATION)) { 511 long k = (start + addr) >> TARGET_PAGE_BITS; 512 if (!test_and_set_bit(k, dest)) { 513 num_dirty++; 514 } 515 } 516 } 517 } 518 519 return num_dirty; 520 } 521 #endif 522 #endif 523