1 /* 2 * Declarations for cpu physical memory functions 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * later. See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * This header is for use by exec.c and memory.c ONLY. Do not include it. 16 * The functions declared here will be removed soon. 17 */ 18 19 #ifndef RAM_ADDR_H 20 #define RAM_ADDR_H 21 22 #ifndef CONFIG_USER_ONLY 23 #include "cpu.h" 24 #include "sysemu/xen.h" 25 #include "sysemu/tcg.h" 26 #include "exec/ramlist.h" 27 #include "exec/ramblock.h" 28 29 extern uint64_t total_dirty_pages; 30 31 /** 32 * clear_bmap_size: calculate clear bitmap size 33 * 34 * @pages: number of guest pages 35 * @shift: guest page number shift 36 * 37 * Returns: number of bits for the clear bitmap 38 */ 39 static inline long clear_bmap_size(uint64_t pages, uint8_t shift) 40 { 41 return DIV_ROUND_UP(pages, 1UL << shift); 42 } 43 44 /** 45 * clear_bmap_set: set clear bitmap for the page range 46 * 47 * @rb: the ramblock to operate on 48 * @start: the start page number 49 * @size: number of pages to set in the bitmap 50 * 51 * Returns: None 52 */ 53 static inline void clear_bmap_set(RAMBlock *rb, uint64_t start, 54 uint64_t npages) 55 { 56 uint8_t shift = rb->clear_bmap_shift; 57 58 bitmap_set_atomic(rb->clear_bmap, start >> shift, 59 clear_bmap_size(npages, shift)); 60 } 61 62 /** 63 * clear_bmap_test_and_clear: test clear bitmap for the page, clear if set 64 * 65 * @rb: the ramblock to operate on 66 * @page: the page number to check 67 * 68 * Returns: true if the bit was set, false otherwise 69 */ 70 static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) 71 { 72 uint8_t shift = rb->clear_bmap_shift; 73 74 return bitmap_test_and_clear_atomic(rb->clear_bmap, page >> shift, 1); 75 } 76 77 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) 78 { 79 return (b && b->host && offset < b->used_length) ? true : false; 80 } 81 82 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) 83 { 84 assert(offset_in_ramblock(block, offset)); 85 return (char *)block->host + offset; 86 } 87 88 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, 89 RAMBlock *rb) 90 { 91 uint64_t host_addr_offset = 92 (uint64_t)(uintptr_t)(host_addr - (void *)rb->host); 93 return host_addr_offset >> TARGET_PAGE_BITS; 94 } 95 96 bool ramblock_is_pmem(RAMBlock *rb); 97 98 long qemu_minrampagesize(void); 99 long qemu_maxrampagesize(void); 100 101 /** 102 * qemu_ram_alloc_from_file, 103 * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing 104 * file or device 105 * 106 * Parameters: 107 * @size: the size in bytes of the ram block 108 * @mr: the memory region where the ram block is 109 * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM, 110 * RAM_NORESERVE. 111 * @mem_path or @fd: specify the backing file or device 112 * @readonly: true to open @path for reading, false for read/write. 113 * @errp: pointer to Error*, to store an error if it happens 114 * 115 * Return: 116 * On success, return a pointer to the ram block. 117 * On failure, return NULL. 118 */ 119 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 120 uint32_t ram_flags, const char *mem_path, 121 bool readonly, Error **errp); 122 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, 123 uint32_t ram_flags, int fd, off_t offset, 124 bool readonly, Error **errp); 125 126 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 127 MemoryRegion *mr, Error **errp); 128 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags, MemoryRegion *mr, 129 Error **errp); 130 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 131 void (*resized)(const char*, 132 uint64_t length, 133 void *host), 134 MemoryRegion *mr, Error **errp); 135 void qemu_ram_free(RAMBlock *block); 136 137 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); 138 139 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length); 140 141 /* Clear whole block of mem */ 142 static inline void qemu_ram_block_writeback(RAMBlock *block) 143 { 144 qemu_ram_msync(block, 0, block->used_length); 145 } 146 147 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) 148 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) 149 150 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, 151 ram_addr_t length, 152 unsigned client) 153 { 154 DirtyMemoryBlocks *blocks; 155 unsigned long end, page; 156 unsigned long idx, offset, base; 157 bool dirty = false; 158 159 assert(client < DIRTY_MEMORY_NUM); 160 161 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 162 page = start >> TARGET_PAGE_BITS; 163 164 WITH_RCU_READ_LOCK_GUARD() { 165 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 166 167 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 168 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 169 base = page - offset; 170 while (page < end) { 171 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 172 unsigned long num = next - base; 173 unsigned long found = find_next_bit(blocks->blocks[idx], 174 num, offset); 175 if (found < num) { 176 dirty = true; 177 break; 178 } 179 180 page = next; 181 idx++; 182 offset = 0; 183 base += DIRTY_MEMORY_BLOCK_SIZE; 184 } 185 } 186 187 return dirty; 188 } 189 190 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, 191 ram_addr_t length, 192 unsigned client) 193 { 194 DirtyMemoryBlocks *blocks; 195 unsigned long end, page; 196 unsigned long idx, offset, base; 197 bool dirty = true; 198 199 assert(client < DIRTY_MEMORY_NUM); 200 201 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 202 page = start >> TARGET_PAGE_BITS; 203 204 RCU_READ_LOCK_GUARD(); 205 206 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 207 208 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 209 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 210 base = page - offset; 211 while (page < end) { 212 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 213 unsigned long num = next - base; 214 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); 215 if (found < num) { 216 dirty = false; 217 break; 218 } 219 220 page = next; 221 idx++; 222 offset = 0; 223 base += DIRTY_MEMORY_BLOCK_SIZE; 224 } 225 226 return dirty; 227 } 228 229 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, 230 unsigned client) 231 { 232 return cpu_physical_memory_get_dirty(addr, 1, client); 233 } 234 235 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) 236 { 237 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); 238 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); 239 bool migration = 240 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); 241 return !(vga && code && migration); 242 } 243 244 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, 245 ram_addr_t length, 246 uint8_t mask) 247 { 248 uint8_t ret = 0; 249 250 if (mask & (1 << DIRTY_MEMORY_VGA) && 251 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { 252 ret |= (1 << DIRTY_MEMORY_VGA); 253 } 254 if (mask & (1 << DIRTY_MEMORY_CODE) && 255 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { 256 ret |= (1 << DIRTY_MEMORY_CODE); 257 } 258 if (mask & (1 << DIRTY_MEMORY_MIGRATION) && 259 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { 260 ret |= (1 << DIRTY_MEMORY_MIGRATION); 261 } 262 return ret; 263 } 264 265 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, 266 unsigned client) 267 { 268 unsigned long page, idx, offset; 269 DirtyMemoryBlocks *blocks; 270 271 assert(client < DIRTY_MEMORY_NUM); 272 273 page = addr >> TARGET_PAGE_BITS; 274 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 275 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 276 277 RCU_READ_LOCK_GUARD(); 278 279 blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); 280 281 set_bit_atomic(offset, blocks->blocks[idx]); 282 } 283 284 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, 285 ram_addr_t length, 286 uint8_t mask) 287 { 288 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; 289 unsigned long end, page; 290 unsigned long idx, offset, base; 291 int i; 292 293 if (!mask && !xen_enabled()) { 294 return; 295 } 296 297 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 298 page = start >> TARGET_PAGE_BITS; 299 300 WITH_RCU_READ_LOCK_GUARD() { 301 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 302 blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); 303 } 304 305 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 306 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 307 base = page - offset; 308 while (page < end) { 309 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 310 311 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { 312 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], 313 offset, next - page); 314 } 315 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { 316 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], 317 offset, next - page); 318 } 319 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { 320 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], 321 offset, next - page); 322 } 323 324 page = next; 325 idx++; 326 offset = 0; 327 base += DIRTY_MEMORY_BLOCK_SIZE; 328 } 329 } 330 331 xen_hvm_modified_memory(start, length); 332 } 333 334 #if !defined(_WIN32) 335 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, 336 ram_addr_t start, 337 ram_addr_t pages) 338 { 339 unsigned long i, j; 340 unsigned long page_number, c; 341 hwaddr addr; 342 ram_addr_t ram_addr; 343 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; 344 unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; 345 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 346 347 /* start address is aligned at the start of a word? */ 348 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && 349 (hpratio == 1)) { 350 unsigned long **blocks[DIRTY_MEMORY_NUM]; 351 unsigned long idx; 352 unsigned long offset; 353 long k; 354 long nr = BITS_TO_LONGS(pages); 355 356 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; 357 offset = BIT_WORD((start >> TARGET_PAGE_BITS) % 358 DIRTY_MEMORY_BLOCK_SIZE); 359 360 WITH_RCU_READ_LOCK_GUARD() { 361 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 362 blocks[i] = 363 qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; 364 } 365 366 for (k = 0; k < nr; k++) { 367 if (bitmap[k]) { 368 unsigned long temp = leul_to_cpu(bitmap[k]); 369 370 qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); 371 372 if (global_dirty_tracking) { 373 qatomic_or( 374 &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], 375 temp); 376 if (unlikely( 377 global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { 378 total_dirty_pages += ctpopl(temp); 379 } 380 } 381 382 if (tcg_enabled()) { 383 qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], 384 temp); 385 } 386 } 387 388 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 389 offset = 0; 390 idx++; 391 } 392 } 393 } 394 395 xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); 396 } else { 397 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; 398 399 if (!global_dirty_tracking) { 400 clients &= ~(1 << DIRTY_MEMORY_MIGRATION); 401 } 402 403 /* 404 * bitmap-traveling is faster than memory-traveling (for addr...) 405 * especially when most of the memory is not dirty. 406 */ 407 for (i = 0; i < len; i++) { 408 if (bitmap[i] != 0) { 409 c = leul_to_cpu(bitmap[i]); 410 if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { 411 total_dirty_pages += ctpopl(c); 412 } 413 do { 414 j = ctzl(c); 415 c &= ~(1ul << j); 416 page_number = (i * HOST_LONG_BITS + j) * hpratio; 417 addr = page_number * TARGET_PAGE_SIZE; 418 ram_addr = start + addr; 419 cpu_physical_memory_set_dirty_range(ram_addr, 420 TARGET_PAGE_SIZE * hpratio, clients); 421 } while (c != 0); 422 } 423 } 424 } 425 } 426 #endif /* not _WIN32 */ 427 428 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 429 ram_addr_t length, 430 unsigned client); 431 432 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty 433 (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); 434 435 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, 436 ram_addr_t start, 437 ram_addr_t length); 438 439 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, 440 ram_addr_t length) 441 { 442 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); 443 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); 444 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); 445 } 446 447 448 /* Called with RCU critical section */ 449 static inline 450 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, 451 ram_addr_t start, 452 ram_addr_t length) 453 { 454 ram_addr_t addr; 455 unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); 456 uint64_t num_dirty = 0; 457 unsigned long *dest = rb->bmap; 458 459 /* start address and length is aligned at the start of a word? */ 460 if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == 461 (start + rb->offset) && 462 !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { 463 int k; 464 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); 465 unsigned long * const *src; 466 unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; 467 unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % 468 DIRTY_MEMORY_BLOCK_SIZE); 469 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 470 471 src = qatomic_rcu_read( 472 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; 473 474 for (k = page; k < page + nr; k++) { 475 if (src[idx][offset]) { 476 unsigned long bits = qatomic_xchg(&src[idx][offset], 0); 477 unsigned long new_dirty; 478 new_dirty = ~dest[k]; 479 dest[k] |= bits; 480 new_dirty &= bits; 481 num_dirty += ctpopl(new_dirty); 482 } 483 484 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 485 offset = 0; 486 idx++; 487 } 488 } 489 490 if (rb->clear_bmap) { 491 /* 492 * Postpone the dirty bitmap clear to the point before we 493 * really send the pages, also we will split the clear 494 * dirty procedure into smaller chunks. 495 */ 496 clear_bmap_set(rb, start >> TARGET_PAGE_BITS, 497 length >> TARGET_PAGE_BITS); 498 } else { 499 /* Slow path - still do that in a huge chunk */ 500 memory_region_clear_dirty_bitmap(rb->mr, start, length); 501 } 502 } else { 503 ram_addr_t offset = rb->offset; 504 505 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { 506 if (cpu_physical_memory_test_and_clear_dirty( 507 start + addr + offset, 508 TARGET_PAGE_SIZE, 509 DIRTY_MEMORY_MIGRATION)) { 510 long k = (start + addr) >> TARGET_PAGE_BITS; 511 if (!test_and_set_bit(k, dest)) { 512 num_dirty++; 513 } 514 } 515 } 516 } 517 518 return num_dirty; 519 } 520 #endif 521 #endif 522