1 /* 2 * QEMU ram block attributes 3 * 4 * Copyright Intel 5 * 6 * Author: 7 * Chenyi Qiang <chenyi.qiang@intel.com> 8 * 9 * SPDX-License-Identifier: GPL-2.0-or-later 10 */ 11 12 #include "qemu/osdep.h" 13 #include "qemu/error-report.h" 14 #include "system/ramblock.h" 15 #include "trace.h" 16 17 OBJECT_DEFINE_SIMPLE_TYPE_WITH_INTERFACES(RamBlockAttributes, 18 ram_block_attributes, 19 RAM_BLOCK_ATTRIBUTES, 20 OBJECT, 21 { TYPE_RAM_DISCARD_MANAGER }, 22 { }) 23 24 static size_t 25 ram_block_attributes_get_block_size(void) 26 { 27 /* 28 * Because page conversion could be manipulated in the size of at least 4K 29 * or 4K aligned, Use the host page size as the granularity to track the 30 * memory attribute. 31 */ 32 return qemu_real_host_page_size(); 33 } 34 35 36 static bool 37 ram_block_attributes_rdm_is_populated(const RamDiscardManager *rdm, 38 const MemoryRegionSection *section) 39 { 40 const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 41 const size_t block_size = ram_block_attributes_get_block_size(); 42 const uint64_t first_bit = section->offset_within_region / block_size; 43 const uint64_t last_bit = 44 first_bit + int128_get64(section->size) / block_size - 1; 45 unsigned long first_discarded_bit; 46 47 first_discarded_bit = find_next_zero_bit(attr->bitmap, last_bit + 1, 48 first_bit); 49 return first_discarded_bit > last_bit; 50 } 51 52 typedef int (*ram_block_attributes_section_cb)(MemoryRegionSection *s, 53 void *arg); 54 55 static int 56 ram_block_attributes_notify_populate_cb(MemoryRegionSection *section, 57 void *arg) 58 { 59 RamDiscardListener *rdl = arg; 60 61 return rdl->notify_populate(rdl, section); 62 } 63 64 static int 65 ram_block_attributes_notify_discard_cb(MemoryRegionSection *section, 66 void *arg) 67 { 68 RamDiscardListener *rdl = arg; 69 70 rdl->notify_discard(rdl, section); 71 return 0; 72 } 73 74 static int 75 ram_block_attributes_for_each_populated_section(const RamBlockAttributes *attr, 76 MemoryRegionSection *section, 77 void *arg, 78 ram_block_attributes_section_cb cb) 79 { 80 unsigned long first_bit, last_bit; 81 uint64_t offset, size; 82 const size_t block_size = ram_block_attributes_get_block_size(); 83 int ret = 0; 84 85 first_bit = section->offset_within_region / block_size; 86 first_bit = find_next_bit(attr->bitmap, attr->bitmap_size, 87 first_bit); 88 89 while (first_bit < attr->bitmap_size) { 90 MemoryRegionSection tmp = *section; 91 92 offset = first_bit * block_size; 93 last_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size, 94 first_bit + 1) - 1; 95 size = (last_bit - first_bit + 1) * block_size; 96 97 if (!memory_region_section_intersect_range(&tmp, offset, size)) { 98 break; 99 } 100 101 ret = cb(&tmp, arg); 102 if (ret) { 103 error_report("%s: Failed to notify RAM discard listener: %s", 104 __func__, strerror(-ret)); 105 break; 106 } 107 108 first_bit = find_next_bit(attr->bitmap, attr->bitmap_size, 109 last_bit + 2); 110 } 111 112 return ret; 113 } 114 115 static int 116 ram_block_attributes_for_each_discarded_section(const RamBlockAttributes *attr, 117 MemoryRegionSection *section, 118 void *arg, 119 ram_block_attributes_section_cb cb) 120 { 121 unsigned long first_bit, last_bit; 122 uint64_t offset, size; 123 const size_t block_size = ram_block_attributes_get_block_size(); 124 int ret = 0; 125 126 first_bit = section->offset_within_region / block_size; 127 first_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size, 128 first_bit); 129 130 while (first_bit < attr->bitmap_size) { 131 MemoryRegionSection tmp = *section; 132 133 offset = first_bit * block_size; 134 last_bit = find_next_bit(attr->bitmap, attr->bitmap_size, 135 first_bit + 1) - 1; 136 size = (last_bit - first_bit + 1) * block_size; 137 138 if (!memory_region_section_intersect_range(&tmp, offset, size)) { 139 break; 140 } 141 142 ret = cb(&tmp, arg); 143 if (ret) { 144 error_report("%s: Failed to notify RAM discard listener: %s", 145 __func__, strerror(-ret)); 146 break; 147 } 148 149 first_bit = find_next_zero_bit(attr->bitmap, 150 attr->bitmap_size, 151 last_bit + 2); 152 } 153 154 return ret; 155 } 156 157 static uint64_t 158 ram_block_attributes_rdm_get_min_granularity(const RamDiscardManager *rdm, 159 const MemoryRegion *mr) 160 { 161 const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 162 163 g_assert(mr == attr->ram_block->mr); 164 return ram_block_attributes_get_block_size(); 165 } 166 167 static void 168 ram_block_attributes_rdm_register_listener(RamDiscardManager *rdm, 169 RamDiscardListener *rdl, 170 MemoryRegionSection *section) 171 { 172 RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 173 int ret; 174 175 g_assert(section->mr == attr->ram_block->mr); 176 rdl->section = memory_region_section_new_copy(section); 177 178 QLIST_INSERT_HEAD(&attr->rdl_list, rdl, next); 179 180 ret = ram_block_attributes_for_each_populated_section(attr, section, rdl, 181 ram_block_attributes_notify_populate_cb); 182 if (ret) { 183 error_report("%s: Failed to register RAM discard listener: %s", 184 __func__, strerror(-ret)); 185 exit(1); 186 } 187 } 188 189 static void 190 ram_block_attributes_rdm_unregister_listener(RamDiscardManager *rdm, 191 RamDiscardListener *rdl) 192 { 193 RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 194 int ret; 195 196 g_assert(rdl->section); 197 g_assert(rdl->section->mr == attr->ram_block->mr); 198 199 if (rdl->double_discard_supported) { 200 rdl->notify_discard(rdl, rdl->section); 201 } else { 202 ret = ram_block_attributes_for_each_populated_section(attr, 203 rdl->section, rdl, ram_block_attributes_notify_discard_cb); 204 if (ret) { 205 error_report("%s: Failed to unregister RAM discard listener: %s", 206 __func__, strerror(-ret)); 207 exit(1); 208 } 209 } 210 211 memory_region_section_free_copy(rdl->section); 212 rdl->section = NULL; 213 QLIST_REMOVE(rdl, next); 214 } 215 216 typedef struct RamBlockAttributesReplayData { 217 ReplayRamDiscardState fn; 218 void *opaque; 219 } RamBlockAttributesReplayData; 220 221 static int ram_block_attributes_rdm_replay_cb(MemoryRegionSection *section, 222 void *arg) 223 { 224 RamBlockAttributesReplayData *data = arg; 225 226 return data->fn(section, data->opaque); 227 } 228 229 static int 230 ram_block_attributes_rdm_replay_populated(const RamDiscardManager *rdm, 231 MemoryRegionSection *section, 232 ReplayRamDiscardState replay_fn, 233 void *opaque) 234 { 235 RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 236 RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque }; 237 238 g_assert(section->mr == attr->ram_block->mr); 239 return ram_block_attributes_for_each_populated_section(attr, section, &data, 240 ram_block_attributes_rdm_replay_cb); 241 } 242 243 static int 244 ram_block_attributes_rdm_replay_discarded(const RamDiscardManager *rdm, 245 MemoryRegionSection *section, 246 ReplayRamDiscardState replay_fn, 247 void *opaque) 248 { 249 RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm); 250 RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque }; 251 252 g_assert(section->mr == attr->ram_block->mr); 253 return ram_block_attributes_for_each_discarded_section(attr, section, &data, 254 ram_block_attributes_rdm_replay_cb); 255 } 256 257 static bool 258 ram_block_attributes_is_valid_range(RamBlockAttributes *attr, uint64_t offset, 259 uint64_t size) 260 { 261 MemoryRegion *mr = attr->ram_block->mr; 262 263 g_assert(mr); 264 265 uint64_t region_size = memory_region_size(mr); 266 const size_t block_size = ram_block_attributes_get_block_size(); 267 268 if (!QEMU_IS_ALIGNED(offset, block_size) || 269 !QEMU_IS_ALIGNED(size, block_size)) { 270 return false; 271 } 272 if (offset + size <= offset) { 273 return false; 274 } 275 if (offset + size > region_size) { 276 return false; 277 } 278 return true; 279 } 280 281 static void ram_block_attributes_notify_discard(RamBlockAttributes *attr, 282 uint64_t offset, 283 uint64_t size) 284 { 285 RamDiscardListener *rdl; 286 287 QLIST_FOREACH(rdl, &attr->rdl_list, next) { 288 MemoryRegionSection tmp = *rdl->section; 289 290 if (!memory_region_section_intersect_range(&tmp, offset, size)) { 291 continue; 292 } 293 rdl->notify_discard(rdl, &tmp); 294 } 295 } 296 297 static int 298 ram_block_attributes_notify_populate(RamBlockAttributes *attr, 299 uint64_t offset, uint64_t size) 300 { 301 RamDiscardListener *rdl; 302 int ret = 0; 303 304 QLIST_FOREACH(rdl, &attr->rdl_list, next) { 305 MemoryRegionSection tmp = *rdl->section; 306 307 if (!memory_region_section_intersect_range(&tmp, offset, size)) { 308 continue; 309 } 310 ret = rdl->notify_populate(rdl, &tmp); 311 if (ret) { 312 break; 313 } 314 } 315 316 return ret; 317 } 318 319 int ram_block_attributes_state_change(RamBlockAttributes *attr, 320 uint64_t offset, uint64_t size, 321 bool to_discard) 322 { 323 const size_t block_size = ram_block_attributes_get_block_size(); 324 const unsigned long first_bit = offset / block_size; 325 const unsigned long nbits = size / block_size; 326 const unsigned long last_bit = first_bit + nbits - 1; 327 const bool is_discarded = find_next_bit(attr->bitmap, attr->bitmap_size, 328 first_bit) > last_bit; 329 const bool is_populated = find_next_zero_bit(attr->bitmap, 330 attr->bitmap_size, first_bit) > last_bit; 331 unsigned long bit; 332 int ret = 0; 333 334 if (!ram_block_attributes_is_valid_range(attr, offset, size)) { 335 error_report("%s, invalid range: offset 0x%" PRIx64 ", size " 336 "0x%" PRIx64, __func__, offset, size); 337 return -EINVAL; 338 } 339 340 trace_ram_block_attributes_state_change(offset, size, 341 is_discarded ? "discarded" : 342 is_populated ? "populated" : 343 "mixture", 344 to_discard ? "discarded" : 345 "populated"); 346 if (to_discard) { 347 if (is_discarded) { 348 /* Already private */ 349 } else if (is_populated) { 350 /* Completely shared */ 351 bitmap_clear(attr->bitmap, first_bit, nbits); 352 ram_block_attributes_notify_discard(attr, offset, size); 353 } else { 354 /* Unexpected mixture: process individual blocks */ 355 for (bit = first_bit; bit < first_bit + nbits; bit++) { 356 if (!test_bit(bit, attr->bitmap)) { 357 continue; 358 } 359 clear_bit(bit, attr->bitmap); 360 ram_block_attributes_notify_discard(attr, bit * block_size, 361 block_size); 362 } 363 } 364 } else { 365 if (is_populated) { 366 /* Already shared */ 367 } else if (is_discarded) { 368 /* Completely private */ 369 bitmap_set(attr->bitmap, first_bit, nbits); 370 ret = ram_block_attributes_notify_populate(attr, offset, size); 371 } else { 372 /* Unexpected mixture: process individual blocks */ 373 for (bit = first_bit; bit < first_bit + nbits; bit++) { 374 if (test_bit(bit, attr->bitmap)) { 375 continue; 376 } 377 set_bit(bit, attr->bitmap); 378 ret = ram_block_attributes_notify_populate(attr, 379 bit * block_size, 380 block_size); 381 if (ret) { 382 break; 383 } 384 } 385 } 386 } 387 388 return ret; 389 } 390 391 RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block) 392 { 393 const int block_size = qemu_real_host_page_size(); 394 RamBlockAttributes *attr; 395 MemoryRegion *mr = ram_block->mr; 396 397 attr = RAM_BLOCK_ATTRIBUTES(object_new(TYPE_RAM_BLOCK_ATTRIBUTES)); 398 399 attr->ram_block = ram_block; 400 if (memory_region_set_ram_discard_manager(mr, RAM_DISCARD_MANAGER(attr))) { 401 object_unref(OBJECT(attr)); 402 return NULL; 403 } 404 attr->bitmap_size = 405 ROUND_UP(int128_get64(mr->size), block_size) / block_size; 406 attr->bitmap = bitmap_new(attr->bitmap_size); 407 408 return attr; 409 } 410 411 void ram_block_attributes_destroy(RamBlockAttributes *attr) 412 { 413 g_assert(attr); 414 415 g_free(attr->bitmap); 416 memory_region_set_ram_discard_manager(attr->ram_block->mr, NULL); 417 object_unref(OBJECT(attr)); 418 } 419 420 static void ram_block_attributes_init(Object *obj) 421 { 422 RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(obj); 423 424 QLIST_INIT(&attr->rdl_list); 425 } 426 427 static void ram_block_attributes_finalize(Object *obj) 428 { 429 } 430 431 static void ram_block_attributes_class_init(ObjectClass *klass, 432 const void *data) 433 { 434 RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); 435 436 rdmc->get_min_granularity = ram_block_attributes_rdm_get_min_granularity; 437 rdmc->register_listener = ram_block_attributes_rdm_register_listener; 438 rdmc->unregister_listener = ram_block_attributes_rdm_unregister_listener; 439 rdmc->is_populated = ram_block_attributes_rdm_is_populated; 440 rdmc->replay_populated = ram_block_attributes_rdm_replay_populated; 441 rdmc->replay_discarded = ram_block_attributes_rdm_replay_discarded; 442 } 443