1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * access_tracking_perf_test 4 * 5 * Copyright (C) 2021, Google, Inc. 6 * 7 * This test measures the performance effects of KVM's access tracking. 8 * Access tracking is driven by the MMU notifiers test_young, clear_young, and 9 * clear_flush_young. These notifiers do not have a direct userspace API, 10 * however the clear_young notifier can be triggered by marking a pages as idle 11 * in /sys/kernel/mm/page_idle/bitmap. This test leverages that mechanism to 12 * enable access tracking on guest memory. 13 * 14 * To measure performance this test runs a VM with a configurable number of 15 * vCPUs that each touch every page in disjoint regions of memory. Performance 16 * is measured in the time it takes all vCPUs to finish touching their 17 * predefined region. 18 * 19 * Note that a deterministic correctness test of access tracking is not possible 20 * by using page_idle as it exists today. This is for a few reasons: 21 * 22 * 1. page_idle only issues clear_young notifiers, which lack a TLB flush. This 23 * means subsequent guest accesses are not guaranteed to see page table 24 * updates made by KVM until some time in the future. 25 * 26 * 2. page_idle only operates on LRU pages. Newly allocated pages are not 27 * immediately allocated to LRU lists. Instead they are held in a "pagevec", 28 * which is drained to LRU lists some time in the future. There is no 29 * userspace API to force this drain to occur. 30 * 31 * These limitations are worked around in this test by using a large enough 32 * region of memory for each vCPU such that the number of translations cached in 33 * the TLB and the number of pages held in pagevecs are a small fraction of the 34 * overall workload. And if either of those conditions are not true this test 35 * will fail rather than silently passing. 36 */ 37 #include <inttypes.h> 38 #include <limits.h> 39 #include <pthread.h> 40 #include <sys/mman.h> 41 #include <sys/types.h> 42 #include <sys/stat.h> 43 44 #include "kvm_util.h" 45 #include "test_util.h" 46 #include "perf_test_util.h" 47 #include "guest_modes.h" 48 49 /* Global variable used to synchronize all of the vCPU threads. */ 50 static int iteration; 51 52 /* Defines what vCPU threads should do during a given iteration. */ 53 static enum { 54 /* Run the vCPU to access all its memory. */ 55 ITERATION_ACCESS_MEMORY, 56 /* Mark the vCPU's memory idle in page_idle. */ 57 ITERATION_MARK_IDLE, 58 } iteration_work; 59 60 /* Set to true when vCPU threads should exit. */ 61 static bool done; 62 63 /* The iteration that was last completed by each vCPU. */ 64 static int vcpu_last_completed_iteration[KVM_MAX_VCPUS]; 65 66 /* Whether to overlap the regions of memory vCPUs access. */ 67 static bool overlap_memory_access; 68 69 struct test_params { 70 /* The backing source for the region of memory. */ 71 enum vm_mem_backing_src_type backing_src; 72 73 /* The amount of memory to allocate for each vCPU. */ 74 uint64_t vcpu_memory_bytes; 75 76 /* The number of vCPUs to create in the VM. */ 77 int nr_vcpus; 78 }; 79 80 static uint64_t pread_uint64(int fd, const char *filename, uint64_t index) 81 { 82 uint64_t value; 83 off_t offset = index * sizeof(value); 84 85 TEST_ASSERT(pread(fd, &value, sizeof(value), offset) == sizeof(value), 86 "pread from %s offset 0x%" PRIx64 " failed!", 87 filename, offset); 88 89 return value; 90 91 } 92 93 #define PAGEMAP_PRESENT (1ULL << 63) 94 #define PAGEMAP_PFN_MASK ((1ULL << 55) - 1) 95 96 static uint64_t lookup_pfn(int pagemap_fd, struct kvm_vm *vm, uint64_t gva) 97 { 98 uint64_t hva = (uint64_t) addr_gva2hva(vm, gva); 99 uint64_t entry; 100 uint64_t pfn; 101 102 entry = pread_uint64(pagemap_fd, "pagemap", hva / getpagesize()); 103 if (!(entry & PAGEMAP_PRESENT)) 104 return 0; 105 106 pfn = entry & PAGEMAP_PFN_MASK; 107 __TEST_REQUIRE(pfn, "Looking up PFNs requires CAP_SYS_ADMIN"); 108 109 return pfn; 110 } 111 112 static bool is_page_idle(int page_idle_fd, uint64_t pfn) 113 { 114 uint64_t bits = pread_uint64(page_idle_fd, "page_idle", pfn / 64); 115 116 return !!((bits >> (pfn % 64)) & 1); 117 } 118 119 static void mark_page_idle(int page_idle_fd, uint64_t pfn) 120 { 121 uint64_t bits = 1ULL << (pfn % 64); 122 123 TEST_ASSERT(pwrite(page_idle_fd, &bits, 8, 8 * (pfn / 64)) == 8, 124 "Set page_idle bits for PFN 0x%" PRIx64, pfn); 125 } 126 127 static void mark_vcpu_memory_idle(struct kvm_vm *vm, 128 struct perf_test_vcpu_args *vcpu_args) 129 { 130 int vcpu_idx = vcpu_args->vcpu_idx; 131 uint64_t base_gva = vcpu_args->gva; 132 uint64_t pages = vcpu_args->pages; 133 uint64_t page; 134 uint64_t still_idle = 0; 135 uint64_t no_pfn = 0; 136 int page_idle_fd; 137 int pagemap_fd; 138 139 /* If vCPUs are using an overlapping region, let vCPU 0 mark it idle. */ 140 if (overlap_memory_access && vcpu_idx) 141 return; 142 143 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR); 144 TEST_ASSERT(page_idle_fd > 0, "Failed to open page_idle."); 145 146 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 147 TEST_ASSERT(pagemap_fd > 0, "Failed to open pagemap."); 148 149 for (page = 0; page < pages; page++) { 150 uint64_t gva = base_gva + page * perf_test_args.guest_page_size; 151 uint64_t pfn = lookup_pfn(pagemap_fd, vm, gva); 152 153 if (!pfn) { 154 no_pfn++; 155 continue; 156 } 157 158 if (is_page_idle(page_idle_fd, pfn)) { 159 still_idle++; 160 continue; 161 } 162 163 mark_page_idle(page_idle_fd, pfn); 164 } 165 166 /* 167 * Assumption: Less than 1% of pages are going to be swapped out from 168 * under us during this test. 169 */ 170 TEST_ASSERT(no_pfn < pages / 100, 171 "vCPU %d: No PFN for %" PRIu64 " out of %" PRIu64 " pages.", 172 vcpu_idx, no_pfn, pages); 173 174 /* 175 * Test that at least 90% of memory has been marked idle (the rest might 176 * not be marked idle because the pages have not yet made it to an LRU 177 * list or the translations are still cached in the TLB). 90% is 178 * arbitrary; high enough that we ensure most memory access went through 179 * access tracking but low enough as to not make the test too brittle 180 * over time and across architectures. 181 */ 182 TEST_ASSERT(still_idle < pages / 10, 183 "vCPU%d: Too many pages still idle (%"PRIu64 " out of %" 184 PRIu64 ").\n", 185 vcpu_idx, still_idle, pages); 186 187 close(page_idle_fd); 188 close(pagemap_fd); 189 } 190 191 static void assert_ucall(struct kvm_vcpu *vcpu, uint64_t expected_ucall) 192 { 193 struct ucall uc; 194 uint64_t actual_ucall = get_ucall(vcpu, &uc); 195 196 TEST_ASSERT(expected_ucall == actual_ucall, 197 "Guest exited unexpectedly (expected ucall %" PRIu64 198 ", got %" PRIu64 ")", 199 expected_ucall, actual_ucall); 200 } 201 202 static bool spin_wait_for_next_iteration(int *current_iteration) 203 { 204 int last_iteration = *current_iteration; 205 206 do { 207 if (READ_ONCE(done)) 208 return false; 209 210 *current_iteration = READ_ONCE(iteration); 211 } while (last_iteration == *current_iteration); 212 213 return true; 214 } 215 216 static void vcpu_thread_main(struct perf_test_vcpu_args *vcpu_args) 217 { 218 struct kvm_vcpu *vcpu = vcpu_args->vcpu; 219 struct kvm_vm *vm = perf_test_args.vm; 220 int vcpu_idx = vcpu_args->vcpu_idx; 221 int current_iteration = 0; 222 223 while (spin_wait_for_next_iteration(¤t_iteration)) { 224 switch (READ_ONCE(iteration_work)) { 225 case ITERATION_ACCESS_MEMORY: 226 vcpu_run(vcpu); 227 assert_ucall(vcpu, UCALL_SYNC); 228 break; 229 case ITERATION_MARK_IDLE: 230 mark_vcpu_memory_idle(vm, vcpu_args); 231 break; 232 }; 233 234 vcpu_last_completed_iteration[vcpu_idx] = current_iteration; 235 } 236 } 237 238 static void spin_wait_for_vcpu(int vcpu_idx, int target_iteration) 239 { 240 while (READ_ONCE(vcpu_last_completed_iteration[vcpu_idx]) != 241 target_iteration) { 242 continue; 243 } 244 } 245 246 /* The type of memory accesses to perform in the VM. */ 247 enum access_type { 248 ACCESS_READ, 249 ACCESS_WRITE, 250 }; 251 252 static void run_iteration(struct kvm_vm *vm, int nr_vcpus, const char *description) 253 { 254 struct timespec ts_start; 255 struct timespec ts_elapsed; 256 int next_iteration, i; 257 258 /* Kick off the vCPUs by incrementing iteration. */ 259 next_iteration = ++iteration; 260 261 clock_gettime(CLOCK_MONOTONIC, &ts_start); 262 263 /* Wait for all vCPUs to finish the iteration. */ 264 for (i = 0; i < nr_vcpus; i++) 265 spin_wait_for_vcpu(i, next_iteration); 266 267 ts_elapsed = timespec_elapsed(ts_start); 268 pr_info("%-30s: %ld.%09lds\n", 269 description, ts_elapsed.tv_sec, ts_elapsed.tv_nsec); 270 } 271 272 static void access_memory(struct kvm_vm *vm, int nr_vcpus, 273 enum access_type access, const char *description) 274 { 275 perf_test_set_wr_fract(vm, (access == ACCESS_READ) ? INT_MAX : 1); 276 iteration_work = ITERATION_ACCESS_MEMORY; 277 run_iteration(vm, nr_vcpus, description); 278 } 279 280 static void mark_memory_idle(struct kvm_vm *vm, int nr_vcpus) 281 { 282 /* 283 * Even though this parallelizes the work across vCPUs, this is still a 284 * very slow operation because page_idle forces the test to mark one pfn 285 * at a time and the clear_young notifier serializes on the KVM MMU 286 * lock. 287 */ 288 pr_debug("Marking VM memory idle (slow)...\n"); 289 iteration_work = ITERATION_MARK_IDLE; 290 run_iteration(vm, nr_vcpus, "Mark memory idle"); 291 } 292 293 static void run_test(enum vm_guest_mode mode, void *arg) 294 { 295 struct test_params *params = arg; 296 struct kvm_vm *vm; 297 int nr_vcpus = params->nr_vcpus; 298 299 vm = perf_test_create_vm(mode, nr_vcpus, params->vcpu_memory_bytes, 1, 300 params->backing_src, !overlap_memory_access); 301 302 perf_test_start_vcpu_threads(nr_vcpus, vcpu_thread_main); 303 304 pr_info("\n"); 305 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Populating memory"); 306 307 /* As a control, read and write to the populated memory first. */ 308 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to populated memory"); 309 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from populated memory"); 310 311 /* Repeat on memory that has been marked as idle. */ 312 mark_memory_idle(vm, nr_vcpus); 313 access_memory(vm, nr_vcpus, ACCESS_WRITE, "Writing to idle memory"); 314 mark_memory_idle(vm, nr_vcpus); 315 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory"); 316 317 /* Set done to signal the vCPU threads to exit */ 318 done = true; 319 320 perf_test_join_vcpu_threads(nr_vcpus); 321 perf_test_destroy_vm(vm); 322 } 323 324 static void help(char *name) 325 { 326 puts(""); 327 printf("usage: %s [-h] [-m mode] [-b vcpu_bytes] [-v vcpus] [-o] [-s mem_type]\n", 328 name); 329 puts(""); 330 printf(" -h: Display this help message."); 331 guest_modes_help(); 332 printf(" -b: specify the size of the memory region which should be\n" 333 " dirtied by each vCPU. e.g. 10M or 3G.\n" 334 " (default: 1G)\n"); 335 printf(" -v: specify the number of vCPUs to run.\n"); 336 printf(" -o: Overlap guest memory accesses instead of partitioning\n" 337 " them into a separate region of memory for each vCPU.\n"); 338 backing_src_help("-s"); 339 puts(""); 340 exit(0); 341 } 342 343 int main(int argc, char *argv[]) 344 { 345 struct test_params params = { 346 .backing_src = DEFAULT_VM_MEM_SRC, 347 .vcpu_memory_bytes = DEFAULT_PER_VCPU_MEM_SIZE, 348 .nr_vcpus = 1, 349 }; 350 int page_idle_fd; 351 int opt; 352 353 guest_modes_append_default(); 354 355 while ((opt = getopt(argc, argv, "hm:b:v:os:")) != -1) { 356 switch (opt) { 357 case 'm': 358 guest_modes_cmdline(optarg); 359 break; 360 case 'b': 361 params.vcpu_memory_bytes = parse_size(optarg); 362 break; 363 case 'v': 364 params.nr_vcpus = atoi(optarg); 365 break; 366 case 'o': 367 overlap_memory_access = true; 368 break; 369 case 's': 370 params.backing_src = parse_backing_src_type(optarg); 371 break; 372 case 'h': 373 default: 374 help(argv[0]); 375 break; 376 } 377 } 378 379 page_idle_fd = open("/sys/kernel/mm/page_idle/bitmap", O_RDWR); 380 __TEST_REQUIRE(page_idle_fd >= 0, 381 "CONFIG_IDLE_PAGE_TRACKING is not enabled"); 382 close(page_idle_fd); 383 384 for_each_guest_mode(run_test, ¶ms); 385 386 return 0; 387 } 388