1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM page table test 4 * 5 * Copyright (C) 2021, Huawei, Inc. 6 * 7 * Make sure that THP has been enabled or enough HUGETLB pages with specific 8 * page size have been pre-allocated on your system, if you are planning to 9 * use hugepages to back the guest memory for testing. 10 */ 11 12 #define _GNU_SOURCE /* for program_invocation_name */ 13 14 #include <stdio.h> 15 #include <stdlib.h> 16 #include <time.h> 17 #include <pthread.h> 18 #include <semaphore.h> 19 20 #include "test_util.h" 21 #include "kvm_util.h" 22 #include "processor.h" 23 #include "guest_modes.h" 24 25 #define TEST_MEM_SLOT_INDEX 1 26 27 /* Default size(1GB) of the memory for testing */ 28 #define DEFAULT_TEST_MEM_SIZE (1 << 30) 29 30 /* Default guest test virtual memory offset */ 31 #define DEFAULT_GUEST_TEST_MEM 0xc0000000 32 33 /* Different guest memory accessing stages */ 34 enum test_stage { 35 KVM_BEFORE_MAPPINGS, 36 KVM_CREATE_MAPPINGS, 37 KVM_UPDATE_MAPPINGS, 38 KVM_ADJUST_MAPPINGS, 39 NUM_TEST_STAGES, 40 }; 41 42 static const char * const test_stage_string[] = { 43 "KVM_BEFORE_MAPPINGS", 44 "KVM_CREATE_MAPPINGS", 45 "KVM_UPDATE_MAPPINGS", 46 "KVM_ADJUST_MAPPINGS", 47 }; 48 49 struct test_args { 50 struct kvm_vm *vm; 51 uint64_t guest_test_virt_mem; 52 uint64_t host_page_size; 53 uint64_t host_num_pages; 54 uint64_t large_page_size; 55 uint64_t large_num_pages; 56 uint64_t host_pages_per_lpage; 57 enum vm_mem_backing_src_type src_type; 58 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; 59 }; 60 61 /* 62 * Guest variables. Use addr_gva2hva() if these variables need 63 * to be changed in host. 64 */ 65 static enum test_stage guest_test_stage; 66 67 /* Host variables */ 68 static uint32_t nr_vcpus = 1; 69 static struct test_args test_args; 70 static enum test_stage *current_stage; 71 static bool host_quit; 72 73 /* Whether the test stage is updated, or completed */ 74 static sem_t test_stage_updated; 75 static sem_t test_stage_completed; 76 77 /* 78 * Guest physical memory offset of the testing memory slot. 79 * This will be set to the topmost valid physical address minus 80 * the test memory size. 81 */ 82 static uint64_t guest_test_phys_mem; 83 84 /* 85 * Guest virtual memory offset of the testing memory slot. 86 * Must not conflict with identity mapped test code. 87 */ 88 static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM; 89 90 static void guest_code(bool do_write) 91 { 92 struct test_args *p = &test_args; 93 enum test_stage *current_stage = &guest_test_stage; 94 uint64_t addr; 95 int i, j; 96 97 while (true) { 98 addr = p->guest_test_virt_mem; 99 100 switch (READ_ONCE(*current_stage)) { 101 /* 102 * All vCPU threads will be started in this stage, 103 * where guest code of each vCPU will do nothing. 104 */ 105 case KVM_BEFORE_MAPPINGS: 106 break; 107 108 /* 109 * Before dirty logging, vCPUs concurrently access the first 110 * 8 bytes of each page (host page/large page) within the same 111 * memory region with different accessing types (read/write). 112 * Then KVM will create normal page mappings or huge block 113 * mappings for them. 114 */ 115 case KVM_CREATE_MAPPINGS: 116 for (i = 0; i < p->large_num_pages; i++) { 117 if (do_write) 118 *(uint64_t *)addr = 0x0123456789ABCDEF; 119 else 120 READ_ONCE(*(uint64_t *)addr); 121 122 addr += p->large_page_size; 123 } 124 break; 125 126 /* 127 * During dirty logging, KVM will only update attributes of the 128 * normal page mappings from RO to RW if memory backing src type 129 * is anonymous. In other cases, KVM will split the huge block 130 * mappings into normal page mappings if memory backing src type 131 * is THP or HUGETLB. 132 */ 133 case KVM_UPDATE_MAPPINGS: 134 if (p->src_type == VM_MEM_SRC_ANONYMOUS) { 135 for (i = 0; i < p->host_num_pages; i++) { 136 *(uint64_t *)addr = 0x0123456789ABCDEF; 137 addr += p->host_page_size; 138 } 139 break; 140 } 141 142 for (i = 0; i < p->large_num_pages; i++) { 143 /* 144 * Write to the first host page in each large 145 * page region, and triger break of large pages. 146 */ 147 *(uint64_t *)addr = 0x0123456789ABCDEF; 148 149 /* 150 * Access the middle host pages in each large 151 * page region. Since dirty logging is enabled, 152 * this will create new mappings at the smallest 153 * granularity. 154 */ 155 addr += p->large_page_size / 2; 156 for (j = 0; j < p->host_pages_per_lpage / 2; j++) { 157 READ_ONCE(*(uint64_t *)addr); 158 addr += p->host_page_size; 159 } 160 } 161 break; 162 163 /* 164 * After dirty logging is stopped, vCPUs concurrently read 165 * from every single host page. Then KVM will coalesce the 166 * split page mappings back to block mappings. And a TLB 167 * conflict abort could occur here if TLB entries of the 168 * page mappings are not fully invalidated. 169 */ 170 case KVM_ADJUST_MAPPINGS: 171 for (i = 0; i < p->host_num_pages; i++) { 172 READ_ONCE(*(uint64_t *)addr); 173 addr += p->host_page_size; 174 } 175 break; 176 177 default: 178 GUEST_ASSERT(0); 179 } 180 181 GUEST_SYNC(1); 182 } 183 } 184 185 static void *vcpu_worker(void *data) 186 { 187 struct kvm_vcpu *vcpu = data; 188 bool do_write = !(vcpu->id % 2); 189 struct timespec start; 190 struct timespec ts_diff; 191 enum test_stage stage; 192 int ret; 193 194 vcpu_args_set(vcpu, 1, do_write); 195 196 while (!READ_ONCE(host_quit)) { 197 ret = sem_wait(&test_stage_updated); 198 TEST_ASSERT(ret == 0, "Error in sem_wait"); 199 200 if (READ_ONCE(host_quit)) 201 return NULL; 202 203 clock_gettime(CLOCK_MONOTONIC_RAW, &start); 204 ret = _vcpu_run(vcpu); 205 ts_diff = timespec_elapsed(start); 206 207 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret); 208 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC, 209 "Invalid guest sync status: exit_reason=%s\n", 210 exit_reason_str(vcpu->run->exit_reason)); 211 212 pr_debug("Got sync event from vCPU %d\n", vcpu->id); 213 stage = READ_ONCE(*current_stage); 214 215 /* 216 * Here we can know the execution time of every 217 * single vcpu running in different test stages. 218 */ 219 pr_debug("vCPU %d has completed stage %s\n" 220 "execution time is: %ld.%.9lds\n\n", 221 vcpu->id, test_stage_string[stage], 222 ts_diff.tv_sec, ts_diff.tv_nsec); 223 224 ret = sem_post(&test_stage_completed); 225 TEST_ASSERT(ret == 0, "Error in sem_post"); 226 } 227 228 return NULL; 229 } 230 231 struct test_params { 232 uint64_t phys_offset; 233 uint64_t test_mem_size; 234 enum vm_mem_backing_src_type src_type; 235 }; 236 237 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) 238 { 239 int ret; 240 struct test_params *p = arg; 241 enum vm_mem_backing_src_type src_type = p->src_type; 242 uint64_t large_page_size = get_backing_src_pagesz(src_type); 243 uint64_t guest_page_size = vm_guest_mode_params[mode].page_size; 244 uint64_t host_page_size = getpagesize(); 245 uint64_t test_mem_size = p->test_mem_size; 246 uint64_t guest_num_pages; 247 uint64_t alignment; 248 void *host_test_mem; 249 struct kvm_vm *vm; 250 251 /* Align up the test memory size */ 252 alignment = max(large_page_size, guest_page_size); 253 test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1); 254 255 /* Create a VM with enough guest pages */ 256 guest_num_pages = test_mem_size / guest_page_size; 257 vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages, 258 guest_code, test_args.vcpus); 259 260 /* Align down GPA of the testing memslot */ 261 if (!p->phys_offset) 262 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) * 263 guest_page_size; 264 else 265 guest_test_phys_mem = p->phys_offset; 266 #ifdef __s390x__ 267 alignment = max(0x100000UL, alignment); 268 #endif 269 guest_test_phys_mem = align_down(guest_test_phys_mem, alignment); 270 271 /* Set up the shared data structure test_args */ 272 test_args.vm = vm; 273 test_args.guest_test_virt_mem = guest_test_virt_mem; 274 test_args.host_page_size = host_page_size; 275 test_args.host_num_pages = test_mem_size / host_page_size; 276 test_args.large_page_size = large_page_size; 277 test_args.large_num_pages = test_mem_size / large_page_size; 278 test_args.host_pages_per_lpage = large_page_size / host_page_size; 279 test_args.src_type = src_type; 280 281 /* Add an extra memory slot with specified backing src type */ 282 vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem, 283 TEST_MEM_SLOT_INDEX, guest_num_pages, 0); 284 285 /* Do mapping(GVA->GPA) for the testing memory slot */ 286 virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); 287 288 /* Cache the HVA pointer of the region */ 289 host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); 290 291 /* Export shared structure test_args to guest */ 292 ucall_init(vm, NULL); 293 sync_global_to_guest(vm, test_args); 294 295 ret = sem_init(&test_stage_updated, 0, 0); 296 TEST_ASSERT(ret == 0, "Error in sem_init"); 297 298 ret = sem_init(&test_stage_completed, 0, 0); 299 TEST_ASSERT(ret == 0, "Error in sem_init"); 300 301 current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage)); 302 *current_stage = NUM_TEST_STAGES; 303 304 pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); 305 pr_info("Testing memory backing src type: %s\n", 306 vm_mem_backing_src_alias(src_type)->name); 307 pr_info("Testing memory backing src granularity: 0x%lx\n", 308 large_page_size); 309 pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size); 310 pr_info("Guest physical test memory offset: 0x%lx\n", 311 guest_test_phys_mem); 312 pr_info("Host virtual test memory offset: 0x%lx\n", 313 (uint64_t)host_test_mem); 314 pr_info("Number of testing vCPUs: %d\n", nr_vcpus); 315 316 return vm; 317 } 318 319 static void vcpus_complete_new_stage(enum test_stage stage) 320 { 321 int ret; 322 int vcpus; 323 324 /* Wake up all the vcpus to run new test stage */ 325 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { 326 ret = sem_post(&test_stage_updated); 327 TEST_ASSERT(ret == 0, "Error in sem_post"); 328 } 329 pr_debug("All vcpus have been notified to continue\n"); 330 331 /* Wait for all the vcpus to complete new test stage */ 332 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { 333 ret = sem_wait(&test_stage_completed); 334 TEST_ASSERT(ret == 0, "Error in sem_wait"); 335 336 pr_debug("%d vcpus have completed stage %s\n", 337 vcpus + 1, test_stage_string[stage]); 338 } 339 340 pr_debug("All vcpus have completed stage %s\n", 341 test_stage_string[stage]); 342 } 343 344 static void run_test(enum vm_guest_mode mode, void *arg) 345 { 346 pthread_t *vcpu_threads; 347 struct kvm_vm *vm; 348 struct timespec start; 349 struct timespec ts_diff; 350 int ret, i; 351 352 /* Create VM with vCPUs and make some pre-initialization */ 353 vm = pre_init_before_test(mode, arg); 354 355 vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads)); 356 TEST_ASSERT(vcpu_threads, "Memory allocation failed"); 357 358 host_quit = false; 359 *current_stage = KVM_BEFORE_MAPPINGS; 360 361 for (i = 0; i < nr_vcpus; i++) 362 pthread_create(&vcpu_threads[i], NULL, vcpu_worker, 363 test_args.vcpus[i]); 364 365 vcpus_complete_new_stage(*current_stage); 366 pr_info("Started all vCPUs successfully\n"); 367 368 /* Test the stage of KVM creating mappings */ 369 *current_stage = KVM_CREATE_MAPPINGS; 370 371 clock_gettime(CLOCK_MONOTONIC_RAW, &start); 372 vcpus_complete_new_stage(*current_stage); 373 ts_diff = timespec_elapsed(start); 374 375 pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n", 376 ts_diff.tv_sec, ts_diff.tv_nsec); 377 378 /* Test the stage of KVM updating mappings */ 379 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 380 KVM_MEM_LOG_DIRTY_PAGES); 381 382 *current_stage = KVM_UPDATE_MAPPINGS; 383 384 clock_gettime(CLOCK_MONOTONIC_RAW, &start); 385 vcpus_complete_new_stage(*current_stage); 386 ts_diff = timespec_elapsed(start); 387 388 pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n", 389 ts_diff.tv_sec, ts_diff.tv_nsec); 390 391 /* Test the stage of KVM adjusting mappings */ 392 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0); 393 394 *current_stage = KVM_ADJUST_MAPPINGS; 395 396 clock_gettime(CLOCK_MONOTONIC_RAW, &start); 397 vcpus_complete_new_stage(*current_stage); 398 ts_diff = timespec_elapsed(start); 399 400 pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n", 401 ts_diff.tv_sec, ts_diff.tv_nsec); 402 403 /* Tell the vcpu thread to quit */ 404 host_quit = true; 405 for (i = 0; i < nr_vcpus; i++) { 406 ret = sem_post(&test_stage_updated); 407 TEST_ASSERT(ret == 0, "Error in sem_post"); 408 } 409 410 for (i = 0; i < nr_vcpus; i++) 411 pthread_join(vcpu_threads[i], NULL); 412 413 ret = sem_destroy(&test_stage_updated); 414 TEST_ASSERT(ret == 0, "Error in sem_destroy"); 415 416 ret = sem_destroy(&test_stage_completed); 417 TEST_ASSERT(ret == 0, "Error in sem_destroy"); 418 419 free(vcpu_threads); 420 ucall_uninit(vm); 421 kvm_vm_free(vm); 422 } 423 424 static void help(char *name) 425 { 426 puts(""); 427 printf("usage: %s [-h] [-p offset] [-m mode] " 428 "[-b mem-size] [-v vcpus] [-s mem-type]\n", name); 429 puts(""); 430 printf(" -p: specify guest physical test memory offset\n" 431 " Warning: a low offset can conflict with the loaded test code.\n"); 432 guest_modes_help(); 433 printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n" 434 " (default: 1G)\n"); 435 printf(" -v: specify the number of vCPUs to run\n" 436 " (default: 1)\n"); 437 backing_src_help("-s"); 438 puts(""); 439 } 440 441 int main(int argc, char *argv[]) 442 { 443 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); 444 struct test_params p = { 445 .test_mem_size = DEFAULT_TEST_MEM_SIZE, 446 .src_type = DEFAULT_VM_MEM_SRC, 447 }; 448 int opt; 449 450 guest_modes_append_default(); 451 452 while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) { 453 switch (opt) { 454 case 'p': 455 p.phys_offset = strtoull(optarg, NULL, 0); 456 break; 457 case 'm': 458 guest_modes_cmdline(optarg); 459 break; 460 case 'b': 461 p.test_mem_size = parse_size(optarg); 462 break; 463 case 'v': 464 nr_vcpus = atoi(optarg); 465 TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus, 466 "Invalid number of vcpus, must be between 1 and %d", max_vcpus); 467 break; 468 case 's': 469 p.src_type = parse_backing_src_type(optarg); 470 break; 471 case 'h': 472 default: 473 help(argv[0]); 474 exit(0); 475 } 476 } 477 478 for_each_guest_mode(run_test, &p); 479 480 return 0; 481 } 482