1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KVM page table test
4 *
5 * Copyright (C) 2021, Huawei, Inc.
6 *
7 * Make sure that THP has been enabled or enough HUGETLB pages with specific
8 * page size have been pre-allocated on your system, if you are planning to
9 * use hugepages to back the guest memory for testing.
10 */
11
12 #define _GNU_SOURCE /* for program_invocation_name */
13
14 #include <stdio.h>
15 #include <stdlib.h>
16 #include <time.h>
17 #include <pthread.h>
18 #include <semaphore.h>
19
20 #include "test_util.h"
21 #include "kvm_util.h"
22 #include "processor.h"
23 #include "guest_modes.h"
24
25 #define TEST_MEM_SLOT_INDEX 1
26
27 /* Default size(1GB) of the memory for testing */
28 #define DEFAULT_TEST_MEM_SIZE (1 << 30)
29
30 /* Default guest test virtual memory offset */
31 #define DEFAULT_GUEST_TEST_MEM 0xc0000000
32
33 /* Different guest memory accessing stages */
34 enum test_stage {
35 KVM_BEFORE_MAPPINGS,
36 KVM_CREATE_MAPPINGS,
37 KVM_UPDATE_MAPPINGS,
38 KVM_ADJUST_MAPPINGS,
39 NUM_TEST_STAGES,
40 };
41
42 static const char * const test_stage_string[] = {
43 "KVM_BEFORE_MAPPINGS",
44 "KVM_CREATE_MAPPINGS",
45 "KVM_UPDATE_MAPPINGS",
46 "KVM_ADJUST_MAPPINGS",
47 };
48
49 struct test_args {
50 struct kvm_vm *vm;
51 uint64_t guest_test_virt_mem;
52 uint64_t host_page_size;
53 uint64_t host_num_pages;
54 uint64_t large_page_size;
55 uint64_t large_num_pages;
56 uint64_t host_pages_per_lpage;
57 enum vm_mem_backing_src_type src_type;
58 struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
59 };
60
61 /*
62 * Guest variables. Use addr_gva2hva() if these variables need
63 * to be changed in host.
64 */
65 static enum test_stage guest_test_stage;
66
67 /* Host variables */
68 static uint32_t nr_vcpus = 1;
69 static struct test_args test_args;
70 static enum test_stage *current_stage;
71 static bool host_quit;
72
73 /* Whether the test stage is updated, or completed */
74 static sem_t test_stage_updated;
75 static sem_t test_stage_completed;
76
77 /*
78 * Guest physical memory offset of the testing memory slot.
79 * This will be set to the topmost valid physical address minus
80 * the test memory size.
81 */
82 static uint64_t guest_test_phys_mem;
83
84 /*
85 * Guest virtual memory offset of the testing memory slot.
86 * Must not conflict with identity mapped test code.
87 */
88 static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
89
guest_code(bool do_write)90 static void guest_code(bool do_write)
91 {
92 struct test_args *p = &test_args;
93 enum test_stage *current_stage = &guest_test_stage;
94 uint64_t addr;
95 int i, j;
96
97 while (true) {
98 addr = p->guest_test_virt_mem;
99
100 switch (READ_ONCE(*current_stage)) {
101 /*
102 * All vCPU threads will be started in this stage,
103 * where guest code of each vCPU will do nothing.
104 */
105 case KVM_BEFORE_MAPPINGS:
106 break;
107
108 /*
109 * Before dirty logging, vCPUs concurrently access the first
110 * 8 bytes of each page (host page/large page) within the same
111 * memory region with different accessing types (read/write).
112 * Then KVM will create normal page mappings or huge block
113 * mappings for them.
114 */
115 case KVM_CREATE_MAPPINGS:
116 for (i = 0; i < p->large_num_pages; i++) {
117 if (do_write)
118 *(uint64_t *)addr = 0x0123456789ABCDEF;
119 else
120 READ_ONCE(*(uint64_t *)addr);
121
122 addr += p->large_page_size;
123 }
124 break;
125
126 /*
127 * During dirty logging, KVM will only update attributes of the
128 * normal page mappings from RO to RW if memory backing src type
129 * is anonymous. In other cases, KVM will split the huge block
130 * mappings into normal page mappings if memory backing src type
131 * is THP or HUGETLB.
132 */
133 case KVM_UPDATE_MAPPINGS:
134 if (p->src_type == VM_MEM_SRC_ANONYMOUS) {
135 for (i = 0; i < p->host_num_pages; i++) {
136 *(uint64_t *)addr = 0x0123456789ABCDEF;
137 addr += p->host_page_size;
138 }
139 break;
140 }
141
142 for (i = 0; i < p->large_num_pages; i++) {
143 /*
144 * Write to the first host page in each large
145 * page region, and triger break of large pages.
146 */
147 *(uint64_t *)addr = 0x0123456789ABCDEF;
148
149 /*
150 * Access the middle host pages in each large
151 * page region. Since dirty logging is enabled,
152 * this will create new mappings at the smallest
153 * granularity.
154 */
155 addr += p->large_page_size / 2;
156 for (j = 0; j < p->host_pages_per_lpage / 2; j++) {
157 READ_ONCE(*(uint64_t *)addr);
158 addr += p->host_page_size;
159 }
160 }
161 break;
162
163 /*
164 * After dirty logging is stopped, vCPUs concurrently read
165 * from every single host page. Then KVM will coalesce the
166 * split page mappings back to block mappings. And a TLB
167 * conflict abort could occur here if TLB entries of the
168 * page mappings are not fully invalidated.
169 */
170 case KVM_ADJUST_MAPPINGS:
171 for (i = 0; i < p->host_num_pages; i++) {
172 READ_ONCE(*(uint64_t *)addr);
173 addr += p->host_page_size;
174 }
175 break;
176
177 default:
178 GUEST_ASSERT(0);
179 }
180
181 GUEST_SYNC(1);
182 }
183 }
184
vcpu_worker(void * data)185 static void *vcpu_worker(void *data)
186 {
187 struct kvm_vcpu *vcpu = data;
188 bool do_write = !(vcpu->id % 2);
189 struct timespec start;
190 struct timespec ts_diff;
191 enum test_stage stage;
192 int ret;
193
194 vcpu_args_set(vcpu, 1, do_write);
195
196 while (!READ_ONCE(host_quit)) {
197 ret = sem_wait(&test_stage_updated);
198 TEST_ASSERT(ret == 0, "Error in sem_wait");
199
200 if (READ_ONCE(host_quit))
201 return NULL;
202
203 clock_gettime(CLOCK_MONOTONIC, &start);
204 ret = _vcpu_run(vcpu);
205 ts_diff = timespec_elapsed(start);
206
207 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
208 TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
209 "Invalid guest sync status: exit_reason=%s\n",
210 exit_reason_str(vcpu->run->exit_reason));
211
212 pr_debug("Got sync event from vCPU %d\n", vcpu->id);
213 stage = READ_ONCE(*current_stage);
214
215 /*
216 * Here we can know the execution time of every
217 * single vcpu running in different test stages.
218 */
219 pr_debug("vCPU %d has completed stage %s\n"
220 "execution time is: %ld.%.9lds\n\n",
221 vcpu->id, test_stage_string[stage],
222 ts_diff.tv_sec, ts_diff.tv_nsec);
223
224 ret = sem_post(&test_stage_completed);
225 TEST_ASSERT(ret == 0, "Error in sem_post");
226 }
227
228 return NULL;
229 }
230
231 struct test_params {
232 uint64_t phys_offset;
233 uint64_t test_mem_size;
234 enum vm_mem_backing_src_type src_type;
235 };
236
pre_init_before_test(enum vm_guest_mode mode,void * arg)237 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
238 {
239 int ret;
240 struct test_params *p = arg;
241 enum vm_mem_backing_src_type src_type = p->src_type;
242 uint64_t large_page_size = get_backing_src_pagesz(src_type);
243 uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;
244 uint64_t host_page_size = getpagesize();
245 uint64_t test_mem_size = p->test_mem_size;
246 uint64_t guest_num_pages;
247 uint64_t alignment;
248 void *host_test_mem;
249 struct kvm_vm *vm;
250
251 /* Align up the test memory size */
252 alignment = max(large_page_size, guest_page_size);
253 test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1);
254
255 /* Create a VM with enough guest pages */
256 guest_num_pages = test_mem_size / guest_page_size;
257 vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages,
258 guest_code, test_args.vcpus);
259
260 /* Align down GPA of the testing memslot */
261 if (!p->phys_offset)
262 guest_test_phys_mem = (vm->max_gfn - guest_num_pages) *
263 guest_page_size;
264 else
265 guest_test_phys_mem = p->phys_offset;
266 #ifdef __s390x__
267 alignment = max(0x100000UL, alignment);
268 #endif
269 guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);
270
271 /* Set up the shared data structure test_args */
272 test_args.vm = vm;
273 test_args.guest_test_virt_mem = guest_test_virt_mem;
274 test_args.host_page_size = host_page_size;
275 test_args.host_num_pages = test_mem_size / host_page_size;
276 test_args.large_page_size = large_page_size;
277 test_args.large_num_pages = test_mem_size / large_page_size;
278 test_args.host_pages_per_lpage = large_page_size / host_page_size;
279 test_args.src_type = src_type;
280
281 /* Add an extra memory slot with specified backing src type */
282 vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem,
283 TEST_MEM_SLOT_INDEX, guest_num_pages, 0);
284
285 /* Do mapping(GVA->GPA) for the testing memory slot */
286 virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
287
288 /* Cache the HVA pointer of the region */
289 host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
290
291 /* Export shared structure test_args to guest */
292 sync_global_to_guest(vm, test_args);
293
294 ret = sem_init(&test_stage_updated, 0, 0);
295 TEST_ASSERT(ret == 0, "Error in sem_init");
296
297 ret = sem_init(&test_stage_completed, 0, 0);
298 TEST_ASSERT(ret == 0, "Error in sem_init");
299
300 current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));
301 *current_stage = NUM_TEST_STAGES;
302
303 pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
304 pr_info("Testing memory backing src type: %s\n",
305 vm_mem_backing_src_alias(src_type)->name);
306 pr_info("Testing memory backing src granularity: 0x%lx\n",
307 large_page_size);
308 pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size);
309 pr_info("Guest physical test memory offset: 0x%lx\n",
310 guest_test_phys_mem);
311 pr_info("Host virtual test memory offset: 0x%lx\n",
312 (uint64_t)host_test_mem);
313 pr_info("Number of testing vCPUs: %d\n", nr_vcpus);
314
315 return vm;
316 }
317
vcpus_complete_new_stage(enum test_stage stage)318 static void vcpus_complete_new_stage(enum test_stage stage)
319 {
320 int ret;
321 int vcpus;
322
323 /* Wake up all the vcpus to run new test stage */
324 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
325 ret = sem_post(&test_stage_updated);
326 TEST_ASSERT(ret == 0, "Error in sem_post");
327 }
328 pr_debug("All vcpus have been notified to continue\n");
329
330 /* Wait for all the vcpus to complete new test stage */
331 for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
332 ret = sem_wait(&test_stage_completed);
333 TEST_ASSERT(ret == 0, "Error in sem_wait");
334
335 pr_debug("%d vcpus have completed stage %s\n",
336 vcpus + 1, test_stage_string[stage]);
337 }
338
339 pr_debug("All vcpus have completed stage %s\n",
340 test_stage_string[stage]);
341 }
342
run_test(enum vm_guest_mode mode,void * arg)343 static void run_test(enum vm_guest_mode mode, void *arg)
344 {
345 pthread_t *vcpu_threads;
346 struct kvm_vm *vm;
347 struct timespec start;
348 struct timespec ts_diff;
349 int ret, i;
350
351 /* Create VM with vCPUs and make some pre-initialization */
352 vm = pre_init_before_test(mode, arg);
353
354 vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
355 TEST_ASSERT(vcpu_threads, "Memory allocation failed");
356
357 host_quit = false;
358 *current_stage = KVM_BEFORE_MAPPINGS;
359
360 for (i = 0; i < nr_vcpus; i++)
361 pthread_create(&vcpu_threads[i], NULL, vcpu_worker,
362 test_args.vcpus[i]);
363
364 vcpus_complete_new_stage(*current_stage);
365 pr_info("Started all vCPUs successfully\n");
366
367 /* Test the stage of KVM creating mappings */
368 *current_stage = KVM_CREATE_MAPPINGS;
369
370 clock_gettime(CLOCK_MONOTONIC, &start);
371 vcpus_complete_new_stage(*current_stage);
372 ts_diff = timespec_elapsed(start);
373
374 pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
375 ts_diff.tv_sec, ts_diff.tv_nsec);
376
377 /* Test the stage of KVM updating mappings */
378 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX,
379 KVM_MEM_LOG_DIRTY_PAGES);
380
381 *current_stage = KVM_UPDATE_MAPPINGS;
382
383 clock_gettime(CLOCK_MONOTONIC, &start);
384 vcpus_complete_new_stage(*current_stage);
385 ts_diff = timespec_elapsed(start);
386
387 pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
388 ts_diff.tv_sec, ts_diff.tv_nsec);
389
390 /* Test the stage of KVM adjusting mappings */
391 vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0);
392
393 *current_stage = KVM_ADJUST_MAPPINGS;
394
395 clock_gettime(CLOCK_MONOTONIC, &start);
396 vcpus_complete_new_stage(*current_stage);
397 ts_diff = timespec_elapsed(start);
398
399 pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n",
400 ts_diff.tv_sec, ts_diff.tv_nsec);
401
402 /* Tell the vcpu thread to quit */
403 host_quit = true;
404 for (i = 0; i < nr_vcpus; i++) {
405 ret = sem_post(&test_stage_updated);
406 TEST_ASSERT(ret == 0, "Error in sem_post");
407 }
408
409 for (i = 0; i < nr_vcpus; i++)
410 pthread_join(vcpu_threads[i], NULL);
411
412 ret = sem_destroy(&test_stage_updated);
413 TEST_ASSERT(ret == 0, "Error in sem_destroy");
414
415 ret = sem_destroy(&test_stage_completed);
416 TEST_ASSERT(ret == 0, "Error in sem_destroy");
417
418 free(vcpu_threads);
419 kvm_vm_free(vm);
420 }
421
help(char * name)422 static void help(char *name)
423 {
424 puts("");
425 printf("usage: %s [-h] [-p offset] [-m mode] "
426 "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);
427 puts("");
428 printf(" -p: specify guest physical test memory offset\n"
429 " Warning: a low offset can conflict with the loaded test code.\n");
430 guest_modes_help();
431 printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n"
432 " (default: 1G)\n");
433 printf(" -v: specify the number of vCPUs to run\n"
434 " (default: 1)\n");
435 backing_src_help("-s");
436 puts("");
437 }
438
main(int argc,char * argv[])439 int main(int argc, char *argv[])
440 {
441 int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
442 struct test_params p = {
443 .test_mem_size = DEFAULT_TEST_MEM_SIZE,
444 .src_type = DEFAULT_VM_MEM_SRC,
445 };
446 int opt;
447
448 guest_modes_append_default();
449
450 while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {
451 switch (opt) {
452 case 'p':
453 p.phys_offset = strtoull(optarg, NULL, 0);
454 break;
455 case 'm':
456 guest_modes_cmdline(optarg);
457 break;
458 case 'b':
459 p.test_mem_size = parse_size(optarg);
460 break;
461 case 'v':
462 nr_vcpus = atoi_positive("Number of vCPUs", optarg);
463 TEST_ASSERT(nr_vcpus <= max_vcpus,
464 "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
465 break;
466 case 's':
467 p.src_type = parse_backing_src_type(optarg);
468 break;
469 case 'h':
470 default:
471 help(argv[0]);
472 exit(0);
473 }
474 }
475
476 for_each_guest_mode(run_test, &p);
477
478 return 0;
479 }
480