1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * KVM page table test
4  *
5  * Copyright (C) 2021, Huawei, Inc.
6  *
7  * Make sure that THP has been enabled or enough HUGETLB pages with specific
8  * page size have been pre-allocated on your system, if you are planning to
9  * use hugepages to back the guest memory for testing.
10  */
11 
12 #define _GNU_SOURCE /* for program_invocation_name */
13 
14 #include <stdio.h>
15 #include <stdlib.h>
16 #include <time.h>
17 #include <pthread.h>
18 #include <semaphore.h>
19 
20 #include "test_util.h"
21 #include "kvm_util.h"
22 #include "processor.h"
23 #include "guest_modes.h"
24 
25 #define TEST_MEM_SLOT_INDEX             1
26 
27 /* Default size(1GB) of the memory for testing */
28 #define DEFAULT_TEST_MEM_SIZE		(1 << 30)
29 
30 /* Default guest test virtual memory offset */
31 #define DEFAULT_GUEST_TEST_MEM		0xc0000000
32 
33 /* Different guest memory accessing stages */
34 enum test_stage {
35 	KVM_BEFORE_MAPPINGS,
36 	KVM_CREATE_MAPPINGS,
37 	KVM_UPDATE_MAPPINGS,
38 	KVM_ADJUST_MAPPINGS,
39 	NUM_TEST_STAGES,
40 };
41 
42 static const char * const test_stage_string[] = {
43 	"KVM_BEFORE_MAPPINGS",
44 	"KVM_CREATE_MAPPINGS",
45 	"KVM_UPDATE_MAPPINGS",
46 	"KVM_ADJUST_MAPPINGS",
47 };
48 
49 struct test_args {
50 	struct kvm_vm *vm;
51 	uint64_t guest_test_virt_mem;
52 	uint64_t host_page_size;
53 	uint64_t host_num_pages;
54 	uint64_t large_page_size;
55 	uint64_t large_num_pages;
56 	uint64_t host_pages_per_lpage;
57 	enum vm_mem_backing_src_type src_type;
58 	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
59 };
60 
61 /*
62  * Guest variables. Use addr_gva2hva() if these variables need
63  * to be changed in host.
64  */
65 static enum test_stage guest_test_stage;
66 
67 /* Host variables */
68 static uint32_t nr_vcpus = 1;
69 static struct test_args test_args;
70 static enum test_stage *current_stage;
71 static bool host_quit;
72 
73 /* Whether the test stage is updated, or completed */
74 static sem_t test_stage_updated;
75 static sem_t test_stage_completed;
76 
77 /*
78  * Guest physical memory offset of the testing memory slot.
79  * This will be set to the topmost valid physical address minus
80  * the test memory size.
81  */
82 static uint64_t guest_test_phys_mem;
83 
84 /*
85  * Guest virtual memory offset of the testing memory slot.
86  * Must not conflict with identity mapped test code.
87  */
88 static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM;
89 
90 static void guest_code(bool do_write)
91 {
92 	struct test_args *p = &test_args;
93 	enum test_stage *current_stage = &guest_test_stage;
94 	uint64_t addr;
95 	int i, j;
96 
97 	while (true) {
98 		addr = p->guest_test_virt_mem;
99 
100 		switch (READ_ONCE(*current_stage)) {
101 		/*
102 		 * All vCPU threads will be started in this stage,
103 		 * where guest code of each vCPU will do nothing.
104 		 */
105 		case KVM_BEFORE_MAPPINGS:
106 			break;
107 
108 		/*
109 		 * Before dirty logging, vCPUs concurrently access the first
110 		 * 8 bytes of each page (host page/large page) within the same
111 		 * memory region with different accessing types (read/write).
112 		 * Then KVM will create normal page mappings or huge block
113 		 * mappings for them.
114 		 */
115 		case KVM_CREATE_MAPPINGS:
116 			for (i = 0; i < p->large_num_pages; i++) {
117 				if (do_write)
118 					*(uint64_t *)addr = 0x0123456789ABCDEF;
119 				else
120 					READ_ONCE(*(uint64_t *)addr);
121 
122 				addr += p->large_page_size;
123 			}
124 			break;
125 
126 		/*
127 		 * During dirty logging, KVM will only update attributes of the
128 		 * normal page mappings from RO to RW if memory backing src type
129 		 * is anonymous. In other cases, KVM will split the huge block
130 		 * mappings into normal page mappings if memory backing src type
131 		 * is THP or HUGETLB.
132 		 */
133 		case KVM_UPDATE_MAPPINGS:
134 			if (p->src_type == VM_MEM_SRC_ANONYMOUS) {
135 				for (i = 0; i < p->host_num_pages; i++) {
136 					*(uint64_t *)addr = 0x0123456789ABCDEF;
137 					addr += p->host_page_size;
138 				}
139 				break;
140 			}
141 
142 			for (i = 0; i < p->large_num_pages; i++) {
143 				/*
144 				 * Write to the first host page in each large
145 				 * page region, and triger break of large pages.
146 				 */
147 				*(uint64_t *)addr = 0x0123456789ABCDEF;
148 
149 				/*
150 				 * Access the middle host pages in each large
151 				 * page region. Since dirty logging is enabled,
152 				 * this will create new mappings at the smallest
153 				 * granularity.
154 				 */
155 				addr += p->large_page_size / 2;
156 				for (j = 0; j < p->host_pages_per_lpage / 2; j++) {
157 					READ_ONCE(*(uint64_t *)addr);
158 					addr += p->host_page_size;
159 				}
160 			}
161 			break;
162 
163 		/*
164 		 * After dirty logging is stopped, vCPUs concurrently read
165 		 * from every single host page. Then KVM will coalesce the
166 		 * split page mappings back to block mappings. And a TLB
167 		 * conflict abort could occur here if TLB entries of the
168 		 * page mappings are not fully invalidated.
169 		 */
170 		case KVM_ADJUST_MAPPINGS:
171 			for (i = 0; i < p->host_num_pages; i++) {
172 				READ_ONCE(*(uint64_t *)addr);
173 				addr += p->host_page_size;
174 			}
175 			break;
176 
177 		default:
178 			GUEST_ASSERT(0);
179 		}
180 
181 		GUEST_SYNC(1);
182 	}
183 }
184 
185 static void *vcpu_worker(void *data)
186 {
187 	struct kvm_vcpu *vcpu = data;
188 	bool do_write = !(vcpu->id % 2);
189 	struct timespec start;
190 	struct timespec ts_diff;
191 	enum test_stage stage;
192 	int ret;
193 
194 	vcpu_args_set(vcpu, 1, do_write);
195 
196 	while (!READ_ONCE(host_quit)) {
197 		ret = sem_wait(&test_stage_updated);
198 		TEST_ASSERT(ret == 0, "Error in sem_wait");
199 
200 		if (READ_ONCE(host_quit))
201 			return NULL;
202 
203 		clock_gettime(CLOCK_MONOTONIC_RAW, &start);
204 		ret = _vcpu_run(vcpu);
205 		ts_diff = timespec_elapsed(start);
206 
207 		TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
208 		TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
209 			    "Invalid guest sync status: exit_reason=%s\n",
210 			    exit_reason_str(vcpu->run->exit_reason));
211 
212 		pr_debug("Got sync event from vCPU %d\n", vcpu->id);
213 		stage = READ_ONCE(*current_stage);
214 
215 		/*
216 		 * Here we can know the execution time of every
217 		 * single vcpu running in different test stages.
218 		 */
219 		pr_debug("vCPU %d has completed stage %s\n"
220 			 "execution time is: %ld.%.9lds\n\n",
221 			 vcpu->id, test_stage_string[stage],
222 			 ts_diff.tv_sec, ts_diff.tv_nsec);
223 
224 		ret = sem_post(&test_stage_completed);
225 		TEST_ASSERT(ret == 0, "Error in sem_post");
226 	}
227 
228 	return NULL;
229 }
230 
231 struct test_params {
232 	uint64_t phys_offset;
233 	uint64_t test_mem_size;
234 	enum vm_mem_backing_src_type src_type;
235 };
236 
237 static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
238 {
239 	int ret;
240 	struct test_params *p = arg;
241 	enum vm_mem_backing_src_type src_type = p->src_type;
242 	uint64_t large_page_size = get_backing_src_pagesz(src_type);
243 	uint64_t guest_page_size = vm_guest_mode_params[mode].page_size;
244 	uint64_t host_page_size = getpagesize();
245 	uint64_t test_mem_size = p->test_mem_size;
246 	uint64_t guest_num_pages;
247 	uint64_t alignment;
248 	void *host_test_mem;
249 	struct kvm_vm *vm;
250 
251 	/* Align up the test memory size */
252 	alignment = max(large_page_size, guest_page_size);
253 	test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1);
254 
255 	/* Create a VM with enough guest pages */
256 	guest_num_pages = test_mem_size / guest_page_size;
257 	vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages,
258 				    guest_code, test_args.vcpus);
259 
260 	/* Align down GPA of the testing memslot */
261 	if (!p->phys_offset)
262 		guest_test_phys_mem = (vm->max_gfn - guest_num_pages) *
263 				       guest_page_size;
264 	else
265 		guest_test_phys_mem = p->phys_offset;
266 #ifdef __s390x__
267 	alignment = max(0x100000UL, alignment);
268 #endif
269 	guest_test_phys_mem = align_down(guest_test_phys_mem, alignment);
270 
271 	/* Set up the shared data structure test_args */
272 	test_args.vm = vm;
273 	test_args.guest_test_virt_mem = guest_test_virt_mem;
274 	test_args.host_page_size = host_page_size;
275 	test_args.host_num_pages = test_mem_size / host_page_size;
276 	test_args.large_page_size = large_page_size;
277 	test_args.large_num_pages = test_mem_size / large_page_size;
278 	test_args.host_pages_per_lpage = large_page_size / host_page_size;
279 	test_args.src_type = src_type;
280 
281 	/* Add an extra memory slot with specified backing src type */
282 	vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem,
283 				    TEST_MEM_SLOT_INDEX, guest_num_pages, 0);
284 
285 	/* Do mapping(GVA->GPA) for the testing memory slot */
286 	virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages);
287 
288 	/* Cache the HVA pointer of the region */
289 	host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem);
290 
291 	/* Export shared structure test_args to guest */
292 	ucall_init(vm, NULL);
293 	sync_global_to_guest(vm, test_args);
294 
295 	ret = sem_init(&test_stage_updated, 0, 0);
296 	TEST_ASSERT(ret == 0, "Error in sem_init");
297 
298 	ret = sem_init(&test_stage_completed, 0, 0);
299 	TEST_ASSERT(ret == 0, "Error in sem_init");
300 
301 	current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage));
302 	*current_stage = NUM_TEST_STAGES;
303 
304 	pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
305 	pr_info("Testing memory backing src type: %s\n",
306 		vm_mem_backing_src_alias(src_type)->name);
307 	pr_info("Testing memory backing src granularity: 0x%lx\n",
308 		large_page_size);
309 	pr_info("Testing memory size(aligned): 0x%lx\n", test_mem_size);
310 	pr_info("Guest physical test memory offset: 0x%lx\n",
311 		guest_test_phys_mem);
312 	pr_info("Host  virtual  test memory offset: 0x%lx\n",
313 		(uint64_t)host_test_mem);
314 	pr_info("Number of testing vCPUs: %d\n", nr_vcpus);
315 
316 	return vm;
317 }
318 
319 static void vcpus_complete_new_stage(enum test_stage stage)
320 {
321 	int ret;
322 	int vcpus;
323 
324 	/* Wake up all the vcpus to run new test stage */
325 	for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
326 		ret = sem_post(&test_stage_updated);
327 		TEST_ASSERT(ret == 0, "Error in sem_post");
328 	}
329 	pr_debug("All vcpus have been notified to continue\n");
330 
331 	/* Wait for all the vcpus to complete new test stage */
332 	for (vcpus = 0; vcpus < nr_vcpus; vcpus++) {
333 		ret = sem_wait(&test_stage_completed);
334 		TEST_ASSERT(ret == 0, "Error in sem_wait");
335 
336 		pr_debug("%d vcpus have completed stage %s\n",
337 			 vcpus + 1, test_stage_string[stage]);
338 	}
339 
340 	pr_debug("All vcpus have completed stage %s\n",
341 		 test_stage_string[stage]);
342 }
343 
344 static void run_test(enum vm_guest_mode mode, void *arg)
345 {
346 	pthread_t *vcpu_threads;
347 	struct kvm_vm *vm;
348 	struct timespec start;
349 	struct timespec ts_diff;
350 	int ret, i;
351 
352 	/* Create VM with vCPUs and make some pre-initialization */
353 	vm = pre_init_before_test(mode, arg);
354 
355 	vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads));
356 	TEST_ASSERT(vcpu_threads, "Memory allocation failed");
357 
358 	host_quit = false;
359 	*current_stage = KVM_BEFORE_MAPPINGS;
360 
361 	for (i = 0; i < nr_vcpus; i++)
362 		pthread_create(&vcpu_threads[i], NULL, vcpu_worker,
363 			       test_args.vcpus[i]);
364 
365 	vcpus_complete_new_stage(*current_stage);
366 	pr_info("Started all vCPUs successfully\n");
367 
368 	/* Test the stage of KVM creating mappings */
369 	*current_stage = KVM_CREATE_MAPPINGS;
370 
371 	clock_gettime(CLOCK_MONOTONIC_RAW, &start);
372 	vcpus_complete_new_stage(*current_stage);
373 	ts_diff = timespec_elapsed(start);
374 
375 	pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
376 		ts_diff.tv_sec, ts_diff.tv_nsec);
377 
378 	/* Test the stage of KVM updating mappings */
379 	vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX,
380 				KVM_MEM_LOG_DIRTY_PAGES);
381 
382 	*current_stage = KVM_UPDATE_MAPPINGS;
383 
384 	clock_gettime(CLOCK_MONOTONIC_RAW, &start);
385 	vcpus_complete_new_stage(*current_stage);
386 	ts_diff = timespec_elapsed(start);
387 
388 	pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n",
389 		ts_diff.tv_sec, ts_diff.tv_nsec);
390 
391 	/* Test the stage of KVM adjusting mappings */
392 	vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0);
393 
394 	*current_stage = KVM_ADJUST_MAPPINGS;
395 
396 	clock_gettime(CLOCK_MONOTONIC_RAW, &start);
397 	vcpus_complete_new_stage(*current_stage);
398 	ts_diff = timespec_elapsed(start);
399 
400 	pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n",
401 		ts_diff.tv_sec, ts_diff.tv_nsec);
402 
403 	/* Tell the vcpu thread to quit */
404 	host_quit = true;
405 	for (i = 0; i < nr_vcpus; i++) {
406 		ret = sem_post(&test_stage_updated);
407 		TEST_ASSERT(ret == 0, "Error in sem_post");
408 	}
409 
410 	for (i = 0; i < nr_vcpus; i++)
411 		pthread_join(vcpu_threads[i], NULL);
412 
413 	ret = sem_destroy(&test_stage_updated);
414 	TEST_ASSERT(ret == 0, "Error in sem_destroy");
415 
416 	ret = sem_destroy(&test_stage_completed);
417 	TEST_ASSERT(ret == 0, "Error in sem_destroy");
418 
419 	free(vcpu_threads);
420 	ucall_uninit(vm);
421 	kvm_vm_free(vm);
422 }
423 
424 static void help(char *name)
425 {
426 	puts("");
427 	printf("usage: %s [-h] [-p offset] [-m mode] "
428 	       "[-b mem-size] [-v vcpus] [-s mem-type]\n", name);
429 	puts("");
430 	printf(" -p: specify guest physical test memory offset\n"
431 	       "     Warning: a low offset can conflict with the loaded test code.\n");
432 	guest_modes_help();
433 	printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n"
434 	       "     (default: 1G)\n");
435 	printf(" -v: specify the number of vCPUs to run\n"
436 	       "     (default: 1)\n");
437 	backing_src_help("-s");
438 	puts("");
439 }
440 
441 int main(int argc, char *argv[])
442 {
443 	int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
444 	struct test_params p = {
445 		.test_mem_size = DEFAULT_TEST_MEM_SIZE,
446 		.src_type = DEFAULT_VM_MEM_SRC,
447 	};
448 	int opt;
449 
450 	guest_modes_append_default();
451 
452 	while ((opt = getopt(argc, argv, "hp:m:b:v:s:")) != -1) {
453 		switch (opt) {
454 		case 'p':
455 			p.phys_offset = strtoull(optarg, NULL, 0);
456 			break;
457 		case 'm':
458 			guest_modes_cmdline(optarg);
459 			break;
460 		case 'b':
461 			p.test_mem_size = parse_size(optarg);
462 			break;
463 		case 'v':
464 			nr_vcpus = atoi(optarg);
465 			TEST_ASSERT(nr_vcpus > 0 && nr_vcpus <= max_vcpus,
466 				    "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
467 			break;
468 		case 's':
469 			p.src_type = parse_backing_src_type(optarg);
470 			break;
471 		case 'h':
472 		default:
473 			help(argv[0]);
474 			exit(0);
475 		}
476 	}
477 
478 	for_each_guest_mode(run_test, &p);
479 
480 	return 0;
481 }
482