1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * A memslot-related performance benchmark.
4  *
5  * Copyright (C) 2021 Oracle and/or its affiliates.
6  *
7  * Basic guest setup / host vCPU thread code lifted from set_memory_region_test.
8  */
9 #include <pthread.h>
10 #include <sched.h>
11 #include <semaphore.h>
12 #include <stdatomic.h>
13 #include <stdbool.h>
14 #include <stdint.h>
15 #include <stdio.h>
16 #include <stdlib.h>
17 #include <string.h>
18 #include <sys/mman.h>
19 #include <time.h>
20 #include <unistd.h>
21 
22 #include <linux/compiler.h>
23 #include <linux/sizes.h>
24 
25 #include <test_util.h>
26 #include <kvm_util.h>
27 #include <processor.h>
28 
29 #define MEM_EXTRA_SIZE		SZ_64K
30 
31 #define MEM_SIZE		(SZ_512M + MEM_EXTRA_SIZE)
32 #define MEM_GPA			SZ_256M
33 #define MEM_AUX_GPA		MEM_GPA
34 #define MEM_SYNC_GPA		MEM_AUX_GPA
35 #define MEM_TEST_GPA		(MEM_AUX_GPA + MEM_EXTRA_SIZE)
36 #define MEM_TEST_SIZE		(MEM_SIZE - MEM_EXTRA_SIZE)
37 
38 /*
39  * 32 MiB is max size that gets well over 100 iterations on 509 slots.
40  * Considering that each slot needs to have at least one page up to
41  * 8194 slots in use can then be tested (although with slightly
42  * limited resolution).
43  */
44 #define MEM_SIZE_MAP		(SZ_32M + MEM_EXTRA_SIZE)
45 #define MEM_TEST_MAP_SIZE	(MEM_SIZE_MAP - MEM_EXTRA_SIZE)
46 
47 /*
48  * 128 MiB is min size that fills 32k slots with at least one page in each
49  * while at the same time gets 100+ iterations in such test
50  *
51  * 2 MiB chunk size like a typical huge page
52  */
53 #define MEM_TEST_UNMAP_SIZE		SZ_128M
54 #define MEM_TEST_UNMAP_CHUNK_SIZE	SZ_2M
55 
56 /*
57  * For the move active test the middle of the test area is placed on
58  * a memslot boundary: half lies in the memslot being moved, half in
59  * other memslot(s).
60  *
61  * We have different number of memory slots, excluding the reserved
62  * memory slot 0, on various architectures and configurations. The
63  * memory size in this test is calculated by picking the maximal
64  * last memory slot's memory size, with alignment to the largest
65  * supported page size (64KB). In this way, the selected memory
66  * size for this test is compatible with test_memslot_move_prepare().
67  *
68  * architecture   slots    memory-per-slot    memory-on-last-slot
69  * --------------------------------------------------------------
70  * x86-4KB        32763    16KB               160KB
71  * arm64-4KB      32766    16KB               112KB
72  * arm64-16KB     32766    16KB               112KB
73  * arm64-64KB     8192     64KB               128KB
74  */
75 #define MEM_TEST_MOVE_SIZE		(3 * SZ_64K)
76 #define MEM_TEST_MOVE_GPA_DEST		(MEM_GPA + MEM_SIZE)
77 static_assert(MEM_TEST_MOVE_SIZE <= MEM_TEST_SIZE,
78 	      "invalid move test region size");
79 
80 #define MEM_TEST_VAL_1 0x1122334455667788
81 #define MEM_TEST_VAL_2 0x99AABBCCDDEEFF00
82 
83 struct vm_data {
84 	struct kvm_vm *vm;
85 	struct kvm_vcpu *vcpu;
86 	pthread_t vcpu_thread;
87 	uint32_t nslots;
88 	uint64_t npages;
89 	uint64_t pages_per_slot;
90 	void **hva_slots;
91 	bool mmio_ok;
92 	uint64_t mmio_gpa_min;
93 	uint64_t mmio_gpa_max;
94 };
95 
96 struct sync_area {
97 	uint32_t    guest_page_size;
98 	atomic_bool start_flag;
99 	atomic_bool exit_flag;
100 	atomic_bool sync_flag;
101 	void *move_area_ptr;
102 };
103 
104 /*
105  * Technically, we need also for the atomic bool to be address-free, which
106  * is recommended, but not strictly required, by C11 for lockless
107  * implementations.
108  * However, in practice both GCC and Clang fulfill this requirement on
109  * all KVM-supported platforms.
110  */
111 static_assert(ATOMIC_BOOL_LOCK_FREE == 2, "atomic bool is not lockless");
112 
113 static sem_t vcpu_ready;
114 
115 static bool map_unmap_verify;
116 
117 static bool verbose;
118 #define pr_info_v(...)				\
119 	do {					\
120 		if (verbose)			\
121 			pr_info(__VA_ARGS__);	\
122 	} while (0)
123 
124 static void check_mmio_access(struct vm_data *data, struct kvm_run *run)
125 {
126 	TEST_ASSERT(data->mmio_ok, "Unexpected mmio exit");
127 	TEST_ASSERT(run->mmio.is_write, "Unexpected mmio read");
128 	TEST_ASSERT(run->mmio.len == 8,
129 		    "Unexpected exit mmio size = %u", run->mmio.len);
130 	TEST_ASSERT(run->mmio.phys_addr >= data->mmio_gpa_min &&
131 		    run->mmio.phys_addr <= data->mmio_gpa_max,
132 		    "Unexpected exit mmio address = 0x%llx",
133 		    run->mmio.phys_addr);
134 }
135 
136 static void *vcpu_worker(void *__data)
137 {
138 	struct vm_data *data = __data;
139 	struct kvm_vcpu *vcpu = data->vcpu;
140 	struct kvm_run *run = vcpu->run;
141 	struct ucall uc;
142 
143 	while (1) {
144 		vcpu_run(vcpu);
145 
146 		switch (get_ucall(vcpu, &uc)) {
147 		case UCALL_SYNC:
148 			TEST_ASSERT(uc.args[1] == 0,
149 				"Unexpected sync ucall, got %lx",
150 				(ulong)uc.args[1]);
151 			sem_post(&vcpu_ready);
152 			continue;
153 		case UCALL_NONE:
154 			if (run->exit_reason == KVM_EXIT_MMIO)
155 				check_mmio_access(data, run);
156 			else
157 				goto done;
158 			break;
159 		case UCALL_ABORT:
160 			REPORT_GUEST_ASSERT_1(uc, "val = %lu");
161 			break;
162 		case UCALL_DONE:
163 			goto done;
164 		default:
165 			TEST_FAIL("Unknown ucall %lu", uc.cmd);
166 		}
167 	}
168 
169 done:
170 	return NULL;
171 }
172 
173 static void wait_for_vcpu(void)
174 {
175 	struct timespec ts;
176 
177 	TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),
178 		    "clock_gettime() failed: %d\n", errno);
179 
180 	ts.tv_sec += 2;
181 	TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),
182 		    "sem_timedwait() failed: %d\n", errno);
183 }
184 
185 static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages)
186 {
187 	uint64_t gpage, pgoffs;
188 	uint32_t slot, slotoffs;
189 	void *base;
190 	uint32_t guest_page_size = data->vm->page_size;
191 
192 	TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");
193 	TEST_ASSERT(gpa < MEM_GPA + data->npages * guest_page_size,
194 		    "Too high gpa to translate");
195 	gpa -= MEM_GPA;
196 
197 	gpage = gpa / guest_page_size;
198 	pgoffs = gpa % guest_page_size;
199 	slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);
200 	slotoffs = gpage - (slot * data->pages_per_slot);
201 
202 	if (rempages) {
203 		uint64_t slotpages;
204 
205 		if (slot == data->nslots - 1)
206 			slotpages = data->npages - slot * data->pages_per_slot;
207 		else
208 			slotpages = data->pages_per_slot;
209 
210 		TEST_ASSERT(!pgoffs,
211 			    "Asking for remaining pages in slot but gpa not page aligned");
212 		*rempages = slotpages - slotoffs;
213 	}
214 
215 	base = data->hva_slots[slot];
216 	return (uint8_t *)base + slotoffs * guest_page_size + pgoffs;
217 }
218 
219 static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)
220 {
221 	uint32_t guest_page_size = data->vm->page_size;
222 
223 	TEST_ASSERT(slot < data->nslots, "Too high slot number");
224 
225 	return MEM_GPA + slot * data->pages_per_slot * guest_page_size;
226 }
227 
228 static struct vm_data *alloc_vm(void)
229 {
230 	struct vm_data *data;
231 
232 	data = malloc(sizeof(*data));
233 	TEST_ASSERT(data, "malloc(vmdata) failed");
234 
235 	data->vm = NULL;
236 	data->vcpu = NULL;
237 	data->hva_slots = NULL;
238 
239 	return data;
240 }
241 
242 static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
243 			     uint64_t pages_per_slot, uint64_t rempages)
244 {
245 	if (!pages_per_slot)
246 		return false;
247 
248 	if ((pages_per_slot * guest_page_size) % host_page_size)
249 		return false;
250 
251 	if ((rempages * guest_page_size) % host_page_size)
252 		return false;
253 
254 	return true;
255 }
256 
257 
258 static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
259 {
260 	uint32_t guest_page_size = data->vm->page_size;
261 	uint64_t mempages, pages_per_slot, rempages;
262 	uint64_t slots;
263 
264 	mempages = data->npages;
265 	slots = data->nslots;
266 	while (--slots > 1) {
267 		pages_per_slot = mempages / slots;
268 		if (!pages_per_slot)
269 			continue;
270 
271 		rempages = mempages % pages_per_slot;
272 		if (check_slot_pages(host_page_size, guest_page_size,
273 				     pages_per_slot, rempages))
274 			return slots + 1;	/* slot 0 is reserved */
275 	}
276 
277 	return 0;
278 }
279 
280 static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
281 		       void *guest_code, uint64_t mem_size,
282 		       struct timespec *slot_runtime)
283 {
284 	uint64_t mempages, rempages;
285 	uint64_t guest_addr;
286 	uint32_t slot, host_page_size, guest_page_size;
287 	struct timespec tstart;
288 	struct sync_area *sync;
289 
290 	host_page_size = getpagesize();
291 	guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
292 	mempages = mem_size / guest_page_size;
293 
294 	data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
295 	TEST_ASSERT(data->vm->page_size == guest_page_size, "Invalid VM page size");
296 
297 	data->npages = mempages;
298 	TEST_ASSERT(data->npages > 1, "Can't test without any memory");
299 	data->nslots = nslots;
300 	data->pages_per_slot = data->npages / data->nslots;
301 	rempages = data->npages % data->nslots;
302 	if (!check_slot_pages(host_page_size, guest_page_size,
303 			      data->pages_per_slot, rempages)) {
304 		*maxslots = get_max_slots(data, host_page_size);
305 		return false;
306 	}
307 
308 	data->hva_slots = malloc(sizeof(*data->hva_slots) * data->nslots);
309 	TEST_ASSERT(data->hva_slots, "malloc() fail");
310 
311 	data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
312 
313 	pr_info_v("Adding slots 1..%i, each slot with %"PRIu64" pages + %"PRIu64" extra pages last\n",
314 		data->nslots, data->pages_per_slot, rempages);
315 
316 	clock_gettime(CLOCK_MONOTONIC, &tstart);
317 	for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
318 		uint64_t npages;
319 
320 		npages = data->pages_per_slot;
321 		if (slot == data->nslots)
322 			npages += rempages;
323 
324 		vm_userspace_mem_region_add(data->vm, VM_MEM_SRC_ANONYMOUS,
325 					    guest_addr, slot, npages,
326 					    0);
327 		guest_addr += npages * guest_page_size;
328 	}
329 	*slot_runtime = timespec_elapsed(tstart);
330 
331 	for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
332 		uint64_t npages;
333 		uint64_t gpa;
334 
335 		npages = data->pages_per_slot;
336 		if (slot == data->nslots)
337 			npages += rempages;
338 
339 		gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot);
340 		TEST_ASSERT(gpa == guest_addr,
341 			    "vm_phy_pages_alloc() failed\n");
342 
343 		data->hva_slots[slot - 1] = addr_gpa2hva(data->vm, guest_addr);
344 		memset(data->hva_slots[slot - 1], 0, npages * guest_page_size);
345 
346 		guest_addr += npages * guest_page_size;
347 	}
348 
349 	virt_map(data->vm, MEM_GPA, MEM_GPA, data->npages);
350 
351 	sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
352 	atomic_init(&sync->start_flag, false);
353 	atomic_init(&sync->exit_flag, false);
354 	atomic_init(&sync->sync_flag, false);
355 
356 	data->mmio_ok = false;
357 
358 	return true;
359 }
360 
361 static void launch_vm(struct vm_data *data)
362 {
363 	pr_info_v("Launching the test VM\n");
364 
365 	pthread_create(&data->vcpu_thread, NULL, vcpu_worker, data);
366 
367 	/* Ensure the guest thread is spun up. */
368 	wait_for_vcpu();
369 }
370 
371 static void free_vm(struct vm_data *data)
372 {
373 	kvm_vm_free(data->vm);
374 	free(data->hva_slots);
375 	free(data);
376 }
377 
378 static void wait_guest_exit(struct vm_data *data)
379 {
380 	pthread_join(data->vcpu_thread, NULL);
381 }
382 
383 static void let_guest_run(struct sync_area *sync)
384 {
385 	atomic_store_explicit(&sync->start_flag, true, memory_order_release);
386 }
387 
388 static void guest_spin_until_start(void)
389 {
390 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
391 
392 	while (!atomic_load_explicit(&sync->start_flag, memory_order_acquire))
393 		;
394 }
395 
396 static void make_guest_exit(struct sync_area *sync)
397 {
398 	atomic_store_explicit(&sync->exit_flag, true, memory_order_release);
399 }
400 
401 static bool _guest_should_exit(void)
402 {
403 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
404 
405 	return atomic_load_explicit(&sync->exit_flag, memory_order_acquire);
406 }
407 
408 #define guest_should_exit() unlikely(_guest_should_exit())
409 
410 /*
411  * noinline so we can easily see how much time the host spends waiting
412  * for the guest.
413  * For the same reason use alarm() instead of polling clock_gettime()
414  * to implement a wait timeout.
415  */
416 static noinline void host_perform_sync(struct sync_area *sync)
417 {
418 	alarm(2);
419 
420 	atomic_store_explicit(&sync->sync_flag, true, memory_order_release);
421 	while (atomic_load_explicit(&sync->sync_flag, memory_order_acquire))
422 		;
423 
424 	alarm(0);
425 }
426 
427 static bool guest_perform_sync(void)
428 {
429 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
430 	bool expected;
431 
432 	do {
433 		if (guest_should_exit())
434 			return false;
435 
436 		expected = true;
437 	} while (!atomic_compare_exchange_weak_explicit(&sync->sync_flag,
438 							&expected, false,
439 							memory_order_acq_rel,
440 							memory_order_relaxed));
441 
442 	return true;
443 }
444 
445 static void guest_code_test_memslot_move(void)
446 {
447 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
448 	uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
449 	uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);
450 
451 	GUEST_SYNC(0);
452 
453 	guest_spin_until_start();
454 
455 	while (!guest_should_exit()) {
456 		uintptr_t ptr;
457 
458 		for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;
459 		     ptr += page_size)
460 			*(uint64_t *)ptr = MEM_TEST_VAL_1;
461 
462 		/*
463 		 * No host sync here since the MMIO exits are so expensive
464 		 * that the host would spend most of its time waiting for
465 		 * the guest and so instead of measuring memslot move
466 		 * performance we would measure the performance and
467 		 * likelihood of MMIO exits
468 		 */
469 	}
470 
471 	GUEST_DONE();
472 }
473 
474 static void guest_code_test_memslot_map(void)
475 {
476 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
477 	uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
478 
479 	GUEST_SYNC(0);
480 
481 	guest_spin_until_start();
482 
483 	while (1) {
484 		uintptr_t ptr;
485 
486 		for (ptr = MEM_TEST_GPA;
487 		     ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
488 		     ptr += page_size)
489 			*(uint64_t *)ptr = MEM_TEST_VAL_1;
490 
491 		if (!guest_perform_sync())
492 			break;
493 
494 		for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
495 		     ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE;
496 		     ptr += page_size)
497 			*(uint64_t *)ptr = MEM_TEST_VAL_2;
498 
499 		if (!guest_perform_sync())
500 			break;
501 	}
502 
503 	GUEST_DONE();
504 }
505 
506 static void guest_code_test_memslot_unmap(void)
507 {
508 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
509 
510 	GUEST_SYNC(0);
511 
512 	guest_spin_until_start();
513 
514 	while (1) {
515 		uintptr_t ptr = MEM_TEST_GPA;
516 
517 		/*
518 		 * We can afford to access (map) just a small number of pages
519 		 * per host sync as otherwise the host will spend
520 		 * a significant amount of its time waiting for the guest
521 		 * (instead of doing unmap operations), so this will
522 		 * effectively turn this test into a map performance test.
523 		 *
524 		 * Just access a single page to be on the safe side.
525 		 */
526 		*(uint64_t *)ptr = MEM_TEST_VAL_1;
527 
528 		if (!guest_perform_sync())
529 			break;
530 
531 		ptr += MEM_TEST_UNMAP_SIZE / 2;
532 		*(uint64_t *)ptr = MEM_TEST_VAL_2;
533 
534 		if (!guest_perform_sync())
535 			break;
536 	}
537 
538 	GUEST_DONE();
539 }
540 
541 static void guest_code_test_memslot_rw(void)
542 {
543 	struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
544 	uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
545 
546 	GUEST_SYNC(0);
547 
548 	guest_spin_until_start();
549 
550 	while (1) {
551 		uintptr_t ptr;
552 
553 		for (ptr = MEM_TEST_GPA;
554 		     ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size)
555 			*(uint64_t *)ptr = MEM_TEST_VAL_1;
556 
557 		if (!guest_perform_sync())
558 			break;
559 
560 		for (ptr = MEM_TEST_GPA + page_size / 2;
561 		     ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size) {
562 			uint64_t val = *(uint64_t *)ptr;
563 
564 			GUEST_ASSERT_1(val == MEM_TEST_VAL_2, val);
565 			*(uint64_t *)ptr = 0;
566 		}
567 
568 		if (!guest_perform_sync())
569 			break;
570 	}
571 
572 	GUEST_DONE();
573 }
574 
575 static bool test_memslot_move_prepare(struct vm_data *data,
576 				      struct sync_area *sync,
577 				      uint64_t *maxslots, bool isactive)
578 {
579 	uint32_t guest_page_size = data->vm->page_size;
580 	uint64_t movesrcgpa, movetestgpa;
581 
582 	movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
583 
584 	if (isactive) {
585 		uint64_t lastpages;
586 
587 		vm_gpa2hva(data, movesrcgpa, &lastpages);
588 		if (lastpages * guest_page_size < MEM_TEST_MOVE_SIZE / 2) {
589 			*maxslots = 0;
590 			return false;
591 		}
592 	}
593 
594 	movetestgpa = movesrcgpa - (MEM_TEST_MOVE_SIZE / (isactive ? 2 : 1));
595 	sync->move_area_ptr = (void *)movetestgpa;
596 
597 	if (isactive) {
598 		data->mmio_ok = true;
599 		data->mmio_gpa_min = movesrcgpa;
600 		data->mmio_gpa_max = movesrcgpa + MEM_TEST_MOVE_SIZE / 2 - 1;
601 	}
602 
603 	return true;
604 }
605 
606 static bool test_memslot_move_prepare_active(struct vm_data *data,
607 					     struct sync_area *sync,
608 					     uint64_t *maxslots)
609 {
610 	return test_memslot_move_prepare(data, sync, maxslots, true);
611 }
612 
613 static bool test_memslot_move_prepare_inactive(struct vm_data *data,
614 					       struct sync_area *sync,
615 					       uint64_t *maxslots)
616 {
617 	return test_memslot_move_prepare(data, sync, maxslots, false);
618 }
619 
620 static void test_memslot_move_loop(struct vm_data *data, struct sync_area *sync)
621 {
622 	uint64_t movesrcgpa;
623 
624 	movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
625 	vm_mem_region_move(data->vm, data->nslots - 1 + 1,
626 			   MEM_TEST_MOVE_GPA_DEST);
627 	vm_mem_region_move(data->vm, data->nslots - 1 + 1, movesrcgpa);
628 }
629 
630 static void test_memslot_do_unmap(struct vm_data *data,
631 				  uint64_t offsp, uint64_t count)
632 {
633 	uint64_t gpa, ctr;
634 	uint32_t guest_page_size = data->vm->page_size;
635 
636 	for (gpa = MEM_TEST_GPA + offsp * guest_page_size, ctr = 0; ctr < count; ) {
637 		uint64_t npages;
638 		void *hva;
639 		int ret;
640 
641 		hva = vm_gpa2hva(data, gpa, &npages);
642 		TEST_ASSERT(npages, "Empty memory slot at gptr 0x%"PRIx64, gpa);
643 		npages = min(npages, count - ctr);
644 		ret = madvise(hva, npages * guest_page_size, MADV_DONTNEED);
645 		TEST_ASSERT(!ret,
646 			    "madvise(%p, MADV_DONTNEED) on VM memory should not fail for gptr 0x%"PRIx64,
647 			    hva, gpa);
648 		ctr += npages;
649 		gpa += npages * guest_page_size;
650 	}
651 	TEST_ASSERT(ctr == count,
652 		    "madvise(MADV_DONTNEED) should exactly cover all of the requested area");
653 }
654 
655 static void test_memslot_map_unmap_check(struct vm_data *data,
656 					 uint64_t offsp, uint64_t valexp)
657 {
658 	uint64_t gpa;
659 	uint64_t *val;
660 	uint32_t guest_page_size = data->vm->page_size;
661 
662 	if (!map_unmap_verify)
663 		return;
664 
665 	gpa = MEM_TEST_GPA + offsp * guest_page_size;
666 	val = (typeof(val))vm_gpa2hva(data, gpa, NULL);
667 	TEST_ASSERT(*val == valexp,
668 		    "Guest written values should read back correctly before unmap (%"PRIu64" vs %"PRIu64" @ %"PRIx64")",
669 		    *val, valexp, gpa);
670 	*val = 0;
671 }
672 
673 static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
674 {
675 	uint32_t guest_page_size = data->vm->page_size;
676 	uint64_t guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
677 
678 	/*
679 	 * Unmap the second half of the test area while guest writes to (maps)
680 	 * the first half.
681 	 */
682 	test_memslot_do_unmap(data, guest_pages / 2, guest_pages / 2);
683 
684 	/*
685 	 * Wait for the guest to finish writing the first half of the test
686 	 * area, verify the written value on the first and the last page of
687 	 * this area and then unmap it.
688 	 * Meanwhile, the guest is writing to (mapping) the second half of
689 	 * the test area.
690 	 */
691 	host_perform_sync(sync);
692 	test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
693 	test_memslot_map_unmap_check(data, guest_pages / 2 - 1, MEM_TEST_VAL_1);
694 	test_memslot_do_unmap(data, 0, guest_pages / 2);
695 
696 
697 	/*
698 	 * Wait for the guest to finish writing the second half of the test
699 	 * area and verify the written value on the first and the last page
700 	 * of this area.
701 	 * The area will be unmapped at the beginning of the next loop
702 	 * iteration.
703 	 * Meanwhile, the guest is writing to (mapping) the first half of
704 	 * the test area.
705 	 */
706 	host_perform_sync(sync);
707 	test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
708 	test_memslot_map_unmap_check(data, guest_pages - 1, MEM_TEST_VAL_2);
709 }
710 
711 static void test_memslot_unmap_loop_common(struct vm_data *data,
712 					   struct sync_area *sync,
713 					   uint64_t chunk)
714 {
715 	uint32_t guest_page_size = data->vm->page_size;
716 	uint64_t guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
717 	uint64_t ctr;
718 
719 	/*
720 	 * Wait for the guest to finish mapping page(s) in the first half
721 	 * of the test area, verify the written value and then perform unmap
722 	 * of this area.
723 	 * Meanwhile, the guest is writing to (mapping) page(s) in the second
724 	 * half of the test area.
725 	 */
726 	host_perform_sync(sync);
727 	test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
728 	for (ctr = 0; ctr < guest_pages / 2; ctr += chunk)
729 		test_memslot_do_unmap(data, ctr, chunk);
730 
731 	/* Likewise, but for the opposite host / guest areas */
732 	host_perform_sync(sync);
733 	test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
734 	for (ctr = guest_pages / 2; ctr < guest_pages; ctr += chunk)
735 		test_memslot_do_unmap(data, ctr, chunk);
736 }
737 
738 static void test_memslot_unmap_loop(struct vm_data *data,
739 				    struct sync_area *sync)
740 {
741 	uint32_t host_page_size = getpagesize();
742 	uint32_t guest_page_size = data->vm->page_size;
743 	uint64_t guest_chunk_pages = guest_page_size >= host_page_size ?
744 					1 : host_page_size / guest_page_size;
745 
746 	test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
747 }
748 
749 static void test_memslot_unmap_loop_chunked(struct vm_data *data,
750 					    struct sync_area *sync)
751 {
752 	uint32_t guest_page_size = data->vm->page_size;
753 	uint64_t guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
754 
755 	test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
756 }
757 
758 static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
759 {
760 	uint64_t gptr;
761 	uint32_t guest_page_size = data->vm->page_size;
762 
763 	for (gptr = MEM_TEST_GPA + guest_page_size / 2;
764 	     gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size)
765 		*(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
766 
767 	host_perform_sync(sync);
768 
769 	for (gptr = MEM_TEST_GPA;
770 	     gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size) {
771 		uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
772 		uint64_t val = *vptr;
773 
774 		TEST_ASSERT(val == MEM_TEST_VAL_1,
775 			    "Guest written values should read back correctly (is %"PRIu64" @ %"PRIx64")",
776 			    val, gptr);
777 		*vptr = 0;
778 	}
779 
780 	host_perform_sync(sync);
781 }
782 
783 struct test_data {
784 	const char *name;
785 	uint64_t mem_size;
786 	void (*guest_code)(void);
787 	bool (*prepare)(struct vm_data *data, struct sync_area *sync,
788 			uint64_t *maxslots);
789 	void (*loop)(struct vm_data *data, struct sync_area *sync);
790 };
791 
792 static bool test_execute(int nslots, uint64_t *maxslots,
793 			 unsigned int maxtime,
794 			 const struct test_data *tdata,
795 			 uint64_t *nloops,
796 			 struct timespec *slot_runtime,
797 			 struct timespec *guest_runtime)
798 {
799 	uint64_t mem_size = tdata->mem_size ? : MEM_SIZE;
800 	struct vm_data *data;
801 	struct sync_area *sync;
802 	struct timespec tstart;
803 	bool ret = true;
804 
805 	data = alloc_vm();
806 	if (!prepare_vm(data, nslots, maxslots, tdata->guest_code,
807 			mem_size, slot_runtime)) {
808 		ret = false;
809 		goto exit_free;
810 	}
811 
812 	sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
813 
814 	sync->guest_page_size = data->vm->page_size;
815 	if (tdata->prepare &&
816 	    !tdata->prepare(data, sync, maxslots)) {
817 		ret = false;
818 		goto exit_free;
819 	}
820 
821 	launch_vm(data);
822 
823 	clock_gettime(CLOCK_MONOTONIC, &tstart);
824 	let_guest_run(sync);
825 
826 	while (1) {
827 		*guest_runtime = timespec_elapsed(tstart);
828 		if (guest_runtime->tv_sec >= maxtime)
829 			break;
830 
831 		tdata->loop(data, sync);
832 
833 		(*nloops)++;
834 	}
835 
836 	make_guest_exit(sync);
837 	wait_guest_exit(data);
838 
839 exit_free:
840 	free_vm(data);
841 
842 	return ret;
843 }
844 
845 static const struct test_data tests[] = {
846 	{
847 		.name = "map",
848 		.mem_size = MEM_SIZE_MAP,
849 		.guest_code = guest_code_test_memslot_map,
850 		.loop = test_memslot_map_loop,
851 	},
852 	{
853 		.name = "unmap",
854 		.mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
855 		.guest_code = guest_code_test_memslot_unmap,
856 		.loop = test_memslot_unmap_loop,
857 	},
858 	{
859 		.name = "unmap chunked",
860 		.mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
861 		.guest_code = guest_code_test_memslot_unmap,
862 		.loop = test_memslot_unmap_loop_chunked,
863 	},
864 	{
865 		.name = "move active area",
866 		.guest_code = guest_code_test_memslot_move,
867 		.prepare = test_memslot_move_prepare_active,
868 		.loop = test_memslot_move_loop,
869 	},
870 	{
871 		.name = "move inactive area",
872 		.guest_code = guest_code_test_memslot_move,
873 		.prepare = test_memslot_move_prepare_inactive,
874 		.loop = test_memslot_move_loop,
875 	},
876 	{
877 		.name = "RW",
878 		.guest_code = guest_code_test_memslot_rw,
879 		.loop = test_memslot_rw_loop
880 	},
881 };
882 
883 #define NTESTS ARRAY_SIZE(tests)
884 
885 struct test_args {
886 	int tfirst;
887 	int tlast;
888 	int nslots;
889 	int seconds;
890 	int runs;
891 };
892 
893 static void help(char *name, struct test_args *targs)
894 {
895 	int ctr;
896 
897 	pr_info("usage: %s [-h] [-v] [-d] [-s slots] [-f first_test] [-e last_test] [-l test_length] [-r run_count]\n",
898 		name);
899 	pr_info(" -h: print this help screen.\n");
900 	pr_info(" -v: enable verbose mode (not for benchmarking).\n");
901 	pr_info(" -d: enable extra debug checks.\n");
902 	pr_info(" -s: specify memslot count cap (-1 means no cap; currently: %i)\n",
903 		targs->nslots);
904 	pr_info(" -f: specify the first test to run (currently: %i; max %zu)\n",
905 		targs->tfirst, NTESTS - 1);
906 	pr_info(" -e: specify the last test to run (currently: %i; max %zu)\n",
907 		targs->tlast, NTESTS - 1);
908 	pr_info(" -l: specify the test length in seconds (currently: %i)\n",
909 		targs->seconds);
910 	pr_info(" -r: specify the number of runs per test (currently: %i)\n",
911 		targs->runs);
912 
913 	pr_info("\nAvailable tests:\n");
914 	for (ctr = 0; ctr < NTESTS; ctr++)
915 		pr_info("%d: %s\n", ctr, tests[ctr].name);
916 }
917 
918 static bool check_memory_sizes(void)
919 {
920 	uint32_t host_page_size = getpagesize();
921 	uint32_t guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
922 
923 	if (host_page_size > SZ_64K || guest_page_size > SZ_64K) {
924 		pr_info("Unsupported page size on host (0x%x) or guest (0x%x)\n",
925 			host_page_size, guest_page_size);
926 		return false;
927 	}
928 
929 	if (MEM_SIZE % guest_page_size ||
930 	    MEM_TEST_SIZE % guest_page_size) {
931 		pr_info("invalid MEM_SIZE or MEM_TEST_SIZE\n");
932 		return false;
933 	}
934 
935 	if (MEM_SIZE_MAP % guest_page_size		||
936 	    MEM_TEST_MAP_SIZE % guest_page_size		||
937 	    (MEM_TEST_MAP_SIZE / guest_page_size) <= 2	||
938 	    (MEM_TEST_MAP_SIZE / guest_page_size) % 2) {
939 		pr_info("invalid MEM_SIZE_MAP or MEM_TEST_MAP_SIZE\n");
940 		return false;
941 	}
942 
943 	if (MEM_TEST_UNMAP_SIZE > MEM_TEST_SIZE		||
944 	    MEM_TEST_UNMAP_SIZE % guest_page_size	||
945 	    (MEM_TEST_UNMAP_SIZE / guest_page_size) %
946 	    (2 * MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size)) {
947 		pr_info("invalid MEM_TEST_UNMAP_SIZE or MEM_TEST_UNMAP_CHUNK_SIZE\n");
948 		return false;
949 	}
950 
951 	return true;
952 }
953 
954 static bool parse_args(int argc, char *argv[],
955 		       struct test_args *targs)
956 {
957 	uint32_t max_mem_slots;
958 	int opt;
959 
960 	while ((opt = getopt(argc, argv, "hvds:f:e:l:r:")) != -1) {
961 		switch (opt) {
962 		case 'h':
963 		default:
964 			help(argv[0], targs);
965 			return false;
966 		case 'v':
967 			verbose = true;
968 			break;
969 		case 'd':
970 			map_unmap_verify = true;
971 			break;
972 		case 's':
973 			targs->nslots = atoi_paranoid(optarg);
974 			if (targs->nslots <= 1 && targs->nslots != -1) {
975 				pr_info("Slot count cap must be larger than 1 or -1 for no cap\n");
976 				return false;
977 			}
978 			break;
979 		case 'f':
980 			targs->tfirst = atoi_non_negative("First test", optarg);
981 			break;
982 		case 'e':
983 			targs->tlast = atoi_non_negative("Last test", optarg);
984 			if (targs->tlast >= NTESTS) {
985 				pr_info("Last test to run has to be non-negative and less than %zu\n",
986 					NTESTS);
987 				return false;
988 			}
989 			break;
990 		case 'l':
991 			targs->seconds = atoi_non_negative("Test length", optarg);
992 			break;
993 		case 'r':
994 			targs->runs = atoi_positive("Runs per test", optarg);
995 			break;
996 		}
997 	}
998 
999 	if (optind < argc) {
1000 		help(argv[0], targs);
1001 		return false;
1002 	}
1003 
1004 	if (targs->tfirst > targs->tlast) {
1005 		pr_info("First test to run cannot be greater than the last test to run\n");
1006 		return false;
1007 	}
1008 
1009 	max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
1010 	if (max_mem_slots <= 1) {
1011 		pr_info("KVM_CAP_NR_MEMSLOTS should be greater than 1\n");
1012 		return false;
1013 	}
1014 
1015 	/* Memory slot 0 is reserved */
1016 	if (targs->nslots == -1)
1017 		targs->nslots = max_mem_slots - 1;
1018 	else
1019 		targs->nslots = min_t(int, targs->nslots, max_mem_slots) - 1;
1020 
1021 	pr_info_v("Allowed Number of memory slots: %"PRIu32"\n",
1022 		  targs->nslots + 1);
1023 
1024 	return true;
1025 }
1026 
1027 struct test_result {
1028 	struct timespec slot_runtime, guest_runtime, iter_runtime;
1029 	int64_t slottimens, runtimens;
1030 	uint64_t nloops;
1031 };
1032 
1033 static bool test_loop(const struct test_data *data,
1034 		      const struct test_args *targs,
1035 		      struct test_result *rbestslottime,
1036 		      struct test_result *rbestruntime)
1037 {
1038 	uint64_t maxslots;
1039 	struct test_result result;
1040 
1041 	result.nloops = 0;
1042 	if (!test_execute(targs->nslots, &maxslots, targs->seconds, data,
1043 			  &result.nloops,
1044 			  &result.slot_runtime, &result.guest_runtime)) {
1045 		if (maxslots)
1046 			pr_info("Memslot count too high for this test, decrease the cap (max is %"PRIu64")\n",
1047 				maxslots);
1048 		else
1049 			pr_info("Memslot count may be too high for this test, try adjusting the cap\n");
1050 
1051 		return false;
1052 	}
1053 
1054 	pr_info("Test took %ld.%.9lds for slot setup + %ld.%.9lds all iterations\n",
1055 		result.slot_runtime.tv_sec, result.slot_runtime.tv_nsec,
1056 		result.guest_runtime.tv_sec, result.guest_runtime.tv_nsec);
1057 	if (!result.nloops) {
1058 		pr_info("No full loops done - too short test time or system too loaded?\n");
1059 		return true;
1060 	}
1061 
1062 	result.iter_runtime = timespec_div(result.guest_runtime,
1063 					   result.nloops);
1064 	pr_info("Done %"PRIu64" iterations, avg %ld.%.9lds each\n",
1065 		result.nloops,
1066 		result.iter_runtime.tv_sec,
1067 		result.iter_runtime.tv_nsec);
1068 	result.slottimens = timespec_to_ns(result.slot_runtime);
1069 	result.runtimens = timespec_to_ns(result.iter_runtime);
1070 
1071 	/*
1072 	 * Only rank the slot setup time for tests using the whole test memory
1073 	 * area so they are comparable
1074 	 */
1075 	if (!data->mem_size &&
1076 	    (!rbestslottime->slottimens ||
1077 	     result.slottimens < rbestslottime->slottimens))
1078 		*rbestslottime = result;
1079 	if (!rbestruntime->runtimens ||
1080 	    result.runtimens < rbestruntime->runtimens)
1081 		*rbestruntime = result;
1082 
1083 	return true;
1084 }
1085 
1086 int main(int argc, char *argv[])
1087 {
1088 	struct test_args targs = {
1089 		.tfirst = 0,
1090 		.tlast = NTESTS - 1,
1091 		.nslots = -1,
1092 		.seconds = 5,
1093 		.runs = 1,
1094 	};
1095 	struct test_result rbestslottime;
1096 	int tctr;
1097 
1098 	if (!check_memory_sizes())
1099 		return -1;
1100 
1101 	if (!parse_args(argc, argv, &targs))
1102 		return -1;
1103 
1104 	rbestslottime.slottimens = 0;
1105 	for (tctr = targs.tfirst; tctr <= targs.tlast; tctr++) {
1106 		const struct test_data *data = &tests[tctr];
1107 		unsigned int runctr;
1108 		struct test_result rbestruntime;
1109 
1110 		if (tctr > targs.tfirst)
1111 			pr_info("\n");
1112 
1113 		pr_info("Testing %s performance with %i runs, %d seconds each\n",
1114 			data->name, targs.runs, targs.seconds);
1115 
1116 		rbestruntime.runtimens = 0;
1117 		for (runctr = 0; runctr < targs.runs; runctr++)
1118 			if (!test_loop(data, &targs,
1119 				       &rbestslottime, &rbestruntime))
1120 				break;
1121 
1122 		if (rbestruntime.runtimens)
1123 			pr_info("Best runtime result was %ld.%.9lds per iteration (with %"PRIu64" iterations)\n",
1124 				rbestruntime.iter_runtime.tv_sec,
1125 				rbestruntime.iter_runtime.tv_nsec,
1126 				rbestruntime.nloops);
1127 	}
1128 
1129 	if (rbestslottime.slottimens)
1130 		pr_info("Best slot setup time for the whole test area was %ld.%.9lds\n",
1131 			rbestslottime.slot_runtime.tv_sec,
1132 			rbestslottime.slot_runtime.tv_nsec);
1133 
1134 	return 0;
1135 }
1136