1 // SPDX-License-Identifier: GPL-2.0
2 #define _GNU_SOURCE /* for program_invocation_short_name */
3 #include <fcntl.h>
4 #include <pthread.h>
5 #include <sched.h>
6 #include <semaphore.h>
7 #include <signal.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <sys/ioctl.h>
12 #include <sys/mman.h>
13 
14 #include <linux/compiler.h>
15 
16 #include <test_util.h>
17 #include <kvm_util.h>
18 #include <processor.h>
19 
20 #define VCPU_ID 0
21 
22 /*
23  * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a
24  * 2MB sized and aligned region so that the initial region corresponds to
25  * exactly one large page.
26  */
27 #define MEM_REGION_SIZE		0x200000
28 
29 #ifdef __x86_64__
30 /*
31  * Somewhat arbitrary location and slot, intended to not overlap anything.
32  */
33 #define MEM_REGION_GPA		0xc0000000
34 #define MEM_REGION_SLOT		10
35 
36 static const uint64_t MMIO_VAL = 0xbeefull;
37 
38 extern const uint64_t final_rip_start;
39 extern const uint64_t final_rip_end;
40 
41 static sem_t vcpu_ready;
42 
43 static inline uint64_t guest_spin_on_val(uint64_t spin_val)
44 {
45 	uint64_t val;
46 
47 	do {
48 		val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA));
49 	} while (val == spin_val);
50 
51 	GUEST_SYNC(0);
52 	return val;
53 }
54 
55 static void *vcpu_worker(void *data)
56 {
57 	struct kvm_vm *vm = data;
58 	struct kvm_run *run;
59 	struct ucall uc;
60 	uint64_t cmd;
61 
62 	/*
63 	 * Loop until the guest is done.  Re-enter the guest on all MMIO exits,
64 	 * which will occur if the guest attempts to access a memslot after it
65 	 * has been deleted or while it is being moved .
66 	 */
67 	run = vcpu_state(vm, VCPU_ID);
68 
69 	while (1) {
70 		vcpu_run(vm, VCPU_ID);
71 
72 		if (run->exit_reason == KVM_EXIT_IO) {
73 			cmd = get_ucall(vm, VCPU_ID, &uc);
74 			if (cmd != UCALL_SYNC)
75 				break;
76 
77 			sem_post(&vcpu_ready);
78 			continue;
79 		}
80 
81 		if (run->exit_reason != KVM_EXIT_MMIO)
82 			break;
83 
84 		TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write");
85 		TEST_ASSERT(run->mmio.len == 8,
86 			    "Unexpected exit mmio size = %u", run->mmio.len);
87 
88 		TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA,
89 			    "Unexpected exit mmio address = 0x%llx",
90 			    run->mmio.phys_addr);
91 		memcpy(run->mmio.data, &MMIO_VAL, 8);
92 	}
93 
94 	if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT)
95 		TEST_FAIL("%s at %s:%ld, val = %lu", (const char *)uc.args[0],
96 			  __FILE__, uc.args[1], uc.args[2]);
97 
98 	return NULL;
99 }
100 
101 static void wait_for_vcpu(void)
102 {
103 	struct timespec ts;
104 
105 	TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),
106 		    "clock_gettime() failed: %d\n", errno);
107 
108 	ts.tv_sec += 2;
109 	TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),
110 		    "sem_timedwait() failed: %d\n", errno);
111 
112 	/* Wait for the vCPU thread to reenter the guest. */
113 	usleep(100000);
114 }
115 
116 static struct kvm_vm *spawn_vm(pthread_t *vcpu_thread, void *guest_code)
117 {
118 	struct kvm_vm *vm;
119 	uint64_t *hva;
120 	uint64_t gpa;
121 
122 	vm = vm_create_default(VCPU_ID, 0, guest_code);
123 
124 	vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
125 
126 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
127 				    MEM_REGION_GPA, MEM_REGION_SLOT,
128 				    MEM_REGION_SIZE / getpagesize(), 0);
129 
130 	/*
131 	 * Allocate and map two pages so that the GPA accessed by guest_code()
132 	 * stays valid across the memslot move.
133 	 */
134 	gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT);
135 	TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n");
136 
137 	virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2, 0);
138 
139 	/* Ditto for the host mapping so that both pages can be zeroed. */
140 	hva = addr_gpa2hva(vm, MEM_REGION_GPA);
141 	memset(hva, 0, 2 * 4096);
142 
143 	pthread_create(vcpu_thread, NULL, vcpu_worker, vm);
144 
145 	/* Ensure the guest thread is spun up. */
146 	wait_for_vcpu();
147 
148 	return vm;
149 }
150 
151 
152 static void guest_code_move_memory_region(void)
153 {
154 	uint64_t val;
155 
156 	GUEST_SYNC(0);
157 
158 	/*
159 	 * Spin until the memory region is moved to a misaligned address.  This
160 	 * may or may not trigger MMIO, as the window where the memslot is
161 	 * invalid is quite small.
162 	 */
163 	val = guest_spin_on_val(0);
164 	GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val);
165 
166 	/* Spin until the memory region is realigned. */
167 	val = guest_spin_on_val(MMIO_VAL);
168 	GUEST_ASSERT_1(val == 1, val);
169 
170 	GUEST_DONE();
171 }
172 
173 static void test_move_memory_region(void)
174 {
175 	pthread_t vcpu_thread;
176 	struct kvm_vm *vm;
177 	uint64_t *hva;
178 
179 	vm = spawn_vm(&vcpu_thread, guest_code_move_memory_region);
180 
181 	hva = addr_gpa2hva(vm, MEM_REGION_GPA);
182 
183 	/*
184 	 * Shift the region's base GPA.  The guest should not see "2" as the
185 	 * hva->gpa translation is misaligned, i.e. the guest is accessing a
186 	 * different host pfn.
187 	 */
188 	vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096);
189 	WRITE_ONCE(*hva, 2);
190 
191 	/*
192 	 * The guest _might_ see an invalid memslot and trigger MMIO, but it's
193 	 * a tiny window.  Spin and defer the sync until the memslot is
194 	 * restored and guest behavior is once again deterministic.
195 	 */
196 	usleep(100000);
197 
198 	/*
199 	 * Note, value in memory needs to be changed *before* restoring the
200 	 * memslot, else the guest could race the update and see "2".
201 	 */
202 	WRITE_ONCE(*hva, 1);
203 
204 	/* Restore the original base, the guest should see "1". */
205 	vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA);
206 	wait_for_vcpu();
207 	/* Defered sync from when the memslot was misaligned (above). */
208 	wait_for_vcpu();
209 
210 	pthread_join(vcpu_thread, NULL);
211 
212 	kvm_vm_free(vm);
213 }
214 
215 static void guest_code_delete_memory_region(void)
216 {
217 	uint64_t val;
218 
219 	GUEST_SYNC(0);
220 
221 	/* Spin until the memory region is deleted. */
222 	val = guest_spin_on_val(0);
223 	GUEST_ASSERT_1(val == MMIO_VAL, val);
224 
225 	/* Spin until the memory region is recreated. */
226 	val = guest_spin_on_val(MMIO_VAL);
227 	GUEST_ASSERT_1(val == 0, val);
228 
229 	/* Spin until the memory region is deleted. */
230 	val = guest_spin_on_val(0);
231 	GUEST_ASSERT_1(val == MMIO_VAL, val);
232 
233 	asm("1:\n\t"
234 	    ".pushsection .rodata\n\t"
235 	    ".global final_rip_start\n\t"
236 	    "final_rip_start: .quad 1b\n\t"
237 	    ".popsection");
238 
239 	/* Spin indefinitely (until the code memslot is deleted). */
240 	guest_spin_on_val(MMIO_VAL);
241 
242 	asm("1:\n\t"
243 	    ".pushsection .rodata\n\t"
244 	    ".global final_rip_end\n\t"
245 	    "final_rip_end: .quad 1b\n\t"
246 	    ".popsection");
247 
248 	GUEST_ASSERT_1(0, 0);
249 }
250 
251 static void test_delete_memory_region(void)
252 {
253 	pthread_t vcpu_thread;
254 	struct kvm_regs regs;
255 	struct kvm_run *run;
256 	struct kvm_vm *vm;
257 
258 	vm = spawn_vm(&vcpu_thread, guest_code_delete_memory_region);
259 
260 	/* Delete the memory region, the guest should not die. */
261 	vm_mem_region_delete(vm, MEM_REGION_SLOT);
262 	wait_for_vcpu();
263 
264 	/* Recreate the memory region.  The guest should see "0". */
265 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
266 				    MEM_REGION_GPA, MEM_REGION_SLOT,
267 				    MEM_REGION_SIZE / getpagesize(), 0);
268 	wait_for_vcpu();
269 
270 	/* Delete the region again so that there's only one memslot left. */
271 	vm_mem_region_delete(vm, MEM_REGION_SLOT);
272 	wait_for_vcpu();
273 
274 	/*
275 	 * Delete the primary memslot.  This should cause an emulation error or
276 	 * shutdown due to the page tables getting nuked.
277 	 */
278 	vm_mem_region_delete(vm, 0);
279 
280 	pthread_join(vcpu_thread, NULL);
281 
282 	run = vcpu_state(vm, VCPU_ID);
283 
284 	TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN ||
285 		    run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
286 		    "Unexpected exit reason = %d", run->exit_reason);
287 
288 	vcpu_regs_get(vm, VCPU_ID, &regs);
289 
290 	/*
291 	 * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already,
292 	 * so the instruction pointer would point to the reset vector.
293 	 */
294 	if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR)
295 		TEST_ASSERT(regs.rip >= final_rip_start &&
296 			    regs.rip < final_rip_end,
297 			    "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n",
298 			    final_rip_start, final_rip_end, regs.rip);
299 
300 	kvm_vm_free(vm);
301 }
302 
303 static void test_zero_memory_regions(void)
304 {
305 	struct kvm_run *run;
306 	struct kvm_vm *vm;
307 
308 	pr_info("Testing KVM_RUN with zero added memory regions\n");
309 
310 	vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
311 	vm_vcpu_add(vm, VCPU_ID);
312 
313 	TEST_ASSERT(!ioctl(vm_get_fd(vm), KVM_SET_NR_MMU_PAGES, 64),
314 		    "KVM_SET_NR_MMU_PAGES failed, errno = %d\n", errno);
315 	vcpu_run(vm, VCPU_ID);
316 
317 	run = vcpu_state(vm, VCPU_ID);
318 	TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
319 		    "Unexpected exit_reason = %u\n", run->exit_reason);
320 
321 	kvm_vm_free(vm);
322 }
323 #endif /* __x86_64__ */
324 
325 /*
326  * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any
327  * tentative to add further slots should fail.
328  */
329 static void test_add_max_memory_regions(void)
330 {
331 	int ret;
332 	struct kvm_vm *vm;
333 	uint32_t max_mem_slots;
334 	uint32_t slot;
335 	uint64_t guest_addr = 0x0;
336 	uint64_t mem_reg_npages;
337 	void *mem;
338 
339 	max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
340 	TEST_ASSERT(max_mem_slots > 0,
341 		    "KVM_CAP_NR_MEMSLOTS should be greater than 0");
342 	pr_info("Allowed number of memory slots: %i\n", max_mem_slots);
343 
344 	vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
345 
346 	mem_reg_npages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, MEM_REGION_SIZE);
347 
348 	/* Check it can be added memory slots up to the maximum allowed */
349 	pr_info("Adding slots 0..%i, each memory region with %dK size\n",
350 		(max_mem_slots - 1), MEM_REGION_SIZE >> 10);
351 	for (slot = 0; slot < max_mem_slots; slot++) {
352 		vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
353 					    guest_addr, slot, mem_reg_npages,
354 					    0);
355 		guest_addr += MEM_REGION_SIZE;
356 	}
357 
358 	/* Check it cannot be added memory slots beyond the limit */
359 	mem = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE,
360 		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
361 	TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host");
362 
363 	ret = ioctl(vm_get_fd(vm), KVM_SET_USER_MEMORY_REGION,
364 		    &(struct kvm_userspace_memory_region) {slot, 0, guest_addr,
365 		    MEM_REGION_SIZE, (uint64_t) mem});
366 	TEST_ASSERT(ret == -1 && errno == EINVAL,
367 		    "Adding one more memory slot should fail with EINVAL");
368 
369 	munmap(mem, MEM_REGION_SIZE);
370 	kvm_vm_free(vm);
371 }
372 
373 int main(int argc, char *argv[])
374 {
375 #ifdef __x86_64__
376 	int i, loops;
377 #endif
378 
379 	/* Tell stdout not to buffer its content */
380 	setbuf(stdout, NULL);
381 
382 #ifdef __x86_64__
383 	/*
384 	 * FIXME: the zero-memslot test fails on aarch64 and s390x because
385 	 * KVM_RUN fails with ENOEXEC or EFAULT.
386 	 */
387 	test_zero_memory_regions();
388 #endif
389 
390 	test_add_max_memory_regions();
391 
392 #ifdef __x86_64__
393 	if (argc > 1)
394 		loops = atoi(argv[1]);
395 	else
396 		loops = 10;
397 
398 	pr_info("Testing MOVE of in-use region, %d loops\n", loops);
399 	for (i = 0; i < loops; i++)
400 		test_move_memory_region();
401 
402 	pr_info("Testing DELETE of in-use region, %d loops\n", loops);
403 	for (i = 0; i < loops; i++)
404 		test_delete_memory_region();
405 #endif
406 
407 	return 0;
408 }
409