1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * arch_timer.c - Tests the aarch64 timer IRQ functionality
4  *
5  * The test validates both the virtual and physical timer IRQs using
6  * CVAL and TVAL registers. This consitutes the four stages in the test.
7  * The guest's main thread configures the timer interrupt for a stage
8  * and waits for it to fire, with a timeout equal to the timer period.
9  * It asserts that the timeout doesn't exceed the timer period.
10  *
11  * On the other hand, upon receipt of an interrupt, the guest's interrupt
12  * handler validates the interrupt by checking if the architectural state
13  * is in compliance with the specifications.
14  *
15  * The test provides command-line options to configure the timer's
16  * period (-p), number of vCPUs (-n), and iterations per stage (-i).
17  * To stress-test the timer stack even more, an option to migrate the
18  * vCPUs across pCPUs (-m), at a particular rate, is also provided.
19  *
20  * Copyright (c) 2021, Google LLC.
21  */
22 #define _GNU_SOURCE
23 
24 #include <stdlib.h>
25 #include <pthread.h>
26 #include <linux/kvm.h>
27 #include <linux/sizes.h>
28 #include <linux/bitmap.h>
29 #include <sys/sysinfo.h>
30 
31 #include "kvm_util.h"
32 #include "processor.h"
33 #include "delay.h"
34 #include "arch_timer.h"
35 #include "gic.h"
36 #include "vgic.h"
37 
38 #define NR_VCPUS_DEF			4
39 #define NR_TEST_ITERS_DEF		5
40 #define TIMER_TEST_PERIOD_MS_DEF	10
41 #define TIMER_TEST_ERR_MARGIN_US	100
42 #define TIMER_TEST_MIGRATION_FREQ_MS	2
43 
44 struct test_args {
45 	int nr_vcpus;
46 	int nr_iter;
47 	int timer_period_ms;
48 	int migration_freq_ms;
49 	struct kvm_arm_counter_offset offset;
50 };
51 
52 static struct test_args test_args = {
53 	.nr_vcpus = NR_VCPUS_DEF,
54 	.nr_iter = NR_TEST_ITERS_DEF,
55 	.timer_period_ms = TIMER_TEST_PERIOD_MS_DEF,
56 	.migration_freq_ms = TIMER_TEST_MIGRATION_FREQ_MS,
57 	.offset = { .reserved = 1 },
58 };
59 
60 #define msecs_to_usecs(msec)		((msec) * 1000LL)
61 
62 #define GICD_BASE_GPA			0x8000000ULL
63 #define GICR_BASE_GPA			0x80A0000ULL
64 
65 enum guest_stage {
66 	GUEST_STAGE_VTIMER_CVAL = 1,
67 	GUEST_STAGE_VTIMER_TVAL,
68 	GUEST_STAGE_PTIMER_CVAL,
69 	GUEST_STAGE_PTIMER_TVAL,
70 	GUEST_STAGE_MAX,
71 };
72 
73 /* Shared variables between host and guest */
74 struct test_vcpu_shared_data {
75 	int nr_iter;
76 	enum guest_stage guest_stage;
77 	uint64_t xcnt;
78 };
79 
80 static struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
81 static pthread_t pt_vcpu_run[KVM_MAX_VCPUS];
82 static struct test_vcpu_shared_data vcpu_shared_data[KVM_MAX_VCPUS];
83 
84 static int vtimer_irq, ptimer_irq;
85 
86 static unsigned long *vcpu_done_map;
87 static pthread_mutex_t vcpu_done_map_lock;
88 
89 static void
90 guest_configure_timer_action(struct test_vcpu_shared_data *shared_data)
91 {
92 	switch (shared_data->guest_stage) {
93 	case GUEST_STAGE_VTIMER_CVAL:
94 		timer_set_next_cval_ms(VIRTUAL, test_args.timer_period_ms);
95 		shared_data->xcnt = timer_get_cntct(VIRTUAL);
96 		timer_set_ctl(VIRTUAL, CTL_ENABLE);
97 		break;
98 	case GUEST_STAGE_VTIMER_TVAL:
99 		timer_set_next_tval_ms(VIRTUAL, test_args.timer_period_ms);
100 		shared_data->xcnt = timer_get_cntct(VIRTUAL);
101 		timer_set_ctl(VIRTUAL, CTL_ENABLE);
102 		break;
103 	case GUEST_STAGE_PTIMER_CVAL:
104 		timer_set_next_cval_ms(PHYSICAL, test_args.timer_period_ms);
105 		shared_data->xcnt = timer_get_cntct(PHYSICAL);
106 		timer_set_ctl(PHYSICAL, CTL_ENABLE);
107 		break;
108 	case GUEST_STAGE_PTIMER_TVAL:
109 		timer_set_next_tval_ms(PHYSICAL, test_args.timer_period_ms);
110 		shared_data->xcnt = timer_get_cntct(PHYSICAL);
111 		timer_set_ctl(PHYSICAL, CTL_ENABLE);
112 		break;
113 	default:
114 		GUEST_ASSERT(0);
115 	}
116 }
117 
118 static void guest_validate_irq(unsigned int intid,
119 				struct test_vcpu_shared_data *shared_data)
120 {
121 	enum guest_stage stage = shared_data->guest_stage;
122 	uint64_t xcnt = 0, xcnt_diff_us, cval = 0;
123 	unsigned long xctl = 0;
124 	unsigned int timer_irq = 0;
125 	unsigned int accessor;
126 
127 	if (intid == IAR_SPURIOUS)
128 		return;
129 
130 	switch (stage) {
131 	case GUEST_STAGE_VTIMER_CVAL:
132 	case GUEST_STAGE_VTIMER_TVAL:
133 		accessor = VIRTUAL;
134 		timer_irq = vtimer_irq;
135 		break;
136 	case GUEST_STAGE_PTIMER_CVAL:
137 	case GUEST_STAGE_PTIMER_TVAL:
138 		accessor = PHYSICAL;
139 		timer_irq = ptimer_irq;
140 		break;
141 	default:
142 		GUEST_ASSERT(0);
143 		return;
144 	}
145 
146 	xctl = timer_get_ctl(accessor);
147 	if ((xctl & CTL_IMASK) || !(xctl & CTL_ENABLE))
148 		return;
149 
150 	timer_set_ctl(accessor, CTL_IMASK);
151 	xcnt = timer_get_cntct(accessor);
152 	cval = timer_get_cval(accessor);
153 
154 	xcnt_diff_us = cycles_to_usec(xcnt - shared_data->xcnt);
155 
156 	/* Make sure we are dealing with the correct timer IRQ */
157 	GUEST_ASSERT_EQ(intid, timer_irq);
158 
159 	/* Basic 'timer condition met' check */
160 	__GUEST_ASSERT(xcnt >= cval,
161 		       "xcnt = 0x%llx, cval = 0x%llx, xcnt_diff_us = 0x%llx",
162 		       xcnt, cval, xcnt_diff_us);
163 	__GUEST_ASSERT(xctl & CTL_ISTATUS, "xcnt = 0x%llx", xcnt);
164 
165 	WRITE_ONCE(shared_data->nr_iter, shared_data->nr_iter + 1);
166 }
167 
168 static void guest_irq_handler(struct ex_regs *regs)
169 {
170 	unsigned int intid = gic_get_and_ack_irq();
171 	uint32_t cpu = guest_get_vcpuid();
172 	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
173 
174 	guest_validate_irq(intid, shared_data);
175 
176 	gic_set_eoi(intid);
177 }
178 
179 static void guest_run_stage(struct test_vcpu_shared_data *shared_data,
180 				enum guest_stage stage)
181 {
182 	uint32_t irq_iter, config_iter;
183 
184 	shared_data->guest_stage = stage;
185 	shared_data->nr_iter = 0;
186 
187 	for (config_iter = 0; config_iter < test_args.nr_iter; config_iter++) {
188 		/* Setup the next interrupt */
189 		guest_configure_timer_action(shared_data);
190 
191 		/* Setup a timeout for the interrupt to arrive */
192 		udelay(msecs_to_usecs(test_args.timer_period_ms) +
193 			TIMER_TEST_ERR_MARGIN_US);
194 
195 		irq_iter = READ_ONCE(shared_data->nr_iter);
196 		GUEST_ASSERT_EQ(config_iter + 1, irq_iter);
197 	}
198 }
199 
200 static void guest_code(void)
201 {
202 	uint32_t cpu = guest_get_vcpuid();
203 	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[cpu];
204 
205 	local_irq_disable();
206 
207 	gic_init(GIC_V3, test_args.nr_vcpus,
208 		(void *)GICD_BASE_GPA, (void *)GICR_BASE_GPA);
209 
210 	timer_set_ctl(VIRTUAL, CTL_IMASK);
211 	timer_set_ctl(PHYSICAL, CTL_IMASK);
212 
213 	gic_irq_enable(vtimer_irq);
214 	gic_irq_enable(ptimer_irq);
215 	local_irq_enable();
216 
217 	guest_run_stage(shared_data, GUEST_STAGE_VTIMER_CVAL);
218 	guest_run_stage(shared_data, GUEST_STAGE_VTIMER_TVAL);
219 	guest_run_stage(shared_data, GUEST_STAGE_PTIMER_CVAL);
220 	guest_run_stage(shared_data, GUEST_STAGE_PTIMER_TVAL);
221 
222 	GUEST_DONE();
223 }
224 
225 static void *test_vcpu_run(void *arg)
226 {
227 	unsigned int vcpu_idx = (unsigned long)arg;
228 	struct ucall uc;
229 	struct kvm_vcpu *vcpu = vcpus[vcpu_idx];
230 	struct kvm_vm *vm = vcpu->vm;
231 	struct test_vcpu_shared_data *shared_data = &vcpu_shared_data[vcpu_idx];
232 
233 	vcpu_run(vcpu);
234 
235 	/* Currently, any exit from guest is an indication of completion */
236 	pthread_mutex_lock(&vcpu_done_map_lock);
237 	__set_bit(vcpu_idx, vcpu_done_map);
238 	pthread_mutex_unlock(&vcpu_done_map_lock);
239 
240 	switch (get_ucall(vcpu, &uc)) {
241 	case UCALL_SYNC:
242 	case UCALL_DONE:
243 		break;
244 	case UCALL_ABORT:
245 		sync_global_from_guest(vm, *shared_data);
246 		fprintf(stderr, "Guest assert failed,  vcpu %u; stage; %u; iter: %u\n",
247 			vcpu_idx, shared_data->guest_stage, shared_data->nr_iter);
248 		REPORT_GUEST_ASSERT(uc);
249 		break;
250 	default:
251 		TEST_FAIL("Unexpected guest exit\n");
252 	}
253 
254 	return NULL;
255 }
256 
257 static uint32_t test_get_pcpu(void)
258 {
259 	uint32_t pcpu;
260 	unsigned int nproc_conf;
261 	cpu_set_t online_cpuset;
262 
263 	nproc_conf = get_nprocs_conf();
264 	sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset);
265 
266 	/* Randomly find an available pCPU to place a vCPU on */
267 	do {
268 		pcpu = rand() % nproc_conf;
269 	} while (!CPU_ISSET(pcpu, &online_cpuset));
270 
271 	return pcpu;
272 }
273 
274 static int test_migrate_vcpu(unsigned int vcpu_idx)
275 {
276 	int ret;
277 	cpu_set_t cpuset;
278 	uint32_t new_pcpu = test_get_pcpu();
279 
280 	CPU_ZERO(&cpuset);
281 	CPU_SET(new_pcpu, &cpuset);
282 
283 	pr_debug("Migrating vCPU: %u to pCPU: %u\n", vcpu_idx, new_pcpu);
284 
285 	ret = pthread_setaffinity_np(pt_vcpu_run[vcpu_idx],
286 				     sizeof(cpuset), &cpuset);
287 
288 	/* Allow the error where the vCPU thread is already finished */
289 	TEST_ASSERT(ret == 0 || ret == ESRCH,
290 		    "Failed to migrate the vCPU:%u to pCPU: %u; ret: %d\n",
291 		    vcpu_idx, new_pcpu, ret);
292 
293 	return ret;
294 }
295 
296 static void *test_vcpu_migration(void *arg)
297 {
298 	unsigned int i, n_done;
299 	bool vcpu_done;
300 
301 	do {
302 		usleep(msecs_to_usecs(test_args.migration_freq_ms));
303 
304 		for (n_done = 0, i = 0; i < test_args.nr_vcpus; i++) {
305 			pthread_mutex_lock(&vcpu_done_map_lock);
306 			vcpu_done = test_bit(i, vcpu_done_map);
307 			pthread_mutex_unlock(&vcpu_done_map_lock);
308 
309 			if (vcpu_done) {
310 				n_done++;
311 				continue;
312 			}
313 
314 			test_migrate_vcpu(i);
315 		}
316 	} while (test_args.nr_vcpus != n_done);
317 
318 	return NULL;
319 }
320 
321 static void test_run(struct kvm_vm *vm)
322 {
323 	pthread_t pt_vcpu_migration;
324 	unsigned int i;
325 	int ret;
326 
327 	pthread_mutex_init(&vcpu_done_map_lock, NULL);
328 	vcpu_done_map = bitmap_zalloc(test_args.nr_vcpus);
329 	TEST_ASSERT(vcpu_done_map, "Failed to allocate vcpu done bitmap\n");
330 
331 	for (i = 0; i < (unsigned long)test_args.nr_vcpus; i++) {
332 		ret = pthread_create(&pt_vcpu_run[i], NULL, test_vcpu_run,
333 				     (void *)(unsigned long)i);
334 		TEST_ASSERT(!ret, "Failed to create vCPU-%d pthread\n", i);
335 	}
336 
337 	/* Spawn a thread to control the vCPU migrations */
338 	if (test_args.migration_freq_ms) {
339 		srand(time(NULL));
340 
341 		ret = pthread_create(&pt_vcpu_migration, NULL,
342 					test_vcpu_migration, NULL);
343 		TEST_ASSERT(!ret, "Failed to create the migration pthread\n");
344 	}
345 
346 
347 	for (i = 0; i < test_args.nr_vcpus; i++)
348 		pthread_join(pt_vcpu_run[i], NULL);
349 
350 	if (test_args.migration_freq_ms)
351 		pthread_join(pt_vcpu_migration, NULL);
352 
353 	bitmap_free(vcpu_done_map);
354 }
355 
356 static void test_init_timer_irq(struct kvm_vm *vm)
357 {
358 	/* Timer initid should be same for all the vCPUs, so query only vCPU-0 */
359 	vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
360 			     KVM_ARM_VCPU_TIMER_IRQ_PTIMER, &ptimer_irq);
361 	vcpu_device_attr_get(vcpus[0], KVM_ARM_VCPU_TIMER_CTRL,
362 			     KVM_ARM_VCPU_TIMER_IRQ_VTIMER, &vtimer_irq);
363 
364 	sync_global_to_guest(vm, ptimer_irq);
365 	sync_global_to_guest(vm, vtimer_irq);
366 
367 	pr_debug("ptimer_irq: %d; vtimer_irq: %d\n", ptimer_irq, vtimer_irq);
368 }
369 
370 static int gic_fd;
371 
372 static struct kvm_vm *test_vm_create(void)
373 {
374 	struct kvm_vm *vm;
375 	unsigned int i;
376 	int nr_vcpus = test_args.nr_vcpus;
377 
378 	vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
379 
380 	vm_init_descriptor_tables(vm);
381 	vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT, guest_irq_handler);
382 
383 	if (!test_args.offset.reserved) {
384 		if (kvm_has_cap(KVM_CAP_COUNTER_OFFSET))
385 			vm_ioctl(vm, KVM_ARM_SET_COUNTER_OFFSET, &test_args.offset);
386 		else
387 			TEST_FAIL("no support for global offset\n");
388 	}
389 
390 	for (i = 0; i < nr_vcpus; i++)
391 		vcpu_init_descriptor_tables(vcpus[i]);
392 
393 	test_init_timer_irq(vm);
394 	gic_fd = vgic_v3_setup(vm, nr_vcpus, 64, GICD_BASE_GPA, GICR_BASE_GPA);
395 	__TEST_REQUIRE(gic_fd >= 0, "Failed to create vgic-v3");
396 
397 	/* Make all the test's cmdline args visible to the guest */
398 	sync_global_to_guest(vm, test_args);
399 
400 	return vm;
401 }
402 
403 static void test_vm_cleanup(struct kvm_vm *vm)
404 {
405 	close(gic_fd);
406 	kvm_vm_free(vm);
407 }
408 
409 static void test_print_help(char *name)
410 {
411 	pr_info("Usage: %s [-h] [-n nr_vcpus] [-i iterations] [-p timer_period_ms]\n",
412 		name);
413 	pr_info("\t-n: Number of vCPUs to configure (default: %u; max: %u)\n",
414 		NR_VCPUS_DEF, KVM_MAX_VCPUS);
415 	pr_info("\t-i: Number of iterations per stage (default: %u)\n",
416 		NR_TEST_ITERS_DEF);
417 	pr_info("\t-p: Periodicity (in ms) of the guest timer (default: %u)\n",
418 		TIMER_TEST_PERIOD_MS_DEF);
419 	pr_info("\t-m: Frequency (in ms) of vCPUs to migrate to different pCPU. 0 to turn off (default: %u)\n",
420 		TIMER_TEST_MIGRATION_FREQ_MS);
421 	pr_info("\t-o: Counter offset (in counter cycles, default: 0)\n");
422 	pr_info("\t-h: print this help screen\n");
423 }
424 
425 static bool parse_args(int argc, char *argv[])
426 {
427 	int opt;
428 
429 	while ((opt = getopt(argc, argv, "hn:i:p:m:o:")) != -1) {
430 		switch (opt) {
431 		case 'n':
432 			test_args.nr_vcpus = atoi_positive("Number of vCPUs", optarg);
433 			if (test_args.nr_vcpus > KVM_MAX_VCPUS) {
434 				pr_info("Max allowed vCPUs: %u\n",
435 					KVM_MAX_VCPUS);
436 				goto err;
437 			}
438 			break;
439 		case 'i':
440 			test_args.nr_iter = atoi_positive("Number of iterations", optarg);
441 			break;
442 		case 'p':
443 			test_args.timer_period_ms = atoi_positive("Periodicity", optarg);
444 			break;
445 		case 'm':
446 			test_args.migration_freq_ms = atoi_non_negative("Frequency", optarg);
447 			break;
448 		case 'o':
449 			test_args.offset.counter_offset = strtol(optarg, NULL, 0);
450 			test_args.offset.reserved = 0;
451 			break;
452 		case 'h':
453 		default:
454 			goto err;
455 		}
456 	}
457 
458 	return true;
459 
460 err:
461 	test_print_help(argv[0]);
462 	return false;
463 }
464 
465 int main(int argc, char *argv[])
466 {
467 	struct kvm_vm *vm;
468 
469 	if (!parse_args(argc, argv))
470 		exit(KSFT_SKIP);
471 
472 	__TEST_REQUIRE(!test_args.migration_freq_ms || get_nprocs() >= 2,
473 		       "At least two physical CPUs needed for vCPU migration");
474 
475 	vm = test_vm_create();
476 	test_run(vm);
477 	test_vm_cleanup(vm);
478 
479 	return 0;
480 }
481