1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2022 ARM Limited.
4  */
5 
6 #define _GNU_SOURCE
7 #define _POSIX_C_SOURCE 199309L
8 
9 #include <errno.h>
10 #include <getopt.h>
11 #include <poll.h>
12 #include <signal.h>
13 #include <stdbool.h>
14 #include <stddef.h>
15 #include <stdio.h>
16 #include <stdlib.h>
17 #include <string.h>
18 #include <unistd.h>
19 #include <sys/auxv.h>
20 #include <sys/epoll.h>
21 #include <sys/prctl.h>
22 #include <sys/types.h>
23 #include <sys/uio.h>
24 #include <sys/wait.h>
25 #include <asm/hwcap.h>
26 
27 #include "../../kselftest.h"
28 
29 #define MAX_VLS 16
30 
31 struct child_data {
32 	char *name, *output;
33 	pid_t pid;
34 	int stdout;
35 	bool output_seen;
36 	bool exited;
37 	int exit_status;
38 };
39 
40 static int epoll_fd;
41 static struct child_data *children;
42 static struct epoll_event *evs;
43 static int tests;
44 static int num_children;
45 static bool terminate;
46 
47 static int startup_pipe[2];
48 
49 static int num_processors(void)
50 {
51 	long nproc = sysconf(_SC_NPROCESSORS_CONF);
52 	if (nproc < 0) {
53 		perror("Unable to read number of processors\n");
54 		exit(EXIT_FAILURE);
55 	}
56 
57 	return nproc;
58 }
59 
60 static void child_start(struct child_data *child, const char *program)
61 {
62 	int ret, pipefd[2], i;
63 	struct epoll_event ev;
64 
65 	ret = pipe(pipefd);
66 	if (ret != 0)
67 		ksft_exit_fail_msg("Failed to create stdout pipe: %s (%d)\n",
68 				   strerror(errno), errno);
69 
70 	child->pid = fork();
71 	if (child->pid == -1)
72 		ksft_exit_fail_msg("fork() failed: %s (%d)\n",
73 				   strerror(errno), errno);
74 
75 	if (!child->pid) {
76 		/*
77 		 * In child, replace stdout with the pipe, errors to
78 		 * stderr from here as kselftest prints to stdout.
79 		 */
80 		ret = dup2(pipefd[1], 1);
81 		if (ret == -1) {
82 			fprintf(stderr, "dup2() %d\n", errno);
83 			exit(EXIT_FAILURE);
84 		}
85 
86 		/*
87 		 * Duplicate the read side of the startup pipe to
88 		 * FD 3 so we can close everything else.
89 		 */
90 		ret = dup2(startup_pipe[0], 3);
91 		if (ret == -1) {
92 			fprintf(stderr, "dup2() %d\n", errno);
93 			exit(EXIT_FAILURE);
94 		}
95 
96 		/*
97 		 * Very dumb mechanism to clean open FDs other than
98 		 * stdio. We don't want O_CLOEXEC for the pipes...
99 		 */
100 		for (i = 4; i < 8192; i++)
101 			close(i);
102 
103 		/*
104 		 * Read from the startup pipe, there should be no data
105 		 * and we should block until it is closed.  We just
106 		 * carry on on error since this isn't super critical.
107 		 */
108 		ret = read(3, &i, sizeof(i));
109 		if (ret < 0)
110 			fprintf(stderr, "read(startp pipe) failed: %s (%d)\n",
111 				strerror(errno), errno);
112 		if (ret > 0)
113 			fprintf(stderr, "%d bytes of data on startup pipe\n",
114 				ret);
115 		close(3);
116 
117 		ret = execl(program, program, NULL);
118 		fprintf(stderr, "execl(%s) failed: %d (%s)\n",
119 			program, errno, strerror(errno));
120 
121 		exit(EXIT_FAILURE);
122 	} else {
123 		/*
124 		 * In parent, remember the child and close our copy of the
125 		 * write side of stdout.
126 		 */
127 		close(pipefd[1]);
128 		child->stdout = pipefd[0];
129 		child->output = NULL;
130 		child->exited = false;
131 		child->output_seen = false;
132 
133 		ev.events = EPOLLIN | EPOLLHUP;
134 		ev.data.ptr = child;
135 
136 		ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, child->stdout, &ev);
137 		if (ret < 0) {
138 			ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n",
139 					   child->name, strerror(errno), errno);
140 		}
141 	}
142 }
143 
144 static bool child_output_read(struct child_data *child)
145 {
146 	char read_data[1024];
147 	char work[1024];
148 	int ret, len, cur_work, cur_read;
149 
150 	ret = read(child->stdout, read_data, sizeof(read_data));
151 	if (ret < 0) {
152 		if (errno == EINTR)
153 			return true;
154 
155 		ksft_print_msg("%s: read() failed: %s (%d)\n",
156 			       child->name, strerror(errno),
157 			       errno);
158 		return false;
159 	}
160 	len = ret;
161 
162 	child->output_seen = true;
163 
164 	/* Pick up any partial read */
165 	if (child->output) {
166 		strncpy(work, child->output, sizeof(work) - 1);
167 		cur_work = strnlen(work, sizeof(work));
168 		free(child->output);
169 		child->output = NULL;
170 	} else {
171 		cur_work = 0;
172 	}
173 
174 	cur_read = 0;
175 	while (cur_read < len) {
176 		work[cur_work] = read_data[cur_read++];
177 
178 		if (work[cur_work] == '\n') {
179 			work[cur_work] = '\0';
180 			ksft_print_msg("%s: %s\n", child->name, work);
181 			cur_work = 0;
182 		} else {
183 			cur_work++;
184 		}
185 	}
186 
187 	if (cur_work) {
188 		work[cur_work] = '\0';
189 		ret = asprintf(&child->output, "%s", work);
190 		if (ret == -1)
191 			ksft_exit_fail_msg("Out of memory\n");
192 	}
193 
194 	return false;
195 }
196 
197 static void child_output(struct child_data *child, uint32_t events,
198 			 bool flush)
199 {
200 	bool read_more;
201 
202 	if (events & EPOLLIN) {
203 		do {
204 			read_more = child_output_read(child);
205 		} while (read_more);
206 	}
207 
208 	if (events & EPOLLHUP) {
209 		close(child->stdout);
210 		child->stdout = -1;
211 		flush = true;
212 	}
213 
214 	if (flush && child->output) {
215 		ksft_print_msg("%s: %s<EOF>\n", child->name, child->output);
216 		free(child->output);
217 		child->output = NULL;
218 	}
219 }
220 
221 static void child_tickle(struct child_data *child)
222 {
223 	if (child->output_seen && !child->exited)
224 		kill(child->pid, SIGUSR2);
225 }
226 
227 static void child_stop(struct child_data *child)
228 {
229 	if (!child->exited)
230 		kill(child->pid, SIGTERM);
231 }
232 
233 static void child_cleanup(struct child_data *child)
234 {
235 	pid_t ret;
236 	int status;
237 	bool fail = false;
238 
239 	if (!child->exited) {
240 		do {
241 			ret = waitpid(child->pid, &status, 0);
242 			if (ret == -1 && errno == EINTR)
243 				continue;
244 
245 			if (ret == -1) {
246 				ksft_print_msg("waitpid(%d) failed: %s (%d)\n",
247 					       child->pid, strerror(errno),
248 					       errno);
249 				fail = true;
250 				break;
251 			}
252 		} while (!WIFEXITED(status));
253 		child->exit_status = WEXITSTATUS(status);
254 	}
255 
256 	if (!child->output_seen) {
257 		ksft_print_msg("%s no output seen\n", child->name);
258 		fail = true;
259 	}
260 
261 	if (child->exit_status != 0) {
262 		ksft_print_msg("%s exited with error code %d\n",
263 			       child->name, child->exit_status);
264 		fail = true;
265 	}
266 
267 	ksft_test_result(!fail, "%s\n", child->name);
268 }
269 
270 static void handle_child_signal(int sig, siginfo_t *info, void *context)
271 {
272 	int i;
273 	bool found = false;
274 
275 	for (i = 0; i < num_children; i++) {
276 		if (children[i].pid == info->si_pid) {
277 			children[i].exited = true;
278 			children[i].exit_status = info->si_status;
279 			found = true;
280 			break;
281 		}
282 	}
283 
284 	if (!found)
285 		ksft_print_msg("SIGCHLD for unknown PID %d with status %d\n",
286 			       info->si_pid, info->si_status);
287 }
288 
289 static void handle_exit_signal(int sig, siginfo_t *info, void *context)
290 {
291 	int i;
292 
293 	/* If we're already exiting then don't signal again */
294 	if (terminate)
295 		return;
296 
297 	ksft_print_msg("Got signal, exiting...\n");
298 
299 	terminate = true;
300 
301 	/*
302 	 * This should be redundant, the main loop should clean up
303 	 * after us, but for safety stop everything we can here.
304 	 */
305 	for (i = 0; i < num_children; i++)
306 		child_stop(&children[i]);
307 }
308 
309 static void start_fpsimd(struct child_data *child, int cpu, int copy)
310 {
311 	int ret;
312 
313 	ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy);
314 	if (ret == -1)
315 		ksft_exit_fail_msg("asprintf() failed\n");
316 
317 	child_start(child, "./fpsimd-test");
318 
319 	ksft_print_msg("Started %s\n", child->name);
320 }
321 
322 static void start_sve(struct child_data *child, int vl, int cpu)
323 {
324 	int ret;
325 
326 	ret = prctl(PR_SVE_SET_VL, vl | PR_SVE_VL_INHERIT);
327 	if (ret < 0)
328 		ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl);
329 
330 	ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu);
331 	if (ret == -1)
332 		ksft_exit_fail_msg("asprintf() failed\n");
333 
334 	child_start(child, "./sve-test");
335 
336 	ksft_print_msg("Started %s\n", child->name);
337 }
338 
339 static void start_ssve(struct child_data *child, int vl, int cpu)
340 {
341 	int ret;
342 
343 	ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
344 	if (ret == -1)
345 		ksft_exit_fail_msg("asprintf() failed\n");
346 
347 	ret = prctl(PR_SME_SET_VL, vl | PR_SME_VL_INHERIT);
348 	if (ret < 0)
349 		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
350 
351 	child_start(child, "./ssve-test");
352 
353 	ksft_print_msg("Started %s\n", child->name);
354 }
355 
356 static void start_za(struct child_data *child, int vl, int cpu)
357 {
358 	int ret;
359 
360 	ret = prctl(PR_SME_SET_VL, vl | PR_SVE_VL_INHERIT);
361 	if (ret < 0)
362 		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
363 
364 	ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu);
365 	if (ret == -1)
366 		ksft_exit_fail_msg("asprintf() failed\n");
367 
368 	child_start(child, "./za-test");
369 
370 	ksft_print_msg("Started %s\n", child->name);
371 }
372 
373 static void start_zt(struct child_data *child, int cpu)
374 {
375 	int ret;
376 
377 	ret = asprintf(&child->name, "ZT-%d", cpu);
378 	if (ret == -1)
379 		ksft_exit_fail_msg("asprintf() failed\n");
380 
381 	child_start(child, "./zt-test");
382 
383 	ksft_print_msg("Started %s\n", child->name);
384 }
385 
386 static void probe_vls(int vls[], int *vl_count, int set_vl)
387 {
388 	unsigned int vq;
389 	int vl;
390 
391 	*vl_count = 0;
392 
393 	for (vq = SVE_VQ_MAX; vq > 0; vq /= 2) {
394 		vl = prctl(set_vl, vq * 16);
395 		if (vl == -1)
396 			ksft_exit_fail_msg("SET_VL failed: %s (%d)\n",
397 					   strerror(errno), errno);
398 
399 		vl &= PR_SVE_VL_LEN_MASK;
400 
401 		if (*vl_count && (vl == vls[*vl_count - 1]))
402 			break;
403 
404 		vq = sve_vq_from_vl(vl);
405 
406 		vls[*vl_count] = vl;
407 		*vl_count += 1;
408 	}
409 }
410 
411 /* Handle any pending output without blocking */
412 static void drain_output(bool flush)
413 {
414 	int ret = 1;
415 	int i;
416 
417 	while (ret > 0) {
418 		ret = epoll_wait(epoll_fd, evs, tests, 0);
419 		if (ret < 0) {
420 			if (errno == EINTR)
421 				continue;
422 			ksft_print_msg("epoll_wait() failed: %s (%d)\n",
423 				       strerror(errno), errno);
424 		}
425 
426 		for (i = 0; i < ret; i++)
427 			child_output(evs[i].data.ptr, evs[i].events, flush);
428 	}
429 }
430 
431 static const struct option options[] = {
432 	{ "timeout",	required_argument, NULL, 't' },
433 	{ }
434 };
435 
436 int main(int argc, char **argv)
437 {
438 	int ret;
439 	int timeout = 10;
440 	int cpus, i, j, c;
441 	int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
442 	bool all_children_started = false;
443 	int seen_children;
444 	int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
445 	bool have_sme2;
446 	struct sigaction sa;
447 
448 	while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) {
449 		switch (c) {
450 		case 't':
451 			ret = sscanf(optarg, "%d", &timeout);
452 			if (ret != 1)
453 				ksft_exit_fail_msg("Failed to parse timeout %s\n",
454 						   optarg);
455 			break;
456 		default:
457 			ksft_exit_fail_msg("Unknown argument\n");
458 		}
459 	}
460 
461 	cpus = num_processors();
462 	tests = 0;
463 
464 	if (getauxval(AT_HWCAP) & HWCAP_SVE) {
465 		probe_vls(sve_vls, &sve_vl_count, PR_SVE_SET_VL);
466 		tests += sve_vl_count * cpus;
467 	} else {
468 		sve_vl_count = 0;
469 	}
470 
471 	if (getauxval(AT_HWCAP2) & HWCAP2_SME) {
472 		probe_vls(sme_vls, &sme_vl_count, PR_SME_SET_VL);
473 		tests += sme_vl_count * cpus * 2;
474 	} else {
475 		sme_vl_count = 0;
476 	}
477 
478 	if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {
479 		tests += cpus;
480 		have_sme2 = true;
481 	} else {
482 		have_sme2 = false;
483 	}
484 
485 	/* Force context switching if we only have FPSIMD */
486 	if (!sve_vl_count && !sme_vl_count)
487 		fpsimd_per_cpu = 2;
488 	else
489 		fpsimd_per_cpu = 1;
490 	tests += cpus * fpsimd_per_cpu;
491 
492 	ksft_print_header();
493 	ksft_set_plan(tests);
494 
495 	ksft_print_msg("%d CPUs, %d SVE VLs, %d SME VLs, SME2 %s\n",
496 		       cpus, sve_vl_count, sme_vl_count,
497 		       have_sme2 ? "present" : "absent");
498 
499 	if (timeout > 0)
500 		ksft_print_msg("Will run for %ds\n", timeout);
501 	else
502 		ksft_print_msg("Will run until terminated\n");
503 
504 	children = calloc(sizeof(*children), tests);
505 	if (!children)
506 		ksft_exit_fail_msg("Unable to allocate child data\n");
507 
508 	ret = epoll_create1(EPOLL_CLOEXEC);
509 	if (ret < 0)
510 		ksft_exit_fail_msg("epoll_create1() failed: %s (%d)\n",
511 				   strerror(errno), ret);
512 	epoll_fd = ret;
513 
514 	/* Create a pipe which children will block on before execing */
515 	ret = pipe(startup_pipe);
516 	if (ret != 0)
517 		ksft_exit_fail_msg("Failed to create startup pipe: %s (%d)\n",
518 				   strerror(errno), errno);
519 
520 	/* Get signal handers ready before we start any children */
521 	memset(&sa, 0, sizeof(sa));
522 	sa.sa_sigaction = handle_exit_signal;
523 	sa.sa_flags = SA_RESTART | SA_SIGINFO;
524 	sigemptyset(&sa.sa_mask);
525 	ret = sigaction(SIGINT, &sa, NULL);
526 	if (ret < 0)
527 		ksft_print_msg("Failed to install SIGINT handler: %s (%d)\n",
528 			       strerror(errno), errno);
529 	ret = sigaction(SIGTERM, &sa, NULL);
530 	if (ret < 0)
531 		ksft_print_msg("Failed to install SIGTERM handler: %s (%d)\n",
532 			       strerror(errno), errno);
533 	sa.sa_sigaction = handle_child_signal;
534 	ret = sigaction(SIGCHLD, &sa, NULL);
535 	if (ret < 0)
536 		ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n",
537 			       strerror(errno), errno);
538 
539 	evs = calloc(tests, sizeof(*evs));
540 	if (!evs)
541 		ksft_exit_fail_msg("Failed to allocated %d epoll events\n",
542 				   tests);
543 
544 	for (i = 0; i < cpus; i++) {
545 		for (j = 0; j < fpsimd_per_cpu; j++)
546 			start_fpsimd(&children[num_children++], i, j);
547 
548 		for (j = 0; j < sve_vl_count; j++)
549 			start_sve(&children[num_children++], sve_vls[j], i);
550 
551 		for (j = 0; j < sme_vl_count; j++) {
552 			start_ssve(&children[num_children++], sme_vls[j], i);
553 			start_za(&children[num_children++], sme_vls[j], i);
554 		}
555 
556 		if (have_sme2)
557 			start_zt(&children[num_children++], i);
558 	}
559 
560 	/*
561 	 * All children started, close the startup pipe and let them
562 	 * run.
563 	 */
564 	close(startup_pipe[0]);
565 	close(startup_pipe[1]);
566 
567 	for (;;) {
568 		/* Did we get a signal asking us to exit? */
569 		if (terminate)
570 			break;
571 
572 		/*
573 		 * Timeout is counted in seconds with no output, the
574 		 * tests print during startup then are silent when
575 		 * running so this should ensure they all ran enough
576 		 * to install the signal handler, this is especially
577 		 * useful in emulation where we will both be slow and
578 		 * likely to have a large set of VLs.
579 		 */
580 		ret = epoll_wait(epoll_fd, evs, tests, 1000);
581 		if (ret < 0) {
582 			if (errno == EINTR)
583 				continue;
584 			ksft_exit_fail_msg("epoll_wait() failed: %s (%d)\n",
585 					   strerror(errno), errno);
586 		}
587 
588 		/* Output? */
589 		if (ret > 0) {
590 			for (i = 0; i < ret; i++) {
591 				child_output(evs[i].data.ptr, evs[i].events,
592 					     false);
593 			}
594 			continue;
595 		}
596 
597 		/* Otherwise epoll_wait() timed out */
598 
599 		/*
600 		 * If the child processes have not produced output they
601 		 * aren't actually running the tests yet .
602 		 */
603 		if (!all_children_started) {
604 			seen_children = 0;
605 
606 			for (i = 0; i < num_children; i++)
607 				if (children[i].output_seen ||
608 				    children[i].exited)
609 					seen_children++;
610 
611 			if (seen_children != num_children) {
612 				ksft_print_msg("Waiting for %d children\n",
613 					       num_children - seen_children);
614 				continue;
615 			}
616 
617 			all_children_started = true;
618 		}
619 
620 		ksft_print_msg("Sending signals, timeout remaining: %d\n",
621 			       timeout);
622 
623 		for (i = 0; i < num_children; i++)
624 			child_tickle(&children[i]);
625 
626 		/* Negative timeout means run indefinitely */
627 		if (timeout < 0)
628 			continue;
629 		if (--timeout == 0)
630 			break;
631 	}
632 
633 	ksft_print_msg("Finishing up...\n");
634 	terminate = true;
635 
636 	for (i = 0; i < tests; i++)
637 		child_stop(&children[i]);
638 
639 	drain_output(false);
640 
641 	for (i = 0; i < tests; i++)
642 		child_cleanup(&children[i]);
643 
644 	drain_output(true);
645 
646 	ksft_print_cnts();
647 
648 	return 0;
649 }
650