xref: /openbmc/linux/tools/perf/tests/builtin-test.c (revision c2cd9d04)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * builtin-test.c
4  *
5  * Builtin regression testing command: ever growing number of sanity tests
6  */
7 #include <fcntl.h>
8 #include <errno.h>
9 #include <unistd.h>
10 #include <string.h>
11 #include <sys/types.h>
12 #include <dirent.h>
13 #include <sys/wait.h>
14 #include <sys/stat.h>
15 #include "builtin.h"
16 #include "hist.h"
17 #include "intlist.h"
18 #include "tests.h"
19 #include "debug.h"
20 #include "color.h"
21 #include <subcmd/parse-options.h>
22 #include "string2.h"
23 #include "symbol.h"
24 #include <linux/kernel.h>
25 #include <subcmd/exec-cmd.h>
26 
27 static bool dont_fork;
28 
29 struct test __weak arch_tests[] = {
30 	{
31 		.func = NULL,
32 	},
33 };
34 
35 static struct test generic_tests[] = {
36 	{
37 		.desc = "vmlinux symtab matches kallsyms",
38 		.func = test__vmlinux_matches_kallsyms,
39 	},
40 	{
41 		.desc = "Detect openat syscall event",
42 		.func = test__openat_syscall_event,
43 	},
44 	{
45 		.desc = "Detect openat syscall event on all cpus",
46 		.func = test__openat_syscall_event_on_all_cpus,
47 	},
48 	{
49 		.desc = "Read samples using the mmap interface",
50 		.func = test__basic_mmap,
51 	},
52 	{
53 		.desc = "Test data source output",
54 		.func = test__mem,
55 	},
56 	{
57 		.desc = "Parse event definition strings",
58 		.func = test__parse_events,
59 	},
60 	{
61 		.desc = "Simple expression parser",
62 		.func = test__expr,
63 	},
64 	{
65 		.desc = "PERF_RECORD_* events & perf_sample fields",
66 		.func = test__PERF_RECORD,
67 	},
68 	{
69 		.desc = "Parse perf pmu format",
70 		.func = test__pmu,
71 	},
72 	{
73 		.desc = "DSO data read",
74 		.func = test__dso_data,
75 	},
76 	{
77 		.desc = "DSO data cache",
78 		.func = test__dso_data_cache,
79 	},
80 	{
81 		.desc = "DSO data reopen",
82 		.func = test__dso_data_reopen,
83 	},
84 	{
85 		.desc = "Roundtrip evsel->name",
86 		.func = test__perf_evsel__roundtrip_name_test,
87 	},
88 	{
89 		.desc = "Parse sched tracepoints fields",
90 		.func = test__perf_evsel__tp_sched_test,
91 	},
92 	{
93 		.desc = "syscalls:sys_enter_openat event fields",
94 		.func = test__syscall_openat_tp_fields,
95 	},
96 	{
97 		.desc = "Setup struct perf_event_attr",
98 		.func = test__attr,
99 	},
100 	{
101 		.desc = "Match and link multiple hists",
102 		.func = test__hists_link,
103 	},
104 	{
105 		.desc = "'import perf' in python",
106 		.func = test__python_use,
107 	},
108 	{
109 		.desc = "Breakpoint overflow signal handler",
110 		.func = test__bp_signal,
111 		.is_supported = test__bp_signal_is_supported,
112 	},
113 	{
114 		.desc = "Breakpoint overflow sampling",
115 		.func = test__bp_signal_overflow,
116 		.is_supported = test__bp_signal_is_supported,
117 	},
118 	{
119 		.desc = "Breakpoint accounting",
120 		.func = test__bp_accounting,
121 		.is_supported = test__bp_signal_is_supported,
122 	},
123 	{
124 		.desc = "Watchpoint",
125 		.func = test__wp,
126 		.is_supported = test__wp_is_supported,
127 		.subtest = {
128 			.skip_if_fail	= false,
129 			.get_nr		= test__wp_subtest_get_nr,
130 			.get_desc	= test__wp_subtest_get_desc,
131 		},
132 	},
133 	{
134 		.desc = "Number of exit events of a simple workload",
135 		.func = test__task_exit,
136 	},
137 	{
138 		.desc = "Software clock events period values",
139 		.func = test__sw_clock_freq,
140 	},
141 	{
142 		.desc = "Object code reading",
143 		.func = test__code_reading,
144 	},
145 	{
146 		.desc = "Sample parsing",
147 		.func = test__sample_parsing,
148 	},
149 	{
150 		.desc = "Use a dummy software event to keep tracking",
151 		.func = test__keep_tracking,
152 	},
153 	{
154 		.desc = "Parse with no sample_id_all bit set",
155 		.func = test__parse_no_sample_id_all,
156 	},
157 	{
158 		.desc = "Filter hist entries",
159 		.func = test__hists_filter,
160 	},
161 	{
162 		.desc = "Lookup mmap thread",
163 		.func = test__mmap_thread_lookup,
164 	},
165 	{
166 		.desc = "Share thread mg",
167 		.func = test__thread_mg_share,
168 	},
169 	{
170 		.desc = "Sort output of hist entries",
171 		.func = test__hists_output,
172 	},
173 	{
174 		.desc = "Cumulate child hist entries",
175 		.func = test__hists_cumulate,
176 	},
177 	{
178 		.desc = "Track with sched_switch",
179 		.func = test__switch_tracking,
180 	},
181 	{
182 		.desc = "Filter fds with revents mask in a fdarray",
183 		.func = test__fdarray__filter,
184 	},
185 	{
186 		.desc = "Add fd to a fdarray, making it autogrow",
187 		.func = test__fdarray__add,
188 	},
189 	{
190 		.desc = "kmod_path__parse",
191 		.func = test__kmod_path__parse,
192 	},
193 	{
194 		.desc = "Thread map",
195 		.func = test__thread_map,
196 	},
197 	{
198 		.desc = "LLVM search and compile",
199 		.func = test__llvm,
200 		.subtest = {
201 			.skip_if_fail	= true,
202 			.get_nr		= test__llvm_subtest_get_nr,
203 			.get_desc	= test__llvm_subtest_get_desc,
204 		},
205 	},
206 	{
207 		.desc = "Session topology",
208 		.func = test__session_topology,
209 	},
210 	{
211 		.desc = "BPF filter",
212 		.func = test__bpf,
213 		.subtest = {
214 			.skip_if_fail	= true,
215 			.get_nr		= test__bpf_subtest_get_nr,
216 			.get_desc	= test__bpf_subtest_get_desc,
217 		},
218 	},
219 	{
220 		.desc = "Synthesize thread map",
221 		.func = test__thread_map_synthesize,
222 	},
223 	{
224 		.desc = "Remove thread map",
225 		.func = test__thread_map_remove,
226 	},
227 	{
228 		.desc = "Synthesize cpu map",
229 		.func = test__cpu_map_synthesize,
230 	},
231 	{
232 		.desc = "Synthesize stat config",
233 		.func = test__synthesize_stat_config,
234 	},
235 	{
236 		.desc = "Synthesize stat",
237 		.func = test__synthesize_stat,
238 	},
239 	{
240 		.desc = "Synthesize stat round",
241 		.func = test__synthesize_stat_round,
242 	},
243 	{
244 		.desc = "Synthesize attr update",
245 		.func = test__event_update,
246 	},
247 	{
248 		.desc = "Event times",
249 		.func = test__event_times,
250 	},
251 	{
252 		.desc = "Read backward ring buffer",
253 		.func = test__backward_ring_buffer,
254 	},
255 	{
256 		.desc = "Print cpu map",
257 		.func = test__cpu_map_print,
258 	},
259 	{
260 		.desc = "Probe SDT events",
261 		.func = test__sdt_event,
262 	},
263 	{
264 		.desc = "is_printable_array",
265 		.func = test__is_printable_array,
266 	},
267 	{
268 		.desc = "Print bitmap",
269 		.func = test__bitmap_print,
270 	},
271 	{
272 		.desc = "perf hooks",
273 		.func = test__perf_hooks,
274 	},
275 	{
276 		.desc = "builtin clang support",
277 		.func = test__clang,
278 		.subtest = {
279 			.skip_if_fail	= true,
280 			.get_nr		= test__clang_subtest_get_nr,
281 			.get_desc	= test__clang_subtest_get_desc,
282 		}
283 	},
284 	{
285 		.desc = "unit_number__scnprintf",
286 		.func = test__unit_number__scnprint,
287 	},
288 	{
289 		.desc = "mem2node",
290 		.func = test__mem2node,
291 	},
292 	{
293 		.func = NULL,
294 	},
295 };
296 
297 static struct test *tests[] = {
298 	generic_tests,
299 	arch_tests,
300 };
301 
302 static bool perf_test__matches(struct test *test, int curr, int argc, const char *argv[])
303 {
304 	int i;
305 
306 	if (argc == 0)
307 		return true;
308 
309 	for (i = 0; i < argc; ++i) {
310 		char *end;
311 		long nr = strtoul(argv[i], &end, 10);
312 
313 		if (*end == '\0') {
314 			if (nr == curr + 1)
315 				return true;
316 			continue;
317 		}
318 
319 		if (strcasestr(test->desc, argv[i]))
320 			return true;
321 	}
322 
323 	return false;
324 }
325 
326 static int run_test(struct test *test, int subtest)
327 {
328 	int status, err = -1, child = dont_fork ? 0 : fork();
329 	char sbuf[STRERR_BUFSIZE];
330 
331 	if (child < 0) {
332 		pr_err("failed to fork test: %s\n",
333 			str_error_r(errno, sbuf, sizeof(sbuf)));
334 		return -1;
335 	}
336 
337 	if (!child) {
338 		if (!dont_fork) {
339 			pr_debug("test child forked, pid %d\n", getpid());
340 
341 			if (verbose <= 0) {
342 				int nullfd = open("/dev/null", O_WRONLY);
343 
344 				if (nullfd >= 0) {
345 					close(STDERR_FILENO);
346 					close(STDOUT_FILENO);
347 
348 					dup2(nullfd, STDOUT_FILENO);
349 					dup2(STDOUT_FILENO, STDERR_FILENO);
350 					close(nullfd);
351 				}
352 			} else {
353 				signal(SIGSEGV, sighandler_dump_stack);
354 				signal(SIGFPE, sighandler_dump_stack);
355 			}
356 		}
357 
358 		err = test->func(test, subtest);
359 		if (!dont_fork)
360 			exit(err);
361 	}
362 
363 	if (!dont_fork) {
364 		wait(&status);
365 
366 		if (WIFEXITED(status)) {
367 			err = (signed char)WEXITSTATUS(status);
368 			pr_debug("test child finished with %d\n", err);
369 		} else if (WIFSIGNALED(status)) {
370 			err = -1;
371 			pr_debug("test child interrupted\n");
372 		}
373 	}
374 
375 	return err;
376 }
377 
378 #define for_each_test(j, t)	 				\
379 	for (j = 0; j < ARRAY_SIZE(tests); j++)	\
380 		for (t = &tests[j][0]; t->func; t++)
381 
382 static int test_and_print(struct test *t, bool force_skip, int subtest)
383 {
384 	int err;
385 
386 	if (!force_skip) {
387 		pr_debug("\n--- start ---\n");
388 		err = run_test(t, subtest);
389 		pr_debug("---- end ----\n");
390 	} else {
391 		pr_debug("\n--- force skipped ---\n");
392 		err = TEST_SKIP;
393 	}
394 
395 	if (!t->subtest.get_nr)
396 		pr_debug("%s:", t->desc);
397 	else
398 		pr_debug("%s subtest %d:", t->desc, subtest + 1);
399 
400 	switch (err) {
401 	case TEST_OK:
402 		pr_info(" Ok\n");
403 		break;
404 	case TEST_SKIP:
405 		color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
406 		break;
407 	case TEST_FAIL:
408 	default:
409 		color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
410 		break;
411 	}
412 
413 	return err;
414 }
415 
416 static const char *shell_test__description(char *description, size_t size,
417 					   const char *path, const char *name)
418 {
419 	FILE *fp;
420 	char filename[PATH_MAX];
421 
422 	path__join(filename, sizeof(filename), path, name);
423 	fp = fopen(filename, "r");
424 	if (!fp)
425 		return NULL;
426 
427 	/* Skip shebang */
428 	while (fgetc(fp) != '\n');
429 
430 	description = fgets(description, size, fp);
431 	fclose(fp);
432 
433 	return description ? trim(description + 1) : NULL;
434 }
435 
436 #define for_each_shell_test(dir, base, ent)	\
437 	while ((ent = readdir(dir)) != NULL)	\
438 		if (!is_directory(base, ent) && ent->d_name[0] != '.')
439 
440 static const char *shell_tests__dir(char *path, size_t size)
441 {
442 	const char *devel_dirs[] = { "./tools/perf/tests", "./tests", };
443         char *exec_path;
444 	unsigned int i;
445 
446 	for (i = 0; i < ARRAY_SIZE(devel_dirs); ++i) {
447 		struct stat st;
448 		if (!lstat(devel_dirs[i], &st)) {
449 			scnprintf(path, size, "%s/shell", devel_dirs[i]);
450 			if (!lstat(devel_dirs[i], &st))
451 				return path;
452 		}
453 	}
454 
455         /* Then installed path. */
456         exec_path = get_argv_exec_path();
457         scnprintf(path, size, "%s/tests/shell", exec_path);
458 	free(exec_path);
459 	return path;
460 }
461 
462 static int shell_tests__max_desc_width(void)
463 {
464 	DIR *dir;
465 	struct dirent *ent;
466 	char path_dir[PATH_MAX];
467 	const char *path = shell_tests__dir(path_dir, sizeof(path_dir));
468 	int width = 0;
469 
470 	if (path == NULL)
471 		return -1;
472 
473 	dir = opendir(path);
474 	if (!dir)
475 		return -1;
476 
477 	for_each_shell_test(dir, path, ent) {
478 		char bf[256];
479 		const char *desc = shell_test__description(bf, sizeof(bf), path, ent->d_name);
480 
481 		if (desc) {
482 			int len = strlen(desc);
483 
484 			if (width < len)
485 				width = len;
486 		}
487 	}
488 
489 	closedir(dir);
490 	return width;
491 }
492 
493 struct shell_test {
494 	const char *dir;
495 	const char *file;
496 };
497 
498 static int shell_test__run(struct test *test, int subdir __maybe_unused)
499 {
500 	int err;
501 	char script[PATH_MAX];
502 	struct shell_test *st = test->priv;
503 
504 	path__join(script, sizeof(script), st->dir, st->file);
505 
506 	err = system(script);
507 	if (!err)
508 		return TEST_OK;
509 
510 	return WEXITSTATUS(err) == 2 ? TEST_SKIP : TEST_FAIL;
511 }
512 
513 static int run_shell_tests(int argc, const char *argv[], int i, int width)
514 {
515 	DIR *dir;
516 	struct dirent *ent;
517 	char path_dir[PATH_MAX];
518 	struct shell_test st = {
519 		.dir = shell_tests__dir(path_dir, sizeof(path_dir)),
520 	};
521 
522 	if (st.dir == NULL)
523 		return -1;
524 
525 	dir = opendir(st.dir);
526 	if (!dir)
527 		return -1;
528 
529 	for_each_shell_test(dir, st.dir, ent) {
530 		int curr = i++;
531 		char desc[256];
532 		struct test test = {
533 			.desc = shell_test__description(desc, sizeof(desc), st.dir, ent->d_name),
534 			.func = shell_test__run,
535 			.priv = &st,
536 		};
537 
538 		if (!perf_test__matches(&test, curr, argc, argv))
539 			continue;
540 
541 		st.file = ent->d_name;
542 		pr_info("%2d: %-*s:", i, width, test.desc);
543 		test_and_print(&test, false, -1);
544 	}
545 
546 	closedir(dir);
547 	return 0;
548 }
549 
550 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
551 {
552 	struct test *t;
553 	unsigned int j;
554 	int i = 0;
555 	int width = shell_tests__max_desc_width();
556 
557 	for_each_test(j, t) {
558 		int len = strlen(t->desc);
559 
560 		if (width < len)
561 			width = len;
562 	}
563 
564 	for_each_test(j, t) {
565 		int curr = i++, err;
566 
567 		if (!perf_test__matches(t, curr, argc, argv))
568 			continue;
569 
570 		if (t->is_supported && !t->is_supported()) {
571 			pr_debug("%2d: %-*s: Disabled\n", i, width, t->desc);
572 			continue;
573 		}
574 
575 		pr_info("%2d: %-*s:", i, width, t->desc);
576 
577 		if (intlist__find(skiplist, i)) {
578 			color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
579 			continue;
580 		}
581 
582 		if (!t->subtest.get_nr) {
583 			test_and_print(t, false, -1);
584 		} else {
585 			int subn = t->subtest.get_nr();
586 			/*
587 			 * minus 2 to align with normal testcases.
588 			 * For subtest we print additional '.x' in number.
589 			 * for example:
590 			 *
591 			 * 35: Test LLVM searching and compiling                        :
592 			 * 35.1: Basic BPF llvm compiling test                          : Ok
593 			 */
594 			int subw = width > 2 ? width - 2 : width;
595 			bool skip = false;
596 			int subi;
597 
598 			if (subn <= 0) {
599 				color_fprintf(stderr, PERF_COLOR_YELLOW,
600 					      " Skip (not compiled in)\n");
601 				continue;
602 			}
603 			pr_info("\n");
604 
605 			for (subi = 0; subi < subn; subi++) {
606 				int len = strlen(t->subtest.get_desc(subi));
607 
608 				if (subw < len)
609 					subw = len;
610 			}
611 
612 			for (subi = 0; subi < subn; subi++) {
613 				pr_info("%2d.%1d: %-*s:", i, subi + 1, subw,
614 					t->subtest.get_desc(subi));
615 				err = test_and_print(t, skip, subi);
616 				if (err != TEST_OK && t->subtest.skip_if_fail)
617 					skip = true;
618 			}
619 		}
620 	}
621 
622 	return run_shell_tests(argc, argv, i, width);
623 }
624 
625 static int perf_test__list_shell(int argc, const char **argv, int i)
626 {
627 	DIR *dir;
628 	struct dirent *ent;
629 	char path_dir[PATH_MAX];
630 	const char *path = shell_tests__dir(path_dir, sizeof(path_dir));
631 
632 	if (path == NULL)
633 		return -1;
634 
635 	dir = opendir(path);
636 	if (!dir)
637 		return -1;
638 
639 	for_each_shell_test(dir, path, ent) {
640 		int curr = i++;
641 		char bf[256];
642 		struct test t = {
643 			.desc = shell_test__description(bf, sizeof(bf), path, ent->d_name),
644 		};
645 
646 		if (!perf_test__matches(&t, curr, argc, argv))
647 			continue;
648 
649 		pr_info("%2d: %s\n", i, t.desc);
650 	}
651 
652 	closedir(dir);
653 	return 0;
654 }
655 
656 static int perf_test__list(int argc, const char **argv)
657 {
658 	unsigned int j;
659 	struct test *t;
660 	int i = 0;
661 
662 	for_each_test(j, t) {
663 		int curr = i++;
664 
665 		if (!perf_test__matches(t, curr, argc, argv) ||
666 		    (t->is_supported && !t->is_supported()))
667 			continue;
668 
669 		pr_info("%2d: %s\n", i, t->desc);
670 
671 		if (t->subtest.get_nr) {
672 			int subn = t->subtest.get_nr();
673 			int subi;
674 
675 			for (subi = 0; subi < subn; subi++)
676 				pr_info("%2d:%1d: %s\n", i, subi + 1,
677 					t->subtest.get_desc(subi));
678 		}
679 	}
680 
681 	perf_test__list_shell(argc, argv, i);
682 
683 	return 0;
684 }
685 
686 int cmd_test(int argc, const char **argv)
687 {
688 	const char *test_usage[] = {
689 	"perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
690 	NULL,
691 	};
692 	const char *skip = NULL;
693 	const struct option test_options[] = {
694 	OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
695 	OPT_INCR('v', "verbose", &verbose,
696 		    "be more verbose (show symbol address, etc)"),
697 	OPT_BOOLEAN('F', "dont-fork", &dont_fork,
698 		    "Do not fork for testcase"),
699 	OPT_END()
700 	};
701 	const char * const test_subcommands[] = { "list", NULL };
702 	struct intlist *skiplist = NULL;
703         int ret = hists__init();
704 
705         if (ret < 0)
706                 return ret;
707 
708 	argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
709 	if (argc >= 1 && !strcmp(argv[0], "list"))
710 		return perf_test__list(argc - 1, argv + 1);
711 
712 	symbol_conf.priv_size = sizeof(int);
713 	symbol_conf.sort_by_name = true;
714 	symbol_conf.try_vmlinux_path = true;
715 
716 	if (symbol__init(NULL) < 0)
717 		return -1;
718 
719 	if (skip != NULL)
720 		skiplist = intlist__new(skip);
721 
722 	return __cmd_test(argc, argv, skiplist);
723 }
724