1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 #define _GNU_SOURCE
4 #include <argp.h>
5 #include <string.h>
6 #include <stdlib.h>
7 #include <linux/compiler.h>
8 #include <sched.h>
9 #include <pthread.h>
10 #include <dirent.h>
11 #include <signal.h>
12 #include <fcntl.h>
13 #include <unistd.h>
14 #include <sys/time.h>
15 #include <sys/sysinfo.h>
16 #include <sys/stat.h>
17 #include <bpf/libbpf.h>
18 #include <libelf.h>
19 #include <gelf.h>
20 
21 enum stat_id {
22 	VERDICT,
23 	DURATION,
24 	TOTAL_INSNS,
25 	TOTAL_STATES,
26 	PEAK_STATES,
27 	MAX_STATES_PER_INSN,
28 	MARK_READ_MAX_LEN,
29 
30 	FILE_NAME,
31 	PROG_NAME,
32 
33 	ALL_STATS_CNT,
34 	NUM_STATS_CNT = FILE_NAME - VERDICT,
35 };
36 
37 struct verif_stats {
38 	char *file_name;
39 	char *prog_name;
40 
41 	long stats[NUM_STATS_CNT];
42 };
43 
44 struct stat_specs {
45 	int spec_cnt;
46 	enum stat_id ids[ALL_STATS_CNT];
47 	bool asc[ALL_STATS_CNT];
48 	int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */
49 };
50 
51 enum resfmt {
52 	RESFMT_TABLE,
53 	RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */
54 	RESFMT_CSV,
55 };
56 
57 struct filter {
58 	char *file_glob;
59 	char *prog_glob;
60 };
61 
62 static struct env {
63 	char **filenames;
64 	int filename_cnt;
65 	bool verbose;
66 	bool quiet;
67 	int log_level;
68 	enum resfmt out_fmt;
69 	bool comparison_mode;
70 
71 	struct verif_stats *prog_stats;
72 	int prog_stat_cnt;
73 
74 	/* baseline_stats is allocated and used only in comparsion mode */
75 	struct verif_stats *baseline_stats;
76 	int baseline_stat_cnt;
77 
78 	struct stat_specs output_spec;
79 	struct stat_specs sort_spec;
80 
81 	struct filter *allow_filters;
82 	struct filter *deny_filters;
83 	int allow_filter_cnt;
84 	int deny_filter_cnt;
85 
86 	int files_processed;
87 	int files_skipped;
88 	int progs_processed;
89 	int progs_skipped;
90 } env;
91 
92 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
93 {
94 	if (!env.verbose)
95 		return 0;
96 	if (level == LIBBPF_DEBUG /* && !env.verbose */)
97 		return 0;
98 	return vfprintf(stderr, format, args);
99 }
100 
101 const char *argp_program_version = "veristat";
102 const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
103 const char argp_program_doc[] =
104 "veristat    BPF verifier stats collection and comparison tool.\n"
105 "\n"
106 "USAGE: veristat <obj-file> [<obj-file>...]\n"
107 "   OR: veristat -C <baseline.csv> <comparison.csv>\n";
108 
109 static const struct argp_option opts[] = {
110 	{ NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" },
111 	{ "verbose", 'v', NULL, 0, "Verbose mode" },
112 	{ "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode)" },
113 	{ "quiet", 'q', NULL, 0, "Quiet mode" },
114 	{ "emit", 'e', "SPEC", 0, "Specify stats to be emitted" },
115 	{ "sort", 's', "SPEC", 0, "Specify sort order" },
116 	{ "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." },
117 	{ "compare", 'C', NULL, 0, "Comparison mode" },
118 	{ "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." },
119 	{},
120 };
121 
122 static int parse_stats(const char *stats_str, struct stat_specs *specs);
123 static int append_filter(struct filter **filters, int *cnt, const char *str);
124 static int append_filter_file(const char *path);
125 
126 static error_t parse_arg(int key, char *arg, struct argp_state *state)
127 {
128 	void *tmp;
129 	int err;
130 
131 	switch (key) {
132 	case 'h':
133 		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
134 		break;
135 	case 'v':
136 		env.verbose = true;
137 		break;
138 	case 'q':
139 		env.quiet = true;
140 		break;
141 	case 'e':
142 		err = parse_stats(arg, &env.output_spec);
143 		if (err)
144 			return err;
145 		break;
146 	case 's':
147 		err = parse_stats(arg, &env.sort_spec);
148 		if (err)
149 			return err;
150 		break;
151 	case 'o':
152 		if (strcmp(arg, "table") == 0) {
153 			env.out_fmt = RESFMT_TABLE;
154 		} else if (strcmp(arg, "csv") == 0) {
155 			env.out_fmt = RESFMT_CSV;
156 		} else {
157 			fprintf(stderr, "Unrecognized output format '%s'\n", arg);
158 			return -EINVAL;
159 		}
160 		break;
161 	case 'l':
162 		errno = 0;
163 		env.log_level = strtol(arg, NULL, 10);
164 		if (errno) {
165 			fprintf(stderr, "invalid log level: %s\n", arg);
166 			argp_usage(state);
167 		}
168 		break;
169 	case 'C':
170 		env.comparison_mode = true;
171 		break;
172 	case 'f':
173 		if (arg[0] == '@')
174 			err = append_filter_file(arg + 1);
175 		else if (arg[0] == '!')
176 			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);
177 		else
178 			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);
179 		if (err) {
180 			fprintf(stderr, "Failed to collect program filter expressions: %d\n", err);
181 			return err;
182 		}
183 		break;
184 	case ARGP_KEY_ARG:
185 		tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
186 		if (!tmp)
187 			return -ENOMEM;
188 		env.filenames = tmp;
189 		env.filenames[env.filename_cnt] = strdup(arg);
190 		if (!env.filenames[env.filename_cnt])
191 			return -ENOMEM;
192 		env.filename_cnt++;
193 		break;
194 	default:
195 		return ARGP_ERR_UNKNOWN;
196 	}
197 	return 0;
198 }
199 
200 static const struct argp argp = {
201 	.options = opts,
202 	.parser = parse_arg,
203 	.doc = argp_program_doc,
204 };
205 
206 
207 /* Adapted from perf/util/string.c */
208 static bool glob_matches(const char *str, const char *pat)
209 {
210 	while (*str && *pat && *pat != '*') {
211 		if (*str != *pat)
212 			return false;
213 		str++;
214 		pat++;
215 	}
216 	/* Check wild card */
217 	if (*pat == '*') {
218 		while (*pat == '*')
219 			pat++;
220 		if (!*pat) /* Tail wild card matches all */
221 			return true;
222 		while (*str)
223 			if (glob_matches(str++, pat))
224 				return true;
225 	}
226 	return !*str && !*pat;
227 }
228 
229 static bool should_process_file(const char *filename)
230 {
231 	int i;
232 
233 	if (env.deny_filter_cnt > 0) {
234 		for (i = 0; i < env.deny_filter_cnt; i++) {
235 			if (glob_matches(filename, env.deny_filters[i].file_glob))
236 				return false;
237 		}
238 	}
239 
240 	if (env.allow_filter_cnt == 0)
241 		return true;
242 
243 	for (i = 0; i < env.allow_filter_cnt; i++) {
244 		if (glob_matches(filename, env.allow_filters[i].file_glob))
245 			return true;
246 	}
247 
248 	return false;
249 }
250 
251 static bool is_bpf_obj_file(const char *path) {
252 	Elf64_Ehdr *ehdr;
253 	int fd, err = -EINVAL;
254 	Elf *elf = NULL;
255 
256 	fd = open(path, O_RDONLY | O_CLOEXEC);
257 	if (fd < 0)
258 		return true; /* we'll fail later and propagate error */
259 
260 	/* ensure libelf is initialized */
261 	(void)elf_version(EV_CURRENT);
262 
263 	elf = elf_begin(fd, ELF_C_READ, NULL);
264 	if (!elf)
265 		goto cleanup;
266 
267 	if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64)
268 		goto cleanup;
269 
270 	ehdr = elf64_getehdr(elf);
271 	/* Old LLVM set e_machine to EM_NONE */
272 	if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF))
273 		goto cleanup;
274 
275 	err = 0;
276 cleanup:
277 	if (elf)
278 		elf_end(elf);
279 	close(fd);
280 	return err == 0;
281 }
282 
283 static bool should_process_prog(const char *path, const char *prog_name)
284 {
285 	const char *filename = basename(path);
286 	int i;
287 
288 	if (env.deny_filter_cnt > 0) {
289 		for (i = 0; i < env.deny_filter_cnt; i++) {
290 			if (glob_matches(filename, env.deny_filters[i].file_glob))
291 				return false;
292 			if (!env.deny_filters[i].prog_glob)
293 				continue;
294 			if (glob_matches(prog_name, env.deny_filters[i].prog_glob))
295 				return false;
296 		}
297 	}
298 
299 	if (env.allow_filter_cnt == 0)
300 		return true;
301 
302 	for (i = 0; i < env.allow_filter_cnt; i++) {
303 		if (!glob_matches(filename, env.allow_filters[i].file_glob))
304 			continue;
305 		/* if filter specifies only filename glob part, it implicitly
306 		 * allows all progs within that file
307 		 */
308 		if (!env.allow_filters[i].prog_glob)
309 			return true;
310 		if (glob_matches(prog_name, env.allow_filters[i].prog_glob))
311 			return true;
312 	}
313 
314 	return false;
315 }
316 
317 static int append_filter(struct filter **filters, int *cnt, const char *str)
318 {
319 	struct filter *f;
320 	void *tmp;
321 	const char *p;
322 
323 	tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters));
324 	if (!tmp)
325 		return -ENOMEM;
326 	*filters = tmp;
327 
328 	f = &(*filters)[*cnt];
329 	f->file_glob = f->prog_glob = NULL;
330 
331 	/* filter can be specified either as "<obj-glob>" or "<obj-glob>/<prog-glob>" */
332 	p = strchr(str, '/');
333 	if (!p) {
334 		f->file_glob = strdup(str);
335 		if (!f->file_glob)
336 			return -ENOMEM;
337 	} else {
338 		f->file_glob = strndup(str, p - str);
339 		f->prog_glob = strdup(p + 1);
340 		if (!f->file_glob || !f->prog_glob) {
341 			free(f->file_glob);
342 			free(f->prog_glob);
343 			f->file_glob = f->prog_glob = NULL;
344 			return -ENOMEM;
345 		}
346 	}
347 
348 	*cnt = *cnt + 1;
349 	return 0;
350 }
351 
352 static int append_filter_file(const char *path)
353 {
354 	char buf[1024];
355 	FILE *f;
356 	int err = 0;
357 
358 	f = fopen(path, "r");
359 	if (!f) {
360 		err = -errno;
361 		fprintf(stderr, "Failed to open filters in '%s': %d\n", path, err);
362 		return err;
363 	}
364 
365 	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
366 		/* lines starting with # are comments, skip them */
367 		if (buf[0] == '\0' || buf[0] == '#')
368 			continue;
369 		/* lines starting with ! are negative match filters */
370 		if (buf[0] == '!')
371 			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);
372 		else
373 			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);
374 		if (err)
375 			goto cleanup;
376 	}
377 
378 cleanup:
379 	fclose(f);
380 	return err;
381 }
382 
383 static const struct stat_specs default_output_spec = {
384 	.spec_cnt = 7,
385 	.ids = {
386 		FILE_NAME, PROG_NAME, VERDICT, DURATION,
387 		TOTAL_INSNS, TOTAL_STATES, PEAK_STATES,
388 	},
389 };
390 
391 static const struct stat_specs default_sort_spec = {
392 	.spec_cnt = 2,
393 	.ids = {
394 		FILE_NAME, PROG_NAME,
395 	},
396 	.asc = { true, true, },
397 };
398 
399 static struct stat_def {
400 	const char *header;
401 	const char *names[4];
402 	bool asc_by_default;
403 } stat_defs[] = {
404 	[FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */ },
405 	[PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */ },
406 	[VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */ },
407 	[DURATION] = { "Duration (us)", {"duration", "dur"}, },
408 	[TOTAL_INSNS] = { "Total insns", {"total_insns", "insns"}, },
409 	[TOTAL_STATES] = { "Total states", {"total_states", "states"}, },
410 	[PEAK_STATES] = { "Peak states", {"peak_states"}, },
411 	[MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, },
412 	[MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, },
413 };
414 
415 static int parse_stat(const char *stat_name, struct stat_specs *specs)
416 {
417 	int id, i;
418 
419 	if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) {
420 		fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids));
421 		return -E2BIG;
422 	}
423 
424 	for (id = 0; id < ARRAY_SIZE(stat_defs); id++) {
425 		struct stat_def *def = &stat_defs[id];
426 
427 		for (i = 0; i < ARRAY_SIZE(stat_defs[id].names); i++) {
428 			if (!def->names[i] || strcmp(def->names[i], stat_name) != 0)
429 				continue;
430 
431 			specs->ids[specs->spec_cnt] = id;
432 			specs->asc[specs->spec_cnt] = def->asc_by_default;
433 			specs->spec_cnt++;
434 
435 			return 0;
436 		}
437 	}
438 
439 	fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name);
440 	return -ESRCH;
441 }
442 
443 static int parse_stats(const char *stats_str, struct stat_specs *specs)
444 {
445 	char *input, *state = NULL, *next;
446 	int err;
447 
448 	input = strdup(stats_str);
449 	if (!input)
450 		return -ENOMEM;
451 
452 	while ((next = strtok_r(state ? NULL : input, ",", &state))) {
453 		err = parse_stat(next, specs);
454 		if (err)
455 			return err;
456 	}
457 
458 	return 0;
459 }
460 
461 static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt)
462 {
463 	int i;
464 
465 	if (!stats)
466 		return;
467 
468 	for (i = 0; i < stat_cnt; i++) {
469 		free(stats[i].file_name);
470 		free(stats[i].prog_name);
471 	}
472 	free(stats);
473 }
474 
475 static char verif_log_buf[64 * 1024];
476 
477 #define MAX_PARSED_LOG_LINES 100
478 
479 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s)
480 {
481 	const char *cur;
482 	int pos, lines;
483 
484 	buf[buf_sz - 1] = '\0';
485 
486 	for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) {
487 		/* find previous endline or otherwise take the start of log buf */
488 		for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) {
489 		}
490 		/* next time start from end of previous line (or pos goes to <0) */
491 		pos--;
492 		/* if we found endline, point right after endline symbol;
493 		 * otherwise, stay at the beginning of log buf
494 		 */
495 		if (cur[0] == '\n')
496 			cur++;
497 
498 		if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION]))
499 			continue;
500 		if (6 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld",
501 				&s->stats[TOTAL_INSNS],
502 				&s->stats[MAX_STATES_PER_INSN],
503 				&s->stats[TOTAL_STATES],
504 				&s->stats[PEAK_STATES],
505 				&s->stats[MARK_READ_MAX_LEN]))
506 			continue;
507 	}
508 
509 	return 0;
510 }
511 
512 static void fixup_obj(struct bpf_object *obj)
513 {
514 	struct bpf_map *map;
515 
516 	bpf_object__for_each_map(map, obj) {
517 		/* disable pinning */
518 		bpf_map__set_pin_path(map, NULL);
519 
520 		/* fix up map size, if necessary */
521 		switch (bpf_map__type(map)) {
522 		case BPF_MAP_TYPE_SK_STORAGE:
523 		case BPF_MAP_TYPE_TASK_STORAGE:
524 		case BPF_MAP_TYPE_INODE_STORAGE:
525 		case BPF_MAP_TYPE_CGROUP_STORAGE:
526 			break;
527 		default:
528 			if (bpf_map__max_entries(map) == 0)
529 				bpf_map__set_max_entries(map, 1);
530 		}
531 	}
532 }
533 
534 static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
535 {
536 	const char *prog_name = bpf_program__name(prog);
537 	size_t buf_sz = sizeof(verif_log_buf);
538 	char *buf = verif_log_buf;
539 	struct verif_stats *stats;
540 	int err = 0;
541 	void *tmp;
542 
543 	if (!should_process_prog(filename, bpf_program__name(prog))) {
544 		env.progs_skipped++;
545 		return 0;
546 	}
547 
548 	tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));
549 	if (!tmp)
550 		return -ENOMEM;
551 	env.prog_stats = tmp;
552 	stats = &env.prog_stats[env.prog_stat_cnt++];
553 	memset(stats, 0, sizeof(*stats));
554 
555 	if (env.verbose) {
556 		buf_sz = 16 * 1024 * 1024;
557 		buf = malloc(buf_sz);
558 		if (!buf)
559 			return -ENOMEM;
560 		bpf_program__set_log_buf(prog, buf, buf_sz);
561 		bpf_program__set_log_level(prog, env.log_level | 4); /* stats + log */
562 	} else {
563 		bpf_program__set_log_buf(prog, buf, buf_sz);
564 		bpf_program__set_log_level(prog, 4); /* only verifier stats */
565 	}
566 	verif_log_buf[0] = '\0';
567 
568 	/* increase chances of successful BPF object loading */
569 	fixup_obj(obj);
570 
571 	err = bpf_object__load(obj);
572 	env.progs_processed++;
573 
574 	stats->file_name = strdup(basename(filename));
575 	stats->prog_name = strdup(bpf_program__name(prog));
576 	stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */
577 	parse_verif_log(buf, buf_sz, stats);
578 
579 	if (env.verbose) {
580 		printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n",
581 		       filename, prog_name, stats->stats[DURATION],
582 		       err ? "failure" : "success", buf);
583 	}
584 
585 	if (verif_log_buf != buf)
586 		free(buf);
587 
588 	return 0;
589 };
590 
591 static int process_obj(const char *filename)
592 {
593 	struct bpf_object *obj = NULL, *tobj;
594 	struct bpf_program *prog, *tprog, *lprog;
595 	libbpf_print_fn_t old_libbpf_print_fn;
596 	LIBBPF_OPTS(bpf_object_open_opts, opts);
597 	int err = 0, prog_cnt = 0;
598 
599 	if (!should_process_file(basename(filename))) {
600 		if (env.verbose)
601 			printf("Skipping '%s' due to filters...\n", filename);
602 		env.files_skipped++;
603 		return 0;
604 	}
605 	if (!is_bpf_obj_file(filename)) {
606 		if (env.verbose)
607 			printf("Skipping '%s' as it's not a BPF object file...\n", filename);
608 		env.files_skipped++;
609 		return 0;
610 	}
611 
612 	if (!env.quiet && env.out_fmt == RESFMT_TABLE)
613 		printf("Processing '%s'...\n", basename(filename));
614 
615 	old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn);
616 	obj = bpf_object__open_file(filename, &opts);
617 	if (!obj) {
618 		/* if libbpf can't open BPF object file, it could be because
619 		 * that BPF object file is incomplete and has to be statically
620 		 * linked into a final BPF object file; instead of bailing
621 		 * out, report it into stderr, mark it as skipped, and
622 		 * proceeed
623 		 */
624 		fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno);
625 		env.files_skipped++;
626 		err = 0;
627 		goto cleanup;
628 	}
629 
630 	env.files_processed++;
631 
632 	bpf_object__for_each_program(prog, obj) {
633 		prog_cnt++;
634 	}
635 
636 	if (prog_cnt == 1) {
637 		prog = bpf_object__next_program(obj, NULL);
638 		bpf_program__set_autoload(prog, true);
639 		process_prog(filename, obj, prog);
640 		goto cleanup;
641 	}
642 
643 	bpf_object__for_each_program(prog, obj) {
644 		const char *prog_name = bpf_program__name(prog);
645 
646 		tobj = bpf_object__open_file(filename, &opts);
647 		if (!tobj) {
648 			err = -errno;
649 			fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
650 			goto cleanup;
651 		}
652 
653 		bpf_object__for_each_program(tprog, tobj) {
654 			const char *tprog_name = bpf_program__name(tprog);
655 
656 			if (strcmp(prog_name, tprog_name) == 0) {
657 				bpf_program__set_autoload(tprog, true);
658 				lprog = tprog;
659 			} else {
660 				bpf_program__set_autoload(tprog, false);
661 			}
662 		}
663 
664 		process_prog(filename, tobj, lprog);
665 		bpf_object__close(tobj);
666 	}
667 
668 cleanup:
669 	bpf_object__close(obj);
670 	libbpf_set_print(old_libbpf_print_fn);
671 	return err;
672 }
673 
674 static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
675 		    enum stat_id id, bool asc)
676 {
677 	int cmp = 0;
678 
679 	switch (id) {
680 	case FILE_NAME:
681 		cmp = strcmp(s1->file_name, s2->file_name);
682 		break;
683 	case PROG_NAME:
684 		cmp = strcmp(s1->prog_name, s2->prog_name);
685 		break;
686 	case VERDICT:
687 	case DURATION:
688 	case TOTAL_INSNS:
689 	case TOTAL_STATES:
690 	case PEAK_STATES:
691 	case MAX_STATES_PER_INSN:
692 	case MARK_READ_MAX_LEN: {
693 		long v1 = s1->stats[id];
694 		long v2 = s2->stats[id];
695 
696 		if (v1 != v2)
697 			cmp = v1 < v2 ? -1 : 1;
698 		break;
699 	}
700 	default:
701 		fprintf(stderr, "Unrecognized stat #%d\n", id);
702 		exit(1);
703 	}
704 
705 	return asc ? cmp : -cmp;
706 }
707 
708 static int cmp_prog_stats(const void *v1, const void *v2)
709 {
710 	const struct verif_stats *s1 = v1, *s2 = v2;
711 	int i, cmp;
712 
713 	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
714 		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], env.sort_spec.asc[i]);
715 		if (cmp != 0)
716 			return cmp;
717 	}
718 
719 	return 0;
720 }
721 
722 #define HEADER_CHAR '-'
723 #define COLUMN_SEP "  "
724 
725 static void output_header_underlines(void)
726 {
727 	int i, j, len;
728 
729 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
730 		len = env.output_spec.lens[i];
731 
732 		printf("%s", i == 0 ? "" : COLUMN_SEP);
733 		for (j = 0; j < len; j++)
734 			printf("%c", HEADER_CHAR);
735 	}
736 	printf("\n");
737 }
738 
739 static void output_headers(enum resfmt fmt)
740 {
741 	int i, len;
742 
743 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
744 		int id = env.output_spec.ids[i];
745 		int *max_len = &env.output_spec.lens[i];
746 
747 		switch (fmt) {
748 		case RESFMT_TABLE_CALCLEN:
749 			len = snprintf(NULL, 0, "%s", stat_defs[id].header);
750 			if (len > *max_len)
751 				*max_len = len;
752 			break;
753 		case RESFMT_TABLE:
754 			printf("%s%-*s", i == 0 ? "" : COLUMN_SEP,  *max_len, stat_defs[id].header);
755 			if (i == env.output_spec.spec_cnt - 1)
756 				printf("\n");
757 			break;
758 		case RESFMT_CSV:
759 			printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]);
760 			if (i == env.output_spec.spec_cnt - 1)
761 				printf("\n");
762 			break;
763 		}
764 	}
765 
766 	if (fmt == RESFMT_TABLE)
767 		output_header_underlines();
768 }
769 
770 static void prepare_value(const struct verif_stats *s, enum stat_id id,
771 			  const char **str, long *val)
772 {
773 	switch (id) {
774 	case FILE_NAME:
775 		*str = s->file_name;
776 		break;
777 	case PROG_NAME:
778 		*str = s->prog_name;
779 		break;
780 	case VERDICT:
781 		*str = s->stats[VERDICT] ? "success" : "failure";
782 		break;
783 	case DURATION:
784 	case TOTAL_INSNS:
785 	case TOTAL_STATES:
786 	case PEAK_STATES:
787 	case MAX_STATES_PER_INSN:
788 	case MARK_READ_MAX_LEN:
789 		*val = s->stats[id];
790 		break;
791 	default:
792 		fprintf(stderr, "Unrecognized stat #%d\n", id);
793 		exit(1);
794 	}
795 }
796 
797 static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last)
798 {
799 	int i;
800 
801 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
802 		int id = env.output_spec.ids[i];
803 		int *max_len = &env.output_spec.lens[i], len;
804 		const char *str = NULL;
805 		long val = 0;
806 
807 		prepare_value(s, id, &str, &val);
808 
809 		switch (fmt) {
810 		case RESFMT_TABLE_CALCLEN:
811 			if (str)
812 				len = snprintf(NULL, 0, "%s", str);
813 			else
814 				len = snprintf(NULL, 0, "%ld", val);
815 			if (len > *max_len)
816 				*max_len = len;
817 			break;
818 		case RESFMT_TABLE:
819 			if (str)
820 				printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str);
821 			else
822 				printf("%s%*ld", i == 0 ? "" : COLUMN_SEP,  *max_len, val);
823 			if (i == env.output_spec.spec_cnt - 1)
824 				printf("\n");
825 			break;
826 		case RESFMT_CSV:
827 			if (str)
828 				printf("%s%s", i == 0 ? "" : ",", str);
829 			else
830 				printf("%s%ld", i == 0 ? "" : ",", val);
831 			if (i == env.output_spec.spec_cnt - 1)
832 				printf("\n");
833 			break;
834 		}
835 	}
836 
837 	if (last && fmt == RESFMT_TABLE) {
838 		output_header_underlines();
839 		printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n",
840 		       env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped);
841 	}
842 }
843 
844 static int handle_verif_mode(void)
845 {
846 	int i, err;
847 
848 	if (env.filename_cnt == 0) {
849 		fprintf(stderr, "Please provide path to BPF object file!\n");
850 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
851 		return -EINVAL;
852 	}
853 
854 	for (i = 0; i < env.filename_cnt; i++) {
855 		err = process_obj(env.filenames[i]);
856 		if (err) {
857 			fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);
858 			return err;
859 		}
860 	}
861 
862 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
863 
864 	if (env.out_fmt == RESFMT_TABLE) {
865 		/* calculate column widths */
866 		output_headers(RESFMT_TABLE_CALCLEN);
867 		for (i = 0; i < env.prog_stat_cnt; i++)
868 			output_stats(&env.prog_stats[i], RESFMT_TABLE_CALCLEN, false);
869 	}
870 
871 	/* actually output the table */
872 	output_headers(env.out_fmt);
873 	for (i = 0; i < env.prog_stat_cnt; i++) {
874 		output_stats(&env.prog_stats[i], env.out_fmt, i == env.prog_stat_cnt - 1);
875 	}
876 
877 	return 0;
878 }
879 
880 static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st)
881 {
882 	switch (id) {
883 	case FILE_NAME:
884 		st->file_name = strdup(str);
885 		if (!st->file_name)
886 			return -ENOMEM;
887 		break;
888 	case PROG_NAME:
889 		st->prog_name = strdup(str);
890 		if (!st->prog_name)
891 			return -ENOMEM;
892 		break;
893 	case VERDICT:
894 		if (strcmp(str, "success") == 0) {
895 			st->stats[VERDICT] = true;
896 		} else if (strcmp(str, "failure") == 0) {
897 			st->stats[VERDICT] = false;
898 		} else {
899 			fprintf(stderr, "Unrecognized verification verdict '%s'\n", str);
900 			return -EINVAL;
901 		}
902 		break;
903 	case DURATION:
904 	case TOTAL_INSNS:
905 	case TOTAL_STATES:
906 	case PEAK_STATES:
907 	case MAX_STATES_PER_INSN:
908 	case MARK_READ_MAX_LEN: {
909 		long val;
910 		int err, n;
911 
912 		if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) {
913 			err = -errno;
914 			fprintf(stderr, "Failed to parse '%s' as integer\n", str);
915 			return err;
916 		}
917 
918 		st->stats[id] = val;
919 		break;
920 	}
921 	default:
922 		fprintf(stderr, "Unrecognized stat #%d\n", id);
923 		return -EINVAL;
924 	}
925 	return 0;
926 }
927 
928 static int parse_stats_csv(const char *filename, struct stat_specs *specs,
929 			   struct verif_stats **statsp, int *stat_cntp)
930 {
931 	char line[4096];
932 	FILE *f;
933 	int err = 0;
934 	bool header = true;
935 
936 	f = fopen(filename, "r");
937 	if (!f) {
938 		err = -errno;
939 		fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
940 		return err;
941 	}
942 
943 	*stat_cntp = 0;
944 
945 	while (fgets(line, sizeof(line), f)) {
946 		char *input = line, *state = NULL, *next;
947 		struct verif_stats *st = NULL;
948 		int col = 0;
949 
950 		if (!header) {
951 			void *tmp;
952 
953 			tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp));
954 			if (!tmp) {
955 				err = -ENOMEM;
956 				goto cleanup;
957 			}
958 			*statsp = tmp;
959 
960 			st = &(*statsp)[*stat_cntp];
961 			memset(st, 0, sizeof(*st));
962 
963 			*stat_cntp += 1;
964 		}
965 
966 		while ((next = strtok_r(state ? NULL : input, ",\n", &state))) {
967 			if (header) {
968 				/* for the first line, set up spec stats */
969 				err = parse_stat(next, specs);
970 				if (err)
971 					goto cleanup;
972 				continue;
973 			}
974 
975 			/* for all other lines, parse values based on spec */
976 			if (col >= specs->spec_cnt) {
977 				fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n",
978 					col, *stat_cntp, filename);
979 				err = -EINVAL;
980 				goto cleanup;
981 			}
982 			err = parse_stat_value(next, specs->ids[col], st);
983 			if (err)
984 				goto cleanup;
985 			col++;
986 		}
987 
988 		if (header) {
989 			header = false;
990 			continue;
991 		}
992 
993 		if (col < specs->spec_cnt) {
994 			fprintf(stderr, "Not enough columns in row #%d in '%s'\n",
995 				*stat_cntp, filename);
996 			err = -EINVAL;
997 			goto cleanup;
998 		}
999 
1000 		if (!st->file_name || !st->prog_name) {
1001 			fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n",
1002 				*stat_cntp, filename);
1003 			err = -EINVAL;
1004 			goto cleanup;
1005 		}
1006 
1007 		/* in comparison mode we can only check filters after we
1008 		 * parsed entire line; if row should be ignored we pretend we
1009 		 * never parsed it
1010 		 */
1011 		if (!should_process_prog(st->file_name, st->prog_name)) {
1012 			free(st->file_name);
1013 			free(st->prog_name);
1014 			*stat_cntp -= 1;
1015 		}
1016 	}
1017 
1018 	if (!feof(f)) {
1019 		err = -errno;
1020 		fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err);
1021 	}
1022 
1023 cleanup:
1024 	fclose(f);
1025 	return err;
1026 }
1027 
1028 /* empty/zero stats for mismatched rows */
1029 static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" };
1030 
1031 static bool is_key_stat(enum stat_id id)
1032 {
1033 	return id == FILE_NAME || id == PROG_NAME;
1034 }
1035 
1036 static void output_comp_header_underlines(void)
1037 {
1038 	int i, j, k;
1039 
1040 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
1041 		int id = env.output_spec.ids[i];
1042 		int max_j = is_key_stat(id) ? 1 : 3;
1043 
1044 		for (j = 0; j < max_j; j++) {
1045 			int len = env.output_spec.lens[3 * i + j];
1046 
1047 			printf("%s", i + j == 0 ? "" : COLUMN_SEP);
1048 
1049 			for (k = 0; k < len; k++)
1050 				printf("%c", HEADER_CHAR);
1051 		}
1052 	}
1053 	printf("\n");
1054 }
1055 
1056 static void output_comp_headers(enum resfmt fmt)
1057 {
1058 	static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"};
1059 	static const char *name_sfxs[3] = {"_base", "_comp", "_diff"};
1060 	int i, j, len;
1061 
1062 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
1063 		int id = env.output_spec.ids[i];
1064 		/* key stats don't have A/B/DIFF columns, they are common for both data sets */
1065 		int max_j = is_key_stat(id) ? 1 : 3;
1066 
1067 		for (j = 0; j < max_j; j++) {
1068 			int *max_len = &env.output_spec.lens[3 * i + j];
1069 			bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);
1070 			const char *sfx;
1071 
1072 			switch (fmt) {
1073 			case RESFMT_TABLE_CALCLEN:
1074 				sfx = is_key_stat(id) ? "" : table_sfxs[j];
1075 				len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx);
1076 				if (len > *max_len)
1077 					*max_len = len;
1078 				break;
1079 			case RESFMT_TABLE:
1080 				sfx = is_key_stat(id) ? "" : table_sfxs[j];
1081 				printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP,
1082 				       *max_len - (int)strlen(sfx), stat_defs[id].header, sfx);
1083 				if (last)
1084 					printf("\n");
1085 				break;
1086 			case RESFMT_CSV:
1087 				sfx = is_key_stat(id) ? "" : name_sfxs[j];
1088 				printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx);
1089 				if (last)
1090 					printf("\n");
1091 				break;
1092 			}
1093 		}
1094 	}
1095 
1096 	if (fmt == RESFMT_TABLE)
1097 		output_comp_header_underlines();
1098 }
1099 
1100 static void output_comp_stats(const struct verif_stats *base, const struct verif_stats *comp,
1101 			      enum resfmt fmt, bool last)
1102 {
1103 	char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {};
1104 	int i;
1105 
1106 	for (i = 0; i < env.output_spec.spec_cnt; i++) {
1107 		int id = env.output_spec.ids[i], len;
1108 		int *max_len_base = &env.output_spec.lens[3 * i + 0];
1109 		int *max_len_comp = &env.output_spec.lens[3 * i + 1];
1110 		int *max_len_diff = &env.output_spec.lens[3 * i + 2];
1111 		const char *base_str = NULL, *comp_str = NULL;
1112 		long base_val = 0, comp_val = 0, diff_val = 0;
1113 
1114 		prepare_value(base, id, &base_str, &base_val);
1115 		prepare_value(comp, id, &comp_str, &comp_val);
1116 
1117 		/* normalize all the outputs to be in string buffers for simplicity */
1118 		if (is_key_stat(id)) {
1119 			/* key stats (file and program name) are always strings */
1120 			if (base != &fallback_stats)
1121 				snprintf(base_buf, sizeof(base_buf), "%s", base_str);
1122 			else
1123 				snprintf(base_buf, sizeof(base_buf), "%s", comp_str);
1124 		} else if (base_str) {
1125 			snprintf(base_buf, sizeof(base_buf), "%s", base_str);
1126 			snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str);
1127 			if (strcmp(base_str, comp_str) == 0)
1128 				snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH");
1129 			else
1130 				snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH");
1131 		} else {
1132 			double p = 0.0;
1133 
1134 			snprintf(base_buf, sizeof(base_buf), "%ld", base_val);
1135 			snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val);
1136 
1137 			diff_val = comp_val - base_val;
1138 			if (base == &fallback_stats || comp == &fallback_stats || base_val == 0) {
1139 				if (comp_val == base_val)
1140 					p = 0.0; /* avoid +0 (+100%) case */
1141 				else
1142 					p = comp_val < base_val ? -100.0 : 100.0;
1143 			} else {
1144 				 p = diff_val * 100.0 / base_val;
1145 			}
1146 			snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", diff_val, p);
1147 		}
1148 
1149 		switch (fmt) {
1150 		case RESFMT_TABLE_CALCLEN:
1151 			len = strlen(base_buf);
1152 			if (len > *max_len_base)
1153 				*max_len_base = len;
1154 			if (!is_key_stat(id)) {
1155 				len = strlen(comp_buf);
1156 				if (len > *max_len_comp)
1157 					*max_len_comp = len;
1158 				len = strlen(diff_buf);
1159 				if (len > *max_len_diff)
1160 					*max_len_diff = len;
1161 			}
1162 			break;
1163 		case RESFMT_TABLE: {
1164 			/* string outputs are left-aligned, number outputs are right-aligned */
1165 			const char *fmt = base_str ? "%s%-*s" : "%s%*s";
1166 
1167 			printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf);
1168 			if (!is_key_stat(id)) {
1169 				printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf);
1170 				printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf);
1171 			}
1172 			if (i == env.output_spec.spec_cnt - 1)
1173 				printf("\n");
1174 			break;
1175 		}
1176 		case RESFMT_CSV:
1177 			printf("%s%s", i == 0 ? "" : ",", base_buf);
1178 			if (!is_key_stat(id)) {
1179 				printf("%s%s", i == 0 ? "" : ",", comp_buf);
1180 				printf("%s%s", i == 0 ? "" : ",", diff_buf);
1181 			}
1182 			if (i == env.output_spec.spec_cnt - 1)
1183 				printf("\n");
1184 			break;
1185 		}
1186 	}
1187 
1188 	if (last && fmt == RESFMT_TABLE)
1189 		output_comp_header_underlines();
1190 }
1191 
1192 static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp)
1193 {
1194 	int r;
1195 
1196 	r = strcmp(base->file_name, comp->file_name);
1197 	if (r != 0)
1198 		return r;
1199 	return strcmp(base->prog_name, comp->prog_name);
1200 }
1201 
1202 static int handle_comparison_mode(void)
1203 {
1204 	struct stat_specs base_specs = {}, comp_specs = {};
1205 	enum resfmt cur_fmt;
1206 	int err, i, j;
1207 
1208 	if (env.filename_cnt != 2) {
1209 		fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n");
1210 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
1211 		return -EINVAL;
1212 	}
1213 
1214 	err = parse_stats_csv(env.filenames[0], &base_specs,
1215 			      &env.baseline_stats, &env.baseline_stat_cnt);
1216 	if (err) {
1217 		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
1218 		return err;
1219 	}
1220 	err = parse_stats_csv(env.filenames[1], &comp_specs,
1221 			      &env.prog_stats, &env.prog_stat_cnt);
1222 	if (err) {
1223 		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);
1224 		return err;
1225 	}
1226 
1227 	/* To keep it simple we validate that the set and order of stats in
1228 	 * both CSVs are exactly the same. This can be lifted with a bit more
1229 	 * pre-processing later.
1230 	 */
1231 	if (base_specs.spec_cnt != comp_specs.spec_cnt) {
1232 		fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n",
1233 			env.filenames[0], env.filenames[1],
1234 			base_specs.spec_cnt, comp_specs.spec_cnt);
1235 		return -EINVAL;
1236 	}
1237 	for (i = 0; i < base_specs.spec_cnt; i++) {
1238 		if (base_specs.ids[i] != comp_specs.ids[i]) {
1239 			fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n",
1240 				env.filenames[0], env.filenames[1],
1241 				stat_defs[base_specs.ids[i]].names[0],
1242 				stat_defs[comp_specs.ids[i]].names[0]);
1243 			return -EINVAL;
1244 		}
1245 	}
1246 
1247 	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
1248 	qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);
1249 
1250 	/* for human-readable table output we need to do extra pass to
1251 	 * calculate column widths, so we substitute current output format
1252 	 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE
1253 	 * and do everything again.
1254 	 */
1255 	if (env.out_fmt == RESFMT_TABLE)
1256 		cur_fmt = RESFMT_TABLE_CALCLEN;
1257 	else
1258 		cur_fmt = env.out_fmt;
1259 
1260 one_more_time:
1261 	output_comp_headers(cur_fmt);
1262 
1263 	/* If baseline and comparison datasets have different subset of rows
1264 	 * (we match by 'object + prog' as a unique key) then assume
1265 	 * empty/missing/zero value for rows that are missing in the opposite
1266 	 * data set
1267 	 */
1268 	i = j = 0;
1269 	while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {
1270 		bool last = (i == env.baseline_stat_cnt - 1) || (j == env.prog_stat_cnt - 1);
1271 		const struct verif_stats *base, *comp;
1272 		int r;
1273 
1274 		base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;
1275 		comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;
1276 
1277 		if (!base->file_name || !base->prog_name) {
1278 			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
1279 				i, env.filenames[0]);
1280 			return -EINVAL;
1281 		}
1282 		if (!comp->file_name || !comp->prog_name) {
1283 			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
1284 				j, env.filenames[1]);
1285 			return -EINVAL;
1286 		}
1287 
1288 		r = cmp_stats_key(base, comp);
1289 		if (r == 0) {
1290 			output_comp_stats(base, comp, cur_fmt, last);
1291 			i++;
1292 			j++;
1293 		} else if (comp == &fallback_stats || r < 0) {
1294 			output_comp_stats(base, &fallback_stats, cur_fmt, last);
1295 			i++;
1296 		} else {
1297 			output_comp_stats(&fallback_stats, comp, cur_fmt, last);
1298 			j++;
1299 		}
1300 	}
1301 
1302 	if (cur_fmt == RESFMT_TABLE_CALCLEN) {
1303 		cur_fmt = RESFMT_TABLE;
1304 		goto one_more_time; /* ... this time with feeling */
1305 	}
1306 
1307 	return 0;
1308 }
1309 
1310 int main(int argc, char **argv)
1311 {
1312 	int err = 0, i;
1313 
1314 	if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
1315 		return 1;
1316 
1317 	if (env.verbose && env.quiet) {
1318 		fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n");
1319 		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
1320 		return 1;
1321 	}
1322 	if (env.verbose && env.log_level == 0)
1323 		env.log_level = 1;
1324 
1325 	if (env.output_spec.spec_cnt == 0)
1326 		env.output_spec = default_output_spec;
1327 	if (env.sort_spec.spec_cnt == 0)
1328 		env.sort_spec = default_sort_spec;
1329 
1330 	if (env.comparison_mode)
1331 		err = handle_comparison_mode();
1332 	else
1333 		err = handle_verif_mode();
1334 
1335 	free_verif_stats(env.prog_stats, env.prog_stat_cnt);
1336 	free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);
1337 	for (i = 0; i < env.filename_cnt; i++)
1338 		free(env.filenames[i]);
1339 	free(env.filenames);
1340 	for (i = 0; i < env.allow_filter_cnt; i++) {
1341 		free(env.allow_filters[i].file_glob);
1342 		free(env.allow_filters[i].prog_glob);
1343 	}
1344 	free(env.allow_filters);
1345 	for (i = 0; i < env.deny_filter_cnt; i++) {
1346 		free(env.deny_filters[i].file_glob);
1347 		free(env.deny_filters[i].prog_glob);
1348 	}
1349 	free(env.deny_filters);
1350 	return -err;
1351 }
1352