1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19 
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23 
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <linux/tls.h>
30 #include <assert.h>
31 #include <libgen.h>
32 
33 #include <getopt.h>
34 
35 #include <bpf/bpf.h>
36 #include <bpf/libbpf.h>
37 
38 #include "bpf_util.h"
39 #include "cgroup_helpers.h"
40 
41 int running;
42 static void running_handler(int a);
43 
44 #ifndef TCP_ULP
45 # define TCP_ULP 31
46 #endif
47 #ifndef SOL_TLS
48 # define SOL_TLS 282
49 #endif
50 
51 /* randomly selected ports for testing on lo */
52 #define S1_PORT 10000
53 #define S2_PORT 10001
54 
55 #define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.bpf.o"
56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57 #define CG_PATH "/sockmap"
58 
59 /* global sockets */
60 int s1, s2, c1, c2, p1, p2;
61 int test_cnt;
62 int passed;
63 int failed;
64 int map_fd[9];
65 struct bpf_map *maps[9];
66 int prog_fd[11];
67 
68 int txmsg_pass;
69 int txmsg_redir;
70 int txmsg_drop;
71 int txmsg_apply;
72 int txmsg_cork;
73 int txmsg_start;
74 int txmsg_end;
75 int txmsg_start_push;
76 int txmsg_end_push;
77 int txmsg_start_pop;
78 int txmsg_pop;
79 int txmsg_ingress;
80 int txmsg_redir_skb;
81 int txmsg_ktls_skb;
82 int txmsg_ktls_skb_drop;
83 int txmsg_ktls_skb_redir;
84 int ktls;
85 int peek_flag;
86 int skb_use_parser;
87 int txmsg_omit_skb_parser;
88 
89 static const struct option long_options[] = {
90 	{"help",	no_argument,		NULL, 'h' },
91 	{"cgroup",	required_argument,	NULL, 'c' },
92 	{"rate",	required_argument,	NULL, 'r' },
93 	{"verbose",	optional_argument,	NULL, 'v' },
94 	{"iov_count",	required_argument,	NULL, 'i' },
95 	{"length",	required_argument,	NULL, 'l' },
96 	{"test",	required_argument,	NULL, 't' },
97 	{"data_test",   no_argument,		NULL, 'd' },
98 	{"txmsg",		no_argument,	&txmsg_pass,  1  },
99 	{"txmsg_redir",		no_argument,	&txmsg_redir, 1  },
100 	{"txmsg_drop",		no_argument,	&txmsg_drop, 1 },
101 	{"txmsg_apply",	required_argument,	NULL, 'a'},
102 	{"txmsg_cork",	required_argument,	NULL, 'k'},
103 	{"txmsg_start", required_argument,	NULL, 's'},
104 	{"txmsg_end",	required_argument,	NULL, 'e'},
105 	{"txmsg_start_push", required_argument,	NULL, 'p'},
106 	{"txmsg_end_push",   required_argument,	NULL, 'q'},
107 	{"txmsg_start_pop",  required_argument,	NULL, 'w'},
108 	{"txmsg_pop",	     required_argument,	NULL, 'x'},
109 	{"txmsg_ingress", no_argument,		&txmsg_ingress, 1 },
110 	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
111 	{"ktls", no_argument,			&ktls, 1 },
112 	{"peek", no_argument,			&peek_flag, 1 },
113 	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
114 	{"whitelist", required_argument,	NULL, 'n' },
115 	{"blacklist", required_argument,	NULL, 'b' },
116 	{0, 0, NULL, 0 }
117 };
118 
119 struct test_env {
120 	const char *type;
121 	const char *subtest;
122 	const char *prepend;
123 
124 	int test_num;
125 	int subtest_num;
126 
127 	int succ_cnt;
128 	int fail_cnt;
129 	int fail_last;
130 };
131 
132 struct test_env env;
133 
134 struct sockmap_options {
135 	int verbose;
136 	bool base;
137 	bool sendpage;
138 	bool data_test;
139 	bool drop_expected;
140 	bool check_recved_len;
141 	bool tx_wait_mem;
142 	int iov_count;
143 	int iov_length;
144 	int rate;
145 	char *map;
146 	char *whitelist;
147 	char *blacklist;
148 	char *prepend;
149 };
150 
151 struct _test {
152 	char *title;
153 	void (*tester)(int cg_fd, struct sockmap_options *opt);
154 };
155 
156 static void test_start(void)
157 {
158 	env.subtest_num++;
159 }
160 
161 static void test_fail(void)
162 {
163 	env.fail_cnt++;
164 }
165 
166 static void test_pass(void)
167 {
168 	env.succ_cnt++;
169 }
170 
171 static void test_reset(void)
172 {
173 	txmsg_start = txmsg_end = 0;
174 	txmsg_start_pop = txmsg_pop = 0;
175 	txmsg_start_push = txmsg_end_push = 0;
176 	txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 	txmsg_apply = txmsg_cork = 0;
178 	txmsg_ingress = txmsg_redir_skb = 0;
179 	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 	txmsg_omit_skb_parser = 0;
181 	skb_use_parser = 0;
182 }
183 
184 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185 {
186 	env.type = o->map;
187 	env.subtest = t->title;
188 	env.prepend = o->prepend;
189 	env.test_num++;
190 	env.subtest_num = 0;
191 	env.fail_last = env.fail_cnt;
192 	test_reset();
193 	return 0;
194 }
195 
196 static void test_end_subtest(void)
197 {
198 	int error = env.fail_cnt - env.fail_last;
199 	int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200 
201 	if (!error)
202 		test_pass();
203 
204 	fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 		env.test_num, env.subtest_num,
206 		!type ? "sockmap" : "sockhash",
207 		env.prepend ? : "",
208 		env.subtest, error ? "FAIL" : "OK");
209 }
210 
211 static void test_print_results(void)
212 {
213 	fprintf(stdout, "Pass: %d Fail: %d\n",
214 		env.succ_cnt, env.fail_cnt);
215 }
216 
217 static void usage(char *argv[])
218 {
219 	int i;
220 
221 	printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 	printf(" options:\n");
223 	for (i = 0; long_options[i].name != 0; i++) {
224 		printf(" --%-12s", long_options[i].name);
225 		if (long_options[i].flag != NULL)
226 			printf(" flag (internal value:%d)\n",
227 				*long_options[i].flag);
228 		else
229 			printf(" -%c\n", long_options[i].val);
230 	}
231 	printf("\n");
232 }
233 
234 char *sock_to_string(int s)
235 {
236 	if (s == c1)
237 		return "client1";
238 	else if (s == c2)
239 		return "client2";
240 	else if (s == s1)
241 		return "server1";
242 	else if (s == s2)
243 		return "server2";
244 	else if (s == p1)
245 		return "peer1";
246 	else if (s == p2)
247 		return "peer2";
248 	else
249 		return "unknown";
250 }
251 
252 static int sockmap_init_ktls(int verbose, int s)
253 {
254 	struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 		.info = {
256 			.version     = TLS_1_2_VERSION,
257 			.cipher_type = TLS_CIPHER_AES_GCM_128,
258 		},
259 	};
260 	struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 		.info = {
262 			.version     = TLS_1_2_VERSION,
263 			.cipher_type = TLS_CIPHER_AES_GCM_128,
264 		},
265 	};
266 	int so_buf = 6553500;
267 	int err;
268 
269 	err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 	if (err) {
271 		fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 		return -EINVAL;
273 	}
274 	err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 	if (err) {
276 		fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 		return -EINVAL;
278 	}
279 	err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 	if (err) {
281 		fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 		return -EINVAL;
283 	}
284 	err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 	if (err) {
286 		fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 		return -EINVAL;
288 	}
289 	err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 	if (err) {
291 		fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 		return -EINVAL;
293 	}
294 
295 	if (verbose)
296 		fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 	return 0;
298 }
299 static int sockmap_init_sockets(int verbose)
300 {
301 	int i, err, one = 1;
302 	struct sockaddr_in addr;
303 	int *fds[4] = {&s1, &s2, &c1, &c2};
304 
305 	s1 = s2 = p1 = p2 = c1 = c2 = 0;
306 
307 	/* Init sockets */
308 	for (i = 0; i < 4; i++) {
309 		*fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 		if (*fds[i] < 0) {
311 			perror("socket s1 failed()");
312 			return errno;
313 		}
314 	}
315 
316 	/* Allow reuse */
317 	for (i = 0; i < 2; i++) {
318 		err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 				 (char *)&one, sizeof(one));
320 		if (err) {
321 			perror("setsockopt failed()");
322 			return errno;
323 		}
324 	}
325 
326 	/* Non-blocking sockets */
327 	for (i = 0; i < 2; i++) {
328 		err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 		if (err < 0) {
330 			perror("ioctl s1 failed()");
331 			return errno;
332 		}
333 	}
334 
335 	/* Bind server sockets */
336 	memset(&addr, 0, sizeof(struct sockaddr_in));
337 	addr.sin_family = AF_INET;
338 	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339 
340 	addr.sin_port = htons(S1_PORT);
341 	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 	if (err < 0) {
343 		perror("bind s1 failed()");
344 		return errno;
345 	}
346 
347 	addr.sin_port = htons(S2_PORT);
348 	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 	if (err < 0) {
350 		perror("bind s2 failed()");
351 		return errno;
352 	}
353 
354 	/* Listen server sockets */
355 	addr.sin_port = htons(S1_PORT);
356 	err = listen(s1, 32);
357 	if (err < 0) {
358 		perror("listen s1 failed()");
359 		return errno;
360 	}
361 
362 	addr.sin_port = htons(S2_PORT);
363 	err = listen(s2, 32);
364 	if (err < 0) {
365 		perror("listen s1 failed()");
366 		return errno;
367 	}
368 
369 	/* Initiate Connect */
370 	addr.sin_port = htons(S1_PORT);
371 	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 	if (err < 0 && errno != EINPROGRESS) {
373 		perror("connect c1 failed()");
374 		return errno;
375 	}
376 
377 	addr.sin_port = htons(S2_PORT);
378 	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 	if (err < 0 && errno != EINPROGRESS) {
380 		perror("connect c2 failed()");
381 		return errno;
382 	} else if (err < 0) {
383 		err = 0;
384 	}
385 
386 	/* Accept Connecrtions */
387 	p1 = accept(s1, NULL, NULL);
388 	if (p1 < 0) {
389 		perror("accept s1 failed()");
390 		return errno;
391 	}
392 
393 	p2 = accept(s2, NULL, NULL);
394 	if (p2 < 0) {
395 		perror("accept s1 failed()");
396 		return errno;
397 	}
398 
399 	if (verbose > 1) {
400 		printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 		printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 			c1, s1, c2, s2);
403 	}
404 	return 0;
405 }
406 
407 struct msg_stats {
408 	size_t bytes_sent;
409 	size_t bytes_recvd;
410 	struct timespec start;
411 	struct timespec end;
412 };
413 
414 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 			     struct msg_stats *s,
416 			     struct sockmap_options *opt)
417 {
418 	bool drop = opt->drop_expected;
419 	unsigned char k = 0;
420 	FILE *file;
421 	int i, fp;
422 
423 	file = tmpfile();
424 	if (!file) {
425 		perror("create file for sendpage");
426 		return 1;
427 	}
428 	for (i = 0; i < iov_length * cnt; i++, k++)
429 		fwrite(&k, sizeof(char), 1, file);
430 	fflush(file);
431 	fseek(file, 0, SEEK_SET);
432 
433 	fp = fileno(file);
434 
435 	clock_gettime(CLOCK_MONOTONIC, &s->start);
436 	for (i = 0; i < cnt; i++) {
437 		int sent;
438 
439 		errno = 0;
440 		sent = sendfile(fd, fp, NULL, iov_length);
441 
442 		if (!drop && sent < 0) {
443 			perror("sendpage loop error");
444 			fclose(file);
445 			return sent;
446 		} else if (drop && sent >= 0) {
447 			printf("sendpage loop error expected: %i errno %i\n",
448 			       sent, errno);
449 			fclose(file);
450 			return -EIO;
451 		}
452 
453 		if (sent > 0)
454 			s->bytes_sent += sent;
455 	}
456 	clock_gettime(CLOCK_MONOTONIC, &s->end);
457 	fclose(file);
458 	return 0;
459 }
460 
461 static void msg_free_iov(struct msghdr *msg)
462 {
463 	int i;
464 
465 	for (i = 0; i < msg->msg_iovlen; i++)
466 		free(msg->msg_iov[i].iov_base);
467 	free(msg->msg_iov);
468 	msg->msg_iov = NULL;
469 	msg->msg_iovlen = 0;
470 }
471 
472 static int msg_alloc_iov(struct msghdr *msg,
473 			 int iov_count, int iov_length,
474 			 bool data, bool xmit)
475 {
476 	unsigned char k = 0;
477 	struct iovec *iov;
478 	int i;
479 
480 	iov = calloc(iov_count, sizeof(struct iovec));
481 	if (!iov)
482 		return errno;
483 
484 	for (i = 0; i < iov_count; i++) {
485 		unsigned char *d = calloc(iov_length, sizeof(char));
486 
487 		if (!d) {
488 			fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 			goto unwind_iov;
490 		}
491 		iov[i].iov_base = d;
492 		iov[i].iov_len = iov_length;
493 
494 		if (data && xmit) {
495 			int j;
496 
497 			for (j = 0; j < iov_length; j++)
498 				d[j] = k++;
499 		}
500 	}
501 
502 	msg->msg_iov = iov;
503 	msg->msg_iovlen = iov_count;
504 
505 	return 0;
506 unwind_iov:
507 	for (i--; i >= 0 ; i--)
508 		free(msg->msg_iov[i].iov_base);
509 	return -ENOMEM;
510 }
511 
512 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513 {
514 	int i, j = 0, bytes_cnt = 0;
515 	unsigned char k = 0;
516 
517 	for (i = 0; i < msg->msg_iovlen; i++) {
518 		unsigned char *d = msg->msg_iov[i].iov_base;
519 
520 		/* Special case test for skb ingress + ktls */
521 		if (i == 0 && txmsg_ktls_skb) {
522 			if (msg->msg_iov[i].iov_len < 4)
523 				return -EIO;
524 			if (memcmp(d, "PASS", 4) != 0) {
525 				fprintf(stderr,
526 					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 					i, 0, d[0], d[1], d[2], d[3]);
528 				return -EIO;
529 			}
530 			j = 4; /* advance index past PASS header */
531 		}
532 
533 		for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 			if (d[j] != k++) {
535 				fprintf(stderr,
536 					"detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 					i, j, d[j], k - 1, d[j+1], k);
538 				return -EIO;
539 			}
540 			bytes_cnt++;
541 			if (bytes_cnt == chunk_sz) {
542 				k = 0;
543 				bytes_cnt = 0;
544 			}
545 			size--;
546 		}
547 	}
548 	return 0;
549 }
550 
551 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 		    struct msg_stats *s, bool tx,
553 		    struct sockmap_options *opt)
554 {
555 	struct msghdr msg = {0}, msg_peek = {0};
556 	int err, i, flags = MSG_NOSIGNAL;
557 	bool drop = opt->drop_expected;
558 	bool data = opt->data_test;
559 	int iov_alloc_length = iov_length;
560 
561 	if (!tx && opt->check_recved_len)
562 		iov_alloc_length *= 2;
563 
564 	err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
565 	if (err)
566 		goto out_errno;
567 	if (peek_flag) {
568 		err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
569 		if (err)
570 			goto out_errno;
571 	}
572 
573 	if (tx) {
574 		clock_gettime(CLOCK_MONOTONIC, &s->start);
575 		for (i = 0; i < cnt; i++) {
576 			int sent;
577 
578 			errno = 0;
579 			sent = sendmsg(fd, &msg, flags);
580 
581 			if (!drop && sent < 0) {
582 				if (opt->tx_wait_mem && errno == EACCES) {
583 					errno = 0;
584 					goto out_errno;
585 				}
586 				perror("sendmsg loop error");
587 				goto out_errno;
588 			} else if (drop && sent >= 0) {
589 				fprintf(stderr,
590 					"sendmsg loop error expected: %i errno %i\n",
591 					sent, errno);
592 				errno = -EIO;
593 				goto out_errno;
594 			}
595 			if (sent > 0)
596 				s->bytes_sent += sent;
597 		}
598 		clock_gettime(CLOCK_MONOTONIC, &s->end);
599 	} else {
600 		int slct, recvp = 0, recv, max_fd = fd;
601 		float total_bytes, txmsg_pop_total;
602 		int fd_flags = O_NONBLOCK;
603 		struct timeval timeout;
604 		fd_set w;
605 
606 		fcntl(fd, fd_flags);
607 		/* Account for pop bytes noting each iteration of apply will
608 		 * call msg_pop_data helper so we need to account for this
609 		 * by calculating the number of apply iterations. Note user
610 		 * of the tool can create cases where no data is sent by
611 		 * manipulating pop/push/pull/etc. For example txmsg_apply 1
612 		 * with txmsg_pop 1 will try to apply 1B at a time but each
613 		 * iteration will then pop 1B so no data will ever be sent.
614 		 * This is really only useful for testing edge cases in code
615 		 * paths.
616 		 */
617 		total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
618 		if (txmsg_apply)
619 			txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
620 		else
621 			txmsg_pop_total = txmsg_pop * cnt;
622 		total_bytes -= txmsg_pop_total;
623 		err = clock_gettime(CLOCK_MONOTONIC, &s->start);
624 		if (err < 0)
625 			perror("recv start time");
626 		while (s->bytes_recvd < total_bytes) {
627 			if (txmsg_cork) {
628 				timeout.tv_sec = 0;
629 				timeout.tv_usec = 300000;
630 			} else {
631 				timeout.tv_sec = 3;
632 				timeout.tv_usec = 0;
633 			}
634 
635 			/* FD sets */
636 			FD_ZERO(&w);
637 			FD_SET(fd, &w);
638 
639 			slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
640 			if (slct == -1) {
641 				perror("select()");
642 				clock_gettime(CLOCK_MONOTONIC, &s->end);
643 				goto out_errno;
644 			} else if (!slct) {
645 				if (opt->verbose)
646 					fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
647 				errno = -EIO;
648 				clock_gettime(CLOCK_MONOTONIC, &s->end);
649 				goto out_errno;
650 			}
651 
652 			if (opt->tx_wait_mem) {
653 				FD_ZERO(&w);
654 				FD_SET(fd, &w);
655 				slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
656 				errno = 0;
657 				close(fd);
658 				goto out_errno;
659 			}
660 
661 			errno = 0;
662 			if (peek_flag) {
663 				flags |= MSG_PEEK;
664 				recvp = recvmsg(fd, &msg_peek, flags);
665 				if (recvp < 0) {
666 					if (errno != EWOULDBLOCK) {
667 						clock_gettime(CLOCK_MONOTONIC, &s->end);
668 						goto out_errno;
669 					}
670 				}
671 				flags = 0;
672 			}
673 
674 			recv = recvmsg(fd, &msg, flags);
675 			if (recv < 0) {
676 				if (errno != EWOULDBLOCK) {
677 					clock_gettime(CLOCK_MONOTONIC, &s->end);
678 					perror("recv failed()");
679 					goto out_errno;
680 				}
681 			}
682 
683 			s->bytes_recvd += recv;
684 
685 			if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
686 				errno = EMSGSIZE;
687 				fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
688 						s->bytes_recvd, total_bytes);
689 				goto out_errno;
690 			}
691 
692 			if (data) {
693 				int chunk_sz = opt->sendpage ?
694 						iov_length * cnt :
695 						iov_length * iov_count;
696 
697 				errno = msg_verify_data(&msg, recv, chunk_sz);
698 				if (errno) {
699 					perror("data verify msg failed");
700 					goto out_errno;
701 				}
702 				if (recvp) {
703 					errno = msg_verify_data(&msg_peek,
704 								recvp,
705 								chunk_sz);
706 					if (errno) {
707 						perror("data verify msg_peek failed");
708 						goto out_errno;
709 					}
710 				}
711 			}
712 		}
713 		clock_gettime(CLOCK_MONOTONIC, &s->end);
714 	}
715 
716 	msg_free_iov(&msg);
717 	msg_free_iov(&msg_peek);
718 	return err;
719 out_errno:
720 	msg_free_iov(&msg);
721 	msg_free_iov(&msg_peek);
722 	return errno;
723 }
724 
725 static float giga = 1000000000;
726 
727 static inline float sentBps(struct msg_stats s)
728 {
729 	return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
730 }
731 
732 static inline float recvdBps(struct msg_stats s)
733 {
734 	return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
735 }
736 
737 static int sendmsg_test(struct sockmap_options *opt)
738 {
739 	float sent_Bps = 0, recvd_Bps = 0;
740 	int rx_fd, txpid, rxpid, err = 0;
741 	struct msg_stats s = {0};
742 	int iov_count = opt->iov_count;
743 	int iov_buf = opt->iov_length;
744 	int rx_status, tx_status;
745 	int cnt = opt->rate;
746 
747 	errno = 0;
748 
749 	if (opt->base)
750 		rx_fd = p1;
751 	else
752 		rx_fd = p2;
753 
754 	if (ktls) {
755 		/* Redirecting into non-TLS socket which sends into a TLS
756 		 * socket is not a valid test. So in this case lets not
757 		 * enable kTLS but still run the test.
758 		 */
759 		if (!txmsg_redir || txmsg_ingress) {
760 			err = sockmap_init_ktls(opt->verbose, rx_fd);
761 			if (err)
762 				return err;
763 		}
764 		err = sockmap_init_ktls(opt->verbose, c1);
765 		if (err)
766 			return err;
767 	}
768 
769 	if (opt->tx_wait_mem) {
770 		struct timeval timeout;
771 		int rxtx_buf_len = 1024;
772 
773 		timeout.tv_sec = 3;
774 		timeout.tv_usec = 0;
775 
776 		err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
777 		err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
778 		err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
779 		if (err) {
780 			perror("setsockopt failed()");
781 			return errno;
782 		}
783 	}
784 
785 	rxpid = fork();
786 	if (rxpid == 0) {
787 		if (txmsg_pop || txmsg_start_pop)
788 			iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
789 		if (opt->drop_expected || txmsg_ktls_skb_drop)
790 			_exit(0);
791 
792 		if (!iov_buf) /* zero bytes sent case */
793 			_exit(0);
794 
795 		if (opt->sendpage)
796 			iov_count = 1;
797 		err = msg_loop(rx_fd, iov_count, iov_buf,
798 			       cnt, &s, false, opt);
799 		if (opt->verbose > 1)
800 			fprintf(stderr,
801 				"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
802 				iov_count, iov_buf, cnt, err);
803 		if (s.end.tv_sec - s.start.tv_sec) {
804 			sent_Bps = sentBps(s);
805 			recvd_Bps = recvdBps(s);
806 		}
807 		if (opt->verbose > 1)
808 			fprintf(stdout,
809 				"rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
810 				s.bytes_sent, sent_Bps, sent_Bps/giga,
811 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
812 				peek_flag ? "(peek_msg)" : "");
813 		if (err && txmsg_cork)
814 			err = 0;
815 		exit(err ? 1 : 0);
816 	} else if (rxpid == -1) {
817 		perror("msg_loop_rx");
818 		return errno;
819 	}
820 
821 	if (opt->tx_wait_mem)
822 		close(c2);
823 
824 	txpid = fork();
825 	if (txpid == 0) {
826 		if (opt->sendpage)
827 			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
828 		else
829 			err = msg_loop(c1, iov_count, iov_buf,
830 				       cnt, &s, true, opt);
831 
832 		if (err)
833 			fprintf(stderr,
834 				"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
835 				iov_count, iov_buf, cnt, err);
836 		if (s.end.tv_sec - s.start.tv_sec) {
837 			sent_Bps = sentBps(s);
838 			recvd_Bps = recvdBps(s);
839 		}
840 		if (opt->verbose > 1)
841 			fprintf(stdout,
842 				"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
843 				s.bytes_sent, sent_Bps, sent_Bps/giga,
844 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
845 		exit(err ? 1 : 0);
846 	} else if (txpid == -1) {
847 		perror("msg_loop_tx");
848 		return errno;
849 	}
850 
851 	assert(waitpid(rxpid, &rx_status, 0) == rxpid);
852 	assert(waitpid(txpid, &tx_status, 0) == txpid);
853 	if (WIFEXITED(rx_status)) {
854 		err = WEXITSTATUS(rx_status);
855 		if (err) {
856 			fprintf(stderr, "rx thread exited with err %d.\n", err);
857 			goto out;
858 		}
859 	}
860 	if (WIFEXITED(tx_status)) {
861 		err = WEXITSTATUS(tx_status);
862 		if (err)
863 			fprintf(stderr, "tx thread exited with err %d.\n", err);
864 	}
865 out:
866 	return err;
867 }
868 
869 static int forever_ping_pong(int rate, struct sockmap_options *opt)
870 {
871 	struct timeval timeout;
872 	char buf[1024] = {0};
873 	int sc;
874 
875 	timeout.tv_sec = 10;
876 	timeout.tv_usec = 0;
877 
878 	/* Ping/Pong data from client to server */
879 	sc = send(c1, buf, sizeof(buf), 0);
880 	if (sc < 0) {
881 		perror("send failed()");
882 		return sc;
883 	}
884 
885 	do {
886 		int s, rc, i, max_fd = p2;
887 		fd_set w;
888 
889 		/* FD sets */
890 		FD_ZERO(&w);
891 		FD_SET(c1, &w);
892 		FD_SET(c2, &w);
893 		FD_SET(p1, &w);
894 		FD_SET(p2, &w);
895 
896 		s = select(max_fd + 1, &w, NULL, NULL, &timeout);
897 		if (s == -1) {
898 			perror("select()");
899 			break;
900 		} else if (!s) {
901 			fprintf(stderr, "unexpected timeout\n");
902 			break;
903 		}
904 
905 		for (i = 0; i <= max_fd && s > 0; ++i) {
906 			if (!FD_ISSET(i, &w))
907 				continue;
908 
909 			s--;
910 
911 			rc = recv(i, buf, sizeof(buf), 0);
912 			if (rc < 0) {
913 				if (errno != EWOULDBLOCK) {
914 					perror("recv failed()");
915 					return rc;
916 				}
917 			}
918 
919 			if (rc == 0) {
920 				close(i);
921 				break;
922 			}
923 
924 			sc = send(i, buf, rc, 0);
925 			if (sc < 0) {
926 				perror("send failed()");
927 				return sc;
928 			}
929 		}
930 
931 		if (rate)
932 			sleep(rate);
933 
934 		if (opt->verbose) {
935 			printf(".");
936 			fflush(stdout);
937 
938 		}
939 	} while (running);
940 
941 	return 0;
942 }
943 
944 enum {
945 	SELFTESTS,
946 	PING_PONG,
947 	SENDMSG,
948 	BASE,
949 	BASE_SENDPAGE,
950 	SENDPAGE,
951 };
952 
953 static int run_options(struct sockmap_options *options, int cg_fd,  int test)
954 {
955 	int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
956 
957 	/* If base test skip BPF setup */
958 	if (test == BASE || test == BASE_SENDPAGE)
959 		goto run;
960 
961 	/* Attach programs to sockmap */
962 	if (!txmsg_omit_skb_parser) {
963 		err = bpf_prog_attach(prog_fd[0], map_fd[0],
964 				      BPF_SK_SKB_STREAM_PARSER, 0);
965 		if (err) {
966 			fprintf(stderr,
967 				"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
968 				prog_fd[0], map_fd[0], err, strerror(errno));
969 			return err;
970 		}
971 	}
972 
973 	err = bpf_prog_attach(prog_fd[1], map_fd[0],
974 				BPF_SK_SKB_STREAM_VERDICT, 0);
975 	if (err) {
976 		fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
977 			err, strerror(errno));
978 		return err;
979 	}
980 
981 	/* Attach programs to TLS sockmap */
982 	if (txmsg_ktls_skb) {
983 		if (!txmsg_omit_skb_parser) {
984 			err = bpf_prog_attach(prog_fd[0], map_fd[8],
985 					      BPF_SK_SKB_STREAM_PARSER, 0);
986 			if (err) {
987 				fprintf(stderr,
988 					"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
989 					prog_fd[0], map_fd[8], err, strerror(errno));
990 				return err;
991 			}
992 		}
993 
994 		err = bpf_prog_attach(prog_fd[2], map_fd[8],
995 				      BPF_SK_SKB_STREAM_VERDICT, 0);
996 		if (err) {
997 			fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
998 				err, strerror(errno));
999 			return err;
1000 		}
1001 	}
1002 
1003 	/* Attach to cgroups */
1004 	err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1005 	if (err) {
1006 		fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1007 			err, strerror(errno));
1008 		return err;
1009 	}
1010 
1011 run:
1012 	err = sockmap_init_sockets(options->verbose);
1013 	if (err) {
1014 		fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1015 		goto out;
1016 	}
1017 
1018 	/* Attach txmsg program to sockmap */
1019 	if (txmsg_pass)
1020 		tx_prog_fd = prog_fd[4];
1021 	else if (txmsg_redir)
1022 		tx_prog_fd = prog_fd[5];
1023 	else if (txmsg_apply)
1024 		tx_prog_fd = prog_fd[6];
1025 	else if (txmsg_cork)
1026 		tx_prog_fd = prog_fd[7];
1027 	else if (txmsg_drop)
1028 		tx_prog_fd = prog_fd[8];
1029 	else
1030 		tx_prog_fd = 0;
1031 
1032 	if (tx_prog_fd) {
1033 		int redir_fd, i = 0;
1034 
1035 		err = bpf_prog_attach(tx_prog_fd,
1036 				      map_fd[1], BPF_SK_MSG_VERDICT, 0);
1037 		if (err) {
1038 			fprintf(stderr,
1039 				"ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1040 				err, strerror(errno));
1041 			goto out;
1042 		}
1043 
1044 		err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1045 		if (err) {
1046 			fprintf(stderr,
1047 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1048 				err, strerror(errno));
1049 			goto out;
1050 		}
1051 
1052 		if (txmsg_redir)
1053 			redir_fd = c2;
1054 		else
1055 			redir_fd = c1;
1056 
1057 		err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1058 		if (err) {
1059 			fprintf(stderr,
1060 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1061 				err, strerror(errno));
1062 			goto out;
1063 		}
1064 
1065 		if (txmsg_apply) {
1066 			err = bpf_map_update_elem(map_fd[3],
1067 						  &i, &txmsg_apply, BPF_ANY);
1068 			if (err) {
1069 				fprintf(stderr,
1070 					"ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1071 					err, strerror(errno));
1072 				goto out;
1073 			}
1074 		}
1075 
1076 		if (txmsg_cork) {
1077 			err = bpf_map_update_elem(map_fd[4],
1078 						  &i, &txmsg_cork, BPF_ANY);
1079 			if (err) {
1080 				fprintf(stderr,
1081 					"ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1082 					err, strerror(errno));
1083 				goto out;
1084 			}
1085 		}
1086 
1087 		if (txmsg_start) {
1088 			err = bpf_map_update_elem(map_fd[5],
1089 						  &i, &txmsg_start, BPF_ANY);
1090 			if (err) {
1091 				fprintf(stderr,
1092 					"ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1093 					err, strerror(errno));
1094 				goto out;
1095 			}
1096 		}
1097 
1098 		if (txmsg_end) {
1099 			i = 1;
1100 			err = bpf_map_update_elem(map_fd[5],
1101 						  &i, &txmsg_end, BPF_ANY);
1102 			if (err) {
1103 				fprintf(stderr,
1104 					"ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1105 					err, strerror(errno));
1106 				goto out;
1107 			}
1108 		}
1109 
1110 		if (txmsg_start_push) {
1111 			i = 2;
1112 			err = bpf_map_update_elem(map_fd[5],
1113 						  &i, &txmsg_start_push, BPF_ANY);
1114 			if (err) {
1115 				fprintf(stderr,
1116 					"ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1117 					err, strerror(errno));
1118 				goto out;
1119 			}
1120 		}
1121 
1122 		if (txmsg_end_push) {
1123 			i = 3;
1124 			err = bpf_map_update_elem(map_fd[5],
1125 						  &i, &txmsg_end_push, BPF_ANY);
1126 			if (err) {
1127 				fprintf(stderr,
1128 					"ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1129 					txmsg_end_push, i, err, strerror(errno));
1130 				goto out;
1131 			}
1132 		}
1133 
1134 		if (txmsg_start_pop) {
1135 			i = 4;
1136 			err = bpf_map_update_elem(map_fd[5],
1137 						  &i, &txmsg_start_pop, BPF_ANY);
1138 			if (err) {
1139 				fprintf(stderr,
1140 					"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1141 					txmsg_start_pop, i, err, strerror(errno));
1142 				goto out;
1143 			}
1144 		} else {
1145 			i = 4;
1146 			bpf_map_update_elem(map_fd[5],
1147 						  &i, &txmsg_start_pop, BPF_ANY);
1148 		}
1149 
1150 		if (txmsg_pop) {
1151 			i = 5;
1152 			err = bpf_map_update_elem(map_fd[5],
1153 						  &i, &txmsg_pop, BPF_ANY);
1154 			if (err) {
1155 				fprintf(stderr,
1156 					"ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1157 					txmsg_pop, i, err, strerror(errno));
1158 				goto out;
1159 			}
1160 		} else {
1161 			i = 5;
1162 			bpf_map_update_elem(map_fd[5],
1163 					    &i, &txmsg_pop, BPF_ANY);
1164 
1165 		}
1166 
1167 		if (txmsg_ingress) {
1168 			int in = BPF_F_INGRESS;
1169 
1170 			i = 0;
1171 			err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1172 			if (err) {
1173 				fprintf(stderr,
1174 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1175 					err, strerror(errno));
1176 			}
1177 			i = 1;
1178 			err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1179 			if (err) {
1180 				fprintf(stderr,
1181 					"ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1182 					err, strerror(errno));
1183 			}
1184 			err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1185 			if (err) {
1186 				fprintf(stderr,
1187 					"ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1188 					err, strerror(errno));
1189 			}
1190 
1191 			i = 2;
1192 			err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1193 			if (err) {
1194 				fprintf(stderr,
1195 					"ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1196 					err, strerror(errno));
1197 			}
1198 		}
1199 
1200 		if (txmsg_ktls_skb) {
1201 			int ingress = BPF_F_INGRESS;
1202 
1203 			i = 0;
1204 			err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1205 			if (err) {
1206 				fprintf(stderr,
1207 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1208 					err, strerror(errno));
1209 			}
1210 
1211 			if (txmsg_ktls_skb_redir) {
1212 				i = 1;
1213 				err = bpf_map_update_elem(map_fd[7],
1214 							  &i, &ingress, BPF_ANY);
1215 				if (err) {
1216 					fprintf(stderr,
1217 						"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1218 						err, strerror(errno));
1219 				}
1220 			}
1221 
1222 			if (txmsg_ktls_skb_drop) {
1223 				i = 1;
1224 				err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1225 			}
1226 		}
1227 
1228 		if (txmsg_redir_skb) {
1229 			int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1230 					p2 : p1;
1231 			int ingress = BPF_F_INGRESS;
1232 
1233 			i = 0;
1234 			err = bpf_map_update_elem(map_fd[7],
1235 						  &i, &ingress, BPF_ANY);
1236 			if (err) {
1237 				fprintf(stderr,
1238 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1239 					err, strerror(errno));
1240 			}
1241 
1242 			i = 3;
1243 			err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1244 			if (err) {
1245 				fprintf(stderr,
1246 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1247 					err, strerror(errno));
1248 			}
1249 		}
1250 	}
1251 
1252 	if (skb_use_parser) {
1253 		i = 2;
1254 		err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1255 	}
1256 
1257 	if (txmsg_drop)
1258 		options->drop_expected = true;
1259 
1260 	if (test == PING_PONG)
1261 		err = forever_ping_pong(options->rate, options);
1262 	else if (test == SENDMSG) {
1263 		options->base = false;
1264 		options->sendpage = false;
1265 		err = sendmsg_test(options);
1266 	} else if (test == SENDPAGE) {
1267 		options->base = false;
1268 		options->sendpage = true;
1269 		err = sendmsg_test(options);
1270 	} else if (test == BASE) {
1271 		options->base = true;
1272 		options->sendpage = false;
1273 		err = sendmsg_test(options);
1274 	} else if (test == BASE_SENDPAGE) {
1275 		options->base = true;
1276 		options->sendpage = true;
1277 		err = sendmsg_test(options);
1278 	} else
1279 		fprintf(stderr, "unknown test\n");
1280 out:
1281 	/* Detatch and zero all the maps */
1282 	bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1283 	bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1284 	bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1285 	bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1286 	bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1287 
1288 	if (tx_prog_fd >= 0)
1289 		bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1290 
1291 	for (i = 0; i < 8; i++) {
1292 		key = next_key = 0;
1293 		bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1294 		while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1295 			bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1296 			key = next_key;
1297 		}
1298 	}
1299 
1300 	close(s1);
1301 	close(s2);
1302 	close(p1);
1303 	close(p2);
1304 	close(c1);
1305 	close(c2);
1306 	return err;
1307 }
1308 
1309 static char *test_to_str(int test)
1310 {
1311 	switch (test) {
1312 	case SENDMSG:
1313 		return "sendmsg";
1314 	case SENDPAGE:
1315 		return "sendpage";
1316 	}
1317 	return "unknown";
1318 }
1319 
1320 static void append_str(char *dst, const char *src, size_t dst_cap)
1321 {
1322 	size_t avail = dst_cap - strlen(dst);
1323 
1324 	if (avail <= 1) /* just zero byte could be written */
1325 		return;
1326 
1327 	strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1328 }
1329 
1330 #define OPTSTRING 60
1331 static void test_options(char *options)
1332 {
1333 	char tstr[OPTSTRING];
1334 
1335 	memset(options, 0, OPTSTRING);
1336 
1337 	if (txmsg_pass)
1338 		append_str(options, "pass,", OPTSTRING);
1339 	if (txmsg_redir)
1340 		append_str(options, "redir,", OPTSTRING);
1341 	if (txmsg_drop)
1342 		append_str(options, "drop,", OPTSTRING);
1343 	if (txmsg_apply) {
1344 		snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1345 		append_str(options, tstr, OPTSTRING);
1346 	}
1347 	if (txmsg_cork) {
1348 		snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1349 		append_str(options, tstr, OPTSTRING);
1350 	}
1351 	if (txmsg_start) {
1352 		snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1353 		append_str(options, tstr, OPTSTRING);
1354 	}
1355 	if (txmsg_end) {
1356 		snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1357 		append_str(options, tstr, OPTSTRING);
1358 	}
1359 	if (txmsg_start_pop) {
1360 		snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1361 			 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1362 		append_str(options, tstr, OPTSTRING);
1363 	}
1364 	if (txmsg_ingress)
1365 		append_str(options, "ingress,", OPTSTRING);
1366 	if (txmsg_redir_skb)
1367 		append_str(options, "redir_skb,", OPTSTRING);
1368 	if (txmsg_ktls_skb)
1369 		append_str(options, "ktls_skb,", OPTSTRING);
1370 	if (ktls)
1371 		append_str(options, "ktls,", OPTSTRING);
1372 	if (peek_flag)
1373 		append_str(options, "peek,", OPTSTRING);
1374 }
1375 
1376 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1377 {
1378 	char *options = calloc(OPTSTRING, sizeof(char));
1379 	int err;
1380 
1381 	if (test == SENDPAGE)
1382 		opt->sendpage = true;
1383 	else
1384 		opt->sendpage = false;
1385 
1386 	if (txmsg_drop)
1387 		opt->drop_expected = true;
1388 	else
1389 		opt->drop_expected = false;
1390 
1391 	test_options(options);
1392 
1393 	if (opt->verbose) {
1394 		fprintf(stdout,
1395 			" [TEST %i]: (%i, %i, %i, %s, %s): ",
1396 			test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1397 			test_to_str(test), options);
1398 		fflush(stdout);
1399 	}
1400 	err = run_options(opt, cgrp, test);
1401 	if (opt->verbose)
1402 		fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1403 	test_cnt++;
1404 	!err ? passed++ : failed++;
1405 	free(options);
1406 	return err;
1407 }
1408 
1409 static void test_exec(int cgrp, struct sockmap_options *opt)
1410 {
1411 	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1412 	int err;
1413 
1414 	if (type == 0) {
1415 		test_start();
1416 		err = __test_exec(cgrp, SENDMSG, opt);
1417 		if (err)
1418 			test_fail();
1419 	} else {
1420 		test_start();
1421 		err = __test_exec(cgrp, SENDPAGE, opt);
1422 		if (err)
1423 			test_fail();
1424 	}
1425 }
1426 
1427 static void test_send_one(struct sockmap_options *opt, int cgrp)
1428 {
1429 	opt->iov_length = 1;
1430 	opt->iov_count = 1;
1431 	opt->rate = 1;
1432 	test_exec(cgrp, opt);
1433 
1434 	opt->iov_length = 1;
1435 	opt->iov_count = 1024;
1436 	opt->rate = 1;
1437 	test_exec(cgrp, opt);
1438 
1439 	opt->iov_length = 1024;
1440 	opt->iov_count = 1;
1441 	opt->rate = 1;
1442 	test_exec(cgrp, opt);
1443 
1444 }
1445 
1446 static void test_send_many(struct sockmap_options *opt, int cgrp)
1447 {
1448 	opt->iov_length = 3;
1449 	opt->iov_count = 1;
1450 	opt->rate = 512;
1451 	test_exec(cgrp, opt);
1452 
1453 	opt->rate = 100;
1454 	opt->iov_count = 1;
1455 	opt->iov_length = 5;
1456 	test_exec(cgrp, opt);
1457 }
1458 
1459 static void test_send_large(struct sockmap_options *opt, int cgrp)
1460 {
1461 	opt->iov_length = 256;
1462 	opt->iov_count = 1024;
1463 	opt->rate = 2;
1464 	test_exec(cgrp, opt);
1465 }
1466 
1467 static void test_send(struct sockmap_options *opt, int cgrp)
1468 {
1469 	test_send_one(opt, cgrp);
1470 	test_send_many(opt, cgrp);
1471 	test_send_large(opt, cgrp);
1472 	sched_yield();
1473 }
1474 
1475 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1476 {
1477 	/* Test small and large iov_count values with pass/redir/apply/cork */
1478 	txmsg_pass = 1;
1479 	test_send(opt, cgrp);
1480 }
1481 
1482 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1483 {
1484 	txmsg_redir = 1;
1485 	test_send(opt, cgrp);
1486 }
1487 
1488 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1489 {
1490 	txmsg_redir = 1;
1491 	opt->tx_wait_mem = true;
1492 	test_send_large(opt, cgrp);
1493 	opt->tx_wait_mem = false;
1494 }
1495 
1496 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1497 {
1498 	txmsg_drop = 1;
1499 	test_send(opt, cgrp);
1500 }
1501 
1502 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1503 {
1504 	txmsg_pass = txmsg_drop = 0;
1505 	txmsg_ingress = txmsg_redir = 1;
1506 	test_send(opt, cgrp);
1507 }
1508 
1509 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1510 {
1511 	bool data = opt->data_test;
1512 	int k = ktls;
1513 
1514 	opt->data_test = true;
1515 	ktls = 1;
1516 
1517 	txmsg_pass = txmsg_drop = 0;
1518 	txmsg_ingress = txmsg_redir = 0;
1519 	txmsg_ktls_skb = 1;
1520 	txmsg_pass = 1;
1521 
1522 	/* Using data verification so ensure iov layout is
1523 	 * expected from test receiver side. e.g. has enough
1524 	 * bytes to write test code.
1525 	 */
1526 	opt->iov_length = 100;
1527 	opt->iov_count = 1;
1528 	opt->rate = 1;
1529 	test_exec(cgrp, opt);
1530 
1531 	txmsg_ktls_skb_drop = 1;
1532 	test_exec(cgrp, opt);
1533 
1534 	txmsg_ktls_skb_drop = 0;
1535 	txmsg_ktls_skb_redir = 1;
1536 	test_exec(cgrp, opt);
1537 	txmsg_ktls_skb_redir = 0;
1538 
1539 	/* Tests that omit skb_parser */
1540 	txmsg_omit_skb_parser = 1;
1541 	ktls = 0;
1542 	txmsg_ktls_skb = 0;
1543 	test_exec(cgrp, opt);
1544 
1545 	txmsg_ktls_skb_drop = 1;
1546 	test_exec(cgrp, opt);
1547 	txmsg_ktls_skb_drop = 0;
1548 
1549 	txmsg_ktls_skb_redir = 1;
1550 	test_exec(cgrp, opt);
1551 
1552 	ktls = 1;
1553 	test_exec(cgrp, opt);
1554 	txmsg_omit_skb_parser = 0;
1555 
1556 	opt->data_test = data;
1557 	ktls = k;
1558 }
1559 
1560 /* Test cork with hung data. This tests poor usage patterns where
1561  * cork can leave data on the ring if user program is buggy and
1562  * doesn't flush them somehow. They do take some time however
1563  * because they wait for a timeout. Test pass, redir and cork with
1564  * apply logic. Use cork size of 4097 with send_large to avoid
1565  * aligning cork size with send size.
1566  */
1567 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1568 {
1569 	txmsg_pass = 1;
1570 	txmsg_redir = 0;
1571 	txmsg_cork = 4097;
1572 	txmsg_apply = 4097;
1573 	test_send_large(opt, cgrp);
1574 
1575 	txmsg_pass = 0;
1576 	txmsg_redir = 1;
1577 	txmsg_apply = 0;
1578 	txmsg_cork = 4097;
1579 	test_send_large(opt, cgrp);
1580 
1581 	txmsg_pass = 0;
1582 	txmsg_redir = 1;
1583 	txmsg_apply = 4097;
1584 	txmsg_cork = 4097;
1585 	test_send_large(opt, cgrp);
1586 }
1587 
1588 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1589 {
1590 	/* Test basic start/end */
1591 	txmsg_start = 1;
1592 	txmsg_end = 2;
1593 	test_send(opt, cgrp);
1594 
1595 	/* Test >4k pull */
1596 	txmsg_start = 4096;
1597 	txmsg_end = 9182;
1598 	test_send_large(opt, cgrp);
1599 
1600 	/* Test pull + redirect */
1601 	txmsg_redir = 0;
1602 	txmsg_start = 1;
1603 	txmsg_end = 2;
1604 	test_send(opt, cgrp);
1605 
1606 	/* Test pull + cork */
1607 	txmsg_redir = 0;
1608 	txmsg_cork = 512;
1609 	txmsg_start = 1;
1610 	txmsg_end = 2;
1611 	test_send_many(opt, cgrp);
1612 
1613 	/* Test pull + cork + redirect */
1614 	txmsg_redir = 1;
1615 	txmsg_cork = 512;
1616 	txmsg_start = 1;
1617 	txmsg_end = 2;
1618 	test_send_many(opt, cgrp);
1619 }
1620 
1621 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1622 {
1623 	/* Test basic pop */
1624 	txmsg_start_pop = 1;
1625 	txmsg_pop = 2;
1626 	test_send_many(opt, cgrp);
1627 
1628 	/* Test pop with >4k */
1629 	txmsg_start_pop = 4096;
1630 	txmsg_pop = 4096;
1631 	test_send_large(opt, cgrp);
1632 
1633 	/* Test pop + redirect */
1634 	txmsg_redir = 1;
1635 	txmsg_start_pop = 1;
1636 	txmsg_pop = 2;
1637 	test_send_many(opt, cgrp);
1638 
1639 	/* Test pop + cork */
1640 	txmsg_redir = 0;
1641 	txmsg_cork = 512;
1642 	txmsg_start_pop = 1;
1643 	txmsg_pop = 2;
1644 	test_send_many(opt, cgrp);
1645 
1646 	/* Test pop + redirect + cork */
1647 	txmsg_redir = 1;
1648 	txmsg_cork = 4;
1649 	txmsg_start_pop = 1;
1650 	txmsg_pop = 2;
1651 	test_send_many(opt, cgrp);
1652 }
1653 
1654 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1655 {
1656 	/* Test basic push */
1657 	txmsg_start_push = 1;
1658 	txmsg_end_push = 1;
1659 	test_send(opt, cgrp);
1660 
1661 	/* Test push 4kB >4k */
1662 	txmsg_start_push = 4096;
1663 	txmsg_end_push = 4096;
1664 	test_send_large(opt, cgrp);
1665 
1666 	/* Test push + redirect */
1667 	txmsg_redir = 1;
1668 	txmsg_start_push = 1;
1669 	txmsg_end_push = 2;
1670 	test_send_many(opt, cgrp);
1671 
1672 	/* Test push + cork */
1673 	txmsg_redir = 0;
1674 	txmsg_cork = 512;
1675 	txmsg_start_push = 1;
1676 	txmsg_end_push = 2;
1677 	test_send_many(opt, cgrp);
1678 }
1679 
1680 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1681 {
1682 	txmsg_start_push = 1;
1683 	txmsg_end_push = 10;
1684 	txmsg_start_pop = 5;
1685 	txmsg_pop = 4;
1686 	test_send_large(opt, cgrp);
1687 }
1688 
1689 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1690 {
1691 	txmsg_pass = 1;
1692 	txmsg_redir = 0;
1693 	txmsg_apply = 1;
1694 	txmsg_cork = 0;
1695 	test_send_one(opt, cgrp);
1696 
1697 	txmsg_pass = 0;
1698 	txmsg_redir = 1;
1699 	txmsg_apply = 1;
1700 	txmsg_cork = 0;
1701 	test_send_one(opt, cgrp);
1702 
1703 	txmsg_pass = 1;
1704 	txmsg_redir = 0;
1705 	txmsg_apply = 1024;
1706 	txmsg_cork = 0;
1707 	test_send_large(opt, cgrp);
1708 
1709 	txmsg_pass = 0;
1710 	txmsg_redir = 1;
1711 	txmsg_apply = 1024;
1712 	txmsg_cork = 0;
1713 	test_send_large(opt, cgrp);
1714 }
1715 
1716 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1717 {
1718 	txmsg_pass = 1;
1719 	txmsg_redir = 0;
1720 	txmsg_apply = 0;
1721 	txmsg_cork = 1;
1722 	test_send(opt, cgrp);
1723 
1724 	txmsg_pass = 1;
1725 	txmsg_redir = 0;
1726 	txmsg_apply = 1;
1727 	txmsg_cork = 1;
1728 	test_send(opt, cgrp);
1729 }
1730 
1731 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1732 {
1733 	txmsg_pass = 1;
1734 	skb_use_parser = 512;
1735 	if (ktls == 1)
1736 		skb_use_parser = 570;
1737 	opt->iov_length = 256;
1738 	opt->iov_count = 1;
1739 	opt->rate = 2;
1740 	test_exec(cgrp, opt);
1741 }
1742 
1743 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1744 {
1745 	if (ktls == 1)
1746 		return;
1747 	skb_use_parser = 10;
1748 	opt->iov_length = 20;
1749 	opt->iov_count = 1;
1750 	opt->rate = 1;
1751 	opt->check_recved_len = true;
1752 	test_exec(cgrp, opt);
1753 	opt->check_recved_len = false;
1754 }
1755 
1756 char *map_names[] = {
1757 	"sock_map",
1758 	"sock_map_txmsg",
1759 	"sock_map_redir",
1760 	"sock_apply_bytes",
1761 	"sock_cork_bytes",
1762 	"sock_bytes",
1763 	"sock_redir_flags",
1764 	"sock_skb_opts",
1765 	"tls_sock_map",
1766 };
1767 
1768 int prog_attach_type[] = {
1769 	BPF_SK_SKB_STREAM_PARSER,
1770 	BPF_SK_SKB_STREAM_VERDICT,
1771 	BPF_SK_SKB_STREAM_VERDICT,
1772 	BPF_CGROUP_SOCK_OPS,
1773 	BPF_SK_MSG_VERDICT,
1774 	BPF_SK_MSG_VERDICT,
1775 	BPF_SK_MSG_VERDICT,
1776 	BPF_SK_MSG_VERDICT,
1777 	BPF_SK_MSG_VERDICT,
1778 	BPF_SK_MSG_VERDICT,
1779 	BPF_SK_MSG_VERDICT,
1780 };
1781 
1782 int prog_type[] = {
1783 	BPF_PROG_TYPE_SK_SKB,
1784 	BPF_PROG_TYPE_SK_SKB,
1785 	BPF_PROG_TYPE_SK_SKB,
1786 	BPF_PROG_TYPE_SOCK_OPS,
1787 	BPF_PROG_TYPE_SK_MSG,
1788 	BPF_PROG_TYPE_SK_MSG,
1789 	BPF_PROG_TYPE_SK_MSG,
1790 	BPF_PROG_TYPE_SK_MSG,
1791 	BPF_PROG_TYPE_SK_MSG,
1792 	BPF_PROG_TYPE_SK_MSG,
1793 	BPF_PROG_TYPE_SK_MSG,
1794 };
1795 
1796 static int populate_progs(char *bpf_file)
1797 {
1798 	struct bpf_program *prog;
1799 	struct bpf_object *obj;
1800 	int i = 0;
1801 	long err;
1802 
1803 	obj = bpf_object__open(bpf_file);
1804 	err = libbpf_get_error(obj);
1805 	if (err) {
1806 		char err_buf[256];
1807 
1808 		libbpf_strerror(err, err_buf, sizeof(err_buf));
1809 		printf("Unable to load eBPF objects in file '%s' : %s\n",
1810 		       bpf_file, err_buf);
1811 		return -1;
1812 	}
1813 
1814 	bpf_object__for_each_program(prog, obj) {
1815 		bpf_program__set_type(prog, prog_type[i]);
1816 		bpf_program__set_expected_attach_type(prog,
1817 						      prog_attach_type[i]);
1818 		i++;
1819 	}
1820 
1821 	i = bpf_object__load(obj);
1822 	i = 0;
1823 	bpf_object__for_each_program(prog, obj) {
1824 		prog_fd[i] = bpf_program__fd(prog);
1825 		i++;
1826 	}
1827 
1828 	for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1829 		maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1830 		map_fd[i] = bpf_map__fd(maps[i]);
1831 		if (map_fd[i] < 0) {
1832 			fprintf(stderr, "load_bpf_file: (%i) %s\n",
1833 				map_fd[i], strerror(errno));
1834 			return -1;
1835 		}
1836 	}
1837 
1838 	return 0;
1839 }
1840 
1841 struct _test test[] = {
1842 	{"txmsg test passthrough", test_txmsg_pass},
1843 	{"txmsg test redirect", test_txmsg_redir},
1844 	{"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1845 	{"txmsg test drop", test_txmsg_drop},
1846 	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
1847 	{"txmsg test skb", test_txmsg_skb},
1848 	{"txmsg test apply", test_txmsg_apply},
1849 	{"txmsg test cork", test_txmsg_cork},
1850 	{"txmsg test hanging corks", test_txmsg_cork_hangs},
1851 	{"txmsg test push_data", test_txmsg_push},
1852 	{"txmsg test pull-data", test_txmsg_pull},
1853 	{"txmsg test pop-data", test_txmsg_pop},
1854 	{"txmsg test push/pop data", test_txmsg_push_pop},
1855 	{"txmsg test ingress parser", test_txmsg_ingress_parser},
1856 	{"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1857 };
1858 
1859 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1860 {
1861 	char *entry, *ptr;
1862 
1863 	if (!opt->whitelist)
1864 		return 0;
1865 	ptr = strdup(opt->whitelist);
1866 	if (!ptr)
1867 		return -ENOMEM;
1868 	entry = strtok(ptr, ",");
1869 	while (entry) {
1870 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1871 		    strstr(opt->map, entry) != 0 ||
1872 		    strstr(t->title, entry) != 0)
1873 			return 0;
1874 		entry = strtok(NULL, ",");
1875 	}
1876 	return -EINVAL;
1877 }
1878 
1879 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1880 {
1881 	char *entry, *ptr;
1882 
1883 	if (!opt->blacklist)
1884 		return -EINVAL;
1885 	ptr = strdup(opt->blacklist);
1886 	if (!ptr)
1887 		return -ENOMEM;
1888 	entry = strtok(ptr, ",");
1889 	while (entry) {
1890 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1891 		    strstr(opt->map, entry) != 0 ||
1892 		    strstr(t->title, entry) != 0)
1893 			return 0;
1894 		entry = strtok(NULL, ",");
1895 	}
1896 	return -EINVAL;
1897 }
1898 
1899 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1900 {
1901 	int i, err;
1902 
1903 	err = populate_progs(opt->map);
1904 	if (err < 0) {
1905 		fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1906 		return err;
1907 	}
1908 
1909 	/* Tests basic commands and APIs */
1910 	for (i = 0; i < ARRAY_SIZE(test); i++) {
1911 		struct _test t = test[i];
1912 
1913 		if (check_whitelist(&t, opt) != 0)
1914 			continue;
1915 		if (check_blacklist(&t, opt) == 0)
1916 			continue;
1917 
1918 		test_start_subtest(&t, opt);
1919 		t.tester(cg_fd, opt);
1920 		test_end_subtest();
1921 	}
1922 
1923 	return err;
1924 }
1925 
1926 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1927 {
1928 	opt->map = BPF_SOCKMAP_FILENAME;
1929 	__test_selftests(cg_fd, opt);
1930 }
1931 
1932 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1933 {
1934 	opt->map = BPF_SOCKHASH_FILENAME;
1935 	__test_selftests(cg_fd, opt);
1936 }
1937 
1938 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1939 {
1940 	opt->map = BPF_SOCKHASH_FILENAME;
1941 	opt->prepend = "ktls";
1942 	ktls = 1;
1943 	__test_selftests(cg_fd, opt);
1944 	ktls = 0;
1945 }
1946 
1947 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1948 {
1949 
1950 	test_selftests_sockmap(cg_fd, opt);
1951 	test_selftests_sockhash(cg_fd, opt);
1952 	test_selftests_ktls(cg_fd, opt);
1953 	test_print_results();
1954 	return 0;
1955 }
1956 
1957 int main(int argc, char **argv)
1958 {
1959 	int iov_count = 1, length = 1024, rate = 1;
1960 	struct sockmap_options options = {0};
1961 	int opt, longindex, err, cg_fd = 0;
1962 	char *bpf_file = BPF_SOCKMAP_FILENAME;
1963 	int test = SELFTESTS;
1964 	bool cg_created = 0;
1965 
1966 	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1967 				  long_options, &longindex)) != -1) {
1968 		switch (opt) {
1969 		case 's':
1970 			txmsg_start = atoi(optarg);
1971 			break;
1972 		case 'e':
1973 			txmsg_end = atoi(optarg);
1974 			break;
1975 		case 'p':
1976 			txmsg_start_push = atoi(optarg);
1977 			break;
1978 		case 'q':
1979 			txmsg_end_push = atoi(optarg);
1980 			break;
1981 		case 'w':
1982 			txmsg_start_pop = atoi(optarg);
1983 			break;
1984 		case 'x':
1985 			txmsg_pop = atoi(optarg);
1986 			break;
1987 		case 'a':
1988 			txmsg_apply = atoi(optarg);
1989 			break;
1990 		case 'k':
1991 			txmsg_cork = atoi(optarg);
1992 			break;
1993 		case 'c':
1994 			cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1995 			if (cg_fd < 0) {
1996 				fprintf(stderr,
1997 					"ERROR: (%i) open cg path failed: %s\n",
1998 					cg_fd, optarg);
1999 				return cg_fd;
2000 			}
2001 			break;
2002 		case 'r':
2003 			rate = atoi(optarg);
2004 			break;
2005 		case 'v':
2006 			options.verbose = 1;
2007 			if (optarg)
2008 				options.verbose = atoi(optarg);
2009 			break;
2010 		case 'i':
2011 			iov_count = atoi(optarg);
2012 			break;
2013 		case 'l':
2014 			length = atoi(optarg);
2015 			break;
2016 		case 'd':
2017 			options.data_test = true;
2018 			break;
2019 		case 't':
2020 			if (strcmp(optarg, "ping") == 0) {
2021 				test = PING_PONG;
2022 			} else if (strcmp(optarg, "sendmsg") == 0) {
2023 				test = SENDMSG;
2024 			} else if (strcmp(optarg, "base") == 0) {
2025 				test = BASE;
2026 			} else if (strcmp(optarg, "base_sendpage") == 0) {
2027 				test = BASE_SENDPAGE;
2028 			} else if (strcmp(optarg, "sendpage") == 0) {
2029 				test = SENDPAGE;
2030 			} else {
2031 				usage(argv);
2032 				return -1;
2033 			}
2034 			break;
2035 		case 'n':
2036 			options.whitelist = strdup(optarg);
2037 			if (!options.whitelist)
2038 				return -ENOMEM;
2039 			break;
2040 		case 'b':
2041 			options.blacklist = strdup(optarg);
2042 			if (!options.blacklist)
2043 				return -ENOMEM;
2044 		case 0:
2045 			break;
2046 		case 'h':
2047 		default:
2048 			usage(argv);
2049 			return -1;
2050 		}
2051 	}
2052 
2053 	if (!cg_fd) {
2054 		cg_fd = cgroup_setup_and_join(CG_PATH);
2055 		if (cg_fd < 0)
2056 			return cg_fd;
2057 		cg_created = 1;
2058 	}
2059 
2060 	/* Use libbpf 1.0 API mode */
2061 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2062 
2063 	if (test == SELFTESTS) {
2064 		err = test_selftest(cg_fd, &options);
2065 		goto out;
2066 	}
2067 
2068 	err = populate_progs(bpf_file);
2069 	if (err) {
2070 		fprintf(stderr, "populate program: (%s) %s\n",
2071 			bpf_file, strerror(errno));
2072 		return 1;
2073 	}
2074 	running = 1;
2075 
2076 	/* catch SIGINT */
2077 	signal(SIGINT, running_handler);
2078 
2079 	options.iov_count = iov_count;
2080 	options.iov_length = length;
2081 	options.rate = rate;
2082 
2083 	err = run_options(&options, cg_fd, test);
2084 out:
2085 	if (options.whitelist)
2086 		free(options.whitelist);
2087 	if (options.blacklist)
2088 		free(options.blacklist);
2089 	if (cg_created)
2090 		cleanup_cgroup_environment();
2091 	close(cg_fd);
2092 	return err;
2093 }
2094 
2095 void running_handler(int a)
2096 {
2097 	running = 0;
2098 }
2099