1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19 
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23 
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <linux/tls.h>
30 #include <assert.h>
31 #include <libgen.h>
32 
33 #include <getopt.h>
34 
35 #include <bpf/bpf.h>
36 #include <bpf/libbpf.h>
37 
38 #include "bpf_util.h"
39 #include "cgroup_helpers.h"
40 
41 int running;
42 static void running_handler(int a);
43 
44 #ifndef TCP_ULP
45 # define TCP_ULP 31
46 #endif
47 #ifndef SOL_TLS
48 # define SOL_TLS 282
49 #endif
50 
51 /* randomly selected ports for testing on lo */
52 #define S1_PORT 10000
53 #define S2_PORT 10001
54 
55 #define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.bpf.o"
56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57 #define CG_PATH "/sockmap"
58 
59 /* global sockets */
60 int s1, s2, c1, c2, p1, p2;
61 int test_cnt;
62 int passed;
63 int failed;
64 int map_fd[9];
65 struct bpf_map *maps[9];
66 int prog_fd[9];
67 
68 int txmsg_pass;
69 int txmsg_redir;
70 int txmsg_drop;
71 int txmsg_apply;
72 int txmsg_cork;
73 int txmsg_start;
74 int txmsg_end;
75 int txmsg_start_push;
76 int txmsg_end_push;
77 int txmsg_start_pop;
78 int txmsg_pop;
79 int txmsg_ingress;
80 int txmsg_redir_skb;
81 int txmsg_ktls_skb;
82 int txmsg_ktls_skb_drop;
83 int txmsg_ktls_skb_redir;
84 int ktls;
85 int peek_flag;
86 int skb_use_parser;
87 int txmsg_omit_skb_parser;
88 
89 static const struct option long_options[] = {
90 	{"help",	no_argument,		NULL, 'h' },
91 	{"cgroup",	required_argument,	NULL, 'c' },
92 	{"rate",	required_argument,	NULL, 'r' },
93 	{"verbose",	optional_argument,	NULL, 'v' },
94 	{"iov_count",	required_argument,	NULL, 'i' },
95 	{"length",	required_argument,	NULL, 'l' },
96 	{"test",	required_argument,	NULL, 't' },
97 	{"data_test",   no_argument,		NULL, 'd' },
98 	{"txmsg",		no_argument,	&txmsg_pass,  1  },
99 	{"txmsg_redir",		no_argument,	&txmsg_redir, 1  },
100 	{"txmsg_drop",		no_argument,	&txmsg_drop, 1 },
101 	{"txmsg_apply",	required_argument,	NULL, 'a'},
102 	{"txmsg_cork",	required_argument,	NULL, 'k'},
103 	{"txmsg_start", required_argument,	NULL, 's'},
104 	{"txmsg_end",	required_argument,	NULL, 'e'},
105 	{"txmsg_start_push", required_argument,	NULL, 'p'},
106 	{"txmsg_end_push",   required_argument,	NULL, 'q'},
107 	{"txmsg_start_pop",  required_argument,	NULL, 'w'},
108 	{"txmsg_pop",	     required_argument,	NULL, 'x'},
109 	{"txmsg_ingress", no_argument,		&txmsg_ingress, 1 },
110 	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
111 	{"ktls", no_argument,			&ktls, 1 },
112 	{"peek", no_argument,			&peek_flag, 1 },
113 	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
114 	{"whitelist", required_argument,	NULL, 'n' },
115 	{"blacklist", required_argument,	NULL, 'b' },
116 	{0, 0, NULL, 0 }
117 };
118 
119 struct test_env {
120 	const char *type;
121 	const char *subtest;
122 	const char *prepend;
123 
124 	int test_num;
125 	int subtest_num;
126 
127 	int succ_cnt;
128 	int fail_cnt;
129 	int fail_last;
130 };
131 
132 struct test_env env;
133 
134 struct sockmap_options {
135 	int verbose;
136 	bool base;
137 	bool sendpage;
138 	bool data_test;
139 	bool drop_expected;
140 	bool check_recved_len;
141 	bool tx_wait_mem;
142 	int iov_count;
143 	int iov_length;
144 	int rate;
145 	char *map;
146 	char *whitelist;
147 	char *blacklist;
148 	char *prepend;
149 };
150 
151 struct _test {
152 	char *title;
153 	void (*tester)(int cg_fd, struct sockmap_options *opt);
154 };
155 
156 static void test_start(void)
157 {
158 	env.subtest_num++;
159 }
160 
161 static void test_fail(void)
162 {
163 	env.fail_cnt++;
164 }
165 
166 static void test_pass(void)
167 {
168 	env.succ_cnt++;
169 }
170 
171 static void test_reset(void)
172 {
173 	txmsg_start = txmsg_end = 0;
174 	txmsg_start_pop = txmsg_pop = 0;
175 	txmsg_start_push = txmsg_end_push = 0;
176 	txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 	txmsg_apply = txmsg_cork = 0;
178 	txmsg_ingress = txmsg_redir_skb = 0;
179 	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 	txmsg_omit_skb_parser = 0;
181 	skb_use_parser = 0;
182 }
183 
184 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185 {
186 	env.type = o->map;
187 	env.subtest = t->title;
188 	env.prepend = o->prepend;
189 	env.test_num++;
190 	env.subtest_num = 0;
191 	env.fail_last = env.fail_cnt;
192 	test_reset();
193 	return 0;
194 }
195 
196 static void test_end_subtest(void)
197 {
198 	int error = env.fail_cnt - env.fail_last;
199 	int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200 
201 	if (!error)
202 		test_pass();
203 
204 	fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 		env.test_num, env.subtest_num,
206 		!type ? "sockmap" : "sockhash",
207 		env.prepend ? : "",
208 		env.subtest, error ? "FAIL" : "OK");
209 }
210 
211 static void test_print_results(void)
212 {
213 	fprintf(stdout, "Pass: %d Fail: %d\n",
214 		env.succ_cnt, env.fail_cnt);
215 }
216 
217 static void usage(char *argv[])
218 {
219 	int i;
220 
221 	printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 	printf(" options:\n");
223 	for (i = 0; long_options[i].name != 0; i++) {
224 		printf(" --%-12s", long_options[i].name);
225 		if (long_options[i].flag != NULL)
226 			printf(" flag (internal value:%d)\n",
227 				*long_options[i].flag);
228 		else
229 			printf(" -%c\n", long_options[i].val);
230 	}
231 	printf("\n");
232 }
233 
234 char *sock_to_string(int s)
235 {
236 	if (s == c1)
237 		return "client1";
238 	else if (s == c2)
239 		return "client2";
240 	else if (s == s1)
241 		return "server1";
242 	else if (s == s2)
243 		return "server2";
244 	else if (s == p1)
245 		return "peer1";
246 	else if (s == p2)
247 		return "peer2";
248 	else
249 		return "unknown";
250 }
251 
252 static int sockmap_init_ktls(int verbose, int s)
253 {
254 	struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 		.info = {
256 			.version     = TLS_1_2_VERSION,
257 			.cipher_type = TLS_CIPHER_AES_GCM_128,
258 		},
259 	};
260 	struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 		.info = {
262 			.version     = TLS_1_2_VERSION,
263 			.cipher_type = TLS_CIPHER_AES_GCM_128,
264 		},
265 	};
266 	int so_buf = 6553500;
267 	int err;
268 
269 	err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 	if (err) {
271 		fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 		return -EINVAL;
273 	}
274 	err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 	if (err) {
276 		fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 		return -EINVAL;
278 	}
279 	err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 	if (err) {
281 		fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 		return -EINVAL;
283 	}
284 	err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 	if (err) {
286 		fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 		return -EINVAL;
288 	}
289 	err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 	if (err) {
291 		fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 		return -EINVAL;
293 	}
294 
295 	if (verbose)
296 		fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 	return 0;
298 }
299 static int sockmap_init_sockets(int verbose)
300 {
301 	int i, err, one = 1;
302 	struct sockaddr_in addr;
303 	int *fds[4] = {&s1, &s2, &c1, &c2};
304 
305 	s1 = s2 = p1 = p2 = c1 = c2 = 0;
306 
307 	/* Init sockets */
308 	for (i = 0; i < 4; i++) {
309 		*fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 		if (*fds[i] < 0) {
311 			perror("socket s1 failed()");
312 			return errno;
313 		}
314 	}
315 
316 	/* Allow reuse */
317 	for (i = 0; i < 2; i++) {
318 		err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 				 (char *)&one, sizeof(one));
320 		if (err) {
321 			perror("setsockopt failed()");
322 			return errno;
323 		}
324 	}
325 
326 	/* Non-blocking sockets */
327 	for (i = 0; i < 2; i++) {
328 		err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 		if (err < 0) {
330 			perror("ioctl s1 failed()");
331 			return errno;
332 		}
333 	}
334 
335 	/* Bind server sockets */
336 	memset(&addr, 0, sizeof(struct sockaddr_in));
337 	addr.sin_family = AF_INET;
338 	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339 
340 	addr.sin_port = htons(S1_PORT);
341 	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 	if (err < 0) {
343 		perror("bind s1 failed()");
344 		return errno;
345 	}
346 
347 	addr.sin_port = htons(S2_PORT);
348 	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 	if (err < 0) {
350 		perror("bind s2 failed()");
351 		return errno;
352 	}
353 
354 	/* Listen server sockets */
355 	addr.sin_port = htons(S1_PORT);
356 	err = listen(s1, 32);
357 	if (err < 0) {
358 		perror("listen s1 failed()");
359 		return errno;
360 	}
361 
362 	addr.sin_port = htons(S2_PORT);
363 	err = listen(s2, 32);
364 	if (err < 0) {
365 		perror("listen s1 failed()");
366 		return errno;
367 	}
368 
369 	/* Initiate Connect */
370 	addr.sin_port = htons(S1_PORT);
371 	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 	if (err < 0 && errno != EINPROGRESS) {
373 		perror("connect c1 failed()");
374 		return errno;
375 	}
376 
377 	addr.sin_port = htons(S2_PORT);
378 	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 	if (err < 0 && errno != EINPROGRESS) {
380 		perror("connect c2 failed()");
381 		return errno;
382 	} else if (err < 0) {
383 		err = 0;
384 	}
385 
386 	/* Accept Connecrtions */
387 	p1 = accept(s1, NULL, NULL);
388 	if (p1 < 0) {
389 		perror("accept s1 failed()");
390 		return errno;
391 	}
392 
393 	p2 = accept(s2, NULL, NULL);
394 	if (p2 < 0) {
395 		perror("accept s1 failed()");
396 		return errno;
397 	}
398 
399 	if (verbose > 1) {
400 		printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 		printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 			c1, s1, c2, s2);
403 	}
404 	return 0;
405 }
406 
407 struct msg_stats {
408 	size_t bytes_sent;
409 	size_t bytes_recvd;
410 	struct timespec start;
411 	struct timespec end;
412 };
413 
414 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 			     struct msg_stats *s,
416 			     struct sockmap_options *opt)
417 {
418 	bool drop = opt->drop_expected;
419 	unsigned char k = 0;
420 	FILE *file;
421 	int i, fp;
422 
423 	file = tmpfile();
424 	if (!file) {
425 		perror("create file for sendpage");
426 		return 1;
427 	}
428 	for (i = 0; i < iov_length * cnt; i++, k++)
429 		fwrite(&k, sizeof(char), 1, file);
430 	fflush(file);
431 	fseek(file, 0, SEEK_SET);
432 
433 	fp = fileno(file);
434 
435 	clock_gettime(CLOCK_MONOTONIC, &s->start);
436 	for (i = 0; i < cnt; i++) {
437 		int sent;
438 
439 		errno = 0;
440 		sent = sendfile(fd, fp, NULL, iov_length);
441 
442 		if (!drop && sent < 0) {
443 			perror("sendpage loop error");
444 			fclose(file);
445 			return sent;
446 		} else if (drop && sent >= 0) {
447 			printf("sendpage loop error expected: %i errno %i\n",
448 			       sent, errno);
449 			fclose(file);
450 			return -EIO;
451 		}
452 
453 		if (sent > 0)
454 			s->bytes_sent += sent;
455 	}
456 	clock_gettime(CLOCK_MONOTONIC, &s->end);
457 	fclose(file);
458 	return 0;
459 }
460 
461 static void msg_free_iov(struct msghdr *msg)
462 {
463 	int i;
464 
465 	for (i = 0; i < msg->msg_iovlen; i++)
466 		free(msg->msg_iov[i].iov_base);
467 	free(msg->msg_iov);
468 	msg->msg_iov = NULL;
469 	msg->msg_iovlen = 0;
470 }
471 
472 static int msg_alloc_iov(struct msghdr *msg,
473 			 int iov_count, int iov_length,
474 			 bool data, bool xmit)
475 {
476 	unsigned char k = 0;
477 	struct iovec *iov;
478 	int i;
479 
480 	iov = calloc(iov_count, sizeof(struct iovec));
481 	if (!iov)
482 		return errno;
483 
484 	for (i = 0; i < iov_count; i++) {
485 		unsigned char *d = calloc(iov_length, sizeof(char));
486 
487 		if (!d) {
488 			fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 			goto unwind_iov;
490 		}
491 		iov[i].iov_base = d;
492 		iov[i].iov_len = iov_length;
493 
494 		if (data && xmit) {
495 			int j;
496 
497 			for (j = 0; j < iov_length; j++)
498 				d[j] = k++;
499 		}
500 	}
501 
502 	msg->msg_iov = iov;
503 	msg->msg_iovlen = iov_count;
504 
505 	return 0;
506 unwind_iov:
507 	for (i--; i >= 0 ; i--)
508 		free(msg->msg_iov[i].iov_base);
509 	return -ENOMEM;
510 }
511 
512 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513 {
514 	int i, j = 0, bytes_cnt = 0;
515 	unsigned char k = 0;
516 
517 	for (i = 0; i < msg->msg_iovlen; i++) {
518 		unsigned char *d = msg->msg_iov[i].iov_base;
519 
520 		/* Special case test for skb ingress + ktls */
521 		if (i == 0 && txmsg_ktls_skb) {
522 			if (msg->msg_iov[i].iov_len < 4)
523 				return -EIO;
524 			if (memcmp(d, "PASS", 4) != 0) {
525 				fprintf(stderr,
526 					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 					i, 0, d[0], d[1], d[2], d[3]);
528 				return -EIO;
529 			}
530 			j = 4; /* advance index past PASS header */
531 		}
532 
533 		for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 			if (d[j] != k++) {
535 				fprintf(stderr,
536 					"detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 					i, j, d[j], k - 1, d[j+1], k);
538 				return -EIO;
539 			}
540 			bytes_cnt++;
541 			if (bytes_cnt == chunk_sz) {
542 				k = 0;
543 				bytes_cnt = 0;
544 			}
545 			size--;
546 		}
547 	}
548 	return 0;
549 }
550 
551 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 		    struct msg_stats *s, bool tx,
553 		    struct sockmap_options *opt)
554 {
555 	struct msghdr msg = {0}, msg_peek = {0};
556 	int err, i, flags = MSG_NOSIGNAL;
557 	bool drop = opt->drop_expected;
558 	bool data = opt->data_test;
559 	int iov_alloc_length = iov_length;
560 
561 	if (!tx && opt->check_recved_len)
562 		iov_alloc_length *= 2;
563 
564 	err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
565 	if (err)
566 		goto out_errno;
567 	if (peek_flag) {
568 		err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
569 		if (err)
570 			goto out_errno;
571 	}
572 
573 	if (tx) {
574 		clock_gettime(CLOCK_MONOTONIC, &s->start);
575 		for (i = 0; i < cnt; i++) {
576 			int sent;
577 
578 			errno = 0;
579 			sent = sendmsg(fd, &msg, flags);
580 
581 			if (!drop && sent < 0) {
582 				if (opt->tx_wait_mem && errno == EACCES) {
583 					errno = 0;
584 					goto out_errno;
585 				}
586 				perror("sendmsg loop error");
587 				goto out_errno;
588 			} else if (drop && sent >= 0) {
589 				fprintf(stderr,
590 					"sendmsg loop error expected: %i errno %i\n",
591 					sent, errno);
592 				errno = -EIO;
593 				goto out_errno;
594 			}
595 			if (sent > 0)
596 				s->bytes_sent += sent;
597 		}
598 		clock_gettime(CLOCK_MONOTONIC, &s->end);
599 	} else {
600 		int slct, recvp = 0, recv, max_fd = fd;
601 		float total_bytes, txmsg_pop_total;
602 		int fd_flags = O_NONBLOCK;
603 		struct timeval timeout;
604 		fd_set w;
605 
606 		fcntl(fd, fd_flags);
607 		/* Account for pop bytes noting each iteration of apply will
608 		 * call msg_pop_data helper so we need to account for this
609 		 * by calculating the number of apply iterations. Note user
610 		 * of the tool can create cases where no data is sent by
611 		 * manipulating pop/push/pull/etc. For example txmsg_apply 1
612 		 * with txmsg_pop 1 will try to apply 1B at a time but each
613 		 * iteration will then pop 1B so no data will ever be sent.
614 		 * This is really only useful for testing edge cases in code
615 		 * paths.
616 		 */
617 		total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
618 		if (txmsg_apply)
619 			txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
620 		else
621 			txmsg_pop_total = txmsg_pop * cnt;
622 		total_bytes -= txmsg_pop_total;
623 		err = clock_gettime(CLOCK_MONOTONIC, &s->start);
624 		if (err < 0)
625 			perror("recv start time");
626 		while (s->bytes_recvd < total_bytes) {
627 			if (txmsg_cork) {
628 				timeout.tv_sec = 0;
629 				timeout.tv_usec = 300000;
630 			} else {
631 				timeout.tv_sec = 3;
632 				timeout.tv_usec = 0;
633 			}
634 
635 			/* FD sets */
636 			FD_ZERO(&w);
637 			FD_SET(fd, &w);
638 
639 			slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
640 			if (slct == -1) {
641 				perror("select()");
642 				clock_gettime(CLOCK_MONOTONIC, &s->end);
643 				goto out_errno;
644 			} else if (!slct) {
645 				if (opt->verbose)
646 					fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
647 				errno = -EIO;
648 				clock_gettime(CLOCK_MONOTONIC, &s->end);
649 				goto out_errno;
650 			}
651 
652 			if (opt->tx_wait_mem) {
653 				FD_ZERO(&w);
654 				FD_SET(fd, &w);
655 				slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
656 				errno = 0;
657 				close(fd);
658 				goto out_errno;
659 			}
660 
661 			errno = 0;
662 			if (peek_flag) {
663 				flags |= MSG_PEEK;
664 				recvp = recvmsg(fd, &msg_peek, flags);
665 				if (recvp < 0) {
666 					if (errno != EWOULDBLOCK) {
667 						clock_gettime(CLOCK_MONOTONIC, &s->end);
668 						goto out_errno;
669 					}
670 				}
671 				flags = 0;
672 			}
673 
674 			recv = recvmsg(fd, &msg, flags);
675 			if (recv < 0) {
676 				if (errno != EWOULDBLOCK) {
677 					clock_gettime(CLOCK_MONOTONIC, &s->end);
678 					perror("recv failed()");
679 					goto out_errno;
680 				}
681 			}
682 
683 			if (recv > 0)
684 				s->bytes_recvd += recv;
685 
686 			if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
687 				errno = EMSGSIZE;
688 				fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
689 						s->bytes_recvd, total_bytes);
690 				goto out_errno;
691 			}
692 
693 			if (data) {
694 				int chunk_sz = opt->sendpage ?
695 						iov_length * cnt :
696 						iov_length * iov_count;
697 
698 				errno = msg_verify_data(&msg, recv, chunk_sz);
699 				if (errno) {
700 					perror("data verify msg failed");
701 					goto out_errno;
702 				}
703 				if (recvp) {
704 					errno = msg_verify_data(&msg_peek,
705 								recvp,
706 								chunk_sz);
707 					if (errno) {
708 						perror("data verify msg_peek failed");
709 						goto out_errno;
710 					}
711 				}
712 			}
713 		}
714 		clock_gettime(CLOCK_MONOTONIC, &s->end);
715 	}
716 
717 	msg_free_iov(&msg);
718 	msg_free_iov(&msg_peek);
719 	return err;
720 out_errno:
721 	msg_free_iov(&msg);
722 	msg_free_iov(&msg_peek);
723 	return errno;
724 }
725 
726 static float giga = 1000000000;
727 
728 static inline float sentBps(struct msg_stats s)
729 {
730 	return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
731 }
732 
733 static inline float recvdBps(struct msg_stats s)
734 {
735 	return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
736 }
737 
738 static int sendmsg_test(struct sockmap_options *opt)
739 {
740 	float sent_Bps = 0, recvd_Bps = 0;
741 	int rx_fd, txpid, rxpid, err = 0;
742 	struct msg_stats s = {0};
743 	int iov_count = opt->iov_count;
744 	int iov_buf = opt->iov_length;
745 	int rx_status, tx_status;
746 	int cnt = opt->rate;
747 
748 	errno = 0;
749 
750 	if (opt->base)
751 		rx_fd = p1;
752 	else
753 		rx_fd = p2;
754 
755 	if (ktls) {
756 		/* Redirecting into non-TLS socket which sends into a TLS
757 		 * socket is not a valid test. So in this case lets not
758 		 * enable kTLS but still run the test.
759 		 */
760 		if (!txmsg_redir || txmsg_ingress) {
761 			err = sockmap_init_ktls(opt->verbose, rx_fd);
762 			if (err)
763 				return err;
764 		}
765 		err = sockmap_init_ktls(opt->verbose, c1);
766 		if (err)
767 			return err;
768 	}
769 
770 	if (opt->tx_wait_mem) {
771 		struct timeval timeout;
772 		int rxtx_buf_len = 1024;
773 
774 		timeout.tv_sec = 3;
775 		timeout.tv_usec = 0;
776 
777 		err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
778 		err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
779 		err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
780 		if (err) {
781 			perror("setsockopt failed()");
782 			return errno;
783 		}
784 	}
785 
786 	rxpid = fork();
787 	if (rxpid == 0) {
788 		if (txmsg_pop || txmsg_start_pop)
789 			iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
790 		if (opt->drop_expected || txmsg_ktls_skb_drop)
791 			_exit(0);
792 
793 		if (!iov_buf) /* zero bytes sent case */
794 			_exit(0);
795 
796 		if (opt->sendpage)
797 			iov_count = 1;
798 		err = msg_loop(rx_fd, iov_count, iov_buf,
799 			       cnt, &s, false, opt);
800 		if (opt->verbose > 1)
801 			fprintf(stderr,
802 				"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
803 				iov_count, iov_buf, cnt, err);
804 		if (s.end.tv_sec - s.start.tv_sec) {
805 			sent_Bps = sentBps(s);
806 			recvd_Bps = recvdBps(s);
807 		}
808 		if (opt->verbose > 1)
809 			fprintf(stdout,
810 				"rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
811 				s.bytes_sent, sent_Bps, sent_Bps/giga,
812 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
813 				peek_flag ? "(peek_msg)" : "");
814 		if (err && txmsg_cork)
815 			err = 0;
816 		exit(err ? 1 : 0);
817 	} else if (rxpid == -1) {
818 		perror("msg_loop_rx");
819 		return errno;
820 	}
821 
822 	if (opt->tx_wait_mem)
823 		close(c2);
824 
825 	txpid = fork();
826 	if (txpid == 0) {
827 		if (opt->sendpage)
828 			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
829 		else
830 			err = msg_loop(c1, iov_count, iov_buf,
831 				       cnt, &s, true, opt);
832 
833 		if (err)
834 			fprintf(stderr,
835 				"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
836 				iov_count, iov_buf, cnt, err);
837 		if (s.end.tv_sec - s.start.tv_sec) {
838 			sent_Bps = sentBps(s);
839 			recvd_Bps = recvdBps(s);
840 		}
841 		if (opt->verbose > 1)
842 			fprintf(stdout,
843 				"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
844 				s.bytes_sent, sent_Bps, sent_Bps/giga,
845 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
846 		exit(err ? 1 : 0);
847 	} else if (txpid == -1) {
848 		perror("msg_loop_tx");
849 		return errno;
850 	}
851 
852 	assert(waitpid(rxpid, &rx_status, 0) == rxpid);
853 	assert(waitpid(txpid, &tx_status, 0) == txpid);
854 	if (WIFEXITED(rx_status)) {
855 		err = WEXITSTATUS(rx_status);
856 		if (err) {
857 			fprintf(stderr, "rx thread exited with err %d.\n", err);
858 			goto out;
859 		}
860 	}
861 	if (WIFEXITED(tx_status)) {
862 		err = WEXITSTATUS(tx_status);
863 		if (err)
864 			fprintf(stderr, "tx thread exited with err %d.\n", err);
865 	}
866 out:
867 	return err;
868 }
869 
870 static int forever_ping_pong(int rate, struct sockmap_options *opt)
871 {
872 	struct timeval timeout;
873 	char buf[1024] = {0};
874 	int sc;
875 
876 	timeout.tv_sec = 10;
877 	timeout.tv_usec = 0;
878 
879 	/* Ping/Pong data from client to server */
880 	sc = send(c1, buf, sizeof(buf), 0);
881 	if (sc < 0) {
882 		perror("send failed()");
883 		return sc;
884 	}
885 
886 	do {
887 		int s, rc, i, max_fd = p2;
888 		fd_set w;
889 
890 		/* FD sets */
891 		FD_ZERO(&w);
892 		FD_SET(c1, &w);
893 		FD_SET(c2, &w);
894 		FD_SET(p1, &w);
895 		FD_SET(p2, &w);
896 
897 		s = select(max_fd + 1, &w, NULL, NULL, &timeout);
898 		if (s == -1) {
899 			perror("select()");
900 			break;
901 		} else if (!s) {
902 			fprintf(stderr, "unexpected timeout\n");
903 			break;
904 		}
905 
906 		for (i = 0; i <= max_fd && s > 0; ++i) {
907 			if (!FD_ISSET(i, &w))
908 				continue;
909 
910 			s--;
911 
912 			rc = recv(i, buf, sizeof(buf), 0);
913 			if (rc < 0) {
914 				if (errno != EWOULDBLOCK) {
915 					perror("recv failed()");
916 					return rc;
917 				}
918 			}
919 
920 			if (rc == 0) {
921 				close(i);
922 				break;
923 			}
924 
925 			sc = send(i, buf, rc, 0);
926 			if (sc < 0) {
927 				perror("send failed()");
928 				return sc;
929 			}
930 		}
931 
932 		if (rate)
933 			sleep(rate);
934 
935 		if (opt->verbose) {
936 			printf(".");
937 			fflush(stdout);
938 
939 		}
940 	} while (running);
941 
942 	return 0;
943 }
944 
945 enum {
946 	SELFTESTS,
947 	PING_PONG,
948 	SENDMSG,
949 	BASE,
950 	BASE_SENDPAGE,
951 	SENDPAGE,
952 };
953 
954 static int run_options(struct sockmap_options *options, int cg_fd,  int test)
955 {
956 	int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
957 
958 	/* If base test skip BPF setup */
959 	if (test == BASE || test == BASE_SENDPAGE)
960 		goto run;
961 
962 	/* Attach programs to sockmap */
963 	if (!txmsg_omit_skb_parser) {
964 		err = bpf_prog_attach(prog_fd[0], map_fd[0],
965 				      BPF_SK_SKB_STREAM_PARSER, 0);
966 		if (err) {
967 			fprintf(stderr,
968 				"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
969 				prog_fd[0], map_fd[0], err, strerror(errno));
970 			return err;
971 		}
972 	}
973 
974 	err = bpf_prog_attach(prog_fd[1], map_fd[0],
975 				BPF_SK_SKB_STREAM_VERDICT, 0);
976 	if (err) {
977 		fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
978 			err, strerror(errno));
979 		return err;
980 	}
981 
982 	/* Attach programs to TLS sockmap */
983 	if (txmsg_ktls_skb) {
984 		if (!txmsg_omit_skb_parser) {
985 			err = bpf_prog_attach(prog_fd[0], map_fd[8],
986 					      BPF_SK_SKB_STREAM_PARSER, 0);
987 			if (err) {
988 				fprintf(stderr,
989 					"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
990 					prog_fd[0], map_fd[8], err, strerror(errno));
991 				return err;
992 			}
993 		}
994 
995 		err = bpf_prog_attach(prog_fd[2], map_fd[8],
996 				      BPF_SK_SKB_STREAM_VERDICT, 0);
997 		if (err) {
998 			fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
999 				err, strerror(errno));
1000 			return err;
1001 		}
1002 	}
1003 
1004 	/* Attach to cgroups */
1005 	err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1006 	if (err) {
1007 		fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1008 			err, strerror(errno));
1009 		return err;
1010 	}
1011 
1012 run:
1013 	err = sockmap_init_sockets(options->verbose);
1014 	if (err) {
1015 		fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1016 		goto out;
1017 	}
1018 
1019 	/* Attach txmsg program to sockmap */
1020 	if (txmsg_pass)
1021 		tx_prog_fd = prog_fd[4];
1022 	else if (txmsg_redir)
1023 		tx_prog_fd = prog_fd[5];
1024 	else if (txmsg_apply)
1025 		tx_prog_fd = prog_fd[6];
1026 	else if (txmsg_cork)
1027 		tx_prog_fd = prog_fd[7];
1028 	else if (txmsg_drop)
1029 		tx_prog_fd = prog_fd[8];
1030 	else
1031 		tx_prog_fd = 0;
1032 
1033 	if (tx_prog_fd) {
1034 		int redir_fd, i = 0;
1035 
1036 		err = bpf_prog_attach(tx_prog_fd,
1037 				      map_fd[1], BPF_SK_MSG_VERDICT, 0);
1038 		if (err) {
1039 			fprintf(stderr,
1040 				"ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1041 				err, strerror(errno));
1042 			goto out;
1043 		}
1044 
1045 		err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1046 		if (err) {
1047 			fprintf(stderr,
1048 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1049 				err, strerror(errno));
1050 			goto out;
1051 		}
1052 
1053 		if (txmsg_redir)
1054 			redir_fd = c2;
1055 		else
1056 			redir_fd = c1;
1057 
1058 		err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1059 		if (err) {
1060 			fprintf(stderr,
1061 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1062 				err, strerror(errno));
1063 			goto out;
1064 		}
1065 
1066 		if (txmsg_apply) {
1067 			err = bpf_map_update_elem(map_fd[3],
1068 						  &i, &txmsg_apply, BPF_ANY);
1069 			if (err) {
1070 				fprintf(stderr,
1071 					"ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1072 					err, strerror(errno));
1073 				goto out;
1074 			}
1075 		}
1076 
1077 		if (txmsg_cork) {
1078 			err = bpf_map_update_elem(map_fd[4],
1079 						  &i, &txmsg_cork, BPF_ANY);
1080 			if (err) {
1081 				fprintf(stderr,
1082 					"ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1083 					err, strerror(errno));
1084 				goto out;
1085 			}
1086 		}
1087 
1088 		if (txmsg_start) {
1089 			err = bpf_map_update_elem(map_fd[5],
1090 						  &i, &txmsg_start, BPF_ANY);
1091 			if (err) {
1092 				fprintf(stderr,
1093 					"ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1094 					err, strerror(errno));
1095 				goto out;
1096 			}
1097 		}
1098 
1099 		if (txmsg_end) {
1100 			i = 1;
1101 			err = bpf_map_update_elem(map_fd[5],
1102 						  &i, &txmsg_end, BPF_ANY);
1103 			if (err) {
1104 				fprintf(stderr,
1105 					"ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1106 					err, strerror(errno));
1107 				goto out;
1108 			}
1109 		}
1110 
1111 		if (txmsg_start_push) {
1112 			i = 2;
1113 			err = bpf_map_update_elem(map_fd[5],
1114 						  &i, &txmsg_start_push, BPF_ANY);
1115 			if (err) {
1116 				fprintf(stderr,
1117 					"ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1118 					err, strerror(errno));
1119 				goto out;
1120 			}
1121 		}
1122 
1123 		if (txmsg_end_push) {
1124 			i = 3;
1125 			err = bpf_map_update_elem(map_fd[5],
1126 						  &i, &txmsg_end_push, BPF_ANY);
1127 			if (err) {
1128 				fprintf(stderr,
1129 					"ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1130 					txmsg_end_push, i, err, strerror(errno));
1131 				goto out;
1132 			}
1133 		}
1134 
1135 		if (txmsg_start_pop) {
1136 			i = 4;
1137 			err = bpf_map_update_elem(map_fd[5],
1138 						  &i, &txmsg_start_pop, BPF_ANY);
1139 			if (err) {
1140 				fprintf(stderr,
1141 					"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1142 					txmsg_start_pop, i, err, strerror(errno));
1143 				goto out;
1144 			}
1145 		} else {
1146 			i = 4;
1147 			bpf_map_update_elem(map_fd[5],
1148 						  &i, &txmsg_start_pop, BPF_ANY);
1149 		}
1150 
1151 		if (txmsg_pop) {
1152 			i = 5;
1153 			err = bpf_map_update_elem(map_fd[5],
1154 						  &i, &txmsg_pop, BPF_ANY);
1155 			if (err) {
1156 				fprintf(stderr,
1157 					"ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1158 					txmsg_pop, i, err, strerror(errno));
1159 				goto out;
1160 			}
1161 		} else {
1162 			i = 5;
1163 			bpf_map_update_elem(map_fd[5],
1164 					    &i, &txmsg_pop, BPF_ANY);
1165 
1166 		}
1167 
1168 		if (txmsg_ingress) {
1169 			int in = BPF_F_INGRESS;
1170 
1171 			i = 0;
1172 			err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1173 			if (err) {
1174 				fprintf(stderr,
1175 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1176 					err, strerror(errno));
1177 			}
1178 			i = 1;
1179 			err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1180 			if (err) {
1181 				fprintf(stderr,
1182 					"ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1183 					err, strerror(errno));
1184 			}
1185 			err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1186 			if (err) {
1187 				fprintf(stderr,
1188 					"ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1189 					err, strerror(errno));
1190 			}
1191 
1192 			i = 2;
1193 			err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1194 			if (err) {
1195 				fprintf(stderr,
1196 					"ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1197 					err, strerror(errno));
1198 			}
1199 		}
1200 
1201 		if (txmsg_ktls_skb) {
1202 			int ingress = BPF_F_INGRESS;
1203 
1204 			i = 0;
1205 			err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1206 			if (err) {
1207 				fprintf(stderr,
1208 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1209 					err, strerror(errno));
1210 			}
1211 
1212 			if (txmsg_ktls_skb_redir) {
1213 				i = 1;
1214 				err = bpf_map_update_elem(map_fd[7],
1215 							  &i, &ingress, BPF_ANY);
1216 				if (err) {
1217 					fprintf(stderr,
1218 						"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1219 						err, strerror(errno));
1220 				}
1221 			}
1222 
1223 			if (txmsg_ktls_skb_drop) {
1224 				i = 1;
1225 				err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1226 			}
1227 		}
1228 
1229 		if (txmsg_redir_skb) {
1230 			int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1231 					p2 : p1;
1232 			int ingress = BPF_F_INGRESS;
1233 
1234 			i = 0;
1235 			err = bpf_map_update_elem(map_fd[7],
1236 						  &i, &ingress, BPF_ANY);
1237 			if (err) {
1238 				fprintf(stderr,
1239 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1240 					err, strerror(errno));
1241 			}
1242 
1243 			i = 3;
1244 			err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1245 			if (err) {
1246 				fprintf(stderr,
1247 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1248 					err, strerror(errno));
1249 			}
1250 		}
1251 	}
1252 
1253 	if (skb_use_parser) {
1254 		i = 2;
1255 		err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1256 	}
1257 
1258 	if (txmsg_drop)
1259 		options->drop_expected = true;
1260 
1261 	if (test == PING_PONG)
1262 		err = forever_ping_pong(options->rate, options);
1263 	else if (test == SENDMSG) {
1264 		options->base = false;
1265 		options->sendpage = false;
1266 		err = sendmsg_test(options);
1267 	} else if (test == SENDPAGE) {
1268 		options->base = false;
1269 		options->sendpage = true;
1270 		err = sendmsg_test(options);
1271 	} else if (test == BASE) {
1272 		options->base = true;
1273 		options->sendpage = false;
1274 		err = sendmsg_test(options);
1275 	} else if (test == BASE_SENDPAGE) {
1276 		options->base = true;
1277 		options->sendpage = true;
1278 		err = sendmsg_test(options);
1279 	} else
1280 		fprintf(stderr, "unknown test\n");
1281 out:
1282 	/* Detatch and zero all the maps */
1283 	bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1284 	bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1285 	bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1286 	bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1287 	bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1288 
1289 	if (tx_prog_fd >= 0)
1290 		bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1291 
1292 	for (i = 0; i < 8; i++) {
1293 		key = next_key = 0;
1294 		bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1295 		while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1296 			bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1297 			key = next_key;
1298 		}
1299 	}
1300 
1301 	close(s1);
1302 	close(s2);
1303 	close(p1);
1304 	close(p2);
1305 	close(c1);
1306 	close(c2);
1307 	return err;
1308 }
1309 
1310 static char *test_to_str(int test)
1311 {
1312 	switch (test) {
1313 	case SENDMSG:
1314 		return "sendmsg";
1315 	case SENDPAGE:
1316 		return "sendpage";
1317 	}
1318 	return "unknown";
1319 }
1320 
1321 static void append_str(char *dst, const char *src, size_t dst_cap)
1322 {
1323 	size_t avail = dst_cap - strlen(dst);
1324 
1325 	if (avail <= 1) /* just zero byte could be written */
1326 		return;
1327 
1328 	strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1329 }
1330 
1331 #define OPTSTRING 60
1332 static void test_options(char *options)
1333 {
1334 	char tstr[OPTSTRING];
1335 
1336 	memset(options, 0, OPTSTRING);
1337 
1338 	if (txmsg_pass)
1339 		append_str(options, "pass,", OPTSTRING);
1340 	if (txmsg_redir)
1341 		append_str(options, "redir,", OPTSTRING);
1342 	if (txmsg_drop)
1343 		append_str(options, "drop,", OPTSTRING);
1344 	if (txmsg_apply) {
1345 		snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1346 		append_str(options, tstr, OPTSTRING);
1347 	}
1348 	if (txmsg_cork) {
1349 		snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1350 		append_str(options, tstr, OPTSTRING);
1351 	}
1352 	if (txmsg_start) {
1353 		snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1354 		append_str(options, tstr, OPTSTRING);
1355 	}
1356 	if (txmsg_end) {
1357 		snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1358 		append_str(options, tstr, OPTSTRING);
1359 	}
1360 	if (txmsg_start_pop) {
1361 		snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1362 			 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1363 		append_str(options, tstr, OPTSTRING);
1364 	}
1365 	if (txmsg_ingress)
1366 		append_str(options, "ingress,", OPTSTRING);
1367 	if (txmsg_redir_skb)
1368 		append_str(options, "redir_skb,", OPTSTRING);
1369 	if (txmsg_ktls_skb)
1370 		append_str(options, "ktls_skb,", OPTSTRING);
1371 	if (ktls)
1372 		append_str(options, "ktls,", OPTSTRING);
1373 	if (peek_flag)
1374 		append_str(options, "peek,", OPTSTRING);
1375 }
1376 
1377 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1378 {
1379 	char *options = calloc(OPTSTRING, sizeof(char));
1380 	int err;
1381 
1382 	if (test == SENDPAGE)
1383 		opt->sendpage = true;
1384 	else
1385 		opt->sendpage = false;
1386 
1387 	if (txmsg_drop)
1388 		opt->drop_expected = true;
1389 	else
1390 		opt->drop_expected = false;
1391 
1392 	test_options(options);
1393 
1394 	if (opt->verbose) {
1395 		fprintf(stdout,
1396 			" [TEST %i]: (%i, %i, %i, %s, %s): ",
1397 			test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1398 			test_to_str(test), options);
1399 		fflush(stdout);
1400 	}
1401 	err = run_options(opt, cgrp, test);
1402 	if (opt->verbose)
1403 		fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1404 	test_cnt++;
1405 	!err ? passed++ : failed++;
1406 	free(options);
1407 	return err;
1408 }
1409 
1410 static void test_exec(int cgrp, struct sockmap_options *opt)
1411 {
1412 	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1413 	int err;
1414 
1415 	if (type == 0) {
1416 		test_start();
1417 		err = __test_exec(cgrp, SENDMSG, opt);
1418 		if (err)
1419 			test_fail();
1420 	} else {
1421 		test_start();
1422 		err = __test_exec(cgrp, SENDPAGE, opt);
1423 		if (err)
1424 			test_fail();
1425 	}
1426 }
1427 
1428 static void test_send_one(struct sockmap_options *opt, int cgrp)
1429 {
1430 	opt->iov_length = 1;
1431 	opt->iov_count = 1;
1432 	opt->rate = 1;
1433 	test_exec(cgrp, opt);
1434 
1435 	opt->iov_length = 1;
1436 	opt->iov_count = 1024;
1437 	opt->rate = 1;
1438 	test_exec(cgrp, opt);
1439 
1440 	opt->iov_length = 1024;
1441 	opt->iov_count = 1;
1442 	opt->rate = 1;
1443 	test_exec(cgrp, opt);
1444 
1445 }
1446 
1447 static void test_send_many(struct sockmap_options *opt, int cgrp)
1448 {
1449 	opt->iov_length = 3;
1450 	opt->iov_count = 1;
1451 	opt->rate = 512;
1452 	test_exec(cgrp, opt);
1453 
1454 	opt->rate = 100;
1455 	opt->iov_count = 1;
1456 	opt->iov_length = 5;
1457 	test_exec(cgrp, opt);
1458 }
1459 
1460 static void test_send_large(struct sockmap_options *opt, int cgrp)
1461 {
1462 	opt->iov_length = 256;
1463 	opt->iov_count = 1024;
1464 	opt->rate = 2;
1465 	test_exec(cgrp, opt);
1466 }
1467 
1468 static void test_send(struct sockmap_options *opt, int cgrp)
1469 {
1470 	test_send_one(opt, cgrp);
1471 	test_send_many(opt, cgrp);
1472 	test_send_large(opt, cgrp);
1473 	sched_yield();
1474 }
1475 
1476 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1477 {
1478 	/* Test small and large iov_count values with pass/redir/apply/cork */
1479 	txmsg_pass = 1;
1480 	test_send(opt, cgrp);
1481 }
1482 
1483 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1484 {
1485 	txmsg_redir = 1;
1486 	test_send(opt, cgrp);
1487 }
1488 
1489 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1490 {
1491 	txmsg_redir = 1;
1492 	opt->tx_wait_mem = true;
1493 	test_send_large(opt, cgrp);
1494 	opt->tx_wait_mem = false;
1495 }
1496 
1497 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1498 {
1499 	txmsg_drop = 1;
1500 	test_send(opt, cgrp);
1501 }
1502 
1503 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1504 {
1505 	txmsg_pass = txmsg_drop = 0;
1506 	txmsg_ingress = txmsg_redir = 1;
1507 	test_send(opt, cgrp);
1508 }
1509 
1510 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1511 {
1512 	bool data = opt->data_test;
1513 	int k = ktls;
1514 
1515 	opt->data_test = true;
1516 	ktls = 1;
1517 
1518 	txmsg_pass = txmsg_drop = 0;
1519 	txmsg_ingress = txmsg_redir = 0;
1520 	txmsg_ktls_skb = 1;
1521 	txmsg_pass = 1;
1522 
1523 	/* Using data verification so ensure iov layout is
1524 	 * expected from test receiver side. e.g. has enough
1525 	 * bytes to write test code.
1526 	 */
1527 	opt->iov_length = 100;
1528 	opt->iov_count = 1;
1529 	opt->rate = 1;
1530 	test_exec(cgrp, opt);
1531 
1532 	txmsg_ktls_skb_drop = 1;
1533 	test_exec(cgrp, opt);
1534 
1535 	txmsg_ktls_skb_drop = 0;
1536 	txmsg_ktls_skb_redir = 1;
1537 	test_exec(cgrp, opt);
1538 	txmsg_ktls_skb_redir = 0;
1539 
1540 	/* Tests that omit skb_parser */
1541 	txmsg_omit_skb_parser = 1;
1542 	ktls = 0;
1543 	txmsg_ktls_skb = 0;
1544 	test_exec(cgrp, opt);
1545 
1546 	txmsg_ktls_skb_drop = 1;
1547 	test_exec(cgrp, opt);
1548 	txmsg_ktls_skb_drop = 0;
1549 
1550 	txmsg_ktls_skb_redir = 1;
1551 	test_exec(cgrp, opt);
1552 
1553 	ktls = 1;
1554 	test_exec(cgrp, opt);
1555 	txmsg_omit_skb_parser = 0;
1556 
1557 	opt->data_test = data;
1558 	ktls = k;
1559 }
1560 
1561 /* Test cork with hung data. This tests poor usage patterns where
1562  * cork can leave data on the ring if user program is buggy and
1563  * doesn't flush them somehow. They do take some time however
1564  * because they wait for a timeout. Test pass, redir and cork with
1565  * apply logic. Use cork size of 4097 with send_large to avoid
1566  * aligning cork size with send size.
1567  */
1568 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1569 {
1570 	txmsg_pass = 1;
1571 	txmsg_redir = 0;
1572 	txmsg_cork = 4097;
1573 	txmsg_apply = 4097;
1574 	test_send_large(opt, cgrp);
1575 
1576 	txmsg_pass = 0;
1577 	txmsg_redir = 1;
1578 	txmsg_apply = 0;
1579 	txmsg_cork = 4097;
1580 	test_send_large(opt, cgrp);
1581 
1582 	txmsg_pass = 0;
1583 	txmsg_redir = 1;
1584 	txmsg_apply = 4097;
1585 	txmsg_cork = 4097;
1586 	test_send_large(opt, cgrp);
1587 }
1588 
1589 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1590 {
1591 	/* Test basic start/end */
1592 	txmsg_start = 1;
1593 	txmsg_end = 2;
1594 	test_send(opt, cgrp);
1595 
1596 	/* Test >4k pull */
1597 	txmsg_start = 4096;
1598 	txmsg_end = 9182;
1599 	test_send_large(opt, cgrp);
1600 
1601 	/* Test pull + redirect */
1602 	txmsg_redir = 0;
1603 	txmsg_start = 1;
1604 	txmsg_end = 2;
1605 	test_send(opt, cgrp);
1606 
1607 	/* Test pull + cork */
1608 	txmsg_redir = 0;
1609 	txmsg_cork = 512;
1610 	txmsg_start = 1;
1611 	txmsg_end = 2;
1612 	test_send_many(opt, cgrp);
1613 
1614 	/* Test pull + cork + redirect */
1615 	txmsg_redir = 1;
1616 	txmsg_cork = 512;
1617 	txmsg_start = 1;
1618 	txmsg_end = 2;
1619 	test_send_many(opt, cgrp);
1620 }
1621 
1622 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1623 {
1624 	/* Test basic pop */
1625 	txmsg_start_pop = 1;
1626 	txmsg_pop = 2;
1627 	test_send_many(opt, cgrp);
1628 
1629 	/* Test pop with >4k */
1630 	txmsg_start_pop = 4096;
1631 	txmsg_pop = 4096;
1632 	test_send_large(opt, cgrp);
1633 
1634 	/* Test pop + redirect */
1635 	txmsg_redir = 1;
1636 	txmsg_start_pop = 1;
1637 	txmsg_pop = 2;
1638 	test_send_many(opt, cgrp);
1639 
1640 	/* Test pop + cork */
1641 	txmsg_redir = 0;
1642 	txmsg_cork = 512;
1643 	txmsg_start_pop = 1;
1644 	txmsg_pop = 2;
1645 	test_send_many(opt, cgrp);
1646 
1647 	/* Test pop + redirect + cork */
1648 	txmsg_redir = 1;
1649 	txmsg_cork = 4;
1650 	txmsg_start_pop = 1;
1651 	txmsg_pop = 2;
1652 	test_send_many(opt, cgrp);
1653 }
1654 
1655 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1656 {
1657 	/* Test basic push */
1658 	txmsg_start_push = 1;
1659 	txmsg_end_push = 1;
1660 	test_send(opt, cgrp);
1661 
1662 	/* Test push 4kB >4k */
1663 	txmsg_start_push = 4096;
1664 	txmsg_end_push = 4096;
1665 	test_send_large(opt, cgrp);
1666 
1667 	/* Test push + redirect */
1668 	txmsg_redir = 1;
1669 	txmsg_start_push = 1;
1670 	txmsg_end_push = 2;
1671 	test_send_many(opt, cgrp);
1672 
1673 	/* Test push + cork */
1674 	txmsg_redir = 0;
1675 	txmsg_cork = 512;
1676 	txmsg_start_push = 1;
1677 	txmsg_end_push = 2;
1678 	test_send_many(opt, cgrp);
1679 }
1680 
1681 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1682 {
1683 	txmsg_start_push = 1;
1684 	txmsg_end_push = 10;
1685 	txmsg_start_pop = 5;
1686 	txmsg_pop = 4;
1687 	test_send_large(opt, cgrp);
1688 }
1689 
1690 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1691 {
1692 	txmsg_pass = 1;
1693 	txmsg_redir = 0;
1694 	txmsg_ingress = 0;
1695 	txmsg_apply = 1;
1696 	txmsg_cork = 0;
1697 	test_send_one(opt, cgrp);
1698 
1699 	txmsg_pass = 0;
1700 	txmsg_redir = 1;
1701 	txmsg_ingress = 0;
1702 	txmsg_apply = 1;
1703 	txmsg_cork = 0;
1704 	test_send_one(opt, cgrp);
1705 
1706 	txmsg_pass = 0;
1707 	txmsg_redir = 1;
1708 	txmsg_ingress = 1;
1709 	txmsg_apply = 1;
1710 	txmsg_cork = 0;
1711 	test_send_one(opt, cgrp);
1712 
1713 	txmsg_pass = 1;
1714 	txmsg_redir = 0;
1715 	txmsg_ingress = 0;
1716 	txmsg_apply = 1024;
1717 	txmsg_cork = 0;
1718 	test_send_large(opt, cgrp);
1719 
1720 	txmsg_pass = 0;
1721 	txmsg_redir = 1;
1722 	txmsg_ingress = 0;
1723 	txmsg_apply = 1024;
1724 	txmsg_cork = 0;
1725 	test_send_large(opt, cgrp);
1726 
1727 	txmsg_pass = 0;
1728 	txmsg_redir = 1;
1729 	txmsg_ingress = 1;
1730 	txmsg_apply = 1024;
1731 	txmsg_cork = 0;
1732 	test_send_large(opt, cgrp);
1733 }
1734 
1735 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1736 {
1737 	txmsg_pass = 1;
1738 	txmsg_redir = 0;
1739 	txmsg_apply = 0;
1740 	txmsg_cork = 1;
1741 	test_send(opt, cgrp);
1742 
1743 	txmsg_pass = 1;
1744 	txmsg_redir = 0;
1745 	txmsg_apply = 1;
1746 	txmsg_cork = 1;
1747 	test_send(opt, cgrp);
1748 }
1749 
1750 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1751 {
1752 	txmsg_pass = 1;
1753 	skb_use_parser = 512;
1754 	if (ktls == 1)
1755 		skb_use_parser = 570;
1756 	opt->iov_length = 256;
1757 	opt->iov_count = 1;
1758 	opt->rate = 2;
1759 	test_exec(cgrp, opt);
1760 }
1761 
1762 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1763 {
1764 	if (ktls == 1)
1765 		return;
1766 	skb_use_parser = 10;
1767 	opt->iov_length = 20;
1768 	opt->iov_count = 1;
1769 	opt->rate = 1;
1770 	opt->check_recved_len = true;
1771 	test_exec(cgrp, opt);
1772 	opt->check_recved_len = false;
1773 }
1774 
1775 char *map_names[] = {
1776 	"sock_map",
1777 	"sock_map_txmsg",
1778 	"sock_map_redir",
1779 	"sock_apply_bytes",
1780 	"sock_cork_bytes",
1781 	"sock_bytes",
1782 	"sock_redir_flags",
1783 	"sock_skb_opts",
1784 	"tls_sock_map",
1785 };
1786 
1787 int prog_attach_type[] = {
1788 	BPF_SK_SKB_STREAM_PARSER,
1789 	BPF_SK_SKB_STREAM_VERDICT,
1790 	BPF_SK_SKB_STREAM_VERDICT,
1791 	BPF_CGROUP_SOCK_OPS,
1792 	BPF_SK_MSG_VERDICT,
1793 	BPF_SK_MSG_VERDICT,
1794 	BPF_SK_MSG_VERDICT,
1795 	BPF_SK_MSG_VERDICT,
1796 	BPF_SK_MSG_VERDICT,
1797 };
1798 
1799 int prog_type[] = {
1800 	BPF_PROG_TYPE_SK_SKB,
1801 	BPF_PROG_TYPE_SK_SKB,
1802 	BPF_PROG_TYPE_SK_SKB,
1803 	BPF_PROG_TYPE_SOCK_OPS,
1804 	BPF_PROG_TYPE_SK_MSG,
1805 	BPF_PROG_TYPE_SK_MSG,
1806 	BPF_PROG_TYPE_SK_MSG,
1807 	BPF_PROG_TYPE_SK_MSG,
1808 	BPF_PROG_TYPE_SK_MSG,
1809 };
1810 
1811 static int populate_progs(char *bpf_file)
1812 {
1813 	struct bpf_program *prog;
1814 	struct bpf_object *obj;
1815 	int i = 0;
1816 	long err;
1817 
1818 	obj = bpf_object__open(bpf_file);
1819 	err = libbpf_get_error(obj);
1820 	if (err) {
1821 		char err_buf[256];
1822 
1823 		libbpf_strerror(err, err_buf, sizeof(err_buf));
1824 		printf("Unable to load eBPF objects in file '%s' : %s\n",
1825 		       bpf_file, err_buf);
1826 		return -1;
1827 	}
1828 
1829 	bpf_object__for_each_program(prog, obj) {
1830 		bpf_program__set_type(prog, prog_type[i]);
1831 		bpf_program__set_expected_attach_type(prog,
1832 						      prog_attach_type[i]);
1833 		i++;
1834 	}
1835 
1836 	i = bpf_object__load(obj);
1837 	i = 0;
1838 	bpf_object__for_each_program(prog, obj) {
1839 		prog_fd[i] = bpf_program__fd(prog);
1840 		i++;
1841 	}
1842 
1843 	for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1844 		maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1845 		map_fd[i] = bpf_map__fd(maps[i]);
1846 		if (map_fd[i] < 0) {
1847 			fprintf(stderr, "load_bpf_file: (%i) %s\n",
1848 				map_fd[i], strerror(errno));
1849 			return -1;
1850 		}
1851 	}
1852 
1853 	return 0;
1854 }
1855 
1856 struct _test test[] = {
1857 	{"txmsg test passthrough", test_txmsg_pass},
1858 	{"txmsg test redirect", test_txmsg_redir},
1859 	{"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1860 	{"txmsg test drop", test_txmsg_drop},
1861 	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
1862 	{"txmsg test skb", test_txmsg_skb},
1863 	{"txmsg test apply", test_txmsg_apply},
1864 	{"txmsg test cork", test_txmsg_cork},
1865 	{"txmsg test hanging corks", test_txmsg_cork_hangs},
1866 	{"txmsg test push_data", test_txmsg_push},
1867 	{"txmsg test pull-data", test_txmsg_pull},
1868 	{"txmsg test pop-data", test_txmsg_pop},
1869 	{"txmsg test push/pop data", test_txmsg_push_pop},
1870 	{"txmsg test ingress parser", test_txmsg_ingress_parser},
1871 	{"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1872 };
1873 
1874 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1875 {
1876 	char *entry, *ptr;
1877 
1878 	if (!opt->whitelist)
1879 		return 0;
1880 	ptr = strdup(opt->whitelist);
1881 	if (!ptr)
1882 		return -ENOMEM;
1883 	entry = strtok(ptr, ",");
1884 	while (entry) {
1885 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1886 		    strstr(opt->map, entry) != 0 ||
1887 		    strstr(t->title, entry) != 0)
1888 			return 0;
1889 		entry = strtok(NULL, ",");
1890 	}
1891 	return -EINVAL;
1892 }
1893 
1894 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1895 {
1896 	char *entry, *ptr;
1897 
1898 	if (!opt->blacklist)
1899 		return -EINVAL;
1900 	ptr = strdup(opt->blacklist);
1901 	if (!ptr)
1902 		return -ENOMEM;
1903 	entry = strtok(ptr, ",");
1904 	while (entry) {
1905 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1906 		    strstr(opt->map, entry) != 0 ||
1907 		    strstr(t->title, entry) != 0)
1908 			return 0;
1909 		entry = strtok(NULL, ",");
1910 	}
1911 	return -EINVAL;
1912 }
1913 
1914 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1915 {
1916 	int i, err;
1917 
1918 	err = populate_progs(opt->map);
1919 	if (err < 0) {
1920 		fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1921 		return err;
1922 	}
1923 
1924 	/* Tests basic commands and APIs */
1925 	for (i = 0; i < ARRAY_SIZE(test); i++) {
1926 		struct _test t = test[i];
1927 
1928 		if (check_whitelist(&t, opt) != 0)
1929 			continue;
1930 		if (check_blacklist(&t, opt) == 0)
1931 			continue;
1932 
1933 		test_start_subtest(&t, opt);
1934 		t.tester(cg_fd, opt);
1935 		test_end_subtest();
1936 	}
1937 
1938 	return err;
1939 }
1940 
1941 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1942 {
1943 	opt->map = BPF_SOCKMAP_FILENAME;
1944 	__test_selftests(cg_fd, opt);
1945 }
1946 
1947 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1948 {
1949 	opt->map = BPF_SOCKHASH_FILENAME;
1950 	__test_selftests(cg_fd, opt);
1951 }
1952 
1953 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1954 {
1955 	opt->map = BPF_SOCKHASH_FILENAME;
1956 	opt->prepend = "ktls";
1957 	ktls = 1;
1958 	__test_selftests(cg_fd, opt);
1959 	ktls = 0;
1960 }
1961 
1962 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1963 {
1964 
1965 	test_selftests_sockmap(cg_fd, opt);
1966 	test_selftests_sockhash(cg_fd, opt);
1967 	test_selftests_ktls(cg_fd, opt);
1968 	test_print_results();
1969 	return 0;
1970 }
1971 
1972 int main(int argc, char **argv)
1973 {
1974 	int iov_count = 1, length = 1024, rate = 1;
1975 	struct sockmap_options options = {0};
1976 	int opt, longindex, err, cg_fd = 0;
1977 	char *bpf_file = BPF_SOCKMAP_FILENAME;
1978 	int test = SELFTESTS;
1979 	bool cg_created = 0;
1980 
1981 	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1982 				  long_options, &longindex)) != -1) {
1983 		switch (opt) {
1984 		case 's':
1985 			txmsg_start = atoi(optarg);
1986 			break;
1987 		case 'e':
1988 			txmsg_end = atoi(optarg);
1989 			break;
1990 		case 'p':
1991 			txmsg_start_push = atoi(optarg);
1992 			break;
1993 		case 'q':
1994 			txmsg_end_push = atoi(optarg);
1995 			break;
1996 		case 'w':
1997 			txmsg_start_pop = atoi(optarg);
1998 			break;
1999 		case 'x':
2000 			txmsg_pop = atoi(optarg);
2001 			break;
2002 		case 'a':
2003 			txmsg_apply = atoi(optarg);
2004 			break;
2005 		case 'k':
2006 			txmsg_cork = atoi(optarg);
2007 			break;
2008 		case 'c':
2009 			cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2010 			if (cg_fd < 0) {
2011 				fprintf(stderr,
2012 					"ERROR: (%i) open cg path failed: %s\n",
2013 					cg_fd, optarg);
2014 				return cg_fd;
2015 			}
2016 			break;
2017 		case 'r':
2018 			rate = atoi(optarg);
2019 			break;
2020 		case 'v':
2021 			options.verbose = 1;
2022 			if (optarg)
2023 				options.verbose = atoi(optarg);
2024 			break;
2025 		case 'i':
2026 			iov_count = atoi(optarg);
2027 			break;
2028 		case 'l':
2029 			length = atoi(optarg);
2030 			break;
2031 		case 'd':
2032 			options.data_test = true;
2033 			break;
2034 		case 't':
2035 			if (strcmp(optarg, "ping") == 0) {
2036 				test = PING_PONG;
2037 			} else if (strcmp(optarg, "sendmsg") == 0) {
2038 				test = SENDMSG;
2039 			} else if (strcmp(optarg, "base") == 0) {
2040 				test = BASE;
2041 			} else if (strcmp(optarg, "base_sendpage") == 0) {
2042 				test = BASE_SENDPAGE;
2043 			} else if (strcmp(optarg, "sendpage") == 0) {
2044 				test = SENDPAGE;
2045 			} else {
2046 				usage(argv);
2047 				return -1;
2048 			}
2049 			break;
2050 		case 'n':
2051 			options.whitelist = strdup(optarg);
2052 			if (!options.whitelist)
2053 				return -ENOMEM;
2054 			break;
2055 		case 'b':
2056 			options.blacklist = strdup(optarg);
2057 			if (!options.blacklist)
2058 				return -ENOMEM;
2059 		case 0:
2060 			break;
2061 		case 'h':
2062 		default:
2063 			usage(argv);
2064 			return -1;
2065 		}
2066 	}
2067 
2068 	if (!cg_fd) {
2069 		cg_fd = cgroup_setup_and_join(CG_PATH);
2070 		if (cg_fd < 0)
2071 			return cg_fd;
2072 		cg_created = 1;
2073 	}
2074 
2075 	/* Use libbpf 1.0 API mode */
2076 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2077 
2078 	if (test == SELFTESTS) {
2079 		err = test_selftest(cg_fd, &options);
2080 		goto out;
2081 	}
2082 
2083 	err = populate_progs(bpf_file);
2084 	if (err) {
2085 		fprintf(stderr, "populate program: (%s) %s\n",
2086 			bpf_file, strerror(errno));
2087 		return 1;
2088 	}
2089 	running = 1;
2090 
2091 	/* catch SIGINT */
2092 	signal(SIGINT, running_handler);
2093 
2094 	options.iov_count = iov_count;
2095 	options.iov_length = length;
2096 	options.rate = rate;
2097 
2098 	err = run_options(&options, cg_fd, test);
2099 out:
2100 	if (options.whitelist)
2101 		free(options.whitelist);
2102 	if (options.blacklist)
2103 		free(options.blacklist);
2104 	close(cg_fd);
2105 	if (cg_created)
2106 		cleanup_cgroup_environment();
2107 	return err;
2108 }
2109 
2110 void running_handler(int a)
2111 {
2112 	running = 0;
2113 }
2114