1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19 
20 #include <sys/time.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
24 
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
31 #include <assert.h>
32 #include <libgen.h>
33 
34 #include <getopt.h>
35 
36 #include <bpf/bpf.h>
37 #include <bpf/libbpf.h>
38 
39 #include "bpf_util.h"
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
42 
43 int running;
44 static void running_handler(int a);
45 
46 #ifndef TCP_ULP
47 # define TCP_ULP 31
48 #endif
49 #ifndef SOL_TLS
50 # define SOL_TLS 282
51 #endif
52 
53 /* randomly selected ports for testing on lo */
54 #define S1_PORT 10000
55 #define S2_PORT 10001
56 
57 #define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
60 
61 /* global sockets */
62 int s1, s2, c1, c2, p1, p2;
63 int test_cnt;
64 int passed;
65 int failed;
66 int map_fd[9];
67 struct bpf_map *maps[9];
68 int prog_fd[11];
69 
70 int txmsg_pass;
71 int txmsg_redir;
72 int txmsg_drop;
73 int txmsg_apply;
74 int txmsg_cork;
75 int txmsg_start;
76 int txmsg_end;
77 int txmsg_start_push;
78 int txmsg_end_push;
79 int txmsg_start_pop;
80 int txmsg_pop;
81 int txmsg_ingress;
82 int txmsg_redir_skb;
83 int txmsg_ktls_skb;
84 int txmsg_ktls_skb_drop;
85 int txmsg_ktls_skb_redir;
86 int ktls;
87 int peek_flag;
88 int skb_use_parser;
89 int txmsg_omit_skb_parser;
90 
91 static const struct option long_options[] = {
92 	{"help",	no_argument,		NULL, 'h' },
93 	{"cgroup",	required_argument,	NULL, 'c' },
94 	{"rate",	required_argument,	NULL, 'r' },
95 	{"verbose",	optional_argument,	NULL, 'v' },
96 	{"iov_count",	required_argument,	NULL, 'i' },
97 	{"length",	required_argument,	NULL, 'l' },
98 	{"test",	required_argument,	NULL, 't' },
99 	{"data_test",   no_argument,		NULL, 'd' },
100 	{"txmsg",		no_argument,	&txmsg_pass,  1  },
101 	{"txmsg_redir",		no_argument,	&txmsg_redir, 1  },
102 	{"txmsg_drop",		no_argument,	&txmsg_drop, 1 },
103 	{"txmsg_apply",	required_argument,	NULL, 'a'},
104 	{"txmsg_cork",	required_argument,	NULL, 'k'},
105 	{"txmsg_start", required_argument,	NULL, 's'},
106 	{"txmsg_end",	required_argument,	NULL, 'e'},
107 	{"txmsg_start_push", required_argument,	NULL, 'p'},
108 	{"txmsg_end_push",   required_argument,	NULL, 'q'},
109 	{"txmsg_start_pop",  required_argument,	NULL, 'w'},
110 	{"txmsg_pop",	     required_argument,	NULL, 'x'},
111 	{"txmsg_ingress", no_argument,		&txmsg_ingress, 1 },
112 	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
113 	{"ktls", no_argument,			&ktls, 1 },
114 	{"peek", no_argument,			&peek_flag, 1 },
115 	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
116 	{"whitelist", required_argument,	NULL, 'n' },
117 	{"blacklist", required_argument,	NULL, 'b' },
118 	{0, 0, NULL, 0 }
119 };
120 
121 struct test_env {
122 	const char *type;
123 	const char *subtest;
124 	const char *prepend;
125 
126 	int test_num;
127 	int subtest_num;
128 
129 	int succ_cnt;
130 	int fail_cnt;
131 	int fail_last;
132 };
133 
134 struct test_env env;
135 
136 struct sockmap_options {
137 	int verbose;
138 	bool base;
139 	bool sendpage;
140 	bool data_test;
141 	bool drop_expected;
142 	int iov_count;
143 	int iov_length;
144 	int rate;
145 	char *map;
146 	char *whitelist;
147 	char *blacklist;
148 	char *prepend;
149 };
150 
151 struct _test {
152 	char *title;
153 	void (*tester)(int cg_fd, struct sockmap_options *opt);
154 };
155 
156 static void test_start(void)
157 {
158 	env.subtest_num++;
159 }
160 
161 static void test_fail(void)
162 {
163 	env.fail_cnt++;
164 }
165 
166 static void test_pass(void)
167 {
168 	env.succ_cnt++;
169 }
170 
171 static void test_reset(void)
172 {
173 	txmsg_start = txmsg_end = 0;
174 	txmsg_start_pop = txmsg_pop = 0;
175 	txmsg_start_push = txmsg_end_push = 0;
176 	txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 	txmsg_apply = txmsg_cork = 0;
178 	txmsg_ingress = txmsg_redir_skb = 0;
179 	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 	txmsg_omit_skb_parser = 0;
181 	skb_use_parser = 0;
182 }
183 
184 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185 {
186 	env.type = o->map;
187 	env.subtest = t->title;
188 	env.prepend = o->prepend;
189 	env.test_num++;
190 	env.subtest_num = 0;
191 	env.fail_last = env.fail_cnt;
192 	test_reset();
193 	return 0;
194 }
195 
196 static void test_end_subtest(void)
197 {
198 	int error = env.fail_cnt - env.fail_last;
199 	int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200 
201 	if (!error)
202 		test_pass();
203 
204 	fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 		env.test_num, env.subtest_num,
206 		!type ? "sockmap" : "sockhash",
207 		env.prepend ? : "",
208 		env.subtest, error ? "FAIL" : "OK");
209 }
210 
211 static void test_print_results(void)
212 {
213 	fprintf(stdout, "Pass: %d Fail: %d\n",
214 		env.succ_cnt, env.fail_cnt);
215 }
216 
217 static void usage(char *argv[])
218 {
219 	int i;
220 
221 	printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 	printf(" options:\n");
223 	for (i = 0; long_options[i].name != 0; i++) {
224 		printf(" --%-12s", long_options[i].name);
225 		if (long_options[i].flag != NULL)
226 			printf(" flag (internal value:%d)\n",
227 				*long_options[i].flag);
228 		else
229 			printf(" -%c\n", long_options[i].val);
230 	}
231 	printf("\n");
232 }
233 
234 char *sock_to_string(int s)
235 {
236 	if (s == c1)
237 		return "client1";
238 	else if (s == c2)
239 		return "client2";
240 	else if (s == s1)
241 		return "server1";
242 	else if (s == s2)
243 		return "server2";
244 	else if (s == p1)
245 		return "peer1";
246 	else if (s == p2)
247 		return "peer2";
248 	else
249 		return "unknown";
250 }
251 
252 static int sockmap_init_ktls(int verbose, int s)
253 {
254 	struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 		.info = {
256 			.version     = TLS_1_2_VERSION,
257 			.cipher_type = TLS_CIPHER_AES_GCM_128,
258 		},
259 	};
260 	struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 		.info = {
262 			.version     = TLS_1_2_VERSION,
263 			.cipher_type = TLS_CIPHER_AES_GCM_128,
264 		},
265 	};
266 	int so_buf = 6553500;
267 	int err;
268 
269 	err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 	if (err) {
271 		fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 		return -EINVAL;
273 	}
274 	err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 	if (err) {
276 		fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 		return -EINVAL;
278 	}
279 	err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 	if (err) {
281 		fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 		return -EINVAL;
283 	}
284 	err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 	if (err) {
286 		fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 		return -EINVAL;
288 	}
289 	err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 	if (err) {
291 		fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 		return -EINVAL;
293 	}
294 
295 	if (verbose)
296 		fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 	return 0;
298 }
299 static int sockmap_init_sockets(int verbose)
300 {
301 	int i, err, one = 1;
302 	struct sockaddr_in addr;
303 	int *fds[4] = {&s1, &s2, &c1, &c2};
304 
305 	s1 = s2 = p1 = p2 = c1 = c2 = 0;
306 
307 	/* Init sockets */
308 	for (i = 0; i < 4; i++) {
309 		*fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 		if (*fds[i] < 0) {
311 			perror("socket s1 failed()");
312 			return errno;
313 		}
314 	}
315 
316 	/* Allow reuse */
317 	for (i = 0; i < 2; i++) {
318 		err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 				 (char *)&one, sizeof(one));
320 		if (err) {
321 			perror("setsockopt failed()");
322 			return errno;
323 		}
324 	}
325 
326 	/* Non-blocking sockets */
327 	for (i = 0; i < 2; i++) {
328 		err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 		if (err < 0) {
330 			perror("ioctl s1 failed()");
331 			return errno;
332 		}
333 	}
334 
335 	/* Bind server sockets */
336 	memset(&addr, 0, sizeof(struct sockaddr_in));
337 	addr.sin_family = AF_INET;
338 	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339 
340 	addr.sin_port = htons(S1_PORT);
341 	err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 	if (err < 0) {
343 		perror("bind s1 failed()");
344 		return errno;
345 	}
346 
347 	addr.sin_port = htons(S2_PORT);
348 	err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 	if (err < 0) {
350 		perror("bind s2 failed()");
351 		return errno;
352 	}
353 
354 	/* Listen server sockets */
355 	addr.sin_port = htons(S1_PORT);
356 	err = listen(s1, 32);
357 	if (err < 0) {
358 		perror("listen s1 failed()");
359 		return errno;
360 	}
361 
362 	addr.sin_port = htons(S2_PORT);
363 	err = listen(s2, 32);
364 	if (err < 0) {
365 		perror("listen s1 failed()");
366 		return errno;
367 	}
368 
369 	/* Initiate Connect */
370 	addr.sin_port = htons(S1_PORT);
371 	err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 	if (err < 0 && errno != EINPROGRESS) {
373 		perror("connect c1 failed()");
374 		return errno;
375 	}
376 
377 	addr.sin_port = htons(S2_PORT);
378 	err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 	if (err < 0 && errno != EINPROGRESS) {
380 		perror("connect c2 failed()");
381 		return errno;
382 	} else if (err < 0) {
383 		err = 0;
384 	}
385 
386 	/* Accept Connecrtions */
387 	p1 = accept(s1, NULL, NULL);
388 	if (p1 < 0) {
389 		perror("accept s1 failed()");
390 		return errno;
391 	}
392 
393 	p2 = accept(s2, NULL, NULL);
394 	if (p2 < 0) {
395 		perror("accept s1 failed()");
396 		return errno;
397 	}
398 
399 	if (verbose > 1) {
400 		printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 		printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 			c1, s1, c2, s2);
403 	}
404 	return 0;
405 }
406 
407 struct msg_stats {
408 	size_t bytes_sent;
409 	size_t bytes_recvd;
410 	struct timespec start;
411 	struct timespec end;
412 };
413 
414 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 			     struct msg_stats *s,
416 			     struct sockmap_options *opt)
417 {
418 	bool drop = opt->drop_expected;
419 	unsigned char k = 0;
420 	FILE *file;
421 	int i, fp;
422 
423 	file = tmpfile();
424 	if (!file) {
425 		perror("create file for sendpage");
426 		return 1;
427 	}
428 	for (i = 0; i < iov_length * cnt; i++, k++)
429 		fwrite(&k, sizeof(char), 1, file);
430 	fflush(file);
431 	fseek(file, 0, SEEK_SET);
432 
433 	fp = fileno(file);
434 
435 	clock_gettime(CLOCK_MONOTONIC, &s->start);
436 	for (i = 0; i < cnt; i++) {
437 		int sent;
438 
439 		errno = 0;
440 		sent = sendfile(fd, fp, NULL, iov_length);
441 
442 		if (!drop && sent < 0) {
443 			perror("sendpage loop error");
444 			fclose(file);
445 			return sent;
446 		} else if (drop && sent >= 0) {
447 			printf("sendpage loop error expected: %i errno %i\n",
448 			       sent, errno);
449 			fclose(file);
450 			return -EIO;
451 		}
452 
453 		if (sent > 0)
454 			s->bytes_sent += sent;
455 	}
456 	clock_gettime(CLOCK_MONOTONIC, &s->end);
457 	fclose(file);
458 	return 0;
459 }
460 
461 static void msg_free_iov(struct msghdr *msg)
462 {
463 	int i;
464 
465 	for (i = 0; i < msg->msg_iovlen; i++)
466 		free(msg->msg_iov[i].iov_base);
467 	free(msg->msg_iov);
468 	msg->msg_iov = NULL;
469 	msg->msg_iovlen = 0;
470 }
471 
472 static int msg_alloc_iov(struct msghdr *msg,
473 			 int iov_count, int iov_length,
474 			 bool data, bool xmit)
475 {
476 	unsigned char k = 0;
477 	struct iovec *iov;
478 	int i;
479 
480 	iov = calloc(iov_count, sizeof(struct iovec));
481 	if (!iov)
482 		return errno;
483 
484 	for (i = 0; i < iov_count; i++) {
485 		unsigned char *d = calloc(iov_length, sizeof(char));
486 
487 		if (!d) {
488 			fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 			goto unwind_iov;
490 		}
491 		iov[i].iov_base = d;
492 		iov[i].iov_len = iov_length;
493 
494 		if (data && xmit) {
495 			int j;
496 
497 			for (j = 0; j < iov_length; j++)
498 				d[j] = k++;
499 		}
500 	}
501 
502 	msg->msg_iov = iov;
503 	msg->msg_iovlen = iov_count;
504 
505 	return 0;
506 unwind_iov:
507 	for (i--; i >= 0 ; i--)
508 		free(msg->msg_iov[i].iov_base);
509 	return -ENOMEM;
510 }
511 
512 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513 {
514 	int i, j = 0, bytes_cnt = 0;
515 	unsigned char k = 0;
516 
517 	for (i = 0; i < msg->msg_iovlen; i++) {
518 		unsigned char *d = msg->msg_iov[i].iov_base;
519 
520 		/* Special case test for skb ingress + ktls */
521 		if (i == 0 && txmsg_ktls_skb) {
522 			if (msg->msg_iov[i].iov_len < 4)
523 				return -EIO;
524 			if (memcmp(d, "PASS", 4) != 0) {
525 				fprintf(stderr,
526 					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 					i, 0, d[0], d[1], d[2], d[3]);
528 				return -EIO;
529 			}
530 			j = 4; /* advance index past PASS header */
531 		}
532 
533 		for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 			if (d[j] != k++) {
535 				fprintf(stderr,
536 					"detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 					i, j, d[j], k - 1, d[j+1], k);
538 				return -EIO;
539 			}
540 			bytes_cnt++;
541 			if (bytes_cnt == chunk_sz) {
542 				k = 0;
543 				bytes_cnt = 0;
544 			}
545 			size--;
546 		}
547 	}
548 	return 0;
549 }
550 
551 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 		    struct msg_stats *s, bool tx,
553 		    struct sockmap_options *opt)
554 {
555 	struct msghdr msg = {0}, msg_peek = {0};
556 	int err, i, flags = MSG_NOSIGNAL;
557 	bool drop = opt->drop_expected;
558 	bool data = opt->data_test;
559 
560 	err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
561 	if (err)
562 		goto out_errno;
563 	if (peek_flag) {
564 		err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
565 		if (err)
566 			goto out_errno;
567 	}
568 
569 	if (tx) {
570 		clock_gettime(CLOCK_MONOTONIC, &s->start);
571 		for (i = 0; i < cnt; i++) {
572 			int sent;
573 
574 			errno = 0;
575 			sent = sendmsg(fd, &msg, flags);
576 
577 			if (!drop && sent < 0) {
578 				perror("sendmsg loop error");
579 				goto out_errno;
580 			} else if (drop && sent >= 0) {
581 				fprintf(stderr,
582 					"sendmsg loop error expected: %i errno %i\n",
583 					sent, errno);
584 				errno = -EIO;
585 				goto out_errno;
586 			}
587 			if (sent > 0)
588 				s->bytes_sent += sent;
589 		}
590 		clock_gettime(CLOCK_MONOTONIC, &s->end);
591 	} else {
592 		int slct, recvp = 0, recv, max_fd = fd;
593 		float total_bytes, txmsg_pop_total;
594 		int fd_flags = O_NONBLOCK;
595 		struct timeval timeout;
596 		fd_set w;
597 
598 		fcntl(fd, fd_flags);
599 		/* Account for pop bytes noting each iteration of apply will
600 		 * call msg_pop_data helper so we need to account for this
601 		 * by calculating the number of apply iterations. Note user
602 		 * of the tool can create cases where no data is sent by
603 		 * manipulating pop/push/pull/etc. For example txmsg_apply 1
604 		 * with txmsg_pop 1 will try to apply 1B at a time but each
605 		 * iteration will then pop 1B so no data will ever be sent.
606 		 * This is really only useful for testing edge cases in code
607 		 * paths.
608 		 */
609 		total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
610 		if (txmsg_apply)
611 			txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
612 		else
613 			txmsg_pop_total = txmsg_pop * cnt;
614 		total_bytes -= txmsg_pop_total;
615 		err = clock_gettime(CLOCK_MONOTONIC, &s->start);
616 		if (err < 0)
617 			perror("recv start time");
618 		while (s->bytes_recvd < total_bytes) {
619 			if (txmsg_cork) {
620 				timeout.tv_sec = 0;
621 				timeout.tv_usec = 300000;
622 			} else {
623 				timeout.tv_sec = 3;
624 				timeout.tv_usec = 0;
625 			}
626 
627 			/* FD sets */
628 			FD_ZERO(&w);
629 			FD_SET(fd, &w);
630 
631 			slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
632 			if (slct == -1) {
633 				perror("select()");
634 				clock_gettime(CLOCK_MONOTONIC, &s->end);
635 				goto out_errno;
636 			} else if (!slct) {
637 				if (opt->verbose)
638 					fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
639 				errno = -EIO;
640 				clock_gettime(CLOCK_MONOTONIC, &s->end);
641 				goto out_errno;
642 			}
643 
644 			errno = 0;
645 			if (peek_flag) {
646 				flags |= MSG_PEEK;
647 				recvp = recvmsg(fd, &msg_peek, flags);
648 				if (recvp < 0) {
649 					if (errno != EWOULDBLOCK) {
650 						clock_gettime(CLOCK_MONOTONIC, &s->end);
651 						goto out_errno;
652 					}
653 				}
654 				flags = 0;
655 			}
656 
657 			recv = recvmsg(fd, &msg, flags);
658 			if (recv < 0) {
659 				if (errno != EWOULDBLOCK) {
660 					clock_gettime(CLOCK_MONOTONIC, &s->end);
661 					perror("recv failed()");
662 					goto out_errno;
663 				}
664 			}
665 
666 			s->bytes_recvd += recv;
667 
668 			if (data) {
669 				int chunk_sz = opt->sendpage ?
670 						iov_length * cnt :
671 						iov_length * iov_count;
672 
673 				errno = msg_verify_data(&msg, recv, chunk_sz);
674 				if (errno) {
675 					perror("data verify msg failed");
676 					goto out_errno;
677 				}
678 				if (recvp) {
679 					errno = msg_verify_data(&msg_peek,
680 								recvp,
681 								chunk_sz);
682 					if (errno) {
683 						perror("data verify msg_peek failed");
684 						goto out_errno;
685 					}
686 				}
687 			}
688 		}
689 		clock_gettime(CLOCK_MONOTONIC, &s->end);
690 	}
691 
692 	msg_free_iov(&msg);
693 	msg_free_iov(&msg_peek);
694 	return err;
695 out_errno:
696 	msg_free_iov(&msg);
697 	msg_free_iov(&msg_peek);
698 	return errno;
699 }
700 
701 static float giga = 1000000000;
702 
703 static inline float sentBps(struct msg_stats s)
704 {
705 	return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
706 }
707 
708 static inline float recvdBps(struct msg_stats s)
709 {
710 	return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
711 }
712 
713 static int sendmsg_test(struct sockmap_options *opt)
714 {
715 	float sent_Bps = 0, recvd_Bps = 0;
716 	int rx_fd, txpid, rxpid, err = 0;
717 	struct msg_stats s = {0};
718 	int iov_count = opt->iov_count;
719 	int iov_buf = opt->iov_length;
720 	int rx_status, tx_status;
721 	int cnt = opt->rate;
722 
723 	errno = 0;
724 
725 	if (opt->base)
726 		rx_fd = p1;
727 	else
728 		rx_fd = p2;
729 
730 	if (ktls) {
731 		/* Redirecting into non-TLS socket which sends into a TLS
732 		 * socket is not a valid test. So in this case lets not
733 		 * enable kTLS but still run the test.
734 		 */
735 		if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
736 			err = sockmap_init_ktls(opt->verbose, rx_fd);
737 			if (err)
738 				return err;
739 		}
740 		err = sockmap_init_ktls(opt->verbose, c1);
741 		if (err)
742 			return err;
743 	}
744 
745 	rxpid = fork();
746 	if (rxpid == 0) {
747 		iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
748 		if (opt->drop_expected || txmsg_ktls_skb_drop)
749 			_exit(0);
750 
751 		if (!iov_buf) /* zero bytes sent case */
752 			_exit(0);
753 
754 		if (opt->sendpage)
755 			iov_count = 1;
756 		err = msg_loop(rx_fd, iov_count, iov_buf,
757 			       cnt, &s, false, opt);
758 		if (opt->verbose > 1)
759 			fprintf(stderr,
760 				"msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
761 				iov_count, iov_buf, cnt, err);
762 		if (s.end.tv_sec - s.start.tv_sec) {
763 			sent_Bps = sentBps(s);
764 			recvd_Bps = recvdBps(s);
765 		}
766 		if (opt->verbose > 1)
767 			fprintf(stdout,
768 				"rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
769 				s.bytes_sent, sent_Bps, sent_Bps/giga,
770 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
771 				peek_flag ? "(peek_msg)" : "");
772 		if (err && txmsg_cork)
773 			err = 0;
774 		exit(err ? 1 : 0);
775 	} else if (rxpid == -1) {
776 		perror("msg_loop_rx");
777 		return errno;
778 	}
779 
780 	txpid = fork();
781 	if (txpid == 0) {
782 		if (opt->sendpage)
783 			err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
784 		else
785 			err = msg_loop(c1, iov_count, iov_buf,
786 				       cnt, &s, true, opt);
787 
788 		if (err)
789 			fprintf(stderr,
790 				"msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
791 				iov_count, iov_buf, cnt, err);
792 		if (s.end.tv_sec - s.start.tv_sec) {
793 			sent_Bps = sentBps(s);
794 			recvd_Bps = recvdBps(s);
795 		}
796 		if (opt->verbose > 1)
797 			fprintf(stdout,
798 				"tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
799 				s.bytes_sent, sent_Bps, sent_Bps/giga,
800 				s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
801 		exit(err ? 1 : 0);
802 	} else if (txpid == -1) {
803 		perror("msg_loop_tx");
804 		return errno;
805 	}
806 
807 	assert(waitpid(rxpid, &rx_status, 0) == rxpid);
808 	assert(waitpid(txpid, &tx_status, 0) == txpid);
809 	if (WIFEXITED(rx_status)) {
810 		err = WEXITSTATUS(rx_status);
811 		if (err) {
812 			fprintf(stderr, "rx thread exited with err %d.\n", err);
813 			goto out;
814 		}
815 	}
816 	if (WIFEXITED(tx_status)) {
817 		err = WEXITSTATUS(tx_status);
818 		if (err)
819 			fprintf(stderr, "tx thread exited with err %d.\n", err);
820 	}
821 out:
822 	return err;
823 }
824 
825 static int forever_ping_pong(int rate, struct sockmap_options *opt)
826 {
827 	struct timeval timeout;
828 	char buf[1024] = {0};
829 	int sc;
830 
831 	timeout.tv_sec = 10;
832 	timeout.tv_usec = 0;
833 
834 	/* Ping/Pong data from client to server */
835 	sc = send(c1, buf, sizeof(buf), 0);
836 	if (sc < 0) {
837 		perror("send failed()");
838 		return sc;
839 	}
840 
841 	do {
842 		int s, rc, i, max_fd = p2;
843 		fd_set w;
844 
845 		/* FD sets */
846 		FD_ZERO(&w);
847 		FD_SET(c1, &w);
848 		FD_SET(c2, &w);
849 		FD_SET(p1, &w);
850 		FD_SET(p2, &w);
851 
852 		s = select(max_fd + 1, &w, NULL, NULL, &timeout);
853 		if (s == -1) {
854 			perror("select()");
855 			break;
856 		} else if (!s) {
857 			fprintf(stderr, "unexpected timeout\n");
858 			break;
859 		}
860 
861 		for (i = 0; i <= max_fd && s > 0; ++i) {
862 			if (!FD_ISSET(i, &w))
863 				continue;
864 
865 			s--;
866 
867 			rc = recv(i, buf, sizeof(buf), 0);
868 			if (rc < 0) {
869 				if (errno != EWOULDBLOCK) {
870 					perror("recv failed()");
871 					return rc;
872 				}
873 			}
874 
875 			if (rc == 0) {
876 				close(i);
877 				break;
878 			}
879 
880 			sc = send(i, buf, rc, 0);
881 			if (sc < 0) {
882 				perror("send failed()");
883 				return sc;
884 			}
885 		}
886 
887 		if (rate)
888 			sleep(rate);
889 
890 		if (opt->verbose) {
891 			printf(".");
892 			fflush(stdout);
893 
894 		}
895 	} while (running);
896 
897 	return 0;
898 }
899 
900 enum {
901 	SELFTESTS,
902 	PING_PONG,
903 	SENDMSG,
904 	BASE,
905 	BASE_SENDPAGE,
906 	SENDPAGE,
907 };
908 
909 static int run_options(struct sockmap_options *options, int cg_fd,  int test)
910 {
911 	int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
912 
913 	/* If base test skip BPF setup */
914 	if (test == BASE || test == BASE_SENDPAGE)
915 		goto run;
916 
917 	/* Attach programs to sockmap */
918 	if (!txmsg_omit_skb_parser) {
919 		err = bpf_prog_attach(prog_fd[0], map_fd[0],
920 				      BPF_SK_SKB_STREAM_PARSER, 0);
921 		if (err) {
922 			fprintf(stderr,
923 				"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
924 				prog_fd[0], map_fd[0], err, strerror(errno));
925 			return err;
926 		}
927 	}
928 
929 	err = bpf_prog_attach(prog_fd[1], map_fd[0],
930 				BPF_SK_SKB_STREAM_VERDICT, 0);
931 	if (err) {
932 		fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
933 			err, strerror(errno));
934 		return err;
935 	}
936 
937 	/* Attach programs to TLS sockmap */
938 	if (txmsg_ktls_skb) {
939 		if (!txmsg_omit_skb_parser) {
940 			err = bpf_prog_attach(prog_fd[0], map_fd[8],
941 					      BPF_SK_SKB_STREAM_PARSER, 0);
942 			if (err) {
943 				fprintf(stderr,
944 					"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
945 					prog_fd[0], map_fd[8], err, strerror(errno));
946 				return err;
947 			}
948 		}
949 
950 		err = bpf_prog_attach(prog_fd[2], map_fd[8],
951 				      BPF_SK_SKB_STREAM_VERDICT, 0);
952 		if (err) {
953 			fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
954 				err, strerror(errno));
955 			return err;
956 		}
957 	}
958 
959 	/* Attach to cgroups */
960 	err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
961 	if (err) {
962 		fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
963 			err, strerror(errno));
964 		return err;
965 	}
966 
967 run:
968 	err = sockmap_init_sockets(options->verbose);
969 	if (err) {
970 		fprintf(stderr, "ERROR: test socket failed: %d\n", err);
971 		goto out;
972 	}
973 
974 	/* Attach txmsg program to sockmap */
975 	if (txmsg_pass)
976 		tx_prog_fd = prog_fd[4];
977 	else if (txmsg_redir)
978 		tx_prog_fd = prog_fd[5];
979 	else if (txmsg_apply)
980 		tx_prog_fd = prog_fd[6];
981 	else if (txmsg_cork)
982 		tx_prog_fd = prog_fd[7];
983 	else if (txmsg_drop)
984 		tx_prog_fd = prog_fd[8];
985 	else
986 		tx_prog_fd = 0;
987 
988 	if (tx_prog_fd) {
989 		int redir_fd, i = 0;
990 
991 		err = bpf_prog_attach(tx_prog_fd,
992 				      map_fd[1], BPF_SK_MSG_VERDICT, 0);
993 		if (err) {
994 			fprintf(stderr,
995 				"ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
996 				err, strerror(errno));
997 			goto out;
998 		}
999 
1000 		err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1001 		if (err) {
1002 			fprintf(stderr,
1003 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1004 				err, strerror(errno));
1005 			goto out;
1006 		}
1007 
1008 		if (txmsg_redir)
1009 			redir_fd = c2;
1010 		else
1011 			redir_fd = c1;
1012 
1013 		err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1014 		if (err) {
1015 			fprintf(stderr,
1016 				"ERROR: bpf_map_update_elem (txmsg):  %d (%s\n",
1017 				err, strerror(errno));
1018 			goto out;
1019 		}
1020 
1021 		if (txmsg_apply) {
1022 			err = bpf_map_update_elem(map_fd[3],
1023 						  &i, &txmsg_apply, BPF_ANY);
1024 			if (err) {
1025 				fprintf(stderr,
1026 					"ERROR: bpf_map_update_elem (apply_bytes):  %d (%s\n",
1027 					err, strerror(errno));
1028 				goto out;
1029 			}
1030 		}
1031 
1032 		if (txmsg_cork) {
1033 			err = bpf_map_update_elem(map_fd[4],
1034 						  &i, &txmsg_cork, BPF_ANY);
1035 			if (err) {
1036 				fprintf(stderr,
1037 					"ERROR: bpf_map_update_elem (cork_bytes):  %d (%s\n",
1038 					err, strerror(errno));
1039 				goto out;
1040 			}
1041 		}
1042 
1043 		if (txmsg_start) {
1044 			err = bpf_map_update_elem(map_fd[5],
1045 						  &i, &txmsg_start, BPF_ANY);
1046 			if (err) {
1047 				fprintf(stderr,
1048 					"ERROR: bpf_map_update_elem (txmsg_start):  %d (%s)\n",
1049 					err, strerror(errno));
1050 				goto out;
1051 			}
1052 		}
1053 
1054 		if (txmsg_end) {
1055 			i = 1;
1056 			err = bpf_map_update_elem(map_fd[5],
1057 						  &i, &txmsg_end, BPF_ANY);
1058 			if (err) {
1059 				fprintf(stderr,
1060 					"ERROR: bpf_map_update_elem (txmsg_end):  %d (%s)\n",
1061 					err, strerror(errno));
1062 				goto out;
1063 			}
1064 		}
1065 
1066 		if (txmsg_start_push) {
1067 			i = 2;
1068 			err = bpf_map_update_elem(map_fd[5],
1069 						  &i, &txmsg_start_push, BPF_ANY);
1070 			if (err) {
1071 				fprintf(stderr,
1072 					"ERROR: bpf_map_update_elem (txmsg_start_push):  %d (%s)\n",
1073 					err, strerror(errno));
1074 				goto out;
1075 			}
1076 		}
1077 
1078 		if (txmsg_end_push) {
1079 			i = 3;
1080 			err = bpf_map_update_elem(map_fd[5],
1081 						  &i, &txmsg_end_push, BPF_ANY);
1082 			if (err) {
1083 				fprintf(stderr,
1084 					"ERROR: bpf_map_update_elem %i@%i (txmsg_end_push):  %d (%s)\n",
1085 					txmsg_end_push, i, err, strerror(errno));
1086 				goto out;
1087 			}
1088 		}
1089 
1090 		if (txmsg_start_pop) {
1091 			i = 4;
1092 			err = bpf_map_update_elem(map_fd[5],
1093 						  &i, &txmsg_start_pop, BPF_ANY);
1094 			if (err) {
1095 				fprintf(stderr,
1096 					"ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop):  %d (%s)\n",
1097 					txmsg_start_pop, i, err, strerror(errno));
1098 				goto out;
1099 			}
1100 		} else {
1101 			i = 4;
1102 			bpf_map_update_elem(map_fd[5],
1103 						  &i, &txmsg_start_pop, BPF_ANY);
1104 		}
1105 
1106 		if (txmsg_pop) {
1107 			i = 5;
1108 			err = bpf_map_update_elem(map_fd[5],
1109 						  &i, &txmsg_pop, BPF_ANY);
1110 			if (err) {
1111 				fprintf(stderr,
1112 					"ERROR: bpf_map_update_elem %i@%i (txmsg_pop):  %d (%s)\n",
1113 					txmsg_pop, i, err, strerror(errno));
1114 				goto out;
1115 			}
1116 		} else {
1117 			i = 5;
1118 			bpf_map_update_elem(map_fd[5],
1119 					    &i, &txmsg_pop, BPF_ANY);
1120 
1121 		}
1122 
1123 		if (txmsg_ingress) {
1124 			int in = BPF_F_INGRESS;
1125 
1126 			i = 0;
1127 			err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1128 			if (err) {
1129 				fprintf(stderr,
1130 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1131 					err, strerror(errno));
1132 			}
1133 			i = 1;
1134 			err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1135 			if (err) {
1136 				fprintf(stderr,
1137 					"ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1138 					err, strerror(errno));
1139 			}
1140 			err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1141 			if (err) {
1142 				fprintf(stderr,
1143 					"ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1144 					err, strerror(errno));
1145 			}
1146 
1147 			i = 2;
1148 			err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1149 			if (err) {
1150 				fprintf(stderr,
1151 					"ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1152 					err, strerror(errno));
1153 			}
1154 		}
1155 
1156 		if (txmsg_ktls_skb) {
1157 			int ingress = BPF_F_INGRESS;
1158 
1159 			i = 0;
1160 			err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1161 			if (err) {
1162 				fprintf(stderr,
1163 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1164 					err, strerror(errno));
1165 			}
1166 
1167 			if (txmsg_ktls_skb_redir) {
1168 				i = 1;
1169 				err = bpf_map_update_elem(map_fd[7],
1170 							  &i, &ingress, BPF_ANY);
1171 				if (err) {
1172 					fprintf(stderr,
1173 						"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1174 						err, strerror(errno));
1175 				}
1176 			}
1177 
1178 			if (txmsg_ktls_skb_drop) {
1179 				i = 1;
1180 				err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1181 			}
1182 		}
1183 
1184 		if (txmsg_redir_skb) {
1185 			int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1186 					p2 : p1;
1187 			int ingress = BPF_F_INGRESS;
1188 
1189 			i = 0;
1190 			err = bpf_map_update_elem(map_fd[7],
1191 						  &i, &ingress, BPF_ANY);
1192 			if (err) {
1193 				fprintf(stderr,
1194 					"ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1195 					err, strerror(errno));
1196 			}
1197 
1198 			i = 3;
1199 			err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1200 			if (err) {
1201 				fprintf(stderr,
1202 					"ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1203 					err, strerror(errno));
1204 			}
1205 		}
1206 	}
1207 
1208 	if (skb_use_parser) {
1209 		i = 2;
1210 		err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1211 	}
1212 
1213 	if (txmsg_drop)
1214 		options->drop_expected = true;
1215 
1216 	if (test == PING_PONG)
1217 		err = forever_ping_pong(options->rate, options);
1218 	else if (test == SENDMSG) {
1219 		options->base = false;
1220 		options->sendpage = false;
1221 		err = sendmsg_test(options);
1222 	} else if (test == SENDPAGE) {
1223 		options->base = false;
1224 		options->sendpage = true;
1225 		err = sendmsg_test(options);
1226 	} else if (test == BASE) {
1227 		options->base = true;
1228 		options->sendpage = false;
1229 		err = sendmsg_test(options);
1230 	} else if (test == BASE_SENDPAGE) {
1231 		options->base = true;
1232 		options->sendpage = true;
1233 		err = sendmsg_test(options);
1234 	} else
1235 		fprintf(stderr, "unknown test\n");
1236 out:
1237 	/* Detatch and zero all the maps */
1238 	bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1239 	bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1240 	bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1241 	bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1242 	bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1243 
1244 	if (tx_prog_fd >= 0)
1245 		bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1246 
1247 	for (i = 0; i < 8; i++) {
1248 		key = next_key = 0;
1249 		bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1250 		while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1251 			bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1252 			key = next_key;
1253 		}
1254 	}
1255 
1256 	close(s1);
1257 	close(s2);
1258 	close(p1);
1259 	close(p2);
1260 	close(c1);
1261 	close(c2);
1262 	return err;
1263 }
1264 
1265 static char *test_to_str(int test)
1266 {
1267 	switch (test) {
1268 	case SENDMSG:
1269 		return "sendmsg";
1270 	case SENDPAGE:
1271 		return "sendpage";
1272 	}
1273 	return "unknown";
1274 }
1275 
1276 #define OPTSTRING 60
1277 static void test_options(char *options)
1278 {
1279 	char tstr[OPTSTRING];
1280 
1281 	memset(options, 0, OPTSTRING);
1282 
1283 	if (txmsg_pass)
1284 		strncat(options, "pass,", OPTSTRING);
1285 	if (txmsg_redir)
1286 		strncat(options, "redir,", OPTSTRING);
1287 	if (txmsg_drop)
1288 		strncat(options, "drop,", OPTSTRING);
1289 	if (txmsg_apply) {
1290 		snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1291 		strncat(options, tstr, OPTSTRING);
1292 	}
1293 	if (txmsg_cork) {
1294 		snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1295 		strncat(options, tstr, OPTSTRING);
1296 	}
1297 	if (txmsg_start) {
1298 		snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1299 		strncat(options, tstr, OPTSTRING);
1300 	}
1301 	if (txmsg_end) {
1302 		snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1303 		strncat(options, tstr, OPTSTRING);
1304 	}
1305 	if (txmsg_start_pop) {
1306 		snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1307 			 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1308 		strncat(options, tstr, OPTSTRING);
1309 	}
1310 	if (txmsg_ingress)
1311 		strncat(options, "ingress,", OPTSTRING);
1312 	if (txmsg_redir_skb)
1313 		strncat(options, "redir_skb,", OPTSTRING);
1314 	if (txmsg_ktls_skb)
1315 		strncat(options, "ktls_skb,", OPTSTRING);
1316 	if (ktls)
1317 		strncat(options, "ktls,", OPTSTRING);
1318 	if (peek_flag)
1319 		strncat(options, "peek,", OPTSTRING);
1320 }
1321 
1322 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1323 {
1324 	char *options = calloc(OPTSTRING, sizeof(char));
1325 	int err;
1326 
1327 	if (test == SENDPAGE)
1328 		opt->sendpage = true;
1329 	else
1330 		opt->sendpage = false;
1331 
1332 	if (txmsg_drop)
1333 		opt->drop_expected = true;
1334 	else
1335 		opt->drop_expected = false;
1336 
1337 	test_options(options);
1338 
1339 	if (opt->verbose) {
1340 		fprintf(stdout,
1341 			" [TEST %i]: (%i, %i, %i, %s, %s): ",
1342 			test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1343 			test_to_str(test), options);
1344 		fflush(stdout);
1345 	}
1346 	err = run_options(opt, cgrp, test);
1347 	if (opt->verbose)
1348 		fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1349 	test_cnt++;
1350 	!err ? passed++ : failed++;
1351 	free(options);
1352 	return err;
1353 }
1354 
1355 static void test_exec(int cgrp, struct sockmap_options *opt)
1356 {
1357 	int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1358 	int err;
1359 
1360 	if (type == 0) {
1361 		test_start();
1362 		err = __test_exec(cgrp, SENDMSG, opt);
1363 		if (err)
1364 			test_fail();
1365 	} else {
1366 		test_start();
1367 		err = __test_exec(cgrp, SENDPAGE, opt);
1368 		if (err)
1369 			test_fail();
1370 	}
1371 }
1372 
1373 static void test_send_one(struct sockmap_options *opt, int cgrp)
1374 {
1375 	opt->iov_length = 1;
1376 	opt->iov_count = 1;
1377 	opt->rate = 1;
1378 	test_exec(cgrp, opt);
1379 
1380 	opt->iov_length = 1;
1381 	opt->iov_count = 1024;
1382 	opt->rate = 1;
1383 	test_exec(cgrp, opt);
1384 
1385 	opt->iov_length = 1024;
1386 	opt->iov_count = 1;
1387 	opt->rate = 1;
1388 	test_exec(cgrp, opt);
1389 
1390 }
1391 
1392 static void test_send_many(struct sockmap_options *opt, int cgrp)
1393 {
1394 	opt->iov_length = 3;
1395 	opt->iov_count = 1;
1396 	opt->rate = 512;
1397 	test_exec(cgrp, opt);
1398 
1399 	opt->rate = 100;
1400 	opt->iov_count = 1;
1401 	opt->iov_length = 5;
1402 	test_exec(cgrp, opt);
1403 }
1404 
1405 static void test_send_large(struct sockmap_options *opt, int cgrp)
1406 {
1407 	opt->iov_length = 256;
1408 	opt->iov_count = 1024;
1409 	opt->rate = 2;
1410 	test_exec(cgrp, opt);
1411 }
1412 
1413 static void test_send(struct sockmap_options *opt, int cgrp)
1414 {
1415 	test_send_one(opt, cgrp);
1416 	test_send_many(opt, cgrp);
1417 	test_send_large(opt, cgrp);
1418 	sched_yield();
1419 }
1420 
1421 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1422 {
1423 	/* Test small and large iov_count values with pass/redir/apply/cork */
1424 	txmsg_pass = 1;
1425 	test_send(opt, cgrp);
1426 }
1427 
1428 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1429 {
1430 	txmsg_redir = 1;
1431 	test_send(opt, cgrp);
1432 }
1433 
1434 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1435 {
1436 	txmsg_drop = 1;
1437 	test_send(opt, cgrp);
1438 }
1439 
1440 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1441 {
1442 	txmsg_pass = txmsg_drop = 0;
1443 	txmsg_ingress = txmsg_redir = 1;
1444 	test_send(opt, cgrp);
1445 }
1446 
1447 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1448 {
1449 	bool data = opt->data_test;
1450 	int k = ktls;
1451 
1452 	opt->data_test = true;
1453 	ktls = 1;
1454 
1455 	txmsg_pass = txmsg_drop = 0;
1456 	txmsg_ingress = txmsg_redir = 0;
1457 	txmsg_ktls_skb = 1;
1458 	txmsg_pass = 1;
1459 
1460 	/* Using data verification so ensure iov layout is
1461 	 * expected from test receiver side. e.g. has enough
1462 	 * bytes to write test code.
1463 	 */
1464 	opt->iov_length = 100;
1465 	opt->iov_count = 1;
1466 	opt->rate = 1;
1467 	test_exec(cgrp, opt);
1468 
1469 	txmsg_ktls_skb_drop = 1;
1470 	test_exec(cgrp, opt);
1471 
1472 	txmsg_ktls_skb_drop = 0;
1473 	txmsg_ktls_skb_redir = 1;
1474 	test_exec(cgrp, opt);
1475 	txmsg_ktls_skb_redir = 0;
1476 
1477 	/* Tests that omit skb_parser */
1478 	txmsg_omit_skb_parser = 1;
1479 	ktls = 0;
1480 	txmsg_ktls_skb = 0;
1481 	test_exec(cgrp, opt);
1482 
1483 	txmsg_ktls_skb_drop = 1;
1484 	test_exec(cgrp, opt);
1485 	txmsg_ktls_skb_drop = 0;
1486 
1487 	txmsg_ktls_skb_redir = 1;
1488 	test_exec(cgrp, opt);
1489 
1490 	ktls = 1;
1491 	test_exec(cgrp, opt);
1492 	txmsg_omit_skb_parser = 0;
1493 
1494 	opt->data_test = data;
1495 	ktls = k;
1496 }
1497 
1498 /* Test cork with hung data. This tests poor usage patterns where
1499  * cork can leave data on the ring if user program is buggy and
1500  * doesn't flush them somehow. They do take some time however
1501  * because they wait for a timeout. Test pass, redir and cork with
1502  * apply logic. Use cork size of 4097 with send_large to avoid
1503  * aligning cork size with send size.
1504  */
1505 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1506 {
1507 	txmsg_pass = 1;
1508 	txmsg_redir = 0;
1509 	txmsg_cork = 4097;
1510 	txmsg_apply = 4097;
1511 	test_send_large(opt, cgrp);
1512 
1513 	txmsg_pass = 0;
1514 	txmsg_redir = 1;
1515 	txmsg_apply = 0;
1516 	txmsg_cork = 4097;
1517 	test_send_large(opt, cgrp);
1518 
1519 	txmsg_pass = 0;
1520 	txmsg_redir = 1;
1521 	txmsg_apply = 4097;
1522 	txmsg_cork = 4097;
1523 	test_send_large(opt, cgrp);
1524 }
1525 
1526 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1527 {
1528 	/* Test basic start/end */
1529 	txmsg_start = 1;
1530 	txmsg_end = 2;
1531 	test_send(opt, cgrp);
1532 
1533 	/* Test >4k pull */
1534 	txmsg_start = 4096;
1535 	txmsg_end = 9182;
1536 	test_send_large(opt, cgrp);
1537 
1538 	/* Test pull + redirect */
1539 	txmsg_redir = 0;
1540 	txmsg_start = 1;
1541 	txmsg_end = 2;
1542 	test_send(opt, cgrp);
1543 
1544 	/* Test pull + cork */
1545 	txmsg_redir = 0;
1546 	txmsg_cork = 512;
1547 	txmsg_start = 1;
1548 	txmsg_end = 2;
1549 	test_send_many(opt, cgrp);
1550 
1551 	/* Test pull + cork + redirect */
1552 	txmsg_redir = 1;
1553 	txmsg_cork = 512;
1554 	txmsg_start = 1;
1555 	txmsg_end = 2;
1556 	test_send_many(opt, cgrp);
1557 }
1558 
1559 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1560 {
1561 	/* Test basic pop */
1562 	txmsg_start_pop = 1;
1563 	txmsg_pop = 2;
1564 	test_send_many(opt, cgrp);
1565 
1566 	/* Test pop with >4k */
1567 	txmsg_start_pop = 4096;
1568 	txmsg_pop = 4096;
1569 	test_send_large(opt, cgrp);
1570 
1571 	/* Test pop + redirect */
1572 	txmsg_redir = 1;
1573 	txmsg_start_pop = 1;
1574 	txmsg_pop = 2;
1575 	test_send_many(opt, cgrp);
1576 
1577 	/* Test pop + cork */
1578 	txmsg_redir = 0;
1579 	txmsg_cork = 512;
1580 	txmsg_start_pop = 1;
1581 	txmsg_pop = 2;
1582 	test_send_many(opt, cgrp);
1583 
1584 	/* Test pop + redirect + cork */
1585 	txmsg_redir = 1;
1586 	txmsg_cork = 4;
1587 	txmsg_start_pop = 1;
1588 	txmsg_pop = 2;
1589 	test_send_many(opt, cgrp);
1590 }
1591 
1592 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1593 {
1594 	/* Test basic push */
1595 	txmsg_start_push = 1;
1596 	txmsg_end_push = 1;
1597 	test_send(opt, cgrp);
1598 
1599 	/* Test push 4kB >4k */
1600 	txmsg_start_push = 4096;
1601 	txmsg_end_push = 4096;
1602 	test_send_large(opt, cgrp);
1603 
1604 	/* Test push + redirect */
1605 	txmsg_redir = 1;
1606 	txmsg_start_push = 1;
1607 	txmsg_end_push = 2;
1608 	test_send_many(opt, cgrp);
1609 
1610 	/* Test push + cork */
1611 	txmsg_redir = 0;
1612 	txmsg_cork = 512;
1613 	txmsg_start_push = 1;
1614 	txmsg_end_push = 2;
1615 	test_send_many(opt, cgrp);
1616 }
1617 
1618 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1619 {
1620 	txmsg_start_push = 1;
1621 	txmsg_end_push = 10;
1622 	txmsg_start_pop = 5;
1623 	txmsg_pop = 4;
1624 	test_send_large(opt, cgrp);
1625 }
1626 
1627 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1628 {
1629 	txmsg_pass = 1;
1630 	txmsg_redir = 0;
1631 	txmsg_apply = 1;
1632 	txmsg_cork = 0;
1633 	test_send_one(opt, cgrp);
1634 
1635 	txmsg_pass = 0;
1636 	txmsg_redir = 1;
1637 	txmsg_apply = 1;
1638 	txmsg_cork = 0;
1639 	test_send_one(opt, cgrp);
1640 
1641 	txmsg_pass = 1;
1642 	txmsg_redir = 0;
1643 	txmsg_apply = 1024;
1644 	txmsg_cork = 0;
1645 	test_send_large(opt, cgrp);
1646 
1647 	txmsg_pass = 0;
1648 	txmsg_redir = 1;
1649 	txmsg_apply = 1024;
1650 	txmsg_cork = 0;
1651 	test_send_large(opt, cgrp);
1652 }
1653 
1654 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1655 {
1656 	txmsg_pass = 1;
1657 	txmsg_redir = 0;
1658 	txmsg_apply = 0;
1659 	txmsg_cork = 1;
1660 	test_send(opt, cgrp);
1661 
1662 	txmsg_pass = 1;
1663 	txmsg_redir = 0;
1664 	txmsg_apply = 1;
1665 	txmsg_cork = 1;
1666 	test_send(opt, cgrp);
1667 }
1668 
1669 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1670 {
1671 	txmsg_pass = 1;
1672 	skb_use_parser = 512;
1673 	opt->iov_length = 256;
1674 	opt->iov_count = 1;
1675 	opt->rate = 2;
1676 	test_exec(cgrp, opt);
1677 }
1678 
1679 char *map_names[] = {
1680 	"sock_map",
1681 	"sock_map_txmsg",
1682 	"sock_map_redir",
1683 	"sock_apply_bytes",
1684 	"sock_cork_bytes",
1685 	"sock_bytes",
1686 	"sock_redir_flags",
1687 	"sock_skb_opts",
1688 	"tls_sock_map",
1689 };
1690 
1691 int prog_attach_type[] = {
1692 	BPF_SK_SKB_STREAM_PARSER,
1693 	BPF_SK_SKB_STREAM_VERDICT,
1694 	BPF_SK_SKB_STREAM_VERDICT,
1695 	BPF_CGROUP_SOCK_OPS,
1696 	BPF_SK_MSG_VERDICT,
1697 	BPF_SK_MSG_VERDICT,
1698 	BPF_SK_MSG_VERDICT,
1699 	BPF_SK_MSG_VERDICT,
1700 	BPF_SK_MSG_VERDICT,
1701 	BPF_SK_MSG_VERDICT,
1702 	BPF_SK_MSG_VERDICT,
1703 };
1704 
1705 int prog_type[] = {
1706 	BPF_PROG_TYPE_SK_SKB,
1707 	BPF_PROG_TYPE_SK_SKB,
1708 	BPF_PROG_TYPE_SK_SKB,
1709 	BPF_PROG_TYPE_SOCK_OPS,
1710 	BPF_PROG_TYPE_SK_MSG,
1711 	BPF_PROG_TYPE_SK_MSG,
1712 	BPF_PROG_TYPE_SK_MSG,
1713 	BPF_PROG_TYPE_SK_MSG,
1714 	BPF_PROG_TYPE_SK_MSG,
1715 	BPF_PROG_TYPE_SK_MSG,
1716 	BPF_PROG_TYPE_SK_MSG,
1717 };
1718 
1719 static int populate_progs(char *bpf_file)
1720 {
1721 	struct bpf_program *prog;
1722 	struct bpf_object *obj;
1723 	int i = 0;
1724 	long err;
1725 
1726 	obj = bpf_object__open(bpf_file);
1727 	err = libbpf_get_error(obj);
1728 	if (err) {
1729 		char err_buf[256];
1730 
1731 		libbpf_strerror(err, err_buf, sizeof(err_buf));
1732 		printf("Unable to load eBPF objects in file '%s' : %s\n",
1733 		       bpf_file, err_buf);
1734 		return -1;
1735 	}
1736 
1737 	bpf_object__for_each_program(prog, obj) {
1738 		bpf_program__set_type(prog, prog_type[i]);
1739 		bpf_program__set_expected_attach_type(prog,
1740 						      prog_attach_type[i]);
1741 		i++;
1742 	}
1743 
1744 	i = bpf_object__load(obj);
1745 	i = 0;
1746 	bpf_object__for_each_program(prog, obj) {
1747 		prog_fd[i] = bpf_program__fd(prog);
1748 		i++;
1749 	}
1750 
1751 	for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1752 		maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1753 		map_fd[i] = bpf_map__fd(maps[i]);
1754 		if (map_fd[i] < 0) {
1755 			fprintf(stderr, "load_bpf_file: (%i) %s\n",
1756 				map_fd[i], strerror(errno));
1757 			return -1;
1758 		}
1759 	}
1760 
1761 	return 0;
1762 }
1763 
1764 struct _test test[] = {
1765 	{"txmsg test passthrough", test_txmsg_pass},
1766 	{"txmsg test redirect", test_txmsg_redir},
1767 	{"txmsg test drop", test_txmsg_drop},
1768 	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
1769 	{"txmsg test skb", test_txmsg_skb},
1770 	{"txmsg test apply", test_txmsg_apply},
1771 	{"txmsg test cork", test_txmsg_cork},
1772 	{"txmsg test hanging corks", test_txmsg_cork_hangs},
1773 	{"txmsg test push_data", test_txmsg_push},
1774 	{"txmsg test pull-data", test_txmsg_pull},
1775 	{"txmsg test pop-data", test_txmsg_pop},
1776 	{"txmsg test push/pop data", test_txmsg_push_pop},
1777 	{"txmsg text ingress parser", test_txmsg_ingress_parser},
1778 };
1779 
1780 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1781 {
1782 	char *entry, *ptr;
1783 
1784 	if (!opt->whitelist)
1785 		return 0;
1786 	ptr = strdup(opt->whitelist);
1787 	if (!ptr)
1788 		return -ENOMEM;
1789 	entry = strtok(ptr, ",");
1790 	while (entry) {
1791 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1792 		    strstr(opt->map, entry) != 0 ||
1793 		    strstr(t->title, entry) != 0)
1794 			return 0;
1795 		entry = strtok(NULL, ",");
1796 	}
1797 	return -EINVAL;
1798 }
1799 
1800 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1801 {
1802 	char *entry, *ptr;
1803 
1804 	if (!opt->blacklist)
1805 		return -EINVAL;
1806 	ptr = strdup(opt->blacklist);
1807 	if (!ptr)
1808 		return -ENOMEM;
1809 	entry = strtok(ptr, ",");
1810 	while (entry) {
1811 		if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1812 		    strstr(opt->map, entry) != 0 ||
1813 		    strstr(t->title, entry) != 0)
1814 			return 0;
1815 		entry = strtok(NULL, ",");
1816 	}
1817 	return -EINVAL;
1818 }
1819 
1820 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1821 {
1822 	int i, err;
1823 
1824 	err = populate_progs(opt->map);
1825 	if (err < 0) {
1826 		fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1827 		return err;
1828 	}
1829 
1830 	/* Tests basic commands and APIs */
1831 	for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
1832 		struct _test t = test[i];
1833 
1834 		if (check_whitelist(&t, opt) != 0)
1835 			continue;
1836 		if (check_blacklist(&t, opt) == 0)
1837 			continue;
1838 
1839 		test_start_subtest(&t, opt);
1840 		t.tester(cg_fd, opt);
1841 		test_end_subtest();
1842 	}
1843 
1844 	return err;
1845 }
1846 
1847 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1848 {
1849 	opt->map = BPF_SOCKMAP_FILENAME;
1850 	__test_selftests(cg_fd, opt);
1851 }
1852 
1853 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1854 {
1855 	opt->map = BPF_SOCKHASH_FILENAME;
1856 	__test_selftests(cg_fd, opt);
1857 }
1858 
1859 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1860 {
1861 	opt->map = BPF_SOCKHASH_FILENAME;
1862 	opt->prepend = "ktls";
1863 	ktls = 1;
1864 	__test_selftests(cg_fd, opt);
1865 	ktls = 0;
1866 }
1867 
1868 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1869 {
1870 
1871 	test_selftests_sockmap(cg_fd, opt);
1872 	test_selftests_sockhash(cg_fd, opt);
1873 	test_selftests_ktls(cg_fd, opt);
1874 	test_print_results();
1875 	return 0;
1876 }
1877 
1878 int main(int argc, char **argv)
1879 {
1880 	int iov_count = 1, length = 1024, rate = 1;
1881 	struct sockmap_options options = {0};
1882 	int opt, longindex, err, cg_fd = 0;
1883 	char *bpf_file = BPF_SOCKMAP_FILENAME;
1884 	int test = SELFTESTS;
1885 	bool cg_created = 0;
1886 
1887 	while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1888 				  long_options, &longindex)) != -1) {
1889 		switch (opt) {
1890 		case 's':
1891 			txmsg_start = atoi(optarg);
1892 			break;
1893 		case 'e':
1894 			txmsg_end = atoi(optarg);
1895 			break;
1896 		case 'p':
1897 			txmsg_start_push = atoi(optarg);
1898 			break;
1899 		case 'q':
1900 			txmsg_end_push = atoi(optarg);
1901 			break;
1902 		case 'w':
1903 			txmsg_start_pop = atoi(optarg);
1904 			break;
1905 		case 'x':
1906 			txmsg_pop = atoi(optarg);
1907 			break;
1908 		case 'a':
1909 			txmsg_apply = atoi(optarg);
1910 			break;
1911 		case 'k':
1912 			txmsg_cork = atoi(optarg);
1913 			break;
1914 		case 'c':
1915 			cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1916 			if (cg_fd < 0) {
1917 				fprintf(stderr,
1918 					"ERROR: (%i) open cg path failed: %s\n",
1919 					cg_fd, optarg);
1920 				return cg_fd;
1921 			}
1922 			break;
1923 		case 'r':
1924 			rate = atoi(optarg);
1925 			break;
1926 		case 'v':
1927 			options.verbose = 1;
1928 			if (optarg)
1929 				options.verbose = atoi(optarg);
1930 			break;
1931 		case 'i':
1932 			iov_count = atoi(optarg);
1933 			break;
1934 		case 'l':
1935 			length = atoi(optarg);
1936 			break;
1937 		case 'd':
1938 			options.data_test = true;
1939 			break;
1940 		case 't':
1941 			if (strcmp(optarg, "ping") == 0) {
1942 				test = PING_PONG;
1943 			} else if (strcmp(optarg, "sendmsg") == 0) {
1944 				test = SENDMSG;
1945 			} else if (strcmp(optarg, "base") == 0) {
1946 				test = BASE;
1947 			} else if (strcmp(optarg, "base_sendpage") == 0) {
1948 				test = BASE_SENDPAGE;
1949 			} else if (strcmp(optarg, "sendpage") == 0) {
1950 				test = SENDPAGE;
1951 			} else {
1952 				usage(argv);
1953 				return -1;
1954 			}
1955 			break;
1956 		case 'n':
1957 			options.whitelist = strdup(optarg);
1958 			if (!options.whitelist)
1959 				return -ENOMEM;
1960 			break;
1961 		case 'b':
1962 			options.blacklist = strdup(optarg);
1963 			if (!options.blacklist)
1964 				return -ENOMEM;
1965 		case 0:
1966 			break;
1967 		case 'h':
1968 		default:
1969 			usage(argv);
1970 			return -1;
1971 		}
1972 	}
1973 
1974 	if (!cg_fd) {
1975 		cg_fd = cgroup_setup_and_join(CG_PATH);
1976 		if (cg_fd < 0)
1977 			return cg_fd;
1978 		cg_created = 1;
1979 	}
1980 
1981 	if (test == SELFTESTS) {
1982 		err = test_selftest(cg_fd, &options);
1983 		goto out;
1984 	}
1985 
1986 	err = populate_progs(bpf_file);
1987 	if (err) {
1988 		fprintf(stderr, "populate program: (%s) %s\n",
1989 			bpf_file, strerror(errno));
1990 		return 1;
1991 	}
1992 	running = 1;
1993 
1994 	/* catch SIGINT */
1995 	signal(SIGINT, running_handler);
1996 
1997 	options.iov_count = iov_count;
1998 	options.iov_length = length;
1999 	options.rate = rate;
2000 
2001 	err = run_options(&options, cg_fd, test);
2002 out:
2003 	if (options.whitelist)
2004 		free(options.whitelist);
2005 	if (options.blacklist)
2006 		free(options.blacklist);
2007 	if (cg_created)
2008 		cleanup_cgroup_environment();
2009 	close(cg_fd);
2010 	return err;
2011 }
2012 
2013 void running_handler(int a)
2014 {
2015 	running = 0;
2016 }
2017