1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <linux/tls.h>
30 #include <assert.h>
31 #include <libgen.h>
32
33 #include <getopt.h>
34
35 #include <bpf/bpf.h>
36 #include <bpf/libbpf.h>
37
38 #include "bpf_util.h"
39 #include "cgroup_helpers.h"
40
41 int running;
42 static void running_handler(int a);
43
44 #ifndef TCP_ULP
45 # define TCP_ULP 31
46 #endif
47 #ifndef SOL_TLS
48 # define SOL_TLS 282
49 #endif
50
51 /* randomly selected ports for testing on lo */
52 #define S1_PORT 10000
53 #define S2_PORT 10001
54
55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57 #define CG_PATH "/sockmap"
58
59 /* global sockets */
60 int s1, s2, c1, c2, p1, p2;
61 int test_cnt;
62 int passed;
63 int failed;
64 int map_fd[9];
65 struct bpf_map *maps[9];
66 int prog_fd[9];
67
68 int txmsg_pass;
69 int txmsg_redir;
70 int txmsg_drop;
71 int txmsg_apply;
72 int txmsg_cork;
73 int txmsg_start;
74 int txmsg_end;
75 int txmsg_start_push;
76 int txmsg_end_push;
77 int txmsg_start_pop;
78 int txmsg_pop;
79 int txmsg_ingress;
80 int txmsg_redir_skb;
81 int txmsg_ktls_skb;
82 int txmsg_ktls_skb_drop;
83 int txmsg_ktls_skb_redir;
84 int ktls;
85 int peek_flag;
86 int skb_use_parser;
87 int txmsg_omit_skb_parser;
88
89 static const struct option long_options[] = {
90 {"help", no_argument, NULL, 'h' },
91 {"cgroup", required_argument, NULL, 'c' },
92 {"rate", required_argument, NULL, 'r' },
93 {"verbose", optional_argument, NULL, 'v' },
94 {"iov_count", required_argument, NULL, 'i' },
95 {"length", required_argument, NULL, 'l' },
96 {"test", required_argument, NULL, 't' },
97 {"data_test", no_argument, NULL, 'd' },
98 {"txmsg", no_argument, &txmsg_pass, 1 },
99 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
100 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
101 {"txmsg_apply", required_argument, NULL, 'a'},
102 {"txmsg_cork", required_argument, NULL, 'k'},
103 {"txmsg_start", required_argument, NULL, 's'},
104 {"txmsg_end", required_argument, NULL, 'e'},
105 {"txmsg_start_push", required_argument, NULL, 'p'},
106 {"txmsg_end_push", required_argument, NULL, 'q'},
107 {"txmsg_start_pop", required_argument, NULL, 'w'},
108 {"txmsg_pop", required_argument, NULL, 'x'},
109 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
110 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
111 {"ktls", no_argument, &ktls, 1 },
112 {"peek", no_argument, &peek_flag, 1 },
113 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
114 {"whitelist", required_argument, NULL, 'n' },
115 {"blacklist", required_argument, NULL, 'b' },
116 {0, 0, NULL, 0 }
117 };
118
119 struct test_env {
120 const char *type;
121 const char *subtest;
122 const char *prepend;
123
124 int test_num;
125 int subtest_num;
126
127 int succ_cnt;
128 int fail_cnt;
129 int fail_last;
130 };
131
132 struct test_env env;
133
134 struct sockmap_options {
135 int verbose;
136 bool base;
137 bool sendpage;
138 bool data_test;
139 bool drop_expected;
140 bool check_recved_len;
141 bool tx_wait_mem;
142 int iov_count;
143 int iov_length;
144 int rate;
145 char *map;
146 char *whitelist;
147 char *blacklist;
148 char *prepend;
149 };
150
151 struct _test {
152 char *title;
153 void (*tester)(int cg_fd, struct sockmap_options *opt);
154 };
155
test_start(void)156 static void test_start(void)
157 {
158 env.subtest_num++;
159 }
160
test_fail(void)161 static void test_fail(void)
162 {
163 env.fail_cnt++;
164 }
165
test_pass(void)166 static void test_pass(void)
167 {
168 env.succ_cnt++;
169 }
170
test_reset(void)171 static void test_reset(void)
172 {
173 txmsg_start = txmsg_end = 0;
174 txmsg_start_pop = txmsg_pop = 0;
175 txmsg_start_push = txmsg_end_push = 0;
176 txmsg_pass = txmsg_drop = txmsg_redir = 0;
177 txmsg_apply = txmsg_cork = 0;
178 txmsg_ingress = txmsg_redir_skb = 0;
179 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180 txmsg_omit_skb_parser = 0;
181 skb_use_parser = 0;
182 }
183
test_start_subtest(const struct _test * t,struct sockmap_options * o)184 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185 {
186 env.type = o->map;
187 env.subtest = t->title;
188 env.prepend = o->prepend;
189 env.test_num++;
190 env.subtest_num = 0;
191 env.fail_last = env.fail_cnt;
192 test_reset();
193 return 0;
194 }
195
test_end_subtest(void)196 static void test_end_subtest(void)
197 {
198 int error = env.fail_cnt - env.fail_last;
199 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200
201 if (!error)
202 test_pass();
203
204 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205 env.test_num, env.subtest_num,
206 !type ? "sockmap" : "sockhash",
207 env.prepend ? : "",
208 env.subtest, error ? "FAIL" : "OK");
209 }
210
test_print_results(void)211 static void test_print_results(void)
212 {
213 fprintf(stdout, "Pass: %d Fail: %d\n",
214 env.succ_cnt, env.fail_cnt);
215 }
216
usage(char * argv[])217 static void usage(char *argv[])
218 {
219 int i;
220
221 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
222 printf(" options:\n");
223 for (i = 0; long_options[i].name != 0; i++) {
224 printf(" --%-12s", long_options[i].name);
225 if (long_options[i].flag != NULL)
226 printf(" flag (internal value:%d)\n",
227 *long_options[i].flag);
228 else
229 printf(" -%c\n", long_options[i].val);
230 }
231 printf("\n");
232 }
233
sock_to_string(int s)234 char *sock_to_string(int s)
235 {
236 if (s == c1)
237 return "client1";
238 else if (s == c2)
239 return "client2";
240 else if (s == s1)
241 return "server1";
242 else if (s == s2)
243 return "server2";
244 else if (s == p1)
245 return "peer1";
246 else if (s == p2)
247 return "peer2";
248 else
249 return "unknown";
250 }
251
sockmap_init_ktls(int verbose,int s)252 static int sockmap_init_ktls(int verbose, int s)
253 {
254 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255 .info = {
256 .version = TLS_1_2_VERSION,
257 .cipher_type = TLS_CIPHER_AES_GCM_128,
258 },
259 };
260 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261 .info = {
262 .version = TLS_1_2_VERSION,
263 .cipher_type = TLS_CIPHER_AES_GCM_128,
264 },
265 };
266 int so_buf = 6553500;
267 int err;
268
269 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270 if (err) {
271 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272 return -EINVAL;
273 }
274 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275 if (err) {
276 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277 return -EINVAL;
278 }
279 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280 if (err) {
281 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282 return -EINVAL;
283 }
284 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285 if (err) {
286 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287 return -EINVAL;
288 }
289 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290 if (err) {
291 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292 return -EINVAL;
293 }
294
295 if (verbose)
296 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297 return 0;
298 }
sockmap_init_sockets(int verbose)299 static int sockmap_init_sockets(int verbose)
300 {
301 int i, err, one = 1;
302 struct sockaddr_in addr;
303 int *fds[4] = {&s1, &s2, &c1, &c2};
304
305 s1 = s2 = p1 = p2 = c1 = c2 = 0;
306
307 /* Init sockets */
308 for (i = 0; i < 4; i++) {
309 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
310 if (*fds[i] < 0) {
311 perror("socket s1 failed()");
312 return errno;
313 }
314 }
315
316 /* Allow reuse */
317 for (i = 0; i < 2; i++) {
318 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
319 (char *)&one, sizeof(one));
320 if (err) {
321 perror("setsockopt failed()");
322 return errno;
323 }
324 }
325
326 /* Non-blocking sockets */
327 for (i = 0; i < 2; i++) {
328 err = ioctl(*fds[i], FIONBIO, (char *)&one);
329 if (err < 0) {
330 perror("ioctl s1 failed()");
331 return errno;
332 }
333 }
334
335 /* Bind server sockets */
336 memset(&addr, 0, sizeof(struct sockaddr_in));
337 addr.sin_family = AF_INET;
338 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
339
340 addr.sin_port = htons(S1_PORT);
341 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
342 if (err < 0) {
343 perror("bind s1 failed()");
344 return errno;
345 }
346
347 addr.sin_port = htons(S2_PORT);
348 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
349 if (err < 0) {
350 perror("bind s2 failed()");
351 return errno;
352 }
353
354 /* Listen server sockets */
355 addr.sin_port = htons(S1_PORT);
356 err = listen(s1, 32);
357 if (err < 0) {
358 perror("listen s1 failed()");
359 return errno;
360 }
361
362 addr.sin_port = htons(S2_PORT);
363 err = listen(s2, 32);
364 if (err < 0) {
365 perror("listen s1 failed()");
366 return errno;
367 }
368
369 /* Initiate Connect */
370 addr.sin_port = htons(S1_PORT);
371 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
372 if (err < 0 && errno != EINPROGRESS) {
373 perror("connect c1 failed()");
374 return errno;
375 }
376
377 addr.sin_port = htons(S2_PORT);
378 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
379 if (err < 0 && errno != EINPROGRESS) {
380 perror("connect c2 failed()");
381 return errno;
382 } else if (err < 0) {
383 err = 0;
384 }
385
386 /* Accept Connecrtions */
387 p1 = accept(s1, NULL, NULL);
388 if (p1 < 0) {
389 perror("accept s1 failed()");
390 return errno;
391 }
392
393 p2 = accept(s2, NULL, NULL);
394 if (p2 < 0) {
395 perror("accept s1 failed()");
396 return errno;
397 }
398
399 if (verbose > 1) {
400 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
401 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
402 c1, s1, c2, s2);
403 }
404 return 0;
405 }
406
407 struct msg_stats {
408 size_t bytes_sent;
409 size_t bytes_recvd;
410 struct timespec start;
411 struct timespec end;
412 };
413
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)414 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
415 struct msg_stats *s,
416 struct sockmap_options *opt)
417 {
418 bool drop = opt->drop_expected;
419 unsigned char k = 0;
420 FILE *file;
421 int i, fp;
422
423 file = tmpfile();
424 if (!file) {
425 perror("create file for sendpage");
426 return 1;
427 }
428 for (i = 0; i < iov_length * cnt; i++, k++)
429 fwrite(&k, sizeof(char), 1, file);
430 fflush(file);
431 fseek(file, 0, SEEK_SET);
432
433 fp = fileno(file);
434
435 clock_gettime(CLOCK_MONOTONIC, &s->start);
436 for (i = 0; i < cnt; i++) {
437 int sent;
438
439 errno = 0;
440 sent = sendfile(fd, fp, NULL, iov_length);
441
442 if (!drop && sent < 0) {
443 perror("sendpage loop error");
444 fclose(file);
445 return sent;
446 } else if (drop && sent >= 0) {
447 printf("sendpage loop error expected: %i errno %i\n",
448 sent, errno);
449 fclose(file);
450 return -EIO;
451 }
452
453 if (sent > 0)
454 s->bytes_sent += sent;
455 }
456 clock_gettime(CLOCK_MONOTONIC, &s->end);
457 fclose(file);
458 return 0;
459 }
460
msg_free_iov(struct msghdr * msg)461 static void msg_free_iov(struct msghdr *msg)
462 {
463 int i;
464
465 for (i = 0; i < msg->msg_iovlen; i++)
466 free(msg->msg_iov[i].iov_base);
467 free(msg->msg_iov);
468 msg->msg_iov = NULL;
469 msg->msg_iovlen = 0;
470 }
471
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)472 static int msg_alloc_iov(struct msghdr *msg,
473 int iov_count, int iov_length,
474 bool data, bool xmit)
475 {
476 unsigned char k = 0;
477 struct iovec *iov;
478 int i;
479
480 iov = calloc(iov_count, sizeof(struct iovec));
481 if (!iov)
482 return errno;
483
484 for (i = 0; i < iov_count; i++) {
485 unsigned char *d = calloc(iov_length, sizeof(char));
486
487 if (!d) {
488 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489 goto unwind_iov;
490 }
491 iov[i].iov_base = d;
492 iov[i].iov_len = iov_length;
493
494 if (data && xmit) {
495 int j;
496
497 for (j = 0; j < iov_length; j++)
498 d[j] = k++;
499 }
500 }
501
502 msg->msg_iov = iov;
503 msg->msg_iovlen = iov_count;
504
505 return 0;
506 unwind_iov:
507 for (i--; i >= 0 ; i--)
508 free(msg->msg_iov[i].iov_base);
509 return -ENOMEM;
510 }
511
msg_verify_data(struct msghdr * msg,int size,int chunk_sz)512 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513 {
514 int i, j = 0, bytes_cnt = 0;
515 unsigned char k = 0;
516
517 for (i = 0; i < msg->msg_iovlen; i++) {
518 unsigned char *d = msg->msg_iov[i].iov_base;
519
520 /* Special case test for skb ingress + ktls */
521 if (i == 0 && txmsg_ktls_skb) {
522 if (msg->msg_iov[i].iov_len < 4)
523 return -EIO;
524 if (memcmp(d, "PASS", 4) != 0) {
525 fprintf(stderr,
526 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
527 i, 0, d[0], d[1], d[2], d[3]);
528 return -EIO;
529 }
530 j = 4; /* advance index past PASS header */
531 }
532
533 for (; j < msg->msg_iov[i].iov_len && size; j++) {
534 if (d[j] != k++) {
535 fprintf(stderr,
536 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537 i, j, d[j], k - 1, d[j+1], k);
538 return -EIO;
539 }
540 bytes_cnt++;
541 if (bytes_cnt == chunk_sz) {
542 k = 0;
543 bytes_cnt = 0;
544 }
545 size--;
546 }
547 }
548 return 0;
549 }
550
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)551 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552 struct msg_stats *s, bool tx,
553 struct sockmap_options *opt)
554 {
555 struct msghdr msg = {0}, msg_peek = {0};
556 int err, i, flags = MSG_NOSIGNAL;
557 bool drop = opt->drop_expected;
558 bool data = opt->data_test;
559 int iov_alloc_length = iov_length;
560
561 if (!tx && opt->check_recved_len)
562 iov_alloc_length *= 2;
563
564 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
565 if (err)
566 goto out_errno;
567 if (peek_flag) {
568 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
569 if (err)
570 goto out_errno;
571 }
572
573 if (tx) {
574 clock_gettime(CLOCK_MONOTONIC, &s->start);
575 for (i = 0; i < cnt; i++) {
576 int sent;
577
578 errno = 0;
579 sent = sendmsg(fd, &msg, flags);
580
581 if (!drop && sent < 0) {
582 if (opt->tx_wait_mem && errno == EACCES) {
583 errno = 0;
584 goto out_errno;
585 }
586 perror("sendmsg loop error");
587 goto out_errno;
588 } else if (drop && sent >= 0) {
589 fprintf(stderr,
590 "sendmsg loop error expected: %i errno %i\n",
591 sent, errno);
592 errno = -EIO;
593 goto out_errno;
594 }
595 if (sent > 0)
596 s->bytes_sent += sent;
597 }
598 clock_gettime(CLOCK_MONOTONIC, &s->end);
599 } else {
600 int slct, recvp = 0, recv, max_fd = fd;
601 float total_bytes, txmsg_pop_total;
602 int fd_flags = O_NONBLOCK;
603 struct timeval timeout;
604 fd_set w;
605
606 fcntl(fd, fd_flags);
607 /* Account for pop bytes noting each iteration of apply will
608 * call msg_pop_data helper so we need to account for this
609 * by calculating the number of apply iterations. Note user
610 * of the tool can create cases where no data is sent by
611 * manipulating pop/push/pull/etc. For example txmsg_apply 1
612 * with txmsg_pop 1 will try to apply 1B at a time but each
613 * iteration will then pop 1B so no data will ever be sent.
614 * This is really only useful for testing edge cases in code
615 * paths.
616 */
617 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
618 if (txmsg_apply)
619 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
620 else
621 txmsg_pop_total = txmsg_pop * cnt;
622 total_bytes -= txmsg_pop_total;
623 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
624 if (err < 0)
625 perror("recv start time");
626 while (s->bytes_recvd < total_bytes) {
627 if (txmsg_cork) {
628 timeout.tv_sec = 0;
629 timeout.tv_usec = 300000;
630 } else {
631 timeout.tv_sec = 3;
632 timeout.tv_usec = 0;
633 }
634
635 /* FD sets */
636 FD_ZERO(&w);
637 FD_SET(fd, &w);
638
639 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
640 if (slct == -1) {
641 perror("select()");
642 clock_gettime(CLOCK_MONOTONIC, &s->end);
643 goto out_errno;
644 } else if (!slct) {
645 if (opt->verbose)
646 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
647 errno = -EIO;
648 clock_gettime(CLOCK_MONOTONIC, &s->end);
649 goto out_errno;
650 }
651
652 if (opt->tx_wait_mem) {
653 FD_ZERO(&w);
654 FD_SET(fd, &w);
655 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
656 errno = 0;
657 close(fd);
658 goto out_errno;
659 }
660
661 errno = 0;
662 if (peek_flag) {
663 flags |= MSG_PEEK;
664 recvp = recvmsg(fd, &msg_peek, flags);
665 if (recvp < 0) {
666 if (errno != EWOULDBLOCK) {
667 clock_gettime(CLOCK_MONOTONIC, &s->end);
668 goto out_errno;
669 }
670 }
671 flags = 0;
672 }
673
674 recv = recvmsg(fd, &msg, flags);
675 if (recv < 0) {
676 if (errno != EWOULDBLOCK) {
677 clock_gettime(CLOCK_MONOTONIC, &s->end);
678 perror("recv failed()");
679 goto out_errno;
680 }
681 }
682
683 if (recv > 0)
684 s->bytes_recvd += recv;
685
686 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
687 errno = EMSGSIZE;
688 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
689 s->bytes_recvd, total_bytes);
690 goto out_errno;
691 }
692
693 if (data) {
694 int chunk_sz = opt->sendpage ?
695 iov_length * cnt :
696 iov_length * iov_count;
697
698 errno = msg_verify_data(&msg, recv, chunk_sz);
699 if (errno) {
700 perror("data verify msg failed");
701 goto out_errno;
702 }
703 if (recvp) {
704 errno = msg_verify_data(&msg_peek,
705 recvp,
706 chunk_sz);
707 if (errno) {
708 perror("data verify msg_peek failed");
709 goto out_errno;
710 }
711 }
712 }
713 }
714 clock_gettime(CLOCK_MONOTONIC, &s->end);
715 }
716
717 msg_free_iov(&msg);
718 msg_free_iov(&msg_peek);
719 return err;
720 out_errno:
721 msg_free_iov(&msg);
722 msg_free_iov(&msg_peek);
723 return errno;
724 }
725
726 static float giga = 1000000000;
727
sentBps(struct msg_stats s)728 static inline float sentBps(struct msg_stats s)
729 {
730 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
731 }
732
recvdBps(struct msg_stats s)733 static inline float recvdBps(struct msg_stats s)
734 {
735 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
736 }
737
sendmsg_test(struct sockmap_options * opt)738 static int sendmsg_test(struct sockmap_options *opt)
739 {
740 float sent_Bps = 0, recvd_Bps = 0;
741 int rx_fd, txpid, rxpid, err = 0;
742 struct msg_stats s = {0};
743 int iov_count = opt->iov_count;
744 int iov_buf = opt->iov_length;
745 int rx_status, tx_status;
746 int cnt = opt->rate;
747
748 errno = 0;
749
750 if (opt->base)
751 rx_fd = p1;
752 else
753 rx_fd = p2;
754
755 if (ktls) {
756 /* Redirecting into non-TLS socket which sends into a TLS
757 * socket is not a valid test. So in this case lets not
758 * enable kTLS but still run the test.
759 */
760 if (!txmsg_redir || txmsg_ingress) {
761 err = sockmap_init_ktls(opt->verbose, rx_fd);
762 if (err)
763 return err;
764 }
765 err = sockmap_init_ktls(opt->verbose, c1);
766 if (err)
767 return err;
768 }
769
770 if (opt->tx_wait_mem) {
771 struct timeval timeout;
772 int rxtx_buf_len = 1024;
773
774 timeout.tv_sec = 3;
775 timeout.tv_usec = 0;
776
777 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
778 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
779 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
780 if (err) {
781 perror("setsockopt failed()");
782 return errno;
783 }
784 }
785
786 rxpid = fork();
787 if (rxpid == 0) {
788 if (txmsg_pop || txmsg_start_pop)
789 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
790 if (opt->drop_expected || txmsg_ktls_skb_drop)
791 _exit(0);
792
793 if (!iov_buf) /* zero bytes sent case */
794 _exit(0);
795
796 if (opt->sendpage)
797 iov_count = 1;
798 err = msg_loop(rx_fd, iov_count, iov_buf,
799 cnt, &s, false, opt);
800 if (opt->verbose > 1)
801 fprintf(stderr,
802 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
803 iov_count, iov_buf, cnt, err);
804 if (s.end.tv_sec - s.start.tv_sec) {
805 sent_Bps = sentBps(s);
806 recvd_Bps = recvdBps(s);
807 }
808 if (opt->verbose > 1)
809 fprintf(stdout,
810 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
811 s.bytes_sent, sent_Bps, sent_Bps/giga,
812 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
813 peek_flag ? "(peek_msg)" : "");
814 if (err && txmsg_cork)
815 err = 0;
816 exit(err ? 1 : 0);
817 } else if (rxpid == -1) {
818 perror("msg_loop_rx");
819 return errno;
820 }
821
822 if (opt->tx_wait_mem)
823 close(c2);
824
825 txpid = fork();
826 if (txpid == 0) {
827 if (opt->sendpage)
828 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
829 else
830 err = msg_loop(c1, iov_count, iov_buf,
831 cnt, &s, true, opt);
832
833 if (err)
834 fprintf(stderr,
835 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
836 iov_count, iov_buf, cnt, err);
837 if (s.end.tv_sec - s.start.tv_sec) {
838 sent_Bps = sentBps(s);
839 recvd_Bps = recvdBps(s);
840 }
841 if (opt->verbose > 1)
842 fprintf(stdout,
843 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
844 s.bytes_sent, sent_Bps, sent_Bps/giga,
845 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
846 exit(err ? 1 : 0);
847 } else if (txpid == -1) {
848 perror("msg_loop_tx");
849 return errno;
850 }
851
852 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
853 assert(waitpid(txpid, &tx_status, 0) == txpid);
854 if (WIFEXITED(rx_status)) {
855 err = WEXITSTATUS(rx_status);
856 if (err) {
857 fprintf(stderr, "rx thread exited with err %d.\n", err);
858 goto out;
859 }
860 }
861 if (WIFEXITED(tx_status)) {
862 err = WEXITSTATUS(tx_status);
863 if (err)
864 fprintf(stderr, "tx thread exited with err %d.\n", err);
865 }
866 out:
867 return err;
868 }
869
forever_ping_pong(int rate,struct sockmap_options * opt)870 static int forever_ping_pong(int rate, struct sockmap_options *opt)
871 {
872 struct timeval timeout;
873 char buf[1024] = {0};
874 int sc;
875
876 timeout.tv_sec = 10;
877 timeout.tv_usec = 0;
878
879 /* Ping/Pong data from client to server */
880 sc = send(c1, buf, sizeof(buf), 0);
881 if (sc < 0) {
882 perror("send failed()");
883 return sc;
884 }
885
886 do {
887 int s, rc, i, max_fd = p2;
888 fd_set w;
889
890 /* FD sets */
891 FD_ZERO(&w);
892 FD_SET(c1, &w);
893 FD_SET(c2, &w);
894 FD_SET(p1, &w);
895 FD_SET(p2, &w);
896
897 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
898 if (s == -1) {
899 perror("select()");
900 break;
901 } else if (!s) {
902 fprintf(stderr, "unexpected timeout\n");
903 break;
904 }
905
906 for (i = 0; i <= max_fd && s > 0; ++i) {
907 if (!FD_ISSET(i, &w))
908 continue;
909
910 s--;
911
912 rc = recv(i, buf, sizeof(buf), 0);
913 if (rc < 0) {
914 if (errno != EWOULDBLOCK) {
915 perror("recv failed()");
916 return rc;
917 }
918 }
919
920 if (rc == 0) {
921 close(i);
922 break;
923 }
924
925 sc = send(i, buf, rc, 0);
926 if (sc < 0) {
927 perror("send failed()");
928 return sc;
929 }
930 }
931
932 if (rate)
933 sleep(rate);
934
935 if (opt->verbose) {
936 printf(".");
937 fflush(stdout);
938
939 }
940 } while (running);
941
942 return 0;
943 }
944
945 enum {
946 SELFTESTS,
947 PING_PONG,
948 SENDMSG,
949 BASE,
950 BASE_SENDPAGE,
951 SENDPAGE,
952 };
953
run_options(struct sockmap_options * options,int cg_fd,int test)954 static int run_options(struct sockmap_options *options, int cg_fd, int test)
955 {
956 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
957
958 /* If base test skip BPF setup */
959 if (test == BASE || test == BASE_SENDPAGE)
960 goto run;
961
962 /* Attach programs to sockmap */
963 if (!txmsg_omit_skb_parser) {
964 err = bpf_prog_attach(prog_fd[0], map_fd[0],
965 BPF_SK_SKB_STREAM_PARSER, 0);
966 if (err) {
967 fprintf(stderr,
968 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
969 prog_fd[0], map_fd[0], err, strerror(errno));
970 return err;
971 }
972 }
973
974 err = bpf_prog_attach(prog_fd[1], map_fd[0],
975 BPF_SK_SKB_STREAM_VERDICT, 0);
976 if (err) {
977 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
978 err, strerror(errno));
979 return err;
980 }
981
982 /* Attach programs to TLS sockmap */
983 if (txmsg_ktls_skb) {
984 if (!txmsg_omit_skb_parser) {
985 err = bpf_prog_attach(prog_fd[0], map_fd[8],
986 BPF_SK_SKB_STREAM_PARSER, 0);
987 if (err) {
988 fprintf(stderr,
989 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
990 prog_fd[0], map_fd[8], err, strerror(errno));
991 return err;
992 }
993 }
994
995 err = bpf_prog_attach(prog_fd[2], map_fd[8],
996 BPF_SK_SKB_STREAM_VERDICT, 0);
997 if (err) {
998 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
999 err, strerror(errno));
1000 return err;
1001 }
1002 }
1003
1004 /* Attach to cgroups */
1005 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1006 if (err) {
1007 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1008 err, strerror(errno));
1009 return err;
1010 }
1011
1012 run:
1013 err = sockmap_init_sockets(options->verbose);
1014 if (err) {
1015 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1016 goto out;
1017 }
1018
1019 /* Attach txmsg program to sockmap */
1020 if (txmsg_pass)
1021 tx_prog_fd = prog_fd[4];
1022 else if (txmsg_redir)
1023 tx_prog_fd = prog_fd[5];
1024 else if (txmsg_apply)
1025 tx_prog_fd = prog_fd[6];
1026 else if (txmsg_cork)
1027 tx_prog_fd = prog_fd[7];
1028 else if (txmsg_drop)
1029 tx_prog_fd = prog_fd[8];
1030 else
1031 tx_prog_fd = 0;
1032
1033 if (tx_prog_fd) {
1034 int redir_fd, i = 0;
1035
1036 err = bpf_prog_attach(tx_prog_fd,
1037 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1038 if (err) {
1039 fprintf(stderr,
1040 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1041 err, strerror(errno));
1042 goto out;
1043 }
1044
1045 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1046 if (err) {
1047 fprintf(stderr,
1048 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1049 err, strerror(errno));
1050 goto out;
1051 }
1052
1053 if (txmsg_redir)
1054 redir_fd = c2;
1055 else
1056 redir_fd = c1;
1057
1058 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1059 if (err) {
1060 fprintf(stderr,
1061 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1062 err, strerror(errno));
1063 goto out;
1064 }
1065
1066 if (txmsg_apply) {
1067 err = bpf_map_update_elem(map_fd[3],
1068 &i, &txmsg_apply, BPF_ANY);
1069 if (err) {
1070 fprintf(stderr,
1071 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1072 err, strerror(errno));
1073 goto out;
1074 }
1075 }
1076
1077 if (txmsg_cork) {
1078 err = bpf_map_update_elem(map_fd[4],
1079 &i, &txmsg_cork, BPF_ANY);
1080 if (err) {
1081 fprintf(stderr,
1082 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1083 err, strerror(errno));
1084 goto out;
1085 }
1086 }
1087
1088 if (txmsg_start) {
1089 err = bpf_map_update_elem(map_fd[5],
1090 &i, &txmsg_start, BPF_ANY);
1091 if (err) {
1092 fprintf(stderr,
1093 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1094 err, strerror(errno));
1095 goto out;
1096 }
1097 }
1098
1099 if (txmsg_end) {
1100 i = 1;
1101 err = bpf_map_update_elem(map_fd[5],
1102 &i, &txmsg_end, BPF_ANY);
1103 if (err) {
1104 fprintf(stderr,
1105 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1106 err, strerror(errno));
1107 goto out;
1108 }
1109 }
1110
1111 if (txmsg_start_push) {
1112 i = 2;
1113 err = bpf_map_update_elem(map_fd[5],
1114 &i, &txmsg_start_push, BPF_ANY);
1115 if (err) {
1116 fprintf(stderr,
1117 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1118 err, strerror(errno));
1119 goto out;
1120 }
1121 }
1122
1123 if (txmsg_end_push) {
1124 i = 3;
1125 err = bpf_map_update_elem(map_fd[5],
1126 &i, &txmsg_end_push, BPF_ANY);
1127 if (err) {
1128 fprintf(stderr,
1129 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1130 txmsg_end_push, i, err, strerror(errno));
1131 goto out;
1132 }
1133 }
1134
1135 if (txmsg_start_pop) {
1136 i = 4;
1137 err = bpf_map_update_elem(map_fd[5],
1138 &i, &txmsg_start_pop, BPF_ANY);
1139 if (err) {
1140 fprintf(stderr,
1141 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1142 txmsg_start_pop, i, err, strerror(errno));
1143 goto out;
1144 }
1145 } else {
1146 i = 4;
1147 bpf_map_update_elem(map_fd[5],
1148 &i, &txmsg_start_pop, BPF_ANY);
1149 }
1150
1151 if (txmsg_pop) {
1152 i = 5;
1153 err = bpf_map_update_elem(map_fd[5],
1154 &i, &txmsg_pop, BPF_ANY);
1155 if (err) {
1156 fprintf(stderr,
1157 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1158 txmsg_pop, i, err, strerror(errno));
1159 goto out;
1160 }
1161 } else {
1162 i = 5;
1163 bpf_map_update_elem(map_fd[5],
1164 &i, &txmsg_pop, BPF_ANY);
1165
1166 }
1167
1168 if (txmsg_ingress) {
1169 int in = BPF_F_INGRESS;
1170
1171 i = 0;
1172 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1173 if (err) {
1174 fprintf(stderr,
1175 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1176 err, strerror(errno));
1177 }
1178 i = 1;
1179 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1180 if (err) {
1181 fprintf(stderr,
1182 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1183 err, strerror(errno));
1184 }
1185 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1186 if (err) {
1187 fprintf(stderr,
1188 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1189 err, strerror(errno));
1190 }
1191
1192 i = 2;
1193 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1194 if (err) {
1195 fprintf(stderr,
1196 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1197 err, strerror(errno));
1198 }
1199 }
1200
1201 if (txmsg_ktls_skb) {
1202 int ingress = BPF_F_INGRESS;
1203
1204 i = 0;
1205 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1206 if (err) {
1207 fprintf(stderr,
1208 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1209 err, strerror(errno));
1210 }
1211
1212 if (txmsg_ktls_skb_redir) {
1213 i = 1;
1214 err = bpf_map_update_elem(map_fd[7],
1215 &i, &ingress, BPF_ANY);
1216 if (err) {
1217 fprintf(stderr,
1218 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1219 err, strerror(errno));
1220 }
1221 }
1222
1223 if (txmsg_ktls_skb_drop) {
1224 i = 1;
1225 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1226 }
1227 }
1228
1229 if (txmsg_redir_skb) {
1230 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1231 p2 : p1;
1232 int ingress = BPF_F_INGRESS;
1233
1234 i = 0;
1235 err = bpf_map_update_elem(map_fd[7],
1236 &i, &ingress, BPF_ANY);
1237 if (err) {
1238 fprintf(stderr,
1239 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1240 err, strerror(errno));
1241 }
1242
1243 i = 3;
1244 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1245 if (err) {
1246 fprintf(stderr,
1247 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1248 err, strerror(errno));
1249 }
1250 }
1251 }
1252
1253 if (skb_use_parser) {
1254 i = 2;
1255 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1256 }
1257
1258 if (txmsg_drop)
1259 options->drop_expected = true;
1260
1261 if (test == PING_PONG)
1262 err = forever_ping_pong(options->rate, options);
1263 else if (test == SENDMSG) {
1264 options->base = false;
1265 options->sendpage = false;
1266 err = sendmsg_test(options);
1267 } else if (test == SENDPAGE) {
1268 options->base = false;
1269 options->sendpage = true;
1270 err = sendmsg_test(options);
1271 } else if (test == BASE) {
1272 options->base = true;
1273 options->sendpage = false;
1274 err = sendmsg_test(options);
1275 } else if (test == BASE_SENDPAGE) {
1276 options->base = true;
1277 options->sendpage = true;
1278 err = sendmsg_test(options);
1279 } else
1280 fprintf(stderr, "unknown test\n");
1281 out:
1282 /* Detatch and zero all the maps */
1283 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1284 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1285 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1286 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1287 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1288
1289 if (tx_prog_fd >= 0)
1290 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1291
1292 for (i = 0; i < 8; i++) {
1293 key = next_key = 0;
1294 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1295 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1296 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1297 key = next_key;
1298 }
1299 }
1300
1301 close(s1);
1302 close(s2);
1303 close(p1);
1304 close(p2);
1305 close(c1);
1306 close(c2);
1307 return err;
1308 }
1309
test_to_str(int test)1310 static char *test_to_str(int test)
1311 {
1312 switch (test) {
1313 case SENDMSG:
1314 return "sendmsg";
1315 case SENDPAGE:
1316 return "sendpage";
1317 }
1318 return "unknown";
1319 }
1320
append_str(char * dst,const char * src,size_t dst_cap)1321 static void append_str(char *dst, const char *src, size_t dst_cap)
1322 {
1323 size_t avail = dst_cap - strlen(dst);
1324
1325 if (avail <= 1) /* just zero byte could be written */
1326 return;
1327
1328 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1329 }
1330
1331 #define OPTSTRING 60
test_options(char * options)1332 static void test_options(char *options)
1333 {
1334 char tstr[OPTSTRING];
1335
1336 memset(options, 0, OPTSTRING);
1337
1338 if (txmsg_pass)
1339 append_str(options, "pass,", OPTSTRING);
1340 if (txmsg_redir)
1341 append_str(options, "redir,", OPTSTRING);
1342 if (txmsg_drop)
1343 append_str(options, "drop,", OPTSTRING);
1344 if (txmsg_apply) {
1345 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1346 append_str(options, tstr, OPTSTRING);
1347 }
1348 if (txmsg_cork) {
1349 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1350 append_str(options, tstr, OPTSTRING);
1351 }
1352 if (txmsg_start) {
1353 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1354 append_str(options, tstr, OPTSTRING);
1355 }
1356 if (txmsg_end) {
1357 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1358 append_str(options, tstr, OPTSTRING);
1359 }
1360 if (txmsg_start_pop) {
1361 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1362 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1363 append_str(options, tstr, OPTSTRING);
1364 }
1365 if (txmsg_ingress)
1366 append_str(options, "ingress,", OPTSTRING);
1367 if (txmsg_redir_skb)
1368 append_str(options, "redir_skb,", OPTSTRING);
1369 if (txmsg_ktls_skb)
1370 append_str(options, "ktls_skb,", OPTSTRING);
1371 if (ktls)
1372 append_str(options, "ktls,", OPTSTRING);
1373 if (peek_flag)
1374 append_str(options, "peek,", OPTSTRING);
1375 }
1376
__test_exec(int cgrp,int test,struct sockmap_options * opt)1377 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1378 {
1379 char *options = calloc(OPTSTRING, sizeof(char));
1380 int err;
1381
1382 if (test == SENDPAGE)
1383 opt->sendpage = true;
1384 else
1385 opt->sendpage = false;
1386
1387 if (txmsg_drop)
1388 opt->drop_expected = true;
1389 else
1390 opt->drop_expected = false;
1391
1392 test_options(options);
1393
1394 if (opt->verbose) {
1395 fprintf(stdout,
1396 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1397 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1398 test_to_str(test), options);
1399 fflush(stdout);
1400 }
1401 err = run_options(opt, cgrp, test);
1402 if (opt->verbose)
1403 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1404 test_cnt++;
1405 !err ? passed++ : failed++;
1406 free(options);
1407 return err;
1408 }
1409
test_exec(int cgrp,struct sockmap_options * opt)1410 static void test_exec(int cgrp, struct sockmap_options *opt)
1411 {
1412 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1413 int err;
1414
1415 if (type == 0) {
1416 test_start();
1417 err = __test_exec(cgrp, SENDMSG, opt);
1418 if (err)
1419 test_fail();
1420 } else {
1421 test_start();
1422 err = __test_exec(cgrp, SENDPAGE, opt);
1423 if (err)
1424 test_fail();
1425 }
1426 }
1427
test_send_one(struct sockmap_options * opt,int cgrp)1428 static void test_send_one(struct sockmap_options *opt, int cgrp)
1429 {
1430 opt->iov_length = 1;
1431 opt->iov_count = 1;
1432 opt->rate = 1;
1433 test_exec(cgrp, opt);
1434
1435 opt->iov_length = 1;
1436 opt->iov_count = 1024;
1437 opt->rate = 1;
1438 test_exec(cgrp, opt);
1439
1440 opt->iov_length = 1024;
1441 opt->iov_count = 1;
1442 opt->rate = 1;
1443 test_exec(cgrp, opt);
1444
1445 }
1446
test_send_many(struct sockmap_options * opt,int cgrp)1447 static void test_send_many(struct sockmap_options *opt, int cgrp)
1448 {
1449 opt->iov_length = 3;
1450 opt->iov_count = 1;
1451 opt->rate = 512;
1452 test_exec(cgrp, opt);
1453
1454 opt->rate = 100;
1455 opt->iov_count = 1;
1456 opt->iov_length = 5;
1457 test_exec(cgrp, opt);
1458 }
1459
test_send_large(struct sockmap_options * opt,int cgrp)1460 static void test_send_large(struct sockmap_options *opt, int cgrp)
1461 {
1462 opt->iov_length = 256;
1463 opt->iov_count = 1024;
1464 opt->rate = 2;
1465 test_exec(cgrp, opt);
1466 }
1467
test_send(struct sockmap_options * opt,int cgrp)1468 static void test_send(struct sockmap_options *opt, int cgrp)
1469 {
1470 test_send_one(opt, cgrp);
1471 test_send_many(opt, cgrp);
1472 test_send_large(opt, cgrp);
1473 sched_yield();
1474 }
1475
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1476 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1477 {
1478 /* Test small and large iov_count values with pass/redir/apply/cork */
1479 txmsg_pass = 1;
1480 test_send(opt, cgrp);
1481 }
1482
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1483 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1484 {
1485 txmsg_redir = 1;
1486 test_send(opt, cgrp);
1487 }
1488
test_txmsg_redir_wait_sndmem(int cgrp,struct sockmap_options * opt)1489 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1490 {
1491 txmsg_redir = 1;
1492 opt->tx_wait_mem = true;
1493 test_send_large(opt, cgrp);
1494 opt->tx_wait_mem = false;
1495 }
1496
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1497 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1498 {
1499 txmsg_drop = 1;
1500 test_send(opt, cgrp);
1501 }
1502
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1503 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1504 {
1505 txmsg_pass = txmsg_drop = 0;
1506 txmsg_ingress = txmsg_redir = 1;
1507 test_send(opt, cgrp);
1508 }
1509
test_txmsg_skb(int cgrp,struct sockmap_options * opt)1510 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1511 {
1512 bool data = opt->data_test;
1513 int k = ktls;
1514
1515 opt->data_test = true;
1516 ktls = 1;
1517
1518 txmsg_pass = txmsg_drop = 0;
1519 txmsg_ingress = txmsg_redir = 0;
1520 txmsg_ktls_skb = 1;
1521 txmsg_pass = 1;
1522
1523 /* Using data verification so ensure iov layout is
1524 * expected from test receiver side. e.g. has enough
1525 * bytes to write test code.
1526 */
1527 opt->iov_length = 100;
1528 opt->iov_count = 1;
1529 opt->rate = 1;
1530 test_exec(cgrp, opt);
1531
1532 txmsg_ktls_skb_drop = 1;
1533 test_exec(cgrp, opt);
1534
1535 txmsg_ktls_skb_drop = 0;
1536 txmsg_ktls_skb_redir = 1;
1537 test_exec(cgrp, opt);
1538 txmsg_ktls_skb_redir = 0;
1539
1540 /* Tests that omit skb_parser */
1541 txmsg_omit_skb_parser = 1;
1542 ktls = 0;
1543 txmsg_ktls_skb = 0;
1544 test_exec(cgrp, opt);
1545
1546 txmsg_ktls_skb_drop = 1;
1547 test_exec(cgrp, opt);
1548 txmsg_ktls_skb_drop = 0;
1549
1550 txmsg_ktls_skb_redir = 1;
1551 test_exec(cgrp, opt);
1552
1553 ktls = 1;
1554 test_exec(cgrp, opt);
1555 txmsg_omit_skb_parser = 0;
1556
1557 opt->data_test = data;
1558 ktls = k;
1559 }
1560
1561 /* Test cork with hung data. This tests poor usage patterns where
1562 * cork can leave data on the ring if user program is buggy and
1563 * doesn't flush them somehow. They do take some time however
1564 * because they wait for a timeout. Test pass, redir and cork with
1565 * apply logic. Use cork size of 4097 with send_large to avoid
1566 * aligning cork size with send size.
1567 */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1568 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1569 {
1570 txmsg_pass = 1;
1571 txmsg_redir = 0;
1572 txmsg_cork = 4097;
1573 txmsg_apply = 4097;
1574 test_send_large(opt, cgrp);
1575
1576 txmsg_pass = 0;
1577 txmsg_redir = 1;
1578 txmsg_apply = 0;
1579 txmsg_cork = 4097;
1580 test_send_large(opt, cgrp);
1581
1582 txmsg_pass = 0;
1583 txmsg_redir = 1;
1584 txmsg_apply = 4097;
1585 txmsg_cork = 4097;
1586 test_send_large(opt, cgrp);
1587 }
1588
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1589 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1590 {
1591 /* Test basic start/end */
1592 txmsg_start = 1;
1593 txmsg_end = 2;
1594 test_send(opt, cgrp);
1595
1596 /* Test >4k pull */
1597 txmsg_start = 4096;
1598 txmsg_end = 9182;
1599 test_send_large(opt, cgrp);
1600
1601 /* Test pull + redirect */
1602 txmsg_redir = 0;
1603 txmsg_start = 1;
1604 txmsg_end = 2;
1605 test_send(opt, cgrp);
1606
1607 /* Test pull + cork */
1608 txmsg_redir = 0;
1609 txmsg_cork = 512;
1610 txmsg_start = 1;
1611 txmsg_end = 2;
1612 test_send_many(opt, cgrp);
1613
1614 /* Test pull + cork + redirect */
1615 txmsg_redir = 1;
1616 txmsg_cork = 512;
1617 txmsg_start = 1;
1618 txmsg_end = 2;
1619 test_send_many(opt, cgrp);
1620 }
1621
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1622 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1623 {
1624 /* Test basic pop */
1625 txmsg_start_pop = 1;
1626 txmsg_pop = 2;
1627 test_send_many(opt, cgrp);
1628
1629 /* Test pop with >4k */
1630 txmsg_start_pop = 4096;
1631 txmsg_pop = 4096;
1632 test_send_large(opt, cgrp);
1633
1634 /* Test pop + redirect */
1635 txmsg_redir = 1;
1636 txmsg_start_pop = 1;
1637 txmsg_pop = 2;
1638 test_send_many(opt, cgrp);
1639
1640 /* Test pop + cork */
1641 txmsg_redir = 0;
1642 txmsg_cork = 512;
1643 txmsg_start_pop = 1;
1644 txmsg_pop = 2;
1645 test_send_many(opt, cgrp);
1646
1647 /* Test pop + redirect + cork */
1648 txmsg_redir = 1;
1649 txmsg_cork = 4;
1650 txmsg_start_pop = 1;
1651 txmsg_pop = 2;
1652 test_send_many(opt, cgrp);
1653 }
1654
test_txmsg_push(int cgrp,struct sockmap_options * opt)1655 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1656 {
1657 /* Test basic push */
1658 txmsg_start_push = 1;
1659 txmsg_end_push = 1;
1660 test_send(opt, cgrp);
1661
1662 /* Test push 4kB >4k */
1663 txmsg_start_push = 4096;
1664 txmsg_end_push = 4096;
1665 test_send_large(opt, cgrp);
1666
1667 /* Test push + redirect */
1668 txmsg_redir = 1;
1669 txmsg_start_push = 1;
1670 txmsg_end_push = 2;
1671 test_send_many(opt, cgrp);
1672
1673 /* Test push + cork */
1674 txmsg_redir = 0;
1675 txmsg_cork = 512;
1676 txmsg_start_push = 1;
1677 txmsg_end_push = 2;
1678 test_send_many(opt, cgrp);
1679 }
1680
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1681 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1682 {
1683 txmsg_start_push = 1;
1684 txmsg_end_push = 10;
1685 txmsg_start_pop = 5;
1686 txmsg_pop = 4;
1687 test_send_large(opt, cgrp);
1688 }
1689
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1690 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1691 {
1692 txmsg_pass = 1;
1693 txmsg_redir = 0;
1694 txmsg_ingress = 0;
1695 txmsg_apply = 1;
1696 txmsg_cork = 0;
1697 test_send_one(opt, cgrp);
1698
1699 txmsg_pass = 0;
1700 txmsg_redir = 1;
1701 txmsg_ingress = 0;
1702 txmsg_apply = 1;
1703 txmsg_cork = 0;
1704 test_send_one(opt, cgrp);
1705
1706 txmsg_pass = 0;
1707 txmsg_redir = 1;
1708 txmsg_ingress = 1;
1709 txmsg_apply = 1;
1710 txmsg_cork = 0;
1711 test_send_one(opt, cgrp);
1712
1713 txmsg_pass = 1;
1714 txmsg_redir = 0;
1715 txmsg_ingress = 0;
1716 txmsg_apply = 1024;
1717 txmsg_cork = 0;
1718 test_send_large(opt, cgrp);
1719
1720 txmsg_pass = 0;
1721 txmsg_redir = 1;
1722 txmsg_ingress = 0;
1723 txmsg_apply = 1024;
1724 txmsg_cork = 0;
1725 test_send_large(opt, cgrp);
1726
1727 txmsg_pass = 0;
1728 txmsg_redir = 1;
1729 txmsg_ingress = 1;
1730 txmsg_apply = 1024;
1731 txmsg_cork = 0;
1732 test_send_large(opt, cgrp);
1733 }
1734
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1735 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1736 {
1737 txmsg_pass = 1;
1738 txmsg_redir = 0;
1739 txmsg_apply = 0;
1740 txmsg_cork = 1;
1741 test_send(opt, cgrp);
1742
1743 txmsg_pass = 1;
1744 txmsg_redir = 0;
1745 txmsg_apply = 1;
1746 txmsg_cork = 1;
1747 test_send(opt, cgrp);
1748 }
1749
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)1750 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1751 {
1752 txmsg_pass = 1;
1753 skb_use_parser = 512;
1754 if (ktls == 1)
1755 skb_use_parser = 570;
1756 opt->iov_length = 256;
1757 opt->iov_count = 1;
1758 opt->rate = 2;
1759 test_exec(cgrp, opt);
1760 }
1761
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1762 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1763 {
1764 if (ktls == 1)
1765 return;
1766 skb_use_parser = 10;
1767 opt->iov_length = 20;
1768 opt->iov_count = 1;
1769 opt->rate = 1;
1770 opt->check_recved_len = true;
1771 test_exec(cgrp, opt);
1772 opt->check_recved_len = false;
1773 }
1774
1775 char *map_names[] = {
1776 "sock_map",
1777 "sock_map_txmsg",
1778 "sock_map_redir",
1779 "sock_apply_bytes",
1780 "sock_cork_bytes",
1781 "sock_bytes",
1782 "sock_redir_flags",
1783 "sock_skb_opts",
1784 "tls_sock_map",
1785 };
1786
1787 int prog_attach_type[] = {
1788 BPF_SK_SKB_STREAM_PARSER,
1789 BPF_SK_SKB_STREAM_VERDICT,
1790 BPF_SK_SKB_STREAM_VERDICT,
1791 BPF_CGROUP_SOCK_OPS,
1792 BPF_SK_MSG_VERDICT,
1793 BPF_SK_MSG_VERDICT,
1794 BPF_SK_MSG_VERDICT,
1795 BPF_SK_MSG_VERDICT,
1796 BPF_SK_MSG_VERDICT,
1797 };
1798
1799 int prog_type[] = {
1800 BPF_PROG_TYPE_SK_SKB,
1801 BPF_PROG_TYPE_SK_SKB,
1802 BPF_PROG_TYPE_SK_SKB,
1803 BPF_PROG_TYPE_SOCK_OPS,
1804 BPF_PROG_TYPE_SK_MSG,
1805 BPF_PROG_TYPE_SK_MSG,
1806 BPF_PROG_TYPE_SK_MSG,
1807 BPF_PROG_TYPE_SK_MSG,
1808 BPF_PROG_TYPE_SK_MSG,
1809 };
1810
populate_progs(char * bpf_file)1811 static int populate_progs(char *bpf_file)
1812 {
1813 struct bpf_program *prog;
1814 struct bpf_object *obj;
1815 int i = 0;
1816 long err;
1817
1818 obj = bpf_object__open(bpf_file);
1819 err = libbpf_get_error(obj);
1820 if (err) {
1821 char err_buf[256];
1822
1823 libbpf_strerror(err, err_buf, sizeof(err_buf));
1824 printf("Unable to load eBPF objects in file '%s' : %s\n",
1825 bpf_file, err_buf);
1826 return -1;
1827 }
1828
1829 bpf_object__for_each_program(prog, obj) {
1830 bpf_program__set_type(prog, prog_type[i]);
1831 bpf_program__set_expected_attach_type(prog,
1832 prog_attach_type[i]);
1833 i++;
1834 }
1835
1836 i = bpf_object__load(obj);
1837 i = 0;
1838 bpf_object__for_each_program(prog, obj) {
1839 prog_fd[i] = bpf_program__fd(prog);
1840 i++;
1841 }
1842
1843 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1844 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1845 map_fd[i] = bpf_map__fd(maps[i]);
1846 if (map_fd[i] < 0) {
1847 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1848 map_fd[i], strerror(errno));
1849 return -1;
1850 }
1851 }
1852
1853 return 0;
1854 }
1855
1856 struct _test test[] = {
1857 {"txmsg test passthrough", test_txmsg_pass},
1858 {"txmsg test redirect", test_txmsg_redir},
1859 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1860 {"txmsg test drop", test_txmsg_drop},
1861 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1862 {"txmsg test skb", test_txmsg_skb},
1863 {"txmsg test apply", test_txmsg_apply},
1864 {"txmsg test cork", test_txmsg_cork},
1865 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1866 {"txmsg test push_data", test_txmsg_push},
1867 {"txmsg test pull-data", test_txmsg_pull},
1868 {"txmsg test pop-data", test_txmsg_pop},
1869 {"txmsg test push/pop data", test_txmsg_push_pop},
1870 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1871 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1872 };
1873
check_whitelist(struct _test * t,struct sockmap_options * opt)1874 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1875 {
1876 char *entry, *ptr;
1877
1878 if (!opt->whitelist)
1879 return 0;
1880 ptr = strdup(opt->whitelist);
1881 if (!ptr)
1882 return -ENOMEM;
1883 entry = strtok(ptr, ",");
1884 while (entry) {
1885 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1886 strstr(opt->map, entry) != 0 ||
1887 strstr(t->title, entry) != 0)
1888 return 0;
1889 entry = strtok(NULL, ",");
1890 }
1891 return -EINVAL;
1892 }
1893
check_blacklist(struct _test * t,struct sockmap_options * opt)1894 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1895 {
1896 char *entry, *ptr;
1897
1898 if (!opt->blacklist)
1899 return -EINVAL;
1900 ptr = strdup(opt->blacklist);
1901 if (!ptr)
1902 return -ENOMEM;
1903 entry = strtok(ptr, ",");
1904 while (entry) {
1905 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1906 strstr(opt->map, entry) != 0 ||
1907 strstr(t->title, entry) != 0)
1908 return 0;
1909 entry = strtok(NULL, ",");
1910 }
1911 return -EINVAL;
1912 }
1913
__test_selftests(int cg_fd,struct sockmap_options * opt)1914 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1915 {
1916 int i, err;
1917
1918 err = populate_progs(opt->map);
1919 if (err < 0) {
1920 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1921 return err;
1922 }
1923
1924 /* Tests basic commands and APIs */
1925 for (i = 0; i < ARRAY_SIZE(test); i++) {
1926 struct _test t = test[i];
1927
1928 if (check_whitelist(&t, opt) != 0)
1929 continue;
1930 if (check_blacklist(&t, opt) == 0)
1931 continue;
1932
1933 test_start_subtest(&t, opt);
1934 t.tester(cg_fd, opt);
1935 test_end_subtest();
1936 }
1937
1938 return err;
1939 }
1940
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)1941 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1942 {
1943 opt->map = BPF_SOCKMAP_FILENAME;
1944 __test_selftests(cg_fd, opt);
1945 }
1946
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)1947 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1948 {
1949 opt->map = BPF_SOCKHASH_FILENAME;
1950 __test_selftests(cg_fd, opt);
1951 }
1952
test_selftests_ktls(int cg_fd,struct sockmap_options * opt)1953 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1954 {
1955 opt->map = BPF_SOCKHASH_FILENAME;
1956 opt->prepend = "ktls";
1957 ktls = 1;
1958 __test_selftests(cg_fd, opt);
1959 ktls = 0;
1960 }
1961
test_selftest(int cg_fd,struct sockmap_options * opt)1962 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1963 {
1964
1965 test_selftests_sockmap(cg_fd, opt);
1966 test_selftests_sockhash(cg_fd, opt);
1967 test_selftests_ktls(cg_fd, opt);
1968 test_print_results();
1969 return 0;
1970 }
1971
main(int argc,char ** argv)1972 int main(int argc, char **argv)
1973 {
1974 int iov_count = 1, length = 1024, rate = 1;
1975 struct sockmap_options options = {0};
1976 int opt, longindex, err, cg_fd = 0;
1977 char *bpf_file = BPF_SOCKMAP_FILENAME;
1978 int test = SELFTESTS;
1979 bool cg_created = 0;
1980
1981 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1982 long_options, &longindex)) != -1) {
1983 switch (opt) {
1984 case 's':
1985 txmsg_start = atoi(optarg);
1986 break;
1987 case 'e':
1988 txmsg_end = atoi(optarg);
1989 break;
1990 case 'p':
1991 txmsg_start_push = atoi(optarg);
1992 break;
1993 case 'q':
1994 txmsg_end_push = atoi(optarg);
1995 break;
1996 case 'w':
1997 txmsg_start_pop = atoi(optarg);
1998 break;
1999 case 'x':
2000 txmsg_pop = atoi(optarg);
2001 break;
2002 case 'a':
2003 txmsg_apply = atoi(optarg);
2004 break;
2005 case 'k':
2006 txmsg_cork = atoi(optarg);
2007 break;
2008 case 'c':
2009 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2010 if (cg_fd < 0) {
2011 fprintf(stderr,
2012 "ERROR: (%i) open cg path failed: %s\n",
2013 cg_fd, optarg);
2014 return cg_fd;
2015 }
2016 break;
2017 case 'r':
2018 rate = atoi(optarg);
2019 break;
2020 case 'v':
2021 options.verbose = 1;
2022 if (optarg)
2023 options.verbose = atoi(optarg);
2024 break;
2025 case 'i':
2026 iov_count = atoi(optarg);
2027 break;
2028 case 'l':
2029 length = atoi(optarg);
2030 break;
2031 case 'd':
2032 options.data_test = true;
2033 break;
2034 case 't':
2035 if (strcmp(optarg, "ping") == 0) {
2036 test = PING_PONG;
2037 } else if (strcmp(optarg, "sendmsg") == 0) {
2038 test = SENDMSG;
2039 } else if (strcmp(optarg, "base") == 0) {
2040 test = BASE;
2041 } else if (strcmp(optarg, "base_sendpage") == 0) {
2042 test = BASE_SENDPAGE;
2043 } else if (strcmp(optarg, "sendpage") == 0) {
2044 test = SENDPAGE;
2045 } else {
2046 usage(argv);
2047 return -1;
2048 }
2049 break;
2050 case 'n':
2051 options.whitelist = strdup(optarg);
2052 if (!options.whitelist)
2053 return -ENOMEM;
2054 break;
2055 case 'b':
2056 options.blacklist = strdup(optarg);
2057 if (!options.blacklist)
2058 return -ENOMEM;
2059 case 0:
2060 break;
2061 case 'h':
2062 default:
2063 usage(argv);
2064 return -1;
2065 }
2066 }
2067
2068 if (!cg_fd) {
2069 cg_fd = cgroup_setup_and_join(CG_PATH);
2070 if (cg_fd < 0)
2071 return cg_fd;
2072 cg_created = 1;
2073 }
2074
2075 /* Use libbpf 1.0 API mode */
2076 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2077
2078 if (test == SELFTESTS) {
2079 err = test_selftest(cg_fd, &options);
2080 goto out;
2081 }
2082
2083 err = populate_progs(bpf_file);
2084 if (err) {
2085 fprintf(stderr, "populate program: (%s) %s\n",
2086 bpf_file, strerror(errno));
2087 return 1;
2088 }
2089 running = 1;
2090
2091 /* catch SIGINT */
2092 signal(SIGINT, running_handler);
2093
2094 options.iov_count = iov_count;
2095 options.iov_length = length;
2096 options.rate = rate;
2097
2098 err = run_options(&options, cg_fd, test);
2099 out:
2100 if (options.whitelist)
2101 free(options.whitelist);
2102 if (options.blacklist)
2103 free(options.blacklist);
2104 close(cg_fd);
2105 if (cg_created)
2106 cleanup_cgroup_environment();
2107 return err;
2108 }
2109
running_handler(int a)2110 void running_handler(int a)
2111 {
2112 running = 0;
2113 }
2114