1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/types.h>
22 #include <sys/sendfile.h>
23
24 #include <linux/netlink.h>
25 #include <linux/socket.h>
26 #include <linux/sock_diag.h>
27 #include <linux/bpf.h>
28 #include <linux/if_link.h>
29 #include <linux/tls.h>
30 #include <assert.h>
31 #include <libgen.h>
32
33 #include <getopt.h>
34
35 #include <bpf/bpf.h>
36 #include <bpf/libbpf.h>
37
38 #include "bpf_util.h"
39 #include "cgroup_helpers.h"
40
41 int running;
42 static void running_handler(int a);
43
44 #ifndef TCP_ULP
45 # define TCP_ULP 31
46 #endif
47 #ifndef SOL_TLS
48 # define SOL_TLS 282
49 #endif
50
51 /* randomly selected ports for testing on lo */
52 #define S1_PORT 10000
53 #define S2_PORT 10001
54
55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57 #define CG_PATH "/sockmap"
58
59 #define EDATAINTEGRITY 2001
60
61 /* global sockets */
62 int s1, s2, c1, c2, p1, p2;
63 int test_cnt;
64 int passed;
65 int failed;
66 int map_fd[9];
67 struct bpf_map *maps[9];
68 int prog_fd[9];
69
70 int txmsg_pass;
71 int txmsg_redir;
72 int txmsg_drop;
73 int txmsg_apply;
74 int txmsg_cork;
75 int txmsg_start;
76 int txmsg_end;
77 int txmsg_start_push;
78 int txmsg_end_push;
79 int txmsg_start_pop;
80 int txmsg_pop;
81 int txmsg_ingress;
82 int txmsg_redir_skb;
83 int txmsg_ktls_skb;
84 int txmsg_ktls_skb_drop;
85 int txmsg_ktls_skb_redir;
86 int ktls;
87 int peek_flag;
88 int skb_use_parser;
89 int txmsg_omit_skb_parser;
90 int verify_push_start;
91 int verify_push_len;
92 int verify_pop_start;
93 int verify_pop_len;
94
95 static const struct option long_options[] = {
96 {"help", no_argument, NULL, 'h' },
97 {"cgroup", required_argument, NULL, 'c' },
98 {"rate", required_argument, NULL, 'r' },
99 {"verbose", optional_argument, NULL, 'v' },
100 {"iov_count", required_argument, NULL, 'i' },
101 {"length", required_argument, NULL, 'l' },
102 {"test", required_argument, NULL, 't' },
103 {"data_test", no_argument, NULL, 'd' },
104 {"txmsg", no_argument, &txmsg_pass, 1 },
105 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
106 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
107 {"txmsg_apply", required_argument, NULL, 'a'},
108 {"txmsg_cork", required_argument, NULL, 'k'},
109 {"txmsg_start", required_argument, NULL, 's'},
110 {"txmsg_end", required_argument, NULL, 'e'},
111 {"txmsg_start_push", required_argument, NULL, 'p'},
112 {"txmsg_end_push", required_argument, NULL, 'q'},
113 {"txmsg_start_pop", required_argument, NULL, 'w'},
114 {"txmsg_pop", required_argument, NULL, 'x'},
115 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
116 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
117 {"ktls", no_argument, &ktls, 1 },
118 {"peek", no_argument, &peek_flag, 1 },
119 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
120 {"whitelist", required_argument, NULL, 'n' },
121 {"blacklist", required_argument, NULL, 'b' },
122 {0, 0, NULL, 0 }
123 };
124
125 struct test_env {
126 const char *type;
127 const char *subtest;
128 const char *prepend;
129
130 int test_num;
131 int subtest_num;
132
133 int succ_cnt;
134 int fail_cnt;
135 int fail_last;
136 };
137
138 struct test_env env;
139
140 struct sockmap_options {
141 int verbose;
142 bool base;
143 bool sendpage;
144 bool data_test;
145 bool drop_expected;
146 bool check_recved_len;
147 bool tx_wait_mem;
148 int iov_count;
149 int iov_length;
150 int rate;
151 char *map;
152 char *whitelist;
153 char *blacklist;
154 char *prepend;
155 };
156
157 struct _test {
158 char *title;
159 void (*tester)(int cg_fd, struct sockmap_options *opt);
160 };
161
test_start(void)162 static void test_start(void)
163 {
164 env.subtest_num++;
165 }
166
test_fail(void)167 static void test_fail(void)
168 {
169 env.fail_cnt++;
170 }
171
test_pass(void)172 static void test_pass(void)
173 {
174 env.succ_cnt++;
175 }
176
test_reset(void)177 static void test_reset(void)
178 {
179 txmsg_start = txmsg_end = 0;
180 txmsg_start_pop = txmsg_pop = 0;
181 txmsg_start_push = txmsg_end_push = 0;
182 txmsg_pass = txmsg_drop = txmsg_redir = 0;
183 txmsg_apply = txmsg_cork = 0;
184 txmsg_ingress = txmsg_redir_skb = 0;
185 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
186 txmsg_omit_skb_parser = 0;
187 skb_use_parser = 0;
188 }
189
test_start_subtest(const struct _test * t,struct sockmap_options * o)190 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
191 {
192 env.type = o->map;
193 env.subtest = t->title;
194 env.prepend = o->prepend;
195 env.test_num++;
196 env.subtest_num = 0;
197 env.fail_last = env.fail_cnt;
198 test_reset();
199 return 0;
200 }
201
test_end_subtest(void)202 static void test_end_subtest(void)
203 {
204 int error = env.fail_cnt - env.fail_last;
205 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
206
207 if (!error)
208 test_pass();
209
210 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
211 env.test_num, env.subtest_num,
212 !type ? "sockmap" : "sockhash",
213 env.prepend ? : "",
214 env.subtest, error ? "FAIL" : "OK");
215 }
216
test_print_results(void)217 static void test_print_results(void)
218 {
219 fprintf(stdout, "Pass: %d Fail: %d\n",
220 env.succ_cnt, env.fail_cnt);
221 }
222
usage(char * argv[])223 static void usage(char *argv[])
224 {
225 int i;
226
227 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
228 printf(" options:\n");
229 for (i = 0; long_options[i].name != 0; i++) {
230 printf(" --%-12s", long_options[i].name);
231 if (long_options[i].flag != NULL)
232 printf(" flag (internal value:%d)\n",
233 *long_options[i].flag);
234 else
235 printf(" -%c\n", long_options[i].val);
236 }
237 printf("\n");
238 }
239
sock_to_string(int s)240 char *sock_to_string(int s)
241 {
242 if (s == c1)
243 return "client1";
244 else if (s == c2)
245 return "client2";
246 else if (s == s1)
247 return "server1";
248 else if (s == s2)
249 return "server2";
250 else if (s == p1)
251 return "peer1";
252 else if (s == p2)
253 return "peer2";
254 else
255 return "unknown";
256 }
257
sockmap_init_ktls(int verbose,int s)258 static int sockmap_init_ktls(int verbose, int s)
259 {
260 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
261 .info = {
262 .version = TLS_1_2_VERSION,
263 .cipher_type = TLS_CIPHER_AES_GCM_128,
264 },
265 };
266 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
267 .info = {
268 .version = TLS_1_2_VERSION,
269 .cipher_type = TLS_CIPHER_AES_GCM_128,
270 },
271 };
272 int so_buf = 6553500;
273 int err;
274
275 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
276 if (err) {
277 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
278 return -EINVAL;
279 }
280 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
281 if (err) {
282 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
283 return -EINVAL;
284 }
285 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
286 if (err) {
287 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
288 return -EINVAL;
289 }
290 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
291 if (err) {
292 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
293 return -EINVAL;
294 }
295 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
296 if (err) {
297 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
298 return -EINVAL;
299 }
300
301 if (verbose)
302 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
303 return 0;
304 }
sockmap_init_sockets(int verbose)305 static int sockmap_init_sockets(int verbose)
306 {
307 int i, err, one = 1;
308 struct sockaddr_in addr;
309 int *fds[4] = {&s1, &s2, &c1, &c2};
310
311 s1 = s2 = p1 = p2 = c1 = c2 = 0;
312
313 /* Init sockets */
314 for (i = 0; i < 4; i++) {
315 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
316 if (*fds[i] < 0) {
317 perror("socket s1 failed()");
318 return errno;
319 }
320 }
321
322 /* Allow reuse */
323 for (i = 0; i < 2; i++) {
324 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
325 (char *)&one, sizeof(one));
326 if (err) {
327 perror("setsockopt failed()");
328 return errno;
329 }
330 }
331
332 /* Non-blocking sockets */
333 for (i = 0; i < 2; i++) {
334 err = ioctl(*fds[i], FIONBIO, (char *)&one);
335 if (err < 0) {
336 perror("ioctl s1 failed()");
337 return errno;
338 }
339 }
340
341 /* Bind server sockets */
342 memset(&addr, 0, sizeof(struct sockaddr_in));
343 addr.sin_family = AF_INET;
344 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
345
346 addr.sin_port = htons(S1_PORT);
347 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
348 if (err < 0) {
349 perror("bind s1 failed()");
350 return errno;
351 }
352
353 addr.sin_port = htons(S2_PORT);
354 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
355 if (err < 0) {
356 perror("bind s2 failed()");
357 return errno;
358 }
359
360 /* Listen server sockets */
361 addr.sin_port = htons(S1_PORT);
362 err = listen(s1, 32);
363 if (err < 0) {
364 perror("listen s1 failed()");
365 return errno;
366 }
367
368 addr.sin_port = htons(S2_PORT);
369 err = listen(s2, 32);
370 if (err < 0) {
371 perror("listen s1 failed()");
372 return errno;
373 }
374
375 /* Initiate Connect */
376 addr.sin_port = htons(S1_PORT);
377 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
378 if (err < 0 && errno != EINPROGRESS) {
379 perror("connect c1 failed()");
380 return errno;
381 }
382
383 addr.sin_port = htons(S2_PORT);
384 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
385 if (err < 0 && errno != EINPROGRESS) {
386 perror("connect c2 failed()");
387 return errno;
388 } else if (err < 0) {
389 err = 0;
390 }
391
392 /* Accept Connecrtions */
393 p1 = accept(s1, NULL, NULL);
394 if (p1 < 0) {
395 perror("accept s1 failed()");
396 return errno;
397 }
398
399 p2 = accept(s2, NULL, NULL);
400 if (p2 < 0) {
401 perror("accept s1 failed()");
402 return errno;
403 }
404
405 if (verbose > 1) {
406 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
407 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
408 c1, s1, c2, s2);
409 }
410 return 0;
411 }
412
413 struct msg_stats {
414 size_t bytes_sent;
415 size_t bytes_recvd;
416 struct timespec start;
417 struct timespec end;
418 };
419
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)420 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
421 struct msg_stats *s,
422 struct sockmap_options *opt)
423 {
424 bool drop = opt->drop_expected;
425 unsigned char k = 0;
426 int i, j, fp;
427 FILE *file;
428
429 file = tmpfile();
430 if (!file) {
431 perror("create file for sendpage");
432 return 1;
433 }
434 for (i = 0; i < cnt; i++, k = 0) {
435 for (j = 0; j < iov_length; j++, k++)
436 fwrite(&k, sizeof(char), 1, file);
437 }
438 fflush(file);
439 fseek(file, 0, SEEK_SET);
440
441 fp = fileno(file);
442
443 clock_gettime(CLOCK_MONOTONIC, &s->start);
444 for (i = 0; i < cnt; i++) {
445 int sent;
446
447 errno = 0;
448 sent = sendfile(fd, fp, NULL, iov_length);
449
450 if (!drop && sent < 0) {
451 perror("sendpage loop error");
452 fclose(file);
453 return sent;
454 } else if (drop && sent >= 0) {
455 printf("sendpage loop error expected: %i errno %i\n",
456 sent, errno);
457 fclose(file);
458 return -EIO;
459 }
460
461 if (sent > 0)
462 s->bytes_sent += sent;
463 }
464 clock_gettime(CLOCK_MONOTONIC, &s->end);
465 fclose(file);
466 return 0;
467 }
468
msg_free_iov(struct msghdr * msg)469 static void msg_free_iov(struct msghdr *msg)
470 {
471 int i;
472
473 for (i = 0; i < msg->msg_iovlen; i++)
474 free(msg->msg_iov[i].iov_base);
475 free(msg->msg_iov);
476 msg->msg_iov = NULL;
477 msg->msg_iovlen = 0;
478 }
479
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)480 static int msg_alloc_iov(struct msghdr *msg,
481 int iov_count, int iov_length,
482 bool data, bool xmit)
483 {
484 unsigned char k = 0;
485 struct iovec *iov;
486 int i;
487
488 iov = calloc(iov_count, sizeof(struct iovec));
489 if (!iov)
490 return errno;
491
492 for (i = 0; i < iov_count; i++) {
493 unsigned char *d = calloc(iov_length, sizeof(char));
494
495 if (!d) {
496 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
497 goto unwind_iov;
498 }
499 iov[i].iov_base = d;
500 iov[i].iov_len = iov_length;
501
502 if (data && xmit) {
503 int j;
504
505 for (j = 0; j < iov_length; j++)
506 d[j] = k++;
507 }
508 }
509
510 msg->msg_iov = iov;
511 msg->msg_iovlen = iov_count;
512
513 return 0;
514 unwind_iov:
515 for (i--; i >= 0 ; i--)
516 free(msg->msg_iov[i].iov_base);
517 return -ENOMEM;
518 }
519
520 /* In push or pop test, we need to do some calculations for msg_verify_data */
msg_verify_date_prep(void)521 static void msg_verify_date_prep(void)
522 {
523 int push_range_end = txmsg_start_push + txmsg_end_push - 1;
524 int pop_range_end = txmsg_start_pop + txmsg_pop - 1;
525
526 if (txmsg_end_push && txmsg_pop &&
527 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) {
528 /* The push range and the pop range overlap */
529 int overlap_len;
530
531 verify_push_start = txmsg_start_push;
532 verify_pop_start = txmsg_start_pop;
533 if (txmsg_start_push < txmsg_start_pop)
534 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop);
535 else
536 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push);
537 verify_push_len = max(txmsg_end_push - overlap_len, 0);
538 verify_pop_len = max(txmsg_pop - overlap_len, 0);
539 } else {
540 /* Otherwise */
541 verify_push_start = txmsg_start_push;
542 verify_pop_start = txmsg_start_pop;
543 verify_push_len = txmsg_end_push;
544 verify_pop_len = txmsg_pop;
545 }
546 }
547
msg_verify_data(struct msghdr * msg,int size,int chunk_sz,unsigned char * k_p,int * bytes_cnt_p,int * check_cnt_p,int * push_p)548 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz,
549 unsigned char *k_p, int *bytes_cnt_p,
550 int *check_cnt_p, int *push_p)
551 {
552 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p;
553 unsigned char k = *k_p;
554 int i, j;
555
556 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) {
557 unsigned char *d = msg->msg_iov[i].iov_base;
558
559 /* Special case test for skb ingress + ktls */
560 if (i == 0 && txmsg_ktls_skb) {
561 if (msg->msg_iov[i].iov_len < 4)
562 return -EDATAINTEGRITY;
563 if (memcmp(d, "PASS", 4) != 0) {
564 fprintf(stderr,
565 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
566 i, 0, d[0], d[1], d[2], d[3]);
567 return -EDATAINTEGRITY;
568 }
569 j = 4; /* advance index past PASS header */
570 }
571
572 for (; j < msg->msg_iov[i].iov_len && size; j++) {
573 if (push > 0 &&
574 check_cnt == verify_push_start + verify_push_len - push) {
575 int skipped;
576 revisit_push:
577 skipped = push;
578 if (j + push >= msg->msg_iov[i].iov_len)
579 skipped = msg->msg_iov[i].iov_len - j;
580 push -= skipped;
581 size -= skipped;
582 j += skipped - 1;
583 check_cnt += skipped;
584 continue;
585 }
586
587 if (verify_pop_len > 0 && check_cnt == verify_pop_start) {
588 bytes_cnt += verify_pop_len;
589 check_cnt += verify_pop_len;
590 k += verify_pop_len;
591
592 if (bytes_cnt == chunk_sz) {
593 k = 0;
594 bytes_cnt = 0;
595 check_cnt = 0;
596 push = verify_push_len;
597 }
598
599 if (push > 0 &&
600 check_cnt == verify_push_start + verify_push_len - push)
601 goto revisit_push;
602 }
603
604 if (d[j] != k++) {
605 fprintf(stderr,
606 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
607 i, j, d[j], k - 1, d[j+1], k);
608 return -EDATAINTEGRITY;
609 }
610 bytes_cnt++;
611 check_cnt++;
612 if (bytes_cnt == chunk_sz) {
613 k = 0;
614 bytes_cnt = 0;
615 check_cnt = 0;
616 push = verify_push_len;
617 }
618 size--;
619 }
620 }
621 *k_p = k;
622 *bytes_cnt_p = bytes_cnt;
623 *check_cnt_p = check_cnt;
624 *push_p = push;
625 return 0;
626 }
627
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)628 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
629 struct msg_stats *s, bool tx,
630 struct sockmap_options *opt)
631 {
632 struct msghdr msg = {0}, msg_peek = {0};
633 int err, i, flags = MSG_NOSIGNAL;
634 bool drop = opt->drop_expected;
635 bool data = opt->data_test;
636 int iov_alloc_length = iov_length;
637
638 if (!tx && opt->check_recved_len)
639 iov_alloc_length *= 2;
640
641 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
642 if (err)
643 goto out_errno;
644 if (peek_flag) {
645 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
646 if (err)
647 goto out_errno;
648 }
649
650 if (tx) {
651 clock_gettime(CLOCK_MONOTONIC, &s->start);
652 for (i = 0; i < cnt; i++) {
653 int sent;
654
655 errno = 0;
656 sent = sendmsg(fd, &msg, flags);
657
658 if (!drop && sent < 0) {
659 if (opt->tx_wait_mem && errno == EACCES) {
660 errno = 0;
661 goto out_errno;
662 }
663 perror("sendmsg loop error");
664 goto out_errno;
665 } else if (drop && sent >= 0) {
666 fprintf(stderr,
667 "sendmsg loop error expected: %i errno %i\n",
668 sent, errno);
669 errno = -EIO;
670 goto out_errno;
671 }
672 if (sent > 0)
673 s->bytes_sent += sent;
674 }
675 clock_gettime(CLOCK_MONOTONIC, &s->end);
676 } else {
677 float total_bytes, txmsg_pop_total, txmsg_push_total;
678 int slct, recvp = 0, recv, max_fd = fd;
679 int fd_flags = O_NONBLOCK;
680 struct timeval timeout;
681 unsigned char k = 0;
682 int bytes_cnt = 0;
683 int check_cnt = 0;
684 int push = 0;
685 fd_set w;
686
687 fcntl(fd, fd_flags);
688 /* Account for pop bytes noting each iteration of apply will
689 * call msg_pop_data helper so we need to account for this
690 * by calculating the number of apply iterations. Note user
691 * of the tool can create cases where no data is sent by
692 * manipulating pop/push/pull/etc. For example txmsg_apply 1
693 * with txmsg_pop 1 will try to apply 1B at a time but each
694 * iteration will then pop 1B so no data will ever be sent.
695 * This is really only useful for testing edge cases in code
696 * paths.
697 */
698 total_bytes = (float)iov_length * (float)cnt;
699 if (!opt->sendpage)
700 total_bytes *= (float)iov_count;
701 if (txmsg_apply) {
702 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply);
703 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
704 } else {
705 txmsg_push_total = txmsg_end_push * cnt;
706 txmsg_pop_total = txmsg_pop * cnt;
707 }
708 total_bytes += txmsg_push_total;
709 total_bytes -= txmsg_pop_total;
710 if (data) {
711 msg_verify_date_prep();
712 push = verify_push_len;
713 }
714 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
715 if (err < 0)
716 perror("recv start time");
717 while (s->bytes_recvd < total_bytes) {
718 if (txmsg_cork) {
719 timeout.tv_sec = 0;
720 timeout.tv_usec = 300000;
721 } else {
722 timeout.tv_sec = 3;
723 timeout.tv_usec = 0;
724 }
725
726 /* FD sets */
727 FD_ZERO(&w);
728 FD_SET(fd, &w);
729
730 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
731 if (slct == -1) {
732 perror("select()");
733 clock_gettime(CLOCK_MONOTONIC, &s->end);
734 goto out_errno;
735 } else if (!slct) {
736 if (opt->verbose)
737 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
738 errno = -EIO;
739 clock_gettime(CLOCK_MONOTONIC, &s->end);
740 goto out_errno;
741 }
742
743 if (opt->tx_wait_mem) {
744 FD_ZERO(&w);
745 FD_SET(fd, &w);
746 slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
747 errno = 0;
748 close(fd);
749 goto out_errno;
750 }
751
752 errno = 0;
753 if (peek_flag) {
754 flags |= MSG_PEEK;
755 recvp = recvmsg(fd, &msg_peek, flags);
756 if (recvp < 0) {
757 if (errno != EWOULDBLOCK) {
758 clock_gettime(CLOCK_MONOTONIC, &s->end);
759 goto out_errno;
760 }
761 }
762 flags = 0;
763 }
764
765 recv = recvmsg(fd, &msg, flags);
766 if (recv < 0) {
767 if (errno != EWOULDBLOCK) {
768 clock_gettime(CLOCK_MONOTONIC, &s->end);
769 perror("recv failed()");
770 goto out_errno;
771 }
772 }
773
774 if (recv > 0)
775 s->bytes_recvd += recv;
776
777 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
778 errno = EMSGSIZE;
779 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
780 s->bytes_recvd, total_bytes);
781 goto out_errno;
782 }
783
784 if (data) {
785 int chunk_sz = opt->sendpage ?
786 iov_length :
787 iov_length * iov_count;
788
789 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt,
790 &check_cnt, &push);
791 if (errno) {
792 perror("data verify msg failed");
793 goto out_errno;
794 }
795 if (recvp) {
796 errno = msg_verify_data(&msg_peek,
797 recvp,
798 chunk_sz,
799 &k,
800 &bytes_cnt,
801 &check_cnt,
802 &push);
803 if (errno) {
804 perror("data verify msg_peek failed");
805 goto out_errno;
806 }
807 }
808 }
809 }
810 clock_gettime(CLOCK_MONOTONIC, &s->end);
811 }
812
813 msg_free_iov(&msg);
814 msg_free_iov(&msg_peek);
815 return err;
816 out_errno:
817 msg_free_iov(&msg);
818 msg_free_iov(&msg_peek);
819 return errno;
820 }
821
822 static float giga = 1000000000;
823
sentBps(struct msg_stats s)824 static inline float sentBps(struct msg_stats s)
825 {
826 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
827 }
828
recvdBps(struct msg_stats s)829 static inline float recvdBps(struct msg_stats s)
830 {
831 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
832 }
833
sendmsg_test(struct sockmap_options * opt)834 static int sendmsg_test(struct sockmap_options *opt)
835 {
836 float sent_Bps = 0, recvd_Bps = 0;
837 int rx_fd, txpid, rxpid, err = 0;
838 struct msg_stats s = {0};
839 int iov_count = opt->iov_count;
840 int iov_buf = opt->iov_length;
841 int rx_status, tx_status;
842 int cnt = opt->rate;
843
844 errno = 0;
845
846 if (opt->base)
847 rx_fd = p1;
848 else
849 rx_fd = p2;
850
851 if (ktls) {
852 /* Redirecting into non-TLS socket which sends into a TLS
853 * socket is not a valid test. So in this case lets not
854 * enable kTLS but still run the test.
855 */
856 if (!txmsg_redir || txmsg_ingress) {
857 err = sockmap_init_ktls(opt->verbose, rx_fd);
858 if (err)
859 return err;
860 }
861 err = sockmap_init_ktls(opt->verbose, c1);
862 if (err)
863 return err;
864 }
865
866 if (opt->tx_wait_mem) {
867 struct timeval timeout;
868 int rxtx_buf_len = 1024;
869
870 timeout.tv_sec = 3;
871 timeout.tv_usec = 0;
872
873 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
874 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
875 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
876 if (err) {
877 perror("setsockopt failed()");
878 return errno;
879 }
880 }
881
882 rxpid = fork();
883 if (rxpid == 0) {
884 if (opt->drop_expected || txmsg_ktls_skb_drop)
885 _exit(0);
886
887 if (!iov_buf) /* zero bytes sent case */
888 _exit(0);
889
890 if (opt->sendpage)
891 iov_count = 1;
892 err = msg_loop(rx_fd, iov_count, iov_buf,
893 cnt, &s, false, opt);
894 if (opt->verbose > 1)
895 fprintf(stderr,
896 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
897 iov_count, iov_buf, cnt, err);
898 if (s.end.tv_sec - s.start.tv_sec) {
899 sent_Bps = sentBps(s);
900 recvd_Bps = recvdBps(s);
901 }
902 if (opt->verbose > 1)
903 fprintf(stdout,
904 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
905 s.bytes_sent, sent_Bps, sent_Bps/giga,
906 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
907 peek_flag ? "(peek_msg)" : "");
908 if (err && err != -EDATAINTEGRITY && txmsg_cork)
909 err = 0;
910 exit(err ? 1 : 0);
911 } else if (rxpid == -1) {
912 perror("msg_loop_rx");
913 return errno;
914 }
915
916 if (opt->tx_wait_mem)
917 close(c2);
918
919 txpid = fork();
920 if (txpid == 0) {
921 if (opt->sendpage)
922 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
923 else
924 err = msg_loop(c1, iov_count, iov_buf,
925 cnt, &s, true, opt);
926
927 if (err)
928 fprintf(stderr,
929 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
930 iov_count, iov_buf, cnt, err);
931 if (s.end.tv_sec - s.start.tv_sec) {
932 sent_Bps = sentBps(s);
933 recvd_Bps = recvdBps(s);
934 }
935 if (opt->verbose > 1)
936 fprintf(stdout,
937 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
938 s.bytes_sent, sent_Bps, sent_Bps/giga,
939 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
940 exit(err ? 1 : 0);
941 } else if (txpid == -1) {
942 perror("msg_loop_tx");
943 return errno;
944 }
945
946 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
947 assert(waitpid(txpid, &tx_status, 0) == txpid);
948 if (WIFEXITED(rx_status)) {
949 err = WEXITSTATUS(rx_status);
950 if (err) {
951 fprintf(stderr, "rx thread exited with err %d.\n", err);
952 goto out;
953 }
954 }
955 if (WIFEXITED(tx_status)) {
956 err = WEXITSTATUS(tx_status);
957 if (err)
958 fprintf(stderr, "tx thread exited with err %d.\n", err);
959 }
960 out:
961 return err;
962 }
963
forever_ping_pong(int rate,struct sockmap_options * opt)964 static int forever_ping_pong(int rate, struct sockmap_options *opt)
965 {
966 struct timeval timeout;
967 char buf[1024] = {0};
968 int sc;
969
970 timeout.tv_sec = 10;
971 timeout.tv_usec = 0;
972
973 /* Ping/Pong data from client to server */
974 sc = send(c1, buf, sizeof(buf), 0);
975 if (sc < 0) {
976 perror("send failed()");
977 return sc;
978 }
979
980 do {
981 int s, rc, i, max_fd = p2;
982 fd_set w;
983
984 /* FD sets */
985 FD_ZERO(&w);
986 FD_SET(c1, &w);
987 FD_SET(c2, &w);
988 FD_SET(p1, &w);
989 FD_SET(p2, &w);
990
991 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
992 if (s == -1) {
993 perror("select()");
994 break;
995 } else if (!s) {
996 fprintf(stderr, "unexpected timeout\n");
997 break;
998 }
999
1000 for (i = 0; i <= max_fd && s > 0; ++i) {
1001 if (!FD_ISSET(i, &w))
1002 continue;
1003
1004 s--;
1005
1006 rc = recv(i, buf, sizeof(buf), 0);
1007 if (rc < 0) {
1008 if (errno != EWOULDBLOCK) {
1009 perror("recv failed()");
1010 return rc;
1011 }
1012 }
1013
1014 if (rc == 0) {
1015 close(i);
1016 break;
1017 }
1018
1019 sc = send(i, buf, rc, 0);
1020 if (sc < 0) {
1021 perror("send failed()");
1022 return sc;
1023 }
1024 }
1025
1026 if (rate)
1027 sleep(rate);
1028
1029 if (opt->verbose) {
1030 printf(".");
1031 fflush(stdout);
1032
1033 }
1034 } while (running);
1035
1036 return 0;
1037 }
1038
1039 enum {
1040 SELFTESTS,
1041 PING_PONG,
1042 SENDMSG,
1043 BASE,
1044 BASE_SENDPAGE,
1045 SENDPAGE,
1046 };
1047
run_options(struct sockmap_options * options,int cg_fd,int test)1048 static int run_options(struct sockmap_options *options, int cg_fd, int test)
1049 {
1050 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
1051
1052 /* If base test skip BPF setup */
1053 if (test == BASE || test == BASE_SENDPAGE)
1054 goto run;
1055
1056 /* Attach programs to sockmap */
1057 if (!txmsg_omit_skb_parser) {
1058 err = bpf_prog_attach(prog_fd[0], map_fd[0],
1059 BPF_SK_SKB_STREAM_PARSER, 0);
1060 if (err) {
1061 fprintf(stderr,
1062 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
1063 prog_fd[0], map_fd[0], err, strerror(errno));
1064 return err;
1065 }
1066 }
1067
1068 err = bpf_prog_attach(prog_fd[1], map_fd[0],
1069 BPF_SK_SKB_STREAM_VERDICT, 0);
1070 if (err) {
1071 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
1072 err, strerror(errno));
1073 return err;
1074 }
1075
1076 /* Attach programs to TLS sockmap */
1077 if (txmsg_ktls_skb) {
1078 if (!txmsg_omit_skb_parser) {
1079 err = bpf_prog_attach(prog_fd[0], map_fd[8],
1080 BPF_SK_SKB_STREAM_PARSER, 0);
1081 if (err) {
1082 fprintf(stderr,
1083 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
1084 prog_fd[0], map_fd[8], err, strerror(errno));
1085 return err;
1086 }
1087 }
1088
1089 err = bpf_prog_attach(prog_fd[2], map_fd[8],
1090 BPF_SK_SKB_STREAM_VERDICT, 0);
1091 if (err) {
1092 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
1093 err, strerror(errno));
1094 return err;
1095 }
1096 }
1097
1098 /* Attach to cgroups */
1099 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
1100 if (err) {
1101 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
1102 err, strerror(errno));
1103 return err;
1104 }
1105
1106 run:
1107 err = sockmap_init_sockets(options->verbose);
1108 if (err) {
1109 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
1110 goto out;
1111 }
1112
1113 /* Attach txmsg program to sockmap */
1114 if (txmsg_pass)
1115 tx_prog_fd = prog_fd[4];
1116 else if (txmsg_redir)
1117 tx_prog_fd = prog_fd[5];
1118 else if (txmsg_apply)
1119 tx_prog_fd = prog_fd[6];
1120 else if (txmsg_cork)
1121 tx_prog_fd = prog_fd[7];
1122 else if (txmsg_drop)
1123 tx_prog_fd = prog_fd[8];
1124 else
1125 tx_prog_fd = 0;
1126
1127 if (tx_prog_fd) {
1128 int redir_fd, i = 0;
1129
1130 err = bpf_prog_attach(tx_prog_fd,
1131 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1132 if (err) {
1133 fprintf(stderr,
1134 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1135 err, strerror(errno));
1136 goto out;
1137 }
1138
1139 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1140 if (err) {
1141 fprintf(stderr,
1142 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1143 err, strerror(errno));
1144 goto out;
1145 }
1146
1147 if (txmsg_redir)
1148 redir_fd = c2;
1149 else
1150 redir_fd = c1;
1151
1152 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1153 if (err) {
1154 fprintf(stderr,
1155 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1156 err, strerror(errno));
1157 goto out;
1158 }
1159
1160 if (txmsg_apply) {
1161 err = bpf_map_update_elem(map_fd[3],
1162 &i, &txmsg_apply, BPF_ANY);
1163 if (err) {
1164 fprintf(stderr,
1165 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1166 err, strerror(errno));
1167 goto out;
1168 }
1169 }
1170
1171 if (txmsg_cork) {
1172 err = bpf_map_update_elem(map_fd[4],
1173 &i, &txmsg_cork, BPF_ANY);
1174 if (err) {
1175 fprintf(stderr,
1176 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1177 err, strerror(errno));
1178 goto out;
1179 }
1180 }
1181
1182 if (txmsg_start) {
1183 err = bpf_map_update_elem(map_fd[5],
1184 &i, &txmsg_start, BPF_ANY);
1185 if (err) {
1186 fprintf(stderr,
1187 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1188 err, strerror(errno));
1189 goto out;
1190 }
1191 }
1192
1193 if (txmsg_end) {
1194 i = 1;
1195 err = bpf_map_update_elem(map_fd[5],
1196 &i, &txmsg_end, BPF_ANY);
1197 if (err) {
1198 fprintf(stderr,
1199 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1200 err, strerror(errno));
1201 goto out;
1202 }
1203 }
1204
1205 if (txmsg_start_push) {
1206 i = 2;
1207 err = bpf_map_update_elem(map_fd[5],
1208 &i, &txmsg_start_push, BPF_ANY);
1209 if (err) {
1210 fprintf(stderr,
1211 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1212 err, strerror(errno));
1213 goto out;
1214 }
1215 }
1216
1217 if (txmsg_end_push) {
1218 i = 3;
1219 err = bpf_map_update_elem(map_fd[5],
1220 &i, &txmsg_end_push, BPF_ANY);
1221 if (err) {
1222 fprintf(stderr,
1223 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1224 txmsg_end_push, i, err, strerror(errno));
1225 goto out;
1226 }
1227 }
1228
1229 if (txmsg_start_pop) {
1230 i = 4;
1231 err = bpf_map_update_elem(map_fd[5],
1232 &i, &txmsg_start_pop, BPF_ANY);
1233 if (err) {
1234 fprintf(stderr,
1235 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1236 txmsg_start_pop, i, err, strerror(errno));
1237 goto out;
1238 }
1239 } else {
1240 i = 4;
1241 bpf_map_update_elem(map_fd[5],
1242 &i, &txmsg_start_pop, BPF_ANY);
1243 }
1244
1245 if (txmsg_pop) {
1246 i = 5;
1247 err = bpf_map_update_elem(map_fd[5],
1248 &i, &txmsg_pop, BPF_ANY);
1249 if (err) {
1250 fprintf(stderr,
1251 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1252 txmsg_pop, i, err, strerror(errno));
1253 goto out;
1254 }
1255 } else {
1256 i = 5;
1257 bpf_map_update_elem(map_fd[5],
1258 &i, &txmsg_pop, BPF_ANY);
1259
1260 }
1261
1262 if (txmsg_ingress) {
1263 int in = BPF_F_INGRESS;
1264
1265 i = 0;
1266 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1267 if (err) {
1268 fprintf(stderr,
1269 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1270 err, strerror(errno));
1271 }
1272 i = 1;
1273 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1274 if (err) {
1275 fprintf(stderr,
1276 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1277 err, strerror(errno));
1278 }
1279 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1280 if (err) {
1281 fprintf(stderr,
1282 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1283 err, strerror(errno));
1284 }
1285
1286 i = 2;
1287 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1288 if (err) {
1289 fprintf(stderr,
1290 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1291 err, strerror(errno));
1292 }
1293 }
1294
1295 if (txmsg_ktls_skb) {
1296 int ingress = BPF_F_INGRESS;
1297
1298 i = 0;
1299 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1300 if (err) {
1301 fprintf(stderr,
1302 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1303 err, strerror(errno));
1304 }
1305
1306 if (txmsg_ktls_skb_redir) {
1307 i = 1;
1308 err = bpf_map_update_elem(map_fd[7],
1309 &i, &ingress, BPF_ANY);
1310 if (err) {
1311 fprintf(stderr,
1312 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1313 err, strerror(errno));
1314 }
1315 }
1316
1317 if (txmsg_ktls_skb_drop) {
1318 i = 1;
1319 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1320 }
1321 }
1322
1323 if (txmsg_redir_skb) {
1324 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1325 p2 : p1;
1326 int ingress = BPF_F_INGRESS;
1327
1328 i = 0;
1329 err = bpf_map_update_elem(map_fd[7],
1330 &i, &ingress, BPF_ANY);
1331 if (err) {
1332 fprintf(stderr,
1333 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1334 err, strerror(errno));
1335 }
1336
1337 i = 3;
1338 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1339 if (err) {
1340 fprintf(stderr,
1341 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1342 err, strerror(errno));
1343 }
1344 }
1345 }
1346
1347 if (skb_use_parser) {
1348 i = 2;
1349 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1350 }
1351
1352 if (txmsg_drop)
1353 options->drop_expected = true;
1354
1355 if (test == PING_PONG)
1356 err = forever_ping_pong(options->rate, options);
1357 else if (test == SENDMSG) {
1358 options->base = false;
1359 options->sendpage = false;
1360 err = sendmsg_test(options);
1361 } else if (test == SENDPAGE) {
1362 options->base = false;
1363 options->sendpage = true;
1364 err = sendmsg_test(options);
1365 } else if (test == BASE) {
1366 options->base = true;
1367 options->sendpage = false;
1368 err = sendmsg_test(options);
1369 } else if (test == BASE_SENDPAGE) {
1370 options->base = true;
1371 options->sendpage = true;
1372 err = sendmsg_test(options);
1373 } else
1374 fprintf(stderr, "unknown test\n");
1375 out:
1376 /* Detatch and zero all the maps */
1377 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1378 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1379 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1380 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1381 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1382
1383 if (tx_prog_fd >= 0)
1384 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1385
1386 for (i = 0; i < 8; i++) {
1387 key = next_key = 0;
1388 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1389 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1390 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1391 key = next_key;
1392 }
1393 }
1394
1395 close(s1);
1396 close(s2);
1397 close(p1);
1398 close(p2);
1399 close(c1);
1400 close(c2);
1401 return err;
1402 }
1403
test_to_str(int test)1404 static char *test_to_str(int test)
1405 {
1406 switch (test) {
1407 case SENDMSG:
1408 return "sendmsg";
1409 case SENDPAGE:
1410 return "sendpage";
1411 }
1412 return "unknown";
1413 }
1414
append_str(char * dst,const char * src,size_t dst_cap)1415 static void append_str(char *dst, const char *src, size_t dst_cap)
1416 {
1417 size_t avail = dst_cap - strlen(dst);
1418
1419 if (avail <= 1) /* just zero byte could be written */
1420 return;
1421
1422 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1423 }
1424
1425 #define OPTSTRING 60
test_options(char * options)1426 static void test_options(char *options)
1427 {
1428 char tstr[OPTSTRING];
1429
1430 memset(options, 0, OPTSTRING);
1431
1432 if (txmsg_pass)
1433 append_str(options, "pass,", OPTSTRING);
1434 if (txmsg_redir)
1435 append_str(options, "redir,", OPTSTRING);
1436 if (txmsg_drop)
1437 append_str(options, "drop,", OPTSTRING);
1438 if (txmsg_apply) {
1439 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1440 append_str(options, tstr, OPTSTRING);
1441 }
1442 if (txmsg_cork) {
1443 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1444 append_str(options, tstr, OPTSTRING);
1445 }
1446 if (txmsg_start) {
1447 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1448 append_str(options, tstr, OPTSTRING);
1449 }
1450 if (txmsg_end) {
1451 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1452 append_str(options, tstr, OPTSTRING);
1453 }
1454 if (txmsg_start_pop) {
1455 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1456 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1457 append_str(options, tstr, OPTSTRING);
1458 }
1459 if (txmsg_ingress)
1460 append_str(options, "ingress,", OPTSTRING);
1461 if (txmsg_redir_skb)
1462 append_str(options, "redir_skb,", OPTSTRING);
1463 if (txmsg_ktls_skb)
1464 append_str(options, "ktls_skb,", OPTSTRING);
1465 if (ktls)
1466 append_str(options, "ktls,", OPTSTRING);
1467 if (peek_flag)
1468 append_str(options, "peek,", OPTSTRING);
1469 }
1470
__test_exec(int cgrp,int test,struct sockmap_options * opt)1471 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1472 {
1473 char *options = calloc(OPTSTRING, sizeof(char));
1474 int err;
1475
1476 if (test == SENDPAGE)
1477 opt->sendpage = true;
1478 else
1479 opt->sendpage = false;
1480
1481 if (txmsg_drop)
1482 opt->drop_expected = true;
1483 else
1484 opt->drop_expected = false;
1485
1486 test_options(options);
1487
1488 if (opt->verbose) {
1489 fprintf(stdout,
1490 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1491 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1492 test_to_str(test), options);
1493 fflush(stdout);
1494 }
1495 err = run_options(opt, cgrp, test);
1496 if (opt->verbose)
1497 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1498 test_cnt++;
1499 !err ? passed++ : failed++;
1500 free(options);
1501 return err;
1502 }
1503
test_exec(int cgrp,struct sockmap_options * opt)1504 static void test_exec(int cgrp, struct sockmap_options *opt)
1505 {
1506 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1507 int err;
1508
1509 if (type == 0) {
1510 test_start();
1511 err = __test_exec(cgrp, SENDMSG, opt);
1512 if (err)
1513 test_fail();
1514 } else {
1515 test_start();
1516 err = __test_exec(cgrp, SENDPAGE, opt);
1517 if (err)
1518 test_fail();
1519 }
1520 }
1521
test_send_one(struct sockmap_options * opt,int cgrp)1522 static void test_send_one(struct sockmap_options *opt, int cgrp)
1523 {
1524 opt->iov_length = 1;
1525 opt->iov_count = 1;
1526 opt->rate = 1;
1527 test_exec(cgrp, opt);
1528
1529 opt->iov_length = 1;
1530 opt->iov_count = 1024;
1531 opt->rate = 1;
1532 test_exec(cgrp, opt);
1533
1534 opt->iov_length = 1024;
1535 opt->iov_count = 1;
1536 opt->rate = 1;
1537 test_exec(cgrp, opt);
1538
1539 }
1540
test_send_many(struct sockmap_options * opt,int cgrp)1541 static void test_send_many(struct sockmap_options *opt, int cgrp)
1542 {
1543 opt->iov_length = 3;
1544 opt->iov_count = 1;
1545 opt->rate = 512;
1546 test_exec(cgrp, opt);
1547
1548 opt->rate = 100;
1549 opt->iov_count = 1;
1550 opt->iov_length = 5;
1551 test_exec(cgrp, opt);
1552 }
1553
test_send_large(struct sockmap_options * opt,int cgrp)1554 static void test_send_large(struct sockmap_options *opt, int cgrp)
1555 {
1556 opt->iov_length = 8192;
1557 opt->iov_count = 32;
1558 opt->rate = 2;
1559 test_exec(cgrp, opt);
1560 }
1561
test_send(struct sockmap_options * opt,int cgrp)1562 static void test_send(struct sockmap_options *opt, int cgrp)
1563 {
1564 test_send_one(opt, cgrp);
1565 test_send_many(opt, cgrp);
1566 test_send_large(opt, cgrp);
1567 sched_yield();
1568 }
1569
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1570 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1571 {
1572 /* Test small and large iov_count values with pass/redir/apply/cork */
1573 txmsg_pass = 1;
1574 test_send(opt, cgrp);
1575 }
1576
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1577 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1578 {
1579 txmsg_redir = 1;
1580 test_send(opt, cgrp);
1581 }
1582
test_txmsg_redir_wait_sndmem(int cgrp,struct sockmap_options * opt)1583 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1584 {
1585 txmsg_redir = 1;
1586 opt->tx_wait_mem = true;
1587 test_send_large(opt, cgrp);
1588 opt->tx_wait_mem = false;
1589 }
1590
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1591 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1592 {
1593 txmsg_drop = 1;
1594 test_send(opt, cgrp);
1595 }
1596
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1597 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1598 {
1599 txmsg_pass = txmsg_drop = 0;
1600 txmsg_ingress = txmsg_redir = 1;
1601 test_send(opt, cgrp);
1602 }
1603
test_txmsg_skb(int cgrp,struct sockmap_options * opt)1604 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1605 {
1606 bool data = opt->data_test;
1607 int k = ktls;
1608
1609 opt->data_test = true;
1610 ktls = 1;
1611
1612 txmsg_pass = txmsg_drop = 0;
1613 txmsg_ingress = txmsg_redir = 0;
1614 txmsg_ktls_skb = 1;
1615 txmsg_pass = 1;
1616
1617 /* Using data verification so ensure iov layout is
1618 * expected from test receiver side. e.g. has enough
1619 * bytes to write test code.
1620 */
1621 opt->iov_length = 100;
1622 opt->iov_count = 1;
1623 opt->rate = 1;
1624 test_exec(cgrp, opt);
1625
1626 txmsg_ktls_skb_drop = 1;
1627 test_exec(cgrp, opt);
1628
1629 txmsg_ktls_skb_drop = 0;
1630 txmsg_ktls_skb_redir = 1;
1631 test_exec(cgrp, opt);
1632 txmsg_ktls_skb_redir = 0;
1633
1634 /* Tests that omit skb_parser */
1635 txmsg_omit_skb_parser = 1;
1636 ktls = 0;
1637 txmsg_ktls_skb = 0;
1638 test_exec(cgrp, opt);
1639
1640 txmsg_ktls_skb_drop = 1;
1641 test_exec(cgrp, opt);
1642 txmsg_ktls_skb_drop = 0;
1643
1644 txmsg_ktls_skb_redir = 1;
1645 test_exec(cgrp, opt);
1646
1647 ktls = 1;
1648 test_exec(cgrp, opt);
1649 txmsg_omit_skb_parser = 0;
1650
1651 opt->data_test = data;
1652 ktls = k;
1653 }
1654
1655 /* Test cork with hung data. This tests poor usage patterns where
1656 * cork can leave data on the ring if user program is buggy and
1657 * doesn't flush them somehow. They do take some time however
1658 * because they wait for a timeout. Test pass, redir and cork with
1659 * apply logic. Use cork size of 4097 with send_large to avoid
1660 * aligning cork size with send size.
1661 */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1662 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1663 {
1664 txmsg_pass = 1;
1665 txmsg_redir = 0;
1666 txmsg_cork = 4097;
1667 txmsg_apply = 4097;
1668 test_send_large(opt, cgrp);
1669
1670 txmsg_pass = 0;
1671 txmsg_redir = 1;
1672 txmsg_apply = 0;
1673 txmsg_cork = 4097;
1674 test_send_large(opt, cgrp);
1675
1676 txmsg_pass = 0;
1677 txmsg_redir = 1;
1678 txmsg_apply = 4097;
1679 txmsg_cork = 4097;
1680 test_send_large(opt, cgrp);
1681 }
1682
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1683 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1684 {
1685 /* Test basic start/end */
1686 txmsg_pass = 1;
1687 txmsg_start = 1;
1688 txmsg_end = 2;
1689 test_send(opt, cgrp);
1690
1691 /* Test >4k pull */
1692 txmsg_pass = 1;
1693 txmsg_start = 4096;
1694 txmsg_end = 9182;
1695 test_send_large(opt, cgrp);
1696
1697 /* Test pull + redirect */
1698 txmsg_redir = 1;
1699 txmsg_start = 1;
1700 txmsg_end = 2;
1701 test_send(opt, cgrp);
1702
1703 /* Test pull + cork */
1704 txmsg_redir = 0;
1705 txmsg_cork = 512;
1706 txmsg_start = 1;
1707 txmsg_end = 2;
1708 test_send_many(opt, cgrp);
1709
1710 /* Test pull + cork + redirect */
1711 txmsg_redir = 1;
1712 txmsg_cork = 512;
1713 txmsg_start = 1;
1714 txmsg_end = 2;
1715 test_send_many(opt, cgrp);
1716 }
1717
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1718 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1719 {
1720 bool data = opt->data_test;
1721
1722 /* Test basic pop */
1723 txmsg_pass = 1;
1724 txmsg_start_pop = 1;
1725 txmsg_pop = 2;
1726 test_send_many(opt, cgrp);
1727
1728 /* Test pop with >4k */
1729 txmsg_pass = 1;
1730 txmsg_start_pop = 4096;
1731 txmsg_pop = 4096;
1732 test_send_large(opt, cgrp);
1733
1734 /* Test pop + redirect */
1735 txmsg_redir = 1;
1736 txmsg_start_pop = 1;
1737 txmsg_pop = 2;
1738 test_send_many(opt, cgrp);
1739
1740 /* TODO: Test for pop + cork should be different,
1741 * - It makes the layout of the received data difficult
1742 * - It makes it hard to calculate the total_bytes in the recvmsg
1743 * Temporarily skip the data integrity test for this case now.
1744 */
1745 opt->data_test = false;
1746 /* Test pop + cork */
1747 txmsg_redir = 0;
1748 txmsg_cork = 512;
1749 txmsg_start_pop = 1;
1750 txmsg_pop = 2;
1751 test_send_many(opt, cgrp);
1752
1753 /* Test pop + redirect + cork */
1754 txmsg_redir = 1;
1755 txmsg_cork = 4;
1756 txmsg_start_pop = 1;
1757 txmsg_pop = 2;
1758 test_send_many(opt, cgrp);
1759 opt->data_test = data;
1760 }
1761
test_txmsg_push(int cgrp,struct sockmap_options * opt)1762 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1763 {
1764 bool data = opt->data_test;
1765
1766 /* Test basic push */
1767 txmsg_pass = 1;
1768 txmsg_start_push = 1;
1769 txmsg_end_push = 1;
1770 test_send(opt, cgrp);
1771
1772 /* Test push 4kB >4k */
1773 txmsg_pass = 1;
1774 txmsg_start_push = 4096;
1775 txmsg_end_push = 4096;
1776 test_send_large(opt, cgrp);
1777
1778 /* Test push + redirect */
1779 txmsg_redir = 1;
1780 txmsg_start_push = 1;
1781 txmsg_end_push = 2;
1782 test_send_many(opt, cgrp);
1783
1784 /* TODO: Test for push + cork should be different,
1785 * - It makes the layout of the received data difficult
1786 * - It makes it hard to calculate the total_bytes in the recvmsg
1787 * Temporarily skip the data integrity test for this case now.
1788 */
1789 opt->data_test = false;
1790 /* Test push + cork */
1791 txmsg_redir = 0;
1792 txmsg_cork = 512;
1793 txmsg_start_push = 1;
1794 txmsg_end_push = 2;
1795 test_send_many(opt, cgrp);
1796 opt->data_test = data;
1797 }
1798
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1799 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1800 {
1801 txmsg_pass = 1;
1802 txmsg_start_push = 1;
1803 txmsg_end_push = 10;
1804 txmsg_start_pop = 5;
1805 txmsg_pop = 4;
1806 test_send_large(opt, cgrp);
1807 }
1808
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1809 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1810 {
1811 txmsg_pass = 1;
1812 txmsg_redir = 0;
1813 txmsg_ingress = 0;
1814 txmsg_apply = 1;
1815 txmsg_cork = 0;
1816 test_send_one(opt, cgrp);
1817
1818 txmsg_pass = 0;
1819 txmsg_redir = 1;
1820 txmsg_ingress = 0;
1821 txmsg_apply = 1;
1822 txmsg_cork = 0;
1823 test_send_one(opt, cgrp);
1824
1825 txmsg_pass = 0;
1826 txmsg_redir = 1;
1827 txmsg_ingress = 1;
1828 txmsg_apply = 1;
1829 txmsg_cork = 0;
1830 test_send_one(opt, cgrp);
1831
1832 txmsg_pass = 1;
1833 txmsg_redir = 0;
1834 txmsg_ingress = 0;
1835 txmsg_apply = 1024;
1836 txmsg_cork = 0;
1837 test_send_large(opt, cgrp);
1838
1839 txmsg_pass = 0;
1840 txmsg_redir = 1;
1841 txmsg_ingress = 0;
1842 txmsg_apply = 1024;
1843 txmsg_cork = 0;
1844 test_send_large(opt, cgrp);
1845
1846 txmsg_pass = 0;
1847 txmsg_redir = 1;
1848 txmsg_ingress = 1;
1849 txmsg_apply = 1024;
1850 txmsg_cork = 0;
1851 test_send_large(opt, cgrp);
1852 }
1853
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1854 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1855 {
1856 txmsg_pass = 1;
1857 txmsg_redir = 0;
1858 txmsg_apply = 0;
1859 txmsg_cork = 1;
1860 test_send(opt, cgrp);
1861
1862 txmsg_pass = 1;
1863 txmsg_redir = 0;
1864 txmsg_apply = 1;
1865 txmsg_cork = 1;
1866 test_send(opt, cgrp);
1867 }
1868
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)1869 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1870 {
1871 txmsg_pass = 1;
1872 skb_use_parser = 512;
1873 if (ktls == 1)
1874 skb_use_parser = 570;
1875 opt->iov_length = 256;
1876 opt->iov_count = 1;
1877 opt->rate = 2;
1878 test_exec(cgrp, opt);
1879 }
1880
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1881 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1882 {
1883 if (ktls == 1)
1884 return;
1885 skb_use_parser = 10;
1886 opt->iov_length = 20;
1887 opt->iov_count = 1;
1888 opt->rate = 1;
1889 opt->check_recved_len = true;
1890 test_exec(cgrp, opt);
1891 opt->check_recved_len = false;
1892 }
1893
1894 char *map_names[] = {
1895 "sock_map",
1896 "sock_map_txmsg",
1897 "sock_map_redir",
1898 "sock_apply_bytes",
1899 "sock_cork_bytes",
1900 "sock_bytes",
1901 "sock_redir_flags",
1902 "sock_skb_opts",
1903 "tls_sock_map",
1904 };
1905
1906 int prog_attach_type[] = {
1907 BPF_SK_SKB_STREAM_PARSER,
1908 BPF_SK_SKB_STREAM_VERDICT,
1909 BPF_SK_SKB_STREAM_VERDICT,
1910 BPF_CGROUP_SOCK_OPS,
1911 BPF_SK_MSG_VERDICT,
1912 BPF_SK_MSG_VERDICT,
1913 BPF_SK_MSG_VERDICT,
1914 BPF_SK_MSG_VERDICT,
1915 BPF_SK_MSG_VERDICT,
1916 };
1917
1918 int prog_type[] = {
1919 BPF_PROG_TYPE_SK_SKB,
1920 BPF_PROG_TYPE_SK_SKB,
1921 BPF_PROG_TYPE_SK_SKB,
1922 BPF_PROG_TYPE_SOCK_OPS,
1923 BPF_PROG_TYPE_SK_MSG,
1924 BPF_PROG_TYPE_SK_MSG,
1925 BPF_PROG_TYPE_SK_MSG,
1926 BPF_PROG_TYPE_SK_MSG,
1927 BPF_PROG_TYPE_SK_MSG,
1928 };
1929
populate_progs(char * bpf_file)1930 static int populate_progs(char *bpf_file)
1931 {
1932 struct bpf_program *prog;
1933 struct bpf_object *obj;
1934 int i = 0;
1935 long err;
1936
1937 obj = bpf_object__open(bpf_file);
1938 err = libbpf_get_error(obj);
1939 if (err) {
1940 char err_buf[256];
1941
1942 libbpf_strerror(err, err_buf, sizeof(err_buf));
1943 printf("Unable to load eBPF objects in file '%s' : %s\n",
1944 bpf_file, err_buf);
1945 return -1;
1946 }
1947
1948 bpf_object__for_each_program(prog, obj) {
1949 bpf_program__set_type(prog, prog_type[i]);
1950 bpf_program__set_expected_attach_type(prog,
1951 prog_attach_type[i]);
1952 i++;
1953 }
1954
1955 i = bpf_object__load(obj);
1956 i = 0;
1957 bpf_object__for_each_program(prog, obj) {
1958 prog_fd[i] = bpf_program__fd(prog);
1959 i++;
1960 }
1961
1962 for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
1963 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1964 map_fd[i] = bpf_map__fd(maps[i]);
1965 if (map_fd[i] < 0) {
1966 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1967 map_fd[i], strerror(errno));
1968 return -1;
1969 }
1970 }
1971
1972 return 0;
1973 }
1974
1975 struct _test test[] = {
1976 {"txmsg test passthrough", test_txmsg_pass},
1977 {"txmsg test redirect", test_txmsg_redir},
1978 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1979 {"txmsg test drop", test_txmsg_drop},
1980 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1981 {"txmsg test skb", test_txmsg_skb},
1982 {"txmsg test apply", test_txmsg_apply},
1983 {"txmsg test cork", test_txmsg_cork},
1984 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1985 {"txmsg test push_data", test_txmsg_push},
1986 {"txmsg test pull-data", test_txmsg_pull},
1987 {"txmsg test pop-data", test_txmsg_pop},
1988 {"txmsg test push/pop data", test_txmsg_push_pop},
1989 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1990 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1991 };
1992
check_whitelist(struct _test * t,struct sockmap_options * opt)1993 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1994 {
1995 char *entry, *ptr;
1996
1997 if (!opt->whitelist)
1998 return 0;
1999 ptr = strdup(opt->whitelist);
2000 if (!ptr)
2001 return -ENOMEM;
2002 entry = strtok(ptr, ",");
2003 while (entry) {
2004 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2005 strstr(opt->map, entry) != 0 ||
2006 strstr(t->title, entry) != 0)
2007 return 0;
2008 entry = strtok(NULL, ",");
2009 }
2010 return -EINVAL;
2011 }
2012
check_blacklist(struct _test * t,struct sockmap_options * opt)2013 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
2014 {
2015 char *entry, *ptr;
2016
2017 if (!opt->blacklist)
2018 return -EINVAL;
2019 ptr = strdup(opt->blacklist);
2020 if (!ptr)
2021 return -ENOMEM;
2022 entry = strtok(ptr, ",");
2023 while (entry) {
2024 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
2025 strstr(opt->map, entry) != 0 ||
2026 strstr(t->title, entry) != 0)
2027 return 0;
2028 entry = strtok(NULL, ",");
2029 }
2030 return -EINVAL;
2031 }
2032
__test_selftests(int cg_fd,struct sockmap_options * opt)2033 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
2034 {
2035 int i, err;
2036
2037 err = populate_progs(opt->map);
2038 if (err < 0) {
2039 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
2040 return err;
2041 }
2042
2043 /* Tests basic commands and APIs */
2044 for (i = 0; i < ARRAY_SIZE(test); i++) {
2045 struct _test t = test[i];
2046
2047 if (check_whitelist(&t, opt) != 0)
2048 continue;
2049 if (check_blacklist(&t, opt) == 0)
2050 continue;
2051
2052 test_start_subtest(&t, opt);
2053 t.tester(cg_fd, opt);
2054 test_end_subtest();
2055 }
2056
2057 return err;
2058 }
2059
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)2060 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
2061 {
2062 opt->map = BPF_SOCKMAP_FILENAME;
2063 __test_selftests(cg_fd, opt);
2064 }
2065
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)2066 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
2067 {
2068 opt->map = BPF_SOCKHASH_FILENAME;
2069 __test_selftests(cg_fd, opt);
2070 }
2071
test_selftests_ktls(int cg_fd,struct sockmap_options * opt)2072 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
2073 {
2074 opt->map = BPF_SOCKHASH_FILENAME;
2075 opt->prepend = "ktls";
2076 ktls = 1;
2077 __test_selftests(cg_fd, opt);
2078 ktls = 0;
2079 }
2080
test_selftest(int cg_fd,struct sockmap_options * opt)2081 static int test_selftest(int cg_fd, struct sockmap_options *opt)
2082 {
2083
2084 test_selftests_sockmap(cg_fd, opt);
2085 test_selftests_sockhash(cg_fd, opt);
2086 test_selftests_ktls(cg_fd, opt);
2087 test_print_results();
2088 return 0;
2089 }
2090
main(int argc,char ** argv)2091 int main(int argc, char **argv)
2092 {
2093 int iov_count = 1, length = 1024, rate = 1;
2094 struct sockmap_options options = {0};
2095 int opt, longindex, err, cg_fd = 0;
2096 char *bpf_file = BPF_SOCKMAP_FILENAME;
2097 int test = SELFTESTS;
2098 bool cg_created = 0;
2099
2100 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
2101 long_options, &longindex)) != -1) {
2102 switch (opt) {
2103 case 's':
2104 txmsg_start = atoi(optarg);
2105 break;
2106 case 'e':
2107 txmsg_end = atoi(optarg);
2108 break;
2109 case 'p':
2110 txmsg_start_push = atoi(optarg);
2111 break;
2112 case 'q':
2113 txmsg_end_push = atoi(optarg);
2114 break;
2115 case 'w':
2116 txmsg_start_pop = atoi(optarg);
2117 break;
2118 case 'x':
2119 txmsg_pop = atoi(optarg);
2120 break;
2121 case 'a':
2122 txmsg_apply = atoi(optarg);
2123 break;
2124 case 'k':
2125 txmsg_cork = atoi(optarg);
2126 break;
2127 case 'c':
2128 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
2129 if (cg_fd < 0) {
2130 fprintf(stderr,
2131 "ERROR: (%i) open cg path failed: %s\n",
2132 cg_fd, optarg);
2133 return cg_fd;
2134 }
2135 break;
2136 case 'r':
2137 rate = atoi(optarg);
2138 break;
2139 case 'v':
2140 options.verbose = 1;
2141 if (optarg)
2142 options.verbose = atoi(optarg);
2143 break;
2144 case 'i':
2145 iov_count = atoi(optarg);
2146 break;
2147 case 'l':
2148 length = atoi(optarg);
2149 break;
2150 case 'd':
2151 options.data_test = true;
2152 break;
2153 case 't':
2154 if (strcmp(optarg, "ping") == 0) {
2155 test = PING_PONG;
2156 } else if (strcmp(optarg, "sendmsg") == 0) {
2157 test = SENDMSG;
2158 } else if (strcmp(optarg, "base") == 0) {
2159 test = BASE;
2160 } else if (strcmp(optarg, "base_sendpage") == 0) {
2161 test = BASE_SENDPAGE;
2162 } else if (strcmp(optarg, "sendpage") == 0) {
2163 test = SENDPAGE;
2164 } else {
2165 usage(argv);
2166 return -1;
2167 }
2168 break;
2169 case 'n':
2170 options.whitelist = strdup(optarg);
2171 if (!options.whitelist)
2172 return -ENOMEM;
2173 break;
2174 case 'b':
2175 options.blacklist = strdup(optarg);
2176 if (!options.blacklist)
2177 return -ENOMEM;
2178 case 0:
2179 break;
2180 case 'h':
2181 default:
2182 usage(argv);
2183 return -1;
2184 }
2185 }
2186
2187 if (!cg_fd) {
2188 cg_fd = cgroup_setup_and_join(CG_PATH);
2189 if (cg_fd < 0)
2190 return cg_fd;
2191 cg_created = 1;
2192 }
2193
2194 /* Use libbpf 1.0 API mode */
2195 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2196
2197 if (test == SELFTESTS) {
2198 err = test_selftest(cg_fd, &options);
2199 goto out;
2200 }
2201
2202 err = populate_progs(bpf_file);
2203 if (err) {
2204 fprintf(stderr, "populate program: (%s) %s\n",
2205 bpf_file, strerror(errno));
2206 return 1;
2207 }
2208 running = 1;
2209
2210 /* catch SIGINT */
2211 signal(SIGINT, running_handler);
2212
2213 options.iov_count = iov_count;
2214 options.iov_length = length;
2215 options.rate = rate;
2216
2217 err = run_options(&options, cg_fd, test);
2218 out:
2219 if (options.whitelist)
2220 free(options.whitelist);
2221 if (options.blacklist)
2222 free(options.blacklist);
2223 close(cg_fd);
2224 if (cg_created)
2225 cleanup_cgroup_environment();
2226 return err;
2227 }
2228
running_handler(int a)2229 void running_handler(int a)
2230 {
2231 running = 0;
2232 }
2233