116962b24SJohn Fastabend // SPDX-License-Identifier: GPL-2.0 216962b24SJohn Fastabend // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 316962b24SJohn Fastabend #include <stdio.h> 416962b24SJohn Fastabend #include <stdlib.h> 516962b24SJohn Fastabend #include <sys/socket.h> 616962b24SJohn Fastabend #include <sys/ioctl.h> 716962b24SJohn Fastabend #include <sys/select.h> 816962b24SJohn Fastabend #include <netinet/in.h> 916962b24SJohn Fastabend #include <arpa/inet.h> 1016962b24SJohn Fastabend #include <unistd.h> 1116962b24SJohn Fastabend #include <string.h> 1216962b24SJohn Fastabend #include <errno.h> 1316962b24SJohn Fastabend #include <stdbool.h> 1416962b24SJohn Fastabend #include <signal.h> 1516962b24SJohn Fastabend #include <fcntl.h> 1616962b24SJohn Fastabend #include <sys/wait.h> 1716962b24SJohn Fastabend #include <time.h> 1816962b24SJohn Fastabend #include <sched.h> 1916962b24SJohn Fastabend 2016962b24SJohn Fastabend #include <sys/time.h> 2116962b24SJohn Fastabend #include <sys/types.h> 2216962b24SJohn Fastabend #include <sys/sendfile.h> 2316962b24SJohn Fastabend 2416962b24SJohn Fastabend #include <linux/netlink.h> 2516962b24SJohn Fastabend #include <linux/socket.h> 2616962b24SJohn Fastabend #include <linux/sock_diag.h> 2716962b24SJohn Fastabend #include <linux/bpf.h> 2816962b24SJohn Fastabend #include <linux/if_link.h> 29421f4292SDaniel Borkmann #include <linux/tls.h> 3016962b24SJohn Fastabend #include <assert.h> 3116962b24SJohn Fastabend #include <libgen.h> 3216962b24SJohn Fastabend 3316962b24SJohn Fastabend #include <getopt.h> 3416962b24SJohn Fastabend 3516962b24SJohn Fastabend #include <bpf/bpf.h> 3616962b24SJohn Fastabend #include <bpf/libbpf.h> 3716962b24SJohn Fastabend 3816962b24SJohn Fastabend #include "bpf_util.h" 3916962b24SJohn Fastabend #include "cgroup_helpers.h" 4016962b24SJohn Fastabend 4116962b24SJohn Fastabend int running; 4216962b24SJohn Fastabend static void running_handler(int a); 4316962b24SJohn Fastabend 44421f4292SDaniel Borkmann #ifndef TCP_ULP 45421f4292SDaniel Borkmann # define TCP_ULP 31 46421f4292SDaniel Borkmann #endif 47421f4292SDaniel Borkmann #ifndef SOL_TLS 48421f4292SDaniel Borkmann # define SOL_TLS 282 49421f4292SDaniel Borkmann #endif 50421f4292SDaniel Borkmann 5116962b24SJohn Fastabend /* randomly selected ports for testing on lo */ 5216962b24SJohn Fastabend #define S1_PORT 10000 5316962b24SJohn Fastabend #define S2_PORT 10001 5416962b24SJohn Fastabend 55*afef88e6SDaniel Müller #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56*afef88e6SDaniel Müller #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 5716962b24SJohn Fastabend #define CG_PATH "/sockmap" 5816962b24SJohn Fastabend 5916962b24SJohn Fastabend /* global sockets */ 6016962b24SJohn Fastabend int s1, s2, c1, c2, p1, p2; 6116962b24SJohn Fastabend int test_cnt; 6216962b24SJohn Fastabend int passed; 6316962b24SJohn Fastabend int failed; 64463bac5fSJohn Fastabend int map_fd[9]; 65463bac5fSJohn Fastabend struct bpf_map *maps[9]; 6616962b24SJohn Fastabend int prog_fd[11]; 6716962b24SJohn Fastabend 6816962b24SJohn Fastabend int txmsg_pass; 6916962b24SJohn Fastabend int txmsg_redir; 7016962b24SJohn Fastabend int txmsg_drop; 7116962b24SJohn Fastabend int txmsg_apply; 7216962b24SJohn Fastabend int txmsg_cork; 7316962b24SJohn Fastabend int txmsg_start; 7416962b24SJohn Fastabend int txmsg_end; 7584fbfe02SJohn Fastabend int txmsg_start_push; 7684fbfe02SJohn Fastabend int txmsg_end_push; 771ade9abaSJohn Fastabend int txmsg_start_pop; 781ade9abaSJohn Fastabend int txmsg_pop; 7916962b24SJohn Fastabend int txmsg_ingress; 80463bac5fSJohn Fastabend int txmsg_redir_skb; 81463bac5fSJohn Fastabend int txmsg_ktls_skb; 82463bac5fSJohn Fastabend int txmsg_ktls_skb_drop; 83463bac5fSJohn Fastabend int txmsg_ktls_skb_redir; 84e9dd9047SJohn Fastabend int ktls; 85753fb2eeSJohn Fastabend int peek_flag; 8653792fa4SJohn Fastabend int skb_use_parser; 87cdf43c4bSJohn Fastabend int txmsg_omit_skb_parser; 8816962b24SJohn Fastabend 8916962b24SJohn Fastabend static const struct option long_options[] = { 9016962b24SJohn Fastabend {"help", no_argument, NULL, 'h' }, 9116962b24SJohn Fastabend {"cgroup", required_argument, NULL, 'c' }, 9216962b24SJohn Fastabend {"rate", required_argument, NULL, 'r' }, 93b98ca90cSJohn Fastabend {"verbose", optional_argument, NULL, 'v' }, 9416962b24SJohn Fastabend {"iov_count", required_argument, NULL, 'i' }, 9516962b24SJohn Fastabend {"length", required_argument, NULL, 'l' }, 9616962b24SJohn Fastabend {"test", required_argument, NULL, 't' }, 9716962b24SJohn Fastabend {"data_test", no_argument, NULL, 'd' }, 9816962b24SJohn Fastabend {"txmsg", no_argument, &txmsg_pass, 1 }, 9916962b24SJohn Fastabend {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 10016962b24SJohn Fastabend {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 10116962b24SJohn Fastabend {"txmsg_apply", required_argument, NULL, 'a'}, 10216962b24SJohn Fastabend {"txmsg_cork", required_argument, NULL, 'k'}, 10316962b24SJohn Fastabend {"txmsg_start", required_argument, NULL, 's'}, 10416962b24SJohn Fastabend {"txmsg_end", required_argument, NULL, 'e'}, 10584fbfe02SJohn Fastabend {"txmsg_start_push", required_argument, NULL, 'p'}, 10684fbfe02SJohn Fastabend {"txmsg_end_push", required_argument, NULL, 'q'}, 1071ade9abaSJohn Fastabend {"txmsg_start_pop", required_argument, NULL, 'w'}, 1081ade9abaSJohn Fastabend {"txmsg_pop", required_argument, NULL, 'x'}, 10916962b24SJohn Fastabend {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 110463bac5fSJohn Fastabend {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 111e9dd9047SJohn Fastabend {"ktls", no_argument, &ktls, 1 }, 112753fb2eeSJohn Fastabend {"peek", no_argument, &peek_flag, 1 }, 113cdf43c4bSJohn Fastabend {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 114065a74cbSJohn Fastabend {"whitelist", required_argument, NULL, 'n' }, 115a7238f7cSJohn Fastabend {"blacklist", required_argument, NULL, 'b' }, 11616962b24SJohn Fastabend {0, 0, NULL, 0 } 11716962b24SJohn Fastabend }; 11816962b24SJohn Fastabend 119328aa08aSJohn Fastabend struct test_env { 120328aa08aSJohn Fastabend const char *type; 121328aa08aSJohn Fastabend const char *subtest; 12296586dd9SJohn Fastabend const char *prepend; 123328aa08aSJohn Fastabend 124328aa08aSJohn Fastabend int test_num; 125328aa08aSJohn Fastabend int subtest_num; 126328aa08aSJohn Fastabend 127328aa08aSJohn Fastabend int succ_cnt; 128328aa08aSJohn Fastabend int fail_cnt; 129328aa08aSJohn Fastabend int fail_last; 130328aa08aSJohn Fastabend }; 131328aa08aSJohn Fastabend 132328aa08aSJohn Fastabend struct test_env env; 133328aa08aSJohn Fastabend 13496586dd9SJohn Fastabend struct sockmap_options { 13596586dd9SJohn Fastabend int verbose; 13696586dd9SJohn Fastabend bool base; 13796586dd9SJohn Fastabend bool sendpage; 13896586dd9SJohn Fastabend bool data_test; 13996586dd9SJohn Fastabend bool drop_expected; 140d6967214SLiu Jian bool check_recved_len; 14196586dd9SJohn Fastabend int iov_count; 14296586dd9SJohn Fastabend int iov_length; 14396586dd9SJohn Fastabend int rate; 14496586dd9SJohn Fastabend char *map; 14596586dd9SJohn Fastabend char *whitelist; 14696586dd9SJohn Fastabend char *blacklist; 14796586dd9SJohn Fastabend char *prepend; 14896586dd9SJohn Fastabend }; 14996586dd9SJohn Fastabend 15096586dd9SJohn Fastabend struct _test { 15196586dd9SJohn Fastabend char *title; 15296586dd9SJohn Fastabend void (*tester)(int cg_fd, struct sockmap_options *opt); 15396586dd9SJohn Fastabend }; 15496586dd9SJohn Fastabend 155328aa08aSJohn Fastabend static void test_start(void) 156328aa08aSJohn Fastabend { 157328aa08aSJohn Fastabend env.subtest_num++; 158328aa08aSJohn Fastabend } 159328aa08aSJohn Fastabend 160328aa08aSJohn Fastabend static void test_fail(void) 161328aa08aSJohn Fastabend { 162328aa08aSJohn Fastabend env.fail_cnt++; 163328aa08aSJohn Fastabend } 164328aa08aSJohn Fastabend 165328aa08aSJohn Fastabend static void test_pass(void) 166328aa08aSJohn Fastabend { 167328aa08aSJohn Fastabend env.succ_cnt++; 168328aa08aSJohn Fastabend } 169328aa08aSJohn Fastabend 170328aa08aSJohn Fastabend static void test_reset(void) 171328aa08aSJohn Fastabend { 172328aa08aSJohn Fastabend txmsg_start = txmsg_end = 0; 173328aa08aSJohn Fastabend txmsg_start_pop = txmsg_pop = 0; 174328aa08aSJohn Fastabend txmsg_start_push = txmsg_end_push = 0; 175328aa08aSJohn Fastabend txmsg_pass = txmsg_drop = txmsg_redir = 0; 176328aa08aSJohn Fastabend txmsg_apply = txmsg_cork = 0; 177463bac5fSJohn Fastabend txmsg_ingress = txmsg_redir_skb = 0; 178463bac5fSJohn Fastabend txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 179cdf43c4bSJohn Fastabend txmsg_omit_skb_parser = 0; 18053792fa4SJohn Fastabend skb_use_parser = 0; 181328aa08aSJohn Fastabend } 182328aa08aSJohn Fastabend 18396586dd9SJohn Fastabend static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 184328aa08aSJohn Fastabend { 18596586dd9SJohn Fastabend env.type = o->map; 18696586dd9SJohn Fastabend env.subtest = t->title; 18796586dd9SJohn Fastabend env.prepend = o->prepend; 188328aa08aSJohn Fastabend env.test_num++; 189328aa08aSJohn Fastabend env.subtest_num = 0; 190328aa08aSJohn Fastabend env.fail_last = env.fail_cnt; 191328aa08aSJohn Fastabend test_reset(); 192328aa08aSJohn Fastabend return 0; 193328aa08aSJohn Fastabend } 194328aa08aSJohn Fastabend 195328aa08aSJohn Fastabend static void test_end_subtest(void) 196328aa08aSJohn Fastabend { 197328aa08aSJohn Fastabend int error = env.fail_cnt - env.fail_last; 198328aa08aSJohn Fastabend int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 199328aa08aSJohn Fastabend 200328aa08aSJohn Fastabend if (!error) 201328aa08aSJohn Fastabend test_pass(); 202328aa08aSJohn Fastabend 20396586dd9SJohn Fastabend fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 204328aa08aSJohn Fastabend env.test_num, env.subtest_num, 205328aa08aSJohn Fastabend !type ? "sockmap" : "sockhash", 20696586dd9SJohn Fastabend env.prepend ? : "", 207328aa08aSJohn Fastabend env.subtest, error ? "FAIL" : "OK"); 208328aa08aSJohn Fastabend } 209328aa08aSJohn Fastabend 210328aa08aSJohn Fastabend static void test_print_results(void) 211328aa08aSJohn Fastabend { 212328aa08aSJohn Fastabend fprintf(stdout, "Pass: %d Fail: %d\n", 213328aa08aSJohn Fastabend env.succ_cnt, env.fail_cnt); 214328aa08aSJohn Fastabend } 215328aa08aSJohn Fastabend 21616962b24SJohn Fastabend static void usage(char *argv[]) 21716962b24SJohn Fastabend { 21816962b24SJohn Fastabend int i; 21916962b24SJohn Fastabend 22016962b24SJohn Fastabend printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 22116962b24SJohn Fastabend printf(" options:\n"); 22216962b24SJohn Fastabend for (i = 0; long_options[i].name != 0; i++) { 22316962b24SJohn Fastabend printf(" --%-12s", long_options[i].name); 22416962b24SJohn Fastabend if (long_options[i].flag != NULL) 22516962b24SJohn Fastabend printf(" flag (internal value:%d)\n", 22616962b24SJohn Fastabend *long_options[i].flag); 22716962b24SJohn Fastabend else 22816962b24SJohn Fastabend printf(" -%c\n", long_options[i].val); 22916962b24SJohn Fastabend } 23016962b24SJohn Fastabend printf("\n"); 23116962b24SJohn Fastabend } 23216962b24SJohn Fastabend 233e9dd9047SJohn Fastabend char *sock_to_string(int s) 234e9dd9047SJohn Fastabend { 235e9dd9047SJohn Fastabend if (s == c1) 236e9dd9047SJohn Fastabend return "client1"; 237e9dd9047SJohn Fastabend else if (s == c2) 238e9dd9047SJohn Fastabend return "client2"; 239e9dd9047SJohn Fastabend else if (s == s1) 240e9dd9047SJohn Fastabend return "server1"; 241e9dd9047SJohn Fastabend else if (s == s2) 242e9dd9047SJohn Fastabend return "server2"; 243e9dd9047SJohn Fastabend else if (s == p1) 244e9dd9047SJohn Fastabend return "peer1"; 245e9dd9047SJohn Fastabend else if (s == p2) 246e9dd9047SJohn Fastabend return "peer2"; 247e9dd9047SJohn Fastabend else 248e9dd9047SJohn Fastabend return "unknown"; 249e9dd9047SJohn Fastabend } 250e9dd9047SJohn Fastabend 251e9dd9047SJohn Fastabend static int sockmap_init_ktls(int verbose, int s) 252e9dd9047SJohn Fastabend { 253e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_tx = { 254e9dd9047SJohn Fastabend .info = { 255e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION, 256e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128, 257e9dd9047SJohn Fastabend }, 258e9dd9047SJohn Fastabend }; 259e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_rx = { 260e9dd9047SJohn Fastabend .info = { 261e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION, 262e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128, 263e9dd9047SJohn Fastabend }, 264e9dd9047SJohn Fastabend }; 265e9dd9047SJohn Fastabend int so_buf = 6553500; 266e9dd9047SJohn Fastabend int err; 267e9dd9047SJohn Fastabend 268e9dd9047SJohn Fastabend err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 269e9dd9047SJohn Fastabend if (err) { 270e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 271e9dd9047SJohn Fastabend return -EINVAL; 272e9dd9047SJohn Fastabend } 273e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 274e9dd9047SJohn Fastabend if (err) { 275e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 276e9dd9047SJohn Fastabend return -EINVAL; 277e9dd9047SJohn Fastabend } 278e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 279e9dd9047SJohn Fastabend if (err) { 280e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 281e9dd9047SJohn Fastabend return -EINVAL; 282e9dd9047SJohn Fastabend } 283e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 284e9dd9047SJohn Fastabend if (err) { 285e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 286e9dd9047SJohn Fastabend return -EINVAL; 287e9dd9047SJohn Fastabend } 288e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 289e9dd9047SJohn Fastabend if (err) { 290e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 291e9dd9047SJohn Fastabend return -EINVAL; 292e9dd9047SJohn Fastabend } 293e9dd9047SJohn Fastabend 294e9dd9047SJohn Fastabend if (verbose) 295e9dd9047SJohn Fastabend fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 296e9dd9047SJohn Fastabend return 0; 297e9dd9047SJohn Fastabend } 29816962b24SJohn Fastabend static int sockmap_init_sockets(int verbose) 29916962b24SJohn Fastabend { 30016962b24SJohn Fastabend int i, err, one = 1; 30116962b24SJohn Fastabend struct sockaddr_in addr; 30216962b24SJohn Fastabend int *fds[4] = {&s1, &s2, &c1, &c2}; 30316962b24SJohn Fastabend 30416962b24SJohn Fastabend s1 = s2 = p1 = p2 = c1 = c2 = 0; 30516962b24SJohn Fastabend 30616962b24SJohn Fastabend /* Init sockets */ 30716962b24SJohn Fastabend for (i = 0; i < 4; i++) { 30816962b24SJohn Fastabend *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 30916962b24SJohn Fastabend if (*fds[i] < 0) { 31016962b24SJohn Fastabend perror("socket s1 failed()"); 31116962b24SJohn Fastabend return errno; 31216962b24SJohn Fastabend } 31316962b24SJohn Fastabend } 31416962b24SJohn Fastabend 31516962b24SJohn Fastabend /* Allow reuse */ 31616962b24SJohn Fastabend for (i = 0; i < 2; i++) { 31716962b24SJohn Fastabend err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 31816962b24SJohn Fastabend (char *)&one, sizeof(one)); 31916962b24SJohn Fastabend if (err) { 32016962b24SJohn Fastabend perror("setsockopt failed()"); 32116962b24SJohn Fastabend return errno; 32216962b24SJohn Fastabend } 32316962b24SJohn Fastabend } 32416962b24SJohn Fastabend 32516962b24SJohn Fastabend /* Non-blocking sockets */ 32616962b24SJohn Fastabend for (i = 0; i < 2; i++) { 32716962b24SJohn Fastabend err = ioctl(*fds[i], FIONBIO, (char *)&one); 32816962b24SJohn Fastabend if (err < 0) { 32916962b24SJohn Fastabend perror("ioctl s1 failed()"); 33016962b24SJohn Fastabend return errno; 33116962b24SJohn Fastabend } 33216962b24SJohn Fastabend } 33316962b24SJohn Fastabend 33416962b24SJohn Fastabend /* Bind server sockets */ 33516962b24SJohn Fastabend memset(&addr, 0, sizeof(struct sockaddr_in)); 33616962b24SJohn Fastabend addr.sin_family = AF_INET; 33716962b24SJohn Fastabend addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 33816962b24SJohn Fastabend 33916962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 34016962b24SJohn Fastabend err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 34116962b24SJohn Fastabend if (err < 0) { 342e5dc9dd3SJakub Kicinski perror("bind s1 failed()"); 34316962b24SJohn Fastabend return errno; 34416962b24SJohn Fastabend } 34516962b24SJohn Fastabend 34616962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 34716962b24SJohn Fastabend err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 34816962b24SJohn Fastabend if (err < 0) { 349e5dc9dd3SJakub Kicinski perror("bind s2 failed()"); 35016962b24SJohn Fastabend return errno; 35116962b24SJohn Fastabend } 35216962b24SJohn Fastabend 35316962b24SJohn Fastabend /* Listen server sockets */ 35416962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 35516962b24SJohn Fastabend err = listen(s1, 32); 35616962b24SJohn Fastabend if (err < 0) { 357e5dc9dd3SJakub Kicinski perror("listen s1 failed()"); 35816962b24SJohn Fastabend return errno; 35916962b24SJohn Fastabend } 36016962b24SJohn Fastabend 36116962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 36216962b24SJohn Fastabend err = listen(s2, 32); 36316962b24SJohn Fastabend if (err < 0) { 364e5dc9dd3SJakub Kicinski perror("listen s1 failed()"); 36516962b24SJohn Fastabend return errno; 36616962b24SJohn Fastabend } 36716962b24SJohn Fastabend 36816962b24SJohn Fastabend /* Initiate Connect */ 36916962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 37016962b24SJohn Fastabend err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 37116962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 372e5dc9dd3SJakub Kicinski perror("connect c1 failed()"); 37316962b24SJohn Fastabend return errno; 37416962b24SJohn Fastabend } 37516962b24SJohn Fastabend 37616962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 37716962b24SJohn Fastabend err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 37816962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 379e5dc9dd3SJakub Kicinski perror("connect c2 failed()"); 38016962b24SJohn Fastabend return errno; 38116962b24SJohn Fastabend } else if (err < 0) { 38216962b24SJohn Fastabend err = 0; 38316962b24SJohn Fastabend } 38416962b24SJohn Fastabend 38516962b24SJohn Fastabend /* Accept Connecrtions */ 38616962b24SJohn Fastabend p1 = accept(s1, NULL, NULL); 38716962b24SJohn Fastabend if (p1 < 0) { 388e5dc9dd3SJakub Kicinski perror("accept s1 failed()"); 38916962b24SJohn Fastabend return errno; 39016962b24SJohn Fastabend } 39116962b24SJohn Fastabend 39216962b24SJohn Fastabend p2 = accept(s2, NULL, NULL); 39316962b24SJohn Fastabend if (p2 < 0) { 394e5dc9dd3SJakub Kicinski perror("accept s1 failed()"); 39516962b24SJohn Fastabend return errno; 39616962b24SJohn Fastabend } 39716962b24SJohn Fastabend 398b98ca90cSJohn Fastabend if (verbose > 1) { 39916962b24SJohn Fastabend printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 40016962b24SJohn Fastabend printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 40116962b24SJohn Fastabend c1, s1, c2, s2); 40216962b24SJohn Fastabend } 40316962b24SJohn Fastabend return 0; 40416962b24SJohn Fastabend } 40516962b24SJohn Fastabend 40616962b24SJohn Fastabend struct msg_stats { 40716962b24SJohn Fastabend size_t bytes_sent; 40816962b24SJohn Fastabend size_t bytes_recvd; 40916962b24SJohn Fastabend struct timespec start; 41016962b24SJohn Fastabend struct timespec end; 41116962b24SJohn Fastabend }; 41216962b24SJohn Fastabend 41316962b24SJohn Fastabend static int msg_loop_sendpage(int fd, int iov_length, int cnt, 41416962b24SJohn Fastabend struct msg_stats *s, 41516962b24SJohn Fastabend struct sockmap_options *opt) 41616962b24SJohn Fastabend { 41716962b24SJohn Fastabend bool drop = opt->drop_expected; 41816962b24SJohn Fastabend unsigned char k = 0; 41916962b24SJohn Fastabend FILE *file; 42016962b24SJohn Fastabend int i, fp; 42116962b24SJohn Fastabend 422c31dbb1eSLorenz Bauer file = tmpfile(); 4234b67c515SJakub Kicinski if (!file) { 4244b67c515SJakub Kicinski perror("create file for sendpage"); 4254b67c515SJakub Kicinski return 1; 4264b67c515SJakub Kicinski } 42716962b24SJohn Fastabend for (i = 0; i < iov_length * cnt; i++, k++) 42816962b24SJohn Fastabend fwrite(&k, sizeof(char), 1, file); 42916962b24SJohn Fastabend fflush(file); 43016962b24SJohn Fastabend fseek(file, 0, SEEK_SET); 43116962b24SJohn Fastabend 432c31dbb1eSLorenz Bauer fp = fileno(file); 4334b67c515SJakub Kicinski 43416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 43516962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 436248aba1dSJohn Fastabend int sent; 437248aba1dSJohn Fastabend 438248aba1dSJohn Fastabend errno = 0; 439248aba1dSJohn Fastabend sent = sendfile(fd, fp, NULL, iov_length); 44016962b24SJohn Fastabend 44116962b24SJohn Fastabend if (!drop && sent < 0) { 442248aba1dSJohn Fastabend perror("sendpage loop error"); 443c31dbb1eSLorenz Bauer fclose(file); 44416962b24SJohn Fastabend return sent; 44516962b24SJohn Fastabend } else if (drop && sent >= 0) { 446248aba1dSJohn Fastabend printf("sendpage loop error expected: %i errno %i\n", 447248aba1dSJohn Fastabend sent, errno); 448c31dbb1eSLorenz Bauer fclose(file); 44916962b24SJohn Fastabend return -EIO; 45016962b24SJohn Fastabend } 45116962b24SJohn Fastabend 45216962b24SJohn Fastabend if (sent > 0) 45316962b24SJohn Fastabend s->bytes_sent += sent; 45416962b24SJohn Fastabend } 45516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 456c31dbb1eSLorenz Bauer fclose(file); 45716962b24SJohn Fastabend return 0; 45816962b24SJohn Fastabend } 45916962b24SJohn Fastabend 460753fb2eeSJohn Fastabend static void msg_free_iov(struct msghdr *msg) 46116962b24SJohn Fastabend { 462753fb2eeSJohn Fastabend int i; 463753fb2eeSJohn Fastabend 464753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++) 465753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base); 466753fb2eeSJohn Fastabend free(msg->msg_iov); 467753fb2eeSJohn Fastabend msg->msg_iov = NULL; 468753fb2eeSJohn Fastabend msg->msg_iovlen = 0; 469753fb2eeSJohn Fastabend } 470753fb2eeSJohn Fastabend 471753fb2eeSJohn Fastabend static int msg_alloc_iov(struct msghdr *msg, 472753fb2eeSJohn Fastabend int iov_count, int iov_length, 473753fb2eeSJohn Fastabend bool data, bool xmit) 474753fb2eeSJohn Fastabend { 475753fb2eeSJohn Fastabend unsigned char k = 0; 47616962b24SJohn Fastabend struct iovec *iov; 477753fb2eeSJohn Fastabend int i; 47816962b24SJohn Fastabend 47916962b24SJohn Fastabend iov = calloc(iov_count, sizeof(struct iovec)); 48016962b24SJohn Fastabend if (!iov) 48116962b24SJohn Fastabend return errno; 48216962b24SJohn Fastabend 48316962b24SJohn Fastabend for (i = 0; i < iov_count; i++) { 48416962b24SJohn Fastabend unsigned char *d = calloc(iov_length, sizeof(char)); 48516962b24SJohn Fastabend 48616962b24SJohn Fastabend if (!d) { 48716962b24SJohn Fastabend fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 488753fb2eeSJohn Fastabend goto unwind_iov; 48916962b24SJohn Fastabend } 49016962b24SJohn Fastabend iov[i].iov_base = d; 49116962b24SJohn Fastabend iov[i].iov_len = iov_length; 49216962b24SJohn Fastabend 493753fb2eeSJohn Fastabend if (data && xmit) { 49416962b24SJohn Fastabend int j; 49516962b24SJohn Fastabend 49616962b24SJohn Fastabend for (j = 0; j < iov_length; j++) 49716962b24SJohn Fastabend d[j] = k++; 49816962b24SJohn Fastabend } 49916962b24SJohn Fastabend } 50016962b24SJohn Fastabend 501753fb2eeSJohn Fastabend msg->msg_iov = iov; 502753fb2eeSJohn Fastabend msg->msg_iovlen = iov_count; 503753fb2eeSJohn Fastabend 504753fb2eeSJohn Fastabend return 0; 505753fb2eeSJohn Fastabend unwind_iov: 506753fb2eeSJohn Fastabend for (i--; i >= 0 ; i--) 507753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base); 508753fb2eeSJohn Fastabend return -ENOMEM; 509753fb2eeSJohn Fastabend } 510753fb2eeSJohn Fastabend 511753fb2eeSJohn Fastabend static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 512753fb2eeSJohn Fastabend { 513463bac5fSJohn Fastabend int i, j = 0, bytes_cnt = 0; 514753fb2eeSJohn Fastabend unsigned char k = 0; 515753fb2eeSJohn Fastabend 516753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++) { 517753fb2eeSJohn Fastabend unsigned char *d = msg->msg_iov[i].iov_base; 518753fb2eeSJohn Fastabend 519463bac5fSJohn Fastabend /* Special case test for skb ingress + ktls */ 520463bac5fSJohn Fastabend if (i == 0 && txmsg_ktls_skb) { 521463bac5fSJohn Fastabend if (msg->msg_iov[i].iov_len < 4) 522463bac5fSJohn Fastabend return -EIO; 523463bac5fSJohn Fastabend if (memcmp(d, "PASS", 4) != 0) { 524463bac5fSJohn Fastabend fprintf(stderr, 52591274ca5SJohn Fastabend "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 52691274ca5SJohn Fastabend i, 0, d[0], d[1], d[2], d[3]); 527463bac5fSJohn Fastabend return -EIO; 528463bac5fSJohn Fastabend } 52991274ca5SJohn Fastabend j = 4; /* advance index past PASS header */ 530463bac5fSJohn Fastabend } 531463bac5fSJohn Fastabend 532463bac5fSJohn Fastabend for (; j < msg->msg_iov[i].iov_len && size; j++) { 533753fb2eeSJohn Fastabend if (d[j] != k++) { 534753fb2eeSJohn Fastabend fprintf(stderr, 535753fb2eeSJohn Fastabend "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 536753fb2eeSJohn Fastabend i, j, d[j], k - 1, d[j+1], k); 537753fb2eeSJohn Fastabend return -EIO; 538753fb2eeSJohn Fastabend } 539753fb2eeSJohn Fastabend bytes_cnt++; 540753fb2eeSJohn Fastabend if (bytes_cnt == chunk_sz) { 54116962b24SJohn Fastabend k = 0; 542753fb2eeSJohn Fastabend bytes_cnt = 0; 543753fb2eeSJohn Fastabend } 544753fb2eeSJohn Fastabend size--; 545753fb2eeSJohn Fastabend } 546753fb2eeSJohn Fastabend } 547753fb2eeSJohn Fastabend return 0; 548753fb2eeSJohn Fastabend } 549753fb2eeSJohn Fastabend 550753fb2eeSJohn Fastabend static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 551753fb2eeSJohn Fastabend struct msg_stats *s, bool tx, 552753fb2eeSJohn Fastabend struct sockmap_options *opt) 553753fb2eeSJohn Fastabend { 554753fb2eeSJohn Fastabend struct msghdr msg = {0}, msg_peek = {0}; 555753fb2eeSJohn Fastabend int err, i, flags = MSG_NOSIGNAL; 556753fb2eeSJohn Fastabend bool drop = opt->drop_expected; 557753fb2eeSJohn Fastabend bool data = opt->data_test; 558d6967214SLiu Jian int iov_alloc_length = iov_length; 559753fb2eeSJohn Fastabend 560d6967214SLiu Jian if (!tx && opt->check_recved_len) 561d6967214SLiu Jian iov_alloc_length *= 2; 562d6967214SLiu Jian 563d6967214SLiu Jian err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 564753fb2eeSJohn Fastabend if (err) 565753fb2eeSJohn Fastabend goto out_errno; 566753fb2eeSJohn Fastabend if (peek_flag) { 567753fb2eeSJohn Fastabend err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 568753fb2eeSJohn Fastabend if (err) 569753fb2eeSJohn Fastabend goto out_errno; 570753fb2eeSJohn Fastabend } 57116962b24SJohn Fastabend 57216962b24SJohn Fastabend if (tx) { 57316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 57416962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 575248aba1dSJohn Fastabend int sent; 576248aba1dSJohn Fastabend 577248aba1dSJohn Fastabend errno = 0; 578248aba1dSJohn Fastabend sent = sendmsg(fd, &msg, flags); 57916962b24SJohn Fastabend 58016962b24SJohn Fastabend if (!drop && sent < 0) { 581248aba1dSJohn Fastabend perror("sendmsg loop error"); 58216962b24SJohn Fastabend goto out_errno; 58316962b24SJohn Fastabend } else if (drop && sent >= 0) { 584248aba1dSJohn Fastabend fprintf(stderr, 585248aba1dSJohn Fastabend "sendmsg loop error expected: %i errno %i\n", 586248aba1dSJohn Fastabend sent, errno); 58716962b24SJohn Fastabend errno = -EIO; 58816962b24SJohn Fastabend goto out_errno; 58916962b24SJohn Fastabend } 59016962b24SJohn Fastabend if (sent > 0) 59116962b24SJohn Fastabend s->bytes_sent += sent; 59216962b24SJohn Fastabend } 59316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 59416962b24SJohn Fastabend } else { 595753fb2eeSJohn Fastabend int slct, recvp = 0, recv, max_fd = fd; 5961ade9abaSJohn Fastabend float total_bytes, txmsg_pop_total; 59716962b24SJohn Fastabend int fd_flags = O_NONBLOCK; 59816962b24SJohn Fastabend struct timeval timeout; 59916962b24SJohn Fastabend fd_set w; 60016962b24SJohn Fastabend 60116962b24SJohn Fastabend fcntl(fd, fd_flags); 6021ade9abaSJohn Fastabend /* Account for pop bytes noting each iteration of apply will 6031ade9abaSJohn Fastabend * call msg_pop_data helper so we need to account for this 6041ade9abaSJohn Fastabend * by calculating the number of apply iterations. Note user 6051ade9abaSJohn Fastabend * of the tool can create cases where no data is sent by 6061ade9abaSJohn Fastabend * manipulating pop/push/pull/etc. For example txmsg_apply 1 6071ade9abaSJohn Fastabend * with txmsg_pop 1 will try to apply 1B at a time but each 6081ade9abaSJohn Fastabend * iteration will then pop 1B so no data will ever be sent. 6091ade9abaSJohn Fastabend * This is really only useful for testing edge cases in code 6101ade9abaSJohn Fastabend * paths. 6111ade9abaSJohn Fastabend */ 61216962b24SJohn Fastabend total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 6131ade9abaSJohn Fastabend if (txmsg_apply) 61418d4e900SJohn Fastabend txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 61518d4e900SJohn Fastabend else 61618d4e900SJohn Fastabend txmsg_pop_total = txmsg_pop * cnt; 6171ade9abaSJohn Fastabend total_bytes -= txmsg_pop_total; 61816962b24SJohn Fastabend err = clock_gettime(CLOCK_MONOTONIC, &s->start); 61916962b24SJohn Fastabend if (err < 0) 620e5dc9dd3SJakub Kicinski perror("recv start time"); 62116962b24SJohn Fastabend while (s->bytes_recvd < total_bytes) { 622a009f1f3SPrashant Bhole if (txmsg_cork) { 623a18fda1aSJohn Fastabend timeout.tv_sec = 0; 6243c6ed988SDaniel Borkmann timeout.tv_usec = 300000; 625a009f1f3SPrashant Bhole } else { 6261ade9abaSJohn Fastabend timeout.tv_sec = 3; 627a009f1f3SPrashant Bhole timeout.tv_usec = 0; 628a009f1f3SPrashant Bhole } 62916962b24SJohn Fastabend 63016962b24SJohn Fastabend /* FD sets */ 63116962b24SJohn Fastabend FD_ZERO(&w); 63216962b24SJohn Fastabend FD_SET(fd, &w); 63316962b24SJohn Fastabend 63416962b24SJohn Fastabend slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 63516962b24SJohn Fastabend if (slct == -1) { 63616962b24SJohn Fastabend perror("select()"); 63716962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 63816962b24SJohn Fastabend goto out_errno; 63916962b24SJohn Fastabend } else if (!slct) { 64016962b24SJohn Fastabend if (opt->verbose) 6411ade9abaSJohn Fastabend fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 64216962b24SJohn Fastabend errno = -EIO; 64316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 64416962b24SJohn Fastabend goto out_errno; 64516962b24SJohn Fastabend } 64616962b24SJohn Fastabend 647753fb2eeSJohn Fastabend errno = 0; 648753fb2eeSJohn Fastabend if (peek_flag) { 649753fb2eeSJohn Fastabend flags |= MSG_PEEK; 650753fb2eeSJohn Fastabend recvp = recvmsg(fd, &msg_peek, flags); 651753fb2eeSJohn Fastabend if (recvp < 0) { 652753fb2eeSJohn Fastabend if (errno != EWOULDBLOCK) { 653753fb2eeSJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 654753fb2eeSJohn Fastabend goto out_errno; 655753fb2eeSJohn Fastabend } 656753fb2eeSJohn Fastabend } 657753fb2eeSJohn Fastabend flags = 0; 658753fb2eeSJohn Fastabend } 659753fb2eeSJohn Fastabend 66016962b24SJohn Fastabend recv = recvmsg(fd, &msg, flags); 66116962b24SJohn Fastabend if (recv < 0) { 66216962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 66316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 664e5dc9dd3SJakub Kicinski perror("recv failed()"); 66516962b24SJohn Fastabend goto out_errno; 66616962b24SJohn Fastabend } 66716962b24SJohn Fastabend } 66816962b24SJohn Fastabend 66916962b24SJohn Fastabend s->bytes_recvd += recv; 67016962b24SJohn Fastabend 671d6967214SLiu Jian if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 672d6967214SLiu Jian errno = EMSGSIZE; 673d6967214SLiu Jian fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 674d6967214SLiu Jian s->bytes_recvd, total_bytes); 675d6967214SLiu Jian goto out_errno; 676d6967214SLiu Jian } 677d6967214SLiu Jian 678753fb2eeSJohn Fastabend if (data) { 679753fb2eeSJohn Fastabend int chunk_sz = opt->sendpage ? 680753fb2eeSJohn Fastabend iov_length * cnt : 681753fb2eeSJohn Fastabend iov_length * iov_count; 68216962b24SJohn Fastabend 683753fb2eeSJohn Fastabend errno = msg_verify_data(&msg, recv, chunk_sz); 684753fb2eeSJohn Fastabend if (errno) { 685e5dc9dd3SJakub Kicinski perror("data verify msg failed"); 68616962b24SJohn Fastabend goto out_errno; 68716962b24SJohn Fastabend } 688753fb2eeSJohn Fastabend if (recvp) { 689753fb2eeSJohn Fastabend errno = msg_verify_data(&msg_peek, 690753fb2eeSJohn Fastabend recvp, 691753fb2eeSJohn Fastabend chunk_sz); 692753fb2eeSJohn Fastabend if (errno) { 693e5dc9dd3SJakub Kicinski perror("data verify msg_peek failed"); 694753fb2eeSJohn Fastabend goto out_errno; 69516962b24SJohn Fastabend } 69616962b24SJohn Fastabend } 69716962b24SJohn Fastabend } 69816962b24SJohn Fastabend } 69916962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 70016962b24SJohn Fastabend } 70116962b24SJohn Fastabend 702753fb2eeSJohn Fastabend msg_free_iov(&msg); 703753fb2eeSJohn Fastabend msg_free_iov(&msg_peek); 704753fb2eeSJohn Fastabend return err; 70516962b24SJohn Fastabend out_errno: 706753fb2eeSJohn Fastabend msg_free_iov(&msg); 707753fb2eeSJohn Fastabend msg_free_iov(&msg_peek); 70816962b24SJohn Fastabend return errno; 70916962b24SJohn Fastabend } 71016962b24SJohn Fastabend 71116962b24SJohn Fastabend static float giga = 1000000000; 71216962b24SJohn Fastabend 71316962b24SJohn Fastabend static inline float sentBps(struct msg_stats s) 71416962b24SJohn Fastabend { 71516962b24SJohn Fastabend return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 71616962b24SJohn Fastabend } 71716962b24SJohn Fastabend 71816962b24SJohn Fastabend static inline float recvdBps(struct msg_stats s) 71916962b24SJohn Fastabend { 72016962b24SJohn Fastabend return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 72116962b24SJohn Fastabend } 72216962b24SJohn Fastabend 72316962b24SJohn Fastabend static int sendmsg_test(struct sockmap_options *opt) 72416962b24SJohn Fastabend { 72516962b24SJohn Fastabend float sent_Bps = 0, recvd_Bps = 0; 72616962b24SJohn Fastabend int rx_fd, txpid, rxpid, err = 0; 72716962b24SJohn Fastabend struct msg_stats s = {0}; 72816962b24SJohn Fastabend int iov_count = opt->iov_count; 72916962b24SJohn Fastabend int iov_buf = opt->iov_length; 73016edddfeSPrashant Bhole int rx_status, tx_status; 73116962b24SJohn Fastabend int cnt = opt->rate; 73216962b24SJohn Fastabend 73316962b24SJohn Fastabend errno = 0; 73416962b24SJohn Fastabend 73516962b24SJohn Fastabend if (opt->base) 73616962b24SJohn Fastabend rx_fd = p1; 73716962b24SJohn Fastabend else 73816962b24SJohn Fastabend rx_fd = p2; 73916962b24SJohn Fastabend 740e9dd9047SJohn Fastabend if (ktls) { 741e9dd9047SJohn Fastabend /* Redirecting into non-TLS socket which sends into a TLS 742e9dd9047SJohn Fastabend * socket is not a valid test. So in this case lets not 743e9dd9047SJohn Fastabend * enable kTLS but still run the test. 744e9dd9047SJohn Fastabend */ 745bce86231SJiapeng Chong if (!txmsg_redir || txmsg_ingress) { 746e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, rx_fd); 747e9dd9047SJohn Fastabend if (err) 748e9dd9047SJohn Fastabend return err; 749e9dd9047SJohn Fastabend } 750e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, c1); 751e9dd9047SJohn Fastabend if (err) 752e9dd9047SJohn Fastabend return err; 753e9dd9047SJohn Fastabend } 754e9dd9047SJohn Fastabend 75516962b24SJohn Fastabend rxpid = fork(); 75616962b24SJohn Fastabend if (rxpid == 0) { 757d6967214SLiu Jian if (txmsg_pop || txmsg_start_pop) 75818d4e900SJohn Fastabend iov_buf -= (txmsg_pop - txmsg_start_pop + 1); 759463bac5fSJohn Fastabend if (opt->drop_expected || txmsg_ktls_skb_drop) 76018d4e900SJohn Fastabend _exit(0); 76118d4e900SJohn Fastabend 76218d4e900SJohn Fastabend if (!iov_buf) /* zero bytes sent case */ 76318d4e900SJohn Fastabend _exit(0); 76416962b24SJohn Fastabend 76516962b24SJohn Fastabend if (opt->sendpage) 76616962b24SJohn Fastabend iov_count = 1; 76716962b24SJohn Fastabend err = msg_loop(rx_fd, iov_count, iov_buf, 76816962b24SJohn Fastabend cnt, &s, false, opt); 769b98ca90cSJohn Fastabend if (opt->verbose > 1) 77016962b24SJohn Fastabend fprintf(stderr, 77116962b24SJohn Fastabend "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 77216962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 77316962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 77416962b24SJohn Fastabend sent_Bps = sentBps(s); 77516962b24SJohn Fastabend recvd_Bps = recvdBps(s); 77616962b24SJohn Fastabend } 777b98ca90cSJohn Fastabend if (opt->verbose > 1) 77816962b24SJohn Fastabend fprintf(stdout, 779753fb2eeSJohn Fastabend "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 78016962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 781753fb2eeSJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 782753fb2eeSJohn Fastabend peek_flag ? "(peek_msg)" : ""); 78316edddfeSPrashant Bhole if (err && txmsg_cork) 78416edddfeSPrashant Bhole err = 0; 78516edddfeSPrashant Bhole exit(err ? 1 : 0); 78616962b24SJohn Fastabend } else if (rxpid == -1) { 787e5dc9dd3SJakub Kicinski perror("msg_loop_rx"); 78816962b24SJohn Fastabend return errno; 78916962b24SJohn Fastabend } 79016962b24SJohn Fastabend 79116962b24SJohn Fastabend txpid = fork(); 79216962b24SJohn Fastabend if (txpid == 0) { 79316962b24SJohn Fastabend if (opt->sendpage) 79416962b24SJohn Fastabend err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 79516962b24SJohn Fastabend else 79616962b24SJohn Fastabend err = msg_loop(c1, iov_count, iov_buf, 79716962b24SJohn Fastabend cnt, &s, true, opt); 79816962b24SJohn Fastabend 79916962b24SJohn Fastabend if (err) 80016962b24SJohn Fastabend fprintf(stderr, 80116962b24SJohn Fastabend "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 80216962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 80316962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 80416962b24SJohn Fastabend sent_Bps = sentBps(s); 80516962b24SJohn Fastabend recvd_Bps = recvdBps(s); 80616962b24SJohn Fastabend } 807b98ca90cSJohn Fastabend if (opt->verbose > 1) 80816962b24SJohn Fastabend fprintf(stdout, 80916962b24SJohn Fastabend "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 81016962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 81116962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 81216edddfeSPrashant Bhole exit(err ? 1 : 0); 81316962b24SJohn Fastabend } else if (txpid == -1) { 814e5dc9dd3SJakub Kicinski perror("msg_loop_tx"); 81516962b24SJohn Fastabend return errno; 81616962b24SJohn Fastabend } 81716962b24SJohn Fastabend 81816edddfeSPrashant Bhole assert(waitpid(rxpid, &rx_status, 0) == rxpid); 81916edddfeSPrashant Bhole assert(waitpid(txpid, &tx_status, 0) == txpid); 82016edddfeSPrashant Bhole if (WIFEXITED(rx_status)) { 82116edddfeSPrashant Bhole err = WEXITSTATUS(rx_status); 82216edddfeSPrashant Bhole if (err) { 823248aba1dSJohn Fastabend fprintf(stderr, "rx thread exited with err %d.\n", err); 82416edddfeSPrashant Bhole goto out; 82516edddfeSPrashant Bhole } 82616edddfeSPrashant Bhole } 82716edddfeSPrashant Bhole if (WIFEXITED(tx_status)) { 82816edddfeSPrashant Bhole err = WEXITSTATUS(tx_status); 82916edddfeSPrashant Bhole if (err) 830248aba1dSJohn Fastabend fprintf(stderr, "tx thread exited with err %d.\n", err); 83116edddfeSPrashant Bhole } 83216edddfeSPrashant Bhole out: 83316962b24SJohn Fastabend return err; 83416962b24SJohn Fastabend } 83516962b24SJohn Fastabend 83616962b24SJohn Fastabend static int forever_ping_pong(int rate, struct sockmap_options *opt) 83716962b24SJohn Fastabend { 83816962b24SJohn Fastabend struct timeval timeout; 83916962b24SJohn Fastabend char buf[1024] = {0}; 84016962b24SJohn Fastabend int sc; 84116962b24SJohn Fastabend 84216962b24SJohn Fastabend timeout.tv_sec = 10; 84316962b24SJohn Fastabend timeout.tv_usec = 0; 84416962b24SJohn Fastabend 84516962b24SJohn Fastabend /* Ping/Pong data from client to server */ 84616962b24SJohn Fastabend sc = send(c1, buf, sizeof(buf), 0); 84716962b24SJohn Fastabend if (sc < 0) { 848e5dc9dd3SJakub Kicinski perror("send failed()"); 84916962b24SJohn Fastabend return sc; 85016962b24SJohn Fastabend } 85116962b24SJohn Fastabend 85216962b24SJohn Fastabend do { 85316962b24SJohn Fastabend int s, rc, i, max_fd = p2; 85416962b24SJohn Fastabend fd_set w; 85516962b24SJohn Fastabend 85616962b24SJohn Fastabend /* FD sets */ 85716962b24SJohn Fastabend FD_ZERO(&w); 85816962b24SJohn Fastabend FD_SET(c1, &w); 85916962b24SJohn Fastabend FD_SET(c2, &w); 86016962b24SJohn Fastabend FD_SET(p1, &w); 86116962b24SJohn Fastabend FD_SET(p2, &w); 86216962b24SJohn Fastabend 86316962b24SJohn Fastabend s = select(max_fd + 1, &w, NULL, NULL, &timeout); 86416962b24SJohn Fastabend if (s == -1) { 86516962b24SJohn Fastabend perror("select()"); 86616962b24SJohn Fastabend break; 86716962b24SJohn Fastabend } else if (!s) { 86816962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 86916962b24SJohn Fastabend break; 87016962b24SJohn Fastabend } 87116962b24SJohn Fastabend 87216962b24SJohn Fastabend for (i = 0; i <= max_fd && s > 0; ++i) { 87316962b24SJohn Fastabend if (!FD_ISSET(i, &w)) 87416962b24SJohn Fastabend continue; 87516962b24SJohn Fastabend 87616962b24SJohn Fastabend s--; 87716962b24SJohn Fastabend 87816962b24SJohn Fastabend rc = recv(i, buf, sizeof(buf), 0); 87916962b24SJohn Fastabend if (rc < 0) { 88016962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 881e5dc9dd3SJakub Kicinski perror("recv failed()"); 88216962b24SJohn Fastabend return rc; 88316962b24SJohn Fastabend } 88416962b24SJohn Fastabend } 88516962b24SJohn Fastabend 88616962b24SJohn Fastabend if (rc == 0) { 88716962b24SJohn Fastabend close(i); 88816962b24SJohn Fastabend break; 88916962b24SJohn Fastabend } 89016962b24SJohn Fastabend 89116962b24SJohn Fastabend sc = send(i, buf, rc, 0); 89216962b24SJohn Fastabend if (sc < 0) { 893e5dc9dd3SJakub Kicinski perror("send failed()"); 89416962b24SJohn Fastabend return sc; 89516962b24SJohn Fastabend } 89616962b24SJohn Fastabend } 89716962b24SJohn Fastabend 89816962b24SJohn Fastabend if (rate) 89916962b24SJohn Fastabend sleep(rate); 90016962b24SJohn Fastabend 90116962b24SJohn Fastabend if (opt->verbose) { 90216962b24SJohn Fastabend printf("."); 90316962b24SJohn Fastabend fflush(stdout); 90416962b24SJohn Fastabend 90516962b24SJohn Fastabend } 90616962b24SJohn Fastabend } while (running); 90716962b24SJohn Fastabend 90816962b24SJohn Fastabend return 0; 90916962b24SJohn Fastabend } 91016962b24SJohn Fastabend 91116962b24SJohn Fastabend enum { 912b98ca90cSJohn Fastabend SELFTESTS, 91316962b24SJohn Fastabend PING_PONG, 91416962b24SJohn Fastabend SENDMSG, 91516962b24SJohn Fastabend BASE, 91616962b24SJohn Fastabend BASE_SENDPAGE, 91716962b24SJohn Fastabend SENDPAGE, 91816962b24SJohn Fastabend }; 91916962b24SJohn Fastabend 92016962b24SJohn Fastabend static int run_options(struct sockmap_options *options, int cg_fd, int test) 92116962b24SJohn Fastabend { 92216962b24SJohn Fastabend int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 92316962b24SJohn Fastabend 92416962b24SJohn Fastabend /* If base test skip BPF setup */ 92516962b24SJohn Fastabend if (test == BASE || test == BASE_SENDPAGE) 92616962b24SJohn Fastabend goto run; 92716962b24SJohn Fastabend 92816962b24SJohn Fastabend /* Attach programs to sockmap */ 929cdf43c4bSJohn Fastabend if (!txmsg_omit_skb_parser) { 93016962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[0], 93116962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0); 93216962b24SJohn Fastabend if (err) { 93316962b24SJohn Fastabend fprintf(stderr, 93416962b24SJohn Fastabend "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 93516962b24SJohn Fastabend prog_fd[0], map_fd[0], err, strerror(errno)); 93616962b24SJohn Fastabend return err; 93716962b24SJohn Fastabend } 938cdf43c4bSJohn Fastabend } 93916962b24SJohn Fastabend 94016962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[1], map_fd[0], 94116962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0); 94216962b24SJohn Fastabend if (err) { 94316962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 94416962b24SJohn Fastabend err, strerror(errno)); 94516962b24SJohn Fastabend return err; 94616962b24SJohn Fastabend } 94716962b24SJohn Fastabend 948463bac5fSJohn Fastabend /* Attach programs to TLS sockmap */ 949463bac5fSJohn Fastabend if (txmsg_ktls_skb) { 950cdf43c4bSJohn Fastabend if (!txmsg_omit_skb_parser) { 951463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[8], 952463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0); 953463bac5fSJohn Fastabend if (err) { 954463bac5fSJohn Fastabend fprintf(stderr, 955463bac5fSJohn Fastabend "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n", 956463bac5fSJohn Fastabend prog_fd[0], map_fd[8], err, strerror(errno)); 957463bac5fSJohn Fastabend return err; 958463bac5fSJohn Fastabend } 959cdf43c4bSJohn Fastabend } 960463bac5fSJohn Fastabend 961463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[2], map_fd[8], 962463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0); 963463bac5fSJohn Fastabend if (err) { 964463bac5fSJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n", 965463bac5fSJohn Fastabend err, strerror(errno)); 966463bac5fSJohn Fastabend return err; 967463bac5fSJohn Fastabend } 968463bac5fSJohn Fastabend } 969463bac5fSJohn Fastabend 97016962b24SJohn Fastabend /* Attach to cgroups */ 971463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 97216962b24SJohn Fastabend if (err) { 97316962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 97416962b24SJohn Fastabend err, strerror(errno)); 97516962b24SJohn Fastabend return err; 97616962b24SJohn Fastabend } 97716962b24SJohn Fastabend 97816962b24SJohn Fastabend run: 97916962b24SJohn Fastabend err = sockmap_init_sockets(options->verbose); 98016962b24SJohn Fastabend if (err) { 98116962b24SJohn Fastabend fprintf(stderr, "ERROR: test socket failed: %d\n", err); 98216962b24SJohn Fastabend goto out; 98316962b24SJohn Fastabend } 98416962b24SJohn Fastabend 98516962b24SJohn Fastabend /* Attach txmsg program to sockmap */ 98616962b24SJohn Fastabend if (txmsg_pass) 987d79a3212SJohn Fastabend tx_prog_fd = prog_fd[4]; 988463bac5fSJohn Fastabend else if (txmsg_redir) 98916962b24SJohn Fastabend tx_prog_fd = prog_fd[5]; 990463bac5fSJohn Fastabend else if (txmsg_apply) 99116962b24SJohn Fastabend tx_prog_fd = prog_fd[6]; 992463bac5fSJohn Fastabend else if (txmsg_cork) 99316962b24SJohn Fastabend tx_prog_fd = prog_fd[7]; 994463bac5fSJohn Fastabend else if (txmsg_drop) 995463bac5fSJohn Fastabend tx_prog_fd = prog_fd[8]; 99616962b24SJohn Fastabend else 99716962b24SJohn Fastabend tx_prog_fd = 0; 99816962b24SJohn Fastabend 99916962b24SJohn Fastabend if (tx_prog_fd) { 100016962b24SJohn Fastabend int redir_fd, i = 0; 100116962b24SJohn Fastabend 100216962b24SJohn Fastabend err = bpf_prog_attach(tx_prog_fd, 100316962b24SJohn Fastabend map_fd[1], BPF_SK_MSG_VERDICT, 0); 100416962b24SJohn Fastabend if (err) { 100516962b24SJohn Fastabend fprintf(stderr, 100616962b24SJohn Fastabend "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 100716962b24SJohn Fastabend err, strerror(errno)); 100816962b24SJohn Fastabend goto out; 100916962b24SJohn Fastabend } 101016962b24SJohn Fastabend 101116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 101216962b24SJohn Fastabend if (err) { 101316962b24SJohn Fastabend fprintf(stderr, 101416962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 101516962b24SJohn Fastabend err, strerror(errno)); 101616962b24SJohn Fastabend goto out; 101716962b24SJohn Fastabend } 101816962b24SJohn Fastabend 1019d79a3212SJohn Fastabend if (txmsg_redir) 102016962b24SJohn Fastabend redir_fd = c2; 102116962b24SJohn Fastabend else 102216962b24SJohn Fastabend redir_fd = c1; 102316962b24SJohn Fastabend 102416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 102516962b24SJohn Fastabend if (err) { 102616962b24SJohn Fastabend fprintf(stderr, 102716962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 102816962b24SJohn Fastabend err, strerror(errno)); 102916962b24SJohn Fastabend goto out; 103016962b24SJohn Fastabend } 103116962b24SJohn Fastabend 103216962b24SJohn Fastabend if (txmsg_apply) { 103316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[3], 103416962b24SJohn Fastabend &i, &txmsg_apply, BPF_ANY); 103516962b24SJohn Fastabend if (err) { 103616962b24SJohn Fastabend fprintf(stderr, 103716962b24SJohn Fastabend "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 103816962b24SJohn Fastabend err, strerror(errno)); 103916962b24SJohn Fastabend goto out; 104016962b24SJohn Fastabend } 104116962b24SJohn Fastabend } 104216962b24SJohn Fastabend 104316962b24SJohn Fastabend if (txmsg_cork) { 104416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[4], 104516962b24SJohn Fastabend &i, &txmsg_cork, BPF_ANY); 104616962b24SJohn Fastabend if (err) { 104716962b24SJohn Fastabend fprintf(stderr, 104816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 104916962b24SJohn Fastabend err, strerror(errno)); 105016962b24SJohn Fastabend goto out; 105116962b24SJohn Fastabend } 105216962b24SJohn Fastabend } 105316962b24SJohn Fastabend 105416962b24SJohn Fastabend if (txmsg_start) { 105516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 105616962b24SJohn Fastabend &i, &txmsg_start, BPF_ANY); 105716962b24SJohn Fastabend if (err) { 105816962b24SJohn Fastabend fprintf(stderr, 105916962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 106016962b24SJohn Fastabend err, strerror(errno)); 106116962b24SJohn Fastabend goto out; 106216962b24SJohn Fastabend } 106316962b24SJohn Fastabend } 106416962b24SJohn Fastabend 106516962b24SJohn Fastabend if (txmsg_end) { 106616962b24SJohn Fastabend i = 1; 106716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 106816962b24SJohn Fastabend &i, &txmsg_end, BPF_ANY); 106916962b24SJohn Fastabend if (err) { 107016962b24SJohn Fastabend fprintf(stderr, 107116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 107216962b24SJohn Fastabend err, strerror(errno)); 107316962b24SJohn Fastabend goto out; 107416962b24SJohn Fastabend } 107516962b24SJohn Fastabend } 107616962b24SJohn Fastabend 107784fbfe02SJohn Fastabend if (txmsg_start_push) { 107884fbfe02SJohn Fastabend i = 2; 107984fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 108084fbfe02SJohn Fastabend &i, &txmsg_start_push, BPF_ANY); 108184fbfe02SJohn Fastabend if (err) { 108284fbfe02SJohn Fastabend fprintf(stderr, 108384fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 108484fbfe02SJohn Fastabend err, strerror(errno)); 108584fbfe02SJohn Fastabend goto out; 108684fbfe02SJohn Fastabend } 108784fbfe02SJohn Fastabend } 108884fbfe02SJohn Fastabend 108984fbfe02SJohn Fastabend if (txmsg_end_push) { 109084fbfe02SJohn Fastabend i = 3; 109184fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 109284fbfe02SJohn Fastabend &i, &txmsg_end_push, BPF_ANY); 109384fbfe02SJohn Fastabend if (err) { 109484fbfe02SJohn Fastabend fprintf(stderr, 109584fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 109684fbfe02SJohn Fastabend txmsg_end_push, i, err, strerror(errno)); 109784fbfe02SJohn Fastabend goto out; 109884fbfe02SJohn Fastabend } 109984fbfe02SJohn Fastabend } 110084fbfe02SJohn Fastabend 11011ade9abaSJohn Fastabend if (txmsg_start_pop) { 11021ade9abaSJohn Fastabend i = 4; 11031ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5], 11041ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY); 11051ade9abaSJohn Fastabend if (err) { 11061ade9abaSJohn Fastabend fprintf(stderr, 11071ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 11081ade9abaSJohn Fastabend txmsg_start_pop, i, err, strerror(errno)); 11091ade9abaSJohn Fastabend goto out; 11101ade9abaSJohn Fastabend } 11111ade9abaSJohn Fastabend } else { 11121ade9abaSJohn Fastabend i = 4; 11131ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5], 11141ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY); 11151ade9abaSJohn Fastabend } 11161ade9abaSJohn Fastabend 11171ade9abaSJohn Fastabend if (txmsg_pop) { 11181ade9abaSJohn Fastabend i = 5; 11191ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5], 11201ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY); 11211ade9abaSJohn Fastabend if (err) { 11221ade9abaSJohn Fastabend fprintf(stderr, 11231ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 11241ade9abaSJohn Fastabend txmsg_pop, i, err, strerror(errno)); 11251ade9abaSJohn Fastabend goto out; 11261ade9abaSJohn Fastabend } 11271ade9abaSJohn Fastabend } else { 11281ade9abaSJohn Fastabend i = 5; 11291ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5], 11301ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY); 11311ade9abaSJohn Fastabend 11321ade9abaSJohn Fastabend } 11331ade9abaSJohn Fastabend 113416962b24SJohn Fastabend if (txmsg_ingress) { 113516962b24SJohn Fastabend int in = BPF_F_INGRESS; 113616962b24SJohn Fastabend 113716962b24SJohn Fastabend i = 0; 113816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 113916962b24SJohn Fastabend if (err) { 114016962b24SJohn Fastabend fprintf(stderr, 114116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 114216962b24SJohn Fastabend err, strerror(errno)); 114316962b24SJohn Fastabend } 114416962b24SJohn Fastabend i = 1; 114516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 114616962b24SJohn Fastabend if (err) { 114716962b24SJohn Fastabend fprintf(stderr, 114816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 114916962b24SJohn Fastabend err, strerror(errno)); 115016962b24SJohn Fastabend } 115116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 115216962b24SJohn Fastabend if (err) { 115316962b24SJohn Fastabend fprintf(stderr, 115416962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 115516962b24SJohn Fastabend err, strerror(errno)); 115616962b24SJohn Fastabend } 115716962b24SJohn Fastabend 115816962b24SJohn Fastabend i = 2; 115916962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 116016962b24SJohn Fastabend if (err) { 116116962b24SJohn Fastabend fprintf(stderr, 116216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 116316962b24SJohn Fastabend err, strerror(errno)); 116416962b24SJohn Fastabend } 116516962b24SJohn Fastabend } 116616962b24SJohn Fastabend 1167463bac5fSJohn Fastabend if (txmsg_ktls_skb) { 1168463bac5fSJohn Fastabend int ingress = BPF_F_INGRESS; 1169463bac5fSJohn Fastabend 1170463bac5fSJohn Fastabend i = 0; 1171463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1172463bac5fSJohn Fastabend if (err) { 1173463bac5fSJohn Fastabend fprintf(stderr, 1174463bac5fSJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1175463bac5fSJohn Fastabend err, strerror(errno)); 1176463bac5fSJohn Fastabend } 1177463bac5fSJohn Fastabend 1178463bac5fSJohn Fastabend if (txmsg_ktls_skb_redir) { 1179463bac5fSJohn Fastabend i = 1; 1180463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[7], 1181463bac5fSJohn Fastabend &i, &ingress, BPF_ANY); 1182463bac5fSJohn Fastabend if (err) { 1183463bac5fSJohn Fastabend fprintf(stderr, 1184463bac5fSJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1185463bac5fSJohn Fastabend err, strerror(errno)); 1186463bac5fSJohn Fastabend } 1187463bac5fSJohn Fastabend } 1188463bac5fSJohn Fastabend 1189463bac5fSJohn Fastabend if (txmsg_ktls_skb_drop) { 1190463bac5fSJohn Fastabend i = 1; 1191463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1192463bac5fSJohn Fastabend } 1193463bac5fSJohn Fastabend } 1194463bac5fSJohn Fastabend 1195463bac5fSJohn Fastabend if (txmsg_redir_skb) { 119616962b24SJohn Fastabend int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 119716962b24SJohn Fastabend p2 : p1; 119816962b24SJohn Fastabend int ingress = BPF_F_INGRESS; 119916962b24SJohn Fastabend 120016962b24SJohn Fastabend i = 0; 120116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[7], 120216962b24SJohn Fastabend &i, &ingress, BPF_ANY); 120316962b24SJohn Fastabend if (err) { 120416962b24SJohn Fastabend fprintf(stderr, 120516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 120616962b24SJohn Fastabend err, strerror(errno)); 120716962b24SJohn Fastabend } 120816962b24SJohn Fastabend 120916962b24SJohn Fastabend i = 3; 1210463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 121116962b24SJohn Fastabend if (err) { 121216962b24SJohn Fastabend fprintf(stderr, 121316962b24SJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 121416962b24SJohn Fastabend err, strerror(errno)); 121516962b24SJohn Fastabend } 121616962b24SJohn Fastabend } 121716962b24SJohn Fastabend } 121816962b24SJohn Fastabend 121953792fa4SJohn Fastabend if (skb_use_parser) { 122053792fa4SJohn Fastabend i = 2; 122153792fa4SJohn Fastabend err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 122253792fa4SJohn Fastabend } 122353792fa4SJohn Fastabend 122416962b24SJohn Fastabend if (txmsg_drop) 122516962b24SJohn Fastabend options->drop_expected = true; 122616962b24SJohn Fastabend 122716962b24SJohn Fastabend if (test == PING_PONG) 122816962b24SJohn Fastabend err = forever_ping_pong(options->rate, options); 122916962b24SJohn Fastabend else if (test == SENDMSG) { 123016962b24SJohn Fastabend options->base = false; 123116962b24SJohn Fastabend options->sendpage = false; 123216962b24SJohn Fastabend err = sendmsg_test(options); 123316962b24SJohn Fastabend } else if (test == SENDPAGE) { 123416962b24SJohn Fastabend options->base = false; 123516962b24SJohn Fastabend options->sendpage = true; 123616962b24SJohn Fastabend err = sendmsg_test(options); 123716962b24SJohn Fastabend } else if (test == BASE) { 123816962b24SJohn Fastabend options->base = true; 123916962b24SJohn Fastabend options->sendpage = false; 124016962b24SJohn Fastabend err = sendmsg_test(options); 124116962b24SJohn Fastabend } else if (test == BASE_SENDPAGE) { 124216962b24SJohn Fastabend options->base = true; 124316962b24SJohn Fastabend options->sendpage = true; 124416962b24SJohn Fastabend err = sendmsg_test(options); 124516962b24SJohn Fastabend } else 124616962b24SJohn Fastabend fprintf(stderr, "unknown test\n"); 124716962b24SJohn Fastabend out: 124816962b24SJohn Fastabend /* Detatch and zero all the maps */ 1249463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS); 125016962b24SJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 125116962b24SJohn Fastabend bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1252463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER); 1253463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT); 1254463bac5fSJohn Fastabend 125516962b24SJohn Fastabend if (tx_prog_fd >= 0) 125616962b24SJohn Fastabend bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 125716962b24SJohn Fastabend 125816962b24SJohn Fastabend for (i = 0; i < 8; i++) { 125916962b24SJohn Fastabend key = next_key = 0; 126016962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 126116962b24SJohn Fastabend while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 126216962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 126316962b24SJohn Fastabend key = next_key; 126416962b24SJohn Fastabend } 126516962b24SJohn Fastabend } 126616962b24SJohn Fastabend 126716962b24SJohn Fastabend close(s1); 126816962b24SJohn Fastabend close(s2); 126916962b24SJohn Fastabend close(p1); 127016962b24SJohn Fastabend close(p2); 127116962b24SJohn Fastabend close(c1); 127216962b24SJohn Fastabend close(c2); 127316962b24SJohn Fastabend return err; 127416962b24SJohn Fastabend } 127516962b24SJohn Fastabend 127616962b24SJohn Fastabend static char *test_to_str(int test) 127716962b24SJohn Fastabend { 127816962b24SJohn Fastabend switch (test) { 127916962b24SJohn Fastabend case SENDMSG: 128016962b24SJohn Fastabend return "sendmsg"; 128116962b24SJohn Fastabend case SENDPAGE: 128216962b24SJohn Fastabend return "sendpage"; 128316962b24SJohn Fastabend } 128416962b24SJohn Fastabend return "unknown"; 128516962b24SJohn Fastabend } 128616962b24SJohn Fastabend 1287eceae70bSAndrii Nakryiko static void append_str(char *dst, const char *src, size_t dst_cap) 1288eceae70bSAndrii Nakryiko { 1289eceae70bSAndrii Nakryiko size_t avail = dst_cap - strlen(dst); 1290eceae70bSAndrii Nakryiko 1291eceae70bSAndrii Nakryiko if (avail <= 1) /* just zero byte could be written */ 1292eceae70bSAndrii Nakryiko return; 1293eceae70bSAndrii Nakryiko 1294eceae70bSAndrii Nakryiko strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1295eceae70bSAndrii Nakryiko } 1296eceae70bSAndrii Nakryiko 129716962b24SJohn Fastabend #define OPTSTRING 60 129816962b24SJohn Fastabend static void test_options(char *options) 129916962b24SJohn Fastabend { 130073563aa3SPrashant Bhole char tstr[OPTSTRING]; 130173563aa3SPrashant Bhole 130216962b24SJohn Fastabend memset(options, 0, OPTSTRING); 130316962b24SJohn Fastabend 130416962b24SJohn Fastabend if (txmsg_pass) 1305eceae70bSAndrii Nakryiko append_str(options, "pass,", OPTSTRING); 130616962b24SJohn Fastabend if (txmsg_redir) 1307eceae70bSAndrii Nakryiko append_str(options, "redir,", OPTSTRING); 130816962b24SJohn Fastabend if (txmsg_drop) 1309eceae70bSAndrii Nakryiko append_str(options, "drop,", OPTSTRING); 131073563aa3SPrashant Bhole if (txmsg_apply) { 131173563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1312eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING); 131373563aa3SPrashant Bhole } 131473563aa3SPrashant Bhole if (txmsg_cork) { 131573563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1316eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING); 131773563aa3SPrashant Bhole } 131873563aa3SPrashant Bhole if (txmsg_start) { 131973563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1320eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING); 132173563aa3SPrashant Bhole } 132273563aa3SPrashant Bhole if (txmsg_end) { 132373563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1324eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING); 132573563aa3SPrashant Bhole } 13261ade9abaSJohn Fastabend if (txmsg_start_pop) { 13271ade9abaSJohn Fastabend snprintf(tstr, OPTSTRING, "pop (%d,%d),", 13281ade9abaSJohn Fastabend txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1329eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING); 13301ade9abaSJohn Fastabend } 133116962b24SJohn Fastabend if (txmsg_ingress) 1332eceae70bSAndrii Nakryiko append_str(options, "ingress,", OPTSTRING); 1333463bac5fSJohn Fastabend if (txmsg_redir_skb) 1334eceae70bSAndrii Nakryiko append_str(options, "redir_skb,", OPTSTRING); 1335463bac5fSJohn Fastabend if (txmsg_ktls_skb) 1336eceae70bSAndrii Nakryiko append_str(options, "ktls_skb,", OPTSTRING); 1337e9dd9047SJohn Fastabend if (ktls) 1338eceae70bSAndrii Nakryiko append_str(options, "ktls,", OPTSTRING); 1339753fb2eeSJohn Fastabend if (peek_flag) 1340eceae70bSAndrii Nakryiko append_str(options, "peek,", OPTSTRING); 134116962b24SJohn Fastabend } 134216962b24SJohn Fastabend 134316962b24SJohn Fastabend static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 134416962b24SJohn Fastabend { 134573563aa3SPrashant Bhole char *options = calloc(OPTSTRING, sizeof(char)); 134616962b24SJohn Fastabend int err; 134716962b24SJohn Fastabend 134816962b24SJohn Fastabend if (test == SENDPAGE) 134916962b24SJohn Fastabend opt->sendpage = true; 135016962b24SJohn Fastabend else 135116962b24SJohn Fastabend opt->sendpage = false; 135216962b24SJohn Fastabend 135316962b24SJohn Fastabend if (txmsg_drop) 135416962b24SJohn Fastabend opt->drop_expected = true; 135516962b24SJohn Fastabend else 135616962b24SJohn Fastabend opt->drop_expected = false; 135716962b24SJohn Fastabend 135816962b24SJohn Fastabend test_options(options); 135916962b24SJohn Fastabend 1360328aa08aSJohn Fastabend if (opt->verbose) { 136116962b24SJohn Fastabend fprintf(stdout, 136216962b24SJohn Fastabend " [TEST %i]: (%i, %i, %i, %s, %s): ", 136316962b24SJohn Fastabend test_cnt, opt->rate, opt->iov_count, opt->iov_length, 136416962b24SJohn Fastabend test_to_str(test), options); 136516962b24SJohn Fastabend fflush(stdout); 1366328aa08aSJohn Fastabend } 136716962b24SJohn Fastabend err = run_options(opt, cgrp, test); 1368328aa08aSJohn Fastabend if (opt->verbose) 136916962b24SJohn Fastabend fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 137016962b24SJohn Fastabend test_cnt++; 137116962b24SJohn Fastabend !err ? passed++ : failed++; 137216962b24SJohn Fastabend free(options); 137316962b24SJohn Fastabend return err; 137416962b24SJohn Fastabend } 137516962b24SJohn Fastabend 1376328aa08aSJohn Fastabend static void test_exec(int cgrp, struct sockmap_options *opt) 137716962b24SJohn Fastabend { 1378328aa08aSJohn Fastabend int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1379328aa08aSJohn Fastabend int err; 138016962b24SJohn Fastabend 1381328aa08aSJohn Fastabend if (type == 0) { 1382328aa08aSJohn Fastabend test_start(); 1383328aa08aSJohn Fastabend err = __test_exec(cgrp, SENDMSG, opt); 138416962b24SJohn Fastabend if (err) 1385328aa08aSJohn Fastabend test_fail(); 1386328aa08aSJohn Fastabend } else { 1387328aa08aSJohn Fastabend test_start(); 138816962b24SJohn Fastabend err = __test_exec(cgrp, SENDPAGE, opt); 1389328aa08aSJohn Fastabend if (err) 1390328aa08aSJohn Fastabend test_fail(); 1391328aa08aSJohn Fastabend } 139216962b24SJohn Fastabend } 139316962b24SJohn Fastabend 1394328aa08aSJohn Fastabend static void test_send_one(struct sockmap_options *opt, int cgrp) 139516962b24SJohn Fastabend { 139616962b24SJohn Fastabend opt->iov_length = 1; 139716962b24SJohn Fastabend opt->iov_count = 1; 139816962b24SJohn Fastabend opt->rate = 1; 1399328aa08aSJohn Fastabend test_exec(cgrp, opt); 140016962b24SJohn Fastabend 140116962b24SJohn Fastabend opt->iov_length = 1; 140216962b24SJohn Fastabend opt->iov_count = 1024; 140316962b24SJohn Fastabend opt->rate = 1; 1404328aa08aSJohn Fastabend test_exec(cgrp, opt); 140516962b24SJohn Fastabend 140616962b24SJohn Fastabend opt->iov_length = 1024; 140716962b24SJohn Fastabend opt->iov_count = 1; 140816962b24SJohn Fastabend opt->rate = 1; 1409328aa08aSJohn Fastabend test_exec(cgrp, opt); 141016962b24SJohn Fastabend 1411328aa08aSJohn Fastabend } 1412328aa08aSJohn Fastabend 1413328aa08aSJohn Fastabend static void test_send_many(struct sockmap_options *opt, int cgrp) 1414328aa08aSJohn Fastabend { 1415328aa08aSJohn Fastabend opt->iov_length = 3; 141616962b24SJohn Fastabend opt->iov_count = 1; 1417a009f1f3SPrashant Bhole opt->rate = 512; 1418328aa08aSJohn Fastabend test_exec(cgrp, opt); 141916962b24SJohn Fastabend 142016962b24SJohn Fastabend opt->rate = 100; 142116962b24SJohn Fastabend opt->iov_count = 1; 142216962b24SJohn Fastabend opt->iov_length = 5; 1423328aa08aSJohn Fastabend test_exec(cgrp, opt); 142416962b24SJohn Fastabend } 142516962b24SJohn Fastabend 1426328aa08aSJohn Fastabend static void test_send_large(struct sockmap_options *opt, int cgrp) 142716962b24SJohn Fastabend { 1428328aa08aSJohn Fastabend opt->iov_length = 256; 1429328aa08aSJohn Fastabend opt->iov_count = 1024; 1430328aa08aSJohn Fastabend opt->rate = 2; 1431328aa08aSJohn Fastabend test_exec(cgrp, opt); 1432328aa08aSJohn Fastabend } 143316962b24SJohn Fastabend 1434328aa08aSJohn Fastabend static void test_send(struct sockmap_options *opt, int cgrp) 1435328aa08aSJohn Fastabend { 1436328aa08aSJohn Fastabend test_send_one(opt, cgrp); 1437328aa08aSJohn Fastabend test_send_many(opt, cgrp); 1438328aa08aSJohn Fastabend test_send_large(opt, cgrp); 1439328aa08aSJohn Fastabend sched_yield(); 1440328aa08aSJohn Fastabend } 1441328aa08aSJohn Fastabend 1442b98ca90cSJohn Fastabend static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1443328aa08aSJohn Fastabend { 144416962b24SJohn Fastabend /* Test small and large iov_count values with pass/redir/apply/cork */ 144516962b24SJohn Fastabend txmsg_pass = 1; 1446b98ca90cSJohn Fastabend test_send(opt, cgrp); 144716962b24SJohn Fastabend } 144816962b24SJohn Fastabend 1449b98ca90cSJohn Fastabend static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 145016962b24SJohn Fastabend { 1451328aa08aSJohn Fastabend txmsg_redir = 1; 1452b98ca90cSJohn Fastabend test_send(opt, cgrp); 1453328aa08aSJohn Fastabend } 1454328aa08aSJohn Fastabend 1455b98ca90cSJohn Fastabend static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1456328aa08aSJohn Fastabend { 1457328aa08aSJohn Fastabend txmsg_drop = 1; 1458b98ca90cSJohn Fastabend test_send(opt, cgrp); 1459328aa08aSJohn Fastabend } 1460328aa08aSJohn Fastabend 1461b98ca90cSJohn Fastabend static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1462328aa08aSJohn Fastabend { 1463328aa08aSJohn Fastabend txmsg_pass = txmsg_drop = 0; 1464328aa08aSJohn Fastabend txmsg_ingress = txmsg_redir = 1; 1465b98ca90cSJohn Fastabend test_send(opt, cgrp); 1466328aa08aSJohn Fastabend } 1467328aa08aSJohn Fastabend 1468463bac5fSJohn Fastabend static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1469463bac5fSJohn Fastabend { 1470463bac5fSJohn Fastabend bool data = opt->data_test; 1471463bac5fSJohn Fastabend int k = ktls; 1472463bac5fSJohn Fastabend 1473463bac5fSJohn Fastabend opt->data_test = true; 1474463bac5fSJohn Fastabend ktls = 1; 1475463bac5fSJohn Fastabend 1476463bac5fSJohn Fastabend txmsg_pass = txmsg_drop = 0; 1477463bac5fSJohn Fastabend txmsg_ingress = txmsg_redir = 0; 1478463bac5fSJohn Fastabend txmsg_ktls_skb = 1; 1479463bac5fSJohn Fastabend txmsg_pass = 1; 1480463bac5fSJohn Fastabend 1481463bac5fSJohn Fastabend /* Using data verification so ensure iov layout is 1482463bac5fSJohn Fastabend * expected from test receiver side. e.g. has enough 1483463bac5fSJohn Fastabend * bytes to write test code. 1484463bac5fSJohn Fastabend */ 1485463bac5fSJohn Fastabend opt->iov_length = 100; 1486463bac5fSJohn Fastabend opt->iov_count = 1; 1487463bac5fSJohn Fastabend opt->rate = 1; 1488463bac5fSJohn Fastabend test_exec(cgrp, opt); 1489463bac5fSJohn Fastabend 1490463bac5fSJohn Fastabend txmsg_ktls_skb_drop = 1; 1491463bac5fSJohn Fastabend test_exec(cgrp, opt); 1492463bac5fSJohn Fastabend 1493463bac5fSJohn Fastabend txmsg_ktls_skb_drop = 0; 1494463bac5fSJohn Fastabend txmsg_ktls_skb_redir = 1; 1495463bac5fSJohn Fastabend test_exec(cgrp, opt); 1496a24fb420SJohn Fastabend txmsg_ktls_skb_redir = 0; 1497a24fb420SJohn Fastabend 1498a24fb420SJohn Fastabend /* Tests that omit skb_parser */ 1499a24fb420SJohn Fastabend txmsg_omit_skb_parser = 1; 1500a24fb420SJohn Fastabend ktls = 0; 1501a24fb420SJohn Fastabend txmsg_ktls_skb = 0; 1502a24fb420SJohn Fastabend test_exec(cgrp, opt); 1503a24fb420SJohn Fastabend 1504a24fb420SJohn Fastabend txmsg_ktls_skb_drop = 1; 1505a24fb420SJohn Fastabend test_exec(cgrp, opt); 1506a24fb420SJohn Fastabend txmsg_ktls_skb_drop = 0; 1507a24fb420SJohn Fastabend 1508a24fb420SJohn Fastabend txmsg_ktls_skb_redir = 1; 1509a24fb420SJohn Fastabend test_exec(cgrp, opt); 1510a24fb420SJohn Fastabend 1511a24fb420SJohn Fastabend ktls = 1; 1512a24fb420SJohn Fastabend test_exec(cgrp, opt); 1513a24fb420SJohn Fastabend txmsg_omit_skb_parser = 0; 1514463bac5fSJohn Fastabend 1515463bac5fSJohn Fastabend opt->data_test = data; 1516463bac5fSJohn Fastabend ktls = k; 1517463bac5fSJohn Fastabend } 1518463bac5fSJohn Fastabend 1519328aa08aSJohn Fastabend /* Test cork with hung data. This tests poor usage patterns where 1520328aa08aSJohn Fastabend * cork can leave data on the ring if user program is buggy and 1521328aa08aSJohn Fastabend * doesn't flush them somehow. They do take some time however 1522328aa08aSJohn Fastabend * because they wait for a timeout. Test pass, redir and cork with 1523328aa08aSJohn Fastabend * apply logic. Use cork size of 4097 with send_large to avoid 1524328aa08aSJohn Fastabend * aligning cork size with send size. 1525328aa08aSJohn Fastabend */ 1526b98ca90cSJohn Fastabend static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1527328aa08aSJohn Fastabend { 1528328aa08aSJohn Fastabend txmsg_pass = 1; 1529328aa08aSJohn Fastabend txmsg_redir = 0; 1530328aa08aSJohn Fastabend txmsg_cork = 4097; 1531328aa08aSJohn Fastabend txmsg_apply = 4097; 1532b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1533328aa08aSJohn Fastabend 1534328aa08aSJohn Fastabend txmsg_pass = 0; 1535328aa08aSJohn Fastabend txmsg_redir = 1; 1536328aa08aSJohn Fastabend txmsg_apply = 0; 1537328aa08aSJohn Fastabend txmsg_cork = 4097; 1538b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1539328aa08aSJohn Fastabend 1540328aa08aSJohn Fastabend txmsg_pass = 0; 1541328aa08aSJohn Fastabend txmsg_redir = 1; 1542328aa08aSJohn Fastabend txmsg_apply = 4097; 1543328aa08aSJohn Fastabend txmsg_cork = 4097; 1544b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1545328aa08aSJohn Fastabend } 1546328aa08aSJohn Fastabend 1547b98ca90cSJohn Fastabend static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1548328aa08aSJohn Fastabend { 1549328aa08aSJohn Fastabend /* Test basic start/end */ 155016962b24SJohn Fastabend txmsg_start = 1; 155116962b24SJohn Fastabend txmsg_end = 2; 1552b98ca90cSJohn Fastabend test_send(opt, cgrp); 1553328aa08aSJohn Fastabend 1554328aa08aSJohn Fastabend /* Test >4k pull */ 1555328aa08aSJohn Fastabend txmsg_start = 4096; 1556328aa08aSJohn Fastabend txmsg_end = 9182; 1557b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1558328aa08aSJohn Fastabend 1559328aa08aSJohn Fastabend /* Test pull + redirect */ 1560328aa08aSJohn Fastabend txmsg_redir = 0; 1561328aa08aSJohn Fastabend txmsg_start = 1; 1562328aa08aSJohn Fastabend txmsg_end = 2; 1563b98ca90cSJohn Fastabend test_send(opt, cgrp); 1564328aa08aSJohn Fastabend 1565328aa08aSJohn Fastabend /* Test pull + cork */ 1566328aa08aSJohn Fastabend txmsg_redir = 0; 1567328aa08aSJohn Fastabend txmsg_cork = 512; 1568328aa08aSJohn Fastabend txmsg_start = 1; 1569328aa08aSJohn Fastabend txmsg_end = 2; 1570b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1571328aa08aSJohn Fastabend 1572328aa08aSJohn Fastabend /* Test pull + cork + redirect */ 1573328aa08aSJohn Fastabend txmsg_redir = 1; 1574328aa08aSJohn Fastabend txmsg_cork = 512; 1575328aa08aSJohn Fastabend txmsg_start = 1; 1576328aa08aSJohn Fastabend txmsg_end = 2; 1577b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1578328aa08aSJohn Fastabend } 1579328aa08aSJohn Fastabend 1580b98ca90cSJohn Fastabend static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1581328aa08aSJohn Fastabend { 1582328aa08aSJohn Fastabend /* Test basic pop */ 1583328aa08aSJohn Fastabend txmsg_start_pop = 1; 1584328aa08aSJohn Fastabend txmsg_pop = 2; 1585b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1586328aa08aSJohn Fastabend 1587328aa08aSJohn Fastabend /* Test pop with >4k */ 1588328aa08aSJohn Fastabend txmsg_start_pop = 4096; 1589328aa08aSJohn Fastabend txmsg_pop = 4096; 1590b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1591328aa08aSJohn Fastabend 1592328aa08aSJohn Fastabend /* Test pop + redirect */ 1593328aa08aSJohn Fastabend txmsg_redir = 1; 1594328aa08aSJohn Fastabend txmsg_start_pop = 1; 1595328aa08aSJohn Fastabend txmsg_pop = 2; 1596b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1597328aa08aSJohn Fastabend 1598328aa08aSJohn Fastabend /* Test pop + cork */ 1599328aa08aSJohn Fastabend txmsg_redir = 0; 1600328aa08aSJohn Fastabend txmsg_cork = 512; 1601328aa08aSJohn Fastabend txmsg_start_pop = 1; 1602328aa08aSJohn Fastabend txmsg_pop = 2; 1603b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1604328aa08aSJohn Fastabend 1605328aa08aSJohn Fastabend /* Test pop + redirect + cork */ 1606328aa08aSJohn Fastabend txmsg_redir = 1; 1607328aa08aSJohn Fastabend txmsg_cork = 4; 1608328aa08aSJohn Fastabend txmsg_start_pop = 1; 1609328aa08aSJohn Fastabend txmsg_pop = 2; 1610b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 1611328aa08aSJohn Fastabend } 1612328aa08aSJohn Fastabend 1613b98ca90cSJohn Fastabend static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1614328aa08aSJohn Fastabend { 1615328aa08aSJohn Fastabend /* Test basic push */ 1616328aa08aSJohn Fastabend txmsg_start_push = 1; 1617328aa08aSJohn Fastabend txmsg_end_push = 1; 1618b98ca90cSJohn Fastabend test_send(opt, cgrp); 1619328aa08aSJohn Fastabend 1620328aa08aSJohn Fastabend /* Test push 4kB >4k */ 1621328aa08aSJohn Fastabend txmsg_start_push = 4096; 1622328aa08aSJohn Fastabend txmsg_end_push = 4096; 1623b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1624328aa08aSJohn Fastabend 1625328aa08aSJohn Fastabend /* Test push + redirect */ 1626328aa08aSJohn Fastabend txmsg_redir = 1; 162784fbfe02SJohn Fastabend txmsg_start_push = 1; 162884fbfe02SJohn Fastabend txmsg_end_push = 2; 1629b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 16301ade9abaSJohn Fastabend 1631328aa08aSJohn Fastabend /* Test push + cork */ 1632328aa08aSJohn Fastabend txmsg_redir = 0; 1633328aa08aSJohn Fastabend txmsg_cork = 512; 16341ade9abaSJohn Fastabend txmsg_start_push = 1; 1635328aa08aSJohn Fastabend txmsg_end_push = 2; 1636b98ca90cSJohn Fastabend test_send_many(opt, cgrp); 163716962b24SJohn Fastabend } 163816962b24SJohn Fastabend 1639b98ca90cSJohn Fastabend static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1640328aa08aSJohn Fastabend { 1641328aa08aSJohn Fastabend txmsg_start_push = 1; 1642328aa08aSJohn Fastabend txmsg_end_push = 10; 1643328aa08aSJohn Fastabend txmsg_start_pop = 5; 1644328aa08aSJohn Fastabend txmsg_pop = 4; 1645b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 164616962b24SJohn Fastabend } 164716962b24SJohn Fastabend 1648b98ca90cSJohn Fastabend static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1649328aa08aSJohn Fastabend { 1650328aa08aSJohn Fastabend txmsg_pass = 1; 1651328aa08aSJohn Fastabend txmsg_redir = 0; 1652328aa08aSJohn Fastabend txmsg_apply = 1; 1653328aa08aSJohn Fastabend txmsg_cork = 0; 1654b98ca90cSJohn Fastabend test_send_one(opt, cgrp); 16551ade9abaSJohn Fastabend 1656328aa08aSJohn Fastabend txmsg_pass = 0; 1657328aa08aSJohn Fastabend txmsg_redir = 1; 1658328aa08aSJohn Fastabend txmsg_apply = 1; 1659328aa08aSJohn Fastabend txmsg_cork = 0; 1660b98ca90cSJohn Fastabend test_send_one(opt, cgrp); 166116962b24SJohn Fastabend 1662328aa08aSJohn Fastabend txmsg_pass = 1; 1663328aa08aSJohn Fastabend txmsg_redir = 0; 1664328aa08aSJohn Fastabend txmsg_apply = 1024; 1665328aa08aSJohn Fastabend txmsg_cork = 0; 1666b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 16671ade9abaSJohn Fastabend 1668328aa08aSJohn Fastabend txmsg_pass = 0; 1669328aa08aSJohn Fastabend txmsg_redir = 1; 1670328aa08aSJohn Fastabend txmsg_apply = 1024; 1671328aa08aSJohn Fastabend txmsg_cork = 0; 1672b98ca90cSJohn Fastabend test_send_large(opt, cgrp); 1673328aa08aSJohn Fastabend } 167416962b24SJohn Fastabend 1675b98ca90cSJohn Fastabend static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1676328aa08aSJohn Fastabend { 1677328aa08aSJohn Fastabend txmsg_pass = 1; 1678328aa08aSJohn Fastabend txmsg_redir = 0; 1679328aa08aSJohn Fastabend txmsg_apply = 0; 1680328aa08aSJohn Fastabend txmsg_cork = 1; 1681b98ca90cSJohn Fastabend test_send(opt, cgrp); 168216962b24SJohn Fastabend 1683328aa08aSJohn Fastabend txmsg_pass = 1; 1684328aa08aSJohn Fastabend txmsg_redir = 0; 1685328aa08aSJohn Fastabend txmsg_apply = 1; 1686328aa08aSJohn Fastabend txmsg_cork = 1; 1687b98ca90cSJohn Fastabend test_send(opt, cgrp); 168816962b24SJohn Fastabend } 168916962b24SJohn Fastabend 169053792fa4SJohn Fastabend static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 169153792fa4SJohn Fastabend { 169253792fa4SJohn Fastabend txmsg_pass = 1; 169353792fa4SJohn Fastabend skb_use_parser = 512; 1694b556c3fdSLiu Jian if (ktls == 1) 1695b556c3fdSLiu Jian skb_use_parser = 570; 169653792fa4SJohn Fastabend opt->iov_length = 256; 169753792fa4SJohn Fastabend opt->iov_count = 1; 169853792fa4SJohn Fastabend opt->rate = 2; 169953792fa4SJohn Fastabend test_exec(cgrp, opt); 170053792fa4SJohn Fastabend } 170153792fa4SJohn Fastabend 1702d6967214SLiu Jian static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1703d6967214SLiu Jian { 1704d6967214SLiu Jian if (ktls == 1) 1705d6967214SLiu Jian return; 1706d6967214SLiu Jian skb_use_parser = 10; 1707d6967214SLiu Jian opt->iov_length = 20; 1708d6967214SLiu Jian opt->iov_count = 1; 1709d6967214SLiu Jian opt->rate = 1; 1710d6967214SLiu Jian opt->check_recved_len = true; 1711d6967214SLiu Jian test_exec(cgrp, opt); 1712d6967214SLiu Jian opt->check_recved_len = false; 1713d6967214SLiu Jian } 1714d6967214SLiu Jian 171516962b24SJohn Fastabend char *map_names[] = { 171616962b24SJohn Fastabend "sock_map", 171716962b24SJohn Fastabend "sock_map_txmsg", 171816962b24SJohn Fastabend "sock_map_redir", 171916962b24SJohn Fastabend "sock_apply_bytes", 172016962b24SJohn Fastabend "sock_cork_bytes", 172184fbfe02SJohn Fastabend "sock_bytes", 172216962b24SJohn Fastabend "sock_redir_flags", 172316962b24SJohn Fastabend "sock_skb_opts", 1724463bac5fSJohn Fastabend "tls_sock_map", 172516962b24SJohn Fastabend }; 172616962b24SJohn Fastabend 172716962b24SJohn Fastabend int prog_attach_type[] = { 172816962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 172916962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 1730463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 173116962b24SJohn Fastabend BPF_CGROUP_SOCK_OPS, 173216962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173316962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173416962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173516962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173616962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173716962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173816962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 173916962b24SJohn Fastabend }; 174016962b24SJohn Fastabend 174116962b24SJohn Fastabend int prog_type[] = { 174216962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 174316962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 1744463bac5fSJohn Fastabend BPF_PROG_TYPE_SK_SKB, 174516962b24SJohn Fastabend BPF_PROG_TYPE_SOCK_OPS, 174616962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 174716962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 174816962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 174916962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 175016962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 175116962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 175216962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 175316962b24SJohn Fastabend }; 175416962b24SJohn Fastabend 1755b8b394faSJohn Fastabend static int populate_progs(char *bpf_file) 175616962b24SJohn Fastabend { 175716962b24SJohn Fastabend struct bpf_program *prog; 175816962b24SJohn Fastabend struct bpf_object *obj; 175916962b24SJohn Fastabend int i = 0; 176016962b24SJohn Fastabend long err; 176116962b24SJohn Fastabend 176216962b24SJohn Fastabend obj = bpf_object__open(bpf_file); 176316962b24SJohn Fastabend err = libbpf_get_error(obj); 176416962b24SJohn Fastabend if (err) { 176516962b24SJohn Fastabend char err_buf[256]; 176616962b24SJohn Fastabend 176716962b24SJohn Fastabend libbpf_strerror(err, err_buf, sizeof(err_buf)); 176816962b24SJohn Fastabend printf("Unable to load eBPF objects in file '%s' : %s\n", 176916962b24SJohn Fastabend bpf_file, err_buf); 177016962b24SJohn Fastabend return -1; 177116962b24SJohn Fastabend } 177216962b24SJohn Fastabend 177316962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 177416962b24SJohn Fastabend bpf_program__set_type(prog, prog_type[i]); 177516962b24SJohn Fastabend bpf_program__set_expected_attach_type(prog, 177616962b24SJohn Fastabend prog_attach_type[i]); 177716962b24SJohn Fastabend i++; 177816962b24SJohn Fastabend } 177916962b24SJohn Fastabend 178016962b24SJohn Fastabend i = bpf_object__load(obj); 178116962b24SJohn Fastabend i = 0; 178216962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 178316962b24SJohn Fastabend prog_fd[i] = bpf_program__fd(prog); 178416962b24SJohn Fastabend i++; 178516962b24SJohn Fastabend } 178616962b24SJohn Fastabend 1787f98d6dd1SGuo Zhengkui for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 178816962b24SJohn Fastabend maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 178916962b24SJohn Fastabend map_fd[i] = bpf_map__fd(maps[i]); 179016962b24SJohn Fastabend if (map_fd[i] < 0) { 179116962b24SJohn Fastabend fprintf(stderr, "load_bpf_file: (%i) %s\n", 179216962b24SJohn Fastabend map_fd[i], strerror(errno)); 179316962b24SJohn Fastabend return -1; 179416962b24SJohn Fastabend } 179516962b24SJohn Fastabend } 179616962b24SJohn Fastabend 179716962b24SJohn Fastabend return 0; 179816962b24SJohn Fastabend } 179916962b24SJohn Fastabend 1800328aa08aSJohn Fastabend struct _test test[] = { 1801328aa08aSJohn Fastabend {"txmsg test passthrough", test_txmsg_pass}, 1802328aa08aSJohn Fastabend {"txmsg test redirect", test_txmsg_redir}, 1803328aa08aSJohn Fastabend {"txmsg test drop", test_txmsg_drop}, 1804328aa08aSJohn Fastabend {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1805463bac5fSJohn Fastabend {"txmsg test skb", test_txmsg_skb}, 1806328aa08aSJohn Fastabend {"txmsg test apply", test_txmsg_apply}, 1807328aa08aSJohn Fastabend {"txmsg test cork", test_txmsg_cork}, 1808328aa08aSJohn Fastabend {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1809328aa08aSJohn Fastabend {"txmsg test push_data", test_txmsg_push}, 1810328aa08aSJohn Fastabend {"txmsg test pull-data", test_txmsg_pull}, 1811328aa08aSJohn Fastabend {"txmsg test pop-data", test_txmsg_pop}, 1812328aa08aSJohn Fastabend {"txmsg test push/pop data", test_txmsg_push_pop}, 1813d6967214SLiu Jian {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1814d6967214SLiu Jian {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1815328aa08aSJohn Fastabend }; 1816328aa08aSJohn Fastabend 1817065a74cbSJohn Fastabend static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1818065a74cbSJohn Fastabend { 1819065a74cbSJohn Fastabend char *entry, *ptr; 1820065a74cbSJohn Fastabend 1821065a74cbSJohn Fastabend if (!opt->whitelist) 1822065a74cbSJohn Fastabend return 0; 1823065a74cbSJohn Fastabend ptr = strdup(opt->whitelist); 1824065a74cbSJohn Fastabend if (!ptr) 1825065a74cbSJohn Fastabend return -ENOMEM; 1826065a74cbSJohn Fastabend entry = strtok(ptr, ","); 1827065a74cbSJohn Fastabend while (entry) { 182896586dd9SJohn Fastabend if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 182996586dd9SJohn Fastabend strstr(opt->map, entry) != 0 || 183096586dd9SJohn Fastabend strstr(t->title, entry) != 0) 1831065a74cbSJohn Fastabend return 0; 1832065a74cbSJohn Fastabend entry = strtok(NULL, ","); 1833065a74cbSJohn Fastabend } 1834065a74cbSJohn Fastabend return -EINVAL; 1835065a74cbSJohn Fastabend } 1836065a74cbSJohn Fastabend 1837a7238f7cSJohn Fastabend static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1838a7238f7cSJohn Fastabend { 1839a7238f7cSJohn Fastabend char *entry, *ptr; 1840a7238f7cSJohn Fastabend 1841a7238f7cSJohn Fastabend if (!opt->blacklist) 1842a7238f7cSJohn Fastabend return -EINVAL; 1843a7238f7cSJohn Fastabend ptr = strdup(opt->blacklist); 1844a7238f7cSJohn Fastabend if (!ptr) 1845a7238f7cSJohn Fastabend return -ENOMEM; 1846a7238f7cSJohn Fastabend entry = strtok(ptr, ","); 1847a7238f7cSJohn Fastabend while (entry) { 184896586dd9SJohn Fastabend if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 184996586dd9SJohn Fastabend strstr(opt->map, entry) != 0 || 185096586dd9SJohn Fastabend strstr(t->title, entry) != 0) 1851a7238f7cSJohn Fastabend return 0; 1852a7238f7cSJohn Fastabend entry = strtok(NULL, ","); 1853a7238f7cSJohn Fastabend } 1854a7238f7cSJohn Fastabend return -EINVAL; 1855a7238f7cSJohn Fastabend } 1856a7238f7cSJohn Fastabend 1857b98ca90cSJohn Fastabend static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1858328aa08aSJohn Fastabend { 1859328aa08aSJohn Fastabend int i, err; 1860328aa08aSJohn Fastabend 1861b98ca90cSJohn Fastabend err = populate_progs(opt->map); 186216962b24SJohn Fastabend if (err < 0) { 186316962b24SJohn Fastabend fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 186416962b24SJohn Fastabend return err; 186516962b24SJohn Fastabend } 186616962b24SJohn Fastabend 1867328aa08aSJohn Fastabend /* Tests basic commands and APIs */ 1868f98d6dd1SGuo Zhengkui for (i = 0; i < ARRAY_SIZE(test); i++) { 1869328aa08aSJohn Fastabend struct _test t = test[i]; 1870328aa08aSJohn Fastabend 1871a7238f7cSJohn Fastabend if (check_whitelist(&t, opt) != 0) 1872a7238f7cSJohn Fastabend continue; 1873a7238f7cSJohn Fastabend if (check_blacklist(&t, opt) == 0) 1874065a74cbSJohn Fastabend continue; 1875065a74cbSJohn Fastabend 187696586dd9SJohn Fastabend test_start_subtest(&t, opt); 1877b98ca90cSJohn Fastabend t.tester(cg_fd, opt); 1878328aa08aSJohn Fastabend test_end_subtest(); 1879328aa08aSJohn Fastabend } 1880328aa08aSJohn Fastabend 1881328aa08aSJohn Fastabend return err; 1882328aa08aSJohn Fastabend } 1883328aa08aSJohn Fastabend 1884b98ca90cSJohn Fastabend static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1885328aa08aSJohn Fastabend { 1886b98ca90cSJohn Fastabend opt->map = BPF_SOCKMAP_FILENAME; 1887b98ca90cSJohn Fastabend __test_selftests(cg_fd, opt); 1888328aa08aSJohn Fastabend } 1889328aa08aSJohn Fastabend 1890b98ca90cSJohn Fastabend static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1891328aa08aSJohn Fastabend { 1892b98ca90cSJohn Fastabend opt->map = BPF_SOCKHASH_FILENAME; 1893b98ca90cSJohn Fastabend __test_selftests(cg_fd, opt); 1894328aa08aSJohn Fastabend } 1895328aa08aSJohn Fastabend 189696586dd9SJohn Fastabend static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 189796586dd9SJohn Fastabend { 189896586dd9SJohn Fastabend opt->map = BPF_SOCKHASH_FILENAME; 189996586dd9SJohn Fastabend opt->prepend = "ktls"; 190096586dd9SJohn Fastabend ktls = 1; 190196586dd9SJohn Fastabend __test_selftests(cg_fd, opt); 190296586dd9SJohn Fastabend ktls = 0; 190396586dd9SJohn Fastabend } 190496586dd9SJohn Fastabend 1905b98ca90cSJohn Fastabend static int test_selftest(int cg_fd, struct sockmap_options *opt) 1906328aa08aSJohn Fastabend { 190716962b24SJohn Fastabend 1908b98ca90cSJohn Fastabend test_selftests_sockmap(cg_fd, opt); 1909b98ca90cSJohn Fastabend test_selftests_sockhash(cg_fd, opt); 191096586dd9SJohn Fastabend test_selftests_ktls(cg_fd, opt); 1911328aa08aSJohn Fastabend test_print_results(); 1912328aa08aSJohn Fastabend return 0; 1913b8b394faSJohn Fastabend } 1914b8b394faSJohn Fastabend 191516962b24SJohn Fastabend int main(int argc, char **argv) 191616962b24SJohn Fastabend { 191716962b24SJohn Fastabend int iov_count = 1, length = 1024, rate = 1; 191816962b24SJohn Fastabend struct sockmap_options options = {0}; 191916962b24SJohn Fastabend int opt, longindex, err, cg_fd = 0; 1920b8b394faSJohn Fastabend char *bpf_file = BPF_SOCKMAP_FILENAME; 1921b98ca90cSJohn Fastabend int test = SELFTESTS; 192213a5f3ffSJohn Fastabend bool cg_created = 0; 192316962b24SJohn Fastabend 1924a7238f7cSJohn Fastabend while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 192516962b24SJohn Fastabend long_options, &longindex)) != -1) { 192616962b24SJohn Fastabend switch (opt) { 192716962b24SJohn Fastabend case 's': 192816962b24SJohn Fastabend txmsg_start = atoi(optarg); 192916962b24SJohn Fastabend break; 193016962b24SJohn Fastabend case 'e': 193116962b24SJohn Fastabend txmsg_end = atoi(optarg); 193216962b24SJohn Fastabend break; 193384fbfe02SJohn Fastabend case 'p': 193484fbfe02SJohn Fastabend txmsg_start_push = atoi(optarg); 193584fbfe02SJohn Fastabend break; 193684fbfe02SJohn Fastabend case 'q': 193784fbfe02SJohn Fastabend txmsg_end_push = atoi(optarg); 193884fbfe02SJohn Fastabend break; 19391ade9abaSJohn Fastabend case 'w': 19401ade9abaSJohn Fastabend txmsg_start_pop = atoi(optarg); 19411ade9abaSJohn Fastabend break; 19421ade9abaSJohn Fastabend case 'x': 19431ade9abaSJohn Fastabend txmsg_pop = atoi(optarg); 19441ade9abaSJohn Fastabend break; 194516962b24SJohn Fastabend case 'a': 194616962b24SJohn Fastabend txmsg_apply = atoi(optarg); 194716962b24SJohn Fastabend break; 194816962b24SJohn Fastabend case 'k': 194916962b24SJohn Fastabend txmsg_cork = atoi(optarg); 195016962b24SJohn Fastabend break; 195116962b24SJohn Fastabend case 'c': 195216962b24SJohn Fastabend cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 195316962b24SJohn Fastabend if (cg_fd < 0) { 195416962b24SJohn Fastabend fprintf(stderr, 195516962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 195616962b24SJohn Fastabend cg_fd, optarg); 195716962b24SJohn Fastabend return cg_fd; 195816962b24SJohn Fastabend } 195916962b24SJohn Fastabend break; 196016962b24SJohn Fastabend case 'r': 196116962b24SJohn Fastabend rate = atoi(optarg); 196216962b24SJohn Fastabend break; 196316962b24SJohn Fastabend case 'v': 196416962b24SJohn Fastabend options.verbose = 1; 1965b98ca90cSJohn Fastabend if (optarg) 1966b98ca90cSJohn Fastabend options.verbose = atoi(optarg); 196716962b24SJohn Fastabend break; 196816962b24SJohn Fastabend case 'i': 196916962b24SJohn Fastabend iov_count = atoi(optarg); 197016962b24SJohn Fastabend break; 197116962b24SJohn Fastabend case 'l': 197216962b24SJohn Fastabend length = atoi(optarg); 197316962b24SJohn Fastabend break; 197416962b24SJohn Fastabend case 'd': 197516962b24SJohn Fastabend options.data_test = true; 197616962b24SJohn Fastabend break; 197716962b24SJohn Fastabend case 't': 197816962b24SJohn Fastabend if (strcmp(optarg, "ping") == 0) { 197916962b24SJohn Fastabend test = PING_PONG; 198016962b24SJohn Fastabend } else if (strcmp(optarg, "sendmsg") == 0) { 198116962b24SJohn Fastabend test = SENDMSG; 198216962b24SJohn Fastabend } else if (strcmp(optarg, "base") == 0) { 198316962b24SJohn Fastabend test = BASE; 198416962b24SJohn Fastabend } else if (strcmp(optarg, "base_sendpage") == 0) { 198516962b24SJohn Fastabend test = BASE_SENDPAGE; 198616962b24SJohn Fastabend } else if (strcmp(optarg, "sendpage") == 0) { 198716962b24SJohn Fastabend test = SENDPAGE; 198816962b24SJohn Fastabend } else { 198916962b24SJohn Fastabend usage(argv); 199016962b24SJohn Fastabend return -1; 199116962b24SJohn Fastabend } 199216962b24SJohn Fastabend break; 1993065a74cbSJohn Fastabend case 'n': 1994065a74cbSJohn Fastabend options.whitelist = strdup(optarg); 1995065a74cbSJohn Fastabend if (!options.whitelist) 1996065a74cbSJohn Fastabend return -ENOMEM; 1997a7238f7cSJohn Fastabend break; 1998a7238f7cSJohn Fastabend case 'b': 1999a7238f7cSJohn Fastabend options.blacklist = strdup(optarg); 2000a7238f7cSJohn Fastabend if (!options.blacklist) 2001a7238f7cSJohn Fastabend return -ENOMEM; 200216962b24SJohn Fastabend case 0: 200316962b24SJohn Fastabend break; 200416962b24SJohn Fastabend case 'h': 200516962b24SJohn Fastabend default: 200616962b24SJohn Fastabend usage(argv); 200716962b24SJohn Fastabend return -1; 200816962b24SJohn Fastabend } 200916962b24SJohn Fastabend } 201016962b24SJohn Fastabend 201116962b24SJohn Fastabend if (!cg_fd) { 20124939b284SJohn Fastabend cg_fd = cgroup_setup_and_join(CG_PATH); 20134939b284SJohn Fastabend if (cg_fd < 0) 201413a5f3ffSJohn Fastabend return cg_fd; 201513a5f3ffSJohn Fastabend cg_created = 1; 201616962b24SJohn Fastabend } 201716962b24SJohn Fastabend 2018b858ba8cSYafang Shao /* Use libbpf 1.0 API mode */ 2019b858ba8cSYafang Shao libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2020b858ba8cSYafang Shao 2021b98ca90cSJohn Fastabend if (test == SELFTESTS) { 2022b98ca90cSJohn Fastabend err = test_selftest(cg_fd, &options); 2023b98ca90cSJohn Fastabend goto out; 2024b98ca90cSJohn Fastabend } 2025b98ca90cSJohn Fastabend 2026b8b394faSJohn Fastabend err = populate_progs(bpf_file); 202716962b24SJohn Fastabend if (err) { 202816962b24SJohn Fastabend fprintf(stderr, "populate program: (%s) %s\n", 202916962b24SJohn Fastabend bpf_file, strerror(errno)); 203016962b24SJohn Fastabend return 1; 203116962b24SJohn Fastabend } 203216962b24SJohn Fastabend running = 1; 203316962b24SJohn Fastabend 203416962b24SJohn Fastabend /* catch SIGINT */ 203516962b24SJohn Fastabend signal(SIGINT, running_handler); 203616962b24SJohn Fastabend 203716962b24SJohn Fastabend options.iov_count = iov_count; 203816962b24SJohn Fastabend options.iov_length = length; 203916962b24SJohn Fastabend options.rate = rate; 204016962b24SJohn Fastabend 204116962b24SJohn Fastabend err = run_options(&options, cg_fd, test); 2042b98ca90cSJohn Fastabend out: 2043065a74cbSJohn Fastabend if (options.whitelist) 2044065a74cbSJohn Fastabend free(options.whitelist); 2045a7238f7cSJohn Fastabend if (options.blacklist) 2046a7238f7cSJohn Fastabend free(options.blacklist); 204713a5f3ffSJohn Fastabend if (cg_created) 204813a5f3ffSJohn Fastabend cleanup_cgroup_environment(); 204916962b24SJohn Fastabend close(cg_fd); 205016962b24SJohn Fastabend return err; 205116962b24SJohn Fastabend } 205216962b24SJohn Fastabend 205316962b24SJohn Fastabend void running_handler(int a) 205416962b24SJohn Fastabend { 205516962b24SJohn Fastabend running = 0; 205616962b24SJohn Fastabend } 2057