116962b24SJohn Fastabend // SPDX-License-Identifier: GPL-2.0
216962b24SJohn Fastabend // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
316962b24SJohn Fastabend #include <stdio.h>
416962b24SJohn Fastabend #include <stdlib.h>
516962b24SJohn Fastabend #include <sys/socket.h>
616962b24SJohn Fastabend #include <sys/ioctl.h>
716962b24SJohn Fastabend #include <sys/select.h>
816962b24SJohn Fastabend #include <netinet/in.h>
916962b24SJohn Fastabend #include <arpa/inet.h>
1016962b24SJohn Fastabend #include <unistd.h>
1116962b24SJohn Fastabend #include <string.h>
1216962b24SJohn Fastabend #include <errno.h>
1316962b24SJohn Fastabend #include <stdbool.h>
1416962b24SJohn Fastabend #include <signal.h>
1516962b24SJohn Fastabend #include <fcntl.h>
1616962b24SJohn Fastabend #include <sys/wait.h>
1716962b24SJohn Fastabend #include <time.h>
1816962b24SJohn Fastabend #include <sched.h>
1916962b24SJohn Fastabend
2016962b24SJohn Fastabend #include <sys/time.h>
2116962b24SJohn Fastabend #include <sys/types.h>
2216962b24SJohn Fastabend #include <sys/sendfile.h>
2316962b24SJohn Fastabend
2416962b24SJohn Fastabend #include <linux/netlink.h>
2516962b24SJohn Fastabend #include <linux/socket.h>
2616962b24SJohn Fastabend #include <linux/sock_diag.h>
2716962b24SJohn Fastabend #include <linux/bpf.h>
2816962b24SJohn Fastabend #include <linux/if_link.h>
29421f4292SDaniel Borkmann #include <linux/tls.h>
3016962b24SJohn Fastabend #include <assert.h>
3116962b24SJohn Fastabend #include <libgen.h>
3216962b24SJohn Fastabend
3316962b24SJohn Fastabend #include <getopt.h>
3416962b24SJohn Fastabend
3516962b24SJohn Fastabend #include <bpf/bpf.h>
3616962b24SJohn Fastabend #include <bpf/libbpf.h>
3716962b24SJohn Fastabend
3816962b24SJohn Fastabend #include "bpf_util.h"
3916962b24SJohn Fastabend #include "cgroup_helpers.h"
4016962b24SJohn Fastabend
4116962b24SJohn Fastabend int running;
4216962b24SJohn Fastabend static void running_handler(int a);
4316962b24SJohn Fastabend
44421f4292SDaniel Borkmann #ifndef TCP_ULP
45421f4292SDaniel Borkmann # define TCP_ULP 31
46421f4292SDaniel Borkmann #endif
47421f4292SDaniel Borkmann #ifndef SOL_TLS
48421f4292SDaniel Borkmann # define SOL_TLS 282
49421f4292SDaniel Borkmann #endif
50421f4292SDaniel Borkmann
5116962b24SJohn Fastabend /* randomly selected ports for testing on lo */
5216962b24SJohn Fastabend #define S1_PORT 10000
5316962b24SJohn Fastabend #define S2_PORT 10001
5416962b24SJohn Fastabend
55afef88e6SDaniel Müller #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56afef88e6SDaniel Müller #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
5716962b24SJohn Fastabend #define CG_PATH "/sockmap"
5816962b24SJohn Fastabend
5916962b24SJohn Fastabend /* global sockets */
6016962b24SJohn Fastabend int s1, s2, c1, c2, p1, p2;
6116962b24SJohn Fastabend int test_cnt;
6216962b24SJohn Fastabend int passed;
6316962b24SJohn Fastabend int failed;
64463bac5fSJohn Fastabend int map_fd[9];
65463bac5fSJohn Fastabend struct bpf_map *maps[9];
6616962b24SJohn Fastabend int prog_fd[11];
6716962b24SJohn Fastabend
6816962b24SJohn Fastabend int txmsg_pass;
6916962b24SJohn Fastabend int txmsg_redir;
7016962b24SJohn Fastabend int txmsg_drop;
7116962b24SJohn Fastabend int txmsg_apply;
7216962b24SJohn Fastabend int txmsg_cork;
7316962b24SJohn Fastabend int txmsg_start;
7416962b24SJohn Fastabend int txmsg_end;
7584fbfe02SJohn Fastabend int txmsg_start_push;
7684fbfe02SJohn Fastabend int txmsg_end_push;
771ade9abaSJohn Fastabend int txmsg_start_pop;
781ade9abaSJohn Fastabend int txmsg_pop;
7916962b24SJohn Fastabend int txmsg_ingress;
80463bac5fSJohn Fastabend int txmsg_redir_skb;
81463bac5fSJohn Fastabend int txmsg_ktls_skb;
82463bac5fSJohn Fastabend int txmsg_ktls_skb_drop;
83463bac5fSJohn Fastabend int txmsg_ktls_skb_redir;
84e9dd9047SJohn Fastabend int ktls;
85753fb2eeSJohn Fastabend int peek_flag;
8653792fa4SJohn Fastabend int skb_use_parser;
87cdf43c4bSJohn Fastabend int txmsg_omit_skb_parser;
8816962b24SJohn Fastabend
8916962b24SJohn Fastabend static const struct option long_options[] = {
9016962b24SJohn Fastabend {"help", no_argument, NULL, 'h' },
9116962b24SJohn Fastabend {"cgroup", required_argument, NULL, 'c' },
9216962b24SJohn Fastabend {"rate", required_argument, NULL, 'r' },
93b98ca90cSJohn Fastabend {"verbose", optional_argument, NULL, 'v' },
9416962b24SJohn Fastabend {"iov_count", required_argument, NULL, 'i' },
9516962b24SJohn Fastabend {"length", required_argument, NULL, 'l' },
9616962b24SJohn Fastabend {"test", required_argument, NULL, 't' },
9716962b24SJohn Fastabend {"data_test", no_argument, NULL, 'd' },
9816962b24SJohn Fastabend {"txmsg", no_argument, &txmsg_pass, 1 },
9916962b24SJohn Fastabend {"txmsg_redir", no_argument, &txmsg_redir, 1 },
10016962b24SJohn Fastabend {"txmsg_drop", no_argument, &txmsg_drop, 1 },
10116962b24SJohn Fastabend {"txmsg_apply", required_argument, NULL, 'a'},
10216962b24SJohn Fastabend {"txmsg_cork", required_argument, NULL, 'k'},
10316962b24SJohn Fastabend {"txmsg_start", required_argument, NULL, 's'},
10416962b24SJohn Fastabend {"txmsg_end", required_argument, NULL, 'e'},
10584fbfe02SJohn Fastabend {"txmsg_start_push", required_argument, NULL, 'p'},
10684fbfe02SJohn Fastabend {"txmsg_end_push", required_argument, NULL, 'q'},
1071ade9abaSJohn Fastabend {"txmsg_start_pop", required_argument, NULL, 'w'},
1081ade9abaSJohn Fastabend {"txmsg_pop", required_argument, NULL, 'x'},
10916962b24SJohn Fastabend {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
110463bac5fSJohn Fastabend {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
111e9dd9047SJohn Fastabend {"ktls", no_argument, &ktls, 1 },
112753fb2eeSJohn Fastabend {"peek", no_argument, &peek_flag, 1 },
113cdf43c4bSJohn Fastabend {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
114065a74cbSJohn Fastabend {"whitelist", required_argument, NULL, 'n' },
115a7238f7cSJohn Fastabend {"blacklist", required_argument, NULL, 'b' },
11616962b24SJohn Fastabend {0, 0, NULL, 0 }
11716962b24SJohn Fastabend };
11816962b24SJohn Fastabend
119328aa08aSJohn Fastabend struct test_env {
120328aa08aSJohn Fastabend const char *type;
121328aa08aSJohn Fastabend const char *subtest;
12296586dd9SJohn Fastabend const char *prepend;
123328aa08aSJohn Fastabend
124328aa08aSJohn Fastabend int test_num;
125328aa08aSJohn Fastabend int subtest_num;
126328aa08aSJohn Fastabend
127328aa08aSJohn Fastabend int succ_cnt;
128328aa08aSJohn Fastabend int fail_cnt;
129328aa08aSJohn Fastabend int fail_last;
130328aa08aSJohn Fastabend };
131328aa08aSJohn Fastabend
132328aa08aSJohn Fastabend struct test_env env;
133328aa08aSJohn Fastabend
13496586dd9SJohn Fastabend struct sockmap_options {
13596586dd9SJohn Fastabend int verbose;
13696586dd9SJohn Fastabend bool base;
13796586dd9SJohn Fastabend bool sendpage;
13896586dd9SJohn Fastabend bool data_test;
13996586dd9SJohn Fastabend bool drop_expected;
140d6967214SLiu Jian bool check_recved_len;
141043a7356SLiu Jian bool tx_wait_mem;
14296586dd9SJohn Fastabend int iov_count;
14396586dd9SJohn Fastabend int iov_length;
14496586dd9SJohn Fastabend int rate;
14596586dd9SJohn Fastabend char *map;
14696586dd9SJohn Fastabend char *whitelist;
14796586dd9SJohn Fastabend char *blacklist;
14896586dd9SJohn Fastabend char *prepend;
14996586dd9SJohn Fastabend };
15096586dd9SJohn Fastabend
15196586dd9SJohn Fastabend struct _test {
15296586dd9SJohn Fastabend char *title;
15396586dd9SJohn Fastabend void (*tester)(int cg_fd, struct sockmap_options *opt);
15496586dd9SJohn Fastabend };
15596586dd9SJohn Fastabend
test_start(void)156328aa08aSJohn Fastabend static void test_start(void)
157328aa08aSJohn Fastabend {
158328aa08aSJohn Fastabend env.subtest_num++;
159328aa08aSJohn Fastabend }
160328aa08aSJohn Fastabend
test_fail(void)161328aa08aSJohn Fastabend static void test_fail(void)
162328aa08aSJohn Fastabend {
163328aa08aSJohn Fastabend env.fail_cnt++;
164328aa08aSJohn Fastabend }
165328aa08aSJohn Fastabend
test_pass(void)166328aa08aSJohn Fastabend static void test_pass(void)
167328aa08aSJohn Fastabend {
168328aa08aSJohn Fastabend env.succ_cnt++;
169328aa08aSJohn Fastabend }
170328aa08aSJohn Fastabend
test_reset(void)171328aa08aSJohn Fastabend static void test_reset(void)
172328aa08aSJohn Fastabend {
173328aa08aSJohn Fastabend txmsg_start = txmsg_end = 0;
174328aa08aSJohn Fastabend txmsg_start_pop = txmsg_pop = 0;
175328aa08aSJohn Fastabend txmsg_start_push = txmsg_end_push = 0;
176328aa08aSJohn Fastabend txmsg_pass = txmsg_drop = txmsg_redir = 0;
177328aa08aSJohn Fastabend txmsg_apply = txmsg_cork = 0;
178463bac5fSJohn Fastabend txmsg_ingress = txmsg_redir_skb = 0;
179463bac5fSJohn Fastabend txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
180cdf43c4bSJohn Fastabend txmsg_omit_skb_parser = 0;
18153792fa4SJohn Fastabend skb_use_parser = 0;
182328aa08aSJohn Fastabend }
183328aa08aSJohn Fastabend
test_start_subtest(const struct _test * t,struct sockmap_options * o)18496586dd9SJohn Fastabend static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
185328aa08aSJohn Fastabend {
18696586dd9SJohn Fastabend env.type = o->map;
18796586dd9SJohn Fastabend env.subtest = t->title;
18896586dd9SJohn Fastabend env.prepend = o->prepend;
189328aa08aSJohn Fastabend env.test_num++;
190328aa08aSJohn Fastabend env.subtest_num = 0;
191328aa08aSJohn Fastabend env.fail_last = env.fail_cnt;
192328aa08aSJohn Fastabend test_reset();
193328aa08aSJohn Fastabend return 0;
194328aa08aSJohn Fastabend }
195328aa08aSJohn Fastabend
test_end_subtest(void)196328aa08aSJohn Fastabend static void test_end_subtest(void)
197328aa08aSJohn Fastabend {
198328aa08aSJohn Fastabend int error = env.fail_cnt - env.fail_last;
199328aa08aSJohn Fastabend int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
200328aa08aSJohn Fastabend
201328aa08aSJohn Fastabend if (!error)
202328aa08aSJohn Fastabend test_pass();
203328aa08aSJohn Fastabend
20496586dd9SJohn Fastabend fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
205328aa08aSJohn Fastabend env.test_num, env.subtest_num,
206328aa08aSJohn Fastabend !type ? "sockmap" : "sockhash",
20796586dd9SJohn Fastabend env.prepend ? : "",
208328aa08aSJohn Fastabend env.subtest, error ? "FAIL" : "OK");
209328aa08aSJohn Fastabend }
210328aa08aSJohn Fastabend
test_print_results(void)211328aa08aSJohn Fastabend static void test_print_results(void)
212328aa08aSJohn Fastabend {
213328aa08aSJohn Fastabend fprintf(stdout, "Pass: %d Fail: %d\n",
214328aa08aSJohn Fastabend env.succ_cnt, env.fail_cnt);
215328aa08aSJohn Fastabend }
216328aa08aSJohn Fastabend
usage(char * argv[])21716962b24SJohn Fastabend static void usage(char *argv[])
21816962b24SJohn Fastabend {
21916962b24SJohn Fastabend int i;
22016962b24SJohn Fastabend
22116962b24SJohn Fastabend printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
22216962b24SJohn Fastabend printf(" options:\n");
22316962b24SJohn Fastabend for (i = 0; long_options[i].name != 0; i++) {
22416962b24SJohn Fastabend printf(" --%-12s", long_options[i].name);
22516962b24SJohn Fastabend if (long_options[i].flag != NULL)
22616962b24SJohn Fastabend printf(" flag (internal value:%d)\n",
22716962b24SJohn Fastabend *long_options[i].flag);
22816962b24SJohn Fastabend else
22916962b24SJohn Fastabend printf(" -%c\n", long_options[i].val);
23016962b24SJohn Fastabend }
23116962b24SJohn Fastabend printf("\n");
23216962b24SJohn Fastabend }
23316962b24SJohn Fastabend
sock_to_string(int s)234e9dd9047SJohn Fastabend char *sock_to_string(int s)
235e9dd9047SJohn Fastabend {
236e9dd9047SJohn Fastabend if (s == c1)
237e9dd9047SJohn Fastabend return "client1";
238e9dd9047SJohn Fastabend else if (s == c2)
239e9dd9047SJohn Fastabend return "client2";
240e9dd9047SJohn Fastabend else if (s == s1)
241e9dd9047SJohn Fastabend return "server1";
242e9dd9047SJohn Fastabend else if (s == s2)
243e9dd9047SJohn Fastabend return "server2";
244e9dd9047SJohn Fastabend else if (s == p1)
245e9dd9047SJohn Fastabend return "peer1";
246e9dd9047SJohn Fastabend else if (s == p2)
247e9dd9047SJohn Fastabend return "peer2";
248e9dd9047SJohn Fastabend else
249e9dd9047SJohn Fastabend return "unknown";
250e9dd9047SJohn Fastabend }
251e9dd9047SJohn Fastabend
sockmap_init_ktls(int verbose,int s)252e9dd9047SJohn Fastabend static int sockmap_init_ktls(int verbose, int s)
253e9dd9047SJohn Fastabend {
254e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_tx = {
255e9dd9047SJohn Fastabend .info = {
256e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION,
257e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128,
258e9dd9047SJohn Fastabend },
259e9dd9047SJohn Fastabend };
260e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_rx = {
261e9dd9047SJohn Fastabend .info = {
262e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION,
263e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128,
264e9dd9047SJohn Fastabend },
265e9dd9047SJohn Fastabend };
266e9dd9047SJohn Fastabend int so_buf = 6553500;
267e9dd9047SJohn Fastabend int err;
268e9dd9047SJohn Fastabend
269e9dd9047SJohn Fastabend err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
270e9dd9047SJohn Fastabend if (err) {
271e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
272e9dd9047SJohn Fastabend return -EINVAL;
273e9dd9047SJohn Fastabend }
274e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
275e9dd9047SJohn Fastabend if (err) {
276e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
277e9dd9047SJohn Fastabend return -EINVAL;
278e9dd9047SJohn Fastabend }
279e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
280e9dd9047SJohn Fastabend if (err) {
281e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
282e9dd9047SJohn Fastabend return -EINVAL;
283e9dd9047SJohn Fastabend }
284e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
285e9dd9047SJohn Fastabend if (err) {
286e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
287e9dd9047SJohn Fastabend return -EINVAL;
288e9dd9047SJohn Fastabend }
289e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
290e9dd9047SJohn Fastabend if (err) {
291e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
292e9dd9047SJohn Fastabend return -EINVAL;
293e9dd9047SJohn Fastabend }
294e9dd9047SJohn Fastabend
295e9dd9047SJohn Fastabend if (verbose)
296e9dd9047SJohn Fastabend fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
297e9dd9047SJohn Fastabend return 0;
298e9dd9047SJohn Fastabend }
sockmap_init_sockets(int verbose)29916962b24SJohn Fastabend static int sockmap_init_sockets(int verbose)
30016962b24SJohn Fastabend {
30116962b24SJohn Fastabend int i, err, one = 1;
30216962b24SJohn Fastabend struct sockaddr_in addr;
30316962b24SJohn Fastabend int *fds[4] = {&s1, &s2, &c1, &c2};
30416962b24SJohn Fastabend
30516962b24SJohn Fastabend s1 = s2 = p1 = p2 = c1 = c2 = 0;
30616962b24SJohn Fastabend
30716962b24SJohn Fastabend /* Init sockets */
30816962b24SJohn Fastabend for (i = 0; i < 4; i++) {
30916962b24SJohn Fastabend *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
31016962b24SJohn Fastabend if (*fds[i] < 0) {
31116962b24SJohn Fastabend perror("socket s1 failed()");
31216962b24SJohn Fastabend return errno;
31316962b24SJohn Fastabend }
31416962b24SJohn Fastabend }
31516962b24SJohn Fastabend
31616962b24SJohn Fastabend /* Allow reuse */
31716962b24SJohn Fastabend for (i = 0; i < 2; i++) {
31816962b24SJohn Fastabend err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
31916962b24SJohn Fastabend (char *)&one, sizeof(one));
32016962b24SJohn Fastabend if (err) {
32116962b24SJohn Fastabend perror("setsockopt failed()");
32216962b24SJohn Fastabend return errno;
32316962b24SJohn Fastabend }
32416962b24SJohn Fastabend }
32516962b24SJohn Fastabend
32616962b24SJohn Fastabend /* Non-blocking sockets */
32716962b24SJohn Fastabend for (i = 0; i < 2; i++) {
32816962b24SJohn Fastabend err = ioctl(*fds[i], FIONBIO, (char *)&one);
32916962b24SJohn Fastabend if (err < 0) {
33016962b24SJohn Fastabend perror("ioctl s1 failed()");
33116962b24SJohn Fastabend return errno;
33216962b24SJohn Fastabend }
33316962b24SJohn Fastabend }
33416962b24SJohn Fastabend
33516962b24SJohn Fastabend /* Bind server sockets */
33616962b24SJohn Fastabend memset(&addr, 0, sizeof(struct sockaddr_in));
33716962b24SJohn Fastabend addr.sin_family = AF_INET;
33816962b24SJohn Fastabend addr.sin_addr.s_addr = inet_addr("127.0.0.1");
33916962b24SJohn Fastabend
34016962b24SJohn Fastabend addr.sin_port = htons(S1_PORT);
34116962b24SJohn Fastabend err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
34216962b24SJohn Fastabend if (err < 0) {
343e5dc9dd3SJakub Kicinski perror("bind s1 failed()");
34416962b24SJohn Fastabend return errno;
34516962b24SJohn Fastabend }
34616962b24SJohn Fastabend
34716962b24SJohn Fastabend addr.sin_port = htons(S2_PORT);
34816962b24SJohn Fastabend err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
34916962b24SJohn Fastabend if (err < 0) {
350e5dc9dd3SJakub Kicinski perror("bind s2 failed()");
35116962b24SJohn Fastabend return errno;
35216962b24SJohn Fastabend }
35316962b24SJohn Fastabend
35416962b24SJohn Fastabend /* Listen server sockets */
35516962b24SJohn Fastabend addr.sin_port = htons(S1_PORT);
35616962b24SJohn Fastabend err = listen(s1, 32);
35716962b24SJohn Fastabend if (err < 0) {
358e5dc9dd3SJakub Kicinski perror("listen s1 failed()");
35916962b24SJohn Fastabend return errno;
36016962b24SJohn Fastabend }
36116962b24SJohn Fastabend
36216962b24SJohn Fastabend addr.sin_port = htons(S2_PORT);
36316962b24SJohn Fastabend err = listen(s2, 32);
36416962b24SJohn Fastabend if (err < 0) {
365e5dc9dd3SJakub Kicinski perror("listen s1 failed()");
36616962b24SJohn Fastabend return errno;
36716962b24SJohn Fastabend }
36816962b24SJohn Fastabend
36916962b24SJohn Fastabend /* Initiate Connect */
37016962b24SJohn Fastabend addr.sin_port = htons(S1_PORT);
37116962b24SJohn Fastabend err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
37216962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) {
373e5dc9dd3SJakub Kicinski perror("connect c1 failed()");
37416962b24SJohn Fastabend return errno;
37516962b24SJohn Fastabend }
37616962b24SJohn Fastabend
37716962b24SJohn Fastabend addr.sin_port = htons(S2_PORT);
37816962b24SJohn Fastabend err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
37916962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) {
380e5dc9dd3SJakub Kicinski perror("connect c2 failed()");
38116962b24SJohn Fastabend return errno;
38216962b24SJohn Fastabend } else if (err < 0) {
38316962b24SJohn Fastabend err = 0;
38416962b24SJohn Fastabend }
38516962b24SJohn Fastabend
38616962b24SJohn Fastabend /* Accept Connecrtions */
38716962b24SJohn Fastabend p1 = accept(s1, NULL, NULL);
38816962b24SJohn Fastabend if (p1 < 0) {
389e5dc9dd3SJakub Kicinski perror("accept s1 failed()");
39016962b24SJohn Fastabend return errno;
39116962b24SJohn Fastabend }
39216962b24SJohn Fastabend
39316962b24SJohn Fastabend p2 = accept(s2, NULL, NULL);
39416962b24SJohn Fastabend if (p2 < 0) {
395e5dc9dd3SJakub Kicinski perror("accept s1 failed()");
39616962b24SJohn Fastabend return errno;
39716962b24SJohn Fastabend }
39816962b24SJohn Fastabend
399b98ca90cSJohn Fastabend if (verbose > 1) {
40016962b24SJohn Fastabend printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
40116962b24SJohn Fastabend printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
40216962b24SJohn Fastabend c1, s1, c2, s2);
40316962b24SJohn Fastabend }
40416962b24SJohn Fastabend return 0;
40516962b24SJohn Fastabend }
40616962b24SJohn Fastabend
40716962b24SJohn Fastabend struct msg_stats {
40816962b24SJohn Fastabend size_t bytes_sent;
40916962b24SJohn Fastabend size_t bytes_recvd;
41016962b24SJohn Fastabend struct timespec start;
41116962b24SJohn Fastabend struct timespec end;
41216962b24SJohn Fastabend };
41316962b24SJohn Fastabend
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)41416962b24SJohn Fastabend static int msg_loop_sendpage(int fd, int iov_length, int cnt,
41516962b24SJohn Fastabend struct msg_stats *s,
41616962b24SJohn Fastabend struct sockmap_options *opt)
41716962b24SJohn Fastabend {
41816962b24SJohn Fastabend bool drop = opt->drop_expected;
41916962b24SJohn Fastabend unsigned char k = 0;
42016962b24SJohn Fastabend FILE *file;
42116962b24SJohn Fastabend int i, fp;
42216962b24SJohn Fastabend
423c31dbb1eSLorenz Bauer file = tmpfile();
4244b67c515SJakub Kicinski if (!file) {
4254b67c515SJakub Kicinski perror("create file for sendpage");
4264b67c515SJakub Kicinski return 1;
4274b67c515SJakub Kicinski }
42816962b24SJohn Fastabend for (i = 0; i < iov_length * cnt; i++, k++)
42916962b24SJohn Fastabend fwrite(&k, sizeof(char), 1, file);
43016962b24SJohn Fastabend fflush(file);
43116962b24SJohn Fastabend fseek(file, 0, SEEK_SET);
43216962b24SJohn Fastabend
433c31dbb1eSLorenz Bauer fp = fileno(file);
4344b67c515SJakub Kicinski
43516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start);
43616962b24SJohn Fastabend for (i = 0; i < cnt; i++) {
437248aba1dSJohn Fastabend int sent;
438248aba1dSJohn Fastabend
439248aba1dSJohn Fastabend errno = 0;
440248aba1dSJohn Fastabend sent = sendfile(fd, fp, NULL, iov_length);
44116962b24SJohn Fastabend
44216962b24SJohn Fastabend if (!drop && sent < 0) {
443248aba1dSJohn Fastabend perror("sendpage loop error");
444c31dbb1eSLorenz Bauer fclose(file);
44516962b24SJohn Fastabend return sent;
44616962b24SJohn Fastabend } else if (drop && sent >= 0) {
447248aba1dSJohn Fastabend printf("sendpage loop error expected: %i errno %i\n",
448248aba1dSJohn Fastabend sent, errno);
449c31dbb1eSLorenz Bauer fclose(file);
45016962b24SJohn Fastabend return -EIO;
45116962b24SJohn Fastabend }
45216962b24SJohn Fastabend
45316962b24SJohn Fastabend if (sent > 0)
45416962b24SJohn Fastabend s->bytes_sent += sent;
45516962b24SJohn Fastabend }
45616962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
457c31dbb1eSLorenz Bauer fclose(file);
45816962b24SJohn Fastabend return 0;
45916962b24SJohn Fastabend }
46016962b24SJohn Fastabend
msg_free_iov(struct msghdr * msg)461753fb2eeSJohn Fastabend static void msg_free_iov(struct msghdr *msg)
46216962b24SJohn Fastabend {
463753fb2eeSJohn Fastabend int i;
464753fb2eeSJohn Fastabend
465753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++)
466753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base);
467753fb2eeSJohn Fastabend free(msg->msg_iov);
468753fb2eeSJohn Fastabend msg->msg_iov = NULL;
469753fb2eeSJohn Fastabend msg->msg_iovlen = 0;
470753fb2eeSJohn Fastabend }
471753fb2eeSJohn Fastabend
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)472753fb2eeSJohn Fastabend static int msg_alloc_iov(struct msghdr *msg,
473753fb2eeSJohn Fastabend int iov_count, int iov_length,
474753fb2eeSJohn Fastabend bool data, bool xmit)
475753fb2eeSJohn Fastabend {
476753fb2eeSJohn Fastabend unsigned char k = 0;
47716962b24SJohn Fastabend struct iovec *iov;
478753fb2eeSJohn Fastabend int i;
47916962b24SJohn Fastabend
48016962b24SJohn Fastabend iov = calloc(iov_count, sizeof(struct iovec));
48116962b24SJohn Fastabend if (!iov)
48216962b24SJohn Fastabend return errno;
48316962b24SJohn Fastabend
48416962b24SJohn Fastabend for (i = 0; i < iov_count; i++) {
48516962b24SJohn Fastabend unsigned char *d = calloc(iov_length, sizeof(char));
48616962b24SJohn Fastabend
48716962b24SJohn Fastabend if (!d) {
48816962b24SJohn Fastabend fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
489753fb2eeSJohn Fastabend goto unwind_iov;
49016962b24SJohn Fastabend }
49116962b24SJohn Fastabend iov[i].iov_base = d;
49216962b24SJohn Fastabend iov[i].iov_len = iov_length;
49316962b24SJohn Fastabend
494753fb2eeSJohn Fastabend if (data && xmit) {
49516962b24SJohn Fastabend int j;
49616962b24SJohn Fastabend
49716962b24SJohn Fastabend for (j = 0; j < iov_length; j++)
49816962b24SJohn Fastabend d[j] = k++;
49916962b24SJohn Fastabend }
50016962b24SJohn Fastabend }
50116962b24SJohn Fastabend
502753fb2eeSJohn Fastabend msg->msg_iov = iov;
503753fb2eeSJohn Fastabend msg->msg_iovlen = iov_count;
504753fb2eeSJohn Fastabend
505753fb2eeSJohn Fastabend return 0;
506753fb2eeSJohn Fastabend unwind_iov:
507753fb2eeSJohn Fastabend for (i--; i >= 0 ; i--)
508753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base);
509753fb2eeSJohn Fastabend return -ENOMEM;
510753fb2eeSJohn Fastabend }
511753fb2eeSJohn Fastabend
msg_verify_data(struct msghdr * msg,int size,int chunk_sz)512753fb2eeSJohn Fastabend static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
513753fb2eeSJohn Fastabend {
514463bac5fSJohn Fastabend int i, j = 0, bytes_cnt = 0;
515753fb2eeSJohn Fastabend unsigned char k = 0;
516753fb2eeSJohn Fastabend
517753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++) {
518753fb2eeSJohn Fastabend unsigned char *d = msg->msg_iov[i].iov_base;
519753fb2eeSJohn Fastabend
520463bac5fSJohn Fastabend /* Special case test for skb ingress + ktls */
521463bac5fSJohn Fastabend if (i == 0 && txmsg_ktls_skb) {
522463bac5fSJohn Fastabend if (msg->msg_iov[i].iov_len < 4)
523463bac5fSJohn Fastabend return -EIO;
524463bac5fSJohn Fastabend if (memcmp(d, "PASS", 4) != 0) {
525463bac5fSJohn Fastabend fprintf(stderr,
52691274ca5SJohn Fastabend "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
52791274ca5SJohn Fastabend i, 0, d[0], d[1], d[2], d[3]);
528463bac5fSJohn Fastabend return -EIO;
529463bac5fSJohn Fastabend }
53091274ca5SJohn Fastabend j = 4; /* advance index past PASS header */
531463bac5fSJohn Fastabend }
532463bac5fSJohn Fastabend
533463bac5fSJohn Fastabend for (; j < msg->msg_iov[i].iov_len && size; j++) {
534753fb2eeSJohn Fastabend if (d[j] != k++) {
535753fb2eeSJohn Fastabend fprintf(stderr,
536753fb2eeSJohn Fastabend "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
537753fb2eeSJohn Fastabend i, j, d[j], k - 1, d[j+1], k);
538753fb2eeSJohn Fastabend return -EIO;
539753fb2eeSJohn Fastabend }
540753fb2eeSJohn Fastabend bytes_cnt++;
541753fb2eeSJohn Fastabend if (bytes_cnt == chunk_sz) {
54216962b24SJohn Fastabend k = 0;
543753fb2eeSJohn Fastabend bytes_cnt = 0;
544753fb2eeSJohn Fastabend }
545753fb2eeSJohn Fastabend size--;
546753fb2eeSJohn Fastabend }
547753fb2eeSJohn Fastabend }
548753fb2eeSJohn Fastabend return 0;
549753fb2eeSJohn Fastabend }
550753fb2eeSJohn Fastabend
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)551753fb2eeSJohn Fastabend static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
552753fb2eeSJohn Fastabend struct msg_stats *s, bool tx,
553753fb2eeSJohn Fastabend struct sockmap_options *opt)
554753fb2eeSJohn Fastabend {
555753fb2eeSJohn Fastabend struct msghdr msg = {0}, msg_peek = {0};
556753fb2eeSJohn Fastabend int err, i, flags = MSG_NOSIGNAL;
557753fb2eeSJohn Fastabend bool drop = opt->drop_expected;
558753fb2eeSJohn Fastabend bool data = opt->data_test;
559d6967214SLiu Jian int iov_alloc_length = iov_length;
560753fb2eeSJohn Fastabend
561d6967214SLiu Jian if (!tx && opt->check_recved_len)
562d6967214SLiu Jian iov_alloc_length *= 2;
563d6967214SLiu Jian
564d6967214SLiu Jian err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
565753fb2eeSJohn Fastabend if (err)
566753fb2eeSJohn Fastabend goto out_errno;
567753fb2eeSJohn Fastabend if (peek_flag) {
568753fb2eeSJohn Fastabend err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
569753fb2eeSJohn Fastabend if (err)
570753fb2eeSJohn Fastabend goto out_errno;
571753fb2eeSJohn Fastabend }
57216962b24SJohn Fastabend
57316962b24SJohn Fastabend if (tx) {
57416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start);
57516962b24SJohn Fastabend for (i = 0; i < cnt; i++) {
576248aba1dSJohn Fastabend int sent;
577248aba1dSJohn Fastabend
578248aba1dSJohn Fastabend errno = 0;
579248aba1dSJohn Fastabend sent = sendmsg(fd, &msg, flags);
58016962b24SJohn Fastabend
58116962b24SJohn Fastabend if (!drop && sent < 0) {
582043a7356SLiu Jian if (opt->tx_wait_mem && errno == EACCES) {
583043a7356SLiu Jian errno = 0;
584043a7356SLiu Jian goto out_errno;
585043a7356SLiu Jian }
586248aba1dSJohn Fastabend perror("sendmsg loop error");
58716962b24SJohn Fastabend goto out_errno;
58816962b24SJohn Fastabend } else if (drop && sent >= 0) {
589248aba1dSJohn Fastabend fprintf(stderr,
590248aba1dSJohn Fastabend "sendmsg loop error expected: %i errno %i\n",
591248aba1dSJohn Fastabend sent, errno);
59216962b24SJohn Fastabend errno = -EIO;
59316962b24SJohn Fastabend goto out_errno;
59416962b24SJohn Fastabend }
59516962b24SJohn Fastabend if (sent > 0)
59616962b24SJohn Fastabend s->bytes_sent += sent;
59716962b24SJohn Fastabend }
59816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
59916962b24SJohn Fastabend } else {
600753fb2eeSJohn Fastabend int slct, recvp = 0, recv, max_fd = fd;
6011ade9abaSJohn Fastabend float total_bytes, txmsg_pop_total;
60216962b24SJohn Fastabend int fd_flags = O_NONBLOCK;
60316962b24SJohn Fastabend struct timeval timeout;
60416962b24SJohn Fastabend fd_set w;
60516962b24SJohn Fastabend
60616962b24SJohn Fastabend fcntl(fd, fd_flags);
6071ade9abaSJohn Fastabend /* Account for pop bytes noting each iteration of apply will
6081ade9abaSJohn Fastabend * call msg_pop_data helper so we need to account for this
6091ade9abaSJohn Fastabend * by calculating the number of apply iterations. Note user
6101ade9abaSJohn Fastabend * of the tool can create cases where no data is sent by
6111ade9abaSJohn Fastabend * manipulating pop/push/pull/etc. For example txmsg_apply 1
6121ade9abaSJohn Fastabend * with txmsg_pop 1 will try to apply 1B at a time but each
6131ade9abaSJohn Fastabend * iteration will then pop 1B so no data will ever be sent.
6141ade9abaSJohn Fastabend * This is really only useful for testing edge cases in code
6151ade9abaSJohn Fastabend * paths.
6161ade9abaSJohn Fastabend */
61716962b24SJohn Fastabend total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
6181ade9abaSJohn Fastabend if (txmsg_apply)
61918d4e900SJohn Fastabend txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
62018d4e900SJohn Fastabend else
62118d4e900SJohn Fastabend txmsg_pop_total = txmsg_pop * cnt;
6221ade9abaSJohn Fastabend total_bytes -= txmsg_pop_total;
62316962b24SJohn Fastabend err = clock_gettime(CLOCK_MONOTONIC, &s->start);
62416962b24SJohn Fastabend if (err < 0)
625e5dc9dd3SJakub Kicinski perror("recv start time");
62616962b24SJohn Fastabend while (s->bytes_recvd < total_bytes) {
627a009f1f3SPrashant Bhole if (txmsg_cork) {
628a18fda1aSJohn Fastabend timeout.tv_sec = 0;
6293c6ed988SDaniel Borkmann timeout.tv_usec = 300000;
630a009f1f3SPrashant Bhole } else {
6311ade9abaSJohn Fastabend timeout.tv_sec = 3;
632a009f1f3SPrashant Bhole timeout.tv_usec = 0;
633a009f1f3SPrashant Bhole }
63416962b24SJohn Fastabend
63516962b24SJohn Fastabend /* FD sets */
63616962b24SJohn Fastabend FD_ZERO(&w);
63716962b24SJohn Fastabend FD_SET(fd, &w);
63816962b24SJohn Fastabend
63916962b24SJohn Fastabend slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
64016962b24SJohn Fastabend if (slct == -1) {
64116962b24SJohn Fastabend perror("select()");
64216962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
64316962b24SJohn Fastabend goto out_errno;
64416962b24SJohn Fastabend } else if (!slct) {
64516962b24SJohn Fastabend if (opt->verbose)
6461ade9abaSJohn Fastabend fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
64716962b24SJohn Fastabend errno = -EIO;
64816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
64916962b24SJohn Fastabend goto out_errno;
65016962b24SJohn Fastabend }
65116962b24SJohn Fastabend
652043a7356SLiu Jian if (opt->tx_wait_mem) {
653043a7356SLiu Jian FD_ZERO(&w);
654043a7356SLiu Jian FD_SET(fd, &w);
655043a7356SLiu Jian slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
656043a7356SLiu Jian errno = 0;
657043a7356SLiu Jian close(fd);
658043a7356SLiu Jian goto out_errno;
659043a7356SLiu Jian }
660043a7356SLiu Jian
661753fb2eeSJohn Fastabend errno = 0;
662753fb2eeSJohn Fastabend if (peek_flag) {
663753fb2eeSJohn Fastabend flags |= MSG_PEEK;
664753fb2eeSJohn Fastabend recvp = recvmsg(fd, &msg_peek, flags);
665753fb2eeSJohn Fastabend if (recvp < 0) {
666753fb2eeSJohn Fastabend if (errno != EWOULDBLOCK) {
667753fb2eeSJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
668753fb2eeSJohn Fastabend goto out_errno;
669753fb2eeSJohn Fastabend }
670753fb2eeSJohn Fastabend }
671753fb2eeSJohn Fastabend flags = 0;
672753fb2eeSJohn Fastabend }
673753fb2eeSJohn Fastabend
67416962b24SJohn Fastabend recv = recvmsg(fd, &msg, flags);
67516962b24SJohn Fastabend if (recv < 0) {
67616962b24SJohn Fastabend if (errno != EWOULDBLOCK) {
67716962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
678e5dc9dd3SJakub Kicinski perror("recv failed()");
67916962b24SJohn Fastabend goto out_errno;
68016962b24SJohn Fastabend }
68116962b24SJohn Fastabend }
68216962b24SJohn Fastabend
68316962b24SJohn Fastabend s->bytes_recvd += recv;
68416962b24SJohn Fastabend
685d6967214SLiu Jian if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
686d6967214SLiu Jian errno = EMSGSIZE;
687d6967214SLiu Jian fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
688d6967214SLiu Jian s->bytes_recvd, total_bytes);
689d6967214SLiu Jian goto out_errno;
690d6967214SLiu Jian }
691d6967214SLiu Jian
692753fb2eeSJohn Fastabend if (data) {
693753fb2eeSJohn Fastabend int chunk_sz = opt->sendpage ?
694753fb2eeSJohn Fastabend iov_length * cnt :
695753fb2eeSJohn Fastabend iov_length * iov_count;
69616962b24SJohn Fastabend
697753fb2eeSJohn Fastabend errno = msg_verify_data(&msg, recv, chunk_sz);
698753fb2eeSJohn Fastabend if (errno) {
699e5dc9dd3SJakub Kicinski perror("data verify msg failed");
70016962b24SJohn Fastabend goto out_errno;
70116962b24SJohn Fastabend }
702753fb2eeSJohn Fastabend if (recvp) {
703753fb2eeSJohn Fastabend errno = msg_verify_data(&msg_peek,
704753fb2eeSJohn Fastabend recvp,
705753fb2eeSJohn Fastabend chunk_sz);
706753fb2eeSJohn Fastabend if (errno) {
707e5dc9dd3SJakub Kicinski perror("data verify msg_peek failed");
708753fb2eeSJohn Fastabend goto out_errno;
70916962b24SJohn Fastabend }
71016962b24SJohn Fastabend }
71116962b24SJohn Fastabend }
71216962b24SJohn Fastabend }
71316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end);
71416962b24SJohn Fastabend }
71516962b24SJohn Fastabend
716753fb2eeSJohn Fastabend msg_free_iov(&msg);
717753fb2eeSJohn Fastabend msg_free_iov(&msg_peek);
718753fb2eeSJohn Fastabend return err;
71916962b24SJohn Fastabend out_errno:
720753fb2eeSJohn Fastabend msg_free_iov(&msg);
721753fb2eeSJohn Fastabend msg_free_iov(&msg_peek);
72216962b24SJohn Fastabend return errno;
72316962b24SJohn Fastabend }
72416962b24SJohn Fastabend
72516962b24SJohn Fastabend static float giga = 1000000000;
72616962b24SJohn Fastabend
sentBps(struct msg_stats s)72716962b24SJohn Fastabend static inline float sentBps(struct msg_stats s)
72816962b24SJohn Fastabend {
72916962b24SJohn Fastabend return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
73016962b24SJohn Fastabend }
73116962b24SJohn Fastabend
recvdBps(struct msg_stats s)73216962b24SJohn Fastabend static inline float recvdBps(struct msg_stats s)
73316962b24SJohn Fastabend {
73416962b24SJohn Fastabend return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
73516962b24SJohn Fastabend }
73616962b24SJohn Fastabend
sendmsg_test(struct sockmap_options * opt)73716962b24SJohn Fastabend static int sendmsg_test(struct sockmap_options *opt)
73816962b24SJohn Fastabend {
73916962b24SJohn Fastabend float sent_Bps = 0, recvd_Bps = 0;
74016962b24SJohn Fastabend int rx_fd, txpid, rxpid, err = 0;
74116962b24SJohn Fastabend struct msg_stats s = {0};
74216962b24SJohn Fastabend int iov_count = opt->iov_count;
74316962b24SJohn Fastabend int iov_buf = opt->iov_length;
74416edddfeSPrashant Bhole int rx_status, tx_status;
74516962b24SJohn Fastabend int cnt = opt->rate;
74616962b24SJohn Fastabend
74716962b24SJohn Fastabend errno = 0;
74816962b24SJohn Fastabend
74916962b24SJohn Fastabend if (opt->base)
75016962b24SJohn Fastabend rx_fd = p1;
75116962b24SJohn Fastabend else
75216962b24SJohn Fastabend rx_fd = p2;
75316962b24SJohn Fastabend
754e9dd9047SJohn Fastabend if (ktls) {
755e9dd9047SJohn Fastabend /* Redirecting into non-TLS socket which sends into a TLS
756e9dd9047SJohn Fastabend * socket is not a valid test. So in this case lets not
757e9dd9047SJohn Fastabend * enable kTLS but still run the test.
758e9dd9047SJohn Fastabend */
759bce86231SJiapeng Chong if (!txmsg_redir || txmsg_ingress) {
760e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, rx_fd);
761e9dd9047SJohn Fastabend if (err)
762e9dd9047SJohn Fastabend return err;
763e9dd9047SJohn Fastabend }
764e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, c1);
765e9dd9047SJohn Fastabend if (err)
766e9dd9047SJohn Fastabend return err;
767e9dd9047SJohn Fastabend }
768e9dd9047SJohn Fastabend
769043a7356SLiu Jian if (opt->tx_wait_mem) {
770043a7356SLiu Jian struct timeval timeout;
771043a7356SLiu Jian int rxtx_buf_len = 1024;
772043a7356SLiu Jian
773043a7356SLiu Jian timeout.tv_sec = 3;
774043a7356SLiu Jian timeout.tv_usec = 0;
775043a7356SLiu Jian
776043a7356SLiu Jian err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
777043a7356SLiu Jian err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
778043a7356SLiu Jian err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
779043a7356SLiu Jian if (err) {
780043a7356SLiu Jian perror("setsockopt failed()");
781043a7356SLiu Jian return errno;
782043a7356SLiu Jian }
783043a7356SLiu Jian }
784043a7356SLiu Jian
78516962b24SJohn Fastabend rxpid = fork();
78616962b24SJohn Fastabend if (rxpid == 0) {
787d6967214SLiu Jian if (txmsg_pop || txmsg_start_pop)
78818d4e900SJohn Fastabend iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
789463bac5fSJohn Fastabend if (opt->drop_expected || txmsg_ktls_skb_drop)
79018d4e900SJohn Fastabend _exit(0);
79118d4e900SJohn Fastabend
79218d4e900SJohn Fastabend if (!iov_buf) /* zero bytes sent case */
79318d4e900SJohn Fastabend _exit(0);
79416962b24SJohn Fastabend
79516962b24SJohn Fastabend if (opt->sendpage)
79616962b24SJohn Fastabend iov_count = 1;
79716962b24SJohn Fastabend err = msg_loop(rx_fd, iov_count, iov_buf,
79816962b24SJohn Fastabend cnt, &s, false, opt);
799b98ca90cSJohn Fastabend if (opt->verbose > 1)
80016962b24SJohn Fastabend fprintf(stderr,
80116962b24SJohn Fastabend "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
80216962b24SJohn Fastabend iov_count, iov_buf, cnt, err);
80316962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) {
80416962b24SJohn Fastabend sent_Bps = sentBps(s);
80516962b24SJohn Fastabend recvd_Bps = recvdBps(s);
80616962b24SJohn Fastabend }
807b98ca90cSJohn Fastabend if (opt->verbose > 1)
80816962b24SJohn Fastabend fprintf(stdout,
809753fb2eeSJohn Fastabend "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
81016962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga,
811753fb2eeSJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
812753fb2eeSJohn Fastabend peek_flag ? "(peek_msg)" : "");
81316edddfeSPrashant Bhole if (err && txmsg_cork)
81416edddfeSPrashant Bhole err = 0;
81516edddfeSPrashant Bhole exit(err ? 1 : 0);
81616962b24SJohn Fastabend } else if (rxpid == -1) {
817e5dc9dd3SJakub Kicinski perror("msg_loop_rx");
81816962b24SJohn Fastabend return errno;
81916962b24SJohn Fastabend }
82016962b24SJohn Fastabend
821043a7356SLiu Jian if (opt->tx_wait_mem)
822043a7356SLiu Jian close(c2);
823043a7356SLiu Jian
82416962b24SJohn Fastabend txpid = fork();
82516962b24SJohn Fastabend if (txpid == 0) {
82616962b24SJohn Fastabend if (opt->sendpage)
82716962b24SJohn Fastabend err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
82816962b24SJohn Fastabend else
82916962b24SJohn Fastabend err = msg_loop(c1, iov_count, iov_buf,
83016962b24SJohn Fastabend cnt, &s, true, opt);
83116962b24SJohn Fastabend
83216962b24SJohn Fastabend if (err)
83316962b24SJohn Fastabend fprintf(stderr,
83416962b24SJohn Fastabend "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
83516962b24SJohn Fastabend iov_count, iov_buf, cnt, err);
83616962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) {
83716962b24SJohn Fastabend sent_Bps = sentBps(s);
83816962b24SJohn Fastabend recvd_Bps = recvdBps(s);
83916962b24SJohn Fastabend }
840b98ca90cSJohn Fastabend if (opt->verbose > 1)
84116962b24SJohn Fastabend fprintf(stdout,
84216962b24SJohn Fastabend "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
84316962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga,
84416962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
84516edddfeSPrashant Bhole exit(err ? 1 : 0);
84616962b24SJohn Fastabend } else if (txpid == -1) {
847e5dc9dd3SJakub Kicinski perror("msg_loop_tx");
84816962b24SJohn Fastabend return errno;
84916962b24SJohn Fastabend }
85016962b24SJohn Fastabend
85116edddfeSPrashant Bhole assert(waitpid(rxpid, &rx_status, 0) == rxpid);
85216edddfeSPrashant Bhole assert(waitpid(txpid, &tx_status, 0) == txpid);
85316edddfeSPrashant Bhole if (WIFEXITED(rx_status)) {
85416edddfeSPrashant Bhole err = WEXITSTATUS(rx_status);
85516edddfeSPrashant Bhole if (err) {
856248aba1dSJohn Fastabend fprintf(stderr, "rx thread exited with err %d.\n", err);
85716edddfeSPrashant Bhole goto out;
85816edddfeSPrashant Bhole }
85916edddfeSPrashant Bhole }
86016edddfeSPrashant Bhole if (WIFEXITED(tx_status)) {
86116edddfeSPrashant Bhole err = WEXITSTATUS(tx_status);
86216edddfeSPrashant Bhole if (err)
863248aba1dSJohn Fastabend fprintf(stderr, "tx thread exited with err %d.\n", err);
86416edddfeSPrashant Bhole }
86516edddfeSPrashant Bhole out:
86616962b24SJohn Fastabend return err;
86716962b24SJohn Fastabend }
86816962b24SJohn Fastabend
forever_ping_pong(int rate,struct sockmap_options * opt)86916962b24SJohn Fastabend static int forever_ping_pong(int rate, struct sockmap_options *opt)
87016962b24SJohn Fastabend {
87116962b24SJohn Fastabend struct timeval timeout;
87216962b24SJohn Fastabend char buf[1024] = {0};
87316962b24SJohn Fastabend int sc;
87416962b24SJohn Fastabend
87516962b24SJohn Fastabend timeout.tv_sec = 10;
87616962b24SJohn Fastabend timeout.tv_usec = 0;
87716962b24SJohn Fastabend
87816962b24SJohn Fastabend /* Ping/Pong data from client to server */
87916962b24SJohn Fastabend sc = send(c1, buf, sizeof(buf), 0);
88016962b24SJohn Fastabend if (sc < 0) {
881e5dc9dd3SJakub Kicinski perror("send failed()");
88216962b24SJohn Fastabend return sc;
88316962b24SJohn Fastabend }
88416962b24SJohn Fastabend
88516962b24SJohn Fastabend do {
88616962b24SJohn Fastabend int s, rc, i, max_fd = p2;
88716962b24SJohn Fastabend fd_set w;
88816962b24SJohn Fastabend
88916962b24SJohn Fastabend /* FD sets */
89016962b24SJohn Fastabend FD_ZERO(&w);
89116962b24SJohn Fastabend FD_SET(c1, &w);
89216962b24SJohn Fastabend FD_SET(c2, &w);
89316962b24SJohn Fastabend FD_SET(p1, &w);
89416962b24SJohn Fastabend FD_SET(p2, &w);
89516962b24SJohn Fastabend
89616962b24SJohn Fastabend s = select(max_fd + 1, &w, NULL, NULL, &timeout);
89716962b24SJohn Fastabend if (s == -1) {
89816962b24SJohn Fastabend perror("select()");
89916962b24SJohn Fastabend break;
90016962b24SJohn Fastabend } else if (!s) {
90116962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n");
90216962b24SJohn Fastabend break;
90316962b24SJohn Fastabend }
90416962b24SJohn Fastabend
90516962b24SJohn Fastabend for (i = 0; i <= max_fd && s > 0; ++i) {
90616962b24SJohn Fastabend if (!FD_ISSET(i, &w))
90716962b24SJohn Fastabend continue;
90816962b24SJohn Fastabend
90916962b24SJohn Fastabend s--;
91016962b24SJohn Fastabend
91116962b24SJohn Fastabend rc = recv(i, buf, sizeof(buf), 0);
91216962b24SJohn Fastabend if (rc < 0) {
91316962b24SJohn Fastabend if (errno != EWOULDBLOCK) {
914e5dc9dd3SJakub Kicinski perror("recv failed()");
91516962b24SJohn Fastabend return rc;
91616962b24SJohn Fastabend }
91716962b24SJohn Fastabend }
91816962b24SJohn Fastabend
91916962b24SJohn Fastabend if (rc == 0) {
92016962b24SJohn Fastabend close(i);
92116962b24SJohn Fastabend break;
92216962b24SJohn Fastabend }
92316962b24SJohn Fastabend
92416962b24SJohn Fastabend sc = send(i, buf, rc, 0);
92516962b24SJohn Fastabend if (sc < 0) {
926e5dc9dd3SJakub Kicinski perror("send failed()");
92716962b24SJohn Fastabend return sc;
92816962b24SJohn Fastabend }
92916962b24SJohn Fastabend }
93016962b24SJohn Fastabend
93116962b24SJohn Fastabend if (rate)
93216962b24SJohn Fastabend sleep(rate);
93316962b24SJohn Fastabend
93416962b24SJohn Fastabend if (opt->verbose) {
93516962b24SJohn Fastabend printf(".");
93616962b24SJohn Fastabend fflush(stdout);
93716962b24SJohn Fastabend
93816962b24SJohn Fastabend }
93916962b24SJohn Fastabend } while (running);
94016962b24SJohn Fastabend
94116962b24SJohn Fastabend return 0;
94216962b24SJohn Fastabend }
94316962b24SJohn Fastabend
94416962b24SJohn Fastabend enum {
945b98ca90cSJohn Fastabend SELFTESTS,
94616962b24SJohn Fastabend PING_PONG,
94716962b24SJohn Fastabend SENDMSG,
94816962b24SJohn Fastabend BASE,
94916962b24SJohn Fastabend BASE_SENDPAGE,
95016962b24SJohn Fastabend SENDPAGE,
95116962b24SJohn Fastabend };
95216962b24SJohn Fastabend
run_options(struct sockmap_options * options,int cg_fd,int test)95316962b24SJohn Fastabend static int run_options(struct sockmap_options *options, int cg_fd, int test)
95416962b24SJohn Fastabend {
95516962b24SJohn Fastabend int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
95616962b24SJohn Fastabend
95716962b24SJohn Fastabend /* If base test skip BPF setup */
95816962b24SJohn Fastabend if (test == BASE || test == BASE_SENDPAGE)
95916962b24SJohn Fastabend goto run;
96016962b24SJohn Fastabend
96116962b24SJohn Fastabend /* Attach programs to sockmap */
962cdf43c4bSJohn Fastabend if (!txmsg_omit_skb_parser) {
96316962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[0],
96416962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0);
96516962b24SJohn Fastabend if (err) {
96616962b24SJohn Fastabend fprintf(stderr,
96716962b24SJohn Fastabend "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
96816962b24SJohn Fastabend prog_fd[0], map_fd[0], err, strerror(errno));
96916962b24SJohn Fastabend return err;
97016962b24SJohn Fastabend }
971cdf43c4bSJohn Fastabend }
97216962b24SJohn Fastabend
97316962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[1], map_fd[0],
97416962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0);
97516962b24SJohn Fastabend if (err) {
97616962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
97716962b24SJohn Fastabend err, strerror(errno));
97816962b24SJohn Fastabend return err;
97916962b24SJohn Fastabend }
98016962b24SJohn Fastabend
981463bac5fSJohn Fastabend /* Attach programs to TLS sockmap */
982463bac5fSJohn Fastabend if (txmsg_ktls_skb) {
983cdf43c4bSJohn Fastabend if (!txmsg_omit_skb_parser) {
984463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[8],
985463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0);
986463bac5fSJohn Fastabend if (err) {
987463bac5fSJohn Fastabend fprintf(stderr,
988463bac5fSJohn Fastabend "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
989463bac5fSJohn Fastabend prog_fd[0], map_fd[8], err, strerror(errno));
990463bac5fSJohn Fastabend return err;
991463bac5fSJohn Fastabend }
992cdf43c4bSJohn Fastabend }
993463bac5fSJohn Fastabend
994463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[2], map_fd[8],
995463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0);
996463bac5fSJohn Fastabend if (err) {
997463bac5fSJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
998463bac5fSJohn Fastabend err, strerror(errno));
999463bac5fSJohn Fastabend return err;
1000463bac5fSJohn Fastabend }
1001463bac5fSJohn Fastabend }
1002463bac5fSJohn Fastabend
100316962b24SJohn Fastabend /* Attach to cgroups */
1004463bac5fSJohn Fastabend err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
100516962b24SJohn Fastabend if (err) {
100616962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
100716962b24SJohn Fastabend err, strerror(errno));
100816962b24SJohn Fastabend return err;
100916962b24SJohn Fastabend }
101016962b24SJohn Fastabend
101116962b24SJohn Fastabend run:
101216962b24SJohn Fastabend err = sockmap_init_sockets(options->verbose);
101316962b24SJohn Fastabend if (err) {
101416962b24SJohn Fastabend fprintf(stderr, "ERROR: test socket failed: %d\n", err);
101516962b24SJohn Fastabend goto out;
101616962b24SJohn Fastabend }
101716962b24SJohn Fastabend
101816962b24SJohn Fastabend /* Attach txmsg program to sockmap */
101916962b24SJohn Fastabend if (txmsg_pass)
1020d79a3212SJohn Fastabend tx_prog_fd = prog_fd[4];
1021463bac5fSJohn Fastabend else if (txmsg_redir)
102216962b24SJohn Fastabend tx_prog_fd = prog_fd[5];
1023463bac5fSJohn Fastabend else if (txmsg_apply)
102416962b24SJohn Fastabend tx_prog_fd = prog_fd[6];
1025463bac5fSJohn Fastabend else if (txmsg_cork)
102616962b24SJohn Fastabend tx_prog_fd = prog_fd[7];
1027463bac5fSJohn Fastabend else if (txmsg_drop)
1028463bac5fSJohn Fastabend tx_prog_fd = prog_fd[8];
102916962b24SJohn Fastabend else
103016962b24SJohn Fastabend tx_prog_fd = 0;
103116962b24SJohn Fastabend
103216962b24SJohn Fastabend if (tx_prog_fd) {
103316962b24SJohn Fastabend int redir_fd, i = 0;
103416962b24SJohn Fastabend
103516962b24SJohn Fastabend err = bpf_prog_attach(tx_prog_fd,
103616962b24SJohn Fastabend map_fd[1], BPF_SK_MSG_VERDICT, 0);
103716962b24SJohn Fastabend if (err) {
103816962b24SJohn Fastabend fprintf(stderr,
103916962b24SJohn Fastabend "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
104016962b24SJohn Fastabend err, strerror(errno));
104116962b24SJohn Fastabend goto out;
104216962b24SJohn Fastabend }
104316962b24SJohn Fastabend
104416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
104516962b24SJohn Fastabend if (err) {
104616962b24SJohn Fastabend fprintf(stderr,
104716962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
104816962b24SJohn Fastabend err, strerror(errno));
104916962b24SJohn Fastabend goto out;
105016962b24SJohn Fastabend }
105116962b24SJohn Fastabend
1052d79a3212SJohn Fastabend if (txmsg_redir)
105316962b24SJohn Fastabend redir_fd = c2;
105416962b24SJohn Fastabend else
105516962b24SJohn Fastabend redir_fd = c1;
105616962b24SJohn Fastabend
105716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
105816962b24SJohn Fastabend if (err) {
105916962b24SJohn Fastabend fprintf(stderr,
106016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
106116962b24SJohn Fastabend err, strerror(errno));
106216962b24SJohn Fastabend goto out;
106316962b24SJohn Fastabend }
106416962b24SJohn Fastabend
106516962b24SJohn Fastabend if (txmsg_apply) {
106616962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[3],
106716962b24SJohn Fastabend &i, &txmsg_apply, BPF_ANY);
106816962b24SJohn Fastabend if (err) {
106916962b24SJohn Fastabend fprintf(stderr,
107016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
107116962b24SJohn Fastabend err, strerror(errno));
107216962b24SJohn Fastabend goto out;
107316962b24SJohn Fastabend }
107416962b24SJohn Fastabend }
107516962b24SJohn Fastabend
107616962b24SJohn Fastabend if (txmsg_cork) {
107716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[4],
107816962b24SJohn Fastabend &i, &txmsg_cork, BPF_ANY);
107916962b24SJohn Fastabend if (err) {
108016962b24SJohn Fastabend fprintf(stderr,
108116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
108216962b24SJohn Fastabend err, strerror(errno));
108316962b24SJohn Fastabend goto out;
108416962b24SJohn Fastabend }
108516962b24SJohn Fastabend }
108616962b24SJohn Fastabend
108716962b24SJohn Fastabend if (txmsg_start) {
108816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5],
108916962b24SJohn Fastabend &i, &txmsg_start, BPF_ANY);
109016962b24SJohn Fastabend if (err) {
109116962b24SJohn Fastabend fprintf(stderr,
109216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
109316962b24SJohn Fastabend err, strerror(errno));
109416962b24SJohn Fastabend goto out;
109516962b24SJohn Fastabend }
109616962b24SJohn Fastabend }
109716962b24SJohn Fastabend
109816962b24SJohn Fastabend if (txmsg_end) {
109916962b24SJohn Fastabend i = 1;
110016962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5],
110116962b24SJohn Fastabend &i, &txmsg_end, BPF_ANY);
110216962b24SJohn Fastabend if (err) {
110316962b24SJohn Fastabend fprintf(stderr,
110416962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
110516962b24SJohn Fastabend err, strerror(errno));
110616962b24SJohn Fastabend goto out;
110716962b24SJohn Fastabend }
110816962b24SJohn Fastabend }
110916962b24SJohn Fastabend
111084fbfe02SJohn Fastabend if (txmsg_start_push) {
111184fbfe02SJohn Fastabend i = 2;
111284fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5],
111384fbfe02SJohn Fastabend &i, &txmsg_start_push, BPF_ANY);
111484fbfe02SJohn Fastabend if (err) {
111584fbfe02SJohn Fastabend fprintf(stderr,
111684fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
111784fbfe02SJohn Fastabend err, strerror(errno));
111884fbfe02SJohn Fastabend goto out;
111984fbfe02SJohn Fastabend }
112084fbfe02SJohn Fastabend }
112184fbfe02SJohn Fastabend
112284fbfe02SJohn Fastabend if (txmsg_end_push) {
112384fbfe02SJohn Fastabend i = 3;
112484fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5],
112584fbfe02SJohn Fastabend &i, &txmsg_end_push, BPF_ANY);
112684fbfe02SJohn Fastabend if (err) {
112784fbfe02SJohn Fastabend fprintf(stderr,
112884fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
112984fbfe02SJohn Fastabend txmsg_end_push, i, err, strerror(errno));
113084fbfe02SJohn Fastabend goto out;
113184fbfe02SJohn Fastabend }
113284fbfe02SJohn Fastabend }
113384fbfe02SJohn Fastabend
11341ade9abaSJohn Fastabend if (txmsg_start_pop) {
11351ade9abaSJohn Fastabend i = 4;
11361ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5],
11371ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY);
11381ade9abaSJohn Fastabend if (err) {
11391ade9abaSJohn Fastabend fprintf(stderr,
11401ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
11411ade9abaSJohn Fastabend txmsg_start_pop, i, err, strerror(errno));
11421ade9abaSJohn Fastabend goto out;
11431ade9abaSJohn Fastabend }
11441ade9abaSJohn Fastabend } else {
11451ade9abaSJohn Fastabend i = 4;
11461ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5],
11471ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY);
11481ade9abaSJohn Fastabend }
11491ade9abaSJohn Fastabend
11501ade9abaSJohn Fastabend if (txmsg_pop) {
11511ade9abaSJohn Fastabend i = 5;
11521ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5],
11531ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY);
11541ade9abaSJohn Fastabend if (err) {
11551ade9abaSJohn Fastabend fprintf(stderr,
11561ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
11571ade9abaSJohn Fastabend txmsg_pop, i, err, strerror(errno));
11581ade9abaSJohn Fastabend goto out;
11591ade9abaSJohn Fastabend }
11601ade9abaSJohn Fastabend } else {
11611ade9abaSJohn Fastabend i = 5;
11621ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5],
11631ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY);
11641ade9abaSJohn Fastabend
11651ade9abaSJohn Fastabend }
11661ade9abaSJohn Fastabend
116716962b24SJohn Fastabend if (txmsg_ingress) {
116816962b24SJohn Fastabend int in = BPF_F_INGRESS;
116916962b24SJohn Fastabend
117016962b24SJohn Fastabend i = 0;
117116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
117216962b24SJohn Fastabend if (err) {
117316962b24SJohn Fastabend fprintf(stderr,
117416962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
117516962b24SJohn Fastabend err, strerror(errno));
117616962b24SJohn Fastabend }
117716962b24SJohn Fastabend i = 1;
117816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
117916962b24SJohn Fastabend if (err) {
118016962b24SJohn Fastabend fprintf(stderr,
118116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
118216962b24SJohn Fastabend err, strerror(errno));
118316962b24SJohn Fastabend }
118416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
118516962b24SJohn Fastabend if (err) {
118616962b24SJohn Fastabend fprintf(stderr,
118716962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
118816962b24SJohn Fastabend err, strerror(errno));
118916962b24SJohn Fastabend }
119016962b24SJohn Fastabend
119116962b24SJohn Fastabend i = 2;
119216962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
119316962b24SJohn Fastabend if (err) {
119416962b24SJohn Fastabend fprintf(stderr,
119516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
119616962b24SJohn Fastabend err, strerror(errno));
119716962b24SJohn Fastabend }
119816962b24SJohn Fastabend }
119916962b24SJohn Fastabend
1200463bac5fSJohn Fastabend if (txmsg_ktls_skb) {
1201463bac5fSJohn Fastabend int ingress = BPF_F_INGRESS;
1202463bac5fSJohn Fastabend
1203463bac5fSJohn Fastabend i = 0;
1204463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1205463bac5fSJohn Fastabend if (err) {
1206463bac5fSJohn Fastabend fprintf(stderr,
1207463bac5fSJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1208463bac5fSJohn Fastabend err, strerror(errno));
1209463bac5fSJohn Fastabend }
1210463bac5fSJohn Fastabend
1211463bac5fSJohn Fastabend if (txmsg_ktls_skb_redir) {
1212463bac5fSJohn Fastabend i = 1;
1213463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[7],
1214463bac5fSJohn Fastabend &i, &ingress, BPF_ANY);
1215463bac5fSJohn Fastabend if (err) {
1216463bac5fSJohn Fastabend fprintf(stderr,
1217463bac5fSJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1218463bac5fSJohn Fastabend err, strerror(errno));
1219463bac5fSJohn Fastabend }
1220463bac5fSJohn Fastabend }
1221463bac5fSJohn Fastabend
1222463bac5fSJohn Fastabend if (txmsg_ktls_skb_drop) {
1223463bac5fSJohn Fastabend i = 1;
1224463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1225463bac5fSJohn Fastabend }
1226463bac5fSJohn Fastabend }
1227463bac5fSJohn Fastabend
1228463bac5fSJohn Fastabend if (txmsg_redir_skb) {
122916962b24SJohn Fastabend int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
123016962b24SJohn Fastabend p2 : p1;
123116962b24SJohn Fastabend int ingress = BPF_F_INGRESS;
123216962b24SJohn Fastabend
123316962b24SJohn Fastabend i = 0;
123416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[7],
123516962b24SJohn Fastabend &i, &ingress, BPF_ANY);
123616962b24SJohn Fastabend if (err) {
123716962b24SJohn Fastabend fprintf(stderr,
123816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
123916962b24SJohn Fastabend err, strerror(errno));
124016962b24SJohn Fastabend }
124116962b24SJohn Fastabend
124216962b24SJohn Fastabend i = 3;
1243463bac5fSJohn Fastabend err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
124416962b24SJohn Fastabend if (err) {
124516962b24SJohn Fastabend fprintf(stderr,
124616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
124716962b24SJohn Fastabend err, strerror(errno));
124816962b24SJohn Fastabend }
124916962b24SJohn Fastabend }
125016962b24SJohn Fastabend }
125116962b24SJohn Fastabend
125253792fa4SJohn Fastabend if (skb_use_parser) {
125353792fa4SJohn Fastabend i = 2;
125453792fa4SJohn Fastabend err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
125553792fa4SJohn Fastabend }
125653792fa4SJohn Fastabend
125716962b24SJohn Fastabend if (txmsg_drop)
125816962b24SJohn Fastabend options->drop_expected = true;
125916962b24SJohn Fastabend
126016962b24SJohn Fastabend if (test == PING_PONG)
126116962b24SJohn Fastabend err = forever_ping_pong(options->rate, options);
126216962b24SJohn Fastabend else if (test == SENDMSG) {
126316962b24SJohn Fastabend options->base = false;
126416962b24SJohn Fastabend options->sendpage = false;
126516962b24SJohn Fastabend err = sendmsg_test(options);
126616962b24SJohn Fastabend } else if (test == SENDPAGE) {
126716962b24SJohn Fastabend options->base = false;
126816962b24SJohn Fastabend options->sendpage = true;
126916962b24SJohn Fastabend err = sendmsg_test(options);
127016962b24SJohn Fastabend } else if (test == BASE) {
127116962b24SJohn Fastabend options->base = true;
127216962b24SJohn Fastabend options->sendpage = false;
127316962b24SJohn Fastabend err = sendmsg_test(options);
127416962b24SJohn Fastabend } else if (test == BASE_SENDPAGE) {
127516962b24SJohn Fastabend options->base = true;
127616962b24SJohn Fastabend options->sendpage = true;
127716962b24SJohn Fastabend err = sendmsg_test(options);
127816962b24SJohn Fastabend } else
127916962b24SJohn Fastabend fprintf(stderr, "unknown test\n");
128016962b24SJohn Fastabend out:
128116962b24SJohn Fastabend /* Detatch and zero all the maps */
1282463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
128316962b24SJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
128416962b24SJohn Fastabend bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1285463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1286463bac5fSJohn Fastabend bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1287463bac5fSJohn Fastabend
128816962b24SJohn Fastabend if (tx_prog_fd >= 0)
128916962b24SJohn Fastabend bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
129016962b24SJohn Fastabend
129116962b24SJohn Fastabend for (i = 0; i < 8; i++) {
129216962b24SJohn Fastabend key = next_key = 0;
129316962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
129416962b24SJohn Fastabend while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
129516962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
129616962b24SJohn Fastabend key = next_key;
129716962b24SJohn Fastabend }
129816962b24SJohn Fastabend }
129916962b24SJohn Fastabend
130016962b24SJohn Fastabend close(s1);
130116962b24SJohn Fastabend close(s2);
130216962b24SJohn Fastabend close(p1);
130316962b24SJohn Fastabend close(p2);
130416962b24SJohn Fastabend close(c1);
130516962b24SJohn Fastabend close(c2);
130616962b24SJohn Fastabend return err;
130716962b24SJohn Fastabend }
130816962b24SJohn Fastabend
test_to_str(int test)130916962b24SJohn Fastabend static char *test_to_str(int test)
131016962b24SJohn Fastabend {
131116962b24SJohn Fastabend switch (test) {
131216962b24SJohn Fastabend case SENDMSG:
131316962b24SJohn Fastabend return "sendmsg";
131416962b24SJohn Fastabend case SENDPAGE:
131516962b24SJohn Fastabend return "sendpage";
131616962b24SJohn Fastabend }
131716962b24SJohn Fastabend return "unknown";
131816962b24SJohn Fastabend }
131916962b24SJohn Fastabend
append_str(char * dst,const char * src,size_t dst_cap)1320eceae70bSAndrii Nakryiko static void append_str(char *dst, const char *src, size_t dst_cap)
1321eceae70bSAndrii Nakryiko {
1322eceae70bSAndrii Nakryiko size_t avail = dst_cap - strlen(dst);
1323eceae70bSAndrii Nakryiko
1324eceae70bSAndrii Nakryiko if (avail <= 1) /* just zero byte could be written */
1325eceae70bSAndrii Nakryiko return;
1326eceae70bSAndrii Nakryiko
1327eceae70bSAndrii Nakryiko strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1328eceae70bSAndrii Nakryiko }
1329eceae70bSAndrii Nakryiko
133016962b24SJohn Fastabend #define OPTSTRING 60
test_options(char * options)133116962b24SJohn Fastabend static void test_options(char *options)
133216962b24SJohn Fastabend {
133373563aa3SPrashant Bhole char tstr[OPTSTRING];
133473563aa3SPrashant Bhole
133516962b24SJohn Fastabend memset(options, 0, OPTSTRING);
133616962b24SJohn Fastabend
133716962b24SJohn Fastabend if (txmsg_pass)
1338eceae70bSAndrii Nakryiko append_str(options, "pass,", OPTSTRING);
133916962b24SJohn Fastabend if (txmsg_redir)
1340eceae70bSAndrii Nakryiko append_str(options, "redir,", OPTSTRING);
134116962b24SJohn Fastabend if (txmsg_drop)
1342eceae70bSAndrii Nakryiko append_str(options, "drop,", OPTSTRING);
134373563aa3SPrashant Bhole if (txmsg_apply) {
134473563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1345eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING);
134673563aa3SPrashant Bhole }
134773563aa3SPrashant Bhole if (txmsg_cork) {
134873563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1349eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING);
135073563aa3SPrashant Bhole }
135173563aa3SPrashant Bhole if (txmsg_start) {
135273563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1353eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING);
135473563aa3SPrashant Bhole }
135573563aa3SPrashant Bhole if (txmsg_end) {
135673563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1357eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING);
135873563aa3SPrashant Bhole }
13591ade9abaSJohn Fastabend if (txmsg_start_pop) {
13601ade9abaSJohn Fastabend snprintf(tstr, OPTSTRING, "pop (%d,%d),",
13611ade9abaSJohn Fastabend txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1362eceae70bSAndrii Nakryiko append_str(options, tstr, OPTSTRING);
13631ade9abaSJohn Fastabend }
136416962b24SJohn Fastabend if (txmsg_ingress)
1365eceae70bSAndrii Nakryiko append_str(options, "ingress,", OPTSTRING);
1366463bac5fSJohn Fastabend if (txmsg_redir_skb)
1367eceae70bSAndrii Nakryiko append_str(options, "redir_skb,", OPTSTRING);
1368463bac5fSJohn Fastabend if (txmsg_ktls_skb)
1369eceae70bSAndrii Nakryiko append_str(options, "ktls_skb,", OPTSTRING);
1370e9dd9047SJohn Fastabend if (ktls)
1371eceae70bSAndrii Nakryiko append_str(options, "ktls,", OPTSTRING);
1372753fb2eeSJohn Fastabend if (peek_flag)
1373eceae70bSAndrii Nakryiko append_str(options, "peek,", OPTSTRING);
137416962b24SJohn Fastabend }
137516962b24SJohn Fastabend
__test_exec(int cgrp,int test,struct sockmap_options * opt)137616962b24SJohn Fastabend static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
137716962b24SJohn Fastabend {
137873563aa3SPrashant Bhole char *options = calloc(OPTSTRING, sizeof(char));
137916962b24SJohn Fastabend int err;
138016962b24SJohn Fastabend
138116962b24SJohn Fastabend if (test == SENDPAGE)
138216962b24SJohn Fastabend opt->sendpage = true;
138316962b24SJohn Fastabend else
138416962b24SJohn Fastabend opt->sendpage = false;
138516962b24SJohn Fastabend
138616962b24SJohn Fastabend if (txmsg_drop)
138716962b24SJohn Fastabend opt->drop_expected = true;
138816962b24SJohn Fastabend else
138916962b24SJohn Fastabend opt->drop_expected = false;
139016962b24SJohn Fastabend
139116962b24SJohn Fastabend test_options(options);
139216962b24SJohn Fastabend
1393328aa08aSJohn Fastabend if (opt->verbose) {
139416962b24SJohn Fastabend fprintf(stdout,
139516962b24SJohn Fastabend " [TEST %i]: (%i, %i, %i, %s, %s): ",
139616962b24SJohn Fastabend test_cnt, opt->rate, opt->iov_count, opt->iov_length,
139716962b24SJohn Fastabend test_to_str(test), options);
139816962b24SJohn Fastabend fflush(stdout);
1399328aa08aSJohn Fastabend }
140016962b24SJohn Fastabend err = run_options(opt, cgrp, test);
1401328aa08aSJohn Fastabend if (opt->verbose)
140216962b24SJohn Fastabend fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
140316962b24SJohn Fastabend test_cnt++;
140416962b24SJohn Fastabend !err ? passed++ : failed++;
140516962b24SJohn Fastabend free(options);
140616962b24SJohn Fastabend return err;
140716962b24SJohn Fastabend }
140816962b24SJohn Fastabend
test_exec(int cgrp,struct sockmap_options * opt)1409328aa08aSJohn Fastabend static void test_exec(int cgrp, struct sockmap_options *opt)
141016962b24SJohn Fastabend {
1411328aa08aSJohn Fastabend int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1412328aa08aSJohn Fastabend int err;
141316962b24SJohn Fastabend
1414328aa08aSJohn Fastabend if (type == 0) {
1415328aa08aSJohn Fastabend test_start();
1416328aa08aSJohn Fastabend err = __test_exec(cgrp, SENDMSG, opt);
141716962b24SJohn Fastabend if (err)
1418328aa08aSJohn Fastabend test_fail();
1419328aa08aSJohn Fastabend } else {
1420328aa08aSJohn Fastabend test_start();
142116962b24SJohn Fastabend err = __test_exec(cgrp, SENDPAGE, opt);
1422328aa08aSJohn Fastabend if (err)
1423328aa08aSJohn Fastabend test_fail();
1424328aa08aSJohn Fastabend }
142516962b24SJohn Fastabend }
142616962b24SJohn Fastabend
test_send_one(struct sockmap_options * opt,int cgrp)1427328aa08aSJohn Fastabend static void test_send_one(struct sockmap_options *opt, int cgrp)
142816962b24SJohn Fastabend {
142916962b24SJohn Fastabend opt->iov_length = 1;
143016962b24SJohn Fastabend opt->iov_count = 1;
143116962b24SJohn Fastabend opt->rate = 1;
1432328aa08aSJohn Fastabend test_exec(cgrp, opt);
143316962b24SJohn Fastabend
143416962b24SJohn Fastabend opt->iov_length = 1;
143516962b24SJohn Fastabend opt->iov_count = 1024;
143616962b24SJohn Fastabend opt->rate = 1;
1437328aa08aSJohn Fastabend test_exec(cgrp, opt);
143816962b24SJohn Fastabend
143916962b24SJohn Fastabend opt->iov_length = 1024;
144016962b24SJohn Fastabend opt->iov_count = 1;
144116962b24SJohn Fastabend opt->rate = 1;
1442328aa08aSJohn Fastabend test_exec(cgrp, opt);
144316962b24SJohn Fastabend
1444328aa08aSJohn Fastabend }
1445328aa08aSJohn Fastabend
test_send_many(struct sockmap_options * opt,int cgrp)1446328aa08aSJohn Fastabend static void test_send_many(struct sockmap_options *opt, int cgrp)
1447328aa08aSJohn Fastabend {
1448328aa08aSJohn Fastabend opt->iov_length = 3;
144916962b24SJohn Fastabend opt->iov_count = 1;
1450a009f1f3SPrashant Bhole opt->rate = 512;
1451328aa08aSJohn Fastabend test_exec(cgrp, opt);
145216962b24SJohn Fastabend
145316962b24SJohn Fastabend opt->rate = 100;
145416962b24SJohn Fastabend opt->iov_count = 1;
145516962b24SJohn Fastabend opt->iov_length = 5;
1456328aa08aSJohn Fastabend test_exec(cgrp, opt);
145716962b24SJohn Fastabend }
145816962b24SJohn Fastabend
test_send_large(struct sockmap_options * opt,int cgrp)1459328aa08aSJohn Fastabend static void test_send_large(struct sockmap_options *opt, int cgrp)
146016962b24SJohn Fastabend {
1461328aa08aSJohn Fastabend opt->iov_length = 256;
1462328aa08aSJohn Fastabend opt->iov_count = 1024;
1463328aa08aSJohn Fastabend opt->rate = 2;
1464328aa08aSJohn Fastabend test_exec(cgrp, opt);
1465328aa08aSJohn Fastabend }
146616962b24SJohn Fastabend
test_send(struct sockmap_options * opt,int cgrp)1467328aa08aSJohn Fastabend static void test_send(struct sockmap_options *opt, int cgrp)
1468328aa08aSJohn Fastabend {
1469328aa08aSJohn Fastabend test_send_one(opt, cgrp);
1470328aa08aSJohn Fastabend test_send_many(opt, cgrp);
1471328aa08aSJohn Fastabend test_send_large(opt, cgrp);
1472328aa08aSJohn Fastabend sched_yield();
1473328aa08aSJohn Fastabend }
1474328aa08aSJohn Fastabend
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1475b98ca90cSJohn Fastabend static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1476328aa08aSJohn Fastabend {
147716962b24SJohn Fastabend /* Test small and large iov_count values with pass/redir/apply/cork */
147816962b24SJohn Fastabend txmsg_pass = 1;
1479b98ca90cSJohn Fastabend test_send(opt, cgrp);
148016962b24SJohn Fastabend }
148116962b24SJohn Fastabend
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1482b98ca90cSJohn Fastabend static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
148316962b24SJohn Fastabend {
1484328aa08aSJohn Fastabend txmsg_redir = 1;
1485b98ca90cSJohn Fastabend test_send(opt, cgrp);
1486328aa08aSJohn Fastabend }
1487328aa08aSJohn Fastabend
test_txmsg_redir_wait_sndmem(int cgrp,struct sockmap_options * opt)1488043a7356SLiu Jian static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
1489043a7356SLiu Jian {
1490043a7356SLiu Jian txmsg_redir = 1;
1491043a7356SLiu Jian opt->tx_wait_mem = true;
1492043a7356SLiu Jian test_send_large(opt, cgrp);
1493043a7356SLiu Jian opt->tx_wait_mem = false;
1494043a7356SLiu Jian }
1495043a7356SLiu Jian
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1496b98ca90cSJohn Fastabend static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1497328aa08aSJohn Fastabend {
1498328aa08aSJohn Fastabend txmsg_drop = 1;
1499b98ca90cSJohn Fastabend test_send(opt, cgrp);
1500328aa08aSJohn Fastabend }
1501328aa08aSJohn Fastabend
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1502b98ca90cSJohn Fastabend static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1503328aa08aSJohn Fastabend {
1504328aa08aSJohn Fastabend txmsg_pass = txmsg_drop = 0;
1505328aa08aSJohn Fastabend txmsg_ingress = txmsg_redir = 1;
1506b98ca90cSJohn Fastabend test_send(opt, cgrp);
1507328aa08aSJohn Fastabend }
1508328aa08aSJohn Fastabend
test_txmsg_skb(int cgrp,struct sockmap_options * opt)1509463bac5fSJohn Fastabend static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1510463bac5fSJohn Fastabend {
1511463bac5fSJohn Fastabend bool data = opt->data_test;
1512463bac5fSJohn Fastabend int k = ktls;
1513463bac5fSJohn Fastabend
1514463bac5fSJohn Fastabend opt->data_test = true;
1515463bac5fSJohn Fastabend ktls = 1;
1516463bac5fSJohn Fastabend
1517463bac5fSJohn Fastabend txmsg_pass = txmsg_drop = 0;
1518463bac5fSJohn Fastabend txmsg_ingress = txmsg_redir = 0;
1519463bac5fSJohn Fastabend txmsg_ktls_skb = 1;
1520463bac5fSJohn Fastabend txmsg_pass = 1;
1521463bac5fSJohn Fastabend
1522463bac5fSJohn Fastabend /* Using data verification so ensure iov layout is
1523463bac5fSJohn Fastabend * expected from test receiver side. e.g. has enough
1524463bac5fSJohn Fastabend * bytes to write test code.
1525463bac5fSJohn Fastabend */
1526463bac5fSJohn Fastabend opt->iov_length = 100;
1527463bac5fSJohn Fastabend opt->iov_count = 1;
1528463bac5fSJohn Fastabend opt->rate = 1;
1529463bac5fSJohn Fastabend test_exec(cgrp, opt);
1530463bac5fSJohn Fastabend
1531463bac5fSJohn Fastabend txmsg_ktls_skb_drop = 1;
1532463bac5fSJohn Fastabend test_exec(cgrp, opt);
1533463bac5fSJohn Fastabend
1534463bac5fSJohn Fastabend txmsg_ktls_skb_drop = 0;
1535463bac5fSJohn Fastabend txmsg_ktls_skb_redir = 1;
1536463bac5fSJohn Fastabend test_exec(cgrp, opt);
1537a24fb420SJohn Fastabend txmsg_ktls_skb_redir = 0;
1538a24fb420SJohn Fastabend
1539a24fb420SJohn Fastabend /* Tests that omit skb_parser */
1540a24fb420SJohn Fastabend txmsg_omit_skb_parser = 1;
1541a24fb420SJohn Fastabend ktls = 0;
1542a24fb420SJohn Fastabend txmsg_ktls_skb = 0;
1543a24fb420SJohn Fastabend test_exec(cgrp, opt);
1544a24fb420SJohn Fastabend
1545a24fb420SJohn Fastabend txmsg_ktls_skb_drop = 1;
1546a24fb420SJohn Fastabend test_exec(cgrp, opt);
1547a24fb420SJohn Fastabend txmsg_ktls_skb_drop = 0;
1548a24fb420SJohn Fastabend
1549a24fb420SJohn Fastabend txmsg_ktls_skb_redir = 1;
1550a24fb420SJohn Fastabend test_exec(cgrp, opt);
1551a24fb420SJohn Fastabend
1552a24fb420SJohn Fastabend ktls = 1;
1553a24fb420SJohn Fastabend test_exec(cgrp, opt);
1554a24fb420SJohn Fastabend txmsg_omit_skb_parser = 0;
1555463bac5fSJohn Fastabend
1556463bac5fSJohn Fastabend opt->data_test = data;
1557463bac5fSJohn Fastabend ktls = k;
1558463bac5fSJohn Fastabend }
1559463bac5fSJohn Fastabend
1560328aa08aSJohn Fastabend /* Test cork with hung data. This tests poor usage patterns where
1561328aa08aSJohn Fastabend * cork can leave data on the ring if user program is buggy and
1562328aa08aSJohn Fastabend * doesn't flush them somehow. They do take some time however
1563328aa08aSJohn Fastabend * because they wait for a timeout. Test pass, redir and cork with
1564328aa08aSJohn Fastabend * apply logic. Use cork size of 4097 with send_large to avoid
1565328aa08aSJohn Fastabend * aligning cork size with send size.
1566328aa08aSJohn Fastabend */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1567b98ca90cSJohn Fastabend static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1568328aa08aSJohn Fastabend {
1569328aa08aSJohn Fastabend txmsg_pass = 1;
1570328aa08aSJohn Fastabend txmsg_redir = 0;
1571328aa08aSJohn Fastabend txmsg_cork = 4097;
1572328aa08aSJohn Fastabend txmsg_apply = 4097;
1573b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1574328aa08aSJohn Fastabend
1575328aa08aSJohn Fastabend txmsg_pass = 0;
1576328aa08aSJohn Fastabend txmsg_redir = 1;
1577328aa08aSJohn Fastabend txmsg_apply = 0;
1578328aa08aSJohn Fastabend txmsg_cork = 4097;
1579b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1580328aa08aSJohn Fastabend
1581328aa08aSJohn Fastabend txmsg_pass = 0;
1582328aa08aSJohn Fastabend txmsg_redir = 1;
1583328aa08aSJohn Fastabend txmsg_apply = 4097;
1584328aa08aSJohn Fastabend txmsg_cork = 4097;
1585b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1586328aa08aSJohn Fastabend }
1587328aa08aSJohn Fastabend
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1588b98ca90cSJohn Fastabend static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1589328aa08aSJohn Fastabend {
1590328aa08aSJohn Fastabend /* Test basic start/end */
159116962b24SJohn Fastabend txmsg_start = 1;
159216962b24SJohn Fastabend txmsg_end = 2;
1593b98ca90cSJohn Fastabend test_send(opt, cgrp);
1594328aa08aSJohn Fastabend
1595328aa08aSJohn Fastabend /* Test >4k pull */
1596328aa08aSJohn Fastabend txmsg_start = 4096;
1597328aa08aSJohn Fastabend txmsg_end = 9182;
1598b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1599328aa08aSJohn Fastabend
1600328aa08aSJohn Fastabend /* Test pull + redirect */
1601328aa08aSJohn Fastabend txmsg_redir = 0;
1602328aa08aSJohn Fastabend txmsg_start = 1;
1603328aa08aSJohn Fastabend txmsg_end = 2;
1604b98ca90cSJohn Fastabend test_send(opt, cgrp);
1605328aa08aSJohn Fastabend
1606328aa08aSJohn Fastabend /* Test pull + cork */
1607328aa08aSJohn Fastabend txmsg_redir = 0;
1608328aa08aSJohn Fastabend txmsg_cork = 512;
1609328aa08aSJohn Fastabend txmsg_start = 1;
1610328aa08aSJohn Fastabend txmsg_end = 2;
1611b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1612328aa08aSJohn Fastabend
1613328aa08aSJohn Fastabend /* Test pull + cork + redirect */
1614328aa08aSJohn Fastabend txmsg_redir = 1;
1615328aa08aSJohn Fastabend txmsg_cork = 512;
1616328aa08aSJohn Fastabend txmsg_start = 1;
1617328aa08aSJohn Fastabend txmsg_end = 2;
1618b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1619328aa08aSJohn Fastabend }
1620328aa08aSJohn Fastabend
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1621b98ca90cSJohn Fastabend static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1622328aa08aSJohn Fastabend {
1623328aa08aSJohn Fastabend /* Test basic pop */
1624328aa08aSJohn Fastabend txmsg_start_pop = 1;
1625328aa08aSJohn Fastabend txmsg_pop = 2;
1626b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1627328aa08aSJohn Fastabend
1628328aa08aSJohn Fastabend /* Test pop with >4k */
1629328aa08aSJohn Fastabend txmsg_start_pop = 4096;
1630328aa08aSJohn Fastabend txmsg_pop = 4096;
1631b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1632328aa08aSJohn Fastabend
1633328aa08aSJohn Fastabend /* Test pop + redirect */
1634328aa08aSJohn Fastabend txmsg_redir = 1;
1635328aa08aSJohn Fastabend txmsg_start_pop = 1;
1636328aa08aSJohn Fastabend txmsg_pop = 2;
1637b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1638328aa08aSJohn Fastabend
1639328aa08aSJohn Fastabend /* Test pop + cork */
1640328aa08aSJohn Fastabend txmsg_redir = 0;
1641328aa08aSJohn Fastabend txmsg_cork = 512;
1642328aa08aSJohn Fastabend txmsg_start_pop = 1;
1643328aa08aSJohn Fastabend txmsg_pop = 2;
1644b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1645328aa08aSJohn Fastabend
1646328aa08aSJohn Fastabend /* Test pop + redirect + cork */
1647328aa08aSJohn Fastabend txmsg_redir = 1;
1648328aa08aSJohn Fastabend txmsg_cork = 4;
1649328aa08aSJohn Fastabend txmsg_start_pop = 1;
1650328aa08aSJohn Fastabend txmsg_pop = 2;
1651b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
1652328aa08aSJohn Fastabend }
1653328aa08aSJohn Fastabend
test_txmsg_push(int cgrp,struct sockmap_options * opt)1654b98ca90cSJohn Fastabend static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1655328aa08aSJohn Fastabend {
1656328aa08aSJohn Fastabend /* Test basic push */
1657328aa08aSJohn Fastabend txmsg_start_push = 1;
1658328aa08aSJohn Fastabend txmsg_end_push = 1;
1659b98ca90cSJohn Fastabend test_send(opt, cgrp);
1660328aa08aSJohn Fastabend
1661328aa08aSJohn Fastabend /* Test push 4kB >4k */
1662328aa08aSJohn Fastabend txmsg_start_push = 4096;
1663328aa08aSJohn Fastabend txmsg_end_push = 4096;
1664b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1665328aa08aSJohn Fastabend
1666328aa08aSJohn Fastabend /* Test push + redirect */
1667328aa08aSJohn Fastabend txmsg_redir = 1;
166884fbfe02SJohn Fastabend txmsg_start_push = 1;
166984fbfe02SJohn Fastabend txmsg_end_push = 2;
1670b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
16711ade9abaSJohn Fastabend
1672328aa08aSJohn Fastabend /* Test push + cork */
1673328aa08aSJohn Fastabend txmsg_redir = 0;
1674328aa08aSJohn Fastabend txmsg_cork = 512;
16751ade9abaSJohn Fastabend txmsg_start_push = 1;
1676328aa08aSJohn Fastabend txmsg_end_push = 2;
1677b98ca90cSJohn Fastabend test_send_many(opt, cgrp);
167816962b24SJohn Fastabend }
167916962b24SJohn Fastabend
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1680b98ca90cSJohn Fastabend static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1681328aa08aSJohn Fastabend {
1682328aa08aSJohn Fastabend txmsg_start_push = 1;
1683328aa08aSJohn Fastabend txmsg_end_push = 10;
1684328aa08aSJohn Fastabend txmsg_start_pop = 5;
1685328aa08aSJohn Fastabend txmsg_pop = 4;
1686b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
168716962b24SJohn Fastabend }
168816962b24SJohn Fastabend
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1689b98ca90cSJohn Fastabend static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1690328aa08aSJohn Fastabend {
1691328aa08aSJohn Fastabend txmsg_pass = 1;
1692328aa08aSJohn Fastabend txmsg_redir = 0;
169389903dcbSPengcheng Yang txmsg_ingress = 0;
1694328aa08aSJohn Fastabend txmsg_apply = 1;
1695328aa08aSJohn Fastabend txmsg_cork = 0;
1696b98ca90cSJohn Fastabend test_send_one(opt, cgrp);
16971ade9abaSJohn Fastabend
1698328aa08aSJohn Fastabend txmsg_pass = 0;
1699328aa08aSJohn Fastabend txmsg_redir = 1;
170089903dcbSPengcheng Yang txmsg_ingress = 0;
170189903dcbSPengcheng Yang txmsg_apply = 1;
170289903dcbSPengcheng Yang txmsg_cork = 0;
170389903dcbSPengcheng Yang test_send_one(opt, cgrp);
170489903dcbSPengcheng Yang
170589903dcbSPengcheng Yang txmsg_pass = 0;
170689903dcbSPengcheng Yang txmsg_redir = 1;
170789903dcbSPengcheng Yang txmsg_ingress = 1;
1708328aa08aSJohn Fastabend txmsg_apply = 1;
1709328aa08aSJohn Fastabend txmsg_cork = 0;
1710b98ca90cSJohn Fastabend test_send_one(opt, cgrp);
171116962b24SJohn Fastabend
1712328aa08aSJohn Fastabend txmsg_pass = 1;
1713328aa08aSJohn Fastabend txmsg_redir = 0;
171489903dcbSPengcheng Yang txmsg_ingress = 0;
1715328aa08aSJohn Fastabend txmsg_apply = 1024;
1716328aa08aSJohn Fastabend txmsg_cork = 0;
1717b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
17181ade9abaSJohn Fastabend
1719328aa08aSJohn Fastabend txmsg_pass = 0;
1720328aa08aSJohn Fastabend txmsg_redir = 1;
172189903dcbSPengcheng Yang txmsg_ingress = 0;
172289903dcbSPengcheng Yang txmsg_apply = 1024;
172389903dcbSPengcheng Yang txmsg_cork = 0;
172489903dcbSPengcheng Yang test_send_large(opt, cgrp);
172589903dcbSPengcheng Yang
172689903dcbSPengcheng Yang txmsg_pass = 0;
172789903dcbSPengcheng Yang txmsg_redir = 1;
172889903dcbSPengcheng Yang txmsg_ingress = 1;
1729328aa08aSJohn Fastabend txmsg_apply = 1024;
1730328aa08aSJohn Fastabend txmsg_cork = 0;
1731b98ca90cSJohn Fastabend test_send_large(opt, cgrp);
1732328aa08aSJohn Fastabend }
173316962b24SJohn Fastabend
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1734b98ca90cSJohn Fastabend static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1735328aa08aSJohn Fastabend {
1736328aa08aSJohn Fastabend txmsg_pass = 1;
1737328aa08aSJohn Fastabend txmsg_redir = 0;
1738328aa08aSJohn Fastabend txmsg_apply = 0;
1739328aa08aSJohn Fastabend txmsg_cork = 1;
1740b98ca90cSJohn Fastabend test_send(opt, cgrp);
174116962b24SJohn Fastabend
1742328aa08aSJohn Fastabend txmsg_pass = 1;
1743328aa08aSJohn Fastabend txmsg_redir = 0;
1744328aa08aSJohn Fastabend txmsg_apply = 1;
1745328aa08aSJohn Fastabend txmsg_cork = 1;
1746b98ca90cSJohn Fastabend test_send(opt, cgrp);
174716962b24SJohn Fastabend }
174816962b24SJohn Fastabend
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)174953792fa4SJohn Fastabend static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
175053792fa4SJohn Fastabend {
175153792fa4SJohn Fastabend txmsg_pass = 1;
175253792fa4SJohn Fastabend skb_use_parser = 512;
1753b556c3fdSLiu Jian if (ktls == 1)
1754b556c3fdSLiu Jian skb_use_parser = 570;
175553792fa4SJohn Fastabend opt->iov_length = 256;
175653792fa4SJohn Fastabend opt->iov_count = 1;
175753792fa4SJohn Fastabend opt->rate = 2;
175853792fa4SJohn Fastabend test_exec(cgrp, opt);
175953792fa4SJohn Fastabend }
176053792fa4SJohn Fastabend
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1761d6967214SLiu Jian static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1762d6967214SLiu Jian {
1763d6967214SLiu Jian if (ktls == 1)
1764d6967214SLiu Jian return;
1765d6967214SLiu Jian skb_use_parser = 10;
1766d6967214SLiu Jian opt->iov_length = 20;
1767d6967214SLiu Jian opt->iov_count = 1;
1768d6967214SLiu Jian opt->rate = 1;
1769d6967214SLiu Jian opt->check_recved_len = true;
1770d6967214SLiu Jian test_exec(cgrp, opt);
1771d6967214SLiu Jian opt->check_recved_len = false;
1772d6967214SLiu Jian }
1773d6967214SLiu Jian
177416962b24SJohn Fastabend char *map_names[] = {
177516962b24SJohn Fastabend "sock_map",
177616962b24SJohn Fastabend "sock_map_txmsg",
177716962b24SJohn Fastabend "sock_map_redir",
177816962b24SJohn Fastabend "sock_apply_bytes",
177916962b24SJohn Fastabend "sock_cork_bytes",
178084fbfe02SJohn Fastabend "sock_bytes",
178116962b24SJohn Fastabend "sock_redir_flags",
178216962b24SJohn Fastabend "sock_skb_opts",
1783463bac5fSJohn Fastabend "tls_sock_map",
178416962b24SJohn Fastabend };
178516962b24SJohn Fastabend
178616962b24SJohn Fastabend int prog_attach_type[] = {
178716962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER,
178816962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT,
1789463bac5fSJohn Fastabend BPF_SK_SKB_STREAM_VERDICT,
179016962b24SJohn Fastabend BPF_CGROUP_SOCK_OPS,
179116962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179216962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179316962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179416962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179516962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179616962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179716962b24SJohn Fastabend BPF_SK_MSG_VERDICT,
179816962b24SJohn Fastabend };
179916962b24SJohn Fastabend
180016962b24SJohn Fastabend int prog_type[] = {
180116962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB,
180216962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB,
1803463bac5fSJohn Fastabend BPF_PROG_TYPE_SK_SKB,
180416962b24SJohn Fastabend BPF_PROG_TYPE_SOCK_OPS,
180516962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
180616962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
180716962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
180816962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
180916962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
181016962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
181116962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG,
181216962b24SJohn Fastabend };
181316962b24SJohn Fastabend
populate_progs(char * bpf_file)1814b8b394faSJohn Fastabend static int populate_progs(char *bpf_file)
181516962b24SJohn Fastabend {
181616962b24SJohn Fastabend struct bpf_program *prog;
181716962b24SJohn Fastabend struct bpf_object *obj;
181816962b24SJohn Fastabend int i = 0;
181916962b24SJohn Fastabend long err;
182016962b24SJohn Fastabend
182116962b24SJohn Fastabend obj = bpf_object__open(bpf_file);
182216962b24SJohn Fastabend err = libbpf_get_error(obj);
182316962b24SJohn Fastabend if (err) {
182416962b24SJohn Fastabend char err_buf[256];
182516962b24SJohn Fastabend
182616962b24SJohn Fastabend libbpf_strerror(err, err_buf, sizeof(err_buf));
182716962b24SJohn Fastabend printf("Unable to load eBPF objects in file '%s' : %s\n",
182816962b24SJohn Fastabend bpf_file, err_buf);
182916962b24SJohn Fastabend return -1;
183016962b24SJohn Fastabend }
183116962b24SJohn Fastabend
183216962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) {
183316962b24SJohn Fastabend bpf_program__set_type(prog, prog_type[i]);
183416962b24SJohn Fastabend bpf_program__set_expected_attach_type(prog,
183516962b24SJohn Fastabend prog_attach_type[i]);
183616962b24SJohn Fastabend i++;
183716962b24SJohn Fastabend }
183816962b24SJohn Fastabend
183916962b24SJohn Fastabend i = bpf_object__load(obj);
184016962b24SJohn Fastabend i = 0;
184116962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) {
184216962b24SJohn Fastabend prog_fd[i] = bpf_program__fd(prog);
184316962b24SJohn Fastabend i++;
184416962b24SJohn Fastabend }
184516962b24SJohn Fastabend
1846f98d6dd1SGuo Zhengkui for (i = 0; i < ARRAY_SIZE(map_fd); i++) {
184716962b24SJohn Fastabend maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
184816962b24SJohn Fastabend map_fd[i] = bpf_map__fd(maps[i]);
184916962b24SJohn Fastabend if (map_fd[i] < 0) {
185016962b24SJohn Fastabend fprintf(stderr, "load_bpf_file: (%i) %s\n",
185116962b24SJohn Fastabend map_fd[i], strerror(errno));
185216962b24SJohn Fastabend return -1;
185316962b24SJohn Fastabend }
185416962b24SJohn Fastabend }
185516962b24SJohn Fastabend
185616962b24SJohn Fastabend return 0;
185716962b24SJohn Fastabend }
185816962b24SJohn Fastabend
1859328aa08aSJohn Fastabend struct _test test[] = {
1860328aa08aSJohn Fastabend {"txmsg test passthrough", test_txmsg_pass},
1861328aa08aSJohn Fastabend {"txmsg test redirect", test_txmsg_redir},
1862043a7356SLiu Jian {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
1863328aa08aSJohn Fastabend {"txmsg test drop", test_txmsg_drop},
1864328aa08aSJohn Fastabend {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1865463bac5fSJohn Fastabend {"txmsg test skb", test_txmsg_skb},
1866328aa08aSJohn Fastabend {"txmsg test apply", test_txmsg_apply},
1867328aa08aSJohn Fastabend {"txmsg test cork", test_txmsg_cork},
1868328aa08aSJohn Fastabend {"txmsg test hanging corks", test_txmsg_cork_hangs},
1869328aa08aSJohn Fastabend {"txmsg test push_data", test_txmsg_push},
1870328aa08aSJohn Fastabend {"txmsg test pull-data", test_txmsg_pull},
1871328aa08aSJohn Fastabend {"txmsg test pop-data", test_txmsg_pop},
1872328aa08aSJohn Fastabend {"txmsg test push/pop data", test_txmsg_push_pop},
1873d6967214SLiu Jian {"txmsg test ingress parser", test_txmsg_ingress_parser},
1874d6967214SLiu Jian {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1875328aa08aSJohn Fastabend };
1876328aa08aSJohn Fastabend
check_whitelist(struct _test * t,struct sockmap_options * opt)1877065a74cbSJohn Fastabend static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1878065a74cbSJohn Fastabend {
1879065a74cbSJohn Fastabend char *entry, *ptr;
1880065a74cbSJohn Fastabend
1881065a74cbSJohn Fastabend if (!opt->whitelist)
1882065a74cbSJohn Fastabend return 0;
1883065a74cbSJohn Fastabend ptr = strdup(opt->whitelist);
1884065a74cbSJohn Fastabend if (!ptr)
1885065a74cbSJohn Fastabend return -ENOMEM;
1886065a74cbSJohn Fastabend entry = strtok(ptr, ",");
1887065a74cbSJohn Fastabend while (entry) {
188896586dd9SJohn Fastabend if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
188996586dd9SJohn Fastabend strstr(opt->map, entry) != 0 ||
189096586dd9SJohn Fastabend strstr(t->title, entry) != 0)
1891065a74cbSJohn Fastabend return 0;
1892065a74cbSJohn Fastabend entry = strtok(NULL, ",");
1893065a74cbSJohn Fastabend }
1894065a74cbSJohn Fastabend return -EINVAL;
1895065a74cbSJohn Fastabend }
1896065a74cbSJohn Fastabend
check_blacklist(struct _test * t,struct sockmap_options * opt)1897a7238f7cSJohn Fastabend static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1898a7238f7cSJohn Fastabend {
1899a7238f7cSJohn Fastabend char *entry, *ptr;
1900a7238f7cSJohn Fastabend
1901a7238f7cSJohn Fastabend if (!opt->blacklist)
1902a7238f7cSJohn Fastabend return -EINVAL;
1903a7238f7cSJohn Fastabend ptr = strdup(opt->blacklist);
1904a7238f7cSJohn Fastabend if (!ptr)
1905a7238f7cSJohn Fastabend return -ENOMEM;
1906a7238f7cSJohn Fastabend entry = strtok(ptr, ",");
1907a7238f7cSJohn Fastabend while (entry) {
190896586dd9SJohn Fastabend if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
190996586dd9SJohn Fastabend strstr(opt->map, entry) != 0 ||
191096586dd9SJohn Fastabend strstr(t->title, entry) != 0)
1911a7238f7cSJohn Fastabend return 0;
1912a7238f7cSJohn Fastabend entry = strtok(NULL, ",");
1913a7238f7cSJohn Fastabend }
1914a7238f7cSJohn Fastabend return -EINVAL;
1915a7238f7cSJohn Fastabend }
1916a7238f7cSJohn Fastabend
__test_selftests(int cg_fd,struct sockmap_options * opt)1917b98ca90cSJohn Fastabend static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1918328aa08aSJohn Fastabend {
1919328aa08aSJohn Fastabend int i, err;
1920328aa08aSJohn Fastabend
1921b98ca90cSJohn Fastabend err = populate_progs(opt->map);
192216962b24SJohn Fastabend if (err < 0) {
192316962b24SJohn Fastabend fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
192416962b24SJohn Fastabend return err;
192516962b24SJohn Fastabend }
192616962b24SJohn Fastabend
1927328aa08aSJohn Fastabend /* Tests basic commands and APIs */
1928f98d6dd1SGuo Zhengkui for (i = 0; i < ARRAY_SIZE(test); i++) {
1929328aa08aSJohn Fastabend struct _test t = test[i];
1930328aa08aSJohn Fastabend
1931a7238f7cSJohn Fastabend if (check_whitelist(&t, opt) != 0)
1932a7238f7cSJohn Fastabend continue;
1933a7238f7cSJohn Fastabend if (check_blacklist(&t, opt) == 0)
1934065a74cbSJohn Fastabend continue;
1935065a74cbSJohn Fastabend
193696586dd9SJohn Fastabend test_start_subtest(&t, opt);
1937b98ca90cSJohn Fastabend t.tester(cg_fd, opt);
1938328aa08aSJohn Fastabend test_end_subtest();
1939328aa08aSJohn Fastabend }
1940328aa08aSJohn Fastabend
1941328aa08aSJohn Fastabend return err;
1942328aa08aSJohn Fastabend }
1943328aa08aSJohn Fastabend
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)1944b98ca90cSJohn Fastabend static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1945328aa08aSJohn Fastabend {
1946b98ca90cSJohn Fastabend opt->map = BPF_SOCKMAP_FILENAME;
1947b98ca90cSJohn Fastabend __test_selftests(cg_fd, opt);
1948328aa08aSJohn Fastabend }
1949328aa08aSJohn Fastabend
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)1950b98ca90cSJohn Fastabend static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1951328aa08aSJohn Fastabend {
1952b98ca90cSJohn Fastabend opt->map = BPF_SOCKHASH_FILENAME;
1953b98ca90cSJohn Fastabend __test_selftests(cg_fd, opt);
1954328aa08aSJohn Fastabend }
1955328aa08aSJohn Fastabend
test_selftests_ktls(int cg_fd,struct sockmap_options * opt)195696586dd9SJohn Fastabend static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
195796586dd9SJohn Fastabend {
195896586dd9SJohn Fastabend opt->map = BPF_SOCKHASH_FILENAME;
195996586dd9SJohn Fastabend opt->prepend = "ktls";
196096586dd9SJohn Fastabend ktls = 1;
196196586dd9SJohn Fastabend __test_selftests(cg_fd, opt);
196296586dd9SJohn Fastabend ktls = 0;
196396586dd9SJohn Fastabend }
196496586dd9SJohn Fastabend
test_selftest(int cg_fd,struct sockmap_options * opt)1965b98ca90cSJohn Fastabend static int test_selftest(int cg_fd, struct sockmap_options *opt)
1966328aa08aSJohn Fastabend {
196716962b24SJohn Fastabend
1968b98ca90cSJohn Fastabend test_selftests_sockmap(cg_fd, opt);
1969b98ca90cSJohn Fastabend test_selftests_sockhash(cg_fd, opt);
197096586dd9SJohn Fastabend test_selftests_ktls(cg_fd, opt);
1971328aa08aSJohn Fastabend test_print_results();
1972328aa08aSJohn Fastabend return 0;
1973b8b394faSJohn Fastabend }
1974b8b394faSJohn Fastabend
main(int argc,char ** argv)197516962b24SJohn Fastabend int main(int argc, char **argv)
197616962b24SJohn Fastabend {
197716962b24SJohn Fastabend int iov_count = 1, length = 1024, rate = 1;
197816962b24SJohn Fastabend struct sockmap_options options = {0};
197916962b24SJohn Fastabend int opt, longindex, err, cg_fd = 0;
1980b8b394faSJohn Fastabend char *bpf_file = BPF_SOCKMAP_FILENAME;
1981b98ca90cSJohn Fastabend int test = SELFTESTS;
198213a5f3ffSJohn Fastabend bool cg_created = 0;
198316962b24SJohn Fastabend
1984a7238f7cSJohn Fastabend while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
198516962b24SJohn Fastabend long_options, &longindex)) != -1) {
198616962b24SJohn Fastabend switch (opt) {
198716962b24SJohn Fastabend case 's':
198816962b24SJohn Fastabend txmsg_start = atoi(optarg);
198916962b24SJohn Fastabend break;
199016962b24SJohn Fastabend case 'e':
199116962b24SJohn Fastabend txmsg_end = atoi(optarg);
199216962b24SJohn Fastabend break;
199384fbfe02SJohn Fastabend case 'p':
199484fbfe02SJohn Fastabend txmsg_start_push = atoi(optarg);
199584fbfe02SJohn Fastabend break;
199684fbfe02SJohn Fastabend case 'q':
199784fbfe02SJohn Fastabend txmsg_end_push = atoi(optarg);
199884fbfe02SJohn Fastabend break;
19991ade9abaSJohn Fastabend case 'w':
20001ade9abaSJohn Fastabend txmsg_start_pop = atoi(optarg);
20011ade9abaSJohn Fastabend break;
20021ade9abaSJohn Fastabend case 'x':
20031ade9abaSJohn Fastabend txmsg_pop = atoi(optarg);
20041ade9abaSJohn Fastabend break;
200516962b24SJohn Fastabend case 'a':
200616962b24SJohn Fastabend txmsg_apply = atoi(optarg);
200716962b24SJohn Fastabend break;
200816962b24SJohn Fastabend case 'k':
200916962b24SJohn Fastabend txmsg_cork = atoi(optarg);
201016962b24SJohn Fastabend break;
201116962b24SJohn Fastabend case 'c':
201216962b24SJohn Fastabend cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
201316962b24SJohn Fastabend if (cg_fd < 0) {
201416962b24SJohn Fastabend fprintf(stderr,
201516962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n",
201616962b24SJohn Fastabend cg_fd, optarg);
201716962b24SJohn Fastabend return cg_fd;
201816962b24SJohn Fastabend }
201916962b24SJohn Fastabend break;
202016962b24SJohn Fastabend case 'r':
202116962b24SJohn Fastabend rate = atoi(optarg);
202216962b24SJohn Fastabend break;
202316962b24SJohn Fastabend case 'v':
202416962b24SJohn Fastabend options.verbose = 1;
2025b98ca90cSJohn Fastabend if (optarg)
2026b98ca90cSJohn Fastabend options.verbose = atoi(optarg);
202716962b24SJohn Fastabend break;
202816962b24SJohn Fastabend case 'i':
202916962b24SJohn Fastabend iov_count = atoi(optarg);
203016962b24SJohn Fastabend break;
203116962b24SJohn Fastabend case 'l':
203216962b24SJohn Fastabend length = atoi(optarg);
203316962b24SJohn Fastabend break;
203416962b24SJohn Fastabend case 'd':
203516962b24SJohn Fastabend options.data_test = true;
203616962b24SJohn Fastabend break;
203716962b24SJohn Fastabend case 't':
203816962b24SJohn Fastabend if (strcmp(optarg, "ping") == 0) {
203916962b24SJohn Fastabend test = PING_PONG;
204016962b24SJohn Fastabend } else if (strcmp(optarg, "sendmsg") == 0) {
204116962b24SJohn Fastabend test = SENDMSG;
204216962b24SJohn Fastabend } else if (strcmp(optarg, "base") == 0) {
204316962b24SJohn Fastabend test = BASE;
204416962b24SJohn Fastabend } else if (strcmp(optarg, "base_sendpage") == 0) {
204516962b24SJohn Fastabend test = BASE_SENDPAGE;
204616962b24SJohn Fastabend } else if (strcmp(optarg, "sendpage") == 0) {
204716962b24SJohn Fastabend test = SENDPAGE;
204816962b24SJohn Fastabend } else {
204916962b24SJohn Fastabend usage(argv);
205016962b24SJohn Fastabend return -1;
205116962b24SJohn Fastabend }
205216962b24SJohn Fastabend break;
2053065a74cbSJohn Fastabend case 'n':
2054065a74cbSJohn Fastabend options.whitelist = strdup(optarg);
2055065a74cbSJohn Fastabend if (!options.whitelist)
2056065a74cbSJohn Fastabend return -ENOMEM;
2057a7238f7cSJohn Fastabend break;
2058a7238f7cSJohn Fastabend case 'b':
2059a7238f7cSJohn Fastabend options.blacklist = strdup(optarg);
2060a7238f7cSJohn Fastabend if (!options.blacklist)
2061a7238f7cSJohn Fastabend return -ENOMEM;
206216962b24SJohn Fastabend case 0:
206316962b24SJohn Fastabend break;
206416962b24SJohn Fastabend case 'h':
206516962b24SJohn Fastabend default:
206616962b24SJohn Fastabend usage(argv);
206716962b24SJohn Fastabend return -1;
206816962b24SJohn Fastabend }
206916962b24SJohn Fastabend }
207016962b24SJohn Fastabend
207116962b24SJohn Fastabend if (!cg_fd) {
20724939b284SJohn Fastabend cg_fd = cgroup_setup_and_join(CG_PATH);
20734939b284SJohn Fastabend if (cg_fd < 0)
207413a5f3ffSJohn Fastabend return cg_fd;
207513a5f3ffSJohn Fastabend cg_created = 1;
207616962b24SJohn Fastabend }
207716962b24SJohn Fastabend
2078b858ba8cSYafang Shao /* Use libbpf 1.0 API mode */
2079b858ba8cSYafang Shao libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2080b858ba8cSYafang Shao
2081b98ca90cSJohn Fastabend if (test == SELFTESTS) {
2082b98ca90cSJohn Fastabend err = test_selftest(cg_fd, &options);
2083b98ca90cSJohn Fastabend goto out;
2084b98ca90cSJohn Fastabend }
2085b98ca90cSJohn Fastabend
2086b8b394faSJohn Fastabend err = populate_progs(bpf_file);
208716962b24SJohn Fastabend if (err) {
208816962b24SJohn Fastabend fprintf(stderr, "populate program: (%s) %s\n",
208916962b24SJohn Fastabend bpf_file, strerror(errno));
209016962b24SJohn Fastabend return 1;
209116962b24SJohn Fastabend }
209216962b24SJohn Fastabend running = 1;
209316962b24SJohn Fastabend
209416962b24SJohn Fastabend /* catch SIGINT */
209516962b24SJohn Fastabend signal(SIGINT, running_handler);
209616962b24SJohn Fastabend
209716962b24SJohn Fastabend options.iov_count = iov_count;
209816962b24SJohn Fastabend options.iov_length = length;
209916962b24SJohn Fastabend options.rate = rate;
210016962b24SJohn Fastabend
210116962b24SJohn Fastabend err = run_options(&options, cg_fd, test);
2102b98ca90cSJohn Fastabend out:
2103065a74cbSJohn Fastabend if (options.whitelist)
2104065a74cbSJohn Fastabend free(options.whitelist);
2105a7238f7cSJohn Fastabend if (options.blacklist)
2106a7238f7cSJohn Fastabend free(options.blacklist);
2107a7fba17aSGeliang Tang close(cg_fd);
210813a5f3ffSJohn Fastabend if (cg_created)
210913a5f3ffSJohn Fastabend cleanup_cgroup_environment();
211016962b24SJohn Fastabend return err;
211116962b24SJohn Fastabend }
211216962b24SJohn Fastabend
running_handler(int a)211316962b24SJohn Fastabend void running_handler(int a)
211416962b24SJohn Fastabend {
211516962b24SJohn Fastabend running = 0;
211616962b24SJohn Fastabend }
2117