116962b24SJohn Fastabend // SPDX-License-Identifier: GPL-2.0 216962b24SJohn Fastabend // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 316962b24SJohn Fastabend #include <stdio.h> 416962b24SJohn Fastabend #include <stdlib.h> 516962b24SJohn Fastabend #include <sys/socket.h> 616962b24SJohn Fastabend #include <sys/ioctl.h> 716962b24SJohn Fastabend #include <sys/select.h> 816962b24SJohn Fastabend #include <netinet/in.h> 916962b24SJohn Fastabend #include <arpa/inet.h> 1016962b24SJohn Fastabend #include <unistd.h> 1116962b24SJohn Fastabend #include <string.h> 1216962b24SJohn Fastabend #include <errno.h> 1316962b24SJohn Fastabend #include <stdbool.h> 1416962b24SJohn Fastabend #include <signal.h> 1516962b24SJohn Fastabend #include <fcntl.h> 1616962b24SJohn Fastabend #include <sys/wait.h> 1716962b24SJohn Fastabend #include <time.h> 1816962b24SJohn Fastabend #include <sched.h> 1916962b24SJohn Fastabend 2016962b24SJohn Fastabend #include <sys/time.h> 2116962b24SJohn Fastabend #include <sys/resource.h> 2216962b24SJohn Fastabend #include <sys/types.h> 2316962b24SJohn Fastabend #include <sys/sendfile.h> 2416962b24SJohn Fastabend 2516962b24SJohn Fastabend #include <linux/netlink.h> 2616962b24SJohn Fastabend #include <linux/socket.h> 2716962b24SJohn Fastabend #include <linux/sock_diag.h> 2816962b24SJohn Fastabend #include <linux/bpf.h> 2916962b24SJohn Fastabend #include <linux/if_link.h> 30421f4292SDaniel Borkmann #include <linux/tls.h> 3116962b24SJohn Fastabend #include <assert.h> 3216962b24SJohn Fastabend #include <libgen.h> 3316962b24SJohn Fastabend 3416962b24SJohn Fastabend #include <getopt.h> 3516962b24SJohn Fastabend 3616962b24SJohn Fastabend #include <bpf/bpf.h> 3716962b24SJohn Fastabend #include <bpf/libbpf.h> 3816962b24SJohn Fastabend 3916962b24SJohn Fastabend #include "bpf_util.h" 4016962b24SJohn Fastabend #include "bpf_rlimit.h" 4116962b24SJohn Fastabend #include "cgroup_helpers.h" 4216962b24SJohn Fastabend 4316962b24SJohn Fastabend int running; 4416962b24SJohn Fastabend static void running_handler(int a); 4516962b24SJohn Fastabend 46421f4292SDaniel Borkmann #ifndef TCP_ULP 47421f4292SDaniel Borkmann # define TCP_ULP 31 48421f4292SDaniel Borkmann #endif 49421f4292SDaniel Borkmann #ifndef SOL_TLS 50421f4292SDaniel Borkmann # define SOL_TLS 282 51421f4292SDaniel Borkmann #endif 52421f4292SDaniel Borkmann 5316962b24SJohn Fastabend /* randomly selected ports for testing on lo */ 5416962b24SJohn Fastabend #define S1_PORT 10000 5516962b24SJohn Fastabend #define S2_PORT 10001 5616962b24SJohn Fastabend 57b8b394faSJohn Fastabend #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 58b8b394faSJohn Fastabend #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 5916962b24SJohn Fastabend #define CG_PATH "/sockmap" 6016962b24SJohn Fastabend 6116962b24SJohn Fastabend /* global sockets */ 6216962b24SJohn Fastabend int s1, s2, c1, c2, p1, p2; 6316962b24SJohn Fastabend int test_cnt; 6416962b24SJohn Fastabend int passed; 6516962b24SJohn Fastabend int failed; 6616962b24SJohn Fastabend int map_fd[8]; 6716962b24SJohn Fastabend struct bpf_map *maps[8]; 6816962b24SJohn Fastabend int prog_fd[11]; 6916962b24SJohn Fastabend 7016962b24SJohn Fastabend int txmsg_pass; 7116962b24SJohn Fastabend int txmsg_redir; 7216962b24SJohn Fastabend int txmsg_drop; 7316962b24SJohn Fastabend int txmsg_apply; 7416962b24SJohn Fastabend int txmsg_cork; 7516962b24SJohn Fastabend int txmsg_start; 7616962b24SJohn Fastabend int txmsg_end; 7784fbfe02SJohn Fastabend int txmsg_start_push; 7884fbfe02SJohn Fastabend int txmsg_end_push; 791ade9abaSJohn Fastabend int txmsg_start_pop; 801ade9abaSJohn Fastabend int txmsg_pop; 8116962b24SJohn Fastabend int txmsg_ingress; 8216962b24SJohn Fastabend int txmsg_skb; 83e9dd9047SJohn Fastabend int ktls; 84753fb2eeSJohn Fastabend int peek_flag; 8516962b24SJohn Fastabend 8616962b24SJohn Fastabend static const struct option long_options[] = { 8716962b24SJohn Fastabend {"help", no_argument, NULL, 'h' }, 8816962b24SJohn Fastabend {"cgroup", required_argument, NULL, 'c' }, 8916962b24SJohn Fastabend {"rate", required_argument, NULL, 'r' }, 9016962b24SJohn Fastabend {"verbose", no_argument, NULL, 'v' }, 9116962b24SJohn Fastabend {"iov_count", required_argument, NULL, 'i' }, 9216962b24SJohn Fastabend {"length", required_argument, NULL, 'l' }, 9316962b24SJohn Fastabend {"test", required_argument, NULL, 't' }, 9416962b24SJohn Fastabend {"data_test", no_argument, NULL, 'd' }, 9516962b24SJohn Fastabend {"txmsg", no_argument, &txmsg_pass, 1 }, 9616962b24SJohn Fastabend {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 9716962b24SJohn Fastabend {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 9816962b24SJohn Fastabend {"txmsg_apply", required_argument, NULL, 'a'}, 9916962b24SJohn Fastabend {"txmsg_cork", required_argument, NULL, 'k'}, 10016962b24SJohn Fastabend {"txmsg_start", required_argument, NULL, 's'}, 10116962b24SJohn Fastabend {"txmsg_end", required_argument, NULL, 'e'}, 10284fbfe02SJohn Fastabend {"txmsg_start_push", required_argument, NULL, 'p'}, 10384fbfe02SJohn Fastabend {"txmsg_end_push", required_argument, NULL, 'q'}, 1041ade9abaSJohn Fastabend {"txmsg_start_pop", required_argument, NULL, 'w'}, 1051ade9abaSJohn Fastabend {"txmsg_pop", required_argument, NULL, 'x'}, 10616962b24SJohn Fastabend {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 10716962b24SJohn Fastabend {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 108e9dd9047SJohn Fastabend {"ktls", no_argument, &ktls, 1 }, 109753fb2eeSJohn Fastabend {"peek", no_argument, &peek_flag, 1 }, 11016962b24SJohn Fastabend {0, 0, NULL, 0 } 11116962b24SJohn Fastabend }; 11216962b24SJohn Fastabend 11316962b24SJohn Fastabend static void usage(char *argv[]) 11416962b24SJohn Fastabend { 11516962b24SJohn Fastabend int i; 11616962b24SJohn Fastabend 11716962b24SJohn Fastabend printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 11816962b24SJohn Fastabend printf(" options:\n"); 11916962b24SJohn Fastabend for (i = 0; long_options[i].name != 0; i++) { 12016962b24SJohn Fastabend printf(" --%-12s", long_options[i].name); 12116962b24SJohn Fastabend if (long_options[i].flag != NULL) 12216962b24SJohn Fastabend printf(" flag (internal value:%d)\n", 12316962b24SJohn Fastabend *long_options[i].flag); 12416962b24SJohn Fastabend else 12516962b24SJohn Fastabend printf(" -%c\n", long_options[i].val); 12616962b24SJohn Fastabend } 12716962b24SJohn Fastabend printf("\n"); 12816962b24SJohn Fastabend } 12916962b24SJohn Fastabend 130e9dd9047SJohn Fastabend char *sock_to_string(int s) 131e9dd9047SJohn Fastabend { 132e9dd9047SJohn Fastabend if (s == c1) 133e9dd9047SJohn Fastabend return "client1"; 134e9dd9047SJohn Fastabend else if (s == c2) 135e9dd9047SJohn Fastabend return "client2"; 136e9dd9047SJohn Fastabend else if (s == s1) 137e9dd9047SJohn Fastabend return "server1"; 138e9dd9047SJohn Fastabend else if (s == s2) 139e9dd9047SJohn Fastabend return "server2"; 140e9dd9047SJohn Fastabend else if (s == p1) 141e9dd9047SJohn Fastabend return "peer1"; 142e9dd9047SJohn Fastabend else if (s == p2) 143e9dd9047SJohn Fastabend return "peer2"; 144e9dd9047SJohn Fastabend else 145e9dd9047SJohn Fastabend return "unknown"; 146e9dd9047SJohn Fastabend } 147e9dd9047SJohn Fastabend 148e9dd9047SJohn Fastabend static int sockmap_init_ktls(int verbose, int s) 149e9dd9047SJohn Fastabend { 150e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_tx = { 151e9dd9047SJohn Fastabend .info = { 152e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION, 153e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128, 154e9dd9047SJohn Fastabend }, 155e9dd9047SJohn Fastabend }; 156e9dd9047SJohn Fastabend struct tls12_crypto_info_aes_gcm_128 tls_rx = { 157e9dd9047SJohn Fastabend .info = { 158e9dd9047SJohn Fastabend .version = TLS_1_2_VERSION, 159e9dd9047SJohn Fastabend .cipher_type = TLS_CIPHER_AES_GCM_128, 160e9dd9047SJohn Fastabend }, 161e9dd9047SJohn Fastabend }; 162e9dd9047SJohn Fastabend int so_buf = 6553500; 163e9dd9047SJohn Fastabend int err; 164e9dd9047SJohn Fastabend 165e9dd9047SJohn Fastabend err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 166e9dd9047SJohn Fastabend if (err) { 167e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 168e9dd9047SJohn Fastabend return -EINVAL; 169e9dd9047SJohn Fastabend } 170e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 171e9dd9047SJohn Fastabend if (err) { 172e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 173e9dd9047SJohn Fastabend return -EINVAL; 174e9dd9047SJohn Fastabend } 175e9dd9047SJohn Fastabend err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 176e9dd9047SJohn Fastabend if (err) { 177e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 178e9dd9047SJohn Fastabend return -EINVAL; 179e9dd9047SJohn Fastabend } 180e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 181e9dd9047SJohn Fastabend if (err) { 182e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 183e9dd9047SJohn Fastabend return -EINVAL; 184e9dd9047SJohn Fastabend } 185e9dd9047SJohn Fastabend err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 186e9dd9047SJohn Fastabend if (err) { 187e9dd9047SJohn Fastabend fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 188e9dd9047SJohn Fastabend return -EINVAL; 189e9dd9047SJohn Fastabend } 190e9dd9047SJohn Fastabend 191e9dd9047SJohn Fastabend if (verbose) 192e9dd9047SJohn Fastabend fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 193e9dd9047SJohn Fastabend return 0; 194e9dd9047SJohn Fastabend } 19516962b24SJohn Fastabend static int sockmap_init_sockets(int verbose) 19616962b24SJohn Fastabend { 19716962b24SJohn Fastabend int i, err, one = 1; 19816962b24SJohn Fastabend struct sockaddr_in addr; 19916962b24SJohn Fastabend int *fds[4] = {&s1, &s2, &c1, &c2}; 20016962b24SJohn Fastabend 20116962b24SJohn Fastabend s1 = s2 = p1 = p2 = c1 = c2 = 0; 20216962b24SJohn Fastabend 20316962b24SJohn Fastabend /* Init sockets */ 20416962b24SJohn Fastabend for (i = 0; i < 4; i++) { 20516962b24SJohn Fastabend *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 20616962b24SJohn Fastabend if (*fds[i] < 0) { 20716962b24SJohn Fastabend perror("socket s1 failed()"); 20816962b24SJohn Fastabend return errno; 20916962b24SJohn Fastabend } 21016962b24SJohn Fastabend } 21116962b24SJohn Fastabend 21216962b24SJohn Fastabend /* Allow reuse */ 21316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 21416962b24SJohn Fastabend err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 21516962b24SJohn Fastabend (char *)&one, sizeof(one)); 21616962b24SJohn Fastabend if (err) { 21716962b24SJohn Fastabend perror("setsockopt failed()"); 21816962b24SJohn Fastabend return errno; 21916962b24SJohn Fastabend } 22016962b24SJohn Fastabend } 22116962b24SJohn Fastabend 22216962b24SJohn Fastabend /* Non-blocking sockets */ 22316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 22416962b24SJohn Fastabend err = ioctl(*fds[i], FIONBIO, (char *)&one); 22516962b24SJohn Fastabend if (err < 0) { 22616962b24SJohn Fastabend perror("ioctl s1 failed()"); 22716962b24SJohn Fastabend return errno; 22816962b24SJohn Fastabend } 22916962b24SJohn Fastabend } 23016962b24SJohn Fastabend 23116962b24SJohn Fastabend /* Bind server sockets */ 23216962b24SJohn Fastabend memset(&addr, 0, sizeof(struct sockaddr_in)); 23316962b24SJohn Fastabend addr.sin_family = AF_INET; 23416962b24SJohn Fastabend addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 23516962b24SJohn Fastabend 23616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 23716962b24SJohn Fastabend err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 23816962b24SJohn Fastabend if (err < 0) { 239e5dc9dd3SJakub Kicinski perror("bind s1 failed()"); 24016962b24SJohn Fastabend return errno; 24116962b24SJohn Fastabend } 24216962b24SJohn Fastabend 24316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 24416962b24SJohn Fastabend err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 24516962b24SJohn Fastabend if (err < 0) { 246e5dc9dd3SJakub Kicinski perror("bind s2 failed()"); 24716962b24SJohn Fastabend return errno; 24816962b24SJohn Fastabend } 24916962b24SJohn Fastabend 25016962b24SJohn Fastabend /* Listen server sockets */ 25116962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 25216962b24SJohn Fastabend err = listen(s1, 32); 25316962b24SJohn Fastabend if (err < 0) { 254e5dc9dd3SJakub Kicinski perror("listen s1 failed()"); 25516962b24SJohn Fastabend return errno; 25616962b24SJohn Fastabend } 25716962b24SJohn Fastabend 25816962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 25916962b24SJohn Fastabend err = listen(s2, 32); 26016962b24SJohn Fastabend if (err < 0) { 261e5dc9dd3SJakub Kicinski perror("listen s1 failed()"); 26216962b24SJohn Fastabend return errno; 26316962b24SJohn Fastabend } 26416962b24SJohn Fastabend 26516962b24SJohn Fastabend /* Initiate Connect */ 26616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 26716962b24SJohn Fastabend err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 26816962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 269e5dc9dd3SJakub Kicinski perror("connect c1 failed()"); 27016962b24SJohn Fastabend return errno; 27116962b24SJohn Fastabend } 27216962b24SJohn Fastabend 27316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 27416962b24SJohn Fastabend err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 27516962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 276e5dc9dd3SJakub Kicinski perror("connect c2 failed()"); 27716962b24SJohn Fastabend return errno; 27816962b24SJohn Fastabend } else if (err < 0) { 27916962b24SJohn Fastabend err = 0; 28016962b24SJohn Fastabend } 28116962b24SJohn Fastabend 28216962b24SJohn Fastabend /* Accept Connecrtions */ 28316962b24SJohn Fastabend p1 = accept(s1, NULL, NULL); 28416962b24SJohn Fastabend if (p1 < 0) { 285e5dc9dd3SJakub Kicinski perror("accept s1 failed()"); 28616962b24SJohn Fastabend return errno; 28716962b24SJohn Fastabend } 28816962b24SJohn Fastabend 28916962b24SJohn Fastabend p2 = accept(s2, NULL, NULL); 29016962b24SJohn Fastabend if (p2 < 0) { 291e5dc9dd3SJakub Kicinski perror("accept s1 failed()"); 29216962b24SJohn Fastabend return errno; 29316962b24SJohn Fastabend } 29416962b24SJohn Fastabend 29516962b24SJohn Fastabend if (verbose) { 29616962b24SJohn Fastabend printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 29716962b24SJohn Fastabend printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 29816962b24SJohn Fastabend c1, s1, c2, s2); 29916962b24SJohn Fastabend } 30016962b24SJohn Fastabend return 0; 30116962b24SJohn Fastabend } 30216962b24SJohn Fastabend 30316962b24SJohn Fastabend struct msg_stats { 30416962b24SJohn Fastabend size_t bytes_sent; 30516962b24SJohn Fastabend size_t bytes_recvd; 30616962b24SJohn Fastabend struct timespec start; 30716962b24SJohn Fastabend struct timespec end; 30816962b24SJohn Fastabend }; 30916962b24SJohn Fastabend 31016962b24SJohn Fastabend struct sockmap_options { 31116962b24SJohn Fastabend int verbose; 31216962b24SJohn Fastabend bool base; 31316962b24SJohn Fastabend bool sendpage; 31416962b24SJohn Fastabend bool data_test; 31516962b24SJohn Fastabend bool drop_expected; 31616962b24SJohn Fastabend int iov_count; 31716962b24SJohn Fastabend int iov_length; 31816962b24SJohn Fastabend int rate; 31916962b24SJohn Fastabend }; 32016962b24SJohn Fastabend 32116962b24SJohn Fastabend static int msg_loop_sendpage(int fd, int iov_length, int cnt, 32216962b24SJohn Fastabend struct msg_stats *s, 32316962b24SJohn Fastabend struct sockmap_options *opt) 32416962b24SJohn Fastabend { 32516962b24SJohn Fastabend bool drop = opt->drop_expected; 32616962b24SJohn Fastabend unsigned char k = 0; 32716962b24SJohn Fastabend FILE *file; 32816962b24SJohn Fastabend int i, fp; 32916962b24SJohn Fastabend 330c31dbb1eSLorenz Bauer file = tmpfile(); 3314b67c515SJakub Kicinski if (!file) { 3324b67c515SJakub Kicinski perror("create file for sendpage"); 3334b67c515SJakub Kicinski return 1; 3344b67c515SJakub Kicinski } 33516962b24SJohn Fastabend for (i = 0; i < iov_length * cnt; i++, k++) 33616962b24SJohn Fastabend fwrite(&k, sizeof(char), 1, file); 33716962b24SJohn Fastabend fflush(file); 33816962b24SJohn Fastabend fseek(file, 0, SEEK_SET); 33916962b24SJohn Fastabend 340c31dbb1eSLorenz Bauer fp = fileno(file); 3414b67c515SJakub Kicinski 34216962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 34316962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 344248aba1dSJohn Fastabend int sent; 345248aba1dSJohn Fastabend 346248aba1dSJohn Fastabend errno = 0; 347248aba1dSJohn Fastabend sent = sendfile(fd, fp, NULL, iov_length); 34816962b24SJohn Fastabend 34916962b24SJohn Fastabend if (!drop && sent < 0) { 350248aba1dSJohn Fastabend perror("sendpage loop error"); 351c31dbb1eSLorenz Bauer fclose(file); 35216962b24SJohn Fastabend return sent; 35316962b24SJohn Fastabend } else if (drop && sent >= 0) { 354248aba1dSJohn Fastabend printf("sendpage loop error expected: %i errno %i\n", 355248aba1dSJohn Fastabend sent, errno); 356c31dbb1eSLorenz Bauer fclose(file); 35716962b24SJohn Fastabend return -EIO; 35816962b24SJohn Fastabend } 35916962b24SJohn Fastabend 36016962b24SJohn Fastabend if (sent > 0) 36116962b24SJohn Fastabend s->bytes_sent += sent; 36216962b24SJohn Fastabend } 36316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 364c31dbb1eSLorenz Bauer fclose(file); 36516962b24SJohn Fastabend return 0; 36616962b24SJohn Fastabend } 36716962b24SJohn Fastabend 368753fb2eeSJohn Fastabend static void msg_free_iov(struct msghdr *msg) 36916962b24SJohn Fastabend { 370753fb2eeSJohn Fastabend int i; 371753fb2eeSJohn Fastabend 372753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++) 373753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base); 374753fb2eeSJohn Fastabend free(msg->msg_iov); 375753fb2eeSJohn Fastabend msg->msg_iov = NULL; 376753fb2eeSJohn Fastabend msg->msg_iovlen = 0; 377753fb2eeSJohn Fastabend } 378753fb2eeSJohn Fastabend 379753fb2eeSJohn Fastabend static int msg_alloc_iov(struct msghdr *msg, 380753fb2eeSJohn Fastabend int iov_count, int iov_length, 381753fb2eeSJohn Fastabend bool data, bool xmit) 382753fb2eeSJohn Fastabend { 383753fb2eeSJohn Fastabend unsigned char k = 0; 38416962b24SJohn Fastabend struct iovec *iov; 385753fb2eeSJohn Fastabend int i; 38616962b24SJohn Fastabend 38716962b24SJohn Fastabend iov = calloc(iov_count, sizeof(struct iovec)); 38816962b24SJohn Fastabend if (!iov) 38916962b24SJohn Fastabend return errno; 39016962b24SJohn Fastabend 39116962b24SJohn Fastabend for (i = 0; i < iov_count; i++) { 39216962b24SJohn Fastabend unsigned char *d = calloc(iov_length, sizeof(char)); 39316962b24SJohn Fastabend 39416962b24SJohn Fastabend if (!d) { 39516962b24SJohn Fastabend fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 396753fb2eeSJohn Fastabend goto unwind_iov; 39716962b24SJohn Fastabend } 39816962b24SJohn Fastabend iov[i].iov_base = d; 39916962b24SJohn Fastabend iov[i].iov_len = iov_length; 40016962b24SJohn Fastabend 401753fb2eeSJohn Fastabend if (data && xmit) { 40216962b24SJohn Fastabend int j; 40316962b24SJohn Fastabend 40416962b24SJohn Fastabend for (j = 0; j < iov_length; j++) 40516962b24SJohn Fastabend d[j] = k++; 40616962b24SJohn Fastabend } 40716962b24SJohn Fastabend } 40816962b24SJohn Fastabend 409753fb2eeSJohn Fastabend msg->msg_iov = iov; 410753fb2eeSJohn Fastabend msg->msg_iovlen = iov_count; 411753fb2eeSJohn Fastabend 412753fb2eeSJohn Fastabend return 0; 413753fb2eeSJohn Fastabend unwind_iov: 414753fb2eeSJohn Fastabend for (i--; i >= 0 ; i--) 415753fb2eeSJohn Fastabend free(msg->msg_iov[i].iov_base); 416753fb2eeSJohn Fastabend return -ENOMEM; 417753fb2eeSJohn Fastabend } 418753fb2eeSJohn Fastabend 419753fb2eeSJohn Fastabend static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 420753fb2eeSJohn Fastabend { 421753fb2eeSJohn Fastabend int i, j, bytes_cnt = 0; 422753fb2eeSJohn Fastabend unsigned char k = 0; 423753fb2eeSJohn Fastabend 424753fb2eeSJohn Fastabend for (i = 0; i < msg->msg_iovlen; i++) { 425753fb2eeSJohn Fastabend unsigned char *d = msg->msg_iov[i].iov_base; 426753fb2eeSJohn Fastabend 427753fb2eeSJohn Fastabend for (j = 0; 428753fb2eeSJohn Fastabend j < msg->msg_iov[i].iov_len && size; j++) { 429753fb2eeSJohn Fastabend if (d[j] != k++) { 430753fb2eeSJohn Fastabend fprintf(stderr, 431753fb2eeSJohn Fastabend "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 432753fb2eeSJohn Fastabend i, j, d[j], k - 1, d[j+1], k); 433753fb2eeSJohn Fastabend return -EIO; 434753fb2eeSJohn Fastabend } 435753fb2eeSJohn Fastabend bytes_cnt++; 436753fb2eeSJohn Fastabend if (bytes_cnt == chunk_sz) { 43716962b24SJohn Fastabend k = 0; 438753fb2eeSJohn Fastabend bytes_cnt = 0; 439753fb2eeSJohn Fastabend } 440753fb2eeSJohn Fastabend size--; 441753fb2eeSJohn Fastabend } 442753fb2eeSJohn Fastabend } 443753fb2eeSJohn Fastabend return 0; 444753fb2eeSJohn Fastabend } 445753fb2eeSJohn Fastabend 446753fb2eeSJohn Fastabend static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 447753fb2eeSJohn Fastabend struct msg_stats *s, bool tx, 448753fb2eeSJohn Fastabend struct sockmap_options *opt) 449753fb2eeSJohn Fastabend { 450753fb2eeSJohn Fastabend struct msghdr msg = {0}, msg_peek = {0}; 451753fb2eeSJohn Fastabend int err, i, flags = MSG_NOSIGNAL; 452753fb2eeSJohn Fastabend bool drop = opt->drop_expected; 453753fb2eeSJohn Fastabend bool data = opt->data_test; 454753fb2eeSJohn Fastabend 455753fb2eeSJohn Fastabend err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx); 456753fb2eeSJohn Fastabend if (err) 457753fb2eeSJohn Fastabend goto out_errno; 458753fb2eeSJohn Fastabend if (peek_flag) { 459753fb2eeSJohn Fastabend err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 460753fb2eeSJohn Fastabend if (err) 461753fb2eeSJohn Fastabend goto out_errno; 462753fb2eeSJohn Fastabend } 46316962b24SJohn Fastabend 46416962b24SJohn Fastabend if (tx) { 46516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 46616962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 467248aba1dSJohn Fastabend int sent; 468248aba1dSJohn Fastabend 469248aba1dSJohn Fastabend errno = 0; 470248aba1dSJohn Fastabend sent = sendmsg(fd, &msg, flags); 47116962b24SJohn Fastabend 47216962b24SJohn Fastabend if (!drop && sent < 0) { 473248aba1dSJohn Fastabend perror("sendmsg loop error"); 47416962b24SJohn Fastabend goto out_errno; 47516962b24SJohn Fastabend } else if (drop && sent >= 0) { 476248aba1dSJohn Fastabend fprintf(stderr, 477248aba1dSJohn Fastabend "sendmsg loop error expected: %i errno %i\n", 478248aba1dSJohn Fastabend sent, errno); 47916962b24SJohn Fastabend errno = -EIO; 48016962b24SJohn Fastabend goto out_errno; 48116962b24SJohn Fastabend } 48216962b24SJohn Fastabend if (sent > 0) 48316962b24SJohn Fastabend s->bytes_sent += sent; 48416962b24SJohn Fastabend } 48516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 48616962b24SJohn Fastabend } else { 487753fb2eeSJohn Fastabend int slct, recvp = 0, recv, max_fd = fd; 4881ade9abaSJohn Fastabend float total_bytes, txmsg_pop_total; 48916962b24SJohn Fastabend int fd_flags = O_NONBLOCK; 49016962b24SJohn Fastabend struct timeval timeout; 49116962b24SJohn Fastabend fd_set w; 49216962b24SJohn Fastabend 49316962b24SJohn Fastabend fcntl(fd, fd_flags); 4941ade9abaSJohn Fastabend /* Account for pop bytes noting each iteration of apply will 4951ade9abaSJohn Fastabend * call msg_pop_data helper so we need to account for this 4961ade9abaSJohn Fastabend * by calculating the number of apply iterations. Note user 4971ade9abaSJohn Fastabend * of the tool can create cases where no data is sent by 4981ade9abaSJohn Fastabend * manipulating pop/push/pull/etc. For example txmsg_apply 1 4991ade9abaSJohn Fastabend * with txmsg_pop 1 will try to apply 1B at a time but each 5001ade9abaSJohn Fastabend * iteration will then pop 1B so no data will ever be sent. 5011ade9abaSJohn Fastabend * This is really only useful for testing edge cases in code 5021ade9abaSJohn Fastabend * paths. 5031ade9abaSJohn Fastabend */ 50416962b24SJohn Fastabend total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 5051ade9abaSJohn Fastabend txmsg_pop_total = txmsg_pop; 5061ade9abaSJohn Fastabend if (txmsg_apply) 5071ade9abaSJohn Fastabend txmsg_pop_total *= (total_bytes / txmsg_apply); 5081ade9abaSJohn Fastabend total_bytes -= txmsg_pop_total; 50916962b24SJohn Fastabend err = clock_gettime(CLOCK_MONOTONIC, &s->start); 51016962b24SJohn Fastabend if (err < 0) 511e5dc9dd3SJakub Kicinski perror("recv start time"); 51216962b24SJohn Fastabend while (s->bytes_recvd < total_bytes) { 513a009f1f3SPrashant Bhole if (txmsg_cork) { 514a18fda1aSJohn Fastabend timeout.tv_sec = 0; 5153c6ed988SDaniel Borkmann timeout.tv_usec = 300000; 516a009f1f3SPrashant Bhole } else { 5171ade9abaSJohn Fastabend timeout.tv_sec = 3; 518a009f1f3SPrashant Bhole timeout.tv_usec = 0; 519a009f1f3SPrashant Bhole } 52016962b24SJohn Fastabend 52116962b24SJohn Fastabend /* FD sets */ 52216962b24SJohn Fastabend FD_ZERO(&w); 52316962b24SJohn Fastabend FD_SET(fd, &w); 52416962b24SJohn Fastabend 52516962b24SJohn Fastabend slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 52616962b24SJohn Fastabend if (slct == -1) { 52716962b24SJohn Fastabend perror("select()"); 52816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 52916962b24SJohn Fastabend goto out_errno; 53016962b24SJohn Fastabend } else if (!slct) { 53116962b24SJohn Fastabend if (opt->verbose) 5321ade9abaSJohn Fastabend fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 53316962b24SJohn Fastabend errno = -EIO; 53416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 53516962b24SJohn Fastabend goto out_errno; 53616962b24SJohn Fastabend } 53716962b24SJohn Fastabend 538753fb2eeSJohn Fastabend errno = 0; 539753fb2eeSJohn Fastabend if (peek_flag) { 540753fb2eeSJohn Fastabend flags |= MSG_PEEK; 541753fb2eeSJohn Fastabend recvp = recvmsg(fd, &msg_peek, flags); 542753fb2eeSJohn Fastabend if (recvp < 0) { 543753fb2eeSJohn Fastabend if (errno != EWOULDBLOCK) { 544753fb2eeSJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 545753fb2eeSJohn Fastabend goto out_errno; 546753fb2eeSJohn Fastabend } 547753fb2eeSJohn Fastabend } 548753fb2eeSJohn Fastabend flags = 0; 549753fb2eeSJohn Fastabend } 550753fb2eeSJohn Fastabend 55116962b24SJohn Fastabend recv = recvmsg(fd, &msg, flags); 55216962b24SJohn Fastabend if (recv < 0) { 55316962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 55416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 555e5dc9dd3SJakub Kicinski perror("recv failed()"); 55616962b24SJohn Fastabend goto out_errno; 55716962b24SJohn Fastabend } 55816962b24SJohn Fastabend } 55916962b24SJohn Fastabend 56016962b24SJohn Fastabend s->bytes_recvd += recv; 56116962b24SJohn Fastabend 562753fb2eeSJohn Fastabend if (data) { 563753fb2eeSJohn Fastabend int chunk_sz = opt->sendpage ? 564753fb2eeSJohn Fastabend iov_length * cnt : 565753fb2eeSJohn Fastabend iov_length * iov_count; 56616962b24SJohn Fastabend 567753fb2eeSJohn Fastabend errno = msg_verify_data(&msg, recv, chunk_sz); 568753fb2eeSJohn Fastabend if (errno) { 569e5dc9dd3SJakub Kicinski perror("data verify msg failed"); 57016962b24SJohn Fastabend goto out_errno; 57116962b24SJohn Fastabend } 572753fb2eeSJohn Fastabend if (recvp) { 573753fb2eeSJohn Fastabend errno = msg_verify_data(&msg_peek, 574753fb2eeSJohn Fastabend recvp, 575753fb2eeSJohn Fastabend chunk_sz); 576753fb2eeSJohn Fastabend if (errno) { 577e5dc9dd3SJakub Kicinski perror("data verify msg_peek failed"); 578753fb2eeSJohn Fastabend goto out_errno; 57916962b24SJohn Fastabend } 58016962b24SJohn Fastabend } 58116962b24SJohn Fastabend } 58216962b24SJohn Fastabend } 58316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 58416962b24SJohn Fastabend } 58516962b24SJohn Fastabend 586753fb2eeSJohn Fastabend msg_free_iov(&msg); 587753fb2eeSJohn Fastabend msg_free_iov(&msg_peek); 588753fb2eeSJohn Fastabend return err; 58916962b24SJohn Fastabend out_errno: 590753fb2eeSJohn Fastabend msg_free_iov(&msg); 591753fb2eeSJohn Fastabend msg_free_iov(&msg_peek); 59216962b24SJohn Fastabend return errno; 59316962b24SJohn Fastabend } 59416962b24SJohn Fastabend 59516962b24SJohn Fastabend static float giga = 1000000000; 59616962b24SJohn Fastabend 59716962b24SJohn Fastabend static inline float sentBps(struct msg_stats s) 59816962b24SJohn Fastabend { 59916962b24SJohn Fastabend return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 60016962b24SJohn Fastabend } 60116962b24SJohn Fastabend 60216962b24SJohn Fastabend static inline float recvdBps(struct msg_stats s) 60316962b24SJohn Fastabend { 60416962b24SJohn Fastabend return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 60516962b24SJohn Fastabend } 60616962b24SJohn Fastabend 60716962b24SJohn Fastabend static int sendmsg_test(struct sockmap_options *opt) 60816962b24SJohn Fastabend { 60916962b24SJohn Fastabend float sent_Bps = 0, recvd_Bps = 0; 61016962b24SJohn Fastabend int rx_fd, txpid, rxpid, err = 0; 61116962b24SJohn Fastabend struct msg_stats s = {0}; 61216962b24SJohn Fastabend int iov_count = opt->iov_count; 61316962b24SJohn Fastabend int iov_buf = opt->iov_length; 61416edddfeSPrashant Bhole int rx_status, tx_status; 61516962b24SJohn Fastabend int cnt = opt->rate; 61616962b24SJohn Fastabend 61716962b24SJohn Fastabend errno = 0; 61816962b24SJohn Fastabend 61916962b24SJohn Fastabend if (opt->base) 62016962b24SJohn Fastabend rx_fd = p1; 62116962b24SJohn Fastabend else 62216962b24SJohn Fastabend rx_fd = p2; 62316962b24SJohn Fastabend 624e9dd9047SJohn Fastabend if (ktls) { 625e9dd9047SJohn Fastabend /* Redirecting into non-TLS socket which sends into a TLS 626e9dd9047SJohn Fastabend * socket is not a valid test. So in this case lets not 627e9dd9047SJohn Fastabend * enable kTLS but still run the test. 628e9dd9047SJohn Fastabend */ 629e9dd9047SJohn Fastabend if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 630e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, rx_fd); 631e9dd9047SJohn Fastabend if (err) 632e9dd9047SJohn Fastabend return err; 633e9dd9047SJohn Fastabend } 634e9dd9047SJohn Fastabend err = sockmap_init_ktls(opt->verbose, c1); 635e9dd9047SJohn Fastabend if (err) 636e9dd9047SJohn Fastabend return err; 637e9dd9047SJohn Fastabend } 638e9dd9047SJohn Fastabend 63916962b24SJohn Fastabend rxpid = fork(); 64016962b24SJohn Fastabend if (rxpid == 0) { 64116962b24SJohn Fastabend if (opt->drop_expected) 64216edddfeSPrashant Bhole exit(0); 64316962b24SJohn Fastabend 64416962b24SJohn Fastabend if (opt->sendpage) 64516962b24SJohn Fastabend iov_count = 1; 64616962b24SJohn Fastabend err = msg_loop(rx_fd, iov_count, iov_buf, 64716962b24SJohn Fastabend cnt, &s, false, opt); 6481ade9abaSJohn Fastabend if (opt->verbose) 64916962b24SJohn Fastabend fprintf(stderr, 65016962b24SJohn Fastabend "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 65116962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 65216962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 65316962b24SJohn Fastabend sent_Bps = sentBps(s); 65416962b24SJohn Fastabend recvd_Bps = recvdBps(s); 65516962b24SJohn Fastabend } 65616962b24SJohn Fastabend if (opt->verbose) 65716962b24SJohn Fastabend fprintf(stdout, 658753fb2eeSJohn Fastabend "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 65916962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 660753fb2eeSJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 661753fb2eeSJohn Fastabend peek_flag ? "(peek_msg)" : ""); 66216edddfeSPrashant Bhole if (err && txmsg_cork) 66316edddfeSPrashant Bhole err = 0; 66416edddfeSPrashant Bhole exit(err ? 1 : 0); 66516962b24SJohn Fastabend } else if (rxpid == -1) { 666e5dc9dd3SJakub Kicinski perror("msg_loop_rx"); 66716962b24SJohn Fastabend return errno; 66816962b24SJohn Fastabend } 66916962b24SJohn Fastabend 67016962b24SJohn Fastabend txpid = fork(); 67116962b24SJohn Fastabend if (txpid == 0) { 67216962b24SJohn Fastabend if (opt->sendpage) 67316962b24SJohn Fastabend err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 67416962b24SJohn Fastabend else 67516962b24SJohn Fastabend err = msg_loop(c1, iov_count, iov_buf, 67616962b24SJohn Fastabend cnt, &s, true, opt); 67716962b24SJohn Fastabend 67816962b24SJohn Fastabend if (err) 67916962b24SJohn Fastabend fprintf(stderr, 68016962b24SJohn Fastabend "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 68116962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 68216962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 68316962b24SJohn Fastabend sent_Bps = sentBps(s); 68416962b24SJohn Fastabend recvd_Bps = recvdBps(s); 68516962b24SJohn Fastabend } 68616962b24SJohn Fastabend if (opt->verbose) 68716962b24SJohn Fastabend fprintf(stdout, 68816962b24SJohn Fastabend "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 68916962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 69016962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 69116edddfeSPrashant Bhole exit(err ? 1 : 0); 69216962b24SJohn Fastabend } else if (txpid == -1) { 693e5dc9dd3SJakub Kicinski perror("msg_loop_tx"); 69416962b24SJohn Fastabend return errno; 69516962b24SJohn Fastabend } 69616962b24SJohn Fastabend 69716edddfeSPrashant Bhole assert(waitpid(rxpid, &rx_status, 0) == rxpid); 69816edddfeSPrashant Bhole assert(waitpid(txpid, &tx_status, 0) == txpid); 69916edddfeSPrashant Bhole if (WIFEXITED(rx_status)) { 70016edddfeSPrashant Bhole err = WEXITSTATUS(rx_status); 70116edddfeSPrashant Bhole if (err) { 702248aba1dSJohn Fastabend fprintf(stderr, "rx thread exited with err %d.\n", err); 70316edddfeSPrashant Bhole goto out; 70416edddfeSPrashant Bhole } 70516edddfeSPrashant Bhole } 70616edddfeSPrashant Bhole if (WIFEXITED(tx_status)) { 70716edddfeSPrashant Bhole err = WEXITSTATUS(tx_status); 70816edddfeSPrashant Bhole if (err) 709248aba1dSJohn Fastabend fprintf(stderr, "tx thread exited with err %d.\n", err); 71016edddfeSPrashant Bhole } 71116edddfeSPrashant Bhole out: 71216962b24SJohn Fastabend return err; 71316962b24SJohn Fastabend } 71416962b24SJohn Fastabend 71516962b24SJohn Fastabend static int forever_ping_pong(int rate, struct sockmap_options *opt) 71616962b24SJohn Fastabend { 71716962b24SJohn Fastabend struct timeval timeout; 71816962b24SJohn Fastabend char buf[1024] = {0}; 71916962b24SJohn Fastabend int sc; 72016962b24SJohn Fastabend 72116962b24SJohn Fastabend timeout.tv_sec = 10; 72216962b24SJohn Fastabend timeout.tv_usec = 0; 72316962b24SJohn Fastabend 72416962b24SJohn Fastabend /* Ping/Pong data from client to server */ 72516962b24SJohn Fastabend sc = send(c1, buf, sizeof(buf), 0); 72616962b24SJohn Fastabend if (sc < 0) { 727e5dc9dd3SJakub Kicinski perror("send failed()"); 72816962b24SJohn Fastabend return sc; 72916962b24SJohn Fastabend } 73016962b24SJohn Fastabend 73116962b24SJohn Fastabend do { 73216962b24SJohn Fastabend int s, rc, i, max_fd = p2; 73316962b24SJohn Fastabend fd_set w; 73416962b24SJohn Fastabend 73516962b24SJohn Fastabend /* FD sets */ 73616962b24SJohn Fastabend FD_ZERO(&w); 73716962b24SJohn Fastabend FD_SET(c1, &w); 73816962b24SJohn Fastabend FD_SET(c2, &w); 73916962b24SJohn Fastabend FD_SET(p1, &w); 74016962b24SJohn Fastabend FD_SET(p2, &w); 74116962b24SJohn Fastabend 74216962b24SJohn Fastabend s = select(max_fd + 1, &w, NULL, NULL, &timeout); 74316962b24SJohn Fastabend if (s == -1) { 74416962b24SJohn Fastabend perror("select()"); 74516962b24SJohn Fastabend break; 74616962b24SJohn Fastabend } else if (!s) { 74716962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 74816962b24SJohn Fastabend break; 74916962b24SJohn Fastabend } 75016962b24SJohn Fastabend 75116962b24SJohn Fastabend for (i = 0; i <= max_fd && s > 0; ++i) { 75216962b24SJohn Fastabend if (!FD_ISSET(i, &w)) 75316962b24SJohn Fastabend continue; 75416962b24SJohn Fastabend 75516962b24SJohn Fastabend s--; 75616962b24SJohn Fastabend 75716962b24SJohn Fastabend rc = recv(i, buf, sizeof(buf), 0); 75816962b24SJohn Fastabend if (rc < 0) { 75916962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 760e5dc9dd3SJakub Kicinski perror("recv failed()"); 76116962b24SJohn Fastabend return rc; 76216962b24SJohn Fastabend } 76316962b24SJohn Fastabend } 76416962b24SJohn Fastabend 76516962b24SJohn Fastabend if (rc == 0) { 76616962b24SJohn Fastabend close(i); 76716962b24SJohn Fastabend break; 76816962b24SJohn Fastabend } 76916962b24SJohn Fastabend 77016962b24SJohn Fastabend sc = send(i, buf, rc, 0); 77116962b24SJohn Fastabend if (sc < 0) { 772e5dc9dd3SJakub Kicinski perror("send failed()"); 77316962b24SJohn Fastabend return sc; 77416962b24SJohn Fastabend } 77516962b24SJohn Fastabend } 77616962b24SJohn Fastabend 77716962b24SJohn Fastabend if (rate) 77816962b24SJohn Fastabend sleep(rate); 77916962b24SJohn Fastabend 78016962b24SJohn Fastabend if (opt->verbose) { 78116962b24SJohn Fastabend printf("."); 78216962b24SJohn Fastabend fflush(stdout); 78316962b24SJohn Fastabend 78416962b24SJohn Fastabend } 78516962b24SJohn Fastabend } while (running); 78616962b24SJohn Fastabend 78716962b24SJohn Fastabend return 0; 78816962b24SJohn Fastabend } 78916962b24SJohn Fastabend 79016962b24SJohn Fastabend enum { 79116962b24SJohn Fastabend PING_PONG, 79216962b24SJohn Fastabend SENDMSG, 79316962b24SJohn Fastabend BASE, 79416962b24SJohn Fastabend BASE_SENDPAGE, 79516962b24SJohn Fastabend SENDPAGE, 79616962b24SJohn Fastabend }; 79716962b24SJohn Fastabend 79816962b24SJohn Fastabend static int run_options(struct sockmap_options *options, int cg_fd, int test) 79916962b24SJohn Fastabend { 80016962b24SJohn Fastabend int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 80116962b24SJohn Fastabend 80216962b24SJohn Fastabend /* If base test skip BPF setup */ 80316962b24SJohn Fastabend if (test == BASE || test == BASE_SENDPAGE) 80416962b24SJohn Fastabend goto run; 80516962b24SJohn Fastabend 80616962b24SJohn Fastabend /* Attach programs to sockmap */ 80716962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[0], 80816962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0); 80916962b24SJohn Fastabend if (err) { 81016962b24SJohn Fastabend fprintf(stderr, 81116962b24SJohn Fastabend "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 81216962b24SJohn Fastabend prog_fd[0], map_fd[0], err, strerror(errno)); 81316962b24SJohn Fastabend return err; 81416962b24SJohn Fastabend } 81516962b24SJohn Fastabend 81616962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[1], map_fd[0], 81716962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0); 81816962b24SJohn Fastabend if (err) { 81916962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 82016962b24SJohn Fastabend err, strerror(errno)); 82116962b24SJohn Fastabend return err; 82216962b24SJohn Fastabend } 82316962b24SJohn Fastabend 82416962b24SJohn Fastabend /* Attach to cgroups */ 82516962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 82616962b24SJohn Fastabend if (err) { 82716962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 82816962b24SJohn Fastabend err, strerror(errno)); 82916962b24SJohn Fastabend return err; 83016962b24SJohn Fastabend } 83116962b24SJohn Fastabend 83216962b24SJohn Fastabend run: 83316962b24SJohn Fastabend err = sockmap_init_sockets(options->verbose); 83416962b24SJohn Fastabend if (err) { 83516962b24SJohn Fastabend fprintf(stderr, "ERROR: test socket failed: %d\n", err); 83616962b24SJohn Fastabend goto out; 83716962b24SJohn Fastabend } 83816962b24SJohn Fastabend 83916962b24SJohn Fastabend /* Attach txmsg program to sockmap */ 84016962b24SJohn Fastabend if (txmsg_pass) 84116962b24SJohn Fastabend tx_prog_fd = prog_fd[3]; 84216962b24SJohn Fastabend else if (txmsg_redir) 843d79a3212SJohn Fastabend tx_prog_fd = prog_fd[4]; 844d79a3212SJohn Fastabend else if (txmsg_apply) 84516962b24SJohn Fastabend tx_prog_fd = prog_fd[5]; 846d79a3212SJohn Fastabend else if (txmsg_cork) 84716962b24SJohn Fastabend tx_prog_fd = prog_fd[6]; 84816962b24SJohn Fastabend else if (txmsg_drop) 84916962b24SJohn Fastabend tx_prog_fd = prog_fd[7]; 85016962b24SJohn Fastabend else 85116962b24SJohn Fastabend tx_prog_fd = 0; 85216962b24SJohn Fastabend 85316962b24SJohn Fastabend if (tx_prog_fd) { 85416962b24SJohn Fastabend int redir_fd, i = 0; 85516962b24SJohn Fastabend 85616962b24SJohn Fastabend err = bpf_prog_attach(tx_prog_fd, 85716962b24SJohn Fastabend map_fd[1], BPF_SK_MSG_VERDICT, 0); 85816962b24SJohn Fastabend if (err) { 85916962b24SJohn Fastabend fprintf(stderr, 86016962b24SJohn Fastabend "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 86116962b24SJohn Fastabend err, strerror(errno)); 86216962b24SJohn Fastabend goto out; 86316962b24SJohn Fastabend } 86416962b24SJohn Fastabend 86516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 86616962b24SJohn Fastabend if (err) { 86716962b24SJohn Fastabend fprintf(stderr, 86816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 86916962b24SJohn Fastabend err, strerror(errno)); 87016962b24SJohn Fastabend goto out; 87116962b24SJohn Fastabend } 87216962b24SJohn Fastabend 873d79a3212SJohn Fastabend if (txmsg_redir) 87416962b24SJohn Fastabend redir_fd = c2; 87516962b24SJohn Fastabend else 87616962b24SJohn Fastabend redir_fd = c1; 87716962b24SJohn Fastabend 87816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 87916962b24SJohn Fastabend if (err) { 88016962b24SJohn Fastabend fprintf(stderr, 88116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 88216962b24SJohn Fastabend err, strerror(errno)); 88316962b24SJohn Fastabend goto out; 88416962b24SJohn Fastabend } 88516962b24SJohn Fastabend 88616962b24SJohn Fastabend if (txmsg_apply) { 88716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[3], 88816962b24SJohn Fastabend &i, &txmsg_apply, BPF_ANY); 88916962b24SJohn Fastabend if (err) { 89016962b24SJohn Fastabend fprintf(stderr, 89116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 89216962b24SJohn Fastabend err, strerror(errno)); 89316962b24SJohn Fastabend goto out; 89416962b24SJohn Fastabend } 89516962b24SJohn Fastabend } 89616962b24SJohn Fastabend 89716962b24SJohn Fastabend if (txmsg_cork) { 89816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[4], 89916962b24SJohn Fastabend &i, &txmsg_cork, BPF_ANY); 90016962b24SJohn Fastabend if (err) { 90116962b24SJohn Fastabend fprintf(stderr, 90216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 90316962b24SJohn Fastabend err, strerror(errno)); 90416962b24SJohn Fastabend goto out; 90516962b24SJohn Fastabend } 90616962b24SJohn Fastabend } 90716962b24SJohn Fastabend 90816962b24SJohn Fastabend if (txmsg_start) { 90916962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 91016962b24SJohn Fastabend &i, &txmsg_start, BPF_ANY); 91116962b24SJohn Fastabend if (err) { 91216962b24SJohn Fastabend fprintf(stderr, 91316962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 91416962b24SJohn Fastabend err, strerror(errno)); 91516962b24SJohn Fastabend goto out; 91616962b24SJohn Fastabend } 91716962b24SJohn Fastabend } 91816962b24SJohn Fastabend 91916962b24SJohn Fastabend if (txmsg_end) { 92016962b24SJohn Fastabend i = 1; 92116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 92216962b24SJohn Fastabend &i, &txmsg_end, BPF_ANY); 92316962b24SJohn Fastabend if (err) { 92416962b24SJohn Fastabend fprintf(stderr, 92516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 92616962b24SJohn Fastabend err, strerror(errno)); 92716962b24SJohn Fastabend goto out; 92816962b24SJohn Fastabend } 92916962b24SJohn Fastabend } 93016962b24SJohn Fastabend 93184fbfe02SJohn Fastabend if (txmsg_start_push) { 93284fbfe02SJohn Fastabend i = 2; 93384fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 93484fbfe02SJohn Fastabend &i, &txmsg_start_push, BPF_ANY); 93584fbfe02SJohn Fastabend if (err) { 93684fbfe02SJohn Fastabend fprintf(stderr, 93784fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 93884fbfe02SJohn Fastabend err, strerror(errno)); 93984fbfe02SJohn Fastabend goto out; 94084fbfe02SJohn Fastabend } 94184fbfe02SJohn Fastabend } 94284fbfe02SJohn Fastabend 94384fbfe02SJohn Fastabend if (txmsg_end_push) { 94484fbfe02SJohn Fastabend i = 3; 94584fbfe02SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 94684fbfe02SJohn Fastabend &i, &txmsg_end_push, BPF_ANY); 94784fbfe02SJohn Fastabend if (err) { 94884fbfe02SJohn Fastabend fprintf(stderr, 94984fbfe02SJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 95084fbfe02SJohn Fastabend txmsg_end_push, i, err, strerror(errno)); 95184fbfe02SJohn Fastabend goto out; 95284fbfe02SJohn Fastabend } 95384fbfe02SJohn Fastabend } 95484fbfe02SJohn Fastabend 9551ade9abaSJohn Fastabend if (txmsg_start_pop) { 9561ade9abaSJohn Fastabend i = 4; 9571ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5], 9581ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY); 9591ade9abaSJohn Fastabend if (err) { 9601ade9abaSJohn Fastabend fprintf(stderr, 9611ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 9621ade9abaSJohn Fastabend txmsg_start_pop, i, err, strerror(errno)); 9631ade9abaSJohn Fastabend goto out; 9641ade9abaSJohn Fastabend } 9651ade9abaSJohn Fastabend } else { 9661ade9abaSJohn Fastabend i = 4; 9671ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5], 9681ade9abaSJohn Fastabend &i, &txmsg_start_pop, BPF_ANY); 9691ade9abaSJohn Fastabend } 9701ade9abaSJohn Fastabend 9711ade9abaSJohn Fastabend if (txmsg_pop) { 9721ade9abaSJohn Fastabend i = 5; 9731ade9abaSJohn Fastabend err = bpf_map_update_elem(map_fd[5], 9741ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY); 9751ade9abaSJohn Fastabend if (err) { 9761ade9abaSJohn Fastabend fprintf(stderr, 9771ade9abaSJohn Fastabend "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 9781ade9abaSJohn Fastabend txmsg_pop, i, err, strerror(errno)); 9791ade9abaSJohn Fastabend goto out; 9801ade9abaSJohn Fastabend } 9811ade9abaSJohn Fastabend } else { 9821ade9abaSJohn Fastabend i = 5; 9831ade9abaSJohn Fastabend bpf_map_update_elem(map_fd[5], 9841ade9abaSJohn Fastabend &i, &txmsg_pop, BPF_ANY); 9851ade9abaSJohn Fastabend 9861ade9abaSJohn Fastabend } 9871ade9abaSJohn Fastabend 98816962b24SJohn Fastabend if (txmsg_ingress) { 98916962b24SJohn Fastabend int in = BPF_F_INGRESS; 99016962b24SJohn Fastabend 99116962b24SJohn Fastabend i = 0; 99216962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 99316962b24SJohn Fastabend if (err) { 99416962b24SJohn Fastabend fprintf(stderr, 99516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 99616962b24SJohn Fastabend err, strerror(errno)); 99716962b24SJohn Fastabend } 99816962b24SJohn Fastabend i = 1; 99916962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 100016962b24SJohn Fastabend if (err) { 100116962b24SJohn Fastabend fprintf(stderr, 100216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 100316962b24SJohn Fastabend err, strerror(errno)); 100416962b24SJohn Fastabend } 100516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 100616962b24SJohn Fastabend if (err) { 100716962b24SJohn Fastabend fprintf(stderr, 100816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 100916962b24SJohn Fastabend err, strerror(errno)); 101016962b24SJohn Fastabend } 101116962b24SJohn Fastabend 101216962b24SJohn Fastabend i = 2; 101316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 101416962b24SJohn Fastabend if (err) { 101516962b24SJohn Fastabend fprintf(stderr, 101616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 101716962b24SJohn Fastabend err, strerror(errno)); 101816962b24SJohn Fastabend } 101916962b24SJohn Fastabend } 102016962b24SJohn Fastabend 102116962b24SJohn Fastabend if (txmsg_skb) { 102216962b24SJohn Fastabend int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 102316962b24SJohn Fastabend p2 : p1; 102416962b24SJohn Fastabend int ingress = BPF_F_INGRESS; 102516962b24SJohn Fastabend 102616962b24SJohn Fastabend i = 0; 102716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[7], 102816962b24SJohn Fastabend &i, &ingress, BPF_ANY); 102916962b24SJohn Fastabend if (err) { 103016962b24SJohn Fastabend fprintf(stderr, 103116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 103216962b24SJohn Fastabend err, strerror(errno)); 103316962b24SJohn Fastabend } 103416962b24SJohn Fastabend 103516962b24SJohn Fastabend i = 3; 103616962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[0], 103716962b24SJohn Fastabend &i, &skb_fd, BPF_ANY); 103816962b24SJohn Fastabend if (err) { 103916962b24SJohn Fastabend fprintf(stderr, 104016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 104116962b24SJohn Fastabend err, strerror(errno)); 104216962b24SJohn Fastabend } 104316962b24SJohn Fastabend } 104416962b24SJohn Fastabend } 104516962b24SJohn Fastabend 104616962b24SJohn Fastabend if (txmsg_drop) 104716962b24SJohn Fastabend options->drop_expected = true; 104816962b24SJohn Fastabend 104916962b24SJohn Fastabend if (test == PING_PONG) 105016962b24SJohn Fastabend err = forever_ping_pong(options->rate, options); 105116962b24SJohn Fastabend else if (test == SENDMSG) { 105216962b24SJohn Fastabend options->base = false; 105316962b24SJohn Fastabend options->sendpage = false; 105416962b24SJohn Fastabend err = sendmsg_test(options); 105516962b24SJohn Fastabend } else if (test == SENDPAGE) { 105616962b24SJohn Fastabend options->base = false; 105716962b24SJohn Fastabend options->sendpage = true; 105816962b24SJohn Fastabend err = sendmsg_test(options); 105916962b24SJohn Fastabend } else if (test == BASE) { 106016962b24SJohn Fastabend options->base = true; 106116962b24SJohn Fastabend options->sendpage = false; 106216962b24SJohn Fastabend err = sendmsg_test(options); 106316962b24SJohn Fastabend } else if (test == BASE_SENDPAGE) { 106416962b24SJohn Fastabend options->base = true; 106516962b24SJohn Fastabend options->sendpage = true; 106616962b24SJohn Fastabend err = sendmsg_test(options); 106716962b24SJohn Fastabend } else 106816962b24SJohn Fastabend fprintf(stderr, "unknown test\n"); 106916962b24SJohn Fastabend out: 107016962b24SJohn Fastabend /* Detatch and zero all the maps */ 107116962b24SJohn Fastabend bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 107216962b24SJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 107316962b24SJohn Fastabend bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 107416962b24SJohn Fastabend if (tx_prog_fd >= 0) 107516962b24SJohn Fastabend bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 107616962b24SJohn Fastabend 107716962b24SJohn Fastabend for (i = 0; i < 8; i++) { 107816962b24SJohn Fastabend key = next_key = 0; 107916962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 108016962b24SJohn Fastabend while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 108116962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 108216962b24SJohn Fastabend key = next_key; 108316962b24SJohn Fastabend } 108416962b24SJohn Fastabend } 108516962b24SJohn Fastabend 108616962b24SJohn Fastabend close(s1); 108716962b24SJohn Fastabend close(s2); 108816962b24SJohn Fastabend close(p1); 108916962b24SJohn Fastabend close(p2); 109016962b24SJohn Fastabend close(c1); 109116962b24SJohn Fastabend close(c2); 109216962b24SJohn Fastabend return err; 109316962b24SJohn Fastabend } 109416962b24SJohn Fastabend 109516962b24SJohn Fastabend static char *test_to_str(int test) 109616962b24SJohn Fastabend { 109716962b24SJohn Fastabend switch (test) { 109816962b24SJohn Fastabend case SENDMSG: 109916962b24SJohn Fastabend return "sendmsg"; 110016962b24SJohn Fastabend case SENDPAGE: 110116962b24SJohn Fastabend return "sendpage"; 110216962b24SJohn Fastabend } 110316962b24SJohn Fastabend return "unknown"; 110416962b24SJohn Fastabend } 110516962b24SJohn Fastabend 110616962b24SJohn Fastabend #define OPTSTRING 60 110716962b24SJohn Fastabend static void test_options(char *options) 110816962b24SJohn Fastabend { 110973563aa3SPrashant Bhole char tstr[OPTSTRING]; 111073563aa3SPrashant Bhole 111116962b24SJohn Fastabend memset(options, 0, OPTSTRING); 111216962b24SJohn Fastabend 111316962b24SJohn Fastabend if (txmsg_pass) 111416962b24SJohn Fastabend strncat(options, "pass,", OPTSTRING); 111516962b24SJohn Fastabend if (txmsg_redir) 111616962b24SJohn Fastabend strncat(options, "redir,", OPTSTRING); 111716962b24SJohn Fastabend if (txmsg_drop) 111816962b24SJohn Fastabend strncat(options, "drop,", OPTSTRING); 111973563aa3SPrashant Bhole if (txmsg_apply) { 112073563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 112173563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 112273563aa3SPrashant Bhole } 112373563aa3SPrashant Bhole if (txmsg_cork) { 112473563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 112573563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 112673563aa3SPrashant Bhole } 112773563aa3SPrashant Bhole if (txmsg_start) { 112873563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 112973563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 113073563aa3SPrashant Bhole } 113173563aa3SPrashant Bhole if (txmsg_end) { 113273563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 113373563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 113473563aa3SPrashant Bhole } 11351ade9abaSJohn Fastabend if (txmsg_start_pop) { 11361ade9abaSJohn Fastabend snprintf(tstr, OPTSTRING, "pop (%d,%d),", 11371ade9abaSJohn Fastabend txmsg_start_pop, txmsg_start_pop + txmsg_pop); 11381ade9abaSJohn Fastabend strncat(options, tstr, OPTSTRING); 11391ade9abaSJohn Fastabend } 114016962b24SJohn Fastabend if (txmsg_ingress) 114116962b24SJohn Fastabend strncat(options, "ingress,", OPTSTRING); 114216962b24SJohn Fastabend if (txmsg_skb) 114316962b24SJohn Fastabend strncat(options, "skb,", OPTSTRING); 1144e9dd9047SJohn Fastabend if (ktls) 1145e9dd9047SJohn Fastabend strncat(options, "ktls,", OPTSTRING); 1146753fb2eeSJohn Fastabend if (peek_flag) 1147753fb2eeSJohn Fastabend strncat(options, "peek,", OPTSTRING); 114816962b24SJohn Fastabend } 114916962b24SJohn Fastabend 115016962b24SJohn Fastabend static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 115116962b24SJohn Fastabend { 115273563aa3SPrashant Bhole char *options = calloc(OPTSTRING, sizeof(char)); 115316962b24SJohn Fastabend int err; 115416962b24SJohn Fastabend 115516962b24SJohn Fastabend if (test == SENDPAGE) 115616962b24SJohn Fastabend opt->sendpage = true; 115716962b24SJohn Fastabend else 115816962b24SJohn Fastabend opt->sendpage = false; 115916962b24SJohn Fastabend 116016962b24SJohn Fastabend if (txmsg_drop) 116116962b24SJohn Fastabend opt->drop_expected = true; 116216962b24SJohn Fastabend else 116316962b24SJohn Fastabend opt->drop_expected = false; 116416962b24SJohn Fastabend 116516962b24SJohn Fastabend test_options(options); 116616962b24SJohn Fastabend 116716962b24SJohn Fastabend fprintf(stdout, 116816962b24SJohn Fastabend "[TEST %i]: (%i, %i, %i, %s, %s): ", 116916962b24SJohn Fastabend test_cnt, opt->rate, opt->iov_count, opt->iov_length, 117016962b24SJohn Fastabend test_to_str(test), options); 117116962b24SJohn Fastabend fflush(stdout); 117216962b24SJohn Fastabend err = run_options(opt, cgrp, test); 117316962b24SJohn Fastabend fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 117416962b24SJohn Fastabend test_cnt++; 117516962b24SJohn Fastabend !err ? passed++ : failed++; 117616962b24SJohn Fastabend free(options); 117716962b24SJohn Fastabend return err; 117816962b24SJohn Fastabend } 117916962b24SJohn Fastabend 118016962b24SJohn Fastabend static int test_exec(int cgrp, struct sockmap_options *opt) 118116962b24SJohn Fastabend { 118216962b24SJohn Fastabend int err = __test_exec(cgrp, SENDMSG, opt); 118316962b24SJohn Fastabend 118416962b24SJohn Fastabend if (err) 118516962b24SJohn Fastabend goto out; 118616962b24SJohn Fastabend 118716962b24SJohn Fastabend err = __test_exec(cgrp, SENDPAGE, opt); 118816962b24SJohn Fastabend out: 118916962b24SJohn Fastabend return err; 119016962b24SJohn Fastabend } 119116962b24SJohn Fastabend 119216962b24SJohn Fastabend static int test_loop(int cgrp) 119316962b24SJohn Fastabend { 119416962b24SJohn Fastabend struct sockmap_options opt; 119516962b24SJohn Fastabend 119616962b24SJohn Fastabend int err, i, l, r; 119716962b24SJohn Fastabend 119816962b24SJohn Fastabend opt.verbose = 0; 119916962b24SJohn Fastabend opt.base = false; 120016962b24SJohn Fastabend opt.sendpage = false; 120116962b24SJohn Fastabend opt.data_test = false; 120216962b24SJohn Fastabend opt.drop_expected = false; 120316962b24SJohn Fastabend opt.iov_count = 0; 120416962b24SJohn Fastabend opt.iov_length = 0; 120516962b24SJohn Fastabend opt.rate = 0; 120616962b24SJohn Fastabend 1207a18fda1aSJohn Fastabend r = 1; 120816962b24SJohn Fastabend for (i = 1; i < 100; i += 33) { 120916962b24SJohn Fastabend for (l = 1; l < 100; l += 33) { 121016962b24SJohn Fastabend opt.rate = r; 121116962b24SJohn Fastabend opt.iov_count = i; 121216962b24SJohn Fastabend opt.iov_length = l; 121316962b24SJohn Fastabend err = test_exec(cgrp, &opt); 121416962b24SJohn Fastabend if (err) 121516962b24SJohn Fastabend goto out; 121616962b24SJohn Fastabend } 121716962b24SJohn Fastabend } 1218a18fda1aSJohn Fastabend sched_yield(); 121916962b24SJohn Fastabend out: 122016962b24SJohn Fastabend return err; 122116962b24SJohn Fastabend } 122216962b24SJohn Fastabend 122316962b24SJohn Fastabend static int test_txmsg(int cgrp) 122416962b24SJohn Fastabend { 122516962b24SJohn Fastabend int err; 122616962b24SJohn Fastabend 1227d79a3212SJohn Fastabend txmsg_pass = txmsg_drop = 0; 122816962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 122916962b24SJohn Fastabend txmsg_ingress = txmsg_skb = 0; 123016962b24SJohn Fastabend 123116962b24SJohn Fastabend txmsg_pass = 1; 123216962b24SJohn Fastabend err = test_loop(cgrp); 123316962b24SJohn Fastabend txmsg_pass = 0; 123416962b24SJohn Fastabend if (err) 123516962b24SJohn Fastabend goto out; 123616962b24SJohn Fastabend 123716962b24SJohn Fastabend txmsg_redir = 1; 123816962b24SJohn Fastabend err = test_loop(cgrp); 123916962b24SJohn Fastabend txmsg_redir = 0; 124016962b24SJohn Fastabend if (err) 124116962b24SJohn Fastabend goto out; 124216962b24SJohn Fastabend 124316962b24SJohn Fastabend txmsg_drop = 1; 124416962b24SJohn Fastabend err = test_loop(cgrp); 124516962b24SJohn Fastabend txmsg_drop = 0; 124616962b24SJohn Fastabend if (err) 124716962b24SJohn Fastabend goto out; 124816962b24SJohn Fastabend 124916962b24SJohn Fastabend txmsg_redir = 1; 125016962b24SJohn Fastabend txmsg_ingress = 1; 125116962b24SJohn Fastabend err = test_loop(cgrp); 125216962b24SJohn Fastabend txmsg_redir = 0; 125316962b24SJohn Fastabend txmsg_ingress = 0; 125416962b24SJohn Fastabend if (err) 125516962b24SJohn Fastabend goto out; 125616962b24SJohn Fastabend out: 125716962b24SJohn Fastabend txmsg_pass = 0; 125816962b24SJohn Fastabend txmsg_redir = 0; 125916962b24SJohn Fastabend txmsg_drop = 0; 126016962b24SJohn Fastabend return err; 126116962b24SJohn Fastabend } 126216962b24SJohn Fastabend 126316962b24SJohn Fastabend static int test_send(struct sockmap_options *opt, int cgrp) 126416962b24SJohn Fastabend { 126516962b24SJohn Fastabend int err; 126616962b24SJohn Fastabend 126716962b24SJohn Fastabend opt->iov_length = 1; 126816962b24SJohn Fastabend opt->iov_count = 1; 126916962b24SJohn Fastabend opt->rate = 1; 127016962b24SJohn Fastabend err = test_exec(cgrp, opt); 127116962b24SJohn Fastabend if (err) 127216962b24SJohn Fastabend goto out; 127316962b24SJohn Fastabend 127416962b24SJohn Fastabend opt->iov_length = 1; 127516962b24SJohn Fastabend opt->iov_count = 1024; 127616962b24SJohn Fastabend opt->rate = 1; 127716962b24SJohn Fastabend err = test_exec(cgrp, opt); 127816962b24SJohn Fastabend if (err) 127916962b24SJohn Fastabend goto out; 128016962b24SJohn Fastabend 128116962b24SJohn Fastabend opt->iov_length = 1024; 128216962b24SJohn Fastabend opt->iov_count = 1; 128316962b24SJohn Fastabend opt->rate = 1; 128416962b24SJohn Fastabend err = test_exec(cgrp, opt); 128516962b24SJohn Fastabend if (err) 128616962b24SJohn Fastabend goto out; 128716962b24SJohn Fastabend 128816962b24SJohn Fastabend opt->iov_length = 1; 128916962b24SJohn Fastabend opt->iov_count = 1; 1290a009f1f3SPrashant Bhole opt->rate = 512; 129116962b24SJohn Fastabend err = test_exec(cgrp, opt); 129216962b24SJohn Fastabend if (err) 129316962b24SJohn Fastabend goto out; 129416962b24SJohn Fastabend 129516962b24SJohn Fastabend opt->iov_length = 256; 129616962b24SJohn Fastabend opt->iov_count = 1024; 1297a009f1f3SPrashant Bhole opt->rate = 2; 129816962b24SJohn Fastabend err = test_exec(cgrp, opt); 129916962b24SJohn Fastabend if (err) 130016962b24SJohn Fastabend goto out; 130116962b24SJohn Fastabend 130216962b24SJohn Fastabend opt->rate = 100; 130316962b24SJohn Fastabend opt->iov_count = 1; 130416962b24SJohn Fastabend opt->iov_length = 5; 130516962b24SJohn Fastabend err = test_exec(cgrp, opt); 130616962b24SJohn Fastabend if (err) 130716962b24SJohn Fastabend goto out; 130816962b24SJohn Fastabend out: 1309a18fda1aSJohn Fastabend sched_yield(); 131016962b24SJohn Fastabend return err; 131116962b24SJohn Fastabend } 131216962b24SJohn Fastabend 131316962b24SJohn Fastabend static int test_mixed(int cgrp) 131416962b24SJohn Fastabend { 131516962b24SJohn Fastabend struct sockmap_options opt = {0}; 131616962b24SJohn Fastabend int err; 131716962b24SJohn Fastabend 1318d79a3212SJohn Fastabend txmsg_pass = txmsg_drop = 0; 131916962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 132016962b24SJohn Fastabend txmsg_start = txmsg_end = 0; 132184fbfe02SJohn Fastabend txmsg_start_push = txmsg_end_push = 0; 13221ade9abaSJohn Fastabend txmsg_start_pop = txmsg_pop = 0; 132384fbfe02SJohn Fastabend 132416962b24SJohn Fastabend /* Test small and large iov_count values with pass/redir/apply/cork */ 132516962b24SJohn Fastabend txmsg_pass = 1; 132616962b24SJohn Fastabend txmsg_redir = 0; 132716962b24SJohn Fastabend txmsg_apply = 1; 132816962b24SJohn Fastabend txmsg_cork = 0; 132916962b24SJohn Fastabend err = test_send(&opt, cgrp); 133016962b24SJohn Fastabend if (err) 133116962b24SJohn Fastabend goto out; 133216962b24SJohn Fastabend 133316962b24SJohn Fastabend txmsg_pass = 1; 133416962b24SJohn Fastabend txmsg_redir = 0; 133516962b24SJohn Fastabend txmsg_apply = 0; 133616962b24SJohn Fastabend txmsg_cork = 1; 133716962b24SJohn Fastabend err = test_send(&opt, cgrp); 133816962b24SJohn Fastabend if (err) 133916962b24SJohn Fastabend goto out; 134016962b24SJohn Fastabend 134116962b24SJohn Fastabend txmsg_pass = 1; 134216962b24SJohn Fastabend txmsg_redir = 0; 134316962b24SJohn Fastabend txmsg_apply = 1; 134416962b24SJohn Fastabend txmsg_cork = 1; 134516962b24SJohn Fastabend err = test_send(&opt, cgrp); 134616962b24SJohn Fastabend if (err) 134716962b24SJohn Fastabend goto out; 134816962b24SJohn Fastabend 134916962b24SJohn Fastabend txmsg_pass = 1; 135016962b24SJohn Fastabend txmsg_redir = 0; 135116962b24SJohn Fastabend txmsg_apply = 1024; 135216962b24SJohn Fastabend txmsg_cork = 0; 135316962b24SJohn Fastabend err = test_send(&opt, cgrp); 135416962b24SJohn Fastabend if (err) 135516962b24SJohn Fastabend goto out; 135616962b24SJohn Fastabend 135716962b24SJohn Fastabend txmsg_pass = 1; 135816962b24SJohn Fastabend txmsg_redir = 0; 135916962b24SJohn Fastabend txmsg_apply = 0; 136016962b24SJohn Fastabend txmsg_cork = 1024; 136116962b24SJohn Fastabend err = test_send(&opt, cgrp); 136216962b24SJohn Fastabend if (err) 136316962b24SJohn Fastabend goto out; 136416962b24SJohn Fastabend 136516962b24SJohn Fastabend txmsg_pass = 1; 136616962b24SJohn Fastabend txmsg_redir = 0; 136716962b24SJohn Fastabend txmsg_apply = 1024; 136816962b24SJohn Fastabend txmsg_cork = 1024; 136916962b24SJohn Fastabend err = test_send(&opt, cgrp); 137016962b24SJohn Fastabend if (err) 137116962b24SJohn Fastabend goto out; 137216962b24SJohn Fastabend 137316962b24SJohn Fastabend txmsg_pass = 1; 137416962b24SJohn Fastabend txmsg_redir = 0; 137516962b24SJohn Fastabend txmsg_cork = 4096; 137616962b24SJohn Fastabend txmsg_apply = 4096; 137716962b24SJohn Fastabend err = test_send(&opt, cgrp); 137816962b24SJohn Fastabend if (err) 137916962b24SJohn Fastabend goto out; 138016962b24SJohn Fastabend 138116962b24SJohn Fastabend txmsg_pass = 0; 138216962b24SJohn Fastabend txmsg_redir = 1; 138316962b24SJohn Fastabend txmsg_apply = 1; 138416962b24SJohn Fastabend txmsg_cork = 0; 138516962b24SJohn Fastabend err = test_send(&opt, cgrp); 138616962b24SJohn Fastabend if (err) 138716962b24SJohn Fastabend goto out; 138816962b24SJohn Fastabend 138916962b24SJohn Fastabend txmsg_pass = 0; 139016962b24SJohn Fastabend txmsg_redir = 1; 139116962b24SJohn Fastabend txmsg_apply = 0; 139216962b24SJohn Fastabend txmsg_cork = 1; 139316962b24SJohn Fastabend err = test_send(&opt, cgrp); 139416962b24SJohn Fastabend if (err) 139516962b24SJohn Fastabend goto out; 139616962b24SJohn Fastabend 139716962b24SJohn Fastabend txmsg_pass = 0; 139816962b24SJohn Fastabend txmsg_redir = 1; 139916962b24SJohn Fastabend txmsg_apply = 1024; 140016962b24SJohn Fastabend txmsg_cork = 0; 140116962b24SJohn Fastabend err = test_send(&opt, cgrp); 140216962b24SJohn Fastabend if (err) 140316962b24SJohn Fastabend goto out; 140416962b24SJohn Fastabend 140516962b24SJohn Fastabend txmsg_pass = 0; 140616962b24SJohn Fastabend txmsg_redir = 1; 140716962b24SJohn Fastabend txmsg_apply = 0; 140816962b24SJohn Fastabend txmsg_cork = 1024; 140916962b24SJohn Fastabend err = test_send(&opt, cgrp); 141016962b24SJohn Fastabend if (err) 141116962b24SJohn Fastabend goto out; 141216962b24SJohn Fastabend 141316962b24SJohn Fastabend txmsg_pass = 0; 141416962b24SJohn Fastabend txmsg_redir = 1; 141516962b24SJohn Fastabend txmsg_apply = 1024; 141616962b24SJohn Fastabend txmsg_cork = 1024; 141716962b24SJohn Fastabend err = test_send(&opt, cgrp); 141816962b24SJohn Fastabend if (err) 141916962b24SJohn Fastabend goto out; 142016962b24SJohn Fastabend 142116962b24SJohn Fastabend txmsg_pass = 0; 142216962b24SJohn Fastabend txmsg_redir = 1; 142316962b24SJohn Fastabend txmsg_cork = 4096; 142416962b24SJohn Fastabend txmsg_apply = 4096; 142516962b24SJohn Fastabend err = test_send(&opt, cgrp); 142616962b24SJohn Fastabend if (err) 142716962b24SJohn Fastabend goto out; 142816962b24SJohn Fastabend out: 142916962b24SJohn Fastabend return err; 143016962b24SJohn Fastabend } 143116962b24SJohn Fastabend 143216962b24SJohn Fastabend static int test_start_end(int cgrp) 143316962b24SJohn Fastabend { 143416962b24SJohn Fastabend struct sockmap_options opt = {0}; 143516962b24SJohn Fastabend int err, i; 143616962b24SJohn Fastabend 143716962b24SJohn Fastabend /* Test basic start/end with lots of iov_count and iov_lengths */ 143816962b24SJohn Fastabend txmsg_start = 1; 143916962b24SJohn Fastabend txmsg_end = 2; 144084fbfe02SJohn Fastabend txmsg_start_push = 1; 144184fbfe02SJohn Fastabend txmsg_end_push = 2; 14421ade9abaSJohn Fastabend txmsg_start_pop = 1; 14431ade9abaSJohn Fastabend txmsg_pop = 1; 14441ade9abaSJohn Fastabend err = test_txmsg(cgrp); 14451ade9abaSJohn Fastabend if (err) 14461ade9abaSJohn Fastabend goto out; 14471ade9abaSJohn Fastabend 14481ade9abaSJohn Fastabend /* Cut a byte of pushed data but leave reamining in place */ 14491ade9abaSJohn Fastabend txmsg_start = 1; 14501ade9abaSJohn Fastabend txmsg_end = 2; 14511ade9abaSJohn Fastabend txmsg_start_push = 1; 14521ade9abaSJohn Fastabend txmsg_end_push = 3; 14531ade9abaSJohn Fastabend txmsg_start_pop = 1; 14541ade9abaSJohn Fastabend txmsg_pop = 1; 145516962b24SJohn Fastabend err = test_txmsg(cgrp); 145616962b24SJohn Fastabend if (err) 145716962b24SJohn Fastabend goto out; 145816962b24SJohn Fastabend 145916962b24SJohn Fastabend /* Test start/end with cork */ 146016962b24SJohn Fastabend opt.rate = 16; 146116962b24SJohn Fastabend opt.iov_count = 1; 146216962b24SJohn Fastabend opt.iov_length = 100; 146316962b24SJohn Fastabend txmsg_cork = 1600; 146416962b24SJohn Fastabend 14651ade9abaSJohn Fastabend txmsg_start_pop = 0; 14661ade9abaSJohn Fastabend txmsg_pop = 0; 14671ade9abaSJohn Fastabend 1468a18fda1aSJohn Fastabend for (i = 99; i <= 1600; i += 500) { 146916962b24SJohn Fastabend txmsg_start = 0; 147016962b24SJohn Fastabend txmsg_end = i; 147184fbfe02SJohn Fastabend txmsg_start_push = 0; 147284fbfe02SJohn Fastabend txmsg_end_push = i; 147316962b24SJohn Fastabend err = test_exec(cgrp, &opt); 147416962b24SJohn Fastabend if (err) 147516962b24SJohn Fastabend goto out; 147616962b24SJohn Fastabend } 147716962b24SJohn Fastabend 14781ade9abaSJohn Fastabend /* Test pop data in middle of cork */ 14791ade9abaSJohn Fastabend for (i = 99; i <= 1600; i += 500) { 14801ade9abaSJohn Fastabend txmsg_start_pop = 10; 14811ade9abaSJohn Fastabend txmsg_pop = i; 14821ade9abaSJohn Fastabend err = test_exec(cgrp, &opt); 14831ade9abaSJohn Fastabend if (err) 14841ade9abaSJohn Fastabend goto out; 14851ade9abaSJohn Fastabend } 14861ade9abaSJohn Fastabend txmsg_start_pop = 0; 14871ade9abaSJohn Fastabend txmsg_pop = 0; 14881ade9abaSJohn Fastabend 148916962b24SJohn Fastabend /* Test start/end with cork but pull data in middle */ 1490a18fda1aSJohn Fastabend for (i = 199; i <= 1600; i += 500) { 149116962b24SJohn Fastabend txmsg_start = 100; 149216962b24SJohn Fastabend txmsg_end = i; 149384fbfe02SJohn Fastabend txmsg_start_push = 100; 149484fbfe02SJohn Fastabend txmsg_end_push = i; 149516962b24SJohn Fastabend err = test_exec(cgrp, &opt); 149616962b24SJohn Fastabend if (err) 149716962b24SJohn Fastabend goto out; 149816962b24SJohn Fastabend } 149916962b24SJohn Fastabend 150016962b24SJohn Fastabend /* Test start/end with cork pulling last sg entry */ 150116962b24SJohn Fastabend txmsg_start = 1500; 150216962b24SJohn Fastabend txmsg_end = 1600; 150384fbfe02SJohn Fastabend txmsg_start_push = 1500; 150484fbfe02SJohn Fastabend txmsg_end_push = 1600; 150516962b24SJohn Fastabend err = test_exec(cgrp, &opt); 150616962b24SJohn Fastabend if (err) 150716962b24SJohn Fastabend goto out; 150816962b24SJohn Fastabend 15091ade9abaSJohn Fastabend /* Test pop with cork pulling last sg entry */ 15101ade9abaSJohn Fastabend txmsg_start_pop = 1500; 15111ade9abaSJohn Fastabend txmsg_pop = 1600; 15121ade9abaSJohn Fastabend err = test_exec(cgrp, &opt); 15131ade9abaSJohn Fastabend if (err) 15141ade9abaSJohn Fastabend goto out; 15151ade9abaSJohn Fastabend txmsg_start_pop = 0; 15161ade9abaSJohn Fastabend txmsg_pop = 0; 15171ade9abaSJohn Fastabend 151816962b24SJohn Fastabend /* Test start/end pull of single byte in last page */ 151916962b24SJohn Fastabend txmsg_start = 1111; 152016962b24SJohn Fastabend txmsg_end = 1112; 152184fbfe02SJohn Fastabend txmsg_start_push = 1111; 152284fbfe02SJohn Fastabend txmsg_end_push = 1112; 152316962b24SJohn Fastabend err = test_exec(cgrp, &opt); 152416962b24SJohn Fastabend if (err) 152516962b24SJohn Fastabend goto out; 152616962b24SJohn Fastabend 15271ade9abaSJohn Fastabend /* Test pop of single byte in last page */ 15281ade9abaSJohn Fastabend txmsg_start_pop = 1111; 15291ade9abaSJohn Fastabend txmsg_pop = 1112; 15301ade9abaSJohn Fastabend err = test_exec(cgrp, &opt); 15311ade9abaSJohn Fastabend if (err) 15321ade9abaSJohn Fastabend goto out; 15331ade9abaSJohn Fastabend 153416962b24SJohn Fastabend /* Test start/end with end < start */ 153516962b24SJohn Fastabend txmsg_start = 1111; 153616962b24SJohn Fastabend txmsg_end = 0; 153784fbfe02SJohn Fastabend txmsg_start_push = 1111; 153884fbfe02SJohn Fastabend txmsg_end_push = 0; 153916962b24SJohn Fastabend err = test_exec(cgrp, &opt); 154016962b24SJohn Fastabend if (err) 154116962b24SJohn Fastabend goto out; 154216962b24SJohn Fastabend 154316962b24SJohn Fastabend /* Test start/end with end > data */ 154416962b24SJohn Fastabend txmsg_start = 0; 154516962b24SJohn Fastabend txmsg_end = 1601; 154684fbfe02SJohn Fastabend txmsg_start_push = 0; 154784fbfe02SJohn Fastabend txmsg_end_push = 1601; 154816962b24SJohn Fastabend err = test_exec(cgrp, &opt); 154916962b24SJohn Fastabend if (err) 155016962b24SJohn Fastabend goto out; 155116962b24SJohn Fastabend 155216962b24SJohn Fastabend /* Test start/end with start > data */ 155316962b24SJohn Fastabend txmsg_start = 1601; 155416962b24SJohn Fastabend txmsg_end = 1600; 155584fbfe02SJohn Fastabend txmsg_start_push = 1601; 155684fbfe02SJohn Fastabend txmsg_end_push = 1600; 155716962b24SJohn Fastabend err = test_exec(cgrp, &opt); 15581ade9abaSJohn Fastabend if (err) 15591ade9abaSJohn Fastabend goto out; 156016962b24SJohn Fastabend 15611ade9abaSJohn Fastabend /* Test pop with start > data */ 15621ade9abaSJohn Fastabend txmsg_start_pop = 1601; 15631ade9abaSJohn Fastabend txmsg_pop = 1; 15641ade9abaSJohn Fastabend err = test_exec(cgrp, &opt); 15651ade9abaSJohn Fastabend if (err) 15661ade9abaSJohn Fastabend goto out; 15671ade9abaSJohn Fastabend 15681ade9abaSJohn Fastabend /* Test pop with pop range > data */ 15691ade9abaSJohn Fastabend txmsg_start_pop = 1599; 15701ade9abaSJohn Fastabend txmsg_pop = 10; 15711ade9abaSJohn Fastabend err = test_exec(cgrp, &opt); 157216962b24SJohn Fastabend out: 157316962b24SJohn Fastabend txmsg_start = 0; 157416962b24SJohn Fastabend txmsg_end = 0; 1575a18fda1aSJohn Fastabend sched_yield(); 157616962b24SJohn Fastabend return err; 157716962b24SJohn Fastabend } 157816962b24SJohn Fastabend 157916962b24SJohn Fastabend char *map_names[] = { 158016962b24SJohn Fastabend "sock_map", 158116962b24SJohn Fastabend "sock_map_txmsg", 158216962b24SJohn Fastabend "sock_map_redir", 158316962b24SJohn Fastabend "sock_apply_bytes", 158416962b24SJohn Fastabend "sock_cork_bytes", 158584fbfe02SJohn Fastabend "sock_bytes", 158616962b24SJohn Fastabend "sock_redir_flags", 158716962b24SJohn Fastabend "sock_skb_opts", 158816962b24SJohn Fastabend }; 158916962b24SJohn Fastabend 159016962b24SJohn Fastabend int prog_attach_type[] = { 159116962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 159216962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 159316962b24SJohn Fastabend BPF_CGROUP_SOCK_OPS, 159416962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 159516962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 159616962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 159716962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 159816962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 159916962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 160016962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 160116962b24SJohn Fastabend }; 160216962b24SJohn Fastabend 160316962b24SJohn Fastabend int prog_type[] = { 160416962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 160516962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 160616962b24SJohn Fastabend BPF_PROG_TYPE_SOCK_OPS, 160716962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 160816962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 160916962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 161016962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 161116962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 161216962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 161316962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 161416962b24SJohn Fastabend }; 161516962b24SJohn Fastabend 1616b8b394faSJohn Fastabend static int populate_progs(char *bpf_file) 161716962b24SJohn Fastabend { 161816962b24SJohn Fastabend struct bpf_program *prog; 161916962b24SJohn Fastabend struct bpf_object *obj; 162016962b24SJohn Fastabend int i = 0; 162116962b24SJohn Fastabend long err; 162216962b24SJohn Fastabend 162316962b24SJohn Fastabend obj = bpf_object__open(bpf_file); 162416962b24SJohn Fastabend err = libbpf_get_error(obj); 162516962b24SJohn Fastabend if (err) { 162616962b24SJohn Fastabend char err_buf[256]; 162716962b24SJohn Fastabend 162816962b24SJohn Fastabend libbpf_strerror(err, err_buf, sizeof(err_buf)); 162916962b24SJohn Fastabend printf("Unable to load eBPF objects in file '%s' : %s\n", 163016962b24SJohn Fastabend bpf_file, err_buf); 163116962b24SJohn Fastabend return -1; 163216962b24SJohn Fastabend } 163316962b24SJohn Fastabend 163416962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 163516962b24SJohn Fastabend bpf_program__set_type(prog, prog_type[i]); 163616962b24SJohn Fastabend bpf_program__set_expected_attach_type(prog, 163716962b24SJohn Fastabend prog_attach_type[i]); 163816962b24SJohn Fastabend i++; 163916962b24SJohn Fastabend } 164016962b24SJohn Fastabend 164116962b24SJohn Fastabend i = bpf_object__load(obj); 164216962b24SJohn Fastabend i = 0; 164316962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 164416962b24SJohn Fastabend prog_fd[i] = bpf_program__fd(prog); 164516962b24SJohn Fastabend i++; 164616962b24SJohn Fastabend } 164716962b24SJohn Fastabend 164816962b24SJohn Fastabend for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 164916962b24SJohn Fastabend maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 165016962b24SJohn Fastabend map_fd[i] = bpf_map__fd(maps[i]); 165116962b24SJohn Fastabend if (map_fd[i] < 0) { 165216962b24SJohn Fastabend fprintf(stderr, "load_bpf_file: (%i) %s\n", 165316962b24SJohn Fastabend map_fd[i], strerror(errno)); 165416962b24SJohn Fastabend return -1; 165516962b24SJohn Fastabend } 165616962b24SJohn Fastabend } 165716962b24SJohn Fastabend 165816962b24SJohn Fastabend return 0; 165916962b24SJohn Fastabend } 166016962b24SJohn Fastabend 16617d2c6cfcSJohn Fastabend static int __test_suite(int cg_fd, char *bpf_file) 166216962b24SJohn Fastabend { 16637d2c6cfcSJohn Fastabend int err, cleanup = cg_fd; 166416962b24SJohn Fastabend 1665b8b394faSJohn Fastabend err = populate_progs(bpf_file); 166616962b24SJohn Fastabend if (err < 0) { 166716962b24SJohn Fastabend fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 166816962b24SJohn Fastabend return err; 166916962b24SJohn Fastabend } 167016962b24SJohn Fastabend 16717d2c6cfcSJohn Fastabend if (cg_fd < 0) { 167216962b24SJohn Fastabend if (setup_cgroup_environment()) { 167316962b24SJohn Fastabend fprintf(stderr, "ERROR: cgroup env failed\n"); 167416962b24SJohn Fastabend return -EINVAL; 167516962b24SJohn Fastabend } 167616962b24SJohn Fastabend 167716962b24SJohn Fastabend cg_fd = create_and_get_cgroup(CG_PATH); 167816962b24SJohn Fastabend if (cg_fd < 0) { 167916962b24SJohn Fastabend fprintf(stderr, 168016962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 168116962b24SJohn Fastabend cg_fd, optarg); 168216962b24SJohn Fastabend return cg_fd; 168316962b24SJohn Fastabend } 168416962b24SJohn Fastabend 1685035b37ffSPrashant Bhole if (join_cgroup(CG_PATH)) { 1686035b37ffSPrashant Bhole fprintf(stderr, "ERROR: failed to join cgroup\n"); 1687035b37ffSPrashant Bhole return -EINVAL; 1688035b37ffSPrashant Bhole } 16897d2c6cfcSJohn Fastabend } 1690035b37ffSPrashant Bhole 169116962b24SJohn Fastabend /* Tests basic commands and APIs with range of iov values */ 169284fbfe02SJohn Fastabend txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0; 169316962b24SJohn Fastabend err = test_txmsg(cg_fd); 169416962b24SJohn Fastabend if (err) 169516962b24SJohn Fastabend goto out; 169616962b24SJohn Fastabend 169716962b24SJohn Fastabend /* Tests interesting combinations of APIs used together */ 169816962b24SJohn Fastabend err = test_mixed(cg_fd); 169916962b24SJohn Fastabend if (err) 170016962b24SJohn Fastabend goto out; 170116962b24SJohn Fastabend 170216962b24SJohn Fastabend /* Tests pull_data API using start/end API */ 170316962b24SJohn Fastabend err = test_start_end(cg_fd); 170416962b24SJohn Fastabend if (err) 170516962b24SJohn Fastabend goto out; 170616962b24SJohn Fastabend 170716962b24SJohn Fastabend out: 170816962b24SJohn Fastabend printf("Summary: %i PASSED %i FAILED\n", passed, failed); 17097d2c6cfcSJohn Fastabend if (cleanup < 0) { 1710b8b394faSJohn Fastabend cleanup_cgroup_environment(); 171116962b24SJohn Fastabend close(cg_fd); 17127d2c6cfcSJohn Fastabend } 171316962b24SJohn Fastabend return err; 171416962b24SJohn Fastabend } 171516962b24SJohn Fastabend 17167d2c6cfcSJohn Fastabend static int test_suite(int cg_fd) 1717b8b394faSJohn Fastabend { 1718b8b394faSJohn Fastabend int err; 1719b8b394faSJohn Fastabend 17207d2c6cfcSJohn Fastabend err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME); 1721b8b394faSJohn Fastabend if (err) 1722b8b394faSJohn Fastabend goto out; 17237d2c6cfcSJohn Fastabend err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME); 1724b8b394faSJohn Fastabend out: 17257d2c6cfcSJohn Fastabend if (cg_fd > -1) 17267d2c6cfcSJohn Fastabend close(cg_fd); 1727b8b394faSJohn Fastabend return err; 1728b8b394faSJohn Fastabend } 1729b8b394faSJohn Fastabend 173016962b24SJohn Fastabend int main(int argc, char **argv) 173116962b24SJohn Fastabend { 173216962b24SJohn Fastabend int iov_count = 1, length = 1024, rate = 1; 173316962b24SJohn Fastabend struct sockmap_options options = {0}; 173416962b24SJohn Fastabend int opt, longindex, err, cg_fd = 0; 1735b8b394faSJohn Fastabend char *bpf_file = BPF_SOCKMAP_FILENAME; 173616962b24SJohn Fastabend int test = PING_PONG; 173713a5f3ffSJohn Fastabend bool cg_created = 0; 173816962b24SJohn Fastabend 173916962b24SJohn Fastabend if (argc < 2) 17407d2c6cfcSJohn Fastabend return test_suite(-1); 174116962b24SJohn Fastabend 174284fbfe02SJohn Fastabend while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:", 174316962b24SJohn Fastabend long_options, &longindex)) != -1) { 174416962b24SJohn Fastabend switch (opt) { 174516962b24SJohn Fastabend case 's': 174616962b24SJohn Fastabend txmsg_start = atoi(optarg); 174716962b24SJohn Fastabend break; 174816962b24SJohn Fastabend case 'e': 174916962b24SJohn Fastabend txmsg_end = atoi(optarg); 175016962b24SJohn Fastabend break; 175184fbfe02SJohn Fastabend case 'p': 175284fbfe02SJohn Fastabend txmsg_start_push = atoi(optarg); 175384fbfe02SJohn Fastabend break; 175484fbfe02SJohn Fastabend case 'q': 175584fbfe02SJohn Fastabend txmsg_end_push = atoi(optarg); 175684fbfe02SJohn Fastabend break; 17571ade9abaSJohn Fastabend case 'w': 17581ade9abaSJohn Fastabend txmsg_start_pop = atoi(optarg); 17591ade9abaSJohn Fastabend break; 17601ade9abaSJohn Fastabend case 'x': 17611ade9abaSJohn Fastabend txmsg_pop = atoi(optarg); 17621ade9abaSJohn Fastabend break; 176316962b24SJohn Fastabend case 'a': 176416962b24SJohn Fastabend txmsg_apply = atoi(optarg); 176516962b24SJohn Fastabend break; 176616962b24SJohn Fastabend case 'k': 176716962b24SJohn Fastabend txmsg_cork = atoi(optarg); 176816962b24SJohn Fastabend break; 176916962b24SJohn Fastabend case 'c': 177016962b24SJohn Fastabend cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 177116962b24SJohn Fastabend if (cg_fd < 0) { 177216962b24SJohn Fastabend fprintf(stderr, 177316962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 177416962b24SJohn Fastabend cg_fd, optarg); 177516962b24SJohn Fastabend return cg_fd; 177616962b24SJohn Fastabend } 177716962b24SJohn Fastabend break; 177816962b24SJohn Fastabend case 'r': 177916962b24SJohn Fastabend rate = atoi(optarg); 178016962b24SJohn Fastabend break; 178116962b24SJohn Fastabend case 'v': 178216962b24SJohn Fastabend options.verbose = 1; 178316962b24SJohn Fastabend break; 178416962b24SJohn Fastabend case 'i': 178516962b24SJohn Fastabend iov_count = atoi(optarg); 178616962b24SJohn Fastabend break; 178716962b24SJohn Fastabend case 'l': 178816962b24SJohn Fastabend length = atoi(optarg); 178916962b24SJohn Fastabend break; 179016962b24SJohn Fastabend case 'd': 179116962b24SJohn Fastabend options.data_test = true; 179216962b24SJohn Fastabend break; 179316962b24SJohn Fastabend case 't': 179416962b24SJohn Fastabend if (strcmp(optarg, "ping") == 0) { 179516962b24SJohn Fastabend test = PING_PONG; 179616962b24SJohn Fastabend } else if (strcmp(optarg, "sendmsg") == 0) { 179716962b24SJohn Fastabend test = SENDMSG; 179816962b24SJohn Fastabend } else if (strcmp(optarg, "base") == 0) { 179916962b24SJohn Fastabend test = BASE; 180016962b24SJohn Fastabend } else if (strcmp(optarg, "base_sendpage") == 0) { 180116962b24SJohn Fastabend test = BASE_SENDPAGE; 180216962b24SJohn Fastabend } else if (strcmp(optarg, "sendpage") == 0) { 180316962b24SJohn Fastabend test = SENDPAGE; 180416962b24SJohn Fastabend } else { 180516962b24SJohn Fastabend usage(argv); 180616962b24SJohn Fastabend return -1; 180716962b24SJohn Fastabend } 180816962b24SJohn Fastabend break; 180916962b24SJohn Fastabend case 0: 181016962b24SJohn Fastabend break; 181116962b24SJohn Fastabend case 'h': 181216962b24SJohn Fastabend default: 181316962b24SJohn Fastabend usage(argv); 181416962b24SJohn Fastabend return -1; 181516962b24SJohn Fastabend } 181616962b24SJohn Fastabend } 181716962b24SJohn Fastabend 181816962b24SJohn Fastabend if (!cg_fd) { 181913a5f3ffSJohn Fastabend if (setup_cgroup_environment()) { 182013a5f3ffSJohn Fastabend fprintf(stderr, "ERROR: cgroup env failed\n"); 182113a5f3ffSJohn Fastabend return -EINVAL; 182213a5f3ffSJohn Fastabend } 182313a5f3ffSJohn Fastabend 182413a5f3ffSJohn Fastabend cg_fd = create_and_get_cgroup(CG_PATH); 182513a5f3ffSJohn Fastabend if (cg_fd < 0) { 182613a5f3ffSJohn Fastabend fprintf(stderr, 182713a5f3ffSJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 182813a5f3ffSJohn Fastabend cg_fd, strerror(errno)); 182913a5f3ffSJohn Fastabend return cg_fd; 183013a5f3ffSJohn Fastabend } 183113a5f3ffSJohn Fastabend 183213a5f3ffSJohn Fastabend if (join_cgroup(CG_PATH)) { 183313a5f3ffSJohn Fastabend fprintf(stderr, "ERROR: failed to join cgroup\n"); 183413a5f3ffSJohn Fastabend return -EINVAL; 183513a5f3ffSJohn Fastabend } 183613a5f3ffSJohn Fastabend cg_created = 1; 183716962b24SJohn Fastabend } 183816962b24SJohn Fastabend 1839b8b394faSJohn Fastabend err = populate_progs(bpf_file); 184016962b24SJohn Fastabend if (err) { 184116962b24SJohn Fastabend fprintf(stderr, "populate program: (%s) %s\n", 184216962b24SJohn Fastabend bpf_file, strerror(errno)); 184316962b24SJohn Fastabend return 1; 184416962b24SJohn Fastabend } 184516962b24SJohn Fastabend running = 1; 184616962b24SJohn Fastabend 184716962b24SJohn Fastabend /* catch SIGINT */ 184816962b24SJohn Fastabend signal(SIGINT, running_handler); 184916962b24SJohn Fastabend 185016962b24SJohn Fastabend options.iov_count = iov_count; 185116962b24SJohn Fastabend options.iov_length = length; 185216962b24SJohn Fastabend options.rate = rate; 185316962b24SJohn Fastabend 185416962b24SJohn Fastabend err = run_options(&options, cg_fd, test); 185513a5f3ffSJohn Fastabend 185613a5f3ffSJohn Fastabend if (cg_created) 185713a5f3ffSJohn Fastabend cleanup_cgroup_environment(); 185816962b24SJohn Fastabend close(cg_fd); 185916962b24SJohn Fastabend return err; 186016962b24SJohn Fastabend } 186116962b24SJohn Fastabend 186216962b24SJohn Fastabend void running_handler(int a) 186316962b24SJohn Fastabend { 186416962b24SJohn Fastabend running = 0; 186516962b24SJohn Fastabend } 1866