116962b24SJohn Fastabend // SPDX-License-Identifier: GPL-2.0 216962b24SJohn Fastabend // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 316962b24SJohn Fastabend #include <stdio.h> 416962b24SJohn Fastabend #include <stdlib.h> 516962b24SJohn Fastabend #include <sys/socket.h> 616962b24SJohn Fastabend #include <sys/ioctl.h> 716962b24SJohn Fastabend #include <sys/select.h> 816962b24SJohn Fastabend #include <netinet/in.h> 916962b24SJohn Fastabend #include <arpa/inet.h> 1016962b24SJohn Fastabend #include <unistd.h> 1116962b24SJohn Fastabend #include <string.h> 1216962b24SJohn Fastabend #include <errno.h> 1316962b24SJohn Fastabend #include <sys/ioctl.h> 1416962b24SJohn Fastabend #include <stdbool.h> 1516962b24SJohn Fastabend #include <signal.h> 1616962b24SJohn Fastabend #include <fcntl.h> 1716962b24SJohn Fastabend #include <sys/wait.h> 1816962b24SJohn Fastabend #include <time.h> 1916962b24SJohn Fastabend #include <sched.h> 2016962b24SJohn Fastabend 2116962b24SJohn Fastabend #include <sys/time.h> 2216962b24SJohn Fastabend #include <sys/resource.h> 2316962b24SJohn Fastabend #include <sys/types.h> 2416962b24SJohn Fastabend #include <sys/sendfile.h> 2516962b24SJohn Fastabend 2616962b24SJohn Fastabend #include <linux/netlink.h> 2716962b24SJohn Fastabend #include <linux/socket.h> 2816962b24SJohn Fastabend #include <linux/sock_diag.h> 2916962b24SJohn Fastabend #include <linux/bpf.h> 3016962b24SJohn Fastabend #include <linux/if_link.h> 3116962b24SJohn Fastabend #include <assert.h> 3216962b24SJohn Fastabend #include <libgen.h> 3316962b24SJohn Fastabend 3416962b24SJohn Fastabend #include <getopt.h> 3516962b24SJohn Fastabend 3616962b24SJohn Fastabend #include <bpf/bpf.h> 3716962b24SJohn Fastabend #include <bpf/libbpf.h> 3816962b24SJohn Fastabend 3916962b24SJohn Fastabend #include "bpf_util.h" 4016962b24SJohn Fastabend #include "bpf_rlimit.h" 4116962b24SJohn Fastabend #include "cgroup_helpers.h" 4216962b24SJohn Fastabend 4316962b24SJohn Fastabend int running; 4416962b24SJohn Fastabend static void running_handler(int a); 4516962b24SJohn Fastabend 4616962b24SJohn Fastabend /* randomly selected ports for testing on lo */ 4716962b24SJohn Fastabend #define S1_PORT 10000 4816962b24SJohn Fastabend #define S2_PORT 10001 4916962b24SJohn Fastabend 50b8b394faSJohn Fastabend #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 51b8b394faSJohn Fastabend #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 5216962b24SJohn Fastabend #define CG_PATH "/sockmap" 5316962b24SJohn Fastabend 5416962b24SJohn Fastabend /* global sockets */ 5516962b24SJohn Fastabend int s1, s2, c1, c2, p1, p2; 5616962b24SJohn Fastabend int test_cnt; 5716962b24SJohn Fastabend int passed; 5816962b24SJohn Fastabend int failed; 5916962b24SJohn Fastabend int map_fd[8]; 6016962b24SJohn Fastabend struct bpf_map *maps[8]; 6116962b24SJohn Fastabend int prog_fd[11]; 6216962b24SJohn Fastabend 6316962b24SJohn Fastabend int txmsg_pass; 6416962b24SJohn Fastabend int txmsg_noisy; 6516962b24SJohn Fastabend int txmsg_redir; 6616962b24SJohn Fastabend int txmsg_redir_noisy; 6716962b24SJohn Fastabend int txmsg_drop; 6816962b24SJohn Fastabend int txmsg_apply; 6916962b24SJohn Fastabend int txmsg_cork; 7016962b24SJohn Fastabend int txmsg_start; 7116962b24SJohn Fastabend int txmsg_end; 7216962b24SJohn Fastabend int txmsg_ingress; 7316962b24SJohn Fastabend int txmsg_skb; 7416962b24SJohn Fastabend 7516962b24SJohn Fastabend static const struct option long_options[] = { 7616962b24SJohn Fastabend {"help", no_argument, NULL, 'h' }, 7716962b24SJohn Fastabend {"cgroup", required_argument, NULL, 'c' }, 7816962b24SJohn Fastabend {"rate", required_argument, NULL, 'r' }, 7916962b24SJohn Fastabend {"verbose", no_argument, NULL, 'v' }, 8016962b24SJohn Fastabend {"iov_count", required_argument, NULL, 'i' }, 8116962b24SJohn Fastabend {"length", required_argument, NULL, 'l' }, 8216962b24SJohn Fastabend {"test", required_argument, NULL, 't' }, 8316962b24SJohn Fastabend {"data_test", no_argument, NULL, 'd' }, 8416962b24SJohn Fastabend {"txmsg", no_argument, &txmsg_pass, 1 }, 8516962b24SJohn Fastabend {"txmsg_noisy", no_argument, &txmsg_noisy, 1 }, 8616962b24SJohn Fastabend {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 8716962b24SJohn Fastabend {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1}, 8816962b24SJohn Fastabend {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 8916962b24SJohn Fastabend {"txmsg_apply", required_argument, NULL, 'a'}, 9016962b24SJohn Fastabend {"txmsg_cork", required_argument, NULL, 'k'}, 9116962b24SJohn Fastabend {"txmsg_start", required_argument, NULL, 's'}, 9216962b24SJohn Fastabend {"txmsg_end", required_argument, NULL, 'e'}, 9316962b24SJohn Fastabend {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 9416962b24SJohn Fastabend {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 9516962b24SJohn Fastabend {0, 0, NULL, 0 } 9616962b24SJohn Fastabend }; 9716962b24SJohn Fastabend 9816962b24SJohn Fastabend static void usage(char *argv[]) 9916962b24SJohn Fastabend { 10016962b24SJohn Fastabend int i; 10116962b24SJohn Fastabend 10216962b24SJohn Fastabend printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 10316962b24SJohn Fastabend printf(" options:\n"); 10416962b24SJohn Fastabend for (i = 0; long_options[i].name != 0; i++) { 10516962b24SJohn Fastabend printf(" --%-12s", long_options[i].name); 10616962b24SJohn Fastabend if (long_options[i].flag != NULL) 10716962b24SJohn Fastabend printf(" flag (internal value:%d)\n", 10816962b24SJohn Fastabend *long_options[i].flag); 10916962b24SJohn Fastabend else 11016962b24SJohn Fastabend printf(" -%c\n", long_options[i].val); 11116962b24SJohn Fastabend } 11216962b24SJohn Fastabend printf("\n"); 11316962b24SJohn Fastabend } 11416962b24SJohn Fastabend 11516962b24SJohn Fastabend static int sockmap_init_sockets(int verbose) 11616962b24SJohn Fastabend { 11716962b24SJohn Fastabend int i, err, one = 1; 11816962b24SJohn Fastabend struct sockaddr_in addr; 11916962b24SJohn Fastabend int *fds[4] = {&s1, &s2, &c1, &c2}; 12016962b24SJohn Fastabend 12116962b24SJohn Fastabend s1 = s2 = p1 = p2 = c1 = c2 = 0; 12216962b24SJohn Fastabend 12316962b24SJohn Fastabend /* Init sockets */ 12416962b24SJohn Fastabend for (i = 0; i < 4; i++) { 12516962b24SJohn Fastabend *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 12616962b24SJohn Fastabend if (*fds[i] < 0) { 12716962b24SJohn Fastabend perror("socket s1 failed()"); 12816962b24SJohn Fastabend return errno; 12916962b24SJohn Fastabend } 13016962b24SJohn Fastabend } 13116962b24SJohn Fastabend 13216962b24SJohn Fastabend /* Allow reuse */ 13316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 13416962b24SJohn Fastabend err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 13516962b24SJohn Fastabend (char *)&one, sizeof(one)); 13616962b24SJohn Fastabend if (err) { 13716962b24SJohn Fastabend perror("setsockopt failed()"); 13816962b24SJohn Fastabend return errno; 13916962b24SJohn Fastabend } 14016962b24SJohn Fastabend } 14116962b24SJohn Fastabend 14216962b24SJohn Fastabend /* Non-blocking sockets */ 14316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 14416962b24SJohn Fastabend err = ioctl(*fds[i], FIONBIO, (char *)&one); 14516962b24SJohn Fastabend if (err < 0) { 14616962b24SJohn Fastabend perror("ioctl s1 failed()"); 14716962b24SJohn Fastabend return errno; 14816962b24SJohn Fastabend } 14916962b24SJohn Fastabend } 15016962b24SJohn Fastabend 15116962b24SJohn Fastabend /* Bind server sockets */ 15216962b24SJohn Fastabend memset(&addr, 0, sizeof(struct sockaddr_in)); 15316962b24SJohn Fastabend addr.sin_family = AF_INET; 15416962b24SJohn Fastabend addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 15516962b24SJohn Fastabend 15616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 15716962b24SJohn Fastabend err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 15816962b24SJohn Fastabend if (err < 0) { 15916962b24SJohn Fastabend perror("bind s1 failed()\n"); 16016962b24SJohn Fastabend return errno; 16116962b24SJohn Fastabend } 16216962b24SJohn Fastabend 16316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 16416962b24SJohn Fastabend err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 16516962b24SJohn Fastabend if (err < 0) { 16616962b24SJohn Fastabend perror("bind s2 failed()\n"); 16716962b24SJohn Fastabend return errno; 16816962b24SJohn Fastabend } 16916962b24SJohn Fastabend 17016962b24SJohn Fastabend /* Listen server sockets */ 17116962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 17216962b24SJohn Fastabend err = listen(s1, 32); 17316962b24SJohn Fastabend if (err < 0) { 17416962b24SJohn Fastabend perror("listen s1 failed()\n"); 17516962b24SJohn Fastabend return errno; 17616962b24SJohn Fastabend } 17716962b24SJohn Fastabend 17816962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 17916962b24SJohn Fastabend err = listen(s2, 32); 18016962b24SJohn Fastabend if (err < 0) { 18116962b24SJohn Fastabend perror("listen s1 failed()\n"); 18216962b24SJohn Fastabend return errno; 18316962b24SJohn Fastabend } 18416962b24SJohn Fastabend 18516962b24SJohn Fastabend /* Initiate Connect */ 18616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 18716962b24SJohn Fastabend err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 18816962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 18916962b24SJohn Fastabend perror("connect c1 failed()\n"); 19016962b24SJohn Fastabend return errno; 19116962b24SJohn Fastabend } 19216962b24SJohn Fastabend 19316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 19416962b24SJohn Fastabend err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 19516962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 19616962b24SJohn Fastabend perror("connect c2 failed()\n"); 19716962b24SJohn Fastabend return errno; 19816962b24SJohn Fastabend } else if (err < 0) { 19916962b24SJohn Fastabend err = 0; 20016962b24SJohn Fastabend } 20116962b24SJohn Fastabend 20216962b24SJohn Fastabend /* Accept Connecrtions */ 20316962b24SJohn Fastabend p1 = accept(s1, NULL, NULL); 20416962b24SJohn Fastabend if (p1 < 0) { 20516962b24SJohn Fastabend perror("accept s1 failed()\n"); 20616962b24SJohn Fastabend return errno; 20716962b24SJohn Fastabend } 20816962b24SJohn Fastabend 20916962b24SJohn Fastabend p2 = accept(s2, NULL, NULL); 21016962b24SJohn Fastabend if (p2 < 0) { 21116962b24SJohn Fastabend perror("accept s1 failed()\n"); 21216962b24SJohn Fastabend return errno; 21316962b24SJohn Fastabend } 21416962b24SJohn Fastabend 21516962b24SJohn Fastabend if (verbose) { 21616962b24SJohn Fastabend printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 21716962b24SJohn Fastabend printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 21816962b24SJohn Fastabend c1, s1, c2, s2); 21916962b24SJohn Fastabend } 22016962b24SJohn Fastabend return 0; 22116962b24SJohn Fastabend } 22216962b24SJohn Fastabend 22316962b24SJohn Fastabend struct msg_stats { 22416962b24SJohn Fastabend size_t bytes_sent; 22516962b24SJohn Fastabend size_t bytes_recvd; 22616962b24SJohn Fastabend struct timespec start; 22716962b24SJohn Fastabend struct timespec end; 22816962b24SJohn Fastabend }; 22916962b24SJohn Fastabend 23016962b24SJohn Fastabend struct sockmap_options { 23116962b24SJohn Fastabend int verbose; 23216962b24SJohn Fastabend bool base; 23316962b24SJohn Fastabend bool sendpage; 23416962b24SJohn Fastabend bool data_test; 23516962b24SJohn Fastabend bool drop_expected; 23616962b24SJohn Fastabend int iov_count; 23716962b24SJohn Fastabend int iov_length; 23816962b24SJohn Fastabend int rate; 23916962b24SJohn Fastabend }; 24016962b24SJohn Fastabend 24116962b24SJohn Fastabend static int msg_loop_sendpage(int fd, int iov_length, int cnt, 24216962b24SJohn Fastabend struct msg_stats *s, 24316962b24SJohn Fastabend struct sockmap_options *opt) 24416962b24SJohn Fastabend { 24516962b24SJohn Fastabend bool drop = opt->drop_expected; 24616962b24SJohn Fastabend unsigned char k = 0; 24716962b24SJohn Fastabend FILE *file; 24816962b24SJohn Fastabend int i, fp; 24916962b24SJohn Fastabend 25016962b24SJohn Fastabend file = fopen(".sendpage_tst.tmp", "w+"); 25116962b24SJohn Fastabend for (i = 0; i < iov_length * cnt; i++, k++) 25216962b24SJohn Fastabend fwrite(&k, sizeof(char), 1, file); 25316962b24SJohn Fastabend fflush(file); 25416962b24SJohn Fastabend fseek(file, 0, SEEK_SET); 25516962b24SJohn Fastabend fclose(file); 25616962b24SJohn Fastabend 25716962b24SJohn Fastabend fp = open(".sendpage_tst.tmp", O_RDONLY); 25816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 25916962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 26016962b24SJohn Fastabend int sent = sendfile(fd, fp, NULL, iov_length); 26116962b24SJohn Fastabend 26216962b24SJohn Fastabend if (!drop && sent < 0) { 26316962b24SJohn Fastabend perror("send loop error:"); 26416962b24SJohn Fastabend close(fp); 26516962b24SJohn Fastabend return sent; 26616962b24SJohn Fastabend } else if (drop && sent >= 0) { 26716962b24SJohn Fastabend printf("sendpage loop error expected: %i\n", sent); 26816962b24SJohn Fastabend close(fp); 26916962b24SJohn Fastabend return -EIO; 27016962b24SJohn Fastabend } 27116962b24SJohn Fastabend 27216962b24SJohn Fastabend if (sent > 0) 27316962b24SJohn Fastabend s->bytes_sent += sent; 27416962b24SJohn Fastabend } 27516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 27616962b24SJohn Fastabend close(fp); 27716962b24SJohn Fastabend return 0; 27816962b24SJohn Fastabend } 27916962b24SJohn Fastabend 28016962b24SJohn Fastabend static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 28116962b24SJohn Fastabend struct msg_stats *s, bool tx, 28216962b24SJohn Fastabend struct sockmap_options *opt) 28316962b24SJohn Fastabend { 28416962b24SJohn Fastabend struct msghdr msg = {0}; 28516962b24SJohn Fastabend int err, i, flags = MSG_NOSIGNAL; 28616962b24SJohn Fastabend struct iovec *iov; 28716962b24SJohn Fastabend unsigned char k; 28816962b24SJohn Fastabend bool data_test = opt->data_test; 28916962b24SJohn Fastabend bool drop = opt->drop_expected; 29016962b24SJohn Fastabend 29116962b24SJohn Fastabend iov = calloc(iov_count, sizeof(struct iovec)); 29216962b24SJohn Fastabend if (!iov) 29316962b24SJohn Fastabend return errno; 29416962b24SJohn Fastabend 29516962b24SJohn Fastabend k = 0; 29616962b24SJohn Fastabend for (i = 0; i < iov_count; i++) { 29716962b24SJohn Fastabend unsigned char *d = calloc(iov_length, sizeof(char)); 29816962b24SJohn Fastabend 29916962b24SJohn Fastabend if (!d) { 30016962b24SJohn Fastabend fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 30116962b24SJohn Fastabend goto out_errno; 30216962b24SJohn Fastabend } 30316962b24SJohn Fastabend iov[i].iov_base = d; 30416962b24SJohn Fastabend iov[i].iov_len = iov_length; 30516962b24SJohn Fastabend 30616962b24SJohn Fastabend if (data_test && tx) { 30716962b24SJohn Fastabend int j; 30816962b24SJohn Fastabend 30916962b24SJohn Fastabend for (j = 0; j < iov_length; j++) 31016962b24SJohn Fastabend d[j] = k++; 31116962b24SJohn Fastabend } 31216962b24SJohn Fastabend } 31316962b24SJohn Fastabend 31416962b24SJohn Fastabend msg.msg_iov = iov; 31516962b24SJohn Fastabend msg.msg_iovlen = iov_count; 31616962b24SJohn Fastabend k = 0; 31716962b24SJohn Fastabend 31816962b24SJohn Fastabend if (tx) { 31916962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 32016962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 32116962b24SJohn Fastabend int sent = sendmsg(fd, &msg, flags); 32216962b24SJohn Fastabend 32316962b24SJohn Fastabend if (!drop && sent < 0) { 32416962b24SJohn Fastabend perror("send loop error:"); 32516962b24SJohn Fastabend goto out_errno; 32616962b24SJohn Fastabend } else if (drop && sent >= 0) { 32716962b24SJohn Fastabend printf("send loop error expected: %i\n", sent); 32816962b24SJohn Fastabend errno = -EIO; 32916962b24SJohn Fastabend goto out_errno; 33016962b24SJohn Fastabend } 33116962b24SJohn Fastabend if (sent > 0) 33216962b24SJohn Fastabend s->bytes_sent += sent; 33316962b24SJohn Fastabend } 33416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 33516962b24SJohn Fastabend } else { 33616962b24SJohn Fastabend int slct, recv, max_fd = fd; 33716962b24SJohn Fastabend int fd_flags = O_NONBLOCK; 33816962b24SJohn Fastabend struct timeval timeout; 33916962b24SJohn Fastabend float total_bytes; 34016962b24SJohn Fastabend fd_set w; 34116962b24SJohn Fastabend 34216962b24SJohn Fastabend fcntl(fd, fd_flags); 34316962b24SJohn Fastabend total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 34416962b24SJohn Fastabend err = clock_gettime(CLOCK_MONOTONIC, &s->start); 34516962b24SJohn Fastabend if (err < 0) 34616962b24SJohn Fastabend perror("recv start time: "); 34716962b24SJohn Fastabend while (s->bytes_recvd < total_bytes) { 348a18fda1aSJohn Fastabend timeout.tv_sec = 0; 349a18fda1aSJohn Fastabend timeout.tv_usec = 10; 35016962b24SJohn Fastabend 35116962b24SJohn Fastabend /* FD sets */ 35216962b24SJohn Fastabend FD_ZERO(&w); 35316962b24SJohn Fastabend FD_SET(fd, &w); 35416962b24SJohn Fastabend 35516962b24SJohn Fastabend slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 35616962b24SJohn Fastabend if (slct == -1) { 35716962b24SJohn Fastabend perror("select()"); 35816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 35916962b24SJohn Fastabend goto out_errno; 36016962b24SJohn Fastabend } else if (!slct) { 36116962b24SJohn Fastabend if (opt->verbose) 36216962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 36316962b24SJohn Fastabend errno = -EIO; 36416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 36516962b24SJohn Fastabend goto out_errno; 36616962b24SJohn Fastabend } 36716962b24SJohn Fastabend 36816962b24SJohn Fastabend recv = recvmsg(fd, &msg, flags); 36916962b24SJohn Fastabend if (recv < 0) { 37016962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 37116962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 37216962b24SJohn Fastabend perror("recv failed()\n"); 37316962b24SJohn Fastabend goto out_errno; 37416962b24SJohn Fastabend } 37516962b24SJohn Fastabend } 37616962b24SJohn Fastabend 37716962b24SJohn Fastabend s->bytes_recvd += recv; 37816962b24SJohn Fastabend 37916962b24SJohn Fastabend if (data_test) { 38016962b24SJohn Fastabend int j; 38116962b24SJohn Fastabend 38216962b24SJohn Fastabend for (i = 0; i < msg.msg_iovlen; i++) { 38316962b24SJohn Fastabend unsigned char *d = iov[i].iov_base; 38416962b24SJohn Fastabend 38516962b24SJohn Fastabend for (j = 0; 38616962b24SJohn Fastabend j < iov[i].iov_len && recv; j++) { 38716962b24SJohn Fastabend if (d[j] != k++) { 38816962b24SJohn Fastabend errno = -EIO; 38916962b24SJohn Fastabend fprintf(stderr, 39016962b24SJohn Fastabend "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 39116962b24SJohn Fastabend i, j, d[j], k - 1, d[j+1], k + 1); 39216962b24SJohn Fastabend goto out_errno; 39316962b24SJohn Fastabend } 39416962b24SJohn Fastabend recv--; 39516962b24SJohn Fastabend } 39616962b24SJohn Fastabend } 39716962b24SJohn Fastabend } 39816962b24SJohn Fastabend } 39916962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 40016962b24SJohn Fastabend } 40116962b24SJohn Fastabend 40216962b24SJohn Fastabend for (i = 0; i < iov_count; i++) 40316962b24SJohn Fastabend free(iov[i].iov_base); 40416962b24SJohn Fastabend free(iov); 40516962b24SJohn Fastabend return 0; 40616962b24SJohn Fastabend out_errno: 40716962b24SJohn Fastabend for (i = 0; i < iov_count; i++) 40816962b24SJohn Fastabend free(iov[i].iov_base); 40916962b24SJohn Fastabend free(iov); 41016962b24SJohn Fastabend return errno; 41116962b24SJohn Fastabend } 41216962b24SJohn Fastabend 41316962b24SJohn Fastabend static float giga = 1000000000; 41416962b24SJohn Fastabend 41516962b24SJohn Fastabend static inline float sentBps(struct msg_stats s) 41616962b24SJohn Fastabend { 41716962b24SJohn Fastabend return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 41816962b24SJohn Fastabend } 41916962b24SJohn Fastabend 42016962b24SJohn Fastabend static inline float recvdBps(struct msg_stats s) 42116962b24SJohn Fastabend { 42216962b24SJohn Fastabend return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 42316962b24SJohn Fastabend } 42416962b24SJohn Fastabend 42516962b24SJohn Fastabend static int sendmsg_test(struct sockmap_options *opt) 42616962b24SJohn Fastabend { 42716962b24SJohn Fastabend float sent_Bps = 0, recvd_Bps = 0; 42816962b24SJohn Fastabend int rx_fd, txpid, rxpid, err = 0; 42916962b24SJohn Fastabend struct msg_stats s = {0}; 43016962b24SJohn Fastabend int iov_count = opt->iov_count; 43116962b24SJohn Fastabend int iov_buf = opt->iov_length; 43216962b24SJohn Fastabend int cnt = opt->rate; 43316962b24SJohn Fastabend int status; 43416962b24SJohn Fastabend 43516962b24SJohn Fastabend errno = 0; 43616962b24SJohn Fastabend 43716962b24SJohn Fastabend if (opt->base) 43816962b24SJohn Fastabend rx_fd = p1; 43916962b24SJohn Fastabend else 44016962b24SJohn Fastabend rx_fd = p2; 44116962b24SJohn Fastabend 44216962b24SJohn Fastabend rxpid = fork(); 44316962b24SJohn Fastabend if (rxpid == 0) { 44416962b24SJohn Fastabend if (opt->drop_expected) 44516962b24SJohn Fastabend exit(1); 44616962b24SJohn Fastabend 44716962b24SJohn Fastabend if (opt->sendpage) 44816962b24SJohn Fastabend iov_count = 1; 44916962b24SJohn Fastabend err = msg_loop(rx_fd, iov_count, iov_buf, 45016962b24SJohn Fastabend cnt, &s, false, opt); 45116962b24SJohn Fastabend if (err && opt->verbose) 45216962b24SJohn Fastabend fprintf(stderr, 45316962b24SJohn Fastabend "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 45416962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 45516962b24SJohn Fastabend shutdown(p2, SHUT_RDWR); 45616962b24SJohn Fastabend shutdown(p1, SHUT_RDWR); 45716962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 45816962b24SJohn Fastabend sent_Bps = sentBps(s); 45916962b24SJohn Fastabend recvd_Bps = recvdBps(s); 46016962b24SJohn Fastabend } 46116962b24SJohn Fastabend if (opt->verbose) 46216962b24SJohn Fastabend fprintf(stdout, 46316962b24SJohn Fastabend "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n", 46416962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 46516962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 46616962b24SJohn Fastabend exit(1); 46716962b24SJohn Fastabend } else if (rxpid == -1) { 46816962b24SJohn Fastabend perror("msg_loop_rx: "); 46916962b24SJohn Fastabend return errno; 47016962b24SJohn Fastabend } 47116962b24SJohn Fastabend 47216962b24SJohn Fastabend txpid = fork(); 47316962b24SJohn Fastabend if (txpid == 0) { 47416962b24SJohn Fastabend if (opt->sendpage) 47516962b24SJohn Fastabend err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 47616962b24SJohn Fastabend else 47716962b24SJohn Fastabend err = msg_loop(c1, iov_count, iov_buf, 47816962b24SJohn Fastabend cnt, &s, true, opt); 47916962b24SJohn Fastabend 48016962b24SJohn Fastabend if (err) 48116962b24SJohn Fastabend fprintf(stderr, 48216962b24SJohn Fastabend "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 48316962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 48416962b24SJohn Fastabend shutdown(c1, SHUT_RDWR); 48516962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 48616962b24SJohn Fastabend sent_Bps = sentBps(s); 48716962b24SJohn Fastabend recvd_Bps = recvdBps(s); 48816962b24SJohn Fastabend } 48916962b24SJohn Fastabend if (opt->verbose) 49016962b24SJohn Fastabend fprintf(stdout, 49116962b24SJohn Fastabend "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 49216962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 49316962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 49416962b24SJohn Fastabend exit(1); 49516962b24SJohn Fastabend } else if (txpid == -1) { 49616962b24SJohn Fastabend perror("msg_loop_tx: "); 49716962b24SJohn Fastabend return errno; 49816962b24SJohn Fastabend } 49916962b24SJohn Fastabend 50016962b24SJohn Fastabend assert(waitpid(rxpid, &status, 0) == rxpid); 50116962b24SJohn Fastabend assert(waitpid(txpid, &status, 0) == txpid); 50216962b24SJohn Fastabend return err; 50316962b24SJohn Fastabend } 50416962b24SJohn Fastabend 50516962b24SJohn Fastabend static int forever_ping_pong(int rate, struct sockmap_options *opt) 50616962b24SJohn Fastabend { 50716962b24SJohn Fastabend struct timeval timeout; 50816962b24SJohn Fastabend char buf[1024] = {0}; 50916962b24SJohn Fastabend int sc; 51016962b24SJohn Fastabend 51116962b24SJohn Fastabend timeout.tv_sec = 10; 51216962b24SJohn Fastabend timeout.tv_usec = 0; 51316962b24SJohn Fastabend 51416962b24SJohn Fastabend /* Ping/Pong data from client to server */ 51516962b24SJohn Fastabend sc = send(c1, buf, sizeof(buf), 0); 51616962b24SJohn Fastabend if (sc < 0) { 51716962b24SJohn Fastabend perror("send failed()\n"); 51816962b24SJohn Fastabend return sc; 51916962b24SJohn Fastabend } 52016962b24SJohn Fastabend 52116962b24SJohn Fastabend do { 52216962b24SJohn Fastabend int s, rc, i, max_fd = p2; 52316962b24SJohn Fastabend fd_set w; 52416962b24SJohn Fastabend 52516962b24SJohn Fastabend /* FD sets */ 52616962b24SJohn Fastabend FD_ZERO(&w); 52716962b24SJohn Fastabend FD_SET(c1, &w); 52816962b24SJohn Fastabend FD_SET(c2, &w); 52916962b24SJohn Fastabend FD_SET(p1, &w); 53016962b24SJohn Fastabend FD_SET(p2, &w); 53116962b24SJohn Fastabend 53216962b24SJohn Fastabend s = select(max_fd + 1, &w, NULL, NULL, &timeout); 53316962b24SJohn Fastabend if (s == -1) { 53416962b24SJohn Fastabend perror("select()"); 53516962b24SJohn Fastabend break; 53616962b24SJohn Fastabend } else if (!s) { 53716962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 53816962b24SJohn Fastabend break; 53916962b24SJohn Fastabend } 54016962b24SJohn Fastabend 54116962b24SJohn Fastabend for (i = 0; i <= max_fd && s > 0; ++i) { 54216962b24SJohn Fastabend if (!FD_ISSET(i, &w)) 54316962b24SJohn Fastabend continue; 54416962b24SJohn Fastabend 54516962b24SJohn Fastabend s--; 54616962b24SJohn Fastabend 54716962b24SJohn Fastabend rc = recv(i, buf, sizeof(buf), 0); 54816962b24SJohn Fastabend if (rc < 0) { 54916962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 55016962b24SJohn Fastabend perror("recv failed()\n"); 55116962b24SJohn Fastabend return rc; 55216962b24SJohn Fastabend } 55316962b24SJohn Fastabend } 55416962b24SJohn Fastabend 55516962b24SJohn Fastabend if (rc == 0) { 55616962b24SJohn Fastabend close(i); 55716962b24SJohn Fastabend break; 55816962b24SJohn Fastabend } 55916962b24SJohn Fastabend 56016962b24SJohn Fastabend sc = send(i, buf, rc, 0); 56116962b24SJohn Fastabend if (sc < 0) { 56216962b24SJohn Fastabend perror("send failed()\n"); 56316962b24SJohn Fastabend return sc; 56416962b24SJohn Fastabend } 56516962b24SJohn Fastabend } 56616962b24SJohn Fastabend 56716962b24SJohn Fastabend if (rate) 56816962b24SJohn Fastabend sleep(rate); 56916962b24SJohn Fastabend 57016962b24SJohn Fastabend if (opt->verbose) { 57116962b24SJohn Fastabend printf("."); 57216962b24SJohn Fastabend fflush(stdout); 57316962b24SJohn Fastabend 57416962b24SJohn Fastabend } 57516962b24SJohn Fastabend } while (running); 57616962b24SJohn Fastabend 57716962b24SJohn Fastabend return 0; 57816962b24SJohn Fastabend } 57916962b24SJohn Fastabend 58016962b24SJohn Fastabend enum { 58116962b24SJohn Fastabend PING_PONG, 58216962b24SJohn Fastabend SENDMSG, 58316962b24SJohn Fastabend BASE, 58416962b24SJohn Fastabend BASE_SENDPAGE, 58516962b24SJohn Fastabend SENDPAGE, 58616962b24SJohn Fastabend }; 58716962b24SJohn Fastabend 58816962b24SJohn Fastabend static int run_options(struct sockmap_options *options, int cg_fd, int test) 58916962b24SJohn Fastabend { 59016962b24SJohn Fastabend int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 59116962b24SJohn Fastabend 59216962b24SJohn Fastabend /* If base test skip BPF setup */ 59316962b24SJohn Fastabend if (test == BASE || test == BASE_SENDPAGE) 59416962b24SJohn Fastabend goto run; 59516962b24SJohn Fastabend 59616962b24SJohn Fastabend /* Attach programs to sockmap */ 59716962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[0], 59816962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0); 59916962b24SJohn Fastabend if (err) { 60016962b24SJohn Fastabend fprintf(stderr, 60116962b24SJohn Fastabend "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 60216962b24SJohn Fastabend prog_fd[0], map_fd[0], err, strerror(errno)); 60316962b24SJohn Fastabend return err; 60416962b24SJohn Fastabend } 60516962b24SJohn Fastabend 60616962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[1], map_fd[0], 60716962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0); 60816962b24SJohn Fastabend if (err) { 60916962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 61016962b24SJohn Fastabend err, strerror(errno)); 61116962b24SJohn Fastabend return err; 61216962b24SJohn Fastabend } 61316962b24SJohn Fastabend 61416962b24SJohn Fastabend /* Attach to cgroups */ 61516962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 61616962b24SJohn Fastabend if (err) { 61716962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 61816962b24SJohn Fastabend err, strerror(errno)); 61916962b24SJohn Fastabend return err; 62016962b24SJohn Fastabend } 62116962b24SJohn Fastabend 62216962b24SJohn Fastabend run: 62316962b24SJohn Fastabend err = sockmap_init_sockets(options->verbose); 62416962b24SJohn Fastabend if (err) { 62516962b24SJohn Fastabend fprintf(stderr, "ERROR: test socket failed: %d\n", err); 62616962b24SJohn Fastabend goto out; 62716962b24SJohn Fastabend } 62816962b24SJohn Fastabend 62916962b24SJohn Fastabend /* Attach txmsg program to sockmap */ 63016962b24SJohn Fastabend if (txmsg_pass) 63116962b24SJohn Fastabend tx_prog_fd = prog_fd[3]; 63216962b24SJohn Fastabend else if (txmsg_noisy) 63316962b24SJohn Fastabend tx_prog_fd = prog_fd[4]; 63416962b24SJohn Fastabend else if (txmsg_redir) 63516962b24SJohn Fastabend tx_prog_fd = prog_fd[5]; 63616962b24SJohn Fastabend else if (txmsg_redir_noisy) 63716962b24SJohn Fastabend tx_prog_fd = prog_fd[6]; 63816962b24SJohn Fastabend else if (txmsg_drop) 63916962b24SJohn Fastabend tx_prog_fd = prog_fd[9]; 64016962b24SJohn Fastabend /* apply and cork must be last */ 64116962b24SJohn Fastabend else if (txmsg_apply) 64216962b24SJohn Fastabend tx_prog_fd = prog_fd[7]; 64316962b24SJohn Fastabend else if (txmsg_cork) 64416962b24SJohn Fastabend tx_prog_fd = prog_fd[8]; 64516962b24SJohn Fastabend else 64616962b24SJohn Fastabend tx_prog_fd = 0; 64716962b24SJohn Fastabend 64816962b24SJohn Fastabend if (tx_prog_fd) { 64916962b24SJohn Fastabend int redir_fd, i = 0; 65016962b24SJohn Fastabend 65116962b24SJohn Fastabend err = bpf_prog_attach(tx_prog_fd, 65216962b24SJohn Fastabend map_fd[1], BPF_SK_MSG_VERDICT, 0); 65316962b24SJohn Fastabend if (err) { 65416962b24SJohn Fastabend fprintf(stderr, 65516962b24SJohn Fastabend "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 65616962b24SJohn Fastabend err, strerror(errno)); 65716962b24SJohn Fastabend goto out; 65816962b24SJohn Fastabend } 65916962b24SJohn Fastabend 66016962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 66116962b24SJohn Fastabend if (err) { 66216962b24SJohn Fastabend fprintf(stderr, 66316962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 66416962b24SJohn Fastabend err, strerror(errno)); 66516962b24SJohn Fastabend goto out; 66616962b24SJohn Fastabend } 66716962b24SJohn Fastabend 66816962b24SJohn Fastabend if (txmsg_redir || txmsg_redir_noisy) 66916962b24SJohn Fastabend redir_fd = c2; 67016962b24SJohn Fastabend else 67116962b24SJohn Fastabend redir_fd = c1; 67216962b24SJohn Fastabend 67316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 67416962b24SJohn Fastabend if (err) { 67516962b24SJohn Fastabend fprintf(stderr, 67616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 67716962b24SJohn Fastabend err, strerror(errno)); 67816962b24SJohn Fastabend goto out; 67916962b24SJohn Fastabend } 68016962b24SJohn Fastabend 68116962b24SJohn Fastabend if (txmsg_apply) { 68216962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[3], 68316962b24SJohn Fastabend &i, &txmsg_apply, BPF_ANY); 68416962b24SJohn Fastabend if (err) { 68516962b24SJohn Fastabend fprintf(stderr, 68616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 68716962b24SJohn Fastabend err, strerror(errno)); 68816962b24SJohn Fastabend goto out; 68916962b24SJohn Fastabend } 69016962b24SJohn Fastabend } 69116962b24SJohn Fastabend 69216962b24SJohn Fastabend if (txmsg_cork) { 69316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[4], 69416962b24SJohn Fastabend &i, &txmsg_cork, BPF_ANY); 69516962b24SJohn Fastabend if (err) { 69616962b24SJohn Fastabend fprintf(stderr, 69716962b24SJohn Fastabend "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 69816962b24SJohn Fastabend err, strerror(errno)); 69916962b24SJohn Fastabend goto out; 70016962b24SJohn Fastabend } 70116962b24SJohn Fastabend } 70216962b24SJohn Fastabend 70316962b24SJohn Fastabend if (txmsg_start) { 70416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 70516962b24SJohn Fastabend &i, &txmsg_start, BPF_ANY); 70616962b24SJohn Fastabend if (err) { 70716962b24SJohn Fastabend fprintf(stderr, 70816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 70916962b24SJohn Fastabend err, strerror(errno)); 71016962b24SJohn Fastabend goto out; 71116962b24SJohn Fastabend } 71216962b24SJohn Fastabend } 71316962b24SJohn Fastabend 71416962b24SJohn Fastabend if (txmsg_end) { 71516962b24SJohn Fastabend i = 1; 71616962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 71716962b24SJohn Fastabend &i, &txmsg_end, BPF_ANY); 71816962b24SJohn Fastabend if (err) { 71916962b24SJohn Fastabend fprintf(stderr, 72016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 72116962b24SJohn Fastabend err, strerror(errno)); 72216962b24SJohn Fastabend goto out; 72316962b24SJohn Fastabend } 72416962b24SJohn Fastabend } 72516962b24SJohn Fastabend 72616962b24SJohn Fastabend if (txmsg_ingress) { 72716962b24SJohn Fastabend int in = BPF_F_INGRESS; 72816962b24SJohn Fastabend 72916962b24SJohn Fastabend i = 0; 73016962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 73116962b24SJohn Fastabend if (err) { 73216962b24SJohn Fastabend fprintf(stderr, 73316962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 73416962b24SJohn Fastabend err, strerror(errno)); 73516962b24SJohn Fastabend } 73616962b24SJohn Fastabend i = 1; 73716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 73816962b24SJohn Fastabend if (err) { 73916962b24SJohn Fastabend fprintf(stderr, 74016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 74116962b24SJohn Fastabend err, strerror(errno)); 74216962b24SJohn Fastabend } 74316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 74416962b24SJohn Fastabend if (err) { 74516962b24SJohn Fastabend fprintf(stderr, 74616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 74716962b24SJohn Fastabend err, strerror(errno)); 74816962b24SJohn Fastabend } 74916962b24SJohn Fastabend 75016962b24SJohn Fastabend i = 2; 75116962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 75216962b24SJohn Fastabend if (err) { 75316962b24SJohn Fastabend fprintf(stderr, 75416962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 75516962b24SJohn Fastabend err, strerror(errno)); 75616962b24SJohn Fastabend } 75716962b24SJohn Fastabend } 75816962b24SJohn Fastabend 75916962b24SJohn Fastabend if (txmsg_skb) { 76016962b24SJohn Fastabend int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 76116962b24SJohn Fastabend p2 : p1; 76216962b24SJohn Fastabend int ingress = BPF_F_INGRESS; 76316962b24SJohn Fastabend 76416962b24SJohn Fastabend i = 0; 76516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[7], 76616962b24SJohn Fastabend &i, &ingress, BPF_ANY); 76716962b24SJohn Fastabend if (err) { 76816962b24SJohn Fastabend fprintf(stderr, 76916962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 77016962b24SJohn Fastabend err, strerror(errno)); 77116962b24SJohn Fastabend } 77216962b24SJohn Fastabend 77316962b24SJohn Fastabend i = 3; 77416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[0], 77516962b24SJohn Fastabend &i, &skb_fd, BPF_ANY); 77616962b24SJohn Fastabend if (err) { 77716962b24SJohn Fastabend fprintf(stderr, 77816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 77916962b24SJohn Fastabend err, strerror(errno)); 78016962b24SJohn Fastabend } 78116962b24SJohn Fastabend } 78216962b24SJohn Fastabend } 78316962b24SJohn Fastabend 78416962b24SJohn Fastabend if (txmsg_drop) 78516962b24SJohn Fastabend options->drop_expected = true; 78616962b24SJohn Fastabend 78716962b24SJohn Fastabend if (test == PING_PONG) 78816962b24SJohn Fastabend err = forever_ping_pong(options->rate, options); 78916962b24SJohn Fastabend else if (test == SENDMSG) { 79016962b24SJohn Fastabend options->base = false; 79116962b24SJohn Fastabend options->sendpage = false; 79216962b24SJohn Fastabend err = sendmsg_test(options); 79316962b24SJohn Fastabend } else if (test == SENDPAGE) { 79416962b24SJohn Fastabend options->base = false; 79516962b24SJohn Fastabend options->sendpage = true; 79616962b24SJohn Fastabend err = sendmsg_test(options); 79716962b24SJohn Fastabend } else if (test == BASE) { 79816962b24SJohn Fastabend options->base = true; 79916962b24SJohn Fastabend options->sendpage = false; 80016962b24SJohn Fastabend err = sendmsg_test(options); 80116962b24SJohn Fastabend } else if (test == BASE_SENDPAGE) { 80216962b24SJohn Fastabend options->base = true; 80316962b24SJohn Fastabend options->sendpage = true; 80416962b24SJohn Fastabend err = sendmsg_test(options); 80516962b24SJohn Fastabend } else 80616962b24SJohn Fastabend fprintf(stderr, "unknown test\n"); 80716962b24SJohn Fastabend out: 80816962b24SJohn Fastabend /* Detatch and zero all the maps */ 80916962b24SJohn Fastabend bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 81016962b24SJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 81116962b24SJohn Fastabend bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 81216962b24SJohn Fastabend if (tx_prog_fd >= 0) 81316962b24SJohn Fastabend bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 81416962b24SJohn Fastabend 81516962b24SJohn Fastabend for (i = 0; i < 8; i++) { 81616962b24SJohn Fastabend key = next_key = 0; 81716962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 81816962b24SJohn Fastabend while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 81916962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 82016962b24SJohn Fastabend key = next_key; 82116962b24SJohn Fastabend } 82216962b24SJohn Fastabend } 82316962b24SJohn Fastabend 82416962b24SJohn Fastabend close(s1); 82516962b24SJohn Fastabend close(s2); 82616962b24SJohn Fastabend close(p1); 82716962b24SJohn Fastabend close(p2); 82816962b24SJohn Fastabend close(c1); 82916962b24SJohn Fastabend close(c2); 83016962b24SJohn Fastabend return err; 83116962b24SJohn Fastabend } 83216962b24SJohn Fastabend 83316962b24SJohn Fastabend static char *test_to_str(int test) 83416962b24SJohn Fastabend { 83516962b24SJohn Fastabend switch (test) { 83616962b24SJohn Fastabend case SENDMSG: 83716962b24SJohn Fastabend return "sendmsg"; 83816962b24SJohn Fastabend case SENDPAGE: 83916962b24SJohn Fastabend return "sendpage"; 84016962b24SJohn Fastabend } 84116962b24SJohn Fastabend return "unknown"; 84216962b24SJohn Fastabend } 84316962b24SJohn Fastabend 84416962b24SJohn Fastabend #define OPTSTRING 60 84516962b24SJohn Fastabend static void test_options(char *options) 84616962b24SJohn Fastabend { 84716962b24SJohn Fastabend memset(options, 0, OPTSTRING); 84816962b24SJohn Fastabend 84916962b24SJohn Fastabend if (txmsg_pass) 85016962b24SJohn Fastabend strncat(options, "pass,", OPTSTRING); 85116962b24SJohn Fastabend if (txmsg_noisy) 85216962b24SJohn Fastabend strncat(options, "pass_noisy,", OPTSTRING); 85316962b24SJohn Fastabend if (txmsg_redir) 85416962b24SJohn Fastabend strncat(options, "redir,", OPTSTRING); 85516962b24SJohn Fastabend if (txmsg_redir_noisy) 85616962b24SJohn Fastabend strncat(options, "redir_noisy,", OPTSTRING); 85716962b24SJohn Fastabend if (txmsg_drop) 85816962b24SJohn Fastabend strncat(options, "drop,", OPTSTRING); 85916962b24SJohn Fastabend if (txmsg_apply) 86016962b24SJohn Fastabend strncat(options, "apply,", OPTSTRING); 86116962b24SJohn Fastabend if (txmsg_cork) 86216962b24SJohn Fastabend strncat(options, "cork,", OPTSTRING); 86316962b24SJohn Fastabend if (txmsg_start) 86416962b24SJohn Fastabend strncat(options, "start,", OPTSTRING); 86516962b24SJohn Fastabend if (txmsg_end) 86616962b24SJohn Fastabend strncat(options, "end,", OPTSTRING); 86716962b24SJohn Fastabend if (txmsg_ingress) 86816962b24SJohn Fastabend strncat(options, "ingress,", OPTSTRING); 86916962b24SJohn Fastabend if (txmsg_skb) 87016962b24SJohn Fastabend strncat(options, "skb,", OPTSTRING); 87116962b24SJohn Fastabend } 87216962b24SJohn Fastabend 87316962b24SJohn Fastabend static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 87416962b24SJohn Fastabend { 87516962b24SJohn Fastabend char *options = calloc(60, sizeof(char)); 87616962b24SJohn Fastabend int err; 87716962b24SJohn Fastabend 87816962b24SJohn Fastabend if (test == SENDPAGE) 87916962b24SJohn Fastabend opt->sendpage = true; 88016962b24SJohn Fastabend else 88116962b24SJohn Fastabend opt->sendpage = false; 88216962b24SJohn Fastabend 88316962b24SJohn Fastabend if (txmsg_drop) 88416962b24SJohn Fastabend opt->drop_expected = true; 88516962b24SJohn Fastabend else 88616962b24SJohn Fastabend opt->drop_expected = false; 88716962b24SJohn Fastabend 88816962b24SJohn Fastabend test_options(options); 88916962b24SJohn Fastabend 89016962b24SJohn Fastabend fprintf(stdout, 89116962b24SJohn Fastabend "[TEST %i]: (%i, %i, %i, %s, %s): ", 89216962b24SJohn Fastabend test_cnt, opt->rate, opt->iov_count, opt->iov_length, 89316962b24SJohn Fastabend test_to_str(test), options); 89416962b24SJohn Fastabend fflush(stdout); 89516962b24SJohn Fastabend err = run_options(opt, cgrp, test); 89616962b24SJohn Fastabend fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 89716962b24SJohn Fastabend test_cnt++; 89816962b24SJohn Fastabend !err ? passed++ : failed++; 89916962b24SJohn Fastabend free(options); 90016962b24SJohn Fastabend return err; 90116962b24SJohn Fastabend } 90216962b24SJohn Fastabend 90316962b24SJohn Fastabend static int test_exec(int cgrp, struct sockmap_options *opt) 90416962b24SJohn Fastabend { 90516962b24SJohn Fastabend int err = __test_exec(cgrp, SENDMSG, opt); 90616962b24SJohn Fastabend 90716962b24SJohn Fastabend if (err) 90816962b24SJohn Fastabend goto out; 90916962b24SJohn Fastabend 91016962b24SJohn Fastabend err = __test_exec(cgrp, SENDPAGE, opt); 91116962b24SJohn Fastabend out: 91216962b24SJohn Fastabend return err; 91316962b24SJohn Fastabend } 91416962b24SJohn Fastabend 91516962b24SJohn Fastabend static int test_loop(int cgrp) 91616962b24SJohn Fastabend { 91716962b24SJohn Fastabend struct sockmap_options opt; 91816962b24SJohn Fastabend 91916962b24SJohn Fastabend int err, i, l, r; 92016962b24SJohn Fastabend 92116962b24SJohn Fastabend opt.verbose = 0; 92216962b24SJohn Fastabend opt.base = false; 92316962b24SJohn Fastabend opt.sendpage = false; 92416962b24SJohn Fastabend opt.data_test = false; 92516962b24SJohn Fastabend opt.drop_expected = false; 92616962b24SJohn Fastabend opt.iov_count = 0; 92716962b24SJohn Fastabend opt.iov_length = 0; 92816962b24SJohn Fastabend opt.rate = 0; 92916962b24SJohn Fastabend 930a18fda1aSJohn Fastabend r = 1; 93116962b24SJohn Fastabend for (i = 1; i < 100; i += 33) { 93216962b24SJohn Fastabend for (l = 1; l < 100; l += 33) { 93316962b24SJohn Fastabend opt.rate = r; 93416962b24SJohn Fastabend opt.iov_count = i; 93516962b24SJohn Fastabend opt.iov_length = l; 93616962b24SJohn Fastabend err = test_exec(cgrp, &opt); 93716962b24SJohn Fastabend if (err) 93816962b24SJohn Fastabend goto out; 93916962b24SJohn Fastabend } 94016962b24SJohn Fastabend } 941a18fda1aSJohn Fastabend sched_yield(); 94216962b24SJohn Fastabend out: 94316962b24SJohn Fastabend return err; 94416962b24SJohn Fastabend } 94516962b24SJohn Fastabend 94616962b24SJohn Fastabend static int test_txmsg(int cgrp) 94716962b24SJohn Fastabend { 94816962b24SJohn Fastabend int err; 94916962b24SJohn Fastabend 95016962b24SJohn Fastabend txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 95116962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 95216962b24SJohn Fastabend txmsg_ingress = txmsg_skb = 0; 95316962b24SJohn Fastabend 95416962b24SJohn Fastabend txmsg_pass = 1; 95516962b24SJohn Fastabend err = test_loop(cgrp); 95616962b24SJohn Fastabend txmsg_pass = 0; 95716962b24SJohn Fastabend if (err) 95816962b24SJohn Fastabend goto out; 95916962b24SJohn Fastabend 96016962b24SJohn Fastabend txmsg_redir = 1; 96116962b24SJohn Fastabend err = test_loop(cgrp); 96216962b24SJohn Fastabend txmsg_redir = 0; 96316962b24SJohn Fastabend if (err) 96416962b24SJohn Fastabend goto out; 96516962b24SJohn Fastabend 96616962b24SJohn Fastabend txmsg_drop = 1; 96716962b24SJohn Fastabend err = test_loop(cgrp); 96816962b24SJohn Fastabend txmsg_drop = 0; 96916962b24SJohn Fastabend if (err) 97016962b24SJohn Fastabend goto out; 97116962b24SJohn Fastabend 97216962b24SJohn Fastabend txmsg_redir = 1; 97316962b24SJohn Fastabend txmsg_ingress = 1; 97416962b24SJohn Fastabend err = test_loop(cgrp); 97516962b24SJohn Fastabend txmsg_redir = 0; 97616962b24SJohn Fastabend txmsg_ingress = 0; 97716962b24SJohn Fastabend if (err) 97816962b24SJohn Fastabend goto out; 97916962b24SJohn Fastabend out: 98016962b24SJohn Fastabend txmsg_pass = 0; 98116962b24SJohn Fastabend txmsg_redir = 0; 98216962b24SJohn Fastabend txmsg_drop = 0; 98316962b24SJohn Fastabend return err; 98416962b24SJohn Fastabend } 98516962b24SJohn Fastabend 98616962b24SJohn Fastabend static int test_send(struct sockmap_options *opt, int cgrp) 98716962b24SJohn Fastabend { 98816962b24SJohn Fastabend int err; 98916962b24SJohn Fastabend 99016962b24SJohn Fastabend opt->iov_length = 1; 99116962b24SJohn Fastabend opt->iov_count = 1; 99216962b24SJohn Fastabend opt->rate = 1; 99316962b24SJohn Fastabend err = test_exec(cgrp, opt); 99416962b24SJohn Fastabend if (err) 99516962b24SJohn Fastabend goto out; 99616962b24SJohn Fastabend 99716962b24SJohn Fastabend opt->iov_length = 1; 99816962b24SJohn Fastabend opt->iov_count = 1024; 99916962b24SJohn Fastabend opt->rate = 1; 100016962b24SJohn Fastabend err = test_exec(cgrp, opt); 100116962b24SJohn Fastabend if (err) 100216962b24SJohn Fastabend goto out; 100316962b24SJohn Fastabend 100416962b24SJohn Fastabend opt->iov_length = 1024; 100516962b24SJohn Fastabend opt->iov_count = 1; 100616962b24SJohn Fastabend opt->rate = 1; 100716962b24SJohn Fastabend err = test_exec(cgrp, opt); 100816962b24SJohn Fastabend if (err) 100916962b24SJohn Fastabend goto out; 101016962b24SJohn Fastabend 101116962b24SJohn Fastabend opt->iov_length = 1; 101216962b24SJohn Fastabend opt->iov_count = 1; 101316962b24SJohn Fastabend opt->rate = 1024; 101416962b24SJohn Fastabend err = test_exec(cgrp, opt); 101516962b24SJohn Fastabend if (err) 101616962b24SJohn Fastabend goto out; 101716962b24SJohn Fastabend 101816962b24SJohn Fastabend opt->iov_length = 256; 101916962b24SJohn Fastabend opt->iov_count = 1024; 102016962b24SJohn Fastabend opt->rate = 10; 102116962b24SJohn Fastabend err = test_exec(cgrp, opt); 102216962b24SJohn Fastabend if (err) 102316962b24SJohn Fastabend goto out; 102416962b24SJohn Fastabend 102516962b24SJohn Fastabend opt->rate = 100; 102616962b24SJohn Fastabend opt->iov_count = 1; 102716962b24SJohn Fastabend opt->iov_length = 5; 102816962b24SJohn Fastabend err = test_exec(cgrp, opt); 102916962b24SJohn Fastabend if (err) 103016962b24SJohn Fastabend goto out; 103116962b24SJohn Fastabend out: 1032a18fda1aSJohn Fastabend sched_yield(); 103316962b24SJohn Fastabend return err; 103416962b24SJohn Fastabend } 103516962b24SJohn Fastabend 103616962b24SJohn Fastabend static int test_mixed(int cgrp) 103716962b24SJohn Fastabend { 103816962b24SJohn Fastabend struct sockmap_options opt = {0}; 103916962b24SJohn Fastabend int err; 104016962b24SJohn Fastabend 104116962b24SJohn Fastabend txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 104216962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 104316962b24SJohn Fastabend txmsg_start = txmsg_end = 0; 104416962b24SJohn Fastabend /* Test small and large iov_count values with pass/redir/apply/cork */ 104516962b24SJohn Fastabend txmsg_pass = 1; 104616962b24SJohn Fastabend txmsg_redir = 0; 104716962b24SJohn Fastabend txmsg_apply = 1; 104816962b24SJohn Fastabend txmsg_cork = 0; 104916962b24SJohn Fastabend err = test_send(&opt, cgrp); 105016962b24SJohn Fastabend if (err) 105116962b24SJohn Fastabend goto out; 105216962b24SJohn Fastabend 105316962b24SJohn Fastabend txmsg_pass = 1; 105416962b24SJohn Fastabend txmsg_redir = 0; 105516962b24SJohn Fastabend txmsg_apply = 0; 105616962b24SJohn Fastabend txmsg_cork = 1; 105716962b24SJohn Fastabend err = test_send(&opt, cgrp); 105816962b24SJohn Fastabend if (err) 105916962b24SJohn Fastabend goto out; 106016962b24SJohn Fastabend 106116962b24SJohn Fastabend txmsg_pass = 1; 106216962b24SJohn Fastabend txmsg_redir = 0; 106316962b24SJohn Fastabend txmsg_apply = 1; 106416962b24SJohn Fastabend txmsg_cork = 1; 106516962b24SJohn Fastabend err = test_send(&opt, cgrp); 106616962b24SJohn Fastabend if (err) 106716962b24SJohn Fastabend goto out; 106816962b24SJohn Fastabend 106916962b24SJohn Fastabend txmsg_pass = 1; 107016962b24SJohn Fastabend txmsg_redir = 0; 107116962b24SJohn Fastabend txmsg_apply = 1024; 107216962b24SJohn Fastabend txmsg_cork = 0; 107316962b24SJohn Fastabend err = test_send(&opt, cgrp); 107416962b24SJohn Fastabend if (err) 107516962b24SJohn Fastabend goto out; 107616962b24SJohn Fastabend 107716962b24SJohn Fastabend txmsg_pass = 1; 107816962b24SJohn Fastabend txmsg_redir = 0; 107916962b24SJohn Fastabend txmsg_apply = 0; 108016962b24SJohn Fastabend txmsg_cork = 1024; 108116962b24SJohn Fastabend err = test_send(&opt, cgrp); 108216962b24SJohn Fastabend if (err) 108316962b24SJohn Fastabend goto out; 108416962b24SJohn Fastabend 108516962b24SJohn Fastabend txmsg_pass = 1; 108616962b24SJohn Fastabend txmsg_redir = 0; 108716962b24SJohn Fastabend txmsg_apply = 1024; 108816962b24SJohn Fastabend txmsg_cork = 1024; 108916962b24SJohn Fastabend err = test_send(&opt, cgrp); 109016962b24SJohn Fastabend if (err) 109116962b24SJohn Fastabend goto out; 109216962b24SJohn Fastabend 109316962b24SJohn Fastabend txmsg_pass = 1; 109416962b24SJohn Fastabend txmsg_redir = 0; 109516962b24SJohn Fastabend txmsg_cork = 4096; 109616962b24SJohn Fastabend txmsg_apply = 4096; 109716962b24SJohn Fastabend err = test_send(&opt, cgrp); 109816962b24SJohn Fastabend if (err) 109916962b24SJohn Fastabend goto out; 110016962b24SJohn Fastabend 110116962b24SJohn Fastabend txmsg_pass = 0; 110216962b24SJohn Fastabend txmsg_redir = 1; 110316962b24SJohn Fastabend txmsg_apply = 1; 110416962b24SJohn Fastabend txmsg_cork = 0; 110516962b24SJohn Fastabend err = test_send(&opt, cgrp); 110616962b24SJohn Fastabend if (err) 110716962b24SJohn Fastabend goto out; 110816962b24SJohn Fastabend 110916962b24SJohn Fastabend txmsg_pass = 0; 111016962b24SJohn Fastabend txmsg_redir = 1; 111116962b24SJohn Fastabend txmsg_apply = 0; 111216962b24SJohn Fastabend txmsg_cork = 1; 111316962b24SJohn Fastabend err = test_send(&opt, cgrp); 111416962b24SJohn Fastabend if (err) 111516962b24SJohn Fastabend goto out; 111616962b24SJohn Fastabend 111716962b24SJohn Fastabend txmsg_pass = 0; 111816962b24SJohn Fastabend txmsg_redir = 1; 111916962b24SJohn Fastabend txmsg_apply = 1024; 112016962b24SJohn Fastabend txmsg_cork = 0; 112116962b24SJohn Fastabend err = test_send(&opt, cgrp); 112216962b24SJohn Fastabend if (err) 112316962b24SJohn Fastabend goto out; 112416962b24SJohn Fastabend 112516962b24SJohn Fastabend txmsg_pass = 0; 112616962b24SJohn Fastabend txmsg_redir = 1; 112716962b24SJohn Fastabend txmsg_apply = 0; 112816962b24SJohn Fastabend txmsg_cork = 1024; 112916962b24SJohn Fastabend err = test_send(&opt, cgrp); 113016962b24SJohn Fastabend if (err) 113116962b24SJohn Fastabend goto out; 113216962b24SJohn Fastabend 113316962b24SJohn Fastabend txmsg_pass = 0; 113416962b24SJohn Fastabend txmsg_redir = 1; 113516962b24SJohn Fastabend txmsg_apply = 1024; 113616962b24SJohn Fastabend txmsg_cork = 1024; 113716962b24SJohn Fastabend err = test_send(&opt, cgrp); 113816962b24SJohn Fastabend if (err) 113916962b24SJohn Fastabend goto out; 114016962b24SJohn Fastabend 114116962b24SJohn Fastabend txmsg_pass = 0; 114216962b24SJohn Fastabend txmsg_redir = 1; 114316962b24SJohn Fastabend txmsg_cork = 4096; 114416962b24SJohn Fastabend txmsg_apply = 4096; 114516962b24SJohn Fastabend err = test_send(&opt, cgrp); 114616962b24SJohn Fastabend if (err) 114716962b24SJohn Fastabend goto out; 114816962b24SJohn Fastabend out: 114916962b24SJohn Fastabend return err; 115016962b24SJohn Fastabend } 115116962b24SJohn Fastabend 115216962b24SJohn Fastabend static int test_start_end(int cgrp) 115316962b24SJohn Fastabend { 115416962b24SJohn Fastabend struct sockmap_options opt = {0}; 115516962b24SJohn Fastabend int err, i; 115616962b24SJohn Fastabend 115716962b24SJohn Fastabend /* Test basic start/end with lots of iov_count and iov_lengths */ 115816962b24SJohn Fastabend txmsg_start = 1; 115916962b24SJohn Fastabend txmsg_end = 2; 116016962b24SJohn Fastabend err = test_txmsg(cgrp); 116116962b24SJohn Fastabend if (err) 116216962b24SJohn Fastabend goto out; 116316962b24SJohn Fastabend 116416962b24SJohn Fastabend /* Test start/end with cork */ 116516962b24SJohn Fastabend opt.rate = 16; 116616962b24SJohn Fastabend opt.iov_count = 1; 116716962b24SJohn Fastabend opt.iov_length = 100; 116816962b24SJohn Fastabend txmsg_cork = 1600; 116916962b24SJohn Fastabend 1170a18fda1aSJohn Fastabend for (i = 99; i <= 1600; i += 500) { 117116962b24SJohn Fastabend txmsg_start = 0; 117216962b24SJohn Fastabend txmsg_end = i; 117316962b24SJohn Fastabend err = test_exec(cgrp, &opt); 117416962b24SJohn Fastabend if (err) 117516962b24SJohn Fastabend goto out; 117616962b24SJohn Fastabend } 117716962b24SJohn Fastabend 117816962b24SJohn Fastabend /* Test start/end with cork but pull data in middle */ 1179a18fda1aSJohn Fastabend for (i = 199; i <= 1600; i += 500) { 118016962b24SJohn Fastabend txmsg_start = 100; 118116962b24SJohn Fastabend txmsg_end = i; 118216962b24SJohn Fastabend err = test_exec(cgrp, &opt); 118316962b24SJohn Fastabend if (err) 118416962b24SJohn Fastabend goto out; 118516962b24SJohn Fastabend } 118616962b24SJohn Fastabend 118716962b24SJohn Fastabend /* Test start/end with cork pulling last sg entry */ 118816962b24SJohn Fastabend txmsg_start = 1500; 118916962b24SJohn Fastabend txmsg_end = 1600; 119016962b24SJohn Fastabend err = test_exec(cgrp, &opt); 119116962b24SJohn Fastabend if (err) 119216962b24SJohn Fastabend goto out; 119316962b24SJohn Fastabend 119416962b24SJohn Fastabend /* Test start/end pull of single byte in last page */ 119516962b24SJohn Fastabend txmsg_start = 1111; 119616962b24SJohn Fastabend txmsg_end = 1112; 119716962b24SJohn Fastabend err = test_exec(cgrp, &opt); 119816962b24SJohn Fastabend if (err) 119916962b24SJohn Fastabend goto out; 120016962b24SJohn Fastabend 120116962b24SJohn Fastabend /* Test start/end with end < start */ 120216962b24SJohn Fastabend txmsg_start = 1111; 120316962b24SJohn Fastabend txmsg_end = 0; 120416962b24SJohn Fastabend err = test_exec(cgrp, &opt); 120516962b24SJohn Fastabend if (err) 120616962b24SJohn Fastabend goto out; 120716962b24SJohn Fastabend 120816962b24SJohn Fastabend /* Test start/end with end > data */ 120916962b24SJohn Fastabend txmsg_start = 0; 121016962b24SJohn Fastabend txmsg_end = 1601; 121116962b24SJohn Fastabend err = test_exec(cgrp, &opt); 121216962b24SJohn Fastabend if (err) 121316962b24SJohn Fastabend goto out; 121416962b24SJohn Fastabend 121516962b24SJohn Fastabend /* Test start/end with start > data */ 121616962b24SJohn Fastabend txmsg_start = 1601; 121716962b24SJohn Fastabend txmsg_end = 1600; 121816962b24SJohn Fastabend err = test_exec(cgrp, &opt); 121916962b24SJohn Fastabend 122016962b24SJohn Fastabend out: 122116962b24SJohn Fastabend txmsg_start = 0; 122216962b24SJohn Fastabend txmsg_end = 0; 1223a18fda1aSJohn Fastabend sched_yield(); 122416962b24SJohn Fastabend return err; 122516962b24SJohn Fastabend } 122616962b24SJohn Fastabend 122716962b24SJohn Fastabend char *map_names[] = { 122816962b24SJohn Fastabend "sock_map", 122916962b24SJohn Fastabend "sock_map_txmsg", 123016962b24SJohn Fastabend "sock_map_redir", 123116962b24SJohn Fastabend "sock_apply_bytes", 123216962b24SJohn Fastabend "sock_cork_bytes", 123316962b24SJohn Fastabend "sock_pull_bytes", 123416962b24SJohn Fastabend "sock_redir_flags", 123516962b24SJohn Fastabend "sock_skb_opts", 123616962b24SJohn Fastabend }; 123716962b24SJohn Fastabend 123816962b24SJohn Fastabend int prog_attach_type[] = { 123916962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 124016962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 124116962b24SJohn Fastabend BPF_CGROUP_SOCK_OPS, 124216962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124316962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124416962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124516962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124616962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124716962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124816962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 124916962b24SJohn Fastabend }; 125016962b24SJohn Fastabend 125116962b24SJohn Fastabend int prog_type[] = { 125216962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 125316962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 125416962b24SJohn Fastabend BPF_PROG_TYPE_SOCK_OPS, 125516962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 125616962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 125716962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 125816962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 125916962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 126016962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 126116962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 126216962b24SJohn Fastabend }; 126316962b24SJohn Fastabend 1264b8b394faSJohn Fastabend static int populate_progs(char *bpf_file) 126516962b24SJohn Fastabend { 126616962b24SJohn Fastabend struct bpf_program *prog; 126716962b24SJohn Fastabend struct bpf_object *obj; 126816962b24SJohn Fastabend int i = 0; 126916962b24SJohn Fastabend long err; 127016962b24SJohn Fastabend 127116962b24SJohn Fastabend obj = bpf_object__open(bpf_file); 127216962b24SJohn Fastabend err = libbpf_get_error(obj); 127316962b24SJohn Fastabend if (err) { 127416962b24SJohn Fastabend char err_buf[256]; 127516962b24SJohn Fastabend 127616962b24SJohn Fastabend libbpf_strerror(err, err_buf, sizeof(err_buf)); 127716962b24SJohn Fastabend printf("Unable to load eBPF objects in file '%s' : %s\n", 127816962b24SJohn Fastabend bpf_file, err_buf); 127916962b24SJohn Fastabend return -1; 128016962b24SJohn Fastabend } 128116962b24SJohn Fastabend 128216962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 128316962b24SJohn Fastabend bpf_program__set_type(prog, prog_type[i]); 128416962b24SJohn Fastabend bpf_program__set_expected_attach_type(prog, 128516962b24SJohn Fastabend prog_attach_type[i]); 128616962b24SJohn Fastabend i++; 128716962b24SJohn Fastabend } 128816962b24SJohn Fastabend 128916962b24SJohn Fastabend i = bpf_object__load(obj); 129016962b24SJohn Fastabend i = 0; 129116962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 129216962b24SJohn Fastabend prog_fd[i] = bpf_program__fd(prog); 129316962b24SJohn Fastabend i++; 129416962b24SJohn Fastabend } 129516962b24SJohn Fastabend 129616962b24SJohn Fastabend for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 129716962b24SJohn Fastabend maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 129816962b24SJohn Fastabend map_fd[i] = bpf_map__fd(maps[i]); 129916962b24SJohn Fastabend if (map_fd[i] < 0) { 130016962b24SJohn Fastabend fprintf(stderr, "load_bpf_file: (%i) %s\n", 130116962b24SJohn Fastabend map_fd[i], strerror(errno)); 130216962b24SJohn Fastabend return -1; 130316962b24SJohn Fastabend } 130416962b24SJohn Fastabend } 130516962b24SJohn Fastabend 130616962b24SJohn Fastabend return 0; 130716962b24SJohn Fastabend } 130816962b24SJohn Fastabend 1309b8b394faSJohn Fastabend static int __test_suite(char *bpf_file) 131016962b24SJohn Fastabend { 131116962b24SJohn Fastabend int cg_fd, err; 131216962b24SJohn Fastabend 1313b8b394faSJohn Fastabend err = populate_progs(bpf_file); 131416962b24SJohn Fastabend if (err < 0) { 131516962b24SJohn Fastabend fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 131616962b24SJohn Fastabend return err; 131716962b24SJohn Fastabend } 131816962b24SJohn Fastabend 131916962b24SJohn Fastabend if (setup_cgroup_environment()) { 132016962b24SJohn Fastabend fprintf(stderr, "ERROR: cgroup env failed\n"); 132116962b24SJohn Fastabend return -EINVAL; 132216962b24SJohn Fastabend } 132316962b24SJohn Fastabend 132416962b24SJohn Fastabend cg_fd = create_and_get_cgroup(CG_PATH); 132516962b24SJohn Fastabend if (cg_fd < 0) { 132616962b24SJohn Fastabend fprintf(stderr, 132716962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 132816962b24SJohn Fastabend cg_fd, optarg); 132916962b24SJohn Fastabend return cg_fd; 133016962b24SJohn Fastabend } 133116962b24SJohn Fastabend 133216962b24SJohn Fastabend /* Tests basic commands and APIs with range of iov values */ 133316962b24SJohn Fastabend txmsg_start = txmsg_end = 0; 133416962b24SJohn Fastabend err = test_txmsg(cg_fd); 133516962b24SJohn Fastabend if (err) 133616962b24SJohn Fastabend goto out; 133716962b24SJohn Fastabend 133816962b24SJohn Fastabend /* Tests interesting combinations of APIs used together */ 133916962b24SJohn Fastabend err = test_mixed(cg_fd); 134016962b24SJohn Fastabend if (err) 134116962b24SJohn Fastabend goto out; 134216962b24SJohn Fastabend 134316962b24SJohn Fastabend /* Tests pull_data API using start/end API */ 134416962b24SJohn Fastabend err = test_start_end(cg_fd); 134516962b24SJohn Fastabend if (err) 134616962b24SJohn Fastabend goto out; 134716962b24SJohn Fastabend 134816962b24SJohn Fastabend out: 134916962b24SJohn Fastabend printf("Summary: %i PASSED %i FAILED\n", passed, failed); 1350b8b394faSJohn Fastabend cleanup_cgroup_environment(); 135116962b24SJohn Fastabend close(cg_fd); 135216962b24SJohn Fastabend return err; 135316962b24SJohn Fastabend } 135416962b24SJohn Fastabend 1355b8b394faSJohn Fastabend static int test_suite(void) 1356b8b394faSJohn Fastabend { 1357b8b394faSJohn Fastabend int err; 1358b8b394faSJohn Fastabend 1359b8b394faSJohn Fastabend err = __test_suite(BPF_SOCKMAP_FILENAME); 1360b8b394faSJohn Fastabend if (err) 1361b8b394faSJohn Fastabend goto out; 1362b8b394faSJohn Fastabend err = __test_suite(BPF_SOCKHASH_FILENAME); 1363b8b394faSJohn Fastabend out: 1364b8b394faSJohn Fastabend return err; 1365b8b394faSJohn Fastabend } 1366b8b394faSJohn Fastabend 136716962b24SJohn Fastabend int main(int argc, char **argv) 136816962b24SJohn Fastabend { 136916962b24SJohn Fastabend struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY}; 137016962b24SJohn Fastabend int iov_count = 1, length = 1024, rate = 1; 137116962b24SJohn Fastabend struct sockmap_options options = {0}; 137216962b24SJohn Fastabend int opt, longindex, err, cg_fd = 0; 1373b8b394faSJohn Fastabend char *bpf_file = BPF_SOCKMAP_FILENAME; 137416962b24SJohn Fastabend int test = PING_PONG; 137516962b24SJohn Fastabend 137616962b24SJohn Fastabend if (setrlimit(RLIMIT_MEMLOCK, &r)) { 137716962b24SJohn Fastabend perror("setrlimit(RLIMIT_MEMLOCK)"); 137816962b24SJohn Fastabend return 1; 137916962b24SJohn Fastabend } 138016962b24SJohn Fastabend 138116962b24SJohn Fastabend if (argc < 2) 138216962b24SJohn Fastabend return test_suite(); 138316962b24SJohn Fastabend 138416962b24SJohn Fastabend while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:", 138516962b24SJohn Fastabend long_options, &longindex)) != -1) { 138616962b24SJohn Fastabend switch (opt) { 138716962b24SJohn Fastabend case 's': 138816962b24SJohn Fastabend txmsg_start = atoi(optarg); 138916962b24SJohn Fastabend break; 139016962b24SJohn Fastabend case 'e': 139116962b24SJohn Fastabend txmsg_end = atoi(optarg); 139216962b24SJohn Fastabend break; 139316962b24SJohn Fastabend case 'a': 139416962b24SJohn Fastabend txmsg_apply = atoi(optarg); 139516962b24SJohn Fastabend break; 139616962b24SJohn Fastabend case 'k': 139716962b24SJohn Fastabend txmsg_cork = atoi(optarg); 139816962b24SJohn Fastabend break; 139916962b24SJohn Fastabend case 'c': 140016962b24SJohn Fastabend cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 140116962b24SJohn Fastabend if (cg_fd < 0) { 140216962b24SJohn Fastabend fprintf(stderr, 140316962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 140416962b24SJohn Fastabend cg_fd, optarg); 140516962b24SJohn Fastabend return cg_fd; 140616962b24SJohn Fastabend } 140716962b24SJohn Fastabend break; 140816962b24SJohn Fastabend case 'r': 140916962b24SJohn Fastabend rate = atoi(optarg); 141016962b24SJohn Fastabend break; 141116962b24SJohn Fastabend case 'v': 141216962b24SJohn Fastabend options.verbose = 1; 141316962b24SJohn Fastabend break; 141416962b24SJohn Fastabend case 'i': 141516962b24SJohn Fastabend iov_count = atoi(optarg); 141616962b24SJohn Fastabend break; 141716962b24SJohn Fastabend case 'l': 141816962b24SJohn Fastabend length = atoi(optarg); 141916962b24SJohn Fastabend break; 142016962b24SJohn Fastabend case 'd': 142116962b24SJohn Fastabend options.data_test = true; 142216962b24SJohn Fastabend break; 142316962b24SJohn Fastabend case 't': 142416962b24SJohn Fastabend if (strcmp(optarg, "ping") == 0) { 142516962b24SJohn Fastabend test = PING_PONG; 142616962b24SJohn Fastabend } else if (strcmp(optarg, "sendmsg") == 0) { 142716962b24SJohn Fastabend test = SENDMSG; 142816962b24SJohn Fastabend } else if (strcmp(optarg, "base") == 0) { 142916962b24SJohn Fastabend test = BASE; 143016962b24SJohn Fastabend } else if (strcmp(optarg, "base_sendpage") == 0) { 143116962b24SJohn Fastabend test = BASE_SENDPAGE; 143216962b24SJohn Fastabend } else if (strcmp(optarg, "sendpage") == 0) { 143316962b24SJohn Fastabend test = SENDPAGE; 143416962b24SJohn Fastabend } else { 143516962b24SJohn Fastabend usage(argv); 143616962b24SJohn Fastabend return -1; 143716962b24SJohn Fastabend } 143816962b24SJohn Fastabend break; 143916962b24SJohn Fastabend case 0: 144016962b24SJohn Fastabend break; 144116962b24SJohn Fastabend case 'h': 144216962b24SJohn Fastabend default: 144316962b24SJohn Fastabend usage(argv); 144416962b24SJohn Fastabend return -1; 144516962b24SJohn Fastabend } 144616962b24SJohn Fastabend } 144716962b24SJohn Fastabend 144816962b24SJohn Fastabend if (!cg_fd) { 144916962b24SJohn Fastabend fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n", 145016962b24SJohn Fastabend argv[0]); 145116962b24SJohn Fastabend return -1; 145216962b24SJohn Fastabend } 145316962b24SJohn Fastabend 1454b8b394faSJohn Fastabend err = populate_progs(bpf_file); 145516962b24SJohn Fastabend if (err) { 145616962b24SJohn Fastabend fprintf(stderr, "populate program: (%s) %s\n", 145716962b24SJohn Fastabend bpf_file, strerror(errno)); 145816962b24SJohn Fastabend return 1; 145916962b24SJohn Fastabend } 146016962b24SJohn Fastabend running = 1; 146116962b24SJohn Fastabend 146216962b24SJohn Fastabend /* catch SIGINT */ 146316962b24SJohn Fastabend signal(SIGINT, running_handler); 146416962b24SJohn Fastabend 146516962b24SJohn Fastabend options.iov_count = iov_count; 146616962b24SJohn Fastabend options.iov_length = length; 146716962b24SJohn Fastabend options.rate = rate; 146816962b24SJohn Fastabend 146916962b24SJohn Fastabend err = run_options(&options, cg_fd, test); 147016962b24SJohn Fastabend close(cg_fd); 147116962b24SJohn Fastabend return err; 147216962b24SJohn Fastabend } 147316962b24SJohn Fastabend 147416962b24SJohn Fastabend void running_handler(int a) 147516962b24SJohn Fastabend { 147616962b24SJohn Fastabend running = 0; 147716962b24SJohn Fastabend } 1478