116962b24SJohn Fastabend // SPDX-License-Identifier: GPL-2.0 216962b24SJohn Fastabend // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 316962b24SJohn Fastabend #include <stdio.h> 416962b24SJohn Fastabend #include <stdlib.h> 516962b24SJohn Fastabend #include <sys/socket.h> 616962b24SJohn Fastabend #include <sys/ioctl.h> 716962b24SJohn Fastabend #include <sys/select.h> 816962b24SJohn Fastabend #include <netinet/in.h> 916962b24SJohn Fastabend #include <arpa/inet.h> 1016962b24SJohn Fastabend #include <unistd.h> 1116962b24SJohn Fastabend #include <string.h> 1216962b24SJohn Fastabend #include <errno.h> 1316962b24SJohn Fastabend #include <sys/ioctl.h> 1416962b24SJohn Fastabend #include <stdbool.h> 1516962b24SJohn Fastabend #include <signal.h> 1616962b24SJohn Fastabend #include <fcntl.h> 1716962b24SJohn Fastabend #include <sys/wait.h> 1816962b24SJohn Fastabend #include <time.h> 1916962b24SJohn Fastabend #include <sched.h> 2016962b24SJohn Fastabend 2116962b24SJohn Fastabend #include <sys/time.h> 2216962b24SJohn Fastabend #include <sys/resource.h> 2316962b24SJohn Fastabend #include <sys/types.h> 2416962b24SJohn Fastabend #include <sys/sendfile.h> 2516962b24SJohn Fastabend 2616962b24SJohn Fastabend #include <linux/netlink.h> 2716962b24SJohn Fastabend #include <linux/socket.h> 2816962b24SJohn Fastabend #include <linux/sock_diag.h> 2916962b24SJohn Fastabend #include <linux/bpf.h> 3016962b24SJohn Fastabend #include <linux/if_link.h> 3116962b24SJohn Fastabend #include <assert.h> 3216962b24SJohn Fastabend #include <libgen.h> 3316962b24SJohn Fastabend 3416962b24SJohn Fastabend #include <getopt.h> 3516962b24SJohn Fastabend 3616962b24SJohn Fastabend #include <bpf/bpf.h> 3716962b24SJohn Fastabend #include <bpf/libbpf.h> 3816962b24SJohn Fastabend 3916962b24SJohn Fastabend #include "bpf_util.h" 4016962b24SJohn Fastabend #include "bpf_rlimit.h" 4116962b24SJohn Fastabend #include "cgroup_helpers.h" 4216962b24SJohn Fastabend 4316962b24SJohn Fastabend int running; 4416962b24SJohn Fastabend static void running_handler(int a); 4516962b24SJohn Fastabend 4616962b24SJohn Fastabend /* randomly selected ports for testing on lo */ 4716962b24SJohn Fastabend #define S1_PORT 10000 4816962b24SJohn Fastabend #define S2_PORT 10001 4916962b24SJohn Fastabend 50b8b394faSJohn Fastabend #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 51b8b394faSJohn Fastabend #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 5216962b24SJohn Fastabend #define CG_PATH "/sockmap" 5316962b24SJohn Fastabend 5416962b24SJohn Fastabend /* global sockets */ 5516962b24SJohn Fastabend int s1, s2, c1, c2, p1, p2; 5616962b24SJohn Fastabend int test_cnt; 5716962b24SJohn Fastabend int passed; 5816962b24SJohn Fastabend int failed; 5916962b24SJohn Fastabend int map_fd[8]; 6016962b24SJohn Fastabend struct bpf_map *maps[8]; 6116962b24SJohn Fastabend int prog_fd[11]; 6216962b24SJohn Fastabend 6316962b24SJohn Fastabend int txmsg_pass; 6416962b24SJohn Fastabend int txmsg_noisy; 6516962b24SJohn Fastabend int txmsg_redir; 6616962b24SJohn Fastabend int txmsg_redir_noisy; 6716962b24SJohn Fastabend int txmsg_drop; 6816962b24SJohn Fastabend int txmsg_apply; 6916962b24SJohn Fastabend int txmsg_cork; 7016962b24SJohn Fastabend int txmsg_start; 7116962b24SJohn Fastabend int txmsg_end; 7216962b24SJohn Fastabend int txmsg_ingress; 7316962b24SJohn Fastabend int txmsg_skb; 7416962b24SJohn Fastabend 7516962b24SJohn Fastabend static const struct option long_options[] = { 7616962b24SJohn Fastabend {"help", no_argument, NULL, 'h' }, 7716962b24SJohn Fastabend {"cgroup", required_argument, NULL, 'c' }, 7816962b24SJohn Fastabend {"rate", required_argument, NULL, 'r' }, 7916962b24SJohn Fastabend {"verbose", no_argument, NULL, 'v' }, 8016962b24SJohn Fastabend {"iov_count", required_argument, NULL, 'i' }, 8116962b24SJohn Fastabend {"length", required_argument, NULL, 'l' }, 8216962b24SJohn Fastabend {"test", required_argument, NULL, 't' }, 8316962b24SJohn Fastabend {"data_test", no_argument, NULL, 'd' }, 8416962b24SJohn Fastabend {"txmsg", no_argument, &txmsg_pass, 1 }, 8516962b24SJohn Fastabend {"txmsg_noisy", no_argument, &txmsg_noisy, 1 }, 8616962b24SJohn Fastabend {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 8716962b24SJohn Fastabend {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1}, 8816962b24SJohn Fastabend {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 8916962b24SJohn Fastabend {"txmsg_apply", required_argument, NULL, 'a'}, 9016962b24SJohn Fastabend {"txmsg_cork", required_argument, NULL, 'k'}, 9116962b24SJohn Fastabend {"txmsg_start", required_argument, NULL, 's'}, 9216962b24SJohn Fastabend {"txmsg_end", required_argument, NULL, 'e'}, 9316962b24SJohn Fastabend {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 9416962b24SJohn Fastabend {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 9516962b24SJohn Fastabend {0, 0, NULL, 0 } 9616962b24SJohn Fastabend }; 9716962b24SJohn Fastabend 9816962b24SJohn Fastabend static void usage(char *argv[]) 9916962b24SJohn Fastabend { 10016962b24SJohn Fastabend int i; 10116962b24SJohn Fastabend 10216962b24SJohn Fastabend printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 10316962b24SJohn Fastabend printf(" options:\n"); 10416962b24SJohn Fastabend for (i = 0; long_options[i].name != 0; i++) { 10516962b24SJohn Fastabend printf(" --%-12s", long_options[i].name); 10616962b24SJohn Fastabend if (long_options[i].flag != NULL) 10716962b24SJohn Fastabend printf(" flag (internal value:%d)\n", 10816962b24SJohn Fastabend *long_options[i].flag); 10916962b24SJohn Fastabend else 11016962b24SJohn Fastabend printf(" -%c\n", long_options[i].val); 11116962b24SJohn Fastabend } 11216962b24SJohn Fastabend printf("\n"); 11316962b24SJohn Fastabend } 11416962b24SJohn Fastabend 11516962b24SJohn Fastabend static int sockmap_init_sockets(int verbose) 11616962b24SJohn Fastabend { 11716962b24SJohn Fastabend int i, err, one = 1; 11816962b24SJohn Fastabend struct sockaddr_in addr; 11916962b24SJohn Fastabend int *fds[4] = {&s1, &s2, &c1, &c2}; 12016962b24SJohn Fastabend 12116962b24SJohn Fastabend s1 = s2 = p1 = p2 = c1 = c2 = 0; 12216962b24SJohn Fastabend 12316962b24SJohn Fastabend /* Init sockets */ 12416962b24SJohn Fastabend for (i = 0; i < 4; i++) { 12516962b24SJohn Fastabend *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 12616962b24SJohn Fastabend if (*fds[i] < 0) { 12716962b24SJohn Fastabend perror("socket s1 failed()"); 12816962b24SJohn Fastabend return errno; 12916962b24SJohn Fastabend } 13016962b24SJohn Fastabend } 13116962b24SJohn Fastabend 13216962b24SJohn Fastabend /* Allow reuse */ 13316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 13416962b24SJohn Fastabend err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 13516962b24SJohn Fastabend (char *)&one, sizeof(one)); 13616962b24SJohn Fastabend if (err) { 13716962b24SJohn Fastabend perror("setsockopt failed()"); 13816962b24SJohn Fastabend return errno; 13916962b24SJohn Fastabend } 14016962b24SJohn Fastabend } 14116962b24SJohn Fastabend 14216962b24SJohn Fastabend /* Non-blocking sockets */ 14316962b24SJohn Fastabend for (i = 0; i < 2; i++) { 14416962b24SJohn Fastabend err = ioctl(*fds[i], FIONBIO, (char *)&one); 14516962b24SJohn Fastabend if (err < 0) { 14616962b24SJohn Fastabend perror("ioctl s1 failed()"); 14716962b24SJohn Fastabend return errno; 14816962b24SJohn Fastabend } 14916962b24SJohn Fastabend } 15016962b24SJohn Fastabend 15116962b24SJohn Fastabend /* Bind server sockets */ 15216962b24SJohn Fastabend memset(&addr, 0, sizeof(struct sockaddr_in)); 15316962b24SJohn Fastabend addr.sin_family = AF_INET; 15416962b24SJohn Fastabend addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 15516962b24SJohn Fastabend 15616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 15716962b24SJohn Fastabend err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 15816962b24SJohn Fastabend if (err < 0) { 15916962b24SJohn Fastabend perror("bind s1 failed()\n"); 16016962b24SJohn Fastabend return errno; 16116962b24SJohn Fastabend } 16216962b24SJohn Fastabend 16316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 16416962b24SJohn Fastabend err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 16516962b24SJohn Fastabend if (err < 0) { 16616962b24SJohn Fastabend perror("bind s2 failed()\n"); 16716962b24SJohn Fastabend return errno; 16816962b24SJohn Fastabend } 16916962b24SJohn Fastabend 17016962b24SJohn Fastabend /* Listen server sockets */ 17116962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 17216962b24SJohn Fastabend err = listen(s1, 32); 17316962b24SJohn Fastabend if (err < 0) { 17416962b24SJohn Fastabend perror("listen s1 failed()\n"); 17516962b24SJohn Fastabend return errno; 17616962b24SJohn Fastabend } 17716962b24SJohn Fastabend 17816962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 17916962b24SJohn Fastabend err = listen(s2, 32); 18016962b24SJohn Fastabend if (err < 0) { 18116962b24SJohn Fastabend perror("listen s1 failed()\n"); 18216962b24SJohn Fastabend return errno; 18316962b24SJohn Fastabend } 18416962b24SJohn Fastabend 18516962b24SJohn Fastabend /* Initiate Connect */ 18616962b24SJohn Fastabend addr.sin_port = htons(S1_PORT); 18716962b24SJohn Fastabend err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 18816962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 18916962b24SJohn Fastabend perror("connect c1 failed()\n"); 19016962b24SJohn Fastabend return errno; 19116962b24SJohn Fastabend } 19216962b24SJohn Fastabend 19316962b24SJohn Fastabend addr.sin_port = htons(S2_PORT); 19416962b24SJohn Fastabend err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 19516962b24SJohn Fastabend if (err < 0 && errno != EINPROGRESS) { 19616962b24SJohn Fastabend perror("connect c2 failed()\n"); 19716962b24SJohn Fastabend return errno; 19816962b24SJohn Fastabend } else if (err < 0) { 19916962b24SJohn Fastabend err = 0; 20016962b24SJohn Fastabend } 20116962b24SJohn Fastabend 20216962b24SJohn Fastabend /* Accept Connecrtions */ 20316962b24SJohn Fastabend p1 = accept(s1, NULL, NULL); 20416962b24SJohn Fastabend if (p1 < 0) { 20516962b24SJohn Fastabend perror("accept s1 failed()\n"); 20616962b24SJohn Fastabend return errno; 20716962b24SJohn Fastabend } 20816962b24SJohn Fastabend 20916962b24SJohn Fastabend p2 = accept(s2, NULL, NULL); 21016962b24SJohn Fastabend if (p2 < 0) { 21116962b24SJohn Fastabend perror("accept s1 failed()\n"); 21216962b24SJohn Fastabend return errno; 21316962b24SJohn Fastabend } 21416962b24SJohn Fastabend 21516962b24SJohn Fastabend if (verbose) { 21616962b24SJohn Fastabend printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 21716962b24SJohn Fastabend printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 21816962b24SJohn Fastabend c1, s1, c2, s2); 21916962b24SJohn Fastabend } 22016962b24SJohn Fastabend return 0; 22116962b24SJohn Fastabend } 22216962b24SJohn Fastabend 22316962b24SJohn Fastabend struct msg_stats { 22416962b24SJohn Fastabend size_t bytes_sent; 22516962b24SJohn Fastabend size_t bytes_recvd; 22616962b24SJohn Fastabend struct timespec start; 22716962b24SJohn Fastabend struct timespec end; 22816962b24SJohn Fastabend }; 22916962b24SJohn Fastabend 23016962b24SJohn Fastabend struct sockmap_options { 23116962b24SJohn Fastabend int verbose; 23216962b24SJohn Fastabend bool base; 23316962b24SJohn Fastabend bool sendpage; 23416962b24SJohn Fastabend bool data_test; 23516962b24SJohn Fastabend bool drop_expected; 23616962b24SJohn Fastabend int iov_count; 23716962b24SJohn Fastabend int iov_length; 23816962b24SJohn Fastabend int rate; 23916962b24SJohn Fastabend }; 24016962b24SJohn Fastabend 24116962b24SJohn Fastabend static int msg_loop_sendpage(int fd, int iov_length, int cnt, 24216962b24SJohn Fastabend struct msg_stats *s, 24316962b24SJohn Fastabend struct sockmap_options *opt) 24416962b24SJohn Fastabend { 24516962b24SJohn Fastabend bool drop = opt->drop_expected; 24616962b24SJohn Fastabend unsigned char k = 0; 24716962b24SJohn Fastabend FILE *file; 24816962b24SJohn Fastabend int i, fp; 24916962b24SJohn Fastabend 25016962b24SJohn Fastabend file = fopen(".sendpage_tst.tmp", "w+"); 25116962b24SJohn Fastabend for (i = 0; i < iov_length * cnt; i++, k++) 25216962b24SJohn Fastabend fwrite(&k, sizeof(char), 1, file); 25316962b24SJohn Fastabend fflush(file); 25416962b24SJohn Fastabend fseek(file, 0, SEEK_SET); 25516962b24SJohn Fastabend fclose(file); 25616962b24SJohn Fastabend 25716962b24SJohn Fastabend fp = open(".sendpage_tst.tmp", O_RDONLY); 25816962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 25916962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 26016962b24SJohn Fastabend int sent = sendfile(fd, fp, NULL, iov_length); 26116962b24SJohn Fastabend 26216962b24SJohn Fastabend if (!drop && sent < 0) { 26316962b24SJohn Fastabend perror("send loop error:"); 26416962b24SJohn Fastabend close(fp); 26516962b24SJohn Fastabend return sent; 26616962b24SJohn Fastabend } else if (drop && sent >= 0) { 26716962b24SJohn Fastabend printf("sendpage loop error expected: %i\n", sent); 26816962b24SJohn Fastabend close(fp); 26916962b24SJohn Fastabend return -EIO; 27016962b24SJohn Fastabend } 27116962b24SJohn Fastabend 27216962b24SJohn Fastabend if (sent > 0) 27316962b24SJohn Fastabend s->bytes_sent += sent; 27416962b24SJohn Fastabend } 27516962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 27616962b24SJohn Fastabend close(fp); 27716962b24SJohn Fastabend return 0; 27816962b24SJohn Fastabend } 27916962b24SJohn Fastabend 28016962b24SJohn Fastabend static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 28116962b24SJohn Fastabend struct msg_stats *s, bool tx, 28216962b24SJohn Fastabend struct sockmap_options *opt) 28316962b24SJohn Fastabend { 28416962b24SJohn Fastabend struct msghdr msg = {0}; 28516962b24SJohn Fastabend int err, i, flags = MSG_NOSIGNAL; 28616962b24SJohn Fastabend struct iovec *iov; 28716962b24SJohn Fastabend unsigned char k; 28816962b24SJohn Fastabend bool data_test = opt->data_test; 28916962b24SJohn Fastabend bool drop = opt->drop_expected; 29016962b24SJohn Fastabend 29116962b24SJohn Fastabend iov = calloc(iov_count, sizeof(struct iovec)); 29216962b24SJohn Fastabend if (!iov) 29316962b24SJohn Fastabend return errno; 29416962b24SJohn Fastabend 29516962b24SJohn Fastabend k = 0; 29616962b24SJohn Fastabend for (i = 0; i < iov_count; i++) { 29716962b24SJohn Fastabend unsigned char *d = calloc(iov_length, sizeof(char)); 29816962b24SJohn Fastabend 29916962b24SJohn Fastabend if (!d) { 30016962b24SJohn Fastabend fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 30116962b24SJohn Fastabend goto out_errno; 30216962b24SJohn Fastabend } 30316962b24SJohn Fastabend iov[i].iov_base = d; 30416962b24SJohn Fastabend iov[i].iov_len = iov_length; 30516962b24SJohn Fastabend 30616962b24SJohn Fastabend if (data_test && tx) { 30716962b24SJohn Fastabend int j; 30816962b24SJohn Fastabend 30916962b24SJohn Fastabend for (j = 0; j < iov_length; j++) 31016962b24SJohn Fastabend d[j] = k++; 31116962b24SJohn Fastabend } 31216962b24SJohn Fastabend } 31316962b24SJohn Fastabend 31416962b24SJohn Fastabend msg.msg_iov = iov; 31516962b24SJohn Fastabend msg.msg_iovlen = iov_count; 31616962b24SJohn Fastabend k = 0; 31716962b24SJohn Fastabend 31816962b24SJohn Fastabend if (tx) { 31916962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->start); 32016962b24SJohn Fastabend for (i = 0; i < cnt; i++) { 32116962b24SJohn Fastabend int sent = sendmsg(fd, &msg, flags); 32216962b24SJohn Fastabend 32316962b24SJohn Fastabend if (!drop && sent < 0) { 32416962b24SJohn Fastabend perror("send loop error:"); 32516962b24SJohn Fastabend goto out_errno; 32616962b24SJohn Fastabend } else if (drop && sent >= 0) { 32716962b24SJohn Fastabend printf("send loop error expected: %i\n", sent); 32816962b24SJohn Fastabend errno = -EIO; 32916962b24SJohn Fastabend goto out_errno; 33016962b24SJohn Fastabend } 33116962b24SJohn Fastabend if (sent > 0) 33216962b24SJohn Fastabend s->bytes_sent += sent; 33316962b24SJohn Fastabend } 33416962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 33516962b24SJohn Fastabend } else { 33616962b24SJohn Fastabend int slct, recv, max_fd = fd; 33716962b24SJohn Fastabend int fd_flags = O_NONBLOCK; 33816962b24SJohn Fastabend struct timeval timeout; 33916962b24SJohn Fastabend float total_bytes; 340d825e12fSPrashant Bhole int bytes_cnt = 0; 341d825e12fSPrashant Bhole int chunk_sz; 34216962b24SJohn Fastabend fd_set w; 34316962b24SJohn Fastabend 344d825e12fSPrashant Bhole if (opt->sendpage) 345d825e12fSPrashant Bhole chunk_sz = iov_length * cnt; 346d825e12fSPrashant Bhole else 347d825e12fSPrashant Bhole chunk_sz = iov_length * iov_count; 348d825e12fSPrashant Bhole 34916962b24SJohn Fastabend fcntl(fd, fd_flags); 35016962b24SJohn Fastabend total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 35116962b24SJohn Fastabend err = clock_gettime(CLOCK_MONOTONIC, &s->start); 35216962b24SJohn Fastabend if (err < 0) 35316962b24SJohn Fastabend perror("recv start time: "); 35416962b24SJohn Fastabend while (s->bytes_recvd < total_bytes) { 355a009f1f3SPrashant Bhole if (txmsg_cork) { 356a18fda1aSJohn Fastabend timeout.tv_sec = 0; 357*3c6ed988SDaniel Borkmann timeout.tv_usec = 300000; 358a009f1f3SPrashant Bhole } else { 359a009f1f3SPrashant Bhole timeout.tv_sec = 1; 360a009f1f3SPrashant Bhole timeout.tv_usec = 0; 361a009f1f3SPrashant Bhole } 36216962b24SJohn Fastabend 36316962b24SJohn Fastabend /* FD sets */ 36416962b24SJohn Fastabend FD_ZERO(&w); 36516962b24SJohn Fastabend FD_SET(fd, &w); 36616962b24SJohn Fastabend 36716962b24SJohn Fastabend slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 36816962b24SJohn Fastabend if (slct == -1) { 36916962b24SJohn Fastabend perror("select()"); 37016962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 37116962b24SJohn Fastabend goto out_errno; 37216962b24SJohn Fastabend } else if (!slct) { 37316962b24SJohn Fastabend if (opt->verbose) 37416962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 37516962b24SJohn Fastabend errno = -EIO; 37616962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 37716962b24SJohn Fastabend goto out_errno; 37816962b24SJohn Fastabend } 37916962b24SJohn Fastabend 38016962b24SJohn Fastabend recv = recvmsg(fd, &msg, flags); 38116962b24SJohn Fastabend if (recv < 0) { 38216962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 38316962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 38416962b24SJohn Fastabend perror("recv failed()\n"); 38516962b24SJohn Fastabend goto out_errno; 38616962b24SJohn Fastabend } 38716962b24SJohn Fastabend } 38816962b24SJohn Fastabend 38916962b24SJohn Fastabend s->bytes_recvd += recv; 39016962b24SJohn Fastabend 39116962b24SJohn Fastabend if (data_test) { 39216962b24SJohn Fastabend int j; 39316962b24SJohn Fastabend 39416962b24SJohn Fastabend for (i = 0; i < msg.msg_iovlen; i++) { 39516962b24SJohn Fastabend unsigned char *d = iov[i].iov_base; 39616962b24SJohn Fastabend 39716962b24SJohn Fastabend for (j = 0; 39816962b24SJohn Fastabend j < iov[i].iov_len && recv; j++) { 39916962b24SJohn Fastabend if (d[j] != k++) { 40016962b24SJohn Fastabend errno = -EIO; 40116962b24SJohn Fastabend fprintf(stderr, 40216962b24SJohn Fastabend "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 403d825e12fSPrashant Bhole i, j, d[j], k - 1, d[j+1], k); 40416962b24SJohn Fastabend goto out_errno; 40516962b24SJohn Fastabend } 406d825e12fSPrashant Bhole bytes_cnt++; 407d825e12fSPrashant Bhole if (bytes_cnt == chunk_sz) { 408d825e12fSPrashant Bhole k = 0; 409d825e12fSPrashant Bhole bytes_cnt = 0; 410d825e12fSPrashant Bhole } 41116962b24SJohn Fastabend recv--; 41216962b24SJohn Fastabend } 41316962b24SJohn Fastabend } 41416962b24SJohn Fastabend } 41516962b24SJohn Fastabend } 41616962b24SJohn Fastabend clock_gettime(CLOCK_MONOTONIC, &s->end); 41716962b24SJohn Fastabend } 41816962b24SJohn Fastabend 41916962b24SJohn Fastabend for (i = 0; i < iov_count; i++) 42016962b24SJohn Fastabend free(iov[i].iov_base); 42116962b24SJohn Fastabend free(iov); 42216962b24SJohn Fastabend return 0; 42316962b24SJohn Fastabend out_errno: 42416962b24SJohn Fastabend for (i = 0; i < iov_count; i++) 42516962b24SJohn Fastabend free(iov[i].iov_base); 42616962b24SJohn Fastabend free(iov); 42716962b24SJohn Fastabend return errno; 42816962b24SJohn Fastabend } 42916962b24SJohn Fastabend 43016962b24SJohn Fastabend static float giga = 1000000000; 43116962b24SJohn Fastabend 43216962b24SJohn Fastabend static inline float sentBps(struct msg_stats s) 43316962b24SJohn Fastabend { 43416962b24SJohn Fastabend return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 43516962b24SJohn Fastabend } 43616962b24SJohn Fastabend 43716962b24SJohn Fastabend static inline float recvdBps(struct msg_stats s) 43816962b24SJohn Fastabend { 43916962b24SJohn Fastabend return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 44016962b24SJohn Fastabend } 44116962b24SJohn Fastabend 44216962b24SJohn Fastabend static int sendmsg_test(struct sockmap_options *opt) 44316962b24SJohn Fastabend { 44416962b24SJohn Fastabend float sent_Bps = 0, recvd_Bps = 0; 44516962b24SJohn Fastabend int rx_fd, txpid, rxpid, err = 0; 44616962b24SJohn Fastabend struct msg_stats s = {0}; 44716962b24SJohn Fastabend int iov_count = opt->iov_count; 44816962b24SJohn Fastabend int iov_buf = opt->iov_length; 44916edddfeSPrashant Bhole int rx_status, tx_status; 45016962b24SJohn Fastabend int cnt = opt->rate; 45116962b24SJohn Fastabend 45216962b24SJohn Fastabend errno = 0; 45316962b24SJohn Fastabend 45416962b24SJohn Fastabend if (opt->base) 45516962b24SJohn Fastabend rx_fd = p1; 45616962b24SJohn Fastabend else 45716962b24SJohn Fastabend rx_fd = p2; 45816962b24SJohn Fastabend 45916962b24SJohn Fastabend rxpid = fork(); 46016962b24SJohn Fastabend if (rxpid == 0) { 46116962b24SJohn Fastabend if (opt->drop_expected) 46216edddfeSPrashant Bhole exit(0); 46316962b24SJohn Fastabend 46416962b24SJohn Fastabend if (opt->sendpage) 46516962b24SJohn Fastabend iov_count = 1; 46616962b24SJohn Fastabend err = msg_loop(rx_fd, iov_count, iov_buf, 46716962b24SJohn Fastabend cnt, &s, false, opt); 46816962b24SJohn Fastabend if (err && opt->verbose) 46916962b24SJohn Fastabend fprintf(stderr, 47016962b24SJohn Fastabend "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 47116962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 47216962b24SJohn Fastabend shutdown(p2, SHUT_RDWR); 47316962b24SJohn Fastabend shutdown(p1, SHUT_RDWR); 47416962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 47516962b24SJohn Fastabend sent_Bps = sentBps(s); 47616962b24SJohn Fastabend recvd_Bps = recvdBps(s); 47716962b24SJohn Fastabend } 47816962b24SJohn Fastabend if (opt->verbose) 47916962b24SJohn Fastabend fprintf(stdout, 48016962b24SJohn Fastabend "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n", 48116962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 48216962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 48316edddfeSPrashant Bhole if (err && txmsg_cork) 48416edddfeSPrashant Bhole err = 0; 48516edddfeSPrashant Bhole exit(err ? 1 : 0); 48616962b24SJohn Fastabend } else if (rxpid == -1) { 48716962b24SJohn Fastabend perror("msg_loop_rx: "); 48816962b24SJohn Fastabend return errno; 48916962b24SJohn Fastabend } 49016962b24SJohn Fastabend 49116962b24SJohn Fastabend txpid = fork(); 49216962b24SJohn Fastabend if (txpid == 0) { 49316962b24SJohn Fastabend if (opt->sendpage) 49416962b24SJohn Fastabend err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 49516962b24SJohn Fastabend else 49616962b24SJohn Fastabend err = msg_loop(c1, iov_count, iov_buf, 49716962b24SJohn Fastabend cnt, &s, true, opt); 49816962b24SJohn Fastabend 49916962b24SJohn Fastabend if (err) 50016962b24SJohn Fastabend fprintf(stderr, 50116962b24SJohn Fastabend "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 50216962b24SJohn Fastabend iov_count, iov_buf, cnt, err); 50316962b24SJohn Fastabend shutdown(c1, SHUT_RDWR); 50416962b24SJohn Fastabend if (s.end.tv_sec - s.start.tv_sec) { 50516962b24SJohn Fastabend sent_Bps = sentBps(s); 50616962b24SJohn Fastabend recvd_Bps = recvdBps(s); 50716962b24SJohn Fastabend } 50816962b24SJohn Fastabend if (opt->verbose) 50916962b24SJohn Fastabend fprintf(stdout, 51016962b24SJohn Fastabend "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 51116962b24SJohn Fastabend s.bytes_sent, sent_Bps, sent_Bps/giga, 51216962b24SJohn Fastabend s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 51316edddfeSPrashant Bhole exit(err ? 1 : 0); 51416962b24SJohn Fastabend } else if (txpid == -1) { 51516962b24SJohn Fastabend perror("msg_loop_tx: "); 51616962b24SJohn Fastabend return errno; 51716962b24SJohn Fastabend } 51816962b24SJohn Fastabend 51916edddfeSPrashant Bhole assert(waitpid(rxpid, &rx_status, 0) == rxpid); 52016edddfeSPrashant Bhole assert(waitpid(txpid, &tx_status, 0) == txpid); 52116edddfeSPrashant Bhole if (WIFEXITED(rx_status)) { 52216edddfeSPrashant Bhole err = WEXITSTATUS(rx_status); 52316edddfeSPrashant Bhole if (err) { 52416edddfeSPrashant Bhole fprintf(stderr, "rx thread exited with err %d. ", err); 52516edddfeSPrashant Bhole goto out; 52616edddfeSPrashant Bhole } 52716edddfeSPrashant Bhole } 52816edddfeSPrashant Bhole if (WIFEXITED(tx_status)) { 52916edddfeSPrashant Bhole err = WEXITSTATUS(tx_status); 53016edddfeSPrashant Bhole if (err) 53116edddfeSPrashant Bhole fprintf(stderr, "tx thread exited with err %d. ", err); 53216edddfeSPrashant Bhole } 53316edddfeSPrashant Bhole out: 53416962b24SJohn Fastabend return err; 53516962b24SJohn Fastabend } 53616962b24SJohn Fastabend 53716962b24SJohn Fastabend static int forever_ping_pong(int rate, struct sockmap_options *opt) 53816962b24SJohn Fastabend { 53916962b24SJohn Fastabend struct timeval timeout; 54016962b24SJohn Fastabend char buf[1024] = {0}; 54116962b24SJohn Fastabend int sc; 54216962b24SJohn Fastabend 54316962b24SJohn Fastabend timeout.tv_sec = 10; 54416962b24SJohn Fastabend timeout.tv_usec = 0; 54516962b24SJohn Fastabend 54616962b24SJohn Fastabend /* Ping/Pong data from client to server */ 54716962b24SJohn Fastabend sc = send(c1, buf, sizeof(buf), 0); 54816962b24SJohn Fastabend if (sc < 0) { 54916962b24SJohn Fastabend perror("send failed()\n"); 55016962b24SJohn Fastabend return sc; 55116962b24SJohn Fastabend } 55216962b24SJohn Fastabend 55316962b24SJohn Fastabend do { 55416962b24SJohn Fastabend int s, rc, i, max_fd = p2; 55516962b24SJohn Fastabend fd_set w; 55616962b24SJohn Fastabend 55716962b24SJohn Fastabend /* FD sets */ 55816962b24SJohn Fastabend FD_ZERO(&w); 55916962b24SJohn Fastabend FD_SET(c1, &w); 56016962b24SJohn Fastabend FD_SET(c2, &w); 56116962b24SJohn Fastabend FD_SET(p1, &w); 56216962b24SJohn Fastabend FD_SET(p2, &w); 56316962b24SJohn Fastabend 56416962b24SJohn Fastabend s = select(max_fd + 1, &w, NULL, NULL, &timeout); 56516962b24SJohn Fastabend if (s == -1) { 56616962b24SJohn Fastabend perror("select()"); 56716962b24SJohn Fastabend break; 56816962b24SJohn Fastabend } else if (!s) { 56916962b24SJohn Fastabend fprintf(stderr, "unexpected timeout\n"); 57016962b24SJohn Fastabend break; 57116962b24SJohn Fastabend } 57216962b24SJohn Fastabend 57316962b24SJohn Fastabend for (i = 0; i <= max_fd && s > 0; ++i) { 57416962b24SJohn Fastabend if (!FD_ISSET(i, &w)) 57516962b24SJohn Fastabend continue; 57616962b24SJohn Fastabend 57716962b24SJohn Fastabend s--; 57816962b24SJohn Fastabend 57916962b24SJohn Fastabend rc = recv(i, buf, sizeof(buf), 0); 58016962b24SJohn Fastabend if (rc < 0) { 58116962b24SJohn Fastabend if (errno != EWOULDBLOCK) { 58216962b24SJohn Fastabend perror("recv failed()\n"); 58316962b24SJohn Fastabend return rc; 58416962b24SJohn Fastabend } 58516962b24SJohn Fastabend } 58616962b24SJohn Fastabend 58716962b24SJohn Fastabend if (rc == 0) { 58816962b24SJohn Fastabend close(i); 58916962b24SJohn Fastabend break; 59016962b24SJohn Fastabend } 59116962b24SJohn Fastabend 59216962b24SJohn Fastabend sc = send(i, buf, rc, 0); 59316962b24SJohn Fastabend if (sc < 0) { 59416962b24SJohn Fastabend perror("send failed()\n"); 59516962b24SJohn Fastabend return sc; 59616962b24SJohn Fastabend } 59716962b24SJohn Fastabend } 59816962b24SJohn Fastabend 59916962b24SJohn Fastabend if (rate) 60016962b24SJohn Fastabend sleep(rate); 60116962b24SJohn Fastabend 60216962b24SJohn Fastabend if (opt->verbose) { 60316962b24SJohn Fastabend printf("."); 60416962b24SJohn Fastabend fflush(stdout); 60516962b24SJohn Fastabend 60616962b24SJohn Fastabend } 60716962b24SJohn Fastabend } while (running); 60816962b24SJohn Fastabend 60916962b24SJohn Fastabend return 0; 61016962b24SJohn Fastabend } 61116962b24SJohn Fastabend 61216962b24SJohn Fastabend enum { 61316962b24SJohn Fastabend PING_PONG, 61416962b24SJohn Fastabend SENDMSG, 61516962b24SJohn Fastabend BASE, 61616962b24SJohn Fastabend BASE_SENDPAGE, 61716962b24SJohn Fastabend SENDPAGE, 61816962b24SJohn Fastabend }; 61916962b24SJohn Fastabend 62016962b24SJohn Fastabend static int run_options(struct sockmap_options *options, int cg_fd, int test) 62116962b24SJohn Fastabend { 62216962b24SJohn Fastabend int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 62316962b24SJohn Fastabend 62416962b24SJohn Fastabend /* If base test skip BPF setup */ 62516962b24SJohn Fastabend if (test == BASE || test == BASE_SENDPAGE) 62616962b24SJohn Fastabend goto run; 62716962b24SJohn Fastabend 62816962b24SJohn Fastabend /* Attach programs to sockmap */ 62916962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[0], map_fd[0], 63016962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 0); 63116962b24SJohn Fastabend if (err) { 63216962b24SJohn Fastabend fprintf(stderr, 63316962b24SJohn Fastabend "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 63416962b24SJohn Fastabend prog_fd[0], map_fd[0], err, strerror(errno)); 63516962b24SJohn Fastabend return err; 63616962b24SJohn Fastabend } 63716962b24SJohn Fastabend 63816962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[1], map_fd[0], 63916962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 0); 64016962b24SJohn Fastabend if (err) { 64116962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 64216962b24SJohn Fastabend err, strerror(errno)); 64316962b24SJohn Fastabend return err; 64416962b24SJohn Fastabend } 64516962b24SJohn Fastabend 64616962b24SJohn Fastabend /* Attach to cgroups */ 64716962b24SJohn Fastabend err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 64816962b24SJohn Fastabend if (err) { 64916962b24SJohn Fastabend fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 65016962b24SJohn Fastabend err, strerror(errno)); 65116962b24SJohn Fastabend return err; 65216962b24SJohn Fastabend } 65316962b24SJohn Fastabend 65416962b24SJohn Fastabend run: 65516962b24SJohn Fastabend err = sockmap_init_sockets(options->verbose); 65616962b24SJohn Fastabend if (err) { 65716962b24SJohn Fastabend fprintf(stderr, "ERROR: test socket failed: %d\n", err); 65816962b24SJohn Fastabend goto out; 65916962b24SJohn Fastabend } 66016962b24SJohn Fastabend 66116962b24SJohn Fastabend /* Attach txmsg program to sockmap */ 66216962b24SJohn Fastabend if (txmsg_pass) 66316962b24SJohn Fastabend tx_prog_fd = prog_fd[3]; 66416962b24SJohn Fastabend else if (txmsg_noisy) 66516962b24SJohn Fastabend tx_prog_fd = prog_fd[4]; 66616962b24SJohn Fastabend else if (txmsg_redir) 66716962b24SJohn Fastabend tx_prog_fd = prog_fd[5]; 66816962b24SJohn Fastabend else if (txmsg_redir_noisy) 66916962b24SJohn Fastabend tx_prog_fd = prog_fd[6]; 67016962b24SJohn Fastabend else if (txmsg_drop) 67116962b24SJohn Fastabend tx_prog_fd = prog_fd[9]; 67216962b24SJohn Fastabend /* apply and cork must be last */ 67316962b24SJohn Fastabend else if (txmsg_apply) 67416962b24SJohn Fastabend tx_prog_fd = prog_fd[7]; 67516962b24SJohn Fastabend else if (txmsg_cork) 67616962b24SJohn Fastabend tx_prog_fd = prog_fd[8]; 67716962b24SJohn Fastabend else 67816962b24SJohn Fastabend tx_prog_fd = 0; 67916962b24SJohn Fastabend 68016962b24SJohn Fastabend if (tx_prog_fd) { 68116962b24SJohn Fastabend int redir_fd, i = 0; 68216962b24SJohn Fastabend 68316962b24SJohn Fastabend err = bpf_prog_attach(tx_prog_fd, 68416962b24SJohn Fastabend map_fd[1], BPF_SK_MSG_VERDICT, 0); 68516962b24SJohn Fastabend if (err) { 68616962b24SJohn Fastabend fprintf(stderr, 68716962b24SJohn Fastabend "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 68816962b24SJohn Fastabend err, strerror(errno)); 68916962b24SJohn Fastabend goto out; 69016962b24SJohn Fastabend } 69116962b24SJohn Fastabend 69216962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 69316962b24SJohn Fastabend if (err) { 69416962b24SJohn Fastabend fprintf(stderr, 69516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 69616962b24SJohn Fastabend err, strerror(errno)); 69716962b24SJohn Fastabend goto out; 69816962b24SJohn Fastabend } 69916962b24SJohn Fastabend 70016962b24SJohn Fastabend if (txmsg_redir || txmsg_redir_noisy) 70116962b24SJohn Fastabend redir_fd = c2; 70216962b24SJohn Fastabend else 70316962b24SJohn Fastabend redir_fd = c1; 70416962b24SJohn Fastabend 70516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 70616962b24SJohn Fastabend if (err) { 70716962b24SJohn Fastabend fprintf(stderr, 70816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 70916962b24SJohn Fastabend err, strerror(errno)); 71016962b24SJohn Fastabend goto out; 71116962b24SJohn Fastabend } 71216962b24SJohn Fastabend 71316962b24SJohn Fastabend if (txmsg_apply) { 71416962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[3], 71516962b24SJohn Fastabend &i, &txmsg_apply, BPF_ANY); 71616962b24SJohn Fastabend if (err) { 71716962b24SJohn Fastabend fprintf(stderr, 71816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 71916962b24SJohn Fastabend err, strerror(errno)); 72016962b24SJohn Fastabend goto out; 72116962b24SJohn Fastabend } 72216962b24SJohn Fastabend } 72316962b24SJohn Fastabend 72416962b24SJohn Fastabend if (txmsg_cork) { 72516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[4], 72616962b24SJohn Fastabend &i, &txmsg_cork, BPF_ANY); 72716962b24SJohn Fastabend if (err) { 72816962b24SJohn Fastabend fprintf(stderr, 72916962b24SJohn Fastabend "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 73016962b24SJohn Fastabend err, strerror(errno)); 73116962b24SJohn Fastabend goto out; 73216962b24SJohn Fastabend } 73316962b24SJohn Fastabend } 73416962b24SJohn Fastabend 73516962b24SJohn Fastabend if (txmsg_start) { 73616962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 73716962b24SJohn Fastabend &i, &txmsg_start, BPF_ANY); 73816962b24SJohn Fastabend if (err) { 73916962b24SJohn Fastabend fprintf(stderr, 74016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 74116962b24SJohn Fastabend err, strerror(errno)); 74216962b24SJohn Fastabend goto out; 74316962b24SJohn Fastabend } 74416962b24SJohn Fastabend } 74516962b24SJohn Fastabend 74616962b24SJohn Fastabend if (txmsg_end) { 74716962b24SJohn Fastabend i = 1; 74816962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[5], 74916962b24SJohn Fastabend &i, &txmsg_end, BPF_ANY); 75016962b24SJohn Fastabend if (err) { 75116962b24SJohn Fastabend fprintf(stderr, 75216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 75316962b24SJohn Fastabend err, strerror(errno)); 75416962b24SJohn Fastabend goto out; 75516962b24SJohn Fastabend } 75616962b24SJohn Fastabend } 75716962b24SJohn Fastabend 75816962b24SJohn Fastabend if (txmsg_ingress) { 75916962b24SJohn Fastabend int in = BPF_F_INGRESS; 76016962b24SJohn Fastabend 76116962b24SJohn Fastabend i = 0; 76216962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 76316962b24SJohn Fastabend if (err) { 76416962b24SJohn Fastabend fprintf(stderr, 76516962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 76616962b24SJohn Fastabend err, strerror(errno)); 76716962b24SJohn Fastabend } 76816962b24SJohn Fastabend i = 1; 76916962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 77016962b24SJohn Fastabend if (err) { 77116962b24SJohn Fastabend fprintf(stderr, 77216962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 77316962b24SJohn Fastabend err, strerror(errno)); 77416962b24SJohn Fastabend } 77516962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 77616962b24SJohn Fastabend if (err) { 77716962b24SJohn Fastabend fprintf(stderr, 77816962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 77916962b24SJohn Fastabend err, strerror(errno)); 78016962b24SJohn Fastabend } 78116962b24SJohn Fastabend 78216962b24SJohn Fastabend i = 2; 78316962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 78416962b24SJohn Fastabend if (err) { 78516962b24SJohn Fastabend fprintf(stderr, 78616962b24SJohn Fastabend "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 78716962b24SJohn Fastabend err, strerror(errno)); 78816962b24SJohn Fastabend } 78916962b24SJohn Fastabend } 79016962b24SJohn Fastabend 79116962b24SJohn Fastabend if (txmsg_skb) { 79216962b24SJohn Fastabend int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 79316962b24SJohn Fastabend p2 : p1; 79416962b24SJohn Fastabend int ingress = BPF_F_INGRESS; 79516962b24SJohn Fastabend 79616962b24SJohn Fastabend i = 0; 79716962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[7], 79816962b24SJohn Fastabend &i, &ingress, BPF_ANY); 79916962b24SJohn Fastabend if (err) { 80016962b24SJohn Fastabend fprintf(stderr, 80116962b24SJohn Fastabend "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 80216962b24SJohn Fastabend err, strerror(errno)); 80316962b24SJohn Fastabend } 80416962b24SJohn Fastabend 80516962b24SJohn Fastabend i = 3; 80616962b24SJohn Fastabend err = bpf_map_update_elem(map_fd[0], 80716962b24SJohn Fastabend &i, &skb_fd, BPF_ANY); 80816962b24SJohn Fastabend if (err) { 80916962b24SJohn Fastabend fprintf(stderr, 81016962b24SJohn Fastabend "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 81116962b24SJohn Fastabend err, strerror(errno)); 81216962b24SJohn Fastabend } 81316962b24SJohn Fastabend } 81416962b24SJohn Fastabend } 81516962b24SJohn Fastabend 81616962b24SJohn Fastabend if (txmsg_drop) 81716962b24SJohn Fastabend options->drop_expected = true; 81816962b24SJohn Fastabend 81916962b24SJohn Fastabend if (test == PING_PONG) 82016962b24SJohn Fastabend err = forever_ping_pong(options->rate, options); 82116962b24SJohn Fastabend else if (test == SENDMSG) { 82216962b24SJohn Fastabend options->base = false; 82316962b24SJohn Fastabend options->sendpage = false; 82416962b24SJohn Fastabend err = sendmsg_test(options); 82516962b24SJohn Fastabend } else if (test == SENDPAGE) { 82616962b24SJohn Fastabend options->base = false; 82716962b24SJohn Fastabend options->sendpage = true; 82816962b24SJohn Fastabend err = sendmsg_test(options); 82916962b24SJohn Fastabend } else if (test == BASE) { 83016962b24SJohn Fastabend options->base = true; 83116962b24SJohn Fastabend options->sendpage = false; 83216962b24SJohn Fastabend err = sendmsg_test(options); 83316962b24SJohn Fastabend } else if (test == BASE_SENDPAGE) { 83416962b24SJohn Fastabend options->base = true; 83516962b24SJohn Fastabend options->sendpage = true; 83616962b24SJohn Fastabend err = sendmsg_test(options); 83716962b24SJohn Fastabend } else 83816962b24SJohn Fastabend fprintf(stderr, "unknown test\n"); 83916962b24SJohn Fastabend out: 84016962b24SJohn Fastabend /* Detatch and zero all the maps */ 84116962b24SJohn Fastabend bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 84216962b24SJohn Fastabend bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 84316962b24SJohn Fastabend bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 84416962b24SJohn Fastabend if (tx_prog_fd >= 0) 84516962b24SJohn Fastabend bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 84616962b24SJohn Fastabend 84716962b24SJohn Fastabend for (i = 0; i < 8; i++) { 84816962b24SJohn Fastabend key = next_key = 0; 84916962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 85016962b24SJohn Fastabend while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 85116962b24SJohn Fastabend bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 85216962b24SJohn Fastabend key = next_key; 85316962b24SJohn Fastabend } 85416962b24SJohn Fastabend } 85516962b24SJohn Fastabend 85616962b24SJohn Fastabend close(s1); 85716962b24SJohn Fastabend close(s2); 85816962b24SJohn Fastabend close(p1); 85916962b24SJohn Fastabend close(p2); 86016962b24SJohn Fastabend close(c1); 86116962b24SJohn Fastabend close(c2); 86216962b24SJohn Fastabend return err; 86316962b24SJohn Fastabend } 86416962b24SJohn Fastabend 86516962b24SJohn Fastabend static char *test_to_str(int test) 86616962b24SJohn Fastabend { 86716962b24SJohn Fastabend switch (test) { 86816962b24SJohn Fastabend case SENDMSG: 86916962b24SJohn Fastabend return "sendmsg"; 87016962b24SJohn Fastabend case SENDPAGE: 87116962b24SJohn Fastabend return "sendpage"; 87216962b24SJohn Fastabend } 87316962b24SJohn Fastabend return "unknown"; 87416962b24SJohn Fastabend } 87516962b24SJohn Fastabend 87616962b24SJohn Fastabend #define OPTSTRING 60 87716962b24SJohn Fastabend static void test_options(char *options) 87816962b24SJohn Fastabend { 87973563aa3SPrashant Bhole char tstr[OPTSTRING]; 88073563aa3SPrashant Bhole 88116962b24SJohn Fastabend memset(options, 0, OPTSTRING); 88216962b24SJohn Fastabend 88316962b24SJohn Fastabend if (txmsg_pass) 88416962b24SJohn Fastabend strncat(options, "pass,", OPTSTRING); 88516962b24SJohn Fastabend if (txmsg_noisy) 88616962b24SJohn Fastabend strncat(options, "pass_noisy,", OPTSTRING); 88716962b24SJohn Fastabend if (txmsg_redir) 88816962b24SJohn Fastabend strncat(options, "redir,", OPTSTRING); 88916962b24SJohn Fastabend if (txmsg_redir_noisy) 89016962b24SJohn Fastabend strncat(options, "redir_noisy,", OPTSTRING); 89116962b24SJohn Fastabend if (txmsg_drop) 89216962b24SJohn Fastabend strncat(options, "drop,", OPTSTRING); 89373563aa3SPrashant Bhole if (txmsg_apply) { 89473563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 89573563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 89673563aa3SPrashant Bhole } 89773563aa3SPrashant Bhole if (txmsg_cork) { 89873563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 89973563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 90073563aa3SPrashant Bhole } 90173563aa3SPrashant Bhole if (txmsg_start) { 90273563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 90373563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 90473563aa3SPrashant Bhole } 90573563aa3SPrashant Bhole if (txmsg_end) { 90673563aa3SPrashant Bhole snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 90773563aa3SPrashant Bhole strncat(options, tstr, OPTSTRING); 90873563aa3SPrashant Bhole } 90916962b24SJohn Fastabend if (txmsg_ingress) 91016962b24SJohn Fastabend strncat(options, "ingress,", OPTSTRING); 91116962b24SJohn Fastabend if (txmsg_skb) 91216962b24SJohn Fastabend strncat(options, "skb,", OPTSTRING); 91316962b24SJohn Fastabend } 91416962b24SJohn Fastabend 91516962b24SJohn Fastabend static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 91616962b24SJohn Fastabend { 91773563aa3SPrashant Bhole char *options = calloc(OPTSTRING, sizeof(char)); 91816962b24SJohn Fastabend int err; 91916962b24SJohn Fastabend 92016962b24SJohn Fastabend if (test == SENDPAGE) 92116962b24SJohn Fastabend opt->sendpage = true; 92216962b24SJohn Fastabend else 92316962b24SJohn Fastabend opt->sendpage = false; 92416962b24SJohn Fastabend 92516962b24SJohn Fastabend if (txmsg_drop) 92616962b24SJohn Fastabend opt->drop_expected = true; 92716962b24SJohn Fastabend else 92816962b24SJohn Fastabend opt->drop_expected = false; 92916962b24SJohn Fastabend 93016962b24SJohn Fastabend test_options(options); 93116962b24SJohn Fastabend 93216962b24SJohn Fastabend fprintf(stdout, 93316962b24SJohn Fastabend "[TEST %i]: (%i, %i, %i, %s, %s): ", 93416962b24SJohn Fastabend test_cnt, opt->rate, opt->iov_count, opt->iov_length, 93516962b24SJohn Fastabend test_to_str(test), options); 93616962b24SJohn Fastabend fflush(stdout); 93716962b24SJohn Fastabend err = run_options(opt, cgrp, test); 93816962b24SJohn Fastabend fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 93916962b24SJohn Fastabend test_cnt++; 94016962b24SJohn Fastabend !err ? passed++ : failed++; 94116962b24SJohn Fastabend free(options); 94216962b24SJohn Fastabend return err; 94316962b24SJohn Fastabend } 94416962b24SJohn Fastabend 94516962b24SJohn Fastabend static int test_exec(int cgrp, struct sockmap_options *opt) 94616962b24SJohn Fastabend { 94716962b24SJohn Fastabend int err = __test_exec(cgrp, SENDMSG, opt); 94816962b24SJohn Fastabend 94916962b24SJohn Fastabend if (err) 95016962b24SJohn Fastabend goto out; 95116962b24SJohn Fastabend 95216962b24SJohn Fastabend err = __test_exec(cgrp, SENDPAGE, opt); 95316962b24SJohn Fastabend out: 95416962b24SJohn Fastabend return err; 95516962b24SJohn Fastabend } 95616962b24SJohn Fastabend 95716962b24SJohn Fastabend static int test_loop(int cgrp) 95816962b24SJohn Fastabend { 95916962b24SJohn Fastabend struct sockmap_options opt; 96016962b24SJohn Fastabend 96116962b24SJohn Fastabend int err, i, l, r; 96216962b24SJohn Fastabend 96316962b24SJohn Fastabend opt.verbose = 0; 96416962b24SJohn Fastabend opt.base = false; 96516962b24SJohn Fastabend opt.sendpage = false; 96616962b24SJohn Fastabend opt.data_test = false; 96716962b24SJohn Fastabend opt.drop_expected = false; 96816962b24SJohn Fastabend opt.iov_count = 0; 96916962b24SJohn Fastabend opt.iov_length = 0; 97016962b24SJohn Fastabend opt.rate = 0; 97116962b24SJohn Fastabend 972a18fda1aSJohn Fastabend r = 1; 97316962b24SJohn Fastabend for (i = 1; i < 100; i += 33) { 97416962b24SJohn Fastabend for (l = 1; l < 100; l += 33) { 97516962b24SJohn Fastabend opt.rate = r; 97616962b24SJohn Fastabend opt.iov_count = i; 97716962b24SJohn Fastabend opt.iov_length = l; 97816962b24SJohn Fastabend err = test_exec(cgrp, &opt); 97916962b24SJohn Fastabend if (err) 98016962b24SJohn Fastabend goto out; 98116962b24SJohn Fastabend } 98216962b24SJohn Fastabend } 983a18fda1aSJohn Fastabend sched_yield(); 98416962b24SJohn Fastabend out: 98516962b24SJohn Fastabend return err; 98616962b24SJohn Fastabend } 98716962b24SJohn Fastabend 98816962b24SJohn Fastabend static int test_txmsg(int cgrp) 98916962b24SJohn Fastabend { 99016962b24SJohn Fastabend int err; 99116962b24SJohn Fastabend 99216962b24SJohn Fastabend txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 99316962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 99416962b24SJohn Fastabend txmsg_ingress = txmsg_skb = 0; 99516962b24SJohn Fastabend 99616962b24SJohn Fastabend txmsg_pass = 1; 99716962b24SJohn Fastabend err = test_loop(cgrp); 99816962b24SJohn Fastabend txmsg_pass = 0; 99916962b24SJohn Fastabend if (err) 100016962b24SJohn Fastabend goto out; 100116962b24SJohn Fastabend 100216962b24SJohn Fastabend txmsg_redir = 1; 100316962b24SJohn Fastabend err = test_loop(cgrp); 100416962b24SJohn Fastabend txmsg_redir = 0; 100516962b24SJohn Fastabend if (err) 100616962b24SJohn Fastabend goto out; 100716962b24SJohn Fastabend 100816962b24SJohn Fastabend txmsg_drop = 1; 100916962b24SJohn Fastabend err = test_loop(cgrp); 101016962b24SJohn Fastabend txmsg_drop = 0; 101116962b24SJohn Fastabend if (err) 101216962b24SJohn Fastabend goto out; 101316962b24SJohn Fastabend 101416962b24SJohn Fastabend txmsg_redir = 1; 101516962b24SJohn Fastabend txmsg_ingress = 1; 101616962b24SJohn Fastabend err = test_loop(cgrp); 101716962b24SJohn Fastabend txmsg_redir = 0; 101816962b24SJohn Fastabend txmsg_ingress = 0; 101916962b24SJohn Fastabend if (err) 102016962b24SJohn Fastabend goto out; 102116962b24SJohn Fastabend out: 102216962b24SJohn Fastabend txmsg_pass = 0; 102316962b24SJohn Fastabend txmsg_redir = 0; 102416962b24SJohn Fastabend txmsg_drop = 0; 102516962b24SJohn Fastabend return err; 102616962b24SJohn Fastabend } 102716962b24SJohn Fastabend 102816962b24SJohn Fastabend static int test_send(struct sockmap_options *opt, int cgrp) 102916962b24SJohn Fastabend { 103016962b24SJohn Fastabend int err; 103116962b24SJohn Fastabend 103216962b24SJohn Fastabend opt->iov_length = 1; 103316962b24SJohn Fastabend opt->iov_count = 1; 103416962b24SJohn Fastabend opt->rate = 1; 103516962b24SJohn Fastabend err = test_exec(cgrp, opt); 103616962b24SJohn Fastabend if (err) 103716962b24SJohn Fastabend goto out; 103816962b24SJohn Fastabend 103916962b24SJohn Fastabend opt->iov_length = 1; 104016962b24SJohn Fastabend opt->iov_count = 1024; 104116962b24SJohn Fastabend opt->rate = 1; 104216962b24SJohn Fastabend err = test_exec(cgrp, opt); 104316962b24SJohn Fastabend if (err) 104416962b24SJohn Fastabend goto out; 104516962b24SJohn Fastabend 104616962b24SJohn Fastabend opt->iov_length = 1024; 104716962b24SJohn Fastabend opt->iov_count = 1; 104816962b24SJohn Fastabend opt->rate = 1; 104916962b24SJohn Fastabend err = test_exec(cgrp, opt); 105016962b24SJohn Fastabend if (err) 105116962b24SJohn Fastabend goto out; 105216962b24SJohn Fastabend 105316962b24SJohn Fastabend opt->iov_length = 1; 105416962b24SJohn Fastabend opt->iov_count = 1; 1055a009f1f3SPrashant Bhole opt->rate = 512; 105616962b24SJohn Fastabend err = test_exec(cgrp, opt); 105716962b24SJohn Fastabend if (err) 105816962b24SJohn Fastabend goto out; 105916962b24SJohn Fastabend 106016962b24SJohn Fastabend opt->iov_length = 256; 106116962b24SJohn Fastabend opt->iov_count = 1024; 1062a009f1f3SPrashant Bhole opt->rate = 2; 106316962b24SJohn Fastabend err = test_exec(cgrp, opt); 106416962b24SJohn Fastabend if (err) 106516962b24SJohn Fastabend goto out; 106616962b24SJohn Fastabend 106716962b24SJohn Fastabend opt->rate = 100; 106816962b24SJohn Fastabend opt->iov_count = 1; 106916962b24SJohn Fastabend opt->iov_length = 5; 107016962b24SJohn Fastabend err = test_exec(cgrp, opt); 107116962b24SJohn Fastabend if (err) 107216962b24SJohn Fastabend goto out; 107316962b24SJohn Fastabend out: 1074a18fda1aSJohn Fastabend sched_yield(); 107516962b24SJohn Fastabend return err; 107616962b24SJohn Fastabend } 107716962b24SJohn Fastabend 107816962b24SJohn Fastabend static int test_mixed(int cgrp) 107916962b24SJohn Fastabend { 108016962b24SJohn Fastabend struct sockmap_options opt = {0}; 108116962b24SJohn Fastabend int err; 108216962b24SJohn Fastabend 108316962b24SJohn Fastabend txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 108416962b24SJohn Fastabend txmsg_apply = txmsg_cork = 0; 108516962b24SJohn Fastabend txmsg_start = txmsg_end = 0; 108616962b24SJohn Fastabend /* Test small and large iov_count values with pass/redir/apply/cork */ 108716962b24SJohn Fastabend txmsg_pass = 1; 108816962b24SJohn Fastabend txmsg_redir = 0; 108916962b24SJohn Fastabend txmsg_apply = 1; 109016962b24SJohn Fastabend txmsg_cork = 0; 109116962b24SJohn Fastabend err = test_send(&opt, cgrp); 109216962b24SJohn Fastabend if (err) 109316962b24SJohn Fastabend goto out; 109416962b24SJohn Fastabend 109516962b24SJohn Fastabend txmsg_pass = 1; 109616962b24SJohn Fastabend txmsg_redir = 0; 109716962b24SJohn Fastabend txmsg_apply = 0; 109816962b24SJohn Fastabend txmsg_cork = 1; 109916962b24SJohn Fastabend err = test_send(&opt, cgrp); 110016962b24SJohn Fastabend if (err) 110116962b24SJohn Fastabend goto out; 110216962b24SJohn Fastabend 110316962b24SJohn Fastabend txmsg_pass = 1; 110416962b24SJohn Fastabend txmsg_redir = 0; 110516962b24SJohn Fastabend txmsg_apply = 1; 110616962b24SJohn Fastabend txmsg_cork = 1; 110716962b24SJohn Fastabend err = test_send(&opt, cgrp); 110816962b24SJohn Fastabend if (err) 110916962b24SJohn Fastabend goto out; 111016962b24SJohn Fastabend 111116962b24SJohn Fastabend txmsg_pass = 1; 111216962b24SJohn Fastabend txmsg_redir = 0; 111316962b24SJohn Fastabend txmsg_apply = 1024; 111416962b24SJohn Fastabend txmsg_cork = 0; 111516962b24SJohn Fastabend err = test_send(&opt, cgrp); 111616962b24SJohn Fastabend if (err) 111716962b24SJohn Fastabend goto out; 111816962b24SJohn Fastabend 111916962b24SJohn Fastabend txmsg_pass = 1; 112016962b24SJohn Fastabend txmsg_redir = 0; 112116962b24SJohn Fastabend txmsg_apply = 0; 112216962b24SJohn Fastabend txmsg_cork = 1024; 112316962b24SJohn Fastabend err = test_send(&opt, cgrp); 112416962b24SJohn Fastabend if (err) 112516962b24SJohn Fastabend goto out; 112616962b24SJohn Fastabend 112716962b24SJohn Fastabend txmsg_pass = 1; 112816962b24SJohn Fastabend txmsg_redir = 0; 112916962b24SJohn Fastabend txmsg_apply = 1024; 113016962b24SJohn Fastabend txmsg_cork = 1024; 113116962b24SJohn Fastabend err = test_send(&opt, cgrp); 113216962b24SJohn Fastabend if (err) 113316962b24SJohn Fastabend goto out; 113416962b24SJohn Fastabend 113516962b24SJohn Fastabend txmsg_pass = 1; 113616962b24SJohn Fastabend txmsg_redir = 0; 113716962b24SJohn Fastabend txmsg_cork = 4096; 113816962b24SJohn Fastabend txmsg_apply = 4096; 113916962b24SJohn Fastabend err = test_send(&opt, cgrp); 114016962b24SJohn Fastabend if (err) 114116962b24SJohn Fastabend goto out; 114216962b24SJohn Fastabend 114316962b24SJohn Fastabend txmsg_pass = 0; 114416962b24SJohn Fastabend txmsg_redir = 1; 114516962b24SJohn Fastabend txmsg_apply = 1; 114616962b24SJohn Fastabend txmsg_cork = 0; 114716962b24SJohn Fastabend err = test_send(&opt, cgrp); 114816962b24SJohn Fastabend if (err) 114916962b24SJohn Fastabend goto out; 115016962b24SJohn Fastabend 115116962b24SJohn Fastabend txmsg_pass = 0; 115216962b24SJohn Fastabend txmsg_redir = 1; 115316962b24SJohn Fastabend txmsg_apply = 0; 115416962b24SJohn Fastabend txmsg_cork = 1; 115516962b24SJohn Fastabend err = test_send(&opt, cgrp); 115616962b24SJohn Fastabend if (err) 115716962b24SJohn Fastabend goto out; 115816962b24SJohn Fastabend 115916962b24SJohn Fastabend txmsg_pass = 0; 116016962b24SJohn Fastabend txmsg_redir = 1; 116116962b24SJohn Fastabend txmsg_apply = 1024; 116216962b24SJohn Fastabend txmsg_cork = 0; 116316962b24SJohn Fastabend err = test_send(&opt, cgrp); 116416962b24SJohn Fastabend if (err) 116516962b24SJohn Fastabend goto out; 116616962b24SJohn Fastabend 116716962b24SJohn Fastabend txmsg_pass = 0; 116816962b24SJohn Fastabend txmsg_redir = 1; 116916962b24SJohn Fastabend txmsg_apply = 0; 117016962b24SJohn Fastabend txmsg_cork = 1024; 117116962b24SJohn Fastabend err = test_send(&opt, cgrp); 117216962b24SJohn Fastabend if (err) 117316962b24SJohn Fastabend goto out; 117416962b24SJohn Fastabend 117516962b24SJohn Fastabend txmsg_pass = 0; 117616962b24SJohn Fastabend txmsg_redir = 1; 117716962b24SJohn Fastabend txmsg_apply = 1024; 117816962b24SJohn Fastabend txmsg_cork = 1024; 117916962b24SJohn Fastabend err = test_send(&opt, cgrp); 118016962b24SJohn Fastabend if (err) 118116962b24SJohn Fastabend goto out; 118216962b24SJohn Fastabend 118316962b24SJohn Fastabend txmsg_pass = 0; 118416962b24SJohn Fastabend txmsg_redir = 1; 118516962b24SJohn Fastabend txmsg_cork = 4096; 118616962b24SJohn Fastabend txmsg_apply = 4096; 118716962b24SJohn Fastabend err = test_send(&opt, cgrp); 118816962b24SJohn Fastabend if (err) 118916962b24SJohn Fastabend goto out; 119016962b24SJohn Fastabend out: 119116962b24SJohn Fastabend return err; 119216962b24SJohn Fastabend } 119316962b24SJohn Fastabend 119416962b24SJohn Fastabend static int test_start_end(int cgrp) 119516962b24SJohn Fastabend { 119616962b24SJohn Fastabend struct sockmap_options opt = {0}; 119716962b24SJohn Fastabend int err, i; 119816962b24SJohn Fastabend 119916962b24SJohn Fastabend /* Test basic start/end with lots of iov_count and iov_lengths */ 120016962b24SJohn Fastabend txmsg_start = 1; 120116962b24SJohn Fastabend txmsg_end = 2; 120216962b24SJohn Fastabend err = test_txmsg(cgrp); 120316962b24SJohn Fastabend if (err) 120416962b24SJohn Fastabend goto out; 120516962b24SJohn Fastabend 120616962b24SJohn Fastabend /* Test start/end with cork */ 120716962b24SJohn Fastabend opt.rate = 16; 120816962b24SJohn Fastabend opt.iov_count = 1; 120916962b24SJohn Fastabend opt.iov_length = 100; 121016962b24SJohn Fastabend txmsg_cork = 1600; 121116962b24SJohn Fastabend 1212a18fda1aSJohn Fastabend for (i = 99; i <= 1600; i += 500) { 121316962b24SJohn Fastabend txmsg_start = 0; 121416962b24SJohn Fastabend txmsg_end = i; 121516962b24SJohn Fastabend err = test_exec(cgrp, &opt); 121616962b24SJohn Fastabend if (err) 121716962b24SJohn Fastabend goto out; 121816962b24SJohn Fastabend } 121916962b24SJohn Fastabend 122016962b24SJohn Fastabend /* Test start/end with cork but pull data in middle */ 1221a18fda1aSJohn Fastabend for (i = 199; i <= 1600; i += 500) { 122216962b24SJohn Fastabend txmsg_start = 100; 122316962b24SJohn Fastabend txmsg_end = i; 122416962b24SJohn Fastabend err = test_exec(cgrp, &opt); 122516962b24SJohn Fastabend if (err) 122616962b24SJohn Fastabend goto out; 122716962b24SJohn Fastabend } 122816962b24SJohn Fastabend 122916962b24SJohn Fastabend /* Test start/end with cork pulling last sg entry */ 123016962b24SJohn Fastabend txmsg_start = 1500; 123116962b24SJohn Fastabend txmsg_end = 1600; 123216962b24SJohn Fastabend err = test_exec(cgrp, &opt); 123316962b24SJohn Fastabend if (err) 123416962b24SJohn Fastabend goto out; 123516962b24SJohn Fastabend 123616962b24SJohn Fastabend /* Test start/end pull of single byte in last page */ 123716962b24SJohn Fastabend txmsg_start = 1111; 123816962b24SJohn Fastabend txmsg_end = 1112; 123916962b24SJohn Fastabend err = test_exec(cgrp, &opt); 124016962b24SJohn Fastabend if (err) 124116962b24SJohn Fastabend goto out; 124216962b24SJohn Fastabend 124316962b24SJohn Fastabend /* Test start/end with end < start */ 124416962b24SJohn Fastabend txmsg_start = 1111; 124516962b24SJohn Fastabend txmsg_end = 0; 124616962b24SJohn Fastabend err = test_exec(cgrp, &opt); 124716962b24SJohn Fastabend if (err) 124816962b24SJohn Fastabend goto out; 124916962b24SJohn Fastabend 125016962b24SJohn Fastabend /* Test start/end with end > data */ 125116962b24SJohn Fastabend txmsg_start = 0; 125216962b24SJohn Fastabend txmsg_end = 1601; 125316962b24SJohn Fastabend err = test_exec(cgrp, &opt); 125416962b24SJohn Fastabend if (err) 125516962b24SJohn Fastabend goto out; 125616962b24SJohn Fastabend 125716962b24SJohn Fastabend /* Test start/end with start > data */ 125816962b24SJohn Fastabend txmsg_start = 1601; 125916962b24SJohn Fastabend txmsg_end = 1600; 126016962b24SJohn Fastabend err = test_exec(cgrp, &opt); 126116962b24SJohn Fastabend 126216962b24SJohn Fastabend out: 126316962b24SJohn Fastabend txmsg_start = 0; 126416962b24SJohn Fastabend txmsg_end = 0; 1265a18fda1aSJohn Fastabend sched_yield(); 126616962b24SJohn Fastabend return err; 126716962b24SJohn Fastabend } 126816962b24SJohn Fastabend 126916962b24SJohn Fastabend char *map_names[] = { 127016962b24SJohn Fastabend "sock_map", 127116962b24SJohn Fastabend "sock_map_txmsg", 127216962b24SJohn Fastabend "sock_map_redir", 127316962b24SJohn Fastabend "sock_apply_bytes", 127416962b24SJohn Fastabend "sock_cork_bytes", 127516962b24SJohn Fastabend "sock_pull_bytes", 127616962b24SJohn Fastabend "sock_redir_flags", 127716962b24SJohn Fastabend "sock_skb_opts", 127816962b24SJohn Fastabend }; 127916962b24SJohn Fastabend 128016962b24SJohn Fastabend int prog_attach_type[] = { 128116962b24SJohn Fastabend BPF_SK_SKB_STREAM_PARSER, 128216962b24SJohn Fastabend BPF_SK_SKB_STREAM_VERDICT, 128316962b24SJohn Fastabend BPF_CGROUP_SOCK_OPS, 128416962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 128516962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 128616962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 128716962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 128816962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 128916962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 129016962b24SJohn Fastabend BPF_SK_MSG_VERDICT, 129116962b24SJohn Fastabend }; 129216962b24SJohn Fastabend 129316962b24SJohn Fastabend int prog_type[] = { 129416962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 129516962b24SJohn Fastabend BPF_PROG_TYPE_SK_SKB, 129616962b24SJohn Fastabend BPF_PROG_TYPE_SOCK_OPS, 129716962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 129816962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 129916962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 130016962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 130116962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 130216962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 130316962b24SJohn Fastabend BPF_PROG_TYPE_SK_MSG, 130416962b24SJohn Fastabend }; 130516962b24SJohn Fastabend 1306b8b394faSJohn Fastabend static int populate_progs(char *bpf_file) 130716962b24SJohn Fastabend { 130816962b24SJohn Fastabend struct bpf_program *prog; 130916962b24SJohn Fastabend struct bpf_object *obj; 131016962b24SJohn Fastabend int i = 0; 131116962b24SJohn Fastabend long err; 131216962b24SJohn Fastabend 131316962b24SJohn Fastabend obj = bpf_object__open(bpf_file); 131416962b24SJohn Fastabend err = libbpf_get_error(obj); 131516962b24SJohn Fastabend if (err) { 131616962b24SJohn Fastabend char err_buf[256]; 131716962b24SJohn Fastabend 131816962b24SJohn Fastabend libbpf_strerror(err, err_buf, sizeof(err_buf)); 131916962b24SJohn Fastabend printf("Unable to load eBPF objects in file '%s' : %s\n", 132016962b24SJohn Fastabend bpf_file, err_buf); 132116962b24SJohn Fastabend return -1; 132216962b24SJohn Fastabend } 132316962b24SJohn Fastabend 132416962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 132516962b24SJohn Fastabend bpf_program__set_type(prog, prog_type[i]); 132616962b24SJohn Fastabend bpf_program__set_expected_attach_type(prog, 132716962b24SJohn Fastabend prog_attach_type[i]); 132816962b24SJohn Fastabend i++; 132916962b24SJohn Fastabend } 133016962b24SJohn Fastabend 133116962b24SJohn Fastabend i = bpf_object__load(obj); 133216962b24SJohn Fastabend i = 0; 133316962b24SJohn Fastabend bpf_object__for_each_program(prog, obj) { 133416962b24SJohn Fastabend prog_fd[i] = bpf_program__fd(prog); 133516962b24SJohn Fastabend i++; 133616962b24SJohn Fastabend } 133716962b24SJohn Fastabend 133816962b24SJohn Fastabend for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 133916962b24SJohn Fastabend maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 134016962b24SJohn Fastabend map_fd[i] = bpf_map__fd(maps[i]); 134116962b24SJohn Fastabend if (map_fd[i] < 0) { 134216962b24SJohn Fastabend fprintf(stderr, "load_bpf_file: (%i) %s\n", 134316962b24SJohn Fastabend map_fd[i], strerror(errno)); 134416962b24SJohn Fastabend return -1; 134516962b24SJohn Fastabend } 134616962b24SJohn Fastabend } 134716962b24SJohn Fastabend 134816962b24SJohn Fastabend return 0; 134916962b24SJohn Fastabend } 135016962b24SJohn Fastabend 1351b8b394faSJohn Fastabend static int __test_suite(char *bpf_file) 135216962b24SJohn Fastabend { 135316962b24SJohn Fastabend int cg_fd, err; 135416962b24SJohn Fastabend 1355b8b394faSJohn Fastabend err = populate_progs(bpf_file); 135616962b24SJohn Fastabend if (err < 0) { 135716962b24SJohn Fastabend fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 135816962b24SJohn Fastabend return err; 135916962b24SJohn Fastabend } 136016962b24SJohn Fastabend 136116962b24SJohn Fastabend if (setup_cgroup_environment()) { 136216962b24SJohn Fastabend fprintf(stderr, "ERROR: cgroup env failed\n"); 136316962b24SJohn Fastabend return -EINVAL; 136416962b24SJohn Fastabend } 136516962b24SJohn Fastabend 136616962b24SJohn Fastabend cg_fd = create_and_get_cgroup(CG_PATH); 136716962b24SJohn Fastabend if (cg_fd < 0) { 136816962b24SJohn Fastabend fprintf(stderr, 136916962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 137016962b24SJohn Fastabend cg_fd, optarg); 137116962b24SJohn Fastabend return cg_fd; 137216962b24SJohn Fastabend } 137316962b24SJohn Fastabend 1374035b37ffSPrashant Bhole if (join_cgroup(CG_PATH)) { 1375035b37ffSPrashant Bhole fprintf(stderr, "ERROR: failed to join cgroup\n"); 1376035b37ffSPrashant Bhole return -EINVAL; 1377035b37ffSPrashant Bhole } 1378035b37ffSPrashant Bhole 137916962b24SJohn Fastabend /* Tests basic commands and APIs with range of iov values */ 138016962b24SJohn Fastabend txmsg_start = txmsg_end = 0; 138116962b24SJohn Fastabend err = test_txmsg(cg_fd); 138216962b24SJohn Fastabend if (err) 138316962b24SJohn Fastabend goto out; 138416962b24SJohn Fastabend 138516962b24SJohn Fastabend /* Tests interesting combinations of APIs used together */ 138616962b24SJohn Fastabend err = test_mixed(cg_fd); 138716962b24SJohn Fastabend if (err) 138816962b24SJohn Fastabend goto out; 138916962b24SJohn Fastabend 139016962b24SJohn Fastabend /* Tests pull_data API using start/end API */ 139116962b24SJohn Fastabend err = test_start_end(cg_fd); 139216962b24SJohn Fastabend if (err) 139316962b24SJohn Fastabend goto out; 139416962b24SJohn Fastabend 139516962b24SJohn Fastabend out: 139616962b24SJohn Fastabend printf("Summary: %i PASSED %i FAILED\n", passed, failed); 1397b8b394faSJohn Fastabend cleanup_cgroup_environment(); 139816962b24SJohn Fastabend close(cg_fd); 139916962b24SJohn Fastabend return err; 140016962b24SJohn Fastabend } 140116962b24SJohn Fastabend 1402b8b394faSJohn Fastabend static int test_suite(void) 1403b8b394faSJohn Fastabend { 1404b8b394faSJohn Fastabend int err; 1405b8b394faSJohn Fastabend 1406b8b394faSJohn Fastabend err = __test_suite(BPF_SOCKMAP_FILENAME); 1407b8b394faSJohn Fastabend if (err) 1408b8b394faSJohn Fastabend goto out; 1409b8b394faSJohn Fastabend err = __test_suite(BPF_SOCKHASH_FILENAME); 1410b8b394faSJohn Fastabend out: 1411b8b394faSJohn Fastabend return err; 1412b8b394faSJohn Fastabend } 1413b8b394faSJohn Fastabend 141416962b24SJohn Fastabend int main(int argc, char **argv) 141516962b24SJohn Fastabend { 141616962b24SJohn Fastabend int iov_count = 1, length = 1024, rate = 1; 141716962b24SJohn Fastabend struct sockmap_options options = {0}; 141816962b24SJohn Fastabend int opt, longindex, err, cg_fd = 0; 1419b8b394faSJohn Fastabend char *bpf_file = BPF_SOCKMAP_FILENAME; 142016962b24SJohn Fastabend int test = PING_PONG; 142116962b24SJohn Fastabend 142216962b24SJohn Fastabend if (argc < 2) 142316962b24SJohn Fastabend return test_suite(); 142416962b24SJohn Fastabend 142516962b24SJohn Fastabend while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:", 142616962b24SJohn Fastabend long_options, &longindex)) != -1) { 142716962b24SJohn Fastabend switch (opt) { 142816962b24SJohn Fastabend case 's': 142916962b24SJohn Fastabend txmsg_start = atoi(optarg); 143016962b24SJohn Fastabend break; 143116962b24SJohn Fastabend case 'e': 143216962b24SJohn Fastabend txmsg_end = atoi(optarg); 143316962b24SJohn Fastabend break; 143416962b24SJohn Fastabend case 'a': 143516962b24SJohn Fastabend txmsg_apply = atoi(optarg); 143616962b24SJohn Fastabend break; 143716962b24SJohn Fastabend case 'k': 143816962b24SJohn Fastabend txmsg_cork = atoi(optarg); 143916962b24SJohn Fastabend break; 144016962b24SJohn Fastabend case 'c': 144116962b24SJohn Fastabend cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 144216962b24SJohn Fastabend if (cg_fd < 0) { 144316962b24SJohn Fastabend fprintf(stderr, 144416962b24SJohn Fastabend "ERROR: (%i) open cg path failed: %s\n", 144516962b24SJohn Fastabend cg_fd, optarg); 144616962b24SJohn Fastabend return cg_fd; 144716962b24SJohn Fastabend } 144816962b24SJohn Fastabend break; 144916962b24SJohn Fastabend case 'r': 145016962b24SJohn Fastabend rate = atoi(optarg); 145116962b24SJohn Fastabend break; 145216962b24SJohn Fastabend case 'v': 145316962b24SJohn Fastabend options.verbose = 1; 145416962b24SJohn Fastabend break; 145516962b24SJohn Fastabend case 'i': 145616962b24SJohn Fastabend iov_count = atoi(optarg); 145716962b24SJohn Fastabend break; 145816962b24SJohn Fastabend case 'l': 145916962b24SJohn Fastabend length = atoi(optarg); 146016962b24SJohn Fastabend break; 146116962b24SJohn Fastabend case 'd': 146216962b24SJohn Fastabend options.data_test = true; 146316962b24SJohn Fastabend break; 146416962b24SJohn Fastabend case 't': 146516962b24SJohn Fastabend if (strcmp(optarg, "ping") == 0) { 146616962b24SJohn Fastabend test = PING_PONG; 146716962b24SJohn Fastabend } else if (strcmp(optarg, "sendmsg") == 0) { 146816962b24SJohn Fastabend test = SENDMSG; 146916962b24SJohn Fastabend } else if (strcmp(optarg, "base") == 0) { 147016962b24SJohn Fastabend test = BASE; 147116962b24SJohn Fastabend } else if (strcmp(optarg, "base_sendpage") == 0) { 147216962b24SJohn Fastabend test = BASE_SENDPAGE; 147316962b24SJohn Fastabend } else if (strcmp(optarg, "sendpage") == 0) { 147416962b24SJohn Fastabend test = SENDPAGE; 147516962b24SJohn Fastabend } else { 147616962b24SJohn Fastabend usage(argv); 147716962b24SJohn Fastabend return -1; 147816962b24SJohn Fastabend } 147916962b24SJohn Fastabend break; 148016962b24SJohn Fastabend case 0: 148116962b24SJohn Fastabend break; 148216962b24SJohn Fastabend case 'h': 148316962b24SJohn Fastabend default: 148416962b24SJohn Fastabend usage(argv); 148516962b24SJohn Fastabend return -1; 148616962b24SJohn Fastabend } 148716962b24SJohn Fastabend } 148816962b24SJohn Fastabend 148916962b24SJohn Fastabend if (!cg_fd) { 149016962b24SJohn Fastabend fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n", 149116962b24SJohn Fastabend argv[0]); 149216962b24SJohn Fastabend return -1; 149316962b24SJohn Fastabend } 149416962b24SJohn Fastabend 1495b8b394faSJohn Fastabend err = populate_progs(bpf_file); 149616962b24SJohn Fastabend if (err) { 149716962b24SJohn Fastabend fprintf(stderr, "populate program: (%s) %s\n", 149816962b24SJohn Fastabend bpf_file, strerror(errno)); 149916962b24SJohn Fastabend return 1; 150016962b24SJohn Fastabend } 150116962b24SJohn Fastabend running = 1; 150216962b24SJohn Fastabend 150316962b24SJohn Fastabend /* catch SIGINT */ 150416962b24SJohn Fastabend signal(SIGINT, running_handler); 150516962b24SJohn Fastabend 150616962b24SJohn Fastabend options.iov_count = iov_count; 150716962b24SJohn Fastabend options.iov_length = length; 150816962b24SJohn Fastabend options.rate = rate; 150916962b24SJohn Fastabend 151016962b24SJohn Fastabend err = run_options(&options, cg_fd, test); 151116962b24SJohn Fastabend close(cg_fd); 151216962b24SJohn Fastabend return err; 151316962b24SJohn Fastabend } 151416962b24SJohn Fastabend 151516962b24SJohn Fastabend void running_handler(int a) 151616962b24SJohn Fastabend { 151716962b24SJohn Fastabend running = 0; 151816962b24SJohn Fastabend } 1519