1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/resource.h> 22 #include <sys/types.h> 23 #include <sys/sendfile.h> 24 25 #include <linux/netlink.h> 26 #include <linux/socket.h> 27 #include <linux/sock_diag.h> 28 #include <linux/bpf.h> 29 #include <linux/if_link.h> 30 #include <linux/tls.h> 31 #include <assert.h> 32 #include <libgen.h> 33 34 #include <getopt.h> 35 36 #include <bpf/bpf.h> 37 #include <bpf/libbpf.h> 38 39 #include "bpf_util.h" 40 #include "bpf_rlimit.h" 41 #include "cgroup_helpers.h" 42 43 int running; 44 static void running_handler(int a); 45 46 #ifndef TCP_ULP 47 # define TCP_ULP 31 48 #endif 49 #ifndef SOL_TLS 50 # define SOL_TLS 282 51 #endif 52 53 /* randomly selected ports for testing on lo */ 54 #define S1_PORT 10000 55 #define S2_PORT 10001 56 57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 59 #define CG_PATH "/sockmap" 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[8]; 67 struct bpf_map *maps[8]; 68 int prog_fd[11]; 69 70 int txmsg_pass; 71 int txmsg_redir; 72 int txmsg_drop; 73 int txmsg_apply; 74 int txmsg_cork; 75 int txmsg_start; 76 int txmsg_end; 77 int txmsg_start_push; 78 int txmsg_end_push; 79 int txmsg_start_pop; 80 int txmsg_pop; 81 int txmsg_ingress; 82 int txmsg_skb; 83 int ktls; 84 int peek_flag; 85 86 static const struct option long_options[] = { 87 {"help", no_argument, NULL, 'h' }, 88 {"cgroup", required_argument, NULL, 'c' }, 89 {"rate", required_argument, NULL, 'r' }, 90 {"verbose", optional_argument, NULL, 'v' }, 91 {"iov_count", required_argument, NULL, 'i' }, 92 {"length", required_argument, NULL, 'l' }, 93 {"test", required_argument, NULL, 't' }, 94 {"data_test", no_argument, NULL, 'd' }, 95 {"txmsg", no_argument, &txmsg_pass, 1 }, 96 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 97 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 98 {"txmsg_apply", required_argument, NULL, 'a'}, 99 {"txmsg_cork", required_argument, NULL, 'k'}, 100 {"txmsg_start", required_argument, NULL, 's'}, 101 {"txmsg_end", required_argument, NULL, 'e'}, 102 {"txmsg_start_push", required_argument, NULL, 'p'}, 103 {"txmsg_end_push", required_argument, NULL, 'q'}, 104 {"txmsg_start_pop", required_argument, NULL, 'w'}, 105 {"txmsg_pop", required_argument, NULL, 'x'}, 106 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 107 {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 108 {"ktls", no_argument, &ktls, 1 }, 109 {"peek", no_argument, &peek_flag, 1 }, 110 {"whitelist", required_argument, NULL, 'n' }, 111 {"blacklist", required_argument, NULL, 'b' }, 112 {0, 0, NULL, 0 } 113 }; 114 115 struct test_env { 116 const char *type; 117 const char *subtest; 118 const char *prepend; 119 120 int test_num; 121 int subtest_num; 122 123 int succ_cnt; 124 int fail_cnt; 125 int fail_last; 126 }; 127 128 struct test_env env; 129 130 struct sockmap_options { 131 int verbose; 132 bool base; 133 bool sendpage; 134 bool data_test; 135 bool drop_expected; 136 int iov_count; 137 int iov_length; 138 int rate; 139 char *map; 140 char *whitelist; 141 char *blacklist; 142 char *prepend; 143 }; 144 145 struct _test { 146 char *title; 147 void (*tester)(int cg_fd, struct sockmap_options *opt); 148 }; 149 150 static void test_start(void) 151 { 152 env.subtest_num++; 153 } 154 155 static void test_fail(void) 156 { 157 env.fail_cnt++; 158 } 159 160 static void test_pass(void) 161 { 162 env.succ_cnt++; 163 } 164 165 static void test_reset(void) 166 { 167 txmsg_start = txmsg_end = 0; 168 txmsg_start_pop = txmsg_pop = 0; 169 txmsg_start_push = txmsg_end_push = 0; 170 txmsg_pass = txmsg_drop = txmsg_redir = 0; 171 txmsg_apply = txmsg_cork = 0; 172 txmsg_ingress = txmsg_skb = 0; 173 } 174 175 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 176 { 177 env.type = o->map; 178 env.subtest = t->title; 179 env.prepend = o->prepend; 180 env.test_num++; 181 env.subtest_num = 0; 182 env.fail_last = env.fail_cnt; 183 test_reset(); 184 return 0; 185 } 186 187 static void test_end_subtest(void) 188 { 189 int error = env.fail_cnt - env.fail_last; 190 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 191 192 if (!error) 193 test_pass(); 194 195 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 196 env.test_num, env.subtest_num, 197 !type ? "sockmap" : "sockhash", 198 env.prepend ? : "", 199 env.subtest, error ? "FAIL" : "OK"); 200 } 201 202 static void test_print_results(void) 203 { 204 fprintf(stdout, "Pass: %d Fail: %d\n", 205 env.succ_cnt, env.fail_cnt); 206 } 207 208 static void usage(char *argv[]) 209 { 210 int i; 211 212 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 213 printf(" options:\n"); 214 for (i = 0; long_options[i].name != 0; i++) { 215 printf(" --%-12s", long_options[i].name); 216 if (long_options[i].flag != NULL) 217 printf(" flag (internal value:%d)\n", 218 *long_options[i].flag); 219 else 220 printf(" -%c\n", long_options[i].val); 221 } 222 printf("\n"); 223 } 224 225 char *sock_to_string(int s) 226 { 227 if (s == c1) 228 return "client1"; 229 else if (s == c2) 230 return "client2"; 231 else if (s == s1) 232 return "server1"; 233 else if (s == s2) 234 return "server2"; 235 else if (s == p1) 236 return "peer1"; 237 else if (s == p2) 238 return "peer2"; 239 else 240 return "unknown"; 241 } 242 243 static int sockmap_init_ktls(int verbose, int s) 244 { 245 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 246 .info = { 247 .version = TLS_1_2_VERSION, 248 .cipher_type = TLS_CIPHER_AES_GCM_128, 249 }, 250 }; 251 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 252 .info = { 253 .version = TLS_1_2_VERSION, 254 .cipher_type = TLS_CIPHER_AES_GCM_128, 255 }, 256 }; 257 int so_buf = 6553500; 258 int err; 259 260 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 261 if (err) { 262 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 263 return -EINVAL; 264 } 265 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 266 if (err) { 267 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 268 return -EINVAL; 269 } 270 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 271 if (err) { 272 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 273 return -EINVAL; 274 } 275 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 276 if (err) { 277 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 278 return -EINVAL; 279 } 280 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 281 if (err) { 282 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 283 return -EINVAL; 284 } 285 286 if (verbose) 287 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 288 return 0; 289 } 290 static int sockmap_init_sockets(int verbose) 291 { 292 int i, err, one = 1; 293 struct sockaddr_in addr; 294 int *fds[4] = {&s1, &s2, &c1, &c2}; 295 296 s1 = s2 = p1 = p2 = c1 = c2 = 0; 297 298 /* Init sockets */ 299 for (i = 0; i < 4; i++) { 300 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 301 if (*fds[i] < 0) { 302 perror("socket s1 failed()"); 303 return errno; 304 } 305 } 306 307 /* Allow reuse */ 308 for (i = 0; i < 2; i++) { 309 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 310 (char *)&one, sizeof(one)); 311 if (err) { 312 perror("setsockopt failed()"); 313 return errno; 314 } 315 } 316 317 /* Non-blocking sockets */ 318 for (i = 0; i < 2; i++) { 319 err = ioctl(*fds[i], FIONBIO, (char *)&one); 320 if (err < 0) { 321 perror("ioctl s1 failed()"); 322 return errno; 323 } 324 } 325 326 /* Bind server sockets */ 327 memset(&addr, 0, sizeof(struct sockaddr_in)); 328 addr.sin_family = AF_INET; 329 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 330 331 addr.sin_port = htons(S1_PORT); 332 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 333 if (err < 0) { 334 perror("bind s1 failed()"); 335 return errno; 336 } 337 338 addr.sin_port = htons(S2_PORT); 339 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 340 if (err < 0) { 341 perror("bind s2 failed()"); 342 return errno; 343 } 344 345 /* Listen server sockets */ 346 addr.sin_port = htons(S1_PORT); 347 err = listen(s1, 32); 348 if (err < 0) { 349 perror("listen s1 failed()"); 350 return errno; 351 } 352 353 addr.sin_port = htons(S2_PORT); 354 err = listen(s2, 32); 355 if (err < 0) { 356 perror("listen s1 failed()"); 357 return errno; 358 } 359 360 /* Initiate Connect */ 361 addr.sin_port = htons(S1_PORT); 362 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 363 if (err < 0 && errno != EINPROGRESS) { 364 perror("connect c1 failed()"); 365 return errno; 366 } 367 368 addr.sin_port = htons(S2_PORT); 369 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 370 if (err < 0 && errno != EINPROGRESS) { 371 perror("connect c2 failed()"); 372 return errno; 373 } else if (err < 0) { 374 err = 0; 375 } 376 377 /* Accept Connecrtions */ 378 p1 = accept(s1, NULL, NULL); 379 if (p1 < 0) { 380 perror("accept s1 failed()"); 381 return errno; 382 } 383 384 p2 = accept(s2, NULL, NULL); 385 if (p2 < 0) { 386 perror("accept s1 failed()"); 387 return errno; 388 } 389 390 if (verbose > 1) { 391 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 392 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 393 c1, s1, c2, s2); 394 } 395 return 0; 396 } 397 398 struct msg_stats { 399 size_t bytes_sent; 400 size_t bytes_recvd; 401 struct timespec start; 402 struct timespec end; 403 }; 404 405 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 406 struct msg_stats *s, 407 struct sockmap_options *opt) 408 { 409 bool drop = opt->drop_expected; 410 unsigned char k = 0; 411 FILE *file; 412 int i, fp; 413 414 file = tmpfile(); 415 if (!file) { 416 perror("create file for sendpage"); 417 return 1; 418 } 419 for (i = 0; i < iov_length * cnt; i++, k++) 420 fwrite(&k, sizeof(char), 1, file); 421 fflush(file); 422 fseek(file, 0, SEEK_SET); 423 424 fp = fileno(file); 425 426 clock_gettime(CLOCK_MONOTONIC, &s->start); 427 for (i = 0; i < cnt; i++) { 428 int sent; 429 430 errno = 0; 431 sent = sendfile(fd, fp, NULL, iov_length); 432 433 if (!drop && sent < 0) { 434 perror("sendpage loop error"); 435 fclose(file); 436 return sent; 437 } else if (drop && sent >= 0) { 438 printf("sendpage loop error expected: %i errno %i\n", 439 sent, errno); 440 fclose(file); 441 return -EIO; 442 } 443 444 if (sent > 0) 445 s->bytes_sent += sent; 446 } 447 clock_gettime(CLOCK_MONOTONIC, &s->end); 448 fclose(file); 449 return 0; 450 } 451 452 static void msg_free_iov(struct msghdr *msg) 453 { 454 int i; 455 456 for (i = 0; i < msg->msg_iovlen; i++) 457 free(msg->msg_iov[i].iov_base); 458 free(msg->msg_iov); 459 msg->msg_iov = NULL; 460 msg->msg_iovlen = 0; 461 } 462 463 static int msg_alloc_iov(struct msghdr *msg, 464 int iov_count, int iov_length, 465 bool data, bool xmit) 466 { 467 unsigned char k = 0; 468 struct iovec *iov; 469 int i; 470 471 iov = calloc(iov_count, sizeof(struct iovec)); 472 if (!iov) 473 return errno; 474 475 for (i = 0; i < iov_count; i++) { 476 unsigned char *d = calloc(iov_length, sizeof(char)); 477 478 if (!d) { 479 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 480 goto unwind_iov; 481 } 482 iov[i].iov_base = d; 483 iov[i].iov_len = iov_length; 484 485 if (data && xmit) { 486 int j; 487 488 for (j = 0; j < iov_length; j++) 489 d[j] = k++; 490 } 491 } 492 493 msg->msg_iov = iov; 494 msg->msg_iovlen = iov_count; 495 496 return 0; 497 unwind_iov: 498 for (i--; i >= 0 ; i--) 499 free(msg->msg_iov[i].iov_base); 500 return -ENOMEM; 501 } 502 503 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 504 { 505 int i, j, bytes_cnt = 0; 506 unsigned char k = 0; 507 508 for (i = 0; i < msg->msg_iovlen; i++) { 509 unsigned char *d = msg->msg_iov[i].iov_base; 510 511 for (j = 0; 512 j < msg->msg_iov[i].iov_len && size; j++) { 513 if (d[j] != k++) { 514 fprintf(stderr, 515 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 516 i, j, d[j], k - 1, d[j+1], k); 517 return -EIO; 518 } 519 bytes_cnt++; 520 if (bytes_cnt == chunk_sz) { 521 k = 0; 522 bytes_cnt = 0; 523 } 524 size--; 525 } 526 } 527 return 0; 528 } 529 530 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 531 struct msg_stats *s, bool tx, 532 struct sockmap_options *opt) 533 { 534 struct msghdr msg = {0}, msg_peek = {0}; 535 int err, i, flags = MSG_NOSIGNAL; 536 bool drop = opt->drop_expected; 537 bool data = opt->data_test; 538 539 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx); 540 if (err) 541 goto out_errno; 542 if (peek_flag) { 543 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 544 if (err) 545 goto out_errno; 546 } 547 548 if (tx) { 549 clock_gettime(CLOCK_MONOTONIC, &s->start); 550 for (i = 0; i < cnt; i++) { 551 int sent; 552 553 errno = 0; 554 sent = sendmsg(fd, &msg, flags); 555 556 if (!drop && sent < 0) { 557 perror("sendmsg loop error"); 558 goto out_errno; 559 } else if (drop && sent >= 0) { 560 fprintf(stderr, 561 "sendmsg loop error expected: %i errno %i\n", 562 sent, errno); 563 errno = -EIO; 564 goto out_errno; 565 } 566 if (sent > 0) 567 s->bytes_sent += sent; 568 } 569 clock_gettime(CLOCK_MONOTONIC, &s->end); 570 } else { 571 int slct, recvp = 0, recv, max_fd = fd; 572 float total_bytes, txmsg_pop_total; 573 int fd_flags = O_NONBLOCK; 574 struct timeval timeout; 575 fd_set w; 576 577 fcntl(fd, fd_flags); 578 /* Account for pop bytes noting each iteration of apply will 579 * call msg_pop_data helper so we need to account for this 580 * by calculating the number of apply iterations. Note user 581 * of the tool can create cases where no data is sent by 582 * manipulating pop/push/pull/etc. For example txmsg_apply 1 583 * with txmsg_pop 1 will try to apply 1B at a time but each 584 * iteration will then pop 1B so no data will ever be sent. 585 * This is really only useful for testing edge cases in code 586 * paths. 587 */ 588 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 589 if (txmsg_apply) 590 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 591 else 592 txmsg_pop_total = txmsg_pop * cnt; 593 total_bytes -= txmsg_pop_total; 594 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 595 if (err < 0) 596 perror("recv start time"); 597 while (s->bytes_recvd < total_bytes) { 598 if (txmsg_cork) { 599 timeout.tv_sec = 0; 600 timeout.tv_usec = 300000; 601 } else { 602 timeout.tv_sec = 3; 603 timeout.tv_usec = 0; 604 } 605 606 /* FD sets */ 607 FD_ZERO(&w); 608 FD_SET(fd, &w); 609 610 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 611 if (slct == -1) { 612 perror("select()"); 613 clock_gettime(CLOCK_MONOTONIC, &s->end); 614 goto out_errno; 615 } else if (!slct) { 616 if (opt->verbose) 617 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 618 errno = -EIO; 619 clock_gettime(CLOCK_MONOTONIC, &s->end); 620 goto out_errno; 621 } 622 623 errno = 0; 624 if (peek_flag) { 625 flags |= MSG_PEEK; 626 recvp = recvmsg(fd, &msg_peek, flags); 627 if (recvp < 0) { 628 if (errno != EWOULDBLOCK) { 629 clock_gettime(CLOCK_MONOTONIC, &s->end); 630 goto out_errno; 631 } 632 } 633 flags = 0; 634 } 635 636 recv = recvmsg(fd, &msg, flags); 637 if (recv < 0) { 638 if (errno != EWOULDBLOCK) { 639 clock_gettime(CLOCK_MONOTONIC, &s->end); 640 perror("recv failed()"); 641 goto out_errno; 642 } 643 } 644 645 s->bytes_recvd += recv; 646 647 if (data) { 648 int chunk_sz = opt->sendpage ? 649 iov_length * cnt : 650 iov_length * iov_count; 651 652 errno = msg_verify_data(&msg, recv, chunk_sz); 653 if (errno) { 654 perror("data verify msg failed"); 655 goto out_errno; 656 } 657 if (recvp) { 658 errno = msg_verify_data(&msg_peek, 659 recvp, 660 chunk_sz); 661 if (errno) { 662 perror("data verify msg_peek failed"); 663 goto out_errno; 664 } 665 } 666 } 667 } 668 clock_gettime(CLOCK_MONOTONIC, &s->end); 669 } 670 671 msg_free_iov(&msg); 672 msg_free_iov(&msg_peek); 673 return err; 674 out_errno: 675 msg_free_iov(&msg); 676 msg_free_iov(&msg_peek); 677 return errno; 678 } 679 680 static float giga = 1000000000; 681 682 static inline float sentBps(struct msg_stats s) 683 { 684 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 685 } 686 687 static inline float recvdBps(struct msg_stats s) 688 { 689 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 690 } 691 692 static int sendmsg_test(struct sockmap_options *opt) 693 { 694 float sent_Bps = 0, recvd_Bps = 0; 695 int rx_fd, txpid, rxpid, err = 0; 696 struct msg_stats s = {0}; 697 int iov_count = opt->iov_count; 698 int iov_buf = opt->iov_length; 699 int rx_status, tx_status; 700 int cnt = opt->rate; 701 702 errno = 0; 703 704 if (opt->base) 705 rx_fd = p1; 706 else 707 rx_fd = p2; 708 709 if (ktls) { 710 /* Redirecting into non-TLS socket which sends into a TLS 711 * socket is not a valid test. So in this case lets not 712 * enable kTLS but still run the test. 713 */ 714 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 715 err = sockmap_init_ktls(opt->verbose, rx_fd); 716 if (err) 717 return err; 718 } 719 err = sockmap_init_ktls(opt->verbose, c1); 720 if (err) 721 return err; 722 } 723 724 rxpid = fork(); 725 if (rxpid == 0) { 726 iov_buf -= (txmsg_pop - txmsg_start_pop + 1); 727 if (opt->drop_expected) 728 _exit(0); 729 730 if (!iov_buf) /* zero bytes sent case */ 731 _exit(0); 732 733 if (opt->sendpage) 734 iov_count = 1; 735 err = msg_loop(rx_fd, iov_count, iov_buf, 736 cnt, &s, false, opt); 737 if (opt->verbose > 1) 738 fprintf(stderr, 739 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 740 iov_count, iov_buf, cnt, err); 741 if (s.end.tv_sec - s.start.tv_sec) { 742 sent_Bps = sentBps(s); 743 recvd_Bps = recvdBps(s); 744 } 745 if (opt->verbose > 1) 746 fprintf(stdout, 747 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 748 s.bytes_sent, sent_Bps, sent_Bps/giga, 749 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 750 peek_flag ? "(peek_msg)" : ""); 751 if (err && txmsg_cork) 752 err = 0; 753 exit(err ? 1 : 0); 754 } else if (rxpid == -1) { 755 perror("msg_loop_rx"); 756 return errno; 757 } 758 759 txpid = fork(); 760 if (txpid == 0) { 761 if (opt->sendpage) 762 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 763 else 764 err = msg_loop(c1, iov_count, iov_buf, 765 cnt, &s, true, opt); 766 767 if (err) 768 fprintf(stderr, 769 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 770 iov_count, iov_buf, cnt, err); 771 if (s.end.tv_sec - s.start.tv_sec) { 772 sent_Bps = sentBps(s); 773 recvd_Bps = recvdBps(s); 774 } 775 if (opt->verbose > 1) 776 fprintf(stdout, 777 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 778 s.bytes_sent, sent_Bps, sent_Bps/giga, 779 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 780 exit(err ? 1 : 0); 781 } else if (txpid == -1) { 782 perror("msg_loop_tx"); 783 return errno; 784 } 785 786 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 787 assert(waitpid(txpid, &tx_status, 0) == txpid); 788 if (WIFEXITED(rx_status)) { 789 err = WEXITSTATUS(rx_status); 790 if (err) { 791 fprintf(stderr, "rx thread exited with err %d.\n", err); 792 goto out; 793 } 794 } 795 if (WIFEXITED(tx_status)) { 796 err = WEXITSTATUS(tx_status); 797 if (err) 798 fprintf(stderr, "tx thread exited with err %d.\n", err); 799 } 800 out: 801 return err; 802 } 803 804 static int forever_ping_pong(int rate, struct sockmap_options *opt) 805 { 806 struct timeval timeout; 807 char buf[1024] = {0}; 808 int sc; 809 810 timeout.tv_sec = 10; 811 timeout.tv_usec = 0; 812 813 /* Ping/Pong data from client to server */ 814 sc = send(c1, buf, sizeof(buf), 0); 815 if (sc < 0) { 816 perror("send failed()"); 817 return sc; 818 } 819 820 do { 821 int s, rc, i, max_fd = p2; 822 fd_set w; 823 824 /* FD sets */ 825 FD_ZERO(&w); 826 FD_SET(c1, &w); 827 FD_SET(c2, &w); 828 FD_SET(p1, &w); 829 FD_SET(p2, &w); 830 831 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 832 if (s == -1) { 833 perror("select()"); 834 break; 835 } else if (!s) { 836 fprintf(stderr, "unexpected timeout\n"); 837 break; 838 } 839 840 for (i = 0; i <= max_fd && s > 0; ++i) { 841 if (!FD_ISSET(i, &w)) 842 continue; 843 844 s--; 845 846 rc = recv(i, buf, sizeof(buf), 0); 847 if (rc < 0) { 848 if (errno != EWOULDBLOCK) { 849 perror("recv failed()"); 850 return rc; 851 } 852 } 853 854 if (rc == 0) { 855 close(i); 856 break; 857 } 858 859 sc = send(i, buf, rc, 0); 860 if (sc < 0) { 861 perror("send failed()"); 862 return sc; 863 } 864 } 865 866 if (rate) 867 sleep(rate); 868 869 if (opt->verbose) { 870 printf("."); 871 fflush(stdout); 872 873 } 874 } while (running); 875 876 return 0; 877 } 878 879 enum { 880 SELFTESTS, 881 PING_PONG, 882 SENDMSG, 883 BASE, 884 BASE_SENDPAGE, 885 SENDPAGE, 886 }; 887 888 static int run_options(struct sockmap_options *options, int cg_fd, int test) 889 { 890 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 891 892 /* If base test skip BPF setup */ 893 if (test == BASE || test == BASE_SENDPAGE) 894 goto run; 895 896 /* Attach programs to sockmap */ 897 err = bpf_prog_attach(prog_fd[0], map_fd[0], 898 BPF_SK_SKB_STREAM_PARSER, 0); 899 if (err) { 900 fprintf(stderr, 901 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 902 prog_fd[0], map_fd[0], err, strerror(errno)); 903 return err; 904 } 905 906 err = bpf_prog_attach(prog_fd[1], map_fd[0], 907 BPF_SK_SKB_STREAM_VERDICT, 0); 908 if (err) { 909 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 910 err, strerror(errno)); 911 return err; 912 } 913 914 /* Attach to cgroups */ 915 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 916 if (err) { 917 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 918 err, strerror(errno)); 919 return err; 920 } 921 922 run: 923 err = sockmap_init_sockets(options->verbose); 924 if (err) { 925 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 926 goto out; 927 } 928 929 /* Attach txmsg program to sockmap */ 930 if (txmsg_pass) 931 tx_prog_fd = prog_fd[3]; 932 else if (txmsg_redir) 933 tx_prog_fd = prog_fd[4]; 934 else if (txmsg_apply) 935 tx_prog_fd = prog_fd[5]; 936 else if (txmsg_cork) 937 tx_prog_fd = prog_fd[6]; 938 else if (txmsg_drop) 939 tx_prog_fd = prog_fd[7]; 940 else 941 tx_prog_fd = 0; 942 943 if (tx_prog_fd) { 944 int redir_fd, i = 0; 945 946 err = bpf_prog_attach(tx_prog_fd, 947 map_fd[1], BPF_SK_MSG_VERDICT, 0); 948 if (err) { 949 fprintf(stderr, 950 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 951 err, strerror(errno)); 952 goto out; 953 } 954 955 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 956 if (err) { 957 fprintf(stderr, 958 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 959 err, strerror(errno)); 960 goto out; 961 } 962 963 if (txmsg_redir) 964 redir_fd = c2; 965 else 966 redir_fd = c1; 967 968 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 969 if (err) { 970 fprintf(stderr, 971 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 972 err, strerror(errno)); 973 goto out; 974 } 975 976 if (txmsg_apply) { 977 err = bpf_map_update_elem(map_fd[3], 978 &i, &txmsg_apply, BPF_ANY); 979 if (err) { 980 fprintf(stderr, 981 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 982 err, strerror(errno)); 983 goto out; 984 } 985 } 986 987 if (txmsg_cork) { 988 err = bpf_map_update_elem(map_fd[4], 989 &i, &txmsg_cork, BPF_ANY); 990 if (err) { 991 fprintf(stderr, 992 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 993 err, strerror(errno)); 994 goto out; 995 } 996 } 997 998 if (txmsg_start) { 999 err = bpf_map_update_elem(map_fd[5], 1000 &i, &txmsg_start, BPF_ANY); 1001 if (err) { 1002 fprintf(stderr, 1003 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1004 err, strerror(errno)); 1005 goto out; 1006 } 1007 } 1008 1009 if (txmsg_end) { 1010 i = 1; 1011 err = bpf_map_update_elem(map_fd[5], 1012 &i, &txmsg_end, BPF_ANY); 1013 if (err) { 1014 fprintf(stderr, 1015 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1016 err, strerror(errno)); 1017 goto out; 1018 } 1019 } 1020 1021 if (txmsg_start_push) { 1022 i = 2; 1023 err = bpf_map_update_elem(map_fd[5], 1024 &i, &txmsg_start_push, BPF_ANY); 1025 if (err) { 1026 fprintf(stderr, 1027 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1028 err, strerror(errno)); 1029 goto out; 1030 } 1031 } 1032 1033 if (txmsg_end_push) { 1034 i = 3; 1035 err = bpf_map_update_elem(map_fd[5], 1036 &i, &txmsg_end_push, BPF_ANY); 1037 if (err) { 1038 fprintf(stderr, 1039 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1040 txmsg_end_push, i, err, strerror(errno)); 1041 goto out; 1042 } 1043 } 1044 1045 if (txmsg_start_pop) { 1046 i = 4; 1047 err = bpf_map_update_elem(map_fd[5], 1048 &i, &txmsg_start_pop, BPF_ANY); 1049 if (err) { 1050 fprintf(stderr, 1051 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1052 txmsg_start_pop, i, err, strerror(errno)); 1053 goto out; 1054 } 1055 } else { 1056 i = 4; 1057 bpf_map_update_elem(map_fd[5], 1058 &i, &txmsg_start_pop, BPF_ANY); 1059 } 1060 1061 if (txmsg_pop) { 1062 i = 5; 1063 err = bpf_map_update_elem(map_fd[5], 1064 &i, &txmsg_pop, BPF_ANY); 1065 if (err) { 1066 fprintf(stderr, 1067 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1068 txmsg_pop, i, err, strerror(errno)); 1069 goto out; 1070 } 1071 } else { 1072 i = 5; 1073 bpf_map_update_elem(map_fd[5], 1074 &i, &txmsg_pop, BPF_ANY); 1075 1076 } 1077 1078 if (txmsg_ingress) { 1079 int in = BPF_F_INGRESS; 1080 1081 i = 0; 1082 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1083 if (err) { 1084 fprintf(stderr, 1085 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1086 err, strerror(errno)); 1087 } 1088 i = 1; 1089 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1090 if (err) { 1091 fprintf(stderr, 1092 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1093 err, strerror(errno)); 1094 } 1095 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1096 if (err) { 1097 fprintf(stderr, 1098 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1099 err, strerror(errno)); 1100 } 1101 1102 i = 2; 1103 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1104 if (err) { 1105 fprintf(stderr, 1106 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1107 err, strerror(errno)); 1108 } 1109 } 1110 1111 if (txmsg_skb) { 1112 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1113 p2 : p1; 1114 int ingress = BPF_F_INGRESS; 1115 1116 i = 0; 1117 err = bpf_map_update_elem(map_fd[7], 1118 &i, &ingress, BPF_ANY); 1119 if (err) { 1120 fprintf(stderr, 1121 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1122 err, strerror(errno)); 1123 } 1124 1125 i = 3; 1126 err = bpf_map_update_elem(map_fd[0], 1127 &i, &skb_fd, BPF_ANY); 1128 if (err) { 1129 fprintf(stderr, 1130 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1131 err, strerror(errno)); 1132 } 1133 } 1134 } 1135 1136 if (txmsg_drop) 1137 options->drop_expected = true; 1138 1139 if (test == PING_PONG) 1140 err = forever_ping_pong(options->rate, options); 1141 else if (test == SENDMSG) { 1142 options->base = false; 1143 options->sendpage = false; 1144 err = sendmsg_test(options); 1145 } else if (test == SENDPAGE) { 1146 options->base = false; 1147 options->sendpage = true; 1148 err = sendmsg_test(options); 1149 } else if (test == BASE) { 1150 options->base = true; 1151 options->sendpage = false; 1152 err = sendmsg_test(options); 1153 } else if (test == BASE_SENDPAGE) { 1154 options->base = true; 1155 options->sendpage = true; 1156 err = sendmsg_test(options); 1157 } else 1158 fprintf(stderr, "unknown test\n"); 1159 out: 1160 /* Detatch and zero all the maps */ 1161 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 1162 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1163 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1164 if (tx_prog_fd >= 0) 1165 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1166 1167 for (i = 0; i < 8; i++) { 1168 key = next_key = 0; 1169 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1170 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1171 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1172 key = next_key; 1173 } 1174 } 1175 1176 close(s1); 1177 close(s2); 1178 close(p1); 1179 close(p2); 1180 close(c1); 1181 close(c2); 1182 return err; 1183 } 1184 1185 static char *test_to_str(int test) 1186 { 1187 switch (test) { 1188 case SENDMSG: 1189 return "sendmsg"; 1190 case SENDPAGE: 1191 return "sendpage"; 1192 } 1193 return "unknown"; 1194 } 1195 1196 #define OPTSTRING 60 1197 static void test_options(char *options) 1198 { 1199 char tstr[OPTSTRING]; 1200 1201 memset(options, 0, OPTSTRING); 1202 1203 if (txmsg_pass) 1204 strncat(options, "pass,", OPTSTRING); 1205 if (txmsg_redir) 1206 strncat(options, "redir,", OPTSTRING); 1207 if (txmsg_drop) 1208 strncat(options, "drop,", OPTSTRING); 1209 if (txmsg_apply) { 1210 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1211 strncat(options, tstr, OPTSTRING); 1212 } 1213 if (txmsg_cork) { 1214 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1215 strncat(options, tstr, OPTSTRING); 1216 } 1217 if (txmsg_start) { 1218 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1219 strncat(options, tstr, OPTSTRING); 1220 } 1221 if (txmsg_end) { 1222 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1223 strncat(options, tstr, OPTSTRING); 1224 } 1225 if (txmsg_start_pop) { 1226 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1227 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1228 strncat(options, tstr, OPTSTRING); 1229 } 1230 if (txmsg_ingress) 1231 strncat(options, "ingress,", OPTSTRING); 1232 if (txmsg_skb) 1233 strncat(options, "skb,", OPTSTRING); 1234 if (ktls) 1235 strncat(options, "ktls,", OPTSTRING); 1236 if (peek_flag) 1237 strncat(options, "peek,", OPTSTRING); 1238 } 1239 1240 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1241 { 1242 char *options = calloc(OPTSTRING, sizeof(char)); 1243 int err; 1244 1245 if (test == SENDPAGE) 1246 opt->sendpage = true; 1247 else 1248 opt->sendpage = false; 1249 1250 if (txmsg_drop) 1251 opt->drop_expected = true; 1252 else 1253 opt->drop_expected = false; 1254 1255 test_options(options); 1256 1257 if (opt->verbose) { 1258 fprintf(stdout, 1259 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1260 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1261 test_to_str(test), options); 1262 fflush(stdout); 1263 } 1264 err = run_options(opt, cgrp, test); 1265 if (opt->verbose) 1266 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1267 test_cnt++; 1268 !err ? passed++ : failed++; 1269 free(options); 1270 return err; 1271 } 1272 1273 static void test_exec(int cgrp, struct sockmap_options *opt) 1274 { 1275 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1276 int err; 1277 1278 if (type == 0) { 1279 test_start(); 1280 err = __test_exec(cgrp, SENDMSG, opt); 1281 if (err) 1282 test_fail(); 1283 } else { 1284 test_start(); 1285 err = __test_exec(cgrp, SENDPAGE, opt); 1286 if (err) 1287 test_fail(); 1288 } 1289 } 1290 1291 static void test_send_one(struct sockmap_options *opt, int cgrp) 1292 { 1293 opt->iov_length = 1; 1294 opt->iov_count = 1; 1295 opt->rate = 1; 1296 test_exec(cgrp, opt); 1297 1298 opt->iov_length = 1; 1299 opt->iov_count = 1024; 1300 opt->rate = 1; 1301 test_exec(cgrp, opt); 1302 1303 opt->iov_length = 1024; 1304 opt->iov_count = 1; 1305 opt->rate = 1; 1306 test_exec(cgrp, opt); 1307 1308 } 1309 1310 static void test_send_many(struct sockmap_options *opt, int cgrp) 1311 { 1312 opt->iov_length = 3; 1313 opt->iov_count = 1; 1314 opt->rate = 512; 1315 test_exec(cgrp, opt); 1316 1317 opt->rate = 100; 1318 opt->iov_count = 1; 1319 opt->iov_length = 5; 1320 test_exec(cgrp, opt); 1321 } 1322 1323 static void test_send_large(struct sockmap_options *opt, int cgrp) 1324 { 1325 opt->iov_length = 256; 1326 opt->iov_count = 1024; 1327 opt->rate = 2; 1328 test_exec(cgrp, opt); 1329 } 1330 1331 static void test_send(struct sockmap_options *opt, int cgrp) 1332 { 1333 test_send_one(opt, cgrp); 1334 test_send_many(opt, cgrp); 1335 test_send_large(opt, cgrp); 1336 sched_yield(); 1337 } 1338 1339 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1340 { 1341 /* Test small and large iov_count values with pass/redir/apply/cork */ 1342 txmsg_pass = 1; 1343 test_send(opt, cgrp); 1344 } 1345 1346 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1347 { 1348 txmsg_redir = 1; 1349 test_send(opt, cgrp); 1350 } 1351 1352 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1353 { 1354 txmsg_drop = 1; 1355 test_send(opt, cgrp); 1356 } 1357 1358 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1359 { 1360 txmsg_pass = txmsg_drop = 0; 1361 txmsg_ingress = txmsg_redir = 1; 1362 test_send(opt, cgrp); 1363 } 1364 1365 /* Test cork with hung data. This tests poor usage patterns where 1366 * cork can leave data on the ring if user program is buggy and 1367 * doesn't flush them somehow. They do take some time however 1368 * because they wait for a timeout. Test pass, redir and cork with 1369 * apply logic. Use cork size of 4097 with send_large to avoid 1370 * aligning cork size with send size. 1371 */ 1372 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1373 { 1374 txmsg_pass = 1; 1375 txmsg_redir = 0; 1376 txmsg_cork = 4097; 1377 txmsg_apply = 4097; 1378 test_send_large(opt, cgrp); 1379 1380 txmsg_pass = 0; 1381 txmsg_redir = 1; 1382 txmsg_apply = 0; 1383 txmsg_cork = 4097; 1384 test_send_large(opt, cgrp); 1385 1386 txmsg_pass = 0; 1387 txmsg_redir = 1; 1388 txmsg_apply = 4097; 1389 txmsg_cork = 4097; 1390 test_send_large(opt, cgrp); 1391 } 1392 1393 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1394 { 1395 /* Test basic start/end */ 1396 txmsg_start = 1; 1397 txmsg_end = 2; 1398 test_send(opt, cgrp); 1399 1400 /* Test >4k pull */ 1401 txmsg_start = 4096; 1402 txmsg_end = 9182; 1403 test_send_large(opt, cgrp); 1404 1405 /* Test pull + redirect */ 1406 txmsg_redir = 0; 1407 txmsg_start = 1; 1408 txmsg_end = 2; 1409 test_send(opt, cgrp); 1410 1411 /* Test pull + cork */ 1412 txmsg_redir = 0; 1413 txmsg_cork = 512; 1414 txmsg_start = 1; 1415 txmsg_end = 2; 1416 test_send_many(opt, cgrp); 1417 1418 /* Test pull + cork + redirect */ 1419 txmsg_redir = 1; 1420 txmsg_cork = 512; 1421 txmsg_start = 1; 1422 txmsg_end = 2; 1423 test_send_many(opt, cgrp); 1424 } 1425 1426 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1427 { 1428 /* Test basic pop */ 1429 txmsg_start_pop = 1; 1430 txmsg_pop = 2; 1431 test_send_many(opt, cgrp); 1432 1433 /* Test pop with >4k */ 1434 txmsg_start_pop = 4096; 1435 txmsg_pop = 4096; 1436 test_send_large(opt, cgrp); 1437 1438 /* Test pop + redirect */ 1439 txmsg_redir = 1; 1440 txmsg_start_pop = 1; 1441 txmsg_pop = 2; 1442 test_send_many(opt, cgrp); 1443 1444 /* Test pop + cork */ 1445 txmsg_redir = 0; 1446 txmsg_cork = 512; 1447 txmsg_start_pop = 1; 1448 txmsg_pop = 2; 1449 test_send_many(opt, cgrp); 1450 1451 /* Test pop + redirect + cork */ 1452 txmsg_redir = 1; 1453 txmsg_cork = 4; 1454 txmsg_start_pop = 1; 1455 txmsg_pop = 2; 1456 test_send_many(opt, cgrp); 1457 } 1458 1459 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1460 { 1461 /* Test basic push */ 1462 txmsg_start_push = 1; 1463 txmsg_end_push = 1; 1464 test_send(opt, cgrp); 1465 1466 /* Test push 4kB >4k */ 1467 txmsg_start_push = 4096; 1468 txmsg_end_push = 4096; 1469 test_send_large(opt, cgrp); 1470 1471 /* Test push + redirect */ 1472 txmsg_redir = 1; 1473 txmsg_start_push = 1; 1474 txmsg_end_push = 2; 1475 test_send_many(opt, cgrp); 1476 1477 /* Test push + cork */ 1478 txmsg_redir = 0; 1479 txmsg_cork = 512; 1480 txmsg_start_push = 1; 1481 txmsg_end_push = 2; 1482 test_send_many(opt, cgrp); 1483 } 1484 1485 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1486 { 1487 txmsg_start_push = 1; 1488 txmsg_end_push = 10; 1489 txmsg_start_pop = 5; 1490 txmsg_pop = 4; 1491 test_send_large(opt, cgrp); 1492 } 1493 1494 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1495 { 1496 txmsg_pass = 1; 1497 txmsg_redir = 0; 1498 txmsg_apply = 1; 1499 txmsg_cork = 0; 1500 test_send_one(opt, cgrp); 1501 1502 txmsg_pass = 0; 1503 txmsg_redir = 1; 1504 txmsg_apply = 1; 1505 txmsg_cork = 0; 1506 test_send_one(opt, cgrp); 1507 1508 txmsg_pass = 1; 1509 txmsg_redir = 0; 1510 txmsg_apply = 1024; 1511 txmsg_cork = 0; 1512 test_send_large(opt, cgrp); 1513 1514 txmsg_pass = 0; 1515 txmsg_redir = 1; 1516 txmsg_apply = 1024; 1517 txmsg_cork = 0; 1518 test_send_large(opt, cgrp); 1519 } 1520 1521 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1522 { 1523 txmsg_pass = 1; 1524 txmsg_redir = 0; 1525 txmsg_apply = 0; 1526 txmsg_cork = 1; 1527 test_send(opt, cgrp); 1528 1529 txmsg_pass = 1; 1530 txmsg_redir = 0; 1531 txmsg_apply = 1; 1532 txmsg_cork = 1; 1533 test_send(opt, cgrp); 1534 } 1535 1536 char *map_names[] = { 1537 "sock_map", 1538 "sock_map_txmsg", 1539 "sock_map_redir", 1540 "sock_apply_bytes", 1541 "sock_cork_bytes", 1542 "sock_bytes", 1543 "sock_redir_flags", 1544 "sock_skb_opts", 1545 }; 1546 1547 int prog_attach_type[] = { 1548 BPF_SK_SKB_STREAM_PARSER, 1549 BPF_SK_SKB_STREAM_VERDICT, 1550 BPF_CGROUP_SOCK_OPS, 1551 BPF_SK_MSG_VERDICT, 1552 BPF_SK_MSG_VERDICT, 1553 BPF_SK_MSG_VERDICT, 1554 BPF_SK_MSG_VERDICT, 1555 BPF_SK_MSG_VERDICT, 1556 BPF_SK_MSG_VERDICT, 1557 BPF_SK_MSG_VERDICT, 1558 }; 1559 1560 int prog_type[] = { 1561 BPF_PROG_TYPE_SK_SKB, 1562 BPF_PROG_TYPE_SK_SKB, 1563 BPF_PROG_TYPE_SOCK_OPS, 1564 BPF_PROG_TYPE_SK_MSG, 1565 BPF_PROG_TYPE_SK_MSG, 1566 BPF_PROG_TYPE_SK_MSG, 1567 BPF_PROG_TYPE_SK_MSG, 1568 BPF_PROG_TYPE_SK_MSG, 1569 BPF_PROG_TYPE_SK_MSG, 1570 BPF_PROG_TYPE_SK_MSG, 1571 }; 1572 1573 static int populate_progs(char *bpf_file) 1574 { 1575 struct bpf_program *prog; 1576 struct bpf_object *obj; 1577 int i = 0; 1578 long err; 1579 1580 obj = bpf_object__open(bpf_file); 1581 err = libbpf_get_error(obj); 1582 if (err) { 1583 char err_buf[256]; 1584 1585 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1586 printf("Unable to load eBPF objects in file '%s' : %s\n", 1587 bpf_file, err_buf); 1588 return -1; 1589 } 1590 1591 bpf_object__for_each_program(prog, obj) { 1592 bpf_program__set_type(prog, prog_type[i]); 1593 bpf_program__set_expected_attach_type(prog, 1594 prog_attach_type[i]); 1595 i++; 1596 } 1597 1598 i = bpf_object__load(obj); 1599 i = 0; 1600 bpf_object__for_each_program(prog, obj) { 1601 prog_fd[i] = bpf_program__fd(prog); 1602 i++; 1603 } 1604 1605 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 1606 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1607 map_fd[i] = bpf_map__fd(maps[i]); 1608 if (map_fd[i] < 0) { 1609 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1610 map_fd[i], strerror(errno)); 1611 return -1; 1612 } 1613 } 1614 1615 return 0; 1616 } 1617 1618 struct _test test[] = { 1619 {"txmsg test passthrough", test_txmsg_pass}, 1620 {"txmsg test redirect", test_txmsg_redir}, 1621 {"txmsg test drop", test_txmsg_drop}, 1622 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1623 {"txmsg test apply", test_txmsg_apply}, 1624 {"txmsg test cork", test_txmsg_cork}, 1625 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1626 {"txmsg test push_data", test_txmsg_push}, 1627 {"txmsg test pull-data", test_txmsg_pull}, 1628 {"txmsg test pop-data", test_txmsg_pop}, 1629 {"txmsg test push/pop data", test_txmsg_push_pop}, 1630 }; 1631 1632 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1633 { 1634 char *entry, *ptr; 1635 1636 if (!opt->whitelist) 1637 return 0; 1638 ptr = strdup(opt->whitelist); 1639 if (!ptr) 1640 return -ENOMEM; 1641 entry = strtok(ptr, ","); 1642 while (entry) { 1643 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1644 strstr(opt->map, entry) != 0 || 1645 strstr(t->title, entry) != 0) 1646 return 0; 1647 entry = strtok(NULL, ","); 1648 } 1649 return -EINVAL; 1650 } 1651 1652 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 1653 { 1654 char *entry, *ptr; 1655 1656 if (!opt->blacklist) 1657 return -EINVAL; 1658 ptr = strdup(opt->blacklist); 1659 if (!ptr) 1660 return -ENOMEM; 1661 entry = strtok(ptr, ","); 1662 while (entry) { 1663 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 1664 strstr(opt->map, entry) != 0 || 1665 strstr(t->title, entry) != 0) 1666 return 0; 1667 entry = strtok(NULL, ","); 1668 } 1669 return -EINVAL; 1670 } 1671 1672 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 1673 { 1674 int i, err; 1675 1676 err = populate_progs(opt->map); 1677 if (err < 0) { 1678 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1679 return err; 1680 } 1681 1682 /* Tests basic commands and APIs */ 1683 for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) { 1684 struct _test t = test[i]; 1685 1686 if (check_whitelist(&t, opt) != 0) 1687 continue; 1688 if (check_blacklist(&t, opt) == 0) 1689 continue; 1690 1691 test_start_subtest(&t, opt); 1692 t.tester(cg_fd, opt); 1693 test_end_subtest(); 1694 } 1695 1696 return err; 1697 } 1698 1699 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 1700 { 1701 opt->map = BPF_SOCKMAP_FILENAME; 1702 __test_selftests(cg_fd, opt); 1703 } 1704 1705 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 1706 { 1707 opt->map = BPF_SOCKHASH_FILENAME; 1708 __test_selftests(cg_fd, opt); 1709 } 1710 1711 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 1712 { 1713 opt->map = BPF_SOCKHASH_FILENAME; 1714 opt->prepend = "ktls"; 1715 ktls = 1; 1716 __test_selftests(cg_fd, opt); 1717 ktls = 0; 1718 } 1719 1720 static int test_selftest(int cg_fd, struct sockmap_options *opt) 1721 { 1722 1723 test_selftests_sockmap(cg_fd, opt); 1724 test_selftests_sockhash(cg_fd, opt); 1725 test_selftests_ktls(cg_fd, opt); 1726 test_print_results(); 1727 return 0; 1728 } 1729 1730 int main(int argc, char **argv) 1731 { 1732 int iov_count = 1, length = 1024, rate = 1; 1733 struct sockmap_options options = {0}; 1734 int opt, longindex, err, cg_fd = 0; 1735 char *bpf_file = BPF_SOCKMAP_FILENAME; 1736 int test = SELFTESTS; 1737 bool cg_created = 0; 1738 1739 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 1740 long_options, &longindex)) != -1) { 1741 switch (opt) { 1742 case 's': 1743 txmsg_start = atoi(optarg); 1744 break; 1745 case 'e': 1746 txmsg_end = atoi(optarg); 1747 break; 1748 case 'p': 1749 txmsg_start_push = atoi(optarg); 1750 break; 1751 case 'q': 1752 txmsg_end_push = atoi(optarg); 1753 break; 1754 case 'w': 1755 txmsg_start_pop = atoi(optarg); 1756 break; 1757 case 'x': 1758 txmsg_pop = atoi(optarg); 1759 break; 1760 case 'a': 1761 txmsg_apply = atoi(optarg); 1762 break; 1763 case 'k': 1764 txmsg_cork = atoi(optarg); 1765 break; 1766 case 'c': 1767 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1768 if (cg_fd < 0) { 1769 fprintf(stderr, 1770 "ERROR: (%i) open cg path failed: %s\n", 1771 cg_fd, optarg); 1772 return cg_fd; 1773 } 1774 break; 1775 case 'r': 1776 rate = atoi(optarg); 1777 break; 1778 case 'v': 1779 options.verbose = 1; 1780 if (optarg) 1781 options.verbose = atoi(optarg); 1782 break; 1783 case 'i': 1784 iov_count = atoi(optarg); 1785 break; 1786 case 'l': 1787 length = atoi(optarg); 1788 break; 1789 case 'd': 1790 options.data_test = true; 1791 break; 1792 case 't': 1793 if (strcmp(optarg, "ping") == 0) { 1794 test = PING_PONG; 1795 } else if (strcmp(optarg, "sendmsg") == 0) { 1796 test = SENDMSG; 1797 } else if (strcmp(optarg, "base") == 0) { 1798 test = BASE; 1799 } else if (strcmp(optarg, "base_sendpage") == 0) { 1800 test = BASE_SENDPAGE; 1801 } else if (strcmp(optarg, "sendpage") == 0) { 1802 test = SENDPAGE; 1803 } else { 1804 usage(argv); 1805 return -1; 1806 } 1807 break; 1808 case 'n': 1809 options.whitelist = strdup(optarg); 1810 if (!options.whitelist) 1811 return -ENOMEM; 1812 break; 1813 case 'b': 1814 options.blacklist = strdup(optarg); 1815 if (!options.blacklist) 1816 return -ENOMEM; 1817 case 0: 1818 break; 1819 case 'h': 1820 default: 1821 usage(argv); 1822 return -1; 1823 } 1824 } 1825 1826 if (!cg_fd) { 1827 if (setup_cgroup_environment()) { 1828 fprintf(stderr, "ERROR: cgroup env failed\n"); 1829 return -EINVAL; 1830 } 1831 1832 cg_fd = create_and_get_cgroup(CG_PATH); 1833 if (cg_fd < 0) { 1834 fprintf(stderr, 1835 "ERROR: (%i) open cg path failed: %s\n", 1836 cg_fd, strerror(errno)); 1837 return cg_fd; 1838 } 1839 1840 if (join_cgroup(CG_PATH)) { 1841 fprintf(stderr, "ERROR: failed to join cgroup\n"); 1842 return -EINVAL; 1843 } 1844 cg_created = 1; 1845 } 1846 1847 if (test == SELFTESTS) { 1848 err = test_selftest(cg_fd, &options); 1849 goto out; 1850 } 1851 1852 err = populate_progs(bpf_file); 1853 if (err) { 1854 fprintf(stderr, "populate program: (%s) %s\n", 1855 bpf_file, strerror(errno)); 1856 return 1; 1857 } 1858 running = 1; 1859 1860 /* catch SIGINT */ 1861 signal(SIGINT, running_handler); 1862 1863 options.iov_count = iov_count; 1864 options.iov_length = length; 1865 options.rate = rate; 1866 1867 err = run_options(&options, cg_fd, test); 1868 out: 1869 if (options.whitelist) 1870 free(options.whitelist); 1871 if (options.blacklist) 1872 free(options.blacklist); 1873 if (cg_created) 1874 cleanup_cgroup_environment(); 1875 close(cg_fd); 1876 return err; 1877 } 1878 1879 void running_handler(int a) 1880 { 1881 running = 0; 1882 } 1883