1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/types.h> 22 #include <sys/sendfile.h> 23 24 #include <linux/netlink.h> 25 #include <linux/socket.h> 26 #include <linux/sock_diag.h> 27 #include <linux/bpf.h> 28 #include <linux/if_link.h> 29 #include <linux/tls.h> 30 #include <assert.h> 31 #include <libgen.h> 32 33 #include <getopt.h> 34 35 #include <bpf/bpf.h> 36 #include <bpf/libbpf.h> 37 38 #include "bpf_util.h" 39 #include "cgroup_helpers.h" 40 41 int running; 42 static void running_handler(int a); 43 44 #ifndef TCP_ULP 45 # define TCP_ULP 31 46 #endif 47 #ifndef SOL_TLS 48 # define SOL_TLS 282 49 #endif 50 51 /* randomly selected ports for testing on lo */ 52 #define S1_PORT 10000 53 #define S2_PORT 10001 54 55 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" 56 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" 57 #define CG_PATH "/sockmap" 58 59 #define EDATAINTEGRITY 2001 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[9]; 67 struct bpf_map *maps[9]; 68 int prog_fd[9]; 69 70 int txmsg_pass; 71 int txmsg_redir; 72 int txmsg_drop; 73 int txmsg_apply; 74 int txmsg_cork; 75 int txmsg_start; 76 int txmsg_end; 77 int txmsg_start_push; 78 int txmsg_end_push; 79 int txmsg_start_pop; 80 int txmsg_pop; 81 int txmsg_ingress; 82 int txmsg_redir_skb; 83 int txmsg_ktls_skb; 84 int txmsg_ktls_skb_drop; 85 int txmsg_ktls_skb_redir; 86 int ktls; 87 int peek_flag; 88 int skb_use_parser; 89 int txmsg_omit_skb_parser; 90 int verify_push_start; 91 int verify_push_len; 92 int verify_pop_start; 93 int verify_pop_len; 94 95 static const struct option long_options[] = { 96 {"help", no_argument, NULL, 'h' }, 97 {"cgroup", required_argument, NULL, 'c' }, 98 {"rate", required_argument, NULL, 'r' }, 99 {"verbose", optional_argument, NULL, 'v' }, 100 {"iov_count", required_argument, NULL, 'i' }, 101 {"length", required_argument, NULL, 'l' }, 102 {"test", required_argument, NULL, 't' }, 103 {"data_test", no_argument, NULL, 'd' }, 104 {"txmsg", no_argument, &txmsg_pass, 1 }, 105 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 106 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 107 {"txmsg_apply", required_argument, NULL, 'a'}, 108 {"txmsg_cork", required_argument, NULL, 'k'}, 109 {"txmsg_start", required_argument, NULL, 's'}, 110 {"txmsg_end", required_argument, NULL, 'e'}, 111 {"txmsg_start_push", required_argument, NULL, 'p'}, 112 {"txmsg_end_push", required_argument, NULL, 'q'}, 113 {"txmsg_start_pop", required_argument, NULL, 'w'}, 114 {"txmsg_pop", required_argument, NULL, 'x'}, 115 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 116 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 }, 117 {"ktls", no_argument, &ktls, 1 }, 118 {"peek", no_argument, &peek_flag, 1 }, 119 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1}, 120 {"whitelist", required_argument, NULL, 'n' }, 121 {"blacklist", required_argument, NULL, 'b' }, 122 {0, 0, NULL, 0 } 123 }; 124 125 struct test_env { 126 const char *type; 127 const char *subtest; 128 const char *prepend; 129 130 int test_num; 131 int subtest_num; 132 133 int succ_cnt; 134 int fail_cnt; 135 int fail_last; 136 }; 137 138 struct test_env env; 139 140 struct sockmap_options { 141 int verbose; 142 bool base; 143 bool sendpage; 144 bool data_test; 145 bool drop_expected; 146 bool check_recved_len; 147 bool tx_wait_mem; 148 int iov_count; 149 int iov_length; 150 int rate; 151 char *map; 152 char *whitelist; 153 char *blacklist; 154 char *prepend; 155 }; 156 157 struct _test { 158 char *title; 159 void (*tester)(int cg_fd, struct sockmap_options *opt); 160 }; 161 162 static void test_start(void) 163 { 164 env.subtest_num++; 165 } 166 167 static void test_fail(void) 168 { 169 env.fail_cnt++; 170 } 171 172 static void test_pass(void) 173 { 174 env.succ_cnt++; 175 } 176 177 static void test_reset(void) 178 { 179 txmsg_start = txmsg_end = 0; 180 txmsg_start_pop = txmsg_pop = 0; 181 txmsg_start_push = txmsg_end_push = 0; 182 txmsg_pass = txmsg_drop = txmsg_redir = 0; 183 txmsg_apply = txmsg_cork = 0; 184 txmsg_ingress = txmsg_redir_skb = 0; 185 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0; 186 txmsg_omit_skb_parser = 0; 187 skb_use_parser = 0; 188 } 189 190 static int test_start_subtest(const struct _test *t, struct sockmap_options *o) 191 { 192 env.type = o->map; 193 env.subtest = t->title; 194 env.prepend = o->prepend; 195 env.test_num++; 196 env.subtest_num = 0; 197 env.fail_last = env.fail_cnt; 198 test_reset(); 199 return 0; 200 } 201 202 static void test_end_subtest(void) 203 { 204 int error = env.fail_cnt - env.fail_last; 205 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME); 206 207 if (!error) 208 test_pass(); 209 210 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n", 211 env.test_num, env.subtest_num, 212 !type ? "sockmap" : "sockhash", 213 env.prepend ? : "", 214 env.subtest, error ? "FAIL" : "OK"); 215 } 216 217 static void test_print_results(void) 218 { 219 fprintf(stdout, "Pass: %d Fail: %d\n", 220 env.succ_cnt, env.fail_cnt); 221 } 222 223 static void usage(char *argv[]) 224 { 225 int i; 226 227 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 228 printf(" options:\n"); 229 for (i = 0; long_options[i].name != 0; i++) { 230 printf(" --%-12s", long_options[i].name); 231 if (long_options[i].flag != NULL) 232 printf(" flag (internal value:%d)\n", 233 *long_options[i].flag); 234 else 235 printf(" -%c\n", long_options[i].val); 236 } 237 printf("\n"); 238 } 239 240 char *sock_to_string(int s) 241 { 242 if (s == c1) 243 return "client1"; 244 else if (s == c2) 245 return "client2"; 246 else if (s == s1) 247 return "server1"; 248 else if (s == s2) 249 return "server2"; 250 else if (s == p1) 251 return "peer1"; 252 else if (s == p2) 253 return "peer2"; 254 else 255 return "unknown"; 256 } 257 258 static int sockmap_init_ktls(int verbose, int s) 259 { 260 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 261 .info = { 262 .version = TLS_1_2_VERSION, 263 .cipher_type = TLS_CIPHER_AES_GCM_128, 264 }, 265 }; 266 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 267 .info = { 268 .version = TLS_1_2_VERSION, 269 .cipher_type = TLS_CIPHER_AES_GCM_128, 270 }, 271 }; 272 int so_buf = 6553500; 273 int err; 274 275 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 276 if (err) { 277 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 278 return -EINVAL; 279 } 280 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 281 if (err) { 282 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 283 return -EINVAL; 284 } 285 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 286 if (err) { 287 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 288 return -EINVAL; 289 } 290 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 291 if (err) { 292 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 293 return -EINVAL; 294 } 295 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 296 if (err) { 297 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 298 return -EINVAL; 299 } 300 301 if (verbose) 302 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 303 return 0; 304 } 305 static int sockmap_init_sockets(int verbose) 306 { 307 int i, err, one = 1; 308 struct sockaddr_in addr; 309 int *fds[4] = {&s1, &s2, &c1, &c2}; 310 311 s1 = s2 = p1 = p2 = c1 = c2 = 0; 312 313 /* Init sockets */ 314 for (i = 0; i < 4; i++) { 315 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 316 if (*fds[i] < 0) { 317 perror("socket s1 failed()"); 318 return errno; 319 } 320 } 321 322 /* Allow reuse */ 323 for (i = 0; i < 2; i++) { 324 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 325 (char *)&one, sizeof(one)); 326 if (err) { 327 perror("setsockopt failed()"); 328 return errno; 329 } 330 } 331 332 /* Non-blocking sockets */ 333 for (i = 0; i < 2; i++) { 334 err = ioctl(*fds[i], FIONBIO, (char *)&one); 335 if (err < 0) { 336 perror("ioctl s1 failed()"); 337 return errno; 338 } 339 } 340 341 /* Bind server sockets */ 342 memset(&addr, 0, sizeof(struct sockaddr_in)); 343 addr.sin_family = AF_INET; 344 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 345 346 addr.sin_port = htons(S1_PORT); 347 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 348 if (err < 0) { 349 perror("bind s1 failed()"); 350 return errno; 351 } 352 353 addr.sin_port = htons(S2_PORT); 354 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 355 if (err < 0) { 356 perror("bind s2 failed()"); 357 return errno; 358 } 359 360 /* Listen server sockets */ 361 addr.sin_port = htons(S1_PORT); 362 err = listen(s1, 32); 363 if (err < 0) { 364 perror("listen s1 failed()"); 365 return errno; 366 } 367 368 addr.sin_port = htons(S2_PORT); 369 err = listen(s2, 32); 370 if (err < 0) { 371 perror("listen s1 failed()"); 372 return errno; 373 } 374 375 /* Initiate Connect */ 376 addr.sin_port = htons(S1_PORT); 377 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 378 if (err < 0 && errno != EINPROGRESS) { 379 perror("connect c1 failed()"); 380 return errno; 381 } 382 383 addr.sin_port = htons(S2_PORT); 384 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 385 if (err < 0 && errno != EINPROGRESS) { 386 perror("connect c2 failed()"); 387 return errno; 388 } else if (err < 0) { 389 err = 0; 390 } 391 392 /* Accept Connecrtions */ 393 p1 = accept(s1, NULL, NULL); 394 if (p1 < 0) { 395 perror("accept s1 failed()"); 396 return errno; 397 } 398 399 p2 = accept(s2, NULL, NULL); 400 if (p2 < 0) { 401 perror("accept s1 failed()"); 402 return errno; 403 } 404 405 if (verbose > 1) { 406 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 407 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 408 c1, s1, c2, s2); 409 } 410 return 0; 411 } 412 413 struct msg_stats { 414 size_t bytes_sent; 415 size_t bytes_recvd; 416 struct timespec start; 417 struct timespec end; 418 }; 419 420 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 421 struct msg_stats *s, 422 struct sockmap_options *opt) 423 { 424 bool drop = opt->drop_expected; 425 unsigned char k = 0; 426 int i, j, fp; 427 FILE *file; 428 429 file = tmpfile(); 430 if (!file) { 431 perror("create file for sendpage"); 432 return 1; 433 } 434 for (i = 0; i < cnt; i++, k = 0) { 435 for (j = 0; j < iov_length; j++, k++) 436 fwrite(&k, sizeof(char), 1, file); 437 } 438 fflush(file); 439 fseek(file, 0, SEEK_SET); 440 441 fp = fileno(file); 442 443 clock_gettime(CLOCK_MONOTONIC, &s->start); 444 for (i = 0; i < cnt; i++) { 445 int sent; 446 447 errno = 0; 448 sent = sendfile(fd, fp, NULL, iov_length); 449 450 if (!drop && sent < 0) { 451 perror("sendpage loop error"); 452 fclose(file); 453 return sent; 454 } else if (drop && sent >= 0) { 455 printf("sendpage loop error expected: %i errno %i\n", 456 sent, errno); 457 fclose(file); 458 return -EIO; 459 } 460 461 if (sent > 0) 462 s->bytes_sent += sent; 463 } 464 clock_gettime(CLOCK_MONOTONIC, &s->end); 465 fclose(file); 466 return 0; 467 } 468 469 static void msg_free_iov(struct msghdr *msg) 470 { 471 int i; 472 473 for (i = 0; i < msg->msg_iovlen; i++) 474 free(msg->msg_iov[i].iov_base); 475 free(msg->msg_iov); 476 msg->msg_iov = NULL; 477 msg->msg_iovlen = 0; 478 } 479 480 static int msg_alloc_iov(struct msghdr *msg, 481 int iov_count, int iov_length, 482 bool data, bool xmit) 483 { 484 unsigned char k = 0; 485 struct iovec *iov; 486 int i; 487 488 iov = calloc(iov_count, sizeof(struct iovec)); 489 if (!iov) 490 return errno; 491 492 for (i = 0; i < iov_count; i++) { 493 unsigned char *d = calloc(iov_length, sizeof(char)); 494 495 if (!d) { 496 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 497 goto unwind_iov; 498 } 499 iov[i].iov_base = d; 500 iov[i].iov_len = iov_length; 501 502 if (data && xmit) { 503 int j; 504 505 for (j = 0; j < iov_length; j++) 506 d[j] = k++; 507 } 508 } 509 510 msg->msg_iov = iov; 511 msg->msg_iovlen = iov_count; 512 513 return 0; 514 unwind_iov: 515 for (i--; i >= 0 ; i--) 516 free(msg->msg_iov[i].iov_base); 517 return -ENOMEM; 518 } 519 520 /* In push or pop test, we need to do some calculations for msg_verify_data */ 521 static void msg_verify_date_prep(void) 522 { 523 int push_range_end = txmsg_start_push + txmsg_end_push - 1; 524 int pop_range_end = txmsg_start_pop + txmsg_pop - 1; 525 526 if (txmsg_end_push && txmsg_pop && 527 txmsg_start_push <= pop_range_end && txmsg_start_pop <= push_range_end) { 528 /* The push range and the pop range overlap */ 529 int overlap_len; 530 531 verify_push_start = txmsg_start_push; 532 verify_pop_start = txmsg_start_pop; 533 if (txmsg_start_push < txmsg_start_pop) 534 overlap_len = min(push_range_end - txmsg_start_pop + 1, txmsg_pop); 535 else 536 overlap_len = min(pop_range_end - txmsg_start_push + 1, txmsg_end_push); 537 verify_push_len = max(txmsg_end_push - overlap_len, 0); 538 verify_pop_len = max(txmsg_pop - overlap_len, 0); 539 } else { 540 /* Otherwise */ 541 verify_push_start = txmsg_start_push; 542 verify_pop_start = txmsg_start_pop; 543 verify_push_len = txmsg_end_push; 544 verify_pop_len = txmsg_pop; 545 } 546 } 547 548 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz, 549 unsigned char *k_p, int *bytes_cnt_p, 550 int *check_cnt_p, int *push_p) 551 { 552 int bytes_cnt = *bytes_cnt_p, check_cnt = *check_cnt_p, push = *push_p; 553 unsigned char k = *k_p; 554 int i, j; 555 556 for (i = 0, j = 0; i < msg->msg_iovlen && size; i++, j = 0) { 557 unsigned char *d = msg->msg_iov[i].iov_base; 558 559 /* Special case test for skb ingress + ktls */ 560 if (i == 0 && txmsg_ktls_skb) { 561 if (msg->msg_iov[i].iov_len < 4) 562 return -EDATAINTEGRITY; 563 if (memcmp(d, "PASS", 4) != 0) { 564 fprintf(stderr, 565 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", 566 i, 0, d[0], d[1], d[2], d[3]); 567 return -EDATAINTEGRITY; 568 } 569 j = 4; /* advance index past PASS header */ 570 } 571 572 for (; j < msg->msg_iov[i].iov_len && size; j++) { 573 if (push > 0 && 574 check_cnt == verify_push_start + verify_push_len - push) { 575 int skipped; 576 revisit_push: 577 skipped = push; 578 if (j + push >= msg->msg_iov[i].iov_len) 579 skipped = msg->msg_iov[i].iov_len - j; 580 push -= skipped; 581 size -= skipped; 582 j += skipped - 1; 583 check_cnt += skipped; 584 continue; 585 } 586 587 if (verify_pop_len > 0 && check_cnt == verify_pop_start) { 588 bytes_cnt += verify_pop_len; 589 check_cnt += verify_pop_len; 590 k += verify_pop_len; 591 592 if (bytes_cnt == chunk_sz) { 593 k = 0; 594 bytes_cnt = 0; 595 check_cnt = 0; 596 push = verify_push_len; 597 } 598 599 if (push > 0 && 600 check_cnt == verify_push_start + verify_push_len - push) 601 goto revisit_push; 602 } 603 604 if (d[j] != k++) { 605 fprintf(stderr, 606 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 607 i, j, d[j], k - 1, d[j+1], k); 608 return -EDATAINTEGRITY; 609 } 610 bytes_cnt++; 611 check_cnt++; 612 if (bytes_cnt == chunk_sz) { 613 k = 0; 614 bytes_cnt = 0; 615 check_cnt = 0; 616 push = verify_push_len; 617 } 618 size--; 619 } 620 } 621 *k_p = k; 622 *bytes_cnt_p = bytes_cnt; 623 *check_cnt_p = check_cnt; 624 *push_p = push; 625 return 0; 626 } 627 628 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 629 struct msg_stats *s, bool tx, 630 struct sockmap_options *opt) 631 { 632 struct msghdr msg = {0}, msg_peek = {0}; 633 int err, i, flags = MSG_NOSIGNAL; 634 bool drop = opt->drop_expected; 635 bool data = opt->data_test; 636 int iov_alloc_length = iov_length; 637 638 if (!tx && opt->check_recved_len) 639 iov_alloc_length *= 2; 640 641 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx); 642 if (err) 643 goto out_errno; 644 if (peek_flag) { 645 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 646 if (err) 647 goto out_errno; 648 } 649 650 if (tx) { 651 clock_gettime(CLOCK_MONOTONIC, &s->start); 652 for (i = 0; i < cnt; i++) { 653 int sent; 654 655 errno = 0; 656 sent = sendmsg(fd, &msg, flags); 657 658 if (!drop && sent < 0) { 659 if (opt->tx_wait_mem && errno == EACCES) { 660 errno = 0; 661 goto out_errno; 662 } 663 perror("sendmsg loop error"); 664 goto out_errno; 665 } else if (drop && sent >= 0) { 666 fprintf(stderr, 667 "sendmsg loop error expected: %i errno %i\n", 668 sent, errno); 669 errno = -EIO; 670 goto out_errno; 671 } 672 if (sent > 0) 673 s->bytes_sent += sent; 674 } 675 clock_gettime(CLOCK_MONOTONIC, &s->end); 676 } else { 677 float total_bytes, txmsg_pop_total, txmsg_push_total; 678 int slct, recvp = 0, recv, max_fd = fd; 679 int fd_flags = O_NONBLOCK; 680 struct timeval timeout; 681 unsigned char k = 0; 682 int bytes_cnt = 0; 683 int check_cnt = 0; 684 int push = 0; 685 fd_set w; 686 687 fcntl(fd, fd_flags); 688 /* Account for pop bytes noting each iteration of apply will 689 * call msg_pop_data helper so we need to account for this 690 * by calculating the number of apply iterations. Note user 691 * of the tool can create cases where no data is sent by 692 * manipulating pop/push/pull/etc. For example txmsg_apply 1 693 * with txmsg_pop 1 will try to apply 1B at a time but each 694 * iteration will then pop 1B so no data will ever be sent. 695 * This is really only useful for testing edge cases in code 696 * paths. 697 */ 698 total_bytes = (float)iov_length * (float)cnt; 699 if (!opt->sendpage) 700 total_bytes *= (float)iov_count; 701 if (txmsg_apply) { 702 txmsg_push_total = txmsg_end_push * (total_bytes / txmsg_apply); 703 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply); 704 } else { 705 txmsg_push_total = txmsg_end_push * cnt; 706 txmsg_pop_total = txmsg_pop * cnt; 707 } 708 total_bytes += txmsg_push_total; 709 total_bytes -= txmsg_pop_total; 710 if (data) { 711 msg_verify_date_prep(); 712 push = verify_push_len; 713 } 714 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 715 if (err < 0) 716 perror("recv start time"); 717 while (s->bytes_recvd < total_bytes) { 718 if (txmsg_cork) { 719 timeout.tv_sec = 0; 720 timeout.tv_usec = 300000; 721 } else { 722 timeout.tv_sec = 3; 723 timeout.tv_usec = 0; 724 } 725 726 /* FD sets */ 727 FD_ZERO(&w); 728 FD_SET(fd, &w); 729 730 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 731 if (slct == -1) { 732 perror("select()"); 733 clock_gettime(CLOCK_MONOTONIC, &s->end); 734 goto out_errno; 735 } else if (!slct) { 736 if (opt->verbose) 737 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 738 errno = -EIO; 739 clock_gettime(CLOCK_MONOTONIC, &s->end); 740 goto out_errno; 741 } 742 743 if (opt->tx_wait_mem) { 744 FD_ZERO(&w); 745 FD_SET(fd, &w); 746 slct = select(max_fd + 1, NULL, NULL, &w, &timeout); 747 errno = 0; 748 close(fd); 749 goto out_errno; 750 } 751 752 errno = 0; 753 if (peek_flag) { 754 flags |= MSG_PEEK; 755 recvp = recvmsg(fd, &msg_peek, flags); 756 if (recvp < 0) { 757 if (errno != EWOULDBLOCK) { 758 clock_gettime(CLOCK_MONOTONIC, &s->end); 759 goto out_errno; 760 } 761 } 762 flags = 0; 763 } 764 765 recv = recvmsg(fd, &msg, flags); 766 if (recv < 0) { 767 if (errno != EWOULDBLOCK) { 768 clock_gettime(CLOCK_MONOTONIC, &s->end); 769 perror("recv failed()"); 770 goto out_errno; 771 } 772 } 773 774 if (recv > 0) 775 s->bytes_recvd += recv; 776 777 if (opt->check_recved_len && s->bytes_recvd > total_bytes) { 778 errno = EMSGSIZE; 779 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n", 780 s->bytes_recvd, total_bytes); 781 goto out_errno; 782 } 783 784 if (data) { 785 int chunk_sz = opt->sendpage ? 786 iov_length : 787 iov_length * iov_count; 788 789 errno = msg_verify_data(&msg, recv, chunk_sz, &k, &bytes_cnt, 790 &check_cnt, &push); 791 if (errno) { 792 perror("data verify msg failed"); 793 goto out_errno; 794 } 795 if (recvp) { 796 errno = msg_verify_data(&msg_peek, 797 recvp, 798 chunk_sz, 799 &k, 800 &bytes_cnt, 801 &check_cnt, 802 &push); 803 if (errno) { 804 perror("data verify msg_peek failed"); 805 goto out_errno; 806 } 807 } 808 } 809 } 810 clock_gettime(CLOCK_MONOTONIC, &s->end); 811 } 812 813 msg_free_iov(&msg); 814 msg_free_iov(&msg_peek); 815 return err; 816 out_errno: 817 msg_free_iov(&msg); 818 msg_free_iov(&msg_peek); 819 return errno; 820 } 821 822 static float giga = 1000000000; 823 824 static inline float sentBps(struct msg_stats s) 825 { 826 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 827 } 828 829 static inline float recvdBps(struct msg_stats s) 830 { 831 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 832 } 833 834 static int sendmsg_test(struct sockmap_options *opt) 835 { 836 float sent_Bps = 0, recvd_Bps = 0; 837 int rx_fd, txpid, rxpid, err = 0; 838 struct msg_stats s = {0}; 839 int iov_count = opt->iov_count; 840 int iov_buf = opt->iov_length; 841 int rx_status, tx_status; 842 int cnt = opt->rate; 843 844 errno = 0; 845 846 if (opt->base) 847 rx_fd = p1; 848 else 849 rx_fd = p2; 850 851 if (ktls) { 852 /* Redirecting into non-TLS socket which sends into a TLS 853 * socket is not a valid test. So in this case lets not 854 * enable kTLS but still run the test. 855 */ 856 if (!txmsg_redir || txmsg_ingress) { 857 err = sockmap_init_ktls(opt->verbose, rx_fd); 858 if (err) 859 return err; 860 } 861 err = sockmap_init_ktls(opt->verbose, c1); 862 if (err) 863 return err; 864 } 865 866 if (opt->tx_wait_mem) { 867 struct timeval timeout; 868 int rxtx_buf_len = 1024; 869 870 timeout.tv_sec = 3; 871 timeout.tv_usec = 0; 872 873 err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); 874 err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); 875 err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); 876 if (err) { 877 perror("setsockopt failed()"); 878 return errno; 879 } 880 } 881 882 rxpid = fork(); 883 if (rxpid == 0) { 884 if (opt->drop_expected || txmsg_ktls_skb_drop) 885 _exit(0); 886 887 if (!iov_buf) /* zero bytes sent case */ 888 _exit(0); 889 890 if (opt->sendpage) 891 iov_count = 1; 892 err = msg_loop(rx_fd, iov_count, iov_buf, 893 cnt, &s, false, opt); 894 if (opt->verbose > 1) 895 fprintf(stderr, 896 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 897 iov_count, iov_buf, cnt, err); 898 if (s.end.tv_sec - s.start.tv_sec) { 899 sent_Bps = sentBps(s); 900 recvd_Bps = recvdBps(s); 901 } 902 if (opt->verbose > 1) 903 fprintf(stdout, 904 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 905 s.bytes_sent, sent_Bps, sent_Bps/giga, 906 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 907 peek_flag ? "(peek_msg)" : ""); 908 if (err && err != -EDATAINTEGRITY && txmsg_cork) 909 err = 0; 910 exit(err ? 1 : 0); 911 } else if (rxpid == -1) { 912 perror("msg_loop_rx"); 913 return errno; 914 } 915 916 if (opt->tx_wait_mem) 917 close(c2); 918 919 txpid = fork(); 920 if (txpid == 0) { 921 if (opt->sendpage) 922 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 923 else 924 err = msg_loop(c1, iov_count, iov_buf, 925 cnt, &s, true, opt); 926 927 if (err) 928 fprintf(stderr, 929 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 930 iov_count, iov_buf, cnt, err); 931 if (s.end.tv_sec - s.start.tv_sec) { 932 sent_Bps = sentBps(s); 933 recvd_Bps = recvdBps(s); 934 } 935 if (opt->verbose > 1) 936 fprintf(stdout, 937 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 938 s.bytes_sent, sent_Bps, sent_Bps/giga, 939 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 940 exit(err ? 1 : 0); 941 } else if (txpid == -1) { 942 perror("msg_loop_tx"); 943 return errno; 944 } 945 946 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 947 assert(waitpid(txpid, &tx_status, 0) == txpid); 948 if (WIFEXITED(rx_status)) { 949 err = WEXITSTATUS(rx_status); 950 if (err) { 951 fprintf(stderr, "rx thread exited with err %d.\n", err); 952 goto out; 953 } 954 } 955 if (WIFEXITED(tx_status)) { 956 err = WEXITSTATUS(tx_status); 957 if (err) 958 fprintf(stderr, "tx thread exited with err %d.\n", err); 959 } 960 out: 961 return err; 962 } 963 964 static int forever_ping_pong(int rate, struct sockmap_options *opt) 965 { 966 struct timeval timeout; 967 char buf[1024] = {0}; 968 int sc; 969 970 timeout.tv_sec = 10; 971 timeout.tv_usec = 0; 972 973 /* Ping/Pong data from client to server */ 974 sc = send(c1, buf, sizeof(buf), 0); 975 if (sc < 0) { 976 perror("send failed()"); 977 return sc; 978 } 979 980 do { 981 int s, rc, i, max_fd = p2; 982 fd_set w; 983 984 /* FD sets */ 985 FD_ZERO(&w); 986 FD_SET(c1, &w); 987 FD_SET(c2, &w); 988 FD_SET(p1, &w); 989 FD_SET(p2, &w); 990 991 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 992 if (s == -1) { 993 perror("select()"); 994 break; 995 } else if (!s) { 996 fprintf(stderr, "unexpected timeout\n"); 997 break; 998 } 999 1000 for (i = 0; i <= max_fd && s > 0; ++i) { 1001 if (!FD_ISSET(i, &w)) 1002 continue; 1003 1004 s--; 1005 1006 rc = recv(i, buf, sizeof(buf), 0); 1007 if (rc < 0) { 1008 if (errno != EWOULDBLOCK) { 1009 perror("recv failed()"); 1010 return rc; 1011 } 1012 } 1013 1014 if (rc == 0) { 1015 close(i); 1016 break; 1017 } 1018 1019 sc = send(i, buf, rc, 0); 1020 if (sc < 0) { 1021 perror("send failed()"); 1022 return sc; 1023 } 1024 } 1025 1026 if (rate) 1027 sleep(rate); 1028 1029 if (opt->verbose) { 1030 printf("."); 1031 fflush(stdout); 1032 1033 } 1034 } while (running); 1035 1036 return 0; 1037 } 1038 1039 enum { 1040 SELFTESTS, 1041 PING_PONG, 1042 SENDMSG, 1043 BASE, 1044 BASE_SENDPAGE, 1045 SENDPAGE, 1046 }; 1047 1048 static int run_options(struct sockmap_options *options, int cg_fd, int test) 1049 { 1050 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 1051 1052 /* If base test skip BPF setup */ 1053 if (test == BASE || test == BASE_SENDPAGE) 1054 goto run; 1055 1056 /* Attach programs to sockmap */ 1057 if (!txmsg_omit_skb_parser) { 1058 err = bpf_prog_attach(prog_fd[0], map_fd[0], 1059 BPF_SK_SKB_STREAM_PARSER, 0); 1060 if (err) { 1061 fprintf(stderr, 1062 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 1063 prog_fd[0], map_fd[0], err, strerror(errno)); 1064 return err; 1065 } 1066 } 1067 1068 err = bpf_prog_attach(prog_fd[1], map_fd[0], 1069 BPF_SK_SKB_STREAM_VERDICT, 0); 1070 if (err) { 1071 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 1072 err, strerror(errno)); 1073 return err; 1074 } 1075 1076 /* Attach programs to TLS sockmap */ 1077 if (txmsg_ktls_skb) { 1078 if (!txmsg_omit_skb_parser) { 1079 err = bpf_prog_attach(prog_fd[0], map_fd[8], 1080 BPF_SK_SKB_STREAM_PARSER, 0); 1081 if (err) { 1082 fprintf(stderr, 1083 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n", 1084 prog_fd[0], map_fd[8], err, strerror(errno)); 1085 return err; 1086 } 1087 } 1088 1089 err = bpf_prog_attach(prog_fd[2], map_fd[8], 1090 BPF_SK_SKB_STREAM_VERDICT, 0); 1091 if (err) { 1092 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n", 1093 err, strerror(errno)); 1094 return err; 1095 } 1096 } 1097 1098 /* Attach to cgroups */ 1099 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 1100 if (err) { 1101 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 1102 err, strerror(errno)); 1103 return err; 1104 } 1105 1106 run: 1107 err = sockmap_init_sockets(options->verbose); 1108 if (err) { 1109 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 1110 goto out; 1111 } 1112 1113 /* Attach txmsg program to sockmap */ 1114 if (txmsg_pass) 1115 tx_prog_fd = prog_fd[4]; 1116 else if (txmsg_redir) 1117 tx_prog_fd = prog_fd[5]; 1118 else if (txmsg_apply) 1119 tx_prog_fd = prog_fd[6]; 1120 else if (txmsg_cork) 1121 tx_prog_fd = prog_fd[7]; 1122 else if (txmsg_drop) 1123 tx_prog_fd = prog_fd[8]; 1124 else 1125 tx_prog_fd = 0; 1126 1127 if (tx_prog_fd) { 1128 int redir_fd, i = 0; 1129 1130 err = bpf_prog_attach(tx_prog_fd, 1131 map_fd[1], BPF_SK_MSG_VERDICT, 0); 1132 if (err) { 1133 fprintf(stderr, 1134 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 1135 err, strerror(errno)); 1136 goto out; 1137 } 1138 1139 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 1140 if (err) { 1141 fprintf(stderr, 1142 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1143 err, strerror(errno)); 1144 goto out; 1145 } 1146 1147 if (txmsg_redir) 1148 redir_fd = c2; 1149 else 1150 redir_fd = c1; 1151 1152 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 1153 if (err) { 1154 fprintf(stderr, 1155 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 1156 err, strerror(errno)); 1157 goto out; 1158 } 1159 1160 if (txmsg_apply) { 1161 err = bpf_map_update_elem(map_fd[3], 1162 &i, &txmsg_apply, BPF_ANY); 1163 if (err) { 1164 fprintf(stderr, 1165 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 1166 err, strerror(errno)); 1167 goto out; 1168 } 1169 } 1170 1171 if (txmsg_cork) { 1172 err = bpf_map_update_elem(map_fd[4], 1173 &i, &txmsg_cork, BPF_ANY); 1174 if (err) { 1175 fprintf(stderr, 1176 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 1177 err, strerror(errno)); 1178 goto out; 1179 } 1180 } 1181 1182 if (txmsg_start) { 1183 err = bpf_map_update_elem(map_fd[5], 1184 &i, &txmsg_start, BPF_ANY); 1185 if (err) { 1186 fprintf(stderr, 1187 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 1188 err, strerror(errno)); 1189 goto out; 1190 } 1191 } 1192 1193 if (txmsg_end) { 1194 i = 1; 1195 err = bpf_map_update_elem(map_fd[5], 1196 &i, &txmsg_end, BPF_ANY); 1197 if (err) { 1198 fprintf(stderr, 1199 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 1200 err, strerror(errno)); 1201 goto out; 1202 } 1203 } 1204 1205 if (txmsg_start_push) { 1206 i = 2; 1207 err = bpf_map_update_elem(map_fd[5], 1208 &i, &txmsg_start_push, BPF_ANY); 1209 if (err) { 1210 fprintf(stderr, 1211 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 1212 err, strerror(errno)); 1213 goto out; 1214 } 1215 } 1216 1217 if (txmsg_end_push) { 1218 i = 3; 1219 err = bpf_map_update_elem(map_fd[5], 1220 &i, &txmsg_end_push, BPF_ANY); 1221 if (err) { 1222 fprintf(stderr, 1223 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 1224 txmsg_end_push, i, err, strerror(errno)); 1225 goto out; 1226 } 1227 } 1228 1229 if (txmsg_start_pop) { 1230 i = 4; 1231 err = bpf_map_update_elem(map_fd[5], 1232 &i, &txmsg_start_pop, BPF_ANY); 1233 if (err) { 1234 fprintf(stderr, 1235 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 1236 txmsg_start_pop, i, err, strerror(errno)); 1237 goto out; 1238 } 1239 } else { 1240 i = 4; 1241 bpf_map_update_elem(map_fd[5], 1242 &i, &txmsg_start_pop, BPF_ANY); 1243 } 1244 1245 if (txmsg_pop) { 1246 i = 5; 1247 err = bpf_map_update_elem(map_fd[5], 1248 &i, &txmsg_pop, BPF_ANY); 1249 if (err) { 1250 fprintf(stderr, 1251 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 1252 txmsg_pop, i, err, strerror(errno)); 1253 goto out; 1254 } 1255 } else { 1256 i = 5; 1257 bpf_map_update_elem(map_fd[5], 1258 &i, &txmsg_pop, BPF_ANY); 1259 1260 } 1261 1262 if (txmsg_ingress) { 1263 int in = BPF_F_INGRESS; 1264 1265 i = 0; 1266 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 1267 if (err) { 1268 fprintf(stderr, 1269 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1270 err, strerror(errno)); 1271 } 1272 i = 1; 1273 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1274 if (err) { 1275 fprintf(stderr, 1276 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1277 err, strerror(errno)); 1278 } 1279 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1280 if (err) { 1281 fprintf(stderr, 1282 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1283 err, strerror(errno)); 1284 } 1285 1286 i = 2; 1287 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1288 if (err) { 1289 fprintf(stderr, 1290 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1291 err, strerror(errno)); 1292 } 1293 } 1294 1295 if (txmsg_ktls_skb) { 1296 int ingress = BPF_F_INGRESS; 1297 1298 i = 0; 1299 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY); 1300 if (err) { 1301 fprintf(stderr, 1302 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1303 err, strerror(errno)); 1304 } 1305 1306 if (txmsg_ktls_skb_redir) { 1307 i = 1; 1308 err = bpf_map_update_elem(map_fd[7], 1309 &i, &ingress, BPF_ANY); 1310 if (err) { 1311 fprintf(stderr, 1312 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1313 err, strerror(errno)); 1314 } 1315 } 1316 1317 if (txmsg_ktls_skb_drop) { 1318 i = 1; 1319 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY); 1320 } 1321 } 1322 1323 if (txmsg_redir_skb) { 1324 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1325 p2 : p1; 1326 int ingress = BPF_F_INGRESS; 1327 1328 i = 0; 1329 err = bpf_map_update_elem(map_fd[7], 1330 &i, &ingress, BPF_ANY); 1331 if (err) { 1332 fprintf(stderr, 1333 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1334 err, strerror(errno)); 1335 } 1336 1337 i = 3; 1338 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY); 1339 if (err) { 1340 fprintf(stderr, 1341 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1342 err, strerror(errno)); 1343 } 1344 } 1345 } 1346 1347 if (skb_use_parser) { 1348 i = 2; 1349 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY); 1350 } 1351 1352 if (txmsg_drop) 1353 options->drop_expected = true; 1354 1355 if (test == PING_PONG) 1356 err = forever_ping_pong(options->rate, options); 1357 else if (test == SENDMSG) { 1358 options->base = false; 1359 options->sendpage = false; 1360 err = sendmsg_test(options); 1361 } else if (test == SENDPAGE) { 1362 options->base = false; 1363 options->sendpage = true; 1364 err = sendmsg_test(options); 1365 } else if (test == BASE) { 1366 options->base = true; 1367 options->sendpage = false; 1368 err = sendmsg_test(options); 1369 } else if (test == BASE_SENDPAGE) { 1370 options->base = true; 1371 options->sendpage = true; 1372 err = sendmsg_test(options); 1373 } else 1374 fprintf(stderr, "unknown test\n"); 1375 out: 1376 /* Detatch and zero all the maps */ 1377 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS); 1378 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1379 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1380 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER); 1381 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT); 1382 1383 if (tx_prog_fd >= 0) 1384 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1385 1386 for (i = 0; i < 8; i++) { 1387 key = next_key = 0; 1388 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1389 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1390 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1391 key = next_key; 1392 } 1393 } 1394 1395 close(s1); 1396 close(s2); 1397 close(p1); 1398 close(p2); 1399 close(c1); 1400 close(c2); 1401 return err; 1402 } 1403 1404 static char *test_to_str(int test) 1405 { 1406 switch (test) { 1407 case SENDMSG: 1408 return "sendmsg"; 1409 case SENDPAGE: 1410 return "sendpage"; 1411 } 1412 return "unknown"; 1413 } 1414 1415 static void append_str(char *dst, const char *src, size_t dst_cap) 1416 { 1417 size_t avail = dst_cap - strlen(dst); 1418 1419 if (avail <= 1) /* just zero byte could be written */ 1420 return; 1421 1422 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */ 1423 } 1424 1425 #define OPTSTRING 60 1426 static void test_options(char *options) 1427 { 1428 char tstr[OPTSTRING]; 1429 1430 memset(options, 0, OPTSTRING); 1431 1432 if (txmsg_pass) 1433 append_str(options, "pass,", OPTSTRING); 1434 if (txmsg_redir) 1435 append_str(options, "redir,", OPTSTRING); 1436 if (txmsg_drop) 1437 append_str(options, "drop,", OPTSTRING); 1438 if (txmsg_apply) { 1439 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1440 append_str(options, tstr, OPTSTRING); 1441 } 1442 if (txmsg_cork) { 1443 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1444 append_str(options, tstr, OPTSTRING); 1445 } 1446 if (txmsg_start) { 1447 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1448 append_str(options, tstr, OPTSTRING); 1449 } 1450 if (txmsg_end) { 1451 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1452 append_str(options, tstr, OPTSTRING); 1453 } 1454 if (txmsg_start_pop) { 1455 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1456 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1457 append_str(options, tstr, OPTSTRING); 1458 } 1459 if (txmsg_ingress) 1460 append_str(options, "ingress,", OPTSTRING); 1461 if (txmsg_redir_skb) 1462 append_str(options, "redir_skb,", OPTSTRING); 1463 if (txmsg_ktls_skb) 1464 append_str(options, "ktls_skb,", OPTSTRING); 1465 if (ktls) 1466 append_str(options, "ktls,", OPTSTRING); 1467 if (peek_flag) 1468 append_str(options, "peek,", OPTSTRING); 1469 } 1470 1471 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1472 { 1473 char *options = calloc(OPTSTRING, sizeof(char)); 1474 int err; 1475 1476 if (test == SENDPAGE) 1477 opt->sendpage = true; 1478 else 1479 opt->sendpage = false; 1480 1481 if (txmsg_drop) 1482 opt->drop_expected = true; 1483 else 1484 opt->drop_expected = false; 1485 1486 test_options(options); 1487 1488 if (opt->verbose) { 1489 fprintf(stdout, 1490 " [TEST %i]: (%i, %i, %i, %s, %s): ", 1491 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1492 test_to_str(test), options); 1493 fflush(stdout); 1494 } 1495 err = run_options(opt, cgrp, test); 1496 if (opt->verbose) 1497 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED"); 1498 test_cnt++; 1499 !err ? passed++ : failed++; 1500 free(options); 1501 return err; 1502 } 1503 1504 static void test_exec(int cgrp, struct sockmap_options *opt) 1505 { 1506 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME); 1507 int err; 1508 1509 if (type == 0) { 1510 test_start(); 1511 err = __test_exec(cgrp, SENDMSG, opt); 1512 if (err) 1513 test_fail(); 1514 } else { 1515 test_start(); 1516 err = __test_exec(cgrp, SENDPAGE, opt); 1517 if (err) 1518 test_fail(); 1519 } 1520 } 1521 1522 static void test_send_one(struct sockmap_options *opt, int cgrp) 1523 { 1524 opt->iov_length = 1; 1525 opt->iov_count = 1; 1526 opt->rate = 1; 1527 test_exec(cgrp, opt); 1528 1529 opt->iov_length = 1; 1530 opt->iov_count = 1024; 1531 opt->rate = 1; 1532 test_exec(cgrp, opt); 1533 1534 opt->iov_length = 1024; 1535 opt->iov_count = 1; 1536 opt->rate = 1; 1537 test_exec(cgrp, opt); 1538 1539 } 1540 1541 static void test_send_many(struct sockmap_options *opt, int cgrp) 1542 { 1543 opt->iov_length = 3; 1544 opt->iov_count = 1; 1545 opt->rate = 512; 1546 test_exec(cgrp, opt); 1547 1548 opt->rate = 100; 1549 opt->iov_count = 1; 1550 opt->iov_length = 5; 1551 test_exec(cgrp, opt); 1552 } 1553 1554 static void test_send_large(struct sockmap_options *opt, int cgrp) 1555 { 1556 opt->iov_length = 8192; 1557 opt->iov_count = 32; 1558 opt->rate = 2; 1559 test_exec(cgrp, opt); 1560 } 1561 1562 static void test_send(struct sockmap_options *opt, int cgrp) 1563 { 1564 test_send_one(opt, cgrp); 1565 test_send_many(opt, cgrp); 1566 test_send_large(opt, cgrp); 1567 sched_yield(); 1568 } 1569 1570 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt) 1571 { 1572 /* Test small and large iov_count values with pass/redir/apply/cork */ 1573 txmsg_pass = 1; 1574 test_send(opt, cgrp); 1575 } 1576 1577 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) 1578 { 1579 txmsg_redir = 1; 1580 test_send(opt, cgrp); 1581 } 1582 1583 static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) 1584 { 1585 txmsg_redir = 1; 1586 opt->tx_wait_mem = true; 1587 test_send_large(opt, cgrp); 1588 opt->tx_wait_mem = false; 1589 } 1590 1591 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) 1592 { 1593 txmsg_drop = 1; 1594 test_send(opt, cgrp); 1595 } 1596 1597 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt) 1598 { 1599 txmsg_pass = txmsg_drop = 0; 1600 txmsg_ingress = txmsg_redir = 1; 1601 test_send(opt, cgrp); 1602 } 1603 1604 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) 1605 { 1606 bool data = opt->data_test; 1607 int k = ktls; 1608 1609 opt->data_test = true; 1610 ktls = 1; 1611 1612 txmsg_pass = txmsg_drop = 0; 1613 txmsg_ingress = txmsg_redir = 0; 1614 txmsg_ktls_skb = 1; 1615 txmsg_pass = 1; 1616 1617 /* Using data verification so ensure iov layout is 1618 * expected from test receiver side. e.g. has enough 1619 * bytes to write test code. 1620 */ 1621 opt->iov_length = 100; 1622 opt->iov_count = 1; 1623 opt->rate = 1; 1624 test_exec(cgrp, opt); 1625 1626 txmsg_ktls_skb_drop = 1; 1627 test_exec(cgrp, opt); 1628 1629 txmsg_ktls_skb_drop = 0; 1630 txmsg_ktls_skb_redir = 1; 1631 test_exec(cgrp, opt); 1632 txmsg_ktls_skb_redir = 0; 1633 1634 /* Tests that omit skb_parser */ 1635 txmsg_omit_skb_parser = 1; 1636 ktls = 0; 1637 txmsg_ktls_skb = 0; 1638 test_exec(cgrp, opt); 1639 1640 txmsg_ktls_skb_drop = 1; 1641 test_exec(cgrp, opt); 1642 txmsg_ktls_skb_drop = 0; 1643 1644 txmsg_ktls_skb_redir = 1; 1645 test_exec(cgrp, opt); 1646 1647 ktls = 1; 1648 test_exec(cgrp, opt); 1649 txmsg_omit_skb_parser = 0; 1650 1651 opt->data_test = data; 1652 ktls = k; 1653 } 1654 1655 /* Test cork with hung data. This tests poor usage patterns where 1656 * cork can leave data on the ring if user program is buggy and 1657 * doesn't flush them somehow. They do take some time however 1658 * because they wait for a timeout. Test pass, redir and cork with 1659 * apply logic. Use cork size of 4097 with send_large to avoid 1660 * aligning cork size with send size. 1661 */ 1662 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt) 1663 { 1664 txmsg_pass = 1; 1665 txmsg_redir = 0; 1666 txmsg_cork = 4097; 1667 txmsg_apply = 4097; 1668 test_send_large(opt, cgrp); 1669 1670 txmsg_pass = 0; 1671 txmsg_redir = 1; 1672 txmsg_apply = 0; 1673 txmsg_cork = 4097; 1674 test_send_large(opt, cgrp); 1675 1676 txmsg_pass = 0; 1677 txmsg_redir = 1; 1678 txmsg_apply = 4097; 1679 txmsg_cork = 4097; 1680 test_send_large(opt, cgrp); 1681 } 1682 1683 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt) 1684 { 1685 /* Test basic start/end */ 1686 txmsg_pass = 1; 1687 txmsg_start = 1; 1688 txmsg_end = 2; 1689 test_send(opt, cgrp); 1690 1691 /* Test >4k pull */ 1692 txmsg_pass = 1; 1693 txmsg_start = 4096; 1694 txmsg_end = 9182; 1695 test_send_large(opt, cgrp); 1696 1697 /* Test pull + redirect */ 1698 txmsg_redir = 1; 1699 txmsg_start = 1; 1700 txmsg_end = 2; 1701 test_send(opt, cgrp); 1702 1703 /* Test pull + cork */ 1704 txmsg_redir = 0; 1705 txmsg_cork = 512; 1706 txmsg_start = 1; 1707 txmsg_end = 2; 1708 test_send_many(opt, cgrp); 1709 1710 /* Test pull + cork + redirect */ 1711 txmsg_redir = 1; 1712 txmsg_cork = 512; 1713 txmsg_start = 1; 1714 txmsg_end = 2; 1715 test_send_many(opt, cgrp); 1716 } 1717 1718 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt) 1719 { 1720 bool data = opt->data_test; 1721 1722 /* Test basic pop */ 1723 txmsg_pass = 1; 1724 txmsg_start_pop = 1; 1725 txmsg_pop = 2; 1726 test_send_many(opt, cgrp); 1727 1728 /* Test pop with >4k */ 1729 txmsg_pass = 1; 1730 txmsg_start_pop = 4096; 1731 txmsg_pop = 4096; 1732 test_send_large(opt, cgrp); 1733 1734 /* Test pop + redirect */ 1735 txmsg_redir = 1; 1736 txmsg_start_pop = 1; 1737 txmsg_pop = 2; 1738 test_send_many(opt, cgrp); 1739 1740 /* TODO: Test for pop + cork should be different, 1741 * - It makes the layout of the received data difficult 1742 * - It makes it hard to calculate the total_bytes in the recvmsg 1743 * Temporarily skip the data integrity test for this case now. 1744 */ 1745 opt->data_test = false; 1746 /* Test pop + cork */ 1747 txmsg_redir = 0; 1748 txmsg_cork = 512; 1749 txmsg_start_pop = 1; 1750 txmsg_pop = 2; 1751 test_send_many(opt, cgrp); 1752 1753 /* Test pop + redirect + cork */ 1754 txmsg_redir = 1; 1755 txmsg_cork = 4; 1756 txmsg_start_pop = 1; 1757 txmsg_pop = 2; 1758 test_send_many(opt, cgrp); 1759 opt->data_test = data; 1760 } 1761 1762 static void test_txmsg_push(int cgrp, struct sockmap_options *opt) 1763 { 1764 bool data = opt->data_test; 1765 1766 /* Test basic push */ 1767 txmsg_pass = 1; 1768 txmsg_start_push = 1; 1769 txmsg_end_push = 1; 1770 test_send(opt, cgrp); 1771 1772 /* Test push 4kB >4k */ 1773 txmsg_pass = 1; 1774 txmsg_start_push = 4096; 1775 txmsg_end_push = 4096; 1776 test_send_large(opt, cgrp); 1777 1778 /* Test push + redirect */ 1779 txmsg_redir = 1; 1780 txmsg_start_push = 1; 1781 txmsg_end_push = 2; 1782 test_send_many(opt, cgrp); 1783 1784 /* TODO: Test for push + cork should be different, 1785 * - It makes the layout of the received data difficult 1786 * - It makes it hard to calculate the total_bytes in the recvmsg 1787 * Temporarily skip the data integrity test for this case now. 1788 */ 1789 opt->data_test = false; 1790 /* Test push + cork */ 1791 txmsg_redir = 0; 1792 txmsg_cork = 512; 1793 txmsg_start_push = 1; 1794 txmsg_end_push = 2; 1795 test_send_many(opt, cgrp); 1796 opt->data_test = data; 1797 } 1798 1799 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt) 1800 { 1801 txmsg_pass = 1; 1802 txmsg_start_push = 1; 1803 txmsg_end_push = 10; 1804 txmsg_start_pop = 5; 1805 txmsg_pop = 4; 1806 test_send_large(opt, cgrp); 1807 } 1808 1809 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt) 1810 { 1811 txmsg_pass = 1; 1812 txmsg_redir = 0; 1813 txmsg_ingress = 0; 1814 txmsg_apply = 1; 1815 txmsg_cork = 0; 1816 test_send_one(opt, cgrp); 1817 1818 txmsg_pass = 0; 1819 txmsg_redir = 1; 1820 txmsg_ingress = 0; 1821 txmsg_apply = 1; 1822 txmsg_cork = 0; 1823 test_send_one(opt, cgrp); 1824 1825 txmsg_pass = 0; 1826 txmsg_redir = 1; 1827 txmsg_ingress = 1; 1828 txmsg_apply = 1; 1829 txmsg_cork = 0; 1830 test_send_one(opt, cgrp); 1831 1832 txmsg_pass = 1; 1833 txmsg_redir = 0; 1834 txmsg_ingress = 0; 1835 txmsg_apply = 1024; 1836 txmsg_cork = 0; 1837 test_send_large(opt, cgrp); 1838 1839 txmsg_pass = 0; 1840 txmsg_redir = 1; 1841 txmsg_ingress = 0; 1842 txmsg_apply = 1024; 1843 txmsg_cork = 0; 1844 test_send_large(opt, cgrp); 1845 1846 txmsg_pass = 0; 1847 txmsg_redir = 1; 1848 txmsg_ingress = 1; 1849 txmsg_apply = 1024; 1850 txmsg_cork = 0; 1851 test_send_large(opt, cgrp); 1852 } 1853 1854 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt) 1855 { 1856 txmsg_pass = 1; 1857 txmsg_redir = 0; 1858 txmsg_apply = 0; 1859 txmsg_cork = 1; 1860 test_send(opt, cgrp); 1861 1862 txmsg_pass = 1; 1863 txmsg_redir = 0; 1864 txmsg_apply = 1; 1865 txmsg_cork = 1; 1866 test_send(opt, cgrp); 1867 } 1868 1869 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt) 1870 { 1871 txmsg_pass = 1; 1872 skb_use_parser = 512; 1873 if (ktls == 1) 1874 skb_use_parser = 570; 1875 opt->iov_length = 256; 1876 opt->iov_count = 1; 1877 opt->rate = 2; 1878 test_exec(cgrp, opt); 1879 } 1880 1881 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt) 1882 { 1883 if (ktls == 1) 1884 return; 1885 skb_use_parser = 10; 1886 opt->iov_length = 20; 1887 opt->iov_count = 1; 1888 opt->rate = 1; 1889 opt->check_recved_len = true; 1890 test_exec(cgrp, opt); 1891 opt->check_recved_len = false; 1892 } 1893 1894 char *map_names[] = { 1895 "sock_map", 1896 "sock_map_txmsg", 1897 "sock_map_redir", 1898 "sock_apply_bytes", 1899 "sock_cork_bytes", 1900 "sock_bytes", 1901 "sock_redir_flags", 1902 "sock_skb_opts", 1903 "tls_sock_map", 1904 }; 1905 1906 int prog_attach_type[] = { 1907 BPF_SK_SKB_STREAM_PARSER, 1908 BPF_SK_SKB_STREAM_VERDICT, 1909 BPF_SK_SKB_STREAM_VERDICT, 1910 BPF_CGROUP_SOCK_OPS, 1911 BPF_SK_MSG_VERDICT, 1912 BPF_SK_MSG_VERDICT, 1913 BPF_SK_MSG_VERDICT, 1914 BPF_SK_MSG_VERDICT, 1915 BPF_SK_MSG_VERDICT, 1916 }; 1917 1918 int prog_type[] = { 1919 BPF_PROG_TYPE_SK_SKB, 1920 BPF_PROG_TYPE_SK_SKB, 1921 BPF_PROG_TYPE_SK_SKB, 1922 BPF_PROG_TYPE_SOCK_OPS, 1923 BPF_PROG_TYPE_SK_MSG, 1924 BPF_PROG_TYPE_SK_MSG, 1925 BPF_PROG_TYPE_SK_MSG, 1926 BPF_PROG_TYPE_SK_MSG, 1927 BPF_PROG_TYPE_SK_MSG, 1928 }; 1929 1930 static int populate_progs(char *bpf_file) 1931 { 1932 struct bpf_program *prog; 1933 struct bpf_object *obj; 1934 int i = 0; 1935 long err; 1936 1937 obj = bpf_object__open(bpf_file); 1938 err = libbpf_get_error(obj); 1939 if (err) { 1940 char err_buf[256]; 1941 1942 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1943 printf("Unable to load eBPF objects in file '%s' : %s\n", 1944 bpf_file, err_buf); 1945 return -1; 1946 } 1947 1948 bpf_object__for_each_program(prog, obj) { 1949 bpf_program__set_type(prog, prog_type[i]); 1950 bpf_program__set_expected_attach_type(prog, 1951 prog_attach_type[i]); 1952 i++; 1953 } 1954 1955 i = bpf_object__load(obj); 1956 i = 0; 1957 bpf_object__for_each_program(prog, obj) { 1958 prog_fd[i] = bpf_program__fd(prog); 1959 i++; 1960 } 1961 1962 for (i = 0; i < ARRAY_SIZE(map_fd); i++) { 1963 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1964 map_fd[i] = bpf_map__fd(maps[i]); 1965 if (map_fd[i] < 0) { 1966 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1967 map_fd[i], strerror(errno)); 1968 return -1; 1969 } 1970 } 1971 1972 return 0; 1973 } 1974 1975 struct _test test[] = { 1976 {"txmsg test passthrough", test_txmsg_pass}, 1977 {"txmsg test redirect", test_txmsg_redir}, 1978 {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, 1979 {"txmsg test drop", test_txmsg_drop}, 1980 {"txmsg test ingress redirect", test_txmsg_ingress_redir}, 1981 {"txmsg test skb", test_txmsg_skb}, 1982 {"txmsg test apply", test_txmsg_apply}, 1983 {"txmsg test cork", test_txmsg_cork}, 1984 {"txmsg test hanging corks", test_txmsg_cork_hangs}, 1985 {"txmsg test push_data", test_txmsg_push}, 1986 {"txmsg test pull-data", test_txmsg_pull}, 1987 {"txmsg test pop-data", test_txmsg_pop}, 1988 {"txmsg test push/pop data", test_txmsg_push_pop}, 1989 {"txmsg test ingress parser", test_txmsg_ingress_parser}, 1990 {"txmsg test ingress parser2", test_txmsg_ingress_parser2}, 1991 }; 1992 1993 static int check_whitelist(struct _test *t, struct sockmap_options *opt) 1994 { 1995 char *entry, *ptr; 1996 1997 if (!opt->whitelist) 1998 return 0; 1999 ptr = strdup(opt->whitelist); 2000 if (!ptr) 2001 return -ENOMEM; 2002 entry = strtok(ptr, ","); 2003 while (entry) { 2004 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 2005 strstr(opt->map, entry) != 0 || 2006 strstr(t->title, entry) != 0) 2007 return 0; 2008 entry = strtok(NULL, ","); 2009 } 2010 return -EINVAL; 2011 } 2012 2013 static int check_blacklist(struct _test *t, struct sockmap_options *opt) 2014 { 2015 char *entry, *ptr; 2016 2017 if (!opt->blacklist) 2018 return -EINVAL; 2019 ptr = strdup(opt->blacklist); 2020 if (!ptr) 2021 return -ENOMEM; 2022 entry = strtok(ptr, ","); 2023 while (entry) { 2024 if ((opt->prepend && strstr(opt->prepend, entry) != 0) || 2025 strstr(opt->map, entry) != 0 || 2026 strstr(t->title, entry) != 0) 2027 return 0; 2028 entry = strtok(NULL, ","); 2029 } 2030 return -EINVAL; 2031 } 2032 2033 static int __test_selftests(int cg_fd, struct sockmap_options *opt) 2034 { 2035 int i, err; 2036 2037 err = populate_progs(opt->map); 2038 if (err < 0) { 2039 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 2040 return err; 2041 } 2042 2043 /* Tests basic commands and APIs */ 2044 for (i = 0; i < ARRAY_SIZE(test); i++) { 2045 struct _test t = test[i]; 2046 2047 if (check_whitelist(&t, opt) != 0) 2048 continue; 2049 if (check_blacklist(&t, opt) == 0) 2050 continue; 2051 2052 test_start_subtest(&t, opt); 2053 t.tester(cg_fd, opt); 2054 test_end_subtest(); 2055 } 2056 2057 return err; 2058 } 2059 2060 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt) 2061 { 2062 opt->map = BPF_SOCKMAP_FILENAME; 2063 __test_selftests(cg_fd, opt); 2064 } 2065 2066 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt) 2067 { 2068 opt->map = BPF_SOCKHASH_FILENAME; 2069 __test_selftests(cg_fd, opt); 2070 } 2071 2072 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt) 2073 { 2074 opt->map = BPF_SOCKHASH_FILENAME; 2075 opt->prepend = "ktls"; 2076 ktls = 1; 2077 __test_selftests(cg_fd, opt); 2078 ktls = 0; 2079 } 2080 2081 static int test_selftest(int cg_fd, struct sockmap_options *opt) 2082 { 2083 2084 test_selftests_sockmap(cg_fd, opt); 2085 test_selftests_sockhash(cg_fd, opt); 2086 test_selftests_ktls(cg_fd, opt); 2087 test_print_results(); 2088 return 0; 2089 } 2090 2091 int main(int argc, char **argv) 2092 { 2093 int iov_count = 1, length = 1024, rate = 1; 2094 struct sockmap_options options = {0}; 2095 int opt, longindex, err, cg_fd = 0; 2096 char *bpf_file = BPF_SOCKMAP_FILENAME; 2097 int test = SELFTESTS; 2098 bool cg_created = 0; 2099 2100 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:", 2101 long_options, &longindex)) != -1) { 2102 switch (opt) { 2103 case 's': 2104 txmsg_start = atoi(optarg); 2105 break; 2106 case 'e': 2107 txmsg_end = atoi(optarg); 2108 break; 2109 case 'p': 2110 txmsg_start_push = atoi(optarg); 2111 break; 2112 case 'q': 2113 txmsg_end_push = atoi(optarg); 2114 break; 2115 case 'w': 2116 txmsg_start_pop = atoi(optarg); 2117 break; 2118 case 'x': 2119 txmsg_pop = atoi(optarg); 2120 break; 2121 case 'a': 2122 txmsg_apply = atoi(optarg); 2123 break; 2124 case 'k': 2125 txmsg_cork = atoi(optarg); 2126 break; 2127 case 'c': 2128 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 2129 if (cg_fd < 0) { 2130 fprintf(stderr, 2131 "ERROR: (%i) open cg path failed: %s\n", 2132 cg_fd, optarg); 2133 return cg_fd; 2134 } 2135 break; 2136 case 'r': 2137 rate = atoi(optarg); 2138 break; 2139 case 'v': 2140 options.verbose = 1; 2141 if (optarg) 2142 options.verbose = atoi(optarg); 2143 break; 2144 case 'i': 2145 iov_count = atoi(optarg); 2146 break; 2147 case 'l': 2148 length = atoi(optarg); 2149 break; 2150 case 'd': 2151 options.data_test = true; 2152 break; 2153 case 't': 2154 if (strcmp(optarg, "ping") == 0) { 2155 test = PING_PONG; 2156 } else if (strcmp(optarg, "sendmsg") == 0) { 2157 test = SENDMSG; 2158 } else if (strcmp(optarg, "base") == 0) { 2159 test = BASE; 2160 } else if (strcmp(optarg, "base_sendpage") == 0) { 2161 test = BASE_SENDPAGE; 2162 } else if (strcmp(optarg, "sendpage") == 0) { 2163 test = SENDPAGE; 2164 } else { 2165 usage(argv); 2166 return -1; 2167 } 2168 break; 2169 case 'n': 2170 options.whitelist = strdup(optarg); 2171 if (!options.whitelist) 2172 return -ENOMEM; 2173 break; 2174 case 'b': 2175 options.blacklist = strdup(optarg); 2176 if (!options.blacklist) 2177 return -ENOMEM; 2178 case 0: 2179 break; 2180 case 'h': 2181 default: 2182 usage(argv); 2183 return -1; 2184 } 2185 } 2186 2187 if (!cg_fd) { 2188 cg_fd = cgroup_setup_and_join(CG_PATH); 2189 if (cg_fd < 0) 2190 return cg_fd; 2191 cg_created = 1; 2192 } 2193 2194 /* Use libbpf 1.0 API mode */ 2195 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 2196 2197 if (test == SELFTESTS) { 2198 err = test_selftest(cg_fd, &options); 2199 goto out; 2200 } 2201 2202 err = populate_progs(bpf_file); 2203 if (err) { 2204 fprintf(stderr, "populate program: (%s) %s\n", 2205 bpf_file, strerror(errno)); 2206 return 1; 2207 } 2208 running = 1; 2209 2210 /* catch SIGINT */ 2211 signal(SIGINT, running_handler); 2212 2213 options.iov_count = iov_count; 2214 options.iov_length = length; 2215 options.rate = rate; 2216 2217 err = run_options(&options, cg_fd, test); 2218 out: 2219 if (options.whitelist) 2220 free(options.whitelist); 2221 if (options.blacklist) 2222 free(options.blacklist); 2223 close(cg_fd); 2224 if (cg_created) 2225 cleanup_cgroup_environment(); 2226 return err; 2227 } 2228 2229 void running_handler(int a) 2230 { 2231 running = 0; 2232 } 2233