1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <sys/ioctl.h> 14 #include <stdbool.h> 15 #include <signal.h> 16 #include <fcntl.h> 17 #include <sys/wait.h> 18 #include <time.h> 19 #include <sched.h> 20 21 #include <sys/time.h> 22 #include <sys/resource.h> 23 #include <sys/types.h> 24 #include <sys/sendfile.h> 25 26 #include <linux/netlink.h> 27 #include <linux/socket.h> 28 #include <linux/sock_diag.h> 29 #include <linux/bpf.h> 30 #include <linux/if_link.h> 31 #include <linux/tls.h> 32 #include <assert.h> 33 #include <libgen.h> 34 35 #include <getopt.h> 36 37 #include <bpf/bpf.h> 38 #include <bpf/libbpf.h> 39 40 #include "bpf_util.h" 41 #include "bpf_rlimit.h" 42 #include "cgroup_helpers.h" 43 44 int running; 45 static void running_handler(int a); 46 47 #ifndef TCP_ULP 48 # define TCP_ULP 31 49 #endif 50 #ifndef SOL_TLS 51 # define SOL_TLS 282 52 #endif 53 54 /* randomly selected ports for testing on lo */ 55 #define S1_PORT 10000 56 #define S2_PORT 10001 57 58 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 59 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 60 #define CG_PATH "/sockmap" 61 62 /* global sockets */ 63 int s1, s2, c1, c2, p1, p2; 64 int test_cnt; 65 int passed; 66 int failed; 67 int map_fd[8]; 68 struct bpf_map *maps[8]; 69 int prog_fd[11]; 70 71 int txmsg_pass; 72 int txmsg_noisy; 73 int txmsg_redir; 74 int txmsg_redir_noisy; 75 int txmsg_drop; 76 int txmsg_apply; 77 int txmsg_cork; 78 int txmsg_start; 79 int txmsg_end; 80 int txmsg_start_push; 81 int txmsg_end_push; 82 int txmsg_ingress; 83 int txmsg_skb; 84 int ktls; 85 int peek_flag; 86 87 static const struct option long_options[] = { 88 {"help", no_argument, NULL, 'h' }, 89 {"cgroup", required_argument, NULL, 'c' }, 90 {"rate", required_argument, NULL, 'r' }, 91 {"verbose", no_argument, NULL, 'v' }, 92 {"iov_count", required_argument, NULL, 'i' }, 93 {"length", required_argument, NULL, 'l' }, 94 {"test", required_argument, NULL, 't' }, 95 {"data_test", no_argument, NULL, 'd' }, 96 {"txmsg", no_argument, &txmsg_pass, 1 }, 97 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 }, 98 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 99 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1}, 100 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 101 {"txmsg_apply", required_argument, NULL, 'a'}, 102 {"txmsg_cork", required_argument, NULL, 'k'}, 103 {"txmsg_start", required_argument, NULL, 's'}, 104 {"txmsg_end", required_argument, NULL, 'e'}, 105 {"txmsg_start_push", required_argument, NULL, 'p'}, 106 {"txmsg_end_push", required_argument, NULL, 'q'}, 107 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 108 {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 109 {"ktls", no_argument, &ktls, 1 }, 110 {"peek", no_argument, &peek_flag, 1 }, 111 {0, 0, NULL, 0 } 112 }; 113 114 static void usage(char *argv[]) 115 { 116 int i; 117 118 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 119 printf(" options:\n"); 120 for (i = 0; long_options[i].name != 0; i++) { 121 printf(" --%-12s", long_options[i].name); 122 if (long_options[i].flag != NULL) 123 printf(" flag (internal value:%d)\n", 124 *long_options[i].flag); 125 else 126 printf(" -%c\n", long_options[i].val); 127 } 128 printf("\n"); 129 } 130 131 char *sock_to_string(int s) 132 { 133 if (s == c1) 134 return "client1"; 135 else if (s == c2) 136 return "client2"; 137 else if (s == s1) 138 return "server1"; 139 else if (s == s2) 140 return "server2"; 141 else if (s == p1) 142 return "peer1"; 143 else if (s == p2) 144 return "peer2"; 145 else 146 return "unknown"; 147 } 148 149 static int sockmap_init_ktls(int verbose, int s) 150 { 151 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 152 .info = { 153 .version = TLS_1_2_VERSION, 154 .cipher_type = TLS_CIPHER_AES_GCM_128, 155 }, 156 }; 157 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 158 .info = { 159 .version = TLS_1_2_VERSION, 160 .cipher_type = TLS_CIPHER_AES_GCM_128, 161 }, 162 }; 163 int so_buf = 6553500; 164 int err; 165 166 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 167 if (err) { 168 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 169 return -EINVAL; 170 } 171 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 172 if (err) { 173 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 174 return -EINVAL; 175 } 176 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 177 if (err) { 178 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 179 return -EINVAL; 180 } 181 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 182 if (err) { 183 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 184 return -EINVAL; 185 } 186 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 187 if (err) { 188 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 189 return -EINVAL; 190 } 191 192 if (verbose) 193 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 194 return 0; 195 } 196 static int sockmap_init_sockets(int verbose) 197 { 198 int i, err, one = 1; 199 struct sockaddr_in addr; 200 int *fds[4] = {&s1, &s2, &c1, &c2}; 201 202 s1 = s2 = p1 = p2 = c1 = c2 = 0; 203 204 /* Init sockets */ 205 for (i = 0; i < 4; i++) { 206 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 207 if (*fds[i] < 0) { 208 perror("socket s1 failed()"); 209 return errno; 210 } 211 } 212 213 /* Allow reuse */ 214 for (i = 0; i < 2; i++) { 215 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 216 (char *)&one, sizeof(one)); 217 if (err) { 218 perror("setsockopt failed()"); 219 return errno; 220 } 221 } 222 223 /* Non-blocking sockets */ 224 for (i = 0; i < 2; i++) { 225 err = ioctl(*fds[i], FIONBIO, (char *)&one); 226 if (err < 0) { 227 perror("ioctl s1 failed()"); 228 return errno; 229 } 230 } 231 232 /* Bind server sockets */ 233 memset(&addr, 0, sizeof(struct sockaddr_in)); 234 addr.sin_family = AF_INET; 235 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 236 237 addr.sin_port = htons(S1_PORT); 238 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 239 if (err < 0) { 240 perror("bind s1 failed()\n"); 241 return errno; 242 } 243 244 addr.sin_port = htons(S2_PORT); 245 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 246 if (err < 0) { 247 perror("bind s2 failed()\n"); 248 return errno; 249 } 250 251 /* Listen server sockets */ 252 addr.sin_port = htons(S1_PORT); 253 err = listen(s1, 32); 254 if (err < 0) { 255 perror("listen s1 failed()\n"); 256 return errno; 257 } 258 259 addr.sin_port = htons(S2_PORT); 260 err = listen(s2, 32); 261 if (err < 0) { 262 perror("listen s1 failed()\n"); 263 return errno; 264 } 265 266 /* Initiate Connect */ 267 addr.sin_port = htons(S1_PORT); 268 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 269 if (err < 0 && errno != EINPROGRESS) { 270 perror("connect c1 failed()\n"); 271 return errno; 272 } 273 274 addr.sin_port = htons(S2_PORT); 275 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 276 if (err < 0 && errno != EINPROGRESS) { 277 perror("connect c2 failed()\n"); 278 return errno; 279 } else if (err < 0) { 280 err = 0; 281 } 282 283 /* Accept Connecrtions */ 284 p1 = accept(s1, NULL, NULL); 285 if (p1 < 0) { 286 perror("accept s1 failed()\n"); 287 return errno; 288 } 289 290 p2 = accept(s2, NULL, NULL); 291 if (p2 < 0) { 292 perror("accept s1 failed()\n"); 293 return errno; 294 } 295 296 if (verbose) { 297 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 298 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 299 c1, s1, c2, s2); 300 } 301 return 0; 302 } 303 304 struct msg_stats { 305 size_t bytes_sent; 306 size_t bytes_recvd; 307 struct timespec start; 308 struct timespec end; 309 }; 310 311 struct sockmap_options { 312 int verbose; 313 bool base; 314 bool sendpage; 315 bool data_test; 316 bool drop_expected; 317 int iov_count; 318 int iov_length; 319 int rate; 320 }; 321 322 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 323 struct msg_stats *s, 324 struct sockmap_options *opt) 325 { 326 bool drop = opt->drop_expected; 327 unsigned char k = 0; 328 FILE *file; 329 int i, fp; 330 331 file = fopen(".sendpage_tst.tmp", "w+"); 332 for (i = 0; i < iov_length * cnt; i++, k++) 333 fwrite(&k, sizeof(char), 1, file); 334 fflush(file); 335 fseek(file, 0, SEEK_SET); 336 fclose(file); 337 338 fp = open(".sendpage_tst.tmp", O_RDONLY); 339 clock_gettime(CLOCK_MONOTONIC, &s->start); 340 for (i = 0; i < cnt; i++) { 341 int sent = sendfile(fd, fp, NULL, iov_length); 342 343 if (!drop && sent < 0) { 344 perror("send loop error:"); 345 close(fp); 346 return sent; 347 } else if (drop && sent >= 0) { 348 printf("sendpage loop error expected: %i\n", sent); 349 close(fp); 350 return -EIO; 351 } 352 353 if (sent > 0) 354 s->bytes_sent += sent; 355 } 356 clock_gettime(CLOCK_MONOTONIC, &s->end); 357 close(fp); 358 return 0; 359 } 360 361 static void msg_free_iov(struct msghdr *msg) 362 { 363 int i; 364 365 for (i = 0; i < msg->msg_iovlen; i++) 366 free(msg->msg_iov[i].iov_base); 367 free(msg->msg_iov); 368 msg->msg_iov = NULL; 369 msg->msg_iovlen = 0; 370 } 371 372 static int msg_alloc_iov(struct msghdr *msg, 373 int iov_count, int iov_length, 374 bool data, bool xmit) 375 { 376 unsigned char k = 0; 377 struct iovec *iov; 378 int i; 379 380 iov = calloc(iov_count, sizeof(struct iovec)); 381 if (!iov) 382 return errno; 383 384 for (i = 0; i < iov_count; i++) { 385 unsigned char *d = calloc(iov_length, sizeof(char)); 386 387 if (!d) { 388 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 389 goto unwind_iov; 390 } 391 iov[i].iov_base = d; 392 iov[i].iov_len = iov_length; 393 394 if (data && xmit) { 395 int j; 396 397 for (j = 0; j < iov_length; j++) 398 d[j] = k++; 399 } 400 } 401 402 msg->msg_iov = iov; 403 msg->msg_iovlen = iov_count; 404 405 return 0; 406 unwind_iov: 407 for (i--; i >= 0 ; i--) 408 free(msg->msg_iov[i].iov_base); 409 return -ENOMEM; 410 } 411 412 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 413 { 414 int i, j, bytes_cnt = 0; 415 unsigned char k = 0; 416 417 for (i = 0; i < msg->msg_iovlen; i++) { 418 unsigned char *d = msg->msg_iov[i].iov_base; 419 420 for (j = 0; 421 j < msg->msg_iov[i].iov_len && size; j++) { 422 if (d[j] != k++) { 423 fprintf(stderr, 424 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 425 i, j, d[j], k - 1, d[j+1], k); 426 return -EIO; 427 } 428 bytes_cnt++; 429 if (bytes_cnt == chunk_sz) { 430 k = 0; 431 bytes_cnt = 0; 432 } 433 size--; 434 } 435 } 436 return 0; 437 } 438 439 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 440 struct msg_stats *s, bool tx, 441 struct sockmap_options *opt) 442 { 443 struct msghdr msg = {0}, msg_peek = {0}; 444 int err, i, flags = MSG_NOSIGNAL; 445 bool drop = opt->drop_expected; 446 bool data = opt->data_test; 447 448 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx); 449 if (err) 450 goto out_errno; 451 if (peek_flag) { 452 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 453 if (err) 454 goto out_errno; 455 } 456 457 if (tx) { 458 clock_gettime(CLOCK_MONOTONIC, &s->start); 459 for (i = 0; i < cnt; i++) { 460 int sent = sendmsg(fd, &msg, flags); 461 462 if (!drop && sent < 0) { 463 perror("send loop error:"); 464 goto out_errno; 465 } else if (drop && sent >= 0) { 466 printf("send loop error expected: %i\n", sent); 467 errno = -EIO; 468 goto out_errno; 469 } 470 if (sent > 0) 471 s->bytes_sent += sent; 472 } 473 clock_gettime(CLOCK_MONOTONIC, &s->end); 474 } else { 475 int slct, recvp = 0, recv, max_fd = fd; 476 int fd_flags = O_NONBLOCK; 477 struct timeval timeout; 478 float total_bytes; 479 fd_set w; 480 481 fcntl(fd, fd_flags); 482 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 483 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 484 if (err < 0) 485 perror("recv start time: "); 486 while (s->bytes_recvd < total_bytes) { 487 if (txmsg_cork) { 488 timeout.tv_sec = 0; 489 timeout.tv_usec = 300000; 490 } else { 491 timeout.tv_sec = 1; 492 timeout.tv_usec = 0; 493 } 494 495 /* FD sets */ 496 FD_ZERO(&w); 497 FD_SET(fd, &w); 498 499 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 500 if (slct == -1) { 501 perror("select()"); 502 clock_gettime(CLOCK_MONOTONIC, &s->end); 503 goto out_errno; 504 } else if (!slct) { 505 if (opt->verbose) 506 fprintf(stderr, "unexpected timeout\n"); 507 errno = -EIO; 508 clock_gettime(CLOCK_MONOTONIC, &s->end); 509 goto out_errno; 510 } 511 512 errno = 0; 513 if (peek_flag) { 514 flags |= MSG_PEEK; 515 recvp = recvmsg(fd, &msg_peek, flags); 516 if (recvp < 0) { 517 if (errno != EWOULDBLOCK) { 518 clock_gettime(CLOCK_MONOTONIC, &s->end); 519 goto out_errno; 520 } 521 } 522 flags = 0; 523 } 524 525 recv = recvmsg(fd, &msg, flags); 526 if (recv < 0) { 527 if (errno != EWOULDBLOCK) { 528 clock_gettime(CLOCK_MONOTONIC, &s->end); 529 perror("recv failed()\n"); 530 goto out_errno; 531 } 532 } 533 534 s->bytes_recvd += recv; 535 536 if (data) { 537 int chunk_sz = opt->sendpage ? 538 iov_length * cnt : 539 iov_length * iov_count; 540 541 errno = msg_verify_data(&msg, recv, chunk_sz); 542 if (errno) { 543 perror("data verify msg failed\n"); 544 goto out_errno; 545 } 546 if (recvp) { 547 errno = msg_verify_data(&msg_peek, 548 recvp, 549 chunk_sz); 550 if (errno) { 551 perror("data verify msg_peek failed\n"); 552 goto out_errno; 553 } 554 } 555 } 556 } 557 clock_gettime(CLOCK_MONOTONIC, &s->end); 558 } 559 560 msg_free_iov(&msg); 561 msg_free_iov(&msg_peek); 562 return err; 563 out_errno: 564 msg_free_iov(&msg); 565 msg_free_iov(&msg_peek); 566 return errno; 567 } 568 569 static float giga = 1000000000; 570 571 static inline float sentBps(struct msg_stats s) 572 { 573 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 574 } 575 576 static inline float recvdBps(struct msg_stats s) 577 { 578 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 579 } 580 581 static int sendmsg_test(struct sockmap_options *opt) 582 { 583 float sent_Bps = 0, recvd_Bps = 0; 584 int rx_fd, txpid, rxpid, err = 0; 585 struct msg_stats s = {0}; 586 int iov_count = opt->iov_count; 587 int iov_buf = opt->iov_length; 588 int rx_status, tx_status; 589 int cnt = opt->rate; 590 591 errno = 0; 592 593 if (opt->base) 594 rx_fd = p1; 595 else 596 rx_fd = p2; 597 598 if (ktls) { 599 /* Redirecting into non-TLS socket which sends into a TLS 600 * socket is not a valid test. So in this case lets not 601 * enable kTLS but still run the test. 602 */ 603 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 604 err = sockmap_init_ktls(opt->verbose, rx_fd); 605 if (err) 606 return err; 607 } 608 err = sockmap_init_ktls(opt->verbose, c1); 609 if (err) 610 return err; 611 } 612 613 rxpid = fork(); 614 if (rxpid == 0) { 615 if (opt->drop_expected) 616 exit(0); 617 618 if (opt->sendpage) 619 iov_count = 1; 620 err = msg_loop(rx_fd, iov_count, iov_buf, 621 cnt, &s, false, opt); 622 if (err && opt->verbose) 623 fprintf(stderr, 624 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 625 iov_count, iov_buf, cnt, err); 626 if (s.end.tv_sec - s.start.tv_sec) { 627 sent_Bps = sentBps(s); 628 recvd_Bps = recvdBps(s); 629 } 630 if (opt->verbose) 631 fprintf(stdout, 632 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 633 s.bytes_sent, sent_Bps, sent_Bps/giga, 634 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 635 peek_flag ? "(peek_msg)" : ""); 636 if (err && txmsg_cork) 637 err = 0; 638 exit(err ? 1 : 0); 639 } else if (rxpid == -1) { 640 perror("msg_loop_rx: "); 641 return errno; 642 } 643 644 txpid = fork(); 645 if (txpid == 0) { 646 if (opt->sendpage) 647 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 648 else 649 err = msg_loop(c1, iov_count, iov_buf, 650 cnt, &s, true, opt); 651 652 if (err) 653 fprintf(stderr, 654 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 655 iov_count, iov_buf, cnt, err); 656 if (s.end.tv_sec - s.start.tv_sec) { 657 sent_Bps = sentBps(s); 658 recvd_Bps = recvdBps(s); 659 } 660 if (opt->verbose) 661 fprintf(stdout, 662 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 663 s.bytes_sent, sent_Bps, sent_Bps/giga, 664 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 665 exit(err ? 1 : 0); 666 } else if (txpid == -1) { 667 perror("msg_loop_tx: "); 668 return errno; 669 } 670 671 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 672 assert(waitpid(txpid, &tx_status, 0) == txpid); 673 if (WIFEXITED(rx_status)) { 674 err = WEXITSTATUS(rx_status); 675 if (err) { 676 fprintf(stderr, "rx thread exited with err %d. ", err); 677 goto out; 678 } 679 } 680 if (WIFEXITED(tx_status)) { 681 err = WEXITSTATUS(tx_status); 682 if (err) 683 fprintf(stderr, "tx thread exited with err %d. ", err); 684 } 685 out: 686 return err; 687 } 688 689 static int forever_ping_pong(int rate, struct sockmap_options *opt) 690 { 691 struct timeval timeout; 692 char buf[1024] = {0}; 693 int sc; 694 695 timeout.tv_sec = 10; 696 timeout.tv_usec = 0; 697 698 /* Ping/Pong data from client to server */ 699 sc = send(c1, buf, sizeof(buf), 0); 700 if (sc < 0) { 701 perror("send failed()\n"); 702 return sc; 703 } 704 705 do { 706 int s, rc, i, max_fd = p2; 707 fd_set w; 708 709 /* FD sets */ 710 FD_ZERO(&w); 711 FD_SET(c1, &w); 712 FD_SET(c2, &w); 713 FD_SET(p1, &w); 714 FD_SET(p2, &w); 715 716 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 717 if (s == -1) { 718 perror("select()"); 719 break; 720 } else if (!s) { 721 fprintf(stderr, "unexpected timeout\n"); 722 break; 723 } 724 725 for (i = 0; i <= max_fd && s > 0; ++i) { 726 if (!FD_ISSET(i, &w)) 727 continue; 728 729 s--; 730 731 rc = recv(i, buf, sizeof(buf), 0); 732 if (rc < 0) { 733 if (errno != EWOULDBLOCK) { 734 perror("recv failed()\n"); 735 return rc; 736 } 737 } 738 739 if (rc == 0) { 740 close(i); 741 break; 742 } 743 744 sc = send(i, buf, rc, 0); 745 if (sc < 0) { 746 perror("send failed()\n"); 747 return sc; 748 } 749 } 750 751 if (rate) 752 sleep(rate); 753 754 if (opt->verbose) { 755 printf("."); 756 fflush(stdout); 757 758 } 759 } while (running); 760 761 return 0; 762 } 763 764 enum { 765 PING_PONG, 766 SENDMSG, 767 BASE, 768 BASE_SENDPAGE, 769 SENDPAGE, 770 }; 771 772 static int run_options(struct sockmap_options *options, int cg_fd, int test) 773 { 774 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 775 776 /* If base test skip BPF setup */ 777 if (test == BASE || test == BASE_SENDPAGE) 778 goto run; 779 780 /* Attach programs to sockmap */ 781 err = bpf_prog_attach(prog_fd[0], map_fd[0], 782 BPF_SK_SKB_STREAM_PARSER, 0); 783 if (err) { 784 fprintf(stderr, 785 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 786 prog_fd[0], map_fd[0], err, strerror(errno)); 787 return err; 788 } 789 790 err = bpf_prog_attach(prog_fd[1], map_fd[0], 791 BPF_SK_SKB_STREAM_VERDICT, 0); 792 if (err) { 793 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 794 err, strerror(errno)); 795 return err; 796 } 797 798 /* Attach to cgroups */ 799 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 800 if (err) { 801 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 802 err, strerror(errno)); 803 return err; 804 } 805 806 run: 807 err = sockmap_init_sockets(options->verbose); 808 if (err) { 809 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 810 goto out; 811 } 812 813 /* Attach txmsg program to sockmap */ 814 if (txmsg_pass) 815 tx_prog_fd = prog_fd[3]; 816 else if (txmsg_noisy) 817 tx_prog_fd = prog_fd[4]; 818 else if (txmsg_redir) 819 tx_prog_fd = prog_fd[5]; 820 else if (txmsg_redir_noisy) 821 tx_prog_fd = prog_fd[6]; 822 else if (txmsg_drop) 823 tx_prog_fd = prog_fd[9]; 824 /* apply and cork must be last */ 825 else if (txmsg_apply) 826 tx_prog_fd = prog_fd[7]; 827 else if (txmsg_cork) 828 tx_prog_fd = prog_fd[8]; 829 else 830 tx_prog_fd = 0; 831 832 if (tx_prog_fd) { 833 int redir_fd, i = 0; 834 835 err = bpf_prog_attach(tx_prog_fd, 836 map_fd[1], BPF_SK_MSG_VERDICT, 0); 837 if (err) { 838 fprintf(stderr, 839 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 840 err, strerror(errno)); 841 goto out; 842 } 843 844 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 845 if (err) { 846 fprintf(stderr, 847 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 848 err, strerror(errno)); 849 goto out; 850 } 851 852 if (txmsg_redir || txmsg_redir_noisy) 853 redir_fd = c2; 854 else 855 redir_fd = c1; 856 857 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 858 if (err) { 859 fprintf(stderr, 860 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 861 err, strerror(errno)); 862 goto out; 863 } 864 865 if (txmsg_apply) { 866 err = bpf_map_update_elem(map_fd[3], 867 &i, &txmsg_apply, BPF_ANY); 868 if (err) { 869 fprintf(stderr, 870 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 871 err, strerror(errno)); 872 goto out; 873 } 874 } 875 876 if (txmsg_cork) { 877 err = bpf_map_update_elem(map_fd[4], 878 &i, &txmsg_cork, BPF_ANY); 879 if (err) { 880 fprintf(stderr, 881 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 882 err, strerror(errno)); 883 goto out; 884 } 885 } 886 887 if (txmsg_start) { 888 err = bpf_map_update_elem(map_fd[5], 889 &i, &txmsg_start, BPF_ANY); 890 if (err) { 891 fprintf(stderr, 892 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 893 err, strerror(errno)); 894 goto out; 895 } 896 } 897 898 if (txmsg_end) { 899 i = 1; 900 err = bpf_map_update_elem(map_fd[5], 901 &i, &txmsg_end, BPF_ANY); 902 if (err) { 903 fprintf(stderr, 904 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 905 err, strerror(errno)); 906 goto out; 907 } 908 } 909 910 if (txmsg_start_push) { 911 i = 2; 912 err = bpf_map_update_elem(map_fd[5], 913 &i, &txmsg_start_push, BPF_ANY); 914 if (err) { 915 fprintf(stderr, 916 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 917 err, strerror(errno)); 918 goto out; 919 } 920 } 921 922 if (txmsg_end_push) { 923 i = 3; 924 err = bpf_map_update_elem(map_fd[5], 925 &i, &txmsg_end_push, BPF_ANY); 926 if (err) { 927 fprintf(stderr, 928 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 929 txmsg_end_push, i, err, strerror(errno)); 930 goto out; 931 } 932 } 933 934 if (txmsg_ingress) { 935 int in = BPF_F_INGRESS; 936 937 i = 0; 938 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 939 if (err) { 940 fprintf(stderr, 941 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 942 err, strerror(errno)); 943 } 944 i = 1; 945 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 946 if (err) { 947 fprintf(stderr, 948 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 949 err, strerror(errno)); 950 } 951 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 952 if (err) { 953 fprintf(stderr, 954 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 955 err, strerror(errno)); 956 } 957 958 i = 2; 959 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 960 if (err) { 961 fprintf(stderr, 962 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 963 err, strerror(errno)); 964 } 965 } 966 967 if (txmsg_skb) { 968 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 969 p2 : p1; 970 int ingress = BPF_F_INGRESS; 971 972 i = 0; 973 err = bpf_map_update_elem(map_fd[7], 974 &i, &ingress, BPF_ANY); 975 if (err) { 976 fprintf(stderr, 977 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 978 err, strerror(errno)); 979 } 980 981 i = 3; 982 err = bpf_map_update_elem(map_fd[0], 983 &i, &skb_fd, BPF_ANY); 984 if (err) { 985 fprintf(stderr, 986 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 987 err, strerror(errno)); 988 } 989 } 990 } 991 992 if (txmsg_drop) 993 options->drop_expected = true; 994 995 if (test == PING_PONG) 996 err = forever_ping_pong(options->rate, options); 997 else if (test == SENDMSG) { 998 options->base = false; 999 options->sendpage = false; 1000 err = sendmsg_test(options); 1001 } else if (test == SENDPAGE) { 1002 options->base = false; 1003 options->sendpage = true; 1004 err = sendmsg_test(options); 1005 } else if (test == BASE) { 1006 options->base = true; 1007 options->sendpage = false; 1008 err = sendmsg_test(options); 1009 } else if (test == BASE_SENDPAGE) { 1010 options->base = true; 1011 options->sendpage = true; 1012 err = sendmsg_test(options); 1013 } else 1014 fprintf(stderr, "unknown test\n"); 1015 out: 1016 /* Detatch and zero all the maps */ 1017 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 1018 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1019 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1020 if (tx_prog_fd >= 0) 1021 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1022 1023 for (i = 0; i < 8; i++) { 1024 key = next_key = 0; 1025 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1026 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1027 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1028 key = next_key; 1029 } 1030 } 1031 1032 close(s1); 1033 close(s2); 1034 close(p1); 1035 close(p2); 1036 close(c1); 1037 close(c2); 1038 return err; 1039 } 1040 1041 static char *test_to_str(int test) 1042 { 1043 switch (test) { 1044 case SENDMSG: 1045 return "sendmsg"; 1046 case SENDPAGE: 1047 return "sendpage"; 1048 } 1049 return "unknown"; 1050 } 1051 1052 #define OPTSTRING 60 1053 static void test_options(char *options) 1054 { 1055 char tstr[OPTSTRING]; 1056 1057 memset(options, 0, OPTSTRING); 1058 1059 if (txmsg_pass) 1060 strncat(options, "pass,", OPTSTRING); 1061 if (txmsg_noisy) 1062 strncat(options, "pass_noisy,", OPTSTRING); 1063 if (txmsg_redir) 1064 strncat(options, "redir,", OPTSTRING); 1065 if (txmsg_redir_noisy) 1066 strncat(options, "redir_noisy,", OPTSTRING); 1067 if (txmsg_drop) 1068 strncat(options, "drop,", OPTSTRING); 1069 if (txmsg_apply) { 1070 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1071 strncat(options, tstr, OPTSTRING); 1072 } 1073 if (txmsg_cork) { 1074 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1075 strncat(options, tstr, OPTSTRING); 1076 } 1077 if (txmsg_start) { 1078 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1079 strncat(options, tstr, OPTSTRING); 1080 } 1081 if (txmsg_end) { 1082 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1083 strncat(options, tstr, OPTSTRING); 1084 } 1085 if (txmsg_ingress) 1086 strncat(options, "ingress,", OPTSTRING); 1087 if (txmsg_skb) 1088 strncat(options, "skb,", OPTSTRING); 1089 if (ktls) 1090 strncat(options, "ktls,", OPTSTRING); 1091 if (peek_flag) 1092 strncat(options, "peek,", OPTSTRING); 1093 } 1094 1095 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1096 { 1097 char *options = calloc(OPTSTRING, sizeof(char)); 1098 int err; 1099 1100 if (test == SENDPAGE) 1101 opt->sendpage = true; 1102 else 1103 opt->sendpage = false; 1104 1105 if (txmsg_drop) 1106 opt->drop_expected = true; 1107 else 1108 opt->drop_expected = false; 1109 1110 test_options(options); 1111 1112 fprintf(stdout, 1113 "[TEST %i]: (%i, %i, %i, %s, %s): ", 1114 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1115 test_to_str(test), options); 1116 fflush(stdout); 1117 err = run_options(opt, cgrp, test); 1118 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 1119 test_cnt++; 1120 !err ? passed++ : failed++; 1121 free(options); 1122 return err; 1123 } 1124 1125 static int test_exec(int cgrp, struct sockmap_options *opt) 1126 { 1127 int err = __test_exec(cgrp, SENDMSG, opt); 1128 1129 if (err) 1130 goto out; 1131 1132 err = __test_exec(cgrp, SENDPAGE, opt); 1133 out: 1134 return err; 1135 } 1136 1137 static int test_loop(int cgrp) 1138 { 1139 struct sockmap_options opt; 1140 1141 int err, i, l, r; 1142 1143 opt.verbose = 0; 1144 opt.base = false; 1145 opt.sendpage = false; 1146 opt.data_test = false; 1147 opt.drop_expected = false; 1148 opt.iov_count = 0; 1149 opt.iov_length = 0; 1150 opt.rate = 0; 1151 1152 r = 1; 1153 for (i = 1; i < 100; i += 33) { 1154 for (l = 1; l < 100; l += 33) { 1155 opt.rate = r; 1156 opt.iov_count = i; 1157 opt.iov_length = l; 1158 err = test_exec(cgrp, &opt); 1159 if (err) 1160 goto out; 1161 } 1162 } 1163 sched_yield(); 1164 out: 1165 return err; 1166 } 1167 1168 static int test_txmsg(int cgrp) 1169 { 1170 int err; 1171 1172 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1173 txmsg_apply = txmsg_cork = 0; 1174 txmsg_ingress = txmsg_skb = 0; 1175 1176 txmsg_pass = 1; 1177 err = test_loop(cgrp); 1178 txmsg_pass = 0; 1179 if (err) 1180 goto out; 1181 1182 txmsg_redir = 1; 1183 err = test_loop(cgrp); 1184 txmsg_redir = 0; 1185 if (err) 1186 goto out; 1187 1188 txmsg_drop = 1; 1189 err = test_loop(cgrp); 1190 txmsg_drop = 0; 1191 if (err) 1192 goto out; 1193 1194 txmsg_redir = 1; 1195 txmsg_ingress = 1; 1196 err = test_loop(cgrp); 1197 txmsg_redir = 0; 1198 txmsg_ingress = 0; 1199 if (err) 1200 goto out; 1201 out: 1202 txmsg_pass = 0; 1203 txmsg_redir = 0; 1204 txmsg_drop = 0; 1205 return err; 1206 } 1207 1208 static int test_send(struct sockmap_options *opt, int cgrp) 1209 { 1210 int err; 1211 1212 opt->iov_length = 1; 1213 opt->iov_count = 1; 1214 opt->rate = 1; 1215 err = test_exec(cgrp, opt); 1216 if (err) 1217 goto out; 1218 1219 opt->iov_length = 1; 1220 opt->iov_count = 1024; 1221 opt->rate = 1; 1222 err = test_exec(cgrp, opt); 1223 if (err) 1224 goto out; 1225 1226 opt->iov_length = 1024; 1227 opt->iov_count = 1; 1228 opt->rate = 1; 1229 err = test_exec(cgrp, opt); 1230 if (err) 1231 goto out; 1232 1233 opt->iov_length = 1; 1234 opt->iov_count = 1; 1235 opt->rate = 512; 1236 err = test_exec(cgrp, opt); 1237 if (err) 1238 goto out; 1239 1240 opt->iov_length = 256; 1241 opt->iov_count = 1024; 1242 opt->rate = 2; 1243 err = test_exec(cgrp, opt); 1244 if (err) 1245 goto out; 1246 1247 opt->rate = 100; 1248 opt->iov_count = 1; 1249 opt->iov_length = 5; 1250 err = test_exec(cgrp, opt); 1251 if (err) 1252 goto out; 1253 out: 1254 sched_yield(); 1255 return err; 1256 } 1257 1258 static int test_mixed(int cgrp) 1259 { 1260 struct sockmap_options opt = {0}; 1261 int err; 1262 1263 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1264 txmsg_apply = txmsg_cork = 0; 1265 txmsg_start = txmsg_end = 0; 1266 txmsg_start_push = txmsg_end_push = 0; 1267 1268 /* Test small and large iov_count values with pass/redir/apply/cork */ 1269 txmsg_pass = 1; 1270 txmsg_redir = 0; 1271 txmsg_apply = 1; 1272 txmsg_cork = 0; 1273 err = test_send(&opt, cgrp); 1274 if (err) 1275 goto out; 1276 1277 txmsg_pass = 1; 1278 txmsg_redir = 0; 1279 txmsg_apply = 0; 1280 txmsg_cork = 1; 1281 err = test_send(&opt, cgrp); 1282 if (err) 1283 goto out; 1284 1285 txmsg_pass = 1; 1286 txmsg_redir = 0; 1287 txmsg_apply = 1; 1288 txmsg_cork = 1; 1289 err = test_send(&opt, cgrp); 1290 if (err) 1291 goto out; 1292 1293 txmsg_pass = 1; 1294 txmsg_redir = 0; 1295 txmsg_apply = 1024; 1296 txmsg_cork = 0; 1297 err = test_send(&opt, cgrp); 1298 if (err) 1299 goto out; 1300 1301 txmsg_pass = 1; 1302 txmsg_redir = 0; 1303 txmsg_apply = 0; 1304 txmsg_cork = 1024; 1305 err = test_send(&opt, cgrp); 1306 if (err) 1307 goto out; 1308 1309 txmsg_pass = 1; 1310 txmsg_redir = 0; 1311 txmsg_apply = 1024; 1312 txmsg_cork = 1024; 1313 err = test_send(&opt, cgrp); 1314 if (err) 1315 goto out; 1316 1317 txmsg_pass = 1; 1318 txmsg_redir = 0; 1319 txmsg_cork = 4096; 1320 txmsg_apply = 4096; 1321 err = test_send(&opt, cgrp); 1322 if (err) 1323 goto out; 1324 1325 txmsg_pass = 0; 1326 txmsg_redir = 1; 1327 txmsg_apply = 1; 1328 txmsg_cork = 0; 1329 err = test_send(&opt, cgrp); 1330 if (err) 1331 goto out; 1332 1333 txmsg_pass = 0; 1334 txmsg_redir = 1; 1335 txmsg_apply = 0; 1336 txmsg_cork = 1; 1337 err = test_send(&opt, cgrp); 1338 if (err) 1339 goto out; 1340 1341 txmsg_pass = 0; 1342 txmsg_redir = 1; 1343 txmsg_apply = 1024; 1344 txmsg_cork = 0; 1345 err = test_send(&opt, cgrp); 1346 if (err) 1347 goto out; 1348 1349 txmsg_pass = 0; 1350 txmsg_redir = 1; 1351 txmsg_apply = 0; 1352 txmsg_cork = 1024; 1353 err = test_send(&opt, cgrp); 1354 if (err) 1355 goto out; 1356 1357 txmsg_pass = 0; 1358 txmsg_redir = 1; 1359 txmsg_apply = 1024; 1360 txmsg_cork = 1024; 1361 err = test_send(&opt, cgrp); 1362 if (err) 1363 goto out; 1364 1365 txmsg_pass = 0; 1366 txmsg_redir = 1; 1367 txmsg_cork = 4096; 1368 txmsg_apply = 4096; 1369 err = test_send(&opt, cgrp); 1370 if (err) 1371 goto out; 1372 out: 1373 return err; 1374 } 1375 1376 static int test_start_end(int cgrp) 1377 { 1378 struct sockmap_options opt = {0}; 1379 int err, i; 1380 1381 /* Test basic start/end with lots of iov_count and iov_lengths */ 1382 txmsg_start = 1; 1383 txmsg_end = 2; 1384 txmsg_start_push = 1; 1385 txmsg_end_push = 2; 1386 err = test_txmsg(cgrp); 1387 if (err) 1388 goto out; 1389 1390 /* Test start/end with cork */ 1391 opt.rate = 16; 1392 opt.iov_count = 1; 1393 opt.iov_length = 100; 1394 txmsg_cork = 1600; 1395 1396 for (i = 99; i <= 1600; i += 500) { 1397 txmsg_start = 0; 1398 txmsg_end = i; 1399 txmsg_start_push = 0; 1400 txmsg_end_push = i; 1401 err = test_exec(cgrp, &opt); 1402 if (err) 1403 goto out; 1404 } 1405 1406 /* Test start/end with cork but pull data in middle */ 1407 for (i = 199; i <= 1600; i += 500) { 1408 txmsg_start = 100; 1409 txmsg_end = i; 1410 txmsg_start_push = 100; 1411 txmsg_end_push = i; 1412 err = test_exec(cgrp, &opt); 1413 if (err) 1414 goto out; 1415 } 1416 1417 /* Test start/end with cork pulling last sg entry */ 1418 txmsg_start = 1500; 1419 txmsg_end = 1600; 1420 txmsg_start_push = 1500; 1421 txmsg_end_push = 1600; 1422 err = test_exec(cgrp, &opt); 1423 if (err) 1424 goto out; 1425 1426 /* Test start/end pull of single byte in last page */ 1427 txmsg_start = 1111; 1428 txmsg_end = 1112; 1429 txmsg_start_push = 1111; 1430 txmsg_end_push = 1112; 1431 err = test_exec(cgrp, &opt); 1432 if (err) 1433 goto out; 1434 1435 /* Test start/end with end < start */ 1436 txmsg_start = 1111; 1437 txmsg_end = 0; 1438 txmsg_start_push = 1111; 1439 txmsg_end_push = 0; 1440 err = test_exec(cgrp, &opt); 1441 if (err) 1442 goto out; 1443 1444 /* Test start/end with end > data */ 1445 txmsg_start = 0; 1446 txmsg_end = 1601; 1447 txmsg_start_push = 0; 1448 txmsg_end_push = 1601; 1449 err = test_exec(cgrp, &opt); 1450 if (err) 1451 goto out; 1452 1453 /* Test start/end with start > data */ 1454 txmsg_start = 1601; 1455 txmsg_end = 1600; 1456 txmsg_start_push = 1601; 1457 txmsg_end_push = 1600; 1458 err = test_exec(cgrp, &opt); 1459 1460 out: 1461 txmsg_start = 0; 1462 txmsg_end = 0; 1463 sched_yield(); 1464 return err; 1465 } 1466 1467 char *map_names[] = { 1468 "sock_map", 1469 "sock_map_txmsg", 1470 "sock_map_redir", 1471 "sock_apply_bytes", 1472 "sock_cork_bytes", 1473 "sock_bytes", 1474 "sock_redir_flags", 1475 "sock_skb_opts", 1476 }; 1477 1478 int prog_attach_type[] = { 1479 BPF_SK_SKB_STREAM_PARSER, 1480 BPF_SK_SKB_STREAM_VERDICT, 1481 BPF_CGROUP_SOCK_OPS, 1482 BPF_SK_MSG_VERDICT, 1483 BPF_SK_MSG_VERDICT, 1484 BPF_SK_MSG_VERDICT, 1485 BPF_SK_MSG_VERDICT, 1486 BPF_SK_MSG_VERDICT, 1487 BPF_SK_MSG_VERDICT, 1488 BPF_SK_MSG_VERDICT, 1489 }; 1490 1491 int prog_type[] = { 1492 BPF_PROG_TYPE_SK_SKB, 1493 BPF_PROG_TYPE_SK_SKB, 1494 BPF_PROG_TYPE_SOCK_OPS, 1495 BPF_PROG_TYPE_SK_MSG, 1496 BPF_PROG_TYPE_SK_MSG, 1497 BPF_PROG_TYPE_SK_MSG, 1498 BPF_PROG_TYPE_SK_MSG, 1499 BPF_PROG_TYPE_SK_MSG, 1500 BPF_PROG_TYPE_SK_MSG, 1501 BPF_PROG_TYPE_SK_MSG, 1502 }; 1503 1504 static int populate_progs(char *bpf_file) 1505 { 1506 struct bpf_program *prog; 1507 struct bpf_object *obj; 1508 int i = 0; 1509 long err; 1510 1511 obj = bpf_object__open(bpf_file); 1512 err = libbpf_get_error(obj); 1513 if (err) { 1514 char err_buf[256]; 1515 1516 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1517 printf("Unable to load eBPF objects in file '%s' : %s\n", 1518 bpf_file, err_buf); 1519 return -1; 1520 } 1521 1522 bpf_object__for_each_program(prog, obj) { 1523 bpf_program__set_type(prog, prog_type[i]); 1524 bpf_program__set_expected_attach_type(prog, 1525 prog_attach_type[i]); 1526 i++; 1527 } 1528 1529 i = bpf_object__load(obj); 1530 i = 0; 1531 bpf_object__for_each_program(prog, obj) { 1532 prog_fd[i] = bpf_program__fd(prog); 1533 i++; 1534 } 1535 1536 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 1537 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1538 map_fd[i] = bpf_map__fd(maps[i]); 1539 if (map_fd[i] < 0) { 1540 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1541 map_fd[i], strerror(errno)); 1542 return -1; 1543 } 1544 } 1545 1546 return 0; 1547 } 1548 1549 static int __test_suite(int cg_fd, char *bpf_file) 1550 { 1551 int err, cleanup = cg_fd; 1552 1553 err = populate_progs(bpf_file); 1554 if (err < 0) { 1555 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1556 return err; 1557 } 1558 1559 if (cg_fd < 0) { 1560 if (setup_cgroup_environment()) { 1561 fprintf(stderr, "ERROR: cgroup env failed\n"); 1562 return -EINVAL; 1563 } 1564 1565 cg_fd = create_and_get_cgroup(CG_PATH); 1566 if (cg_fd < 0) { 1567 fprintf(stderr, 1568 "ERROR: (%i) open cg path failed: %s\n", 1569 cg_fd, optarg); 1570 return cg_fd; 1571 } 1572 1573 if (join_cgroup(CG_PATH)) { 1574 fprintf(stderr, "ERROR: failed to join cgroup\n"); 1575 return -EINVAL; 1576 } 1577 } 1578 1579 /* Tests basic commands and APIs with range of iov values */ 1580 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0; 1581 err = test_txmsg(cg_fd); 1582 if (err) 1583 goto out; 1584 1585 /* Tests interesting combinations of APIs used together */ 1586 err = test_mixed(cg_fd); 1587 if (err) 1588 goto out; 1589 1590 /* Tests pull_data API using start/end API */ 1591 err = test_start_end(cg_fd); 1592 if (err) 1593 goto out; 1594 1595 out: 1596 printf("Summary: %i PASSED %i FAILED\n", passed, failed); 1597 if (cleanup < 0) { 1598 cleanup_cgroup_environment(); 1599 close(cg_fd); 1600 } 1601 return err; 1602 } 1603 1604 static int test_suite(int cg_fd) 1605 { 1606 int err; 1607 1608 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME); 1609 if (err) 1610 goto out; 1611 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME); 1612 out: 1613 if (cg_fd > -1) 1614 close(cg_fd); 1615 return err; 1616 } 1617 1618 int main(int argc, char **argv) 1619 { 1620 int iov_count = 1, length = 1024, rate = 1; 1621 struct sockmap_options options = {0}; 1622 int opt, longindex, err, cg_fd = 0; 1623 char *bpf_file = BPF_SOCKMAP_FILENAME; 1624 int test = PING_PONG; 1625 1626 if (argc < 2) 1627 return test_suite(-1); 1628 1629 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:", 1630 long_options, &longindex)) != -1) { 1631 switch (opt) { 1632 case 's': 1633 txmsg_start = atoi(optarg); 1634 break; 1635 case 'e': 1636 txmsg_end = atoi(optarg); 1637 break; 1638 case 'p': 1639 txmsg_start_push = atoi(optarg); 1640 break; 1641 case 'q': 1642 txmsg_end_push = atoi(optarg); 1643 break; 1644 case 'a': 1645 txmsg_apply = atoi(optarg); 1646 break; 1647 case 'k': 1648 txmsg_cork = atoi(optarg); 1649 break; 1650 case 'c': 1651 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1652 if (cg_fd < 0) { 1653 fprintf(stderr, 1654 "ERROR: (%i) open cg path failed: %s\n", 1655 cg_fd, optarg); 1656 return cg_fd; 1657 } 1658 break; 1659 case 'r': 1660 rate = atoi(optarg); 1661 break; 1662 case 'v': 1663 options.verbose = 1; 1664 break; 1665 case 'i': 1666 iov_count = atoi(optarg); 1667 break; 1668 case 'l': 1669 length = atoi(optarg); 1670 break; 1671 case 'd': 1672 options.data_test = true; 1673 break; 1674 case 't': 1675 if (strcmp(optarg, "ping") == 0) { 1676 test = PING_PONG; 1677 } else if (strcmp(optarg, "sendmsg") == 0) { 1678 test = SENDMSG; 1679 } else if (strcmp(optarg, "base") == 0) { 1680 test = BASE; 1681 } else if (strcmp(optarg, "base_sendpage") == 0) { 1682 test = BASE_SENDPAGE; 1683 } else if (strcmp(optarg, "sendpage") == 0) { 1684 test = SENDPAGE; 1685 } else { 1686 usage(argv); 1687 return -1; 1688 } 1689 break; 1690 case 0: 1691 break; 1692 case 'h': 1693 default: 1694 usage(argv); 1695 return -1; 1696 } 1697 } 1698 1699 if (argc <= 3 && cg_fd) 1700 return test_suite(cg_fd); 1701 1702 if (!cg_fd) { 1703 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n", 1704 argv[0]); 1705 return -1; 1706 } 1707 1708 err = populate_progs(bpf_file); 1709 if (err) { 1710 fprintf(stderr, "populate program: (%s) %s\n", 1711 bpf_file, strerror(errno)); 1712 return 1; 1713 } 1714 running = 1; 1715 1716 /* catch SIGINT */ 1717 signal(SIGINT, running_handler); 1718 1719 options.iov_count = iov_count; 1720 options.iov_length = length; 1721 options.rate = rate; 1722 1723 err = run_options(&options, cg_fd, test); 1724 close(cg_fd); 1725 return err; 1726 } 1727 1728 void running_handler(int a) 1729 { 1730 running = 0; 1731 } 1732