1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/resource.h> 22 #include <sys/types.h> 23 #include <sys/sendfile.h> 24 25 #include <linux/netlink.h> 26 #include <linux/socket.h> 27 #include <linux/sock_diag.h> 28 #include <linux/bpf.h> 29 #include <linux/if_link.h> 30 #include <linux/tls.h> 31 #include <assert.h> 32 #include <libgen.h> 33 34 #include <getopt.h> 35 36 #include <bpf/bpf.h> 37 #include <bpf/libbpf.h> 38 39 #include "bpf_util.h" 40 #include "bpf_rlimit.h" 41 #include "cgroup_helpers.h" 42 43 int running; 44 static void running_handler(int a); 45 46 #ifndef TCP_ULP 47 # define TCP_ULP 31 48 #endif 49 #ifndef SOL_TLS 50 # define SOL_TLS 282 51 #endif 52 53 /* randomly selected ports for testing on lo */ 54 #define S1_PORT 10000 55 #define S2_PORT 10001 56 57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 59 #define CG_PATH "/sockmap" 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[8]; 67 struct bpf_map *maps[8]; 68 int prog_fd[11]; 69 70 int txmsg_pass; 71 int txmsg_noisy; 72 int txmsg_redir; 73 int txmsg_redir_noisy; 74 int txmsg_drop; 75 int txmsg_apply; 76 int txmsg_cork; 77 int txmsg_start; 78 int txmsg_end; 79 int txmsg_start_push; 80 int txmsg_end_push; 81 int txmsg_start_pop; 82 int txmsg_pop; 83 int txmsg_ingress; 84 int txmsg_skb; 85 int ktls; 86 int peek_flag; 87 88 static const struct option long_options[] = { 89 {"help", no_argument, NULL, 'h' }, 90 {"cgroup", required_argument, NULL, 'c' }, 91 {"rate", required_argument, NULL, 'r' }, 92 {"verbose", no_argument, NULL, 'v' }, 93 {"iov_count", required_argument, NULL, 'i' }, 94 {"length", required_argument, NULL, 'l' }, 95 {"test", required_argument, NULL, 't' }, 96 {"data_test", no_argument, NULL, 'd' }, 97 {"txmsg", no_argument, &txmsg_pass, 1 }, 98 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 }, 99 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 100 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1}, 101 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 102 {"txmsg_apply", required_argument, NULL, 'a'}, 103 {"txmsg_cork", required_argument, NULL, 'k'}, 104 {"txmsg_start", required_argument, NULL, 's'}, 105 {"txmsg_end", required_argument, NULL, 'e'}, 106 {"txmsg_start_push", required_argument, NULL, 'p'}, 107 {"txmsg_end_push", required_argument, NULL, 'q'}, 108 {"txmsg_start_pop", required_argument, NULL, 'w'}, 109 {"txmsg_pop", required_argument, NULL, 'x'}, 110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 111 {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 112 {"ktls", no_argument, &ktls, 1 }, 113 {"peek", no_argument, &peek_flag, 1 }, 114 {0, 0, NULL, 0 } 115 }; 116 117 static void usage(char *argv[]) 118 { 119 int i; 120 121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 122 printf(" options:\n"); 123 for (i = 0; long_options[i].name != 0; i++) { 124 printf(" --%-12s", long_options[i].name); 125 if (long_options[i].flag != NULL) 126 printf(" flag (internal value:%d)\n", 127 *long_options[i].flag); 128 else 129 printf(" -%c\n", long_options[i].val); 130 } 131 printf("\n"); 132 } 133 134 char *sock_to_string(int s) 135 { 136 if (s == c1) 137 return "client1"; 138 else if (s == c2) 139 return "client2"; 140 else if (s == s1) 141 return "server1"; 142 else if (s == s2) 143 return "server2"; 144 else if (s == p1) 145 return "peer1"; 146 else if (s == p2) 147 return "peer2"; 148 else 149 return "unknown"; 150 } 151 152 static int sockmap_init_ktls(int verbose, int s) 153 { 154 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 155 .info = { 156 .version = TLS_1_2_VERSION, 157 .cipher_type = TLS_CIPHER_AES_GCM_128, 158 }, 159 }; 160 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 161 .info = { 162 .version = TLS_1_2_VERSION, 163 .cipher_type = TLS_CIPHER_AES_GCM_128, 164 }, 165 }; 166 int so_buf = 6553500; 167 int err; 168 169 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 170 if (err) { 171 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 172 return -EINVAL; 173 } 174 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 175 if (err) { 176 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 177 return -EINVAL; 178 } 179 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 180 if (err) { 181 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 182 return -EINVAL; 183 } 184 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 185 if (err) { 186 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 187 return -EINVAL; 188 } 189 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 190 if (err) { 191 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 192 return -EINVAL; 193 } 194 195 if (verbose) 196 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 197 return 0; 198 } 199 static int sockmap_init_sockets(int verbose) 200 { 201 int i, err, one = 1; 202 struct sockaddr_in addr; 203 int *fds[4] = {&s1, &s2, &c1, &c2}; 204 205 s1 = s2 = p1 = p2 = c1 = c2 = 0; 206 207 /* Init sockets */ 208 for (i = 0; i < 4; i++) { 209 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 210 if (*fds[i] < 0) { 211 perror("socket s1 failed()"); 212 return errno; 213 } 214 } 215 216 /* Allow reuse */ 217 for (i = 0; i < 2; i++) { 218 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 219 (char *)&one, sizeof(one)); 220 if (err) { 221 perror("setsockopt failed()"); 222 return errno; 223 } 224 } 225 226 /* Non-blocking sockets */ 227 for (i = 0; i < 2; i++) { 228 err = ioctl(*fds[i], FIONBIO, (char *)&one); 229 if (err < 0) { 230 perror("ioctl s1 failed()"); 231 return errno; 232 } 233 } 234 235 /* Bind server sockets */ 236 memset(&addr, 0, sizeof(struct sockaddr_in)); 237 addr.sin_family = AF_INET; 238 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 239 240 addr.sin_port = htons(S1_PORT); 241 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 242 if (err < 0) { 243 perror("bind s1 failed()"); 244 return errno; 245 } 246 247 addr.sin_port = htons(S2_PORT); 248 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 249 if (err < 0) { 250 perror("bind s2 failed()"); 251 return errno; 252 } 253 254 /* Listen server sockets */ 255 addr.sin_port = htons(S1_PORT); 256 err = listen(s1, 32); 257 if (err < 0) { 258 perror("listen s1 failed()"); 259 return errno; 260 } 261 262 addr.sin_port = htons(S2_PORT); 263 err = listen(s2, 32); 264 if (err < 0) { 265 perror("listen s1 failed()"); 266 return errno; 267 } 268 269 /* Initiate Connect */ 270 addr.sin_port = htons(S1_PORT); 271 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 272 if (err < 0 && errno != EINPROGRESS) { 273 perror("connect c1 failed()"); 274 return errno; 275 } 276 277 addr.sin_port = htons(S2_PORT); 278 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 279 if (err < 0 && errno != EINPROGRESS) { 280 perror("connect c2 failed()"); 281 return errno; 282 } else if (err < 0) { 283 err = 0; 284 } 285 286 /* Accept Connecrtions */ 287 p1 = accept(s1, NULL, NULL); 288 if (p1 < 0) { 289 perror("accept s1 failed()"); 290 return errno; 291 } 292 293 p2 = accept(s2, NULL, NULL); 294 if (p2 < 0) { 295 perror("accept s1 failed()"); 296 return errno; 297 } 298 299 if (verbose) { 300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 302 c1, s1, c2, s2); 303 } 304 return 0; 305 } 306 307 struct msg_stats { 308 size_t bytes_sent; 309 size_t bytes_recvd; 310 struct timespec start; 311 struct timespec end; 312 }; 313 314 struct sockmap_options { 315 int verbose; 316 bool base; 317 bool sendpage; 318 bool data_test; 319 bool drop_expected; 320 int iov_count; 321 int iov_length; 322 int rate; 323 }; 324 325 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 326 struct msg_stats *s, 327 struct sockmap_options *opt) 328 { 329 bool drop = opt->drop_expected; 330 unsigned char k = 0; 331 FILE *file; 332 int i, fp; 333 334 file = tmpfile(); 335 if (!file) { 336 perror("create file for sendpage"); 337 return 1; 338 } 339 for (i = 0; i < iov_length * cnt; i++, k++) 340 fwrite(&k, sizeof(char), 1, file); 341 fflush(file); 342 fseek(file, 0, SEEK_SET); 343 344 fp = fileno(file); 345 346 clock_gettime(CLOCK_MONOTONIC, &s->start); 347 for (i = 0; i < cnt; i++) { 348 int sent = sendfile(fd, fp, NULL, iov_length); 349 350 if (!drop && sent < 0) { 351 perror("send loop error"); 352 fclose(file); 353 return sent; 354 } else if (drop && sent >= 0) { 355 printf("sendpage loop error expected: %i\n", sent); 356 fclose(file); 357 return -EIO; 358 } 359 360 if (sent > 0) 361 s->bytes_sent += sent; 362 } 363 clock_gettime(CLOCK_MONOTONIC, &s->end); 364 fclose(file); 365 return 0; 366 } 367 368 static void msg_free_iov(struct msghdr *msg) 369 { 370 int i; 371 372 for (i = 0; i < msg->msg_iovlen; i++) 373 free(msg->msg_iov[i].iov_base); 374 free(msg->msg_iov); 375 msg->msg_iov = NULL; 376 msg->msg_iovlen = 0; 377 } 378 379 static int msg_alloc_iov(struct msghdr *msg, 380 int iov_count, int iov_length, 381 bool data, bool xmit) 382 { 383 unsigned char k = 0; 384 struct iovec *iov; 385 int i; 386 387 iov = calloc(iov_count, sizeof(struct iovec)); 388 if (!iov) 389 return errno; 390 391 for (i = 0; i < iov_count; i++) { 392 unsigned char *d = calloc(iov_length, sizeof(char)); 393 394 if (!d) { 395 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 396 goto unwind_iov; 397 } 398 iov[i].iov_base = d; 399 iov[i].iov_len = iov_length; 400 401 if (data && xmit) { 402 int j; 403 404 for (j = 0; j < iov_length; j++) 405 d[j] = k++; 406 } 407 } 408 409 msg->msg_iov = iov; 410 msg->msg_iovlen = iov_count; 411 412 return 0; 413 unwind_iov: 414 for (i--; i >= 0 ; i--) 415 free(msg->msg_iov[i].iov_base); 416 return -ENOMEM; 417 } 418 419 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 420 { 421 int i, j, bytes_cnt = 0; 422 unsigned char k = 0; 423 424 for (i = 0; i < msg->msg_iovlen; i++) { 425 unsigned char *d = msg->msg_iov[i].iov_base; 426 427 for (j = 0; 428 j < msg->msg_iov[i].iov_len && size; j++) { 429 if (d[j] != k++) { 430 fprintf(stderr, 431 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 432 i, j, d[j], k - 1, d[j+1], k); 433 return -EIO; 434 } 435 bytes_cnt++; 436 if (bytes_cnt == chunk_sz) { 437 k = 0; 438 bytes_cnt = 0; 439 } 440 size--; 441 } 442 } 443 return 0; 444 } 445 446 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 447 struct msg_stats *s, bool tx, 448 struct sockmap_options *opt) 449 { 450 struct msghdr msg = {0}, msg_peek = {0}; 451 int err, i, flags = MSG_NOSIGNAL; 452 bool drop = opt->drop_expected; 453 bool data = opt->data_test; 454 455 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx); 456 if (err) 457 goto out_errno; 458 if (peek_flag) { 459 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 460 if (err) 461 goto out_errno; 462 } 463 464 if (tx) { 465 clock_gettime(CLOCK_MONOTONIC, &s->start); 466 for (i = 0; i < cnt; i++) { 467 int sent = sendmsg(fd, &msg, flags); 468 469 if (!drop && sent < 0) { 470 perror("send loop error"); 471 goto out_errno; 472 } else if (drop && sent >= 0) { 473 printf("send loop error expected: %i\n", sent); 474 errno = -EIO; 475 goto out_errno; 476 } 477 if (sent > 0) 478 s->bytes_sent += sent; 479 } 480 clock_gettime(CLOCK_MONOTONIC, &s->end); 481 } else { 482 int slct, recvp = 0, recv, max_fd = fd; 483 float total_bytes, txmsg_pop_total; 484 int fd_flags = O_NONBLOCK; 485 struct timeval timeout; 486 fd_set w; 487 488 fcntl(fd, fd_flags); 489 /* Account for pop bytes noting each iteration of apply will 490 * call msg_pop_data helper so we need to account for this 491 * by calculating the number of apply iterations. Note user 492 * of the tool can create cases where no data is sent by 493 * manipulating pop/push/pull/etc. For example txmsg_apply 1 494 * with txmsg_pop 1 will try to apply 1B at a time but each 495 * iteration will then pop 1B so no data will ever be sent. 496 * This is really only useful for testing edge cases in code 497 * paths. 498 */ 499 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 500 txmsg_pop_total = txmsg_pop; 501 if (txmsg_apply) 502 txmsg_pop_total *= (total_bytes / txmsg_apply); 503 total_bytes -= txmsg_pop_total; 504 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 505 if (err < 0) 506 perror("recv start time"); 507 while (s->bytes_recvd < total_bytes) { 508 if (txmsg_cork) { 509 timeout.tv_sec = 0; 510 timeout.tv_usec = 300000; 511 } else { 512 timeout.tv_sec = 3; 513 timeout.tv_usec = 0; 514 } 515 516 /* FD sets */ 517 FD_ZERO(&w); 518 FD_SET(fd, &w); 519 520 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 521 if (slct == -1) { 522 perror("select()"); 523 clock_gettime(CLOCK_MONOTONIC, &s->end); 524 goto out_errno; 525 } else if (!slct) { 526 if (opt->verbose) 527 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 528 errno = -EIO; 529 clock_gettime(CLOCK_MONOTONIC, &s->end); 530 goto out_errno; 531 } 532 533 errno = 0; 534 if (peek_flag) { 535 flags |= MSG_PEEK; 536 recvp = recvmsg(fd, &msg_peek, flags); 537 if (recvp < 0) { 538 if (errno != EWOULDBLOCK) { 539 clock_gettime(CLOCK_MONOTONIC, &s->end); 540 goto out_errno; 541 } 542 } 543 flags = 0; 544 } 545 546 recv = recvmsg(fd, &msg, flags); 547 if (recv < 0) { 548 if (errno != EWOULDBLOCK) { 549 clock_gettime(CLOCK_MONOTONIC, &s->end); 550 perror("recv failed()"); 551 goto out_errno; 552 } 553 } 554 555 s->bytes_recvd += recv; 556 557 if (data) { 558 int chunk_sz = opt->sendpage ? 559 iov_length * cnt : 560 iov_length * iov_count; 561 562 errno = msg_verify_data(&msg, recv, chunk_sz); 563 if (errno) { 564 perror("data verify msg failed"); 565 goto out_errno; 566 } 567 if (recvp) { 568 errno = msg_verify_data(&msg_peek, 569 recvp, 570 chunk_sz); 571 if (errno) { 572 perror("data verify msg_peek failed"); 573 goto out_errno; 574 } 575 } 576 } 577 } 578 clock_gettime(CLOCK_MONOTONIC, &s->end); 579 } 580 581 msg_free_iov(&msg); 582 msg_free_iov(&msg_peek); 583 return err; 584 out_errno: 585 msg_free_iov(&msg); 586 msg_free_iov(&msg_peek); 587 return errno; 588 } 589 590 static float giga = 1000000000; 591 592 static inline float sentBps(struct msg_stats s) 593 { 594 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 595 } 596 597 static inline float recvdBps(struct msg_stats s) 598 { 599 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 600 } 601 602 static int sendmsg_test(struct sockmap_options *opt) 603 { 604 float sent_Bps = 0, recvd_Bps = 0; 605 int rx_fd, txpid, rxpid, err = 0; 606 struct msg_stats s = {0}; 607 int iov_count = opt->iov_count; 608 int iov_buf = opt->iov_length; 609 int rx_status, tx_status; 610 int cnt = opt->rate; 611 612 errno = 0; 613 614 if (opt->base) 615 rx_fd = p1; 616 else 617 rx_fd = p2; 618 619 if (ktls) { 620 /* Redirecting into non-TLS socket which sends into a TLS 621 * socket is not a valid test. So in this case lets not 622 * enable kTLS but still run the test. 623 */ 624 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 625 err = sockmap_init_ktls(opt->verbose, rx_fd); 626 if (err) 627 return err; 628 } 629 err = sockmap_init_ktls(opt->verbose, c1); 630 if (err) 631 return err; 632 } 633 634 rxpid = fork(); 635 if (rxpid == 0) { 636 if (opt->drop_expected) 637 exit(0); 638 639 if (opt->sendpage) 640 iov_count = 1; 641 err = msg_loop(rx_fd, iov_count, iov_buf, 642 cnt, &s, false, opt); 643 if (opt->verbose) 644 fprintf(stderr, 645 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 646 iov_count, iov_buf, cnt, err); 647 if (s.end.tv_sec - s.start.tv_sec) { 648 sent_Bps = sentBps(s); 649 recvd_Bps = recvdBps(s); 650 } 651 if (opt->verbose) 652 fprintf(stdout, 653 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 654 s.bytes_sent, sent_Bps, sent_Bps/giga, 655 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 656 peek_flag ? "(peek_msg)" : ""); 657 if (err && txmsg_cork) 658 err = 0; 659 exit(err ? 1 : 0); 660 } else if (rxpid == -1) { 661 perror("msg_loop_rx"); 662 return errno; 663 } 664 665 txpid = fork(); 666 if (txpid == 0) { 667 if (opt->sendpage) 668 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 669 else 670 err = msg_loop(c1, iov_count, iov_buf, 671 cnt, &s, true, opt); 672 673 if (err) 674 fprintf(stderr, 675 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 676 iov_count, iov_buf, cnt, err); 677 if (s.end.tv_sec - s.start.tv_sec) { 678 sent_Bps = sentBps(s); 679 recvd_Bps = recvdBps(s); 680 } 681 if (opt->verbose) 682 fprintf(stdout, 683 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 684 s.bytes_sent, sent_Bps, sent_Bps/giga, 685 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 686 exit(err ? 1 : 0); 687 } else if (txpid == -1) { 688 perror("msg_loop_tx"); 689 return errno; 690 } 691 692 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 693 assert(waitpid(txpid, &tx_status, 0) == txpid); 694 if (WIFEXITED(rx_status)) { 695 err = WEXITSTATUS(rx_status); 696 if (err) { 697 fprintf(stderr, "rx thread exited with err %d. ", err); 698 goto out; 699 } 700 } 701 if (WIFEXITED(tx_status)) { 702 err = WEXITSTATUS(tx_status); 703 if (err) 704 fprintf(stderr, "tx thread exited with err %d. ", err); 705 } 706 out: 707 return err; 708 } 709 710 static int forever_ping_pong(int rate, struct sockmap_options *opt) 711 { 712 struct timeval timeout; 713 char buf[1024] = {0}; 714 int sc; 715 716 timeout.tv_sec = 10; 717 timeout.tv_usec = 0; 718 719 /* Ping/Pong data from client to server */ 720 sc = send(c1, buf, sizeof(buf), 0); 721 if (sc < 0) { 722 perror("send failed()"); 723 return sc; 724 } 725 726 do { 727 int s, rc, i, max_fd = p2; 728 fd_set w; 729 730 /* FD sets */ 731 FD_ZERO(&w); 732 FD_SET(c1, &w); 733 FD_SET(c2, &w); 734 FD_SET(p1, &w); 735 FD_SET(p2, &w); 736 737 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 738 if (s == -1) { 739 perror("select()"); 740 break; 741 } else if (!s) { 742 fprintf(stderr, "unexpected timeout\n"); 743 break; 744 } 745 746 for (i = 0; i <= max_fd && s > 0; ++i) { 747 if (!FD_ISSET(i, &w)) 748 continue; 749 750 s--; 751 752 rc = recv(i, buf, sizeof(buf), 0); 753 if (rc < 0) { 754 if (errno != EWOULDBLOCK) { 755 perror("recv failed()"); 756 return rc; 757 } 758 } 759 760 if (rc == 0) { 761 close(i); 762 break; 763 } 764 765 sc = send(i, buf, rc, 0); 766 if (sc < 0) { 767 perror("send failed()"); 768 return sc; 769 } 770 } 771 772 if (rate) 773 sleep(rate); 774 775 if (opt->verbose) { 776 printf("."); 777 fflush(stdout); 778 779 } 780 } while (running); 781 782 return 0; 783 } 784 785 enum { 786 PING_PONG, 787 SENDMSG, 788 BASE, 789 BASE_SENDPAGE, 790 SENDPAGE, 791 }; 792 793 static int run_options(struct sockmap_options *options, int cg_fd, int test) 794 { 795 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 796 797 /* If base test skip BPF setup */ 798 if (test == BASE || test == BASE_SENDPAGE) 799 goto run; 800 801 /* Attach programs to sockmap */ 802 err = bpf_prog_attach(prog_fd[0], map_fd[0], 803 BPF_SK_SKB_STREAM_PARSER, 0); 804 if (err) { 805 fprintf(stderr, 806 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 807 prog_fd[0], map_fd[0], err, strerror(errno)); 808 return err; 809 } 810 811 err = bpf_prog_attach(prog_fd[1], map_fd[0], 812 BPF_SK_SKB_STREAM_VERDICT, 0); 813 if (err) { 814 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 815 err, strerror(errno)); 816 return err; 817 } 818 819 /* Attach to cgroups */ 820 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 821 if (err) { 822 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 823 err, strerror(errno)); 824 return err; 825 } 826 827 run: 828 err = sockmap_init_sockets(options->verbose); 829 if (err) { 830 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 831 goto out; 832 } 833 834 /* Attach txmsg program to sockmap */ 835 if (txmsg_pass) 836 tx_prog_fd = prog_fd[3]; 837 else if (txmsg_noisy) 838 tx_prog_fd = prog_fd[4]; 839 else if (txmsg_redir) 840 tx_prog_fd = prog_fd[5]; 841 else if (txmsg_redir_noisy) 842 tx_prog_fd = prog_fd[6]; 843 else if (txmsg_drop) 844 tx_prog_fd = prog_fd[9]; 845 /* apply and cork must be last */ 846 else if (txmsg_apply) 847 tx_prog_fd = prog_fd[7]; 848 else if (txmsg_cork) 849 tx_prog_fd = prog_fd[8]; 850 else 851 tx_prog_fd = 0; 852 853 if (tx_prog_fd) { 854 int redir_fd, i = 0; 855 856 err = bpf_prog_attach(tx_prog_fd, 857 map_fd[1], BPF_SK_MSG_VERDICT, 0); 858 if (err) { 859 fprintf(stderr, 860 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 861 err, strerror(errno)); 862 goto out; 863 } 864 865 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 866 if (err) { 867 fprintf(stderr, 868 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 869 err, strerror(errno)); 870 goto out; 871 } 872 873 if (txmsg_redir || txmsg_redir_noisy) 874 redir_fd = c2; 875 else 876 redir_fd = c1; 877 878 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 879 if (err) { 880 fprintf(stderr, 881 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 882 err, strerror(errno)); 883 goto out; 884 } 885 886 if (txmsg_apply) { 887 err = bpf_map_update_elem(map_fd[3], 888 &i, &txmsg_apply, BPF_ANY); 889 if (err) { 890 fprintf(stderr, 891 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 892 err, strerror(errno)); 893 goto out; 894 } 895 } 896 897 if (txmsg_cork) { 898 err = bpf_map_update_elem(map_fd[4], 899 &i, &txmsg_cork, BPF_ANY); 900 if (err) { 901 fprintf(stderr, 902 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 903 err, strerror(errno)); 904 goto out; 905 } 906 } 907 908 if (txmsg_start) { 909 err = bpf_map_update_elem(map_fd[5], 910 &i, &txmsg_start, BPF_ANY); 911 if (err) { 912 fprintf(stderr, 913 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 914 err, strerror(errno)); 915 goto out; 916 } 917 } 918 919 if (txmsg_end) { 920 i = 1; 921 err = bpf_map_update_elem(map_fd[5], 922 &i, &txmsg_end, BPF_ANY); 923 if (err) { 924 fprintf(stderr, 925 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 926 err, strerror(errno)); 927 goto out; 928 } 929 } 930 931 if (txmsg_start_push) { 932 i = 2; 933 err = bpf_map_update_elem(map_fd[5], 934 &i, &txmsg_start_push, BPF_ANY); 935 if (err) { 936 fprintf(stderr, 937 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 938 err, strerror(errno)); 939 goto out; 940 } 941 } 942 943 if (txmsg_end_push) { 944 i = 3; 945 err = bpf_map_update_elem(map_fd[5], 946 &i, &txmsg_end_push, BPF_ANY); 947 if (err) { 948 fprintf(stderr, 949 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 950 txmsg_end_push, i, err, strerror(errno)); 951 goto out; 952 } 953 } 954 955 if (txmsg_start_pop) { 956 i = 4; 957 err = bpf_map_update_elem(map_fd[5], 958 &i, &txmsg_start_pop, BPF_ANY); 959 if (err) { 960 fprintf(stderr, 961 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 962 txmsg_start_pop, i, err, strerror(errno)); 963 goto out; 964 } 965 } else { 966 i = 4; 967 bpf_map_update_elem(map_fd[5], 968 &i, &txmsg_start_pop, BPF_ANY); 969 } 970 971 if (txmsg_pop) { 972 i = 5; 973 err = bpf_map_update_elem(map_fd[5], 974 &i, &txmsg_pop, BPF_ANY); 975 if (err) { 976 fprintf(stderr, 977 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 978 txmsg_pop, i, err, strerror(errno)); 979 goto out; 980 } 981 } else { 982 i = 5; 983 bpf_map_update_elem(map_fd[5], 984 &i, &txmsg_pop, BPF_ANY); 985 986 } 987 988 if (txmsg_ingress) { 989 int in = BPF_F_INGRESS; 990 991 i = 0; 992 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 993 if (err) { 994 fprintf(stderr, 995 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 996 err, strerror(errno)); 997 } 998 i = 1; 999 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 1000 if (err) { 1001 fprintf(stderr, 1002 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 1003 err, strerror(errno)); 1004 } 1005 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1006 if (err) { 1007 fprintf(stderr, 1008 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1009 err, strerror(errno)); 1010 } 1011 1012 i = 2; 1013 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1014 if (err) { 1015 fprintf(stderr, 1016 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1017 err, strerror(errno)); 1018 } 1019 } 1020 1021 if (txmsg_skb) { 1022 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1023 p2 : p1; 1024 int ingress = BPF_F_INGRESS; 1025 1026 i = 0; 1027 err = bpf_map_update_elem(map_fd[7], 1028 &i, &ingress, BPF_ANY); 1029 if (err) { 1030 fprintf(stderr, 1031 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1032 err, strerror(errno)); 1033 } 1034 1035 i = 3; 1036 err = bpf_map_update_elem(map_fd[0], 1037 &i, &skb_fd, BPF_ANY); 1038 if (err) { 1039 fprintf(stderr, 1040 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1041 err, strerror(errno)); 1042 } 1043 } 1044 } 1045 1046 if (txmsg_drop) 1047 options->drop_expected = true; 1048 1049 if (test == PING_PONG) 1050 err = forever_ping_pong(options->rate, options); 1051 else if (test == SENDMSG) { 1052 options->base = false; 1053 options->sendpage = false; 1054 err = sendmsg_test(options); 1055 } else if (test == SENDPAGE) { 1056 options->base = false; 1057 options->sendpage = true; 1058 err = sendmsg_test(options); 1059 } else if (test == BASE) { 1060 options->base = true; 1061 options->sendpage = false; 1062 err = sendmsg_test(options); 1063 } else if (test == BASE_SENDPAGE) { 1064 options->base = true; 1065 options->sendpage = true; 1066 err = sendmsg_test(options); 1067 } else 1068 fprintf(stderr, "unknown test\n"); 1069 out: 1070 /* Detatch and zero all the maps */ 1071 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 1072 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1073 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1074 if (tx_prog_fd >= 0) 1075 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1076 1077 for (i = 0; i < 8; i++) { 1078 key = next_key = 0; 1079 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1080 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1081 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1082 key = next_key; 1083 } 1084 } 1085 1086 close(s1); 1087 close(s2); 1088 close(p1); 1089 close(p2); 1090 close(c1); 1091 close(c2); 1092 return err; 1093 } 1094 1095 static char *test_to_str(int test) 1096 { 1097 switch (test) { 1098 case SENDMSG: 1099 return "sendmsg"; 1100 case SENDPAGE: 1101 return "sendpage"; 1102 } 1103 return "unknown"; 1104 } 1105 1106 #define OPTSTRING 60 1107 static void test_options(char *options) 1108 { 1109 char tstr[OPTSTRING]; 1110 1111 memset(options, 0, OPTSTRING); 1112 1113 if (txmsg_pass) 1114 strncat(options, "pass,", OPTSTRING); 1115 if (txmsg_noisy) 1116 strncat(options, "pass_noisy,", OPTSTRING); 1117 if (txmsg_redir) 1118 strncat(options, "redir,", OPTSTRING); 1119 if (txmsg_redir_noisy) 1120 strncat(options, "redir_noisy,", OPTSTRING); 1121 if (txmsg_drop) 1122 strncat(options, "drop,", OPTSTRING); 1123 if (txmsg_apply) { 1124 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1125 strncat(options, tstr, OPTSTRING); 1126 } 1127 if (txmsg_cork) { 1128 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1129 strncat(options, tstr, OPTSTRING); 1130 } 1131 if (txmsg_start) { 1132 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1133 strncat(options, tstr, OPTSTRING); 1134 } 1135 if (txmsg_end) { 1136 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1137 strncat(options, tstr, OPTSTRING); 1138 } 1139 if (txmsg_start_pop) { 1140 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1141 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1142 strncat(options, tstr, OPTSTRING); 1143 } 1144 if (txmsg_ingress) 1145 strncat(options, "ingress,", OPTSTRING); 1146 if (txmsg_skb) 1147 strncat(options, "skb,", OPTSTRING); 1148 if (ktls) 1149 strncat(options, "ktls,", OPTSTRING); 1150 if (peek_flag) 1151 strncat(options, "peek,", OPTSTRING); 1152 } 1153 1154 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1155 { 1156 char *options = calloc(OPTSTRING, sizeof(char)); 1157 int err; 1158 1159 if (test == SENDPAGE) 1160 opt->sendpage = true; 1161 else 1162 opt->sendpage = false; 1163 1164 if (txmsg_drop) 1165 opt->drop_expected = true; 1166 else 1167 opt->drop_expected = false; 1168 1169 test_options(options); 1170 1171 fprintf(stdout, 1172 "[TEST %i]: (%i, %i, %i, %s, %s): ", 1173 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1174 test_to_str(test), options); 1175 fflush(stdout); 1176 err = run_options(opt, cgrp, test); 1177 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 1178 test_cnt++; 1179 !err ? passed++ : failed++; 1180 free(options); 1181 return err; 1182 } 1183 1184 static int test_exec(int cgrp, struct sockmap_options *opt) 1185 { 1186 int err = __test_exec(cgrp, SENDMSG, opt); 1187 1188 if (err) 1189 goto out; 1190 1191 err = __test_exec(cgrp, SENDPAGE, opt); 1192 out: 1193 return err; 1194 } 1195 1196 static int test_loop(int cgrp) 1197 { 1198 struct sockmap_options opt; 1199 1200 int err, i, l, r; 1201 1202 opt.verbose = 0; 1203 opt.base = false; 1204 opt.sendpage = false; 1205 opt.data_test = false; 1206 opt.drop_expected = false; 1207 opt.iov_count = 0; 1208 opt.iov_length = 0; 1209 opt.rate = 0; 1210 1211 r = 1; 1212 for (i = 1; i < 100; i += 33) { 1213 for (l = 1; l < 100; l += 33) { 1214 opt.rate = r; 1215 opt.iov_count = i; 1216 opt.iov_length = l; 1217 err = test_exec(cgrp, &opt); 1218 if (err) 1219 goto out; 1220 } 1221 } 1222 sched_yield(); 1223 out: 1224 return err; 1225 } 1226 1227 static int test_txmsg(int cgrp) 1228 { 1229 int err; 1230 1231 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1232 txmsg_apply = txmsg_cork = 0; 1233 txmsg_ingress = txmsg_skb = 0; 1234 1235 txmsg_pass = 1; 1236 err = test_loop(cgrp); 1237 txmsg_pass = 0; 1238 if (err) 1239 goto out; 1240 1241 txmsg_redir = 1; 1242 err = test_loop(cgrp); 1243 txmsg_redir = 0; 1244 if (err) 1245 goto out; 1246 1247 txmsg_drop = 1; 1248 err = test_loop(cgrp); 1249 txmsg_drop = 0; 1250 if (err) 1251 goto out; 1252 1253 txmsg_redir = 1; 1254 txmsg_ingress = 1; 1255 err = test_loop(cgrp); 1256 txmsg_redir = 0; 1257 txmsg_ingress = 0; 1258 if (err) 1259 goto out; 1260 out: 1261 txmsg_pass = 0; 1262 txmsg_redir = 0; 1263 txmsg_drop = 0; 1264 return err; 1265 } 1266 1267 static int test_send(struct sockmap_options *opt, int cgrp) 1268 { 1269 int err; 1270 1271 opt->iov_length = 1; 1272 opt->iov_count = 1; 1273 opt->rate = 1; 1274 err = test_exec(cgrp, opt); 1275 if (err) 1276 goto out; 1277 1278 opt->iov_length = 1; 1279 opt->iov_count = 1024; 1280 opt->rate = 1; 1281 err = test_exec(cgrp, opt); 1282 if (err) 1283 goto out; 1284 1285 opt->iov_length = 1024; 1286 opt->iov_count = 1; 1287 opt->rate = 1; 1288 err = test_exec(cgrp, opt); 1289 if (err) 1290 goto out; 1291 1292 opt->iov_length = 1; 1293 opt->iov_count = 1; 1294 opt->rate = 512; 1295 err = test_exec(cgrp, opt); 1296 if (err) 1297 goto out; 1298 1299 opt->iov_length = 256; 1300 opt->iov_count = 1024; 1301 opt->rate = 2; 1302 err = test_exec(cgrp, opt); 1303 if (err) 1304 goto out; 1305 1306 opt->rate = 100; 1307 opt->iov_count = 1; 1308 opt->iov_length = 5; 1309 err = test_exec(cgrp, opt); 1310 if (err) 1311 goto out; 1312 out: 1313 sched_yield(); 1314 return err; 1315 } 1316 1317 static int test_mixed(int cgrp) 1318 { 1319 struct sockmap_options opt = {0}; 1320 int err; 1321 1322 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1323 txmsg_apply = txmsg_cork = 0; 1324 txmsg_start = txmsg_end = 0; 1325 txmsg_start_push = txmsg_end_push = 0; 1326 txmsg_start_pop = txmsg_pop = 0; 1327 1328 /* Test small and large iov_count values with pass/redir/apply/cork */ 1329 txmsg_pass = 1; 1330 txmsg_redir = 0; 1331 txmsg_apply = 1; 1332 txmsg_cork = 0; 1333 err = test_send(&opt, cgrp); 1334 if (err) 1335 goto out; 1336 1337 txmsg_pass = 1; 1338 txmsg_redir = 0; 1339 txmsg_apply = 0; 1340 txmsg_cork = 1; 1341 err = test_send(&opt, cgrp); 1342 if (err) 1343 goto out; 1344 1345 txmsg_pass = 1; 1346 txmsg_redir = 0; 1347 txmsg_apply = 1; 1348 txmsg_cork = 1; 1349 err = test_send(&opt, cgrp); 1350 if (err) 1351 goto out; 1352 1353 txmsg_pass = 1; 1354 txmsg_redir = 0; 1355 txmsg_apply = 1024; 1356 txmsg_cork = 0; 1357 err = test_send(&opt, cgrp); 1358 if (err) 1359 goto out; 1360 1361 txmsg_pass = 1; 1362 txmsg_redir = 0; 1363 txmsg_apply = 0; 1364 txmsg_cork = 1024; 1365 err = test_send(&opt, cgrp); 1366 if (err) 1367 goto out; 1368 1369 txmsg_pass = 1; 1370 txmsg_redir = 0; 1371 txmsg_apply = 1024; 1372 txmsg_cork = 1024; 1373 err = test_send(&opt, cgrp); 1374 if (err) 1375 goto out; 1376 1377 txmsg_pass = 1; 1378 txmsg_redir = 0; 1379 txmsg_cork = 4096; 1380 txmsg_apply = 4096; 1381 err = test_send(&opt, cgrp); 1382 if (err) 1383 goto out; 1384 1385 txmsg_pass = 0; 1386 txmsg_redir = 1; 1387 txmsg_apply = 1; 1388 txmsg_cork = 0; 1389 err = test_send(&opt, cgrp); 1390 if (err) 1391 goto out; 1392 1393 txmsg_pass = 0; 1394 txmsg_redir = 1; 1395 txmsg_apply = 0; 1396 txmsg_cork = 1; 1397 err = test_send(&opt, cgrp); 1398 if (err) 1399 goto out; 1400 1401 txmsg_pass = 0; 1402 txmsg_redir = 1; 1403 txmsg_apply = 1024; 1404 txmsg_cork = 0; 1405 err = test_send(&opt, cgrp); 1406 if (err) 1407 goto out; 1408 1409 txmsg_pass = 0; 1410 txmsg_redir = 1; 1411 txmsg_apply = 0; 1412 txmsg_cork = 1024; 1413 err = test_send(&opt, cgrp); 1414 if (err) 1415 goto out; 1416 1417 txmsg_pass = 0; 1418 txmsg_redir = 1; 1419 txmsg_apply = 1024; 1420 txmsg_cork = 1024; 1421 err = test_send(&opt, cgrp); 1422 if (err) 1423 goto out; 1424 1425 txmsg_pass = 0; 1426 txmsg_redir = 1; 1427 txmsg_cork = 4096; 1428 txmsg_apply = 4096; 1429 err = test_send(&opt, cgrp); 1430 if (err) 1431 goto out; 1432 out: 1433 return err; 1434 } 1435 1436 static int test_start_end(int cgrp) 1437 { 1438 struct sockmap_options opt = {0}; 1439 int err, i; 1440 1441 /* Test basic start/end with lots of iov_count and iov_lengths */ 1442 txmsg_start = 1; 1443 txmsg_end = 2; 1444 txmsg_start_push = 1; 1445 txmsg_end_push = 2; 1446 txmsg_start_pop = 1; 1447 txmsg_pop = 1; 1448 err = test_txmsg(cgrp); 1449 if (err) 1450 goto out; 1451 1452 /* Cut a byte of pushed data but leave reamining in place */ 1453 txmsg_start = 1; 1454 txmsg_end = 2; 1455 txmsg_start_push = 1; 1456 txmsg_end_push = 3; 1457 txmsg_start_pop = 1; 1458 txmsg_pop = 1; 1459 err = test_txmsg(cgrp); 1460 if (err) 1461 goto out; 1462 1463 /* Test start/end with cork */ 1464 opt.rate = 16; 1465 opt.iov_count = 1; 1466 opt.iov_length = 100; 1467 txmsg_cork = 1600; 1468 1469 txmsg_start_pop = 0; 1470 txmsg_pop = 0; 1471 1472 for (i = 99; i <= 1600; i += 500) { 1473 txmsg_start = 0; 1474 txmsg_end = i; 1475 txmsg_start_push = 0; 1476 txmsg_end_push = i; 1477 err = test_exec(cgrp, &opt); 1478 if (err) 1479 goto out; 1480 } 1481 1482 /* Test pop data in middle of cork */ 1483 for (i = 99; i <= 1600; i += 500) { 1484 txmsg_start_pop = 10; 1485 txmsg_pop = i; 1486 err = test_exec(cgrp, &opt); 1487 if (err) 1488 goto out; 1489 } 1490 txmsg_start_pop = 0; 1491 txmsg_pop = 0; 1492 1493 /* Test start/end with cork but pull data in middle */ 1494 for (i = 199; i <= 1600; i += 500) { 1495 txmsg_start = 100; 1496 txmsg_end = i; 1497 txmsg_start_push = 100; 1498 txmsg_end_push = i; 1499 err = test_exec(cgrp, &opt); 1500 if (err) 1501 goto out; 1502 } 1503 1504 /* Test start/end with cork pulling last sg entry */ 1505 txmsg_start = 1500; 1506 txmsg_end = 1600; 1507 txmsg_start_push = 1500; 1508 txmsg_end_push = 1600; 1509 err = test_exec(cgrp, &opt); 1510 if (err) 1511 goto out; 1512 1513 /* Test pop with cork pulling last sg entry */ 1514 txmsg_start_pop = 1500; 1515 txmsg_pop = 1600; 1516 err = test_exec(cgrp, &opt); 1517 if (err) 1518 goto out; 1519 txmsg_start_pop = 0; 1520 txmsg_pop = 0; 1521 1522 /* Test start/end pull of single byte in last page */ 1523 txmsg_start = 1111; 1524 txmsg_end = 1112; 1525 txmsg_start_push = 1111; 1526 txmsg_end_push = 1112; 1527 err = test_exec(cgrp, &opt); 1528 if (err) 1529 goto out; 1530 1531 /* Test pop of single byte in last page */ 1532 txmsg_start_pop = 1111; 1533 txmsg_pop = 1112; 1534 err = test_exec(cgrp, &opt); 1535 if (err) 1536 goto out; 1537 1538 /* Test start/end with end < start */ 1539 txmsg_start = 1111; 1540 txmsg_end = 0; 1541 txmsg_start_push = 1111; 1542 txmsg_end_push = 0; 1543 err = test_exec(cgrp, &opt); 1544 if (err) 1545 goto out; 1546 1547 /* Test start/end with end > data */ 1548 txmsg_start = 0; 1549 txmsg_end = 1601; 1550 txmsg_start_push = 0; 1551 txmsg_end_push = 1601; 1552 err = test_exec(cgrp, &opt); 1553 if (err) 1554 goto out; 1555 1556 /* Test start/end with start > data */ 1557 txmsg_start = 1601; 1558 txmsg_end = 1600; 1559 txmsg_start_push = 1601; 1560 txmsg_end_push = 1600; 1561 err = test_exec(cgrp, &opt); 1562 if (err) 1563 goto out; 1564 1565 /* Test pop with start > data */ 1566 txmsg_start_pop = 1601; 1567 txmsg_pop = 1; 1568 err = test_exec(cgrp, &opt); 1569 if (err) 1570 goto out; 1571 1572 /* Test pop with pop range > data */ 1573 txmsg_start_pop = 1599; 1574 txmsg_pop = 10; 1575 err = test_exec(cgrp, &opt); 1576 out: 1577 txmsg_start = 0; 1578 txmsg_end = 0; 1579 sched_yield(); 1580 return err; 1581 } 1582 1583 char *map_names[] = { 1584 "sock_map", 1585 "sock_map_txmsg", 1586 "sock_map_redir", 1587 "sock_apply_bytes", 1588 "sock_cork_bytes", 1589 "sock_bytes", 1590 "sock_redir_flags", 1591 "sock_skb_opts", 1592 }; 1593 1594 int prog_attach_type[] = { 1595 BPF_SK_SKB_STREAM_PARSER, 1596 BPF_SK_SKB_STREAM_VERDICT, 1597 BPF_CGROUP_SOCK_OPS, 1598 BPF_SK_MSG_VERDICT, 1599 BPF_SK_MSG_VERDICT, 1600 BPF_SK_MSG_VERDICT, 1601 BPF_SK_MSG_VERDICT, 1602 BPF_SK_MSG_VERDICT, 1603 BPF_SK_MSG_VERDICT, 1604 BPF_SK_MSG_VERDICT, 1605 }; 1606 1607 int prog_type[] = { 1608 BPF_PROG_TYPE_SK_SKB, 1609 BPF_PROG_TYPE_SK_SKB, 1610 BPF_PROG_TYPE_SOCK_OPS, 1611 BPF_PROG_TYPE_SK_MSG, 1612 BPF_PROG_TYPE_SK_MSG, 1613 BPF_PROG_TYPE_SK_MSG, 1614 BPF_PROG_TYPE_SK_MSG, 1615 BPF_PROG_TYPE_SK_MSG, 1616 BPF_PROG_TYPE_SK_MSG, 1617 BPF_PROG_TYPE_SK_MSG, 1618 }; 1619 1620 static int populate_progs(char *bpf_file) 1621 { 1622 struct bpf_program *prog; 1623 struct bpf_object *obj; 1624 int i = 0; 1625 long err; 1626 1627 obj = bpf_object__open(bpf_file); 1628 err = libbpf_get_error(obj); 1629 if (err) { 1630 char err_buf[256]; 1631 1632 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1633 printf("Unable to load eBPF objects in file '%s' : %s\n", 1634 bpf_file, err_buf); 1635 return -1; 1636 } 1637 1638 bpf_object__for_each_program(prog, obj) { 1639 bpf_program__set_type(prog, prog_type[i]); 1640 bpf_program__set_expected_attach_type(prog, 1641 prog_attach_type[i]); 1642 i++; 1643 } 1644 1645 i = bpf_object__load(obj); 1646 i = 0; 1647 bpf_object__for_each_program(prog, obj) { 1648 prog_fd[i] = bpf_program__fd(prog); 1649 i++; 1650 } 1651 1652 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 1653 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1654 map_fd[i] = bpf_map__fd(maps[i]); 1655 if (map_fd[i] < 0) { 1656 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1657 map_fd[i], strerror(errno)); 1658 return -1; 1659 } 1660 } 1661 1662 return 0; 1663 } 1664 1665 static int __test_suite(int cg_fd, char *bpf_file) 1666 { 1667 int err, cleanup = cg_fd; 1668 1669 err = populate_progs(bpf_file); 1670 if (err < 0) { 1671 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1672 return err; 1673 } 1674 1675 if (cg_fd < 0) { 1676 if (setup_cgroup_environment()) { 1677 fprintf(stderr, "ERROR: cgroup env failed\n"); 1678 return -EINVAL; 1679 } 1680 1681 cg_fd = create_and_get_cgroup(CG_PATH); 1682 if (cg_fd < 0) { 1683 fprintf(stderr, 1684 "ERROR: (%i) open cg path failed: %s\n", 1685 cg_fd, optarg); 1686 return cg_fd; 1687 } 1688 1689 if (join_cgroup(CG_PATH)) { 1690 fprintf(stderr, "ERROR: failed to join cgroup\n"); 1691 return -EINVAL; 1692 } 1693 } 1694 1695 /* Tests basic commands and APIs with range of iov values */ 1696 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0; 1697 err = test_txmsg(cg_fd); 1698 if (err) 1699 goto out; 1700 1701 /* Tests interesting combinations of APIs used together */ 1702 err = test_mixed(cg_fd); 1703 if (err) 1704 goto out; 1705 1706 /* Tests pull_data API using start/end API */ 1707 err = test_start_end(cg_fd); 1708 if (err) 1709 goto out; 1710 1711 out: 1712 printf("Summary: %i PASSED %i FAILED\n", passed, failed); 1713 if (cleanup < 0) { 1714 cleanup_cgroup_environment(); 1715 close(cg_fd); 1716 } 1717 return err; 1718 } 1719 1720 static int test_suite(int cg_fd) 1721 { 1722 int err; 1723 1724 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME); 1725 if (err) 1726 goto out; 1727 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME); 1728 out: 1729 if (cg_fd > -1) 1730 close(cg_fd); 1731 return err; 1732 } 1733 1734 int main(int argc, char **argv) 1735 { 1736 int iov_count = 1, length = 1024, rate = 1; 1737 struct sockmap_options options = {0}; 1738 int opt, longindex, err, cg_fd = 0; 1739 char *bpf_file = BPF_SOCKMAP_FILENAME; 1740 int test = PING_PONG; 1741 1742 if (argc < 2) 1743 return test_suite(-1); 1744 1745 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:", 1746 long_options, &longindex)) != -1) { 1747 switch (opt) { 1748 case 's': 1749 txmsg_start = atoi(optarg); 1750 break; 1751 case 'e': 1752 txmsg_end = atoi(optarg); 1753 break; 1754 case 'p': 1755 txmsg_start_push = atoi(optarg); 1756 break; 1757 case 'q': 1758 txmsg_end_push = atoi(optarg); 1759 break; 1760 case 'w': 1761 txmsg_start_pop = atoi(optarg); 1762 break; 1763 case 'x': 1764 txmsg_pop = atoi(optarg); 1765 break; 1766 case 'a': 1767 txmsg_apply = atoi(optarg); 1768 break; 1769 case 'k': 1770 txmsg_cork = atoi(optarg); 1771 break; 1772 case 'c': 1773 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1774 if (cg_fd < 0) { 1775 fprintf(stderr, 1776 "ERROR: (%i) open cg path failed: %s\n", 1777 cg_fd, optarg); 1778 return cg_fd; 1779 } 1780 break; 1781 case 'r': 1782 rate = atoi(optarg); 1783 break; 1784 case 'v': 1785 options.verbose = 1; 1786 break; 1787 case 'i': 1788 iov_count = atoi(optarg); 1789 break; 1790 case 'l': 1791 length = atoi(optarg); 1792 break; 1793 case 'd': 1794 options.data_test = true; 1795 break; 1796 case 't': 1797 if (strcmp(optarg, "ping") == 0) { 1798 test = PING_PONG; 1799 } else if (strcmp(optarg, "sendmsg") == 0) { 1800 test = SENDMSG; 1801 } else if (strcmp(optarg, "base") == 0) { 1802 test = BASE; 1803 } else if (strcmp(optarg, "base_sendpage") == 0) { 1804 test = BASE_SENDPAGE; 1805 } else if (strcmp(optarg, "sendpage") == 0) { 1806 test = SENDPAGE; 1807 } else { 1808 usage(argv); 1809 return -1; 1810 } 1811 break; 1812 case 0: 1813 break; 1814 case 'h': 1815 default: 1816 usage(argv); 1817 return -1; 1818 } 1819 } 1820 1821 if (argc <= 3 && cg_fd) 1822 return test_suite(cg_fd); 1823 1824 if (!cg_fd) { 1825 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n", 1826 argv[0]); 1827 return -1; 1828 } 1829 1830 err = populate_progs(bpf_file); 1831 if (err) { 1832 fprintf(stderr, "populate program: (%s) %s\n", 1833 bpf_file, strerror(errno)); 1834 return 1; 1835 } 1836 running = 1; 1837 1838 /* catch SIGINT */ 1839 signal(SIGINT, running_handler); 1840 1841 options.iov_count = iov_count; 1842 options.iov_length = length; 1843 options.rate = rate; 1844 1845 err = run_options(&options, cg_fd, test); 1846 close(cg_fd); 1847 return err; 1848 } 1849 1850 void running_handler(int a) 1851 { 1852 running = 0; 1853 } 1854