1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io 3 #include <stdio.h> 4 #include <stdlib.h> 5 #include <sys/socket.h> 6 #include <sys/ioctl.h> 7 #include <sys/select.h> 8 #include <netinet/in.h> 9 #include <arpa/inet.h> 10 #include <unistd.h> 11 #include <string.h> 12 #include <errno.h> 13 #include <stdbool.h> 14 #include <signal.h> 15 #include <fcntl.h> 16 #include <sys/wait.h> 17 #include <time.h> 18 #include <sched.h> 19 20 #include <sys/time.h> 21 #include <sys/resource.h> 22 #include <sys/types.h> 23 #include <sys/sendfile.h> 24 25 #include <linux/netlink.h> 26 #include <linux/socket.h> 27 #include <linux/sock_diag.h> 28 #include <linux/bpf.h> 29 #include <linux/if_link.h> 30 #include <linux/tls.h> 31 #include <assert.h> 32 #include <libgen.h> 33 34 #include <getopt.h> 35 36 #include <bpf/bpf.h> 37 #include <bpf/libbpf.h> 38 39 #include "bpf_util.h" 40 #include "bpf_rlimit.h" 41 #include "cgroup_helpers.h" 42 43 int running; 44 static void running_handler(int a); 45 46 #ifndef TCP_ULP 47 # define TCP_ULP 31 48 #endif 49 #ifndef SOL_TLS 50 # define SOL_TLS 282 51 #endif 52 53 /* randomly selected ports for testing on lo */ 54 #define S1_PORT 10000 55 #define S2_PORT 10001 56 57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" 58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" 59 #define CG_PATH "/sockmap" 60 61 /* global sockets */ 62 int s1, s2, c1, c2, p1, p2; 63 int test_cnt; 64 int passed; 65 int failed; 66 int map_fd[8]; 67 struct bpf_map *maps[8]; 68 int prog_fd[11]; 69 70 int txmsg_pass; 71 int txmsg_noisy; 72 int txmsg_redir; 73 int txmsg_redir_noisy; 74 int txmsg_drop; 75 int txmsg_apply; 76 int txmsg_cork; 77 int txmsg_start; 78 int txmsg_end; 79 int txmsg_start_push; 80 int txmsg_end_push; 81 int txmsg_start_pop; 82 int txmsg_pop; 83 int txmsg_ingress; 84 int txmsg_skb; 85 int ktls; 86 int peek_flag; 87 88 static const struct option long_options[] = { 89 {"help", no_argument, NULL, 'h' }, 90 {"cgroup", required_argument, NULL, 'c' }, 91 {"rate", required_argument, NULL, 'r' }, 92 {"verbose", no_argument, NULL, 'v' }, 93 {"iov_count", required_argument, NULL, 'i' }, 94 {"length", required_argument, NULL, 'l' }, 95 {"test", required_argument, NULL, 't' }, 96 {"data_test", no_argument, NULL, 'd' }, 97 {"txmsg", no_argument, &txmsg_pass, 1 }, 98 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 }, 99 {"txmsg_redir", no_argument, &txmsg_redir, 1 }, 100 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1}, 101 {"txmsg_drop", no_argument, &txmsg_drop, 1 }, 102 {"txmsg_apply", required_argument, NULL, 'a'}, 103 {"txmsg_cork", required_argument, NULL, 'k'}, 104 {"txmsg_start", required_argument, NULL, 's'}, 105 {"txmsg_end", required_argument, NULL, 'e'}, 106 {"txmsg_start_push", required_argument, NULL, 'p'}, 107 {"txmsg_end_push", required_argument, NULL, 'q'}, 108 {"txmsg_start_pop", required_argument, NULL, 'w'}, 109 {"txmsg_pop", required_argument, NULL, 'x'}, 110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 }, 111 {"txmsg_skb", no_argument, &txmsg_skb, 1 }, 112 {"ktls", no_argument, &ktls, 1 }, 113 {"peek", no_argument, &peek_flag, 1 }, 114 {0, 0, NULL, 0 } 115 }; 116 117 static void usage(char *argv[]) 118 { 119 int i; 120 121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]); 122 printf(" options:\n"); 123 for (i = 0; long_options[i].name != 0; i++) { 124 printf(" --%-12s", long_options[i].name); 125 if (long_options[i].flag != NULL) 126 printf(" flag (internal value:%d)\n", 127 *long_options[i].flag); 128 else 129 printf(" -%c\n", long_options[i].val); 130 } 131 printf("\n"); 132 } 133 134 char *sock_to_string(int s) 135 { 136 if (s == c1) 137 return "client1"; 138 else if (s == c2) 139 return "client2"; 140 else if (s == s1) 141 return "server1"; 142 else if (s == s2) 143 return "server2"; 144 else if (s == p1) 145 return "peer1"; 146 else if (s == p2) 147 return "peer2"; 148 else 149 return "unknown"; 150 } 151 152 static int sockmap_init_ktls(int verbose, int s) 153 { 154 struct tls12_crypto_info_aes_gcm_128 tls_tx = { 155 .info = { 156 .version = TLS_1_2_VERSION, 157 .cipher_type = TLS_CIPHER_AES_GCM_128, 158 }, 159 }; 160 struct tls12_crypto_info_aes_gcm_128 tls_rx = { 161 .info = { 162 .version = TLS_1_2_VERSION, 163 .cipher_type = TLS_CIPHER_AES_GCM_128, 164 }, 165 }; 166 int so_buf = 6553500; 167 int err; 168 169 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls")); 170 if (err) { 171 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err); 172 return -EINVAL; 173 } 174 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx)); 175 if (err) { 176 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err); 177 return -EINVAL; 178 } 179 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx)); 180 if (err) { 181 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err); 182 return -EINVAL; 183 } 184 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf)); 185 if (err) { 186 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err); 187 return -EINVAL; 188 } 189 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf)); 190 if (err) { 191 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err); 192 return -EINVAL; 193 } 194 195 if (verbose) 196 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s)); 197 return 0; 198 } 199 static int sockmap_init_sockets(int verbose) 200 { 201 int i, err, one = 1; 202 struct sockaddr_in addr; 203 int *fds[4] = {&s1, &s2, &c1, &c2}; 204 205 s1 = s2 = p1 = p2 = c1 = c2 = 0; 206 207 /* Init sockets */ 208 for (i = 0; i < 4; i++) { 209 *fds[i] = socket(AF_INET, SOCK_STREAM, 0); 210 if (*fds[i] < 0) { 211 perror("socket s1 failed()"); 212 return errno; 213 } 214 } 215 216 /* Allow reuse */ 217 for (i = 0; i < 2; i++) { 218 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR, 219 (char *)&one, sizeof(one)); 220 if (err) { 221 perror("setsockopt failed()"); 222 return errno; 223 } 224 } 225 226 /* Non-blocking sockets */ 227 for (i = 0; i < 2; i++) { 228 err = ioctl(*fds[i], FIONBIO, (char *)&one); 229 if (err < 0) { 230 perror("ioctl s1 failed()"); 231 return errno; 232 } 233 } 234 235 /* Bind server sockets */ 236 memset(&addr, 0, sizeof(struct sockaddr_in)); 237 addr.sin_family = AF_INET; 238 addr.sin_addr.s_addr = inet_addr("127.0.0.1"); 239 240 addr.sin_port = htons(S1_PORT); 241 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr)); 242 if (err < 0) { 243 perror("bind s1 failed()\n"); 244 return errno; 245 } 246 247 addr.sin_port = htons(S2_PORT); 248 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr)); 249 if (err < 0) { 250 perror("bind s2 failed()\n"); 251 return errno; 252 } 253 254 /* Listen server sockets */ 255 addr.sin_port = htons(S1_PORT); 256 err = listen(s1, 32); 257 if (err < 0) { 258 perror("listen s1 failed()\n"); 259 return errno; 260 } 261 262 addr.sin_port = htons(S2_PORT); 263 err = listen(s2, 32); 264 if (err < 0) { 265 perror("listen s1 failed()\n"); 266 return errno; 267 } 268 269 /* Initiate Connect */ 270 addr.sin_port = htons(S1_PORT); 271 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr)); 272 if (err < 0 && errno != EINPROGRESS) { 273 perror("connect c1 failed()\n"); 274 return errno; 275 } 276 277 addr.sin_port = htons(S2_PORT); 278 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr)); 279 if (err < 0 && errno != EINPROGRESS) { 280 perror("connect c2 failed()\n"); 281 return errno; 282 } else if (err < 0) { 283 err = 0; 284 } 285 286 /* Accept Connecrtions */ 287 p1 = accept(s1, NULL, NULL); 288 if (p1 < 0) { 289 perror("accept s1 failed()\n"); 290 return errno; 291 } 292 293 p2 = accept(s2, NULL, NULL); 294 if (p2 < 0) { 295 perror("accept s1 failed()\n"); 296 return errno; 297 } 298 299 if (verbose) { 300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n"); 301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n", 302 c1, s1, c2, s2); 303 } 304 return 0; 305 } 306 307 struct msg_stats { 308 size_t bytes_sent; 309 size_t bytes_recvd; 310 struct timespec start; 311 struct timespec end; 312 }; 313 314 struct sockmap_options { 315 int verbose; 316 bool base; 317 bool sendpage; 318 bool data_test; 319 bool drop_expected; 320 int iov_count; 321 int iov_length; 322 int rate; 323 }; 324 325 static int msg_loop_sendpage(int fd, int iov_length, int cnt, 326 struct msg_stats *s, 327 struct sockmap_options *opt) 328 { 329 bool drop = opt->drop_expected; 330 unsigned char k = 0; 331 FILE *file; 332 int i, fp; 333 334 file = fopen(".sendpage_tst.tmp", "w+"); 335 for (i = 0; i < iov_length * cnt; i++, k++) 336 fwrite(&k, sizeof(char), 1, file); 337 fflush(file); 338 fseek(file, 0, SEEK_SET); 339 fclose(file); 340 341 fp = open(".sendpage_tst.tmp", O_RDONLY); 342 clock_gettime(CLOCK_MONOTONIC, &s->start); 343 for (i = 0; i < cnt; i++) { 344 int sent = sendfile(fd, fp, NULL, iov_length); 345 346 if (!drop && sent < 0) { 347 perror("send loop error:"); 348 close(fp); 349 return sent; 350 } else if (drop && sent >= 0) { 351 printf("sendpage loop error expected: %i\n", sent); 352 close(fp); 353 return -EIO; 354 } 355 356 if (sent > 0) 357 s->bytes_sent += sent; 358 } 359 clock_gettime(CLOCK_MONOTONIC, &s->end); 360 close(fp); 361 return 0; 362 } 363 364 static void msg_free_iov(struct msghdr *msg) 365 { 366 int i; 367 368 for (i = 0; i < msg->msg_iovlen; i++) 369 free(msg->msg_iov[i].iov_base); 370 free(msg->msg_iov); 371 msg->msg_iov = NULL; 372 msg->msg_iovlen = 0; 373 } 374 375 static int msg_alloc_iov(struct msghdr *msg, 376 int iov_count, int iov_length, 377 bool data, bool xmit) 378 { 379 unsigned char k = 0; 380 struct iovec *iov; 381 int i; 382 383 iov = calloc(iov_count, sizeof(struct iovec)); 384 if (!iov) 385 return errno; 386 387 for (i = 0; i < iov_count; i++) { 388 unsigned char *d = calloc(iov_length, sizeof(char)); 389 390 if (!d) { 391 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count); 392 goto unwind_iov; 393 } 394 iov[i].iov_base = d; 395 iov[i].iov_len = iov_length; 396 397 if (data && xmit) { 398 int j; 399 400 for (j = 0; j < iov_length; j++) 401 d[j] = k++; 402 } 403 } 404 405 msg->msg_iov = iov; 406 msg->msg_iovlen = iov_count; 407 408 return 0; 409 unwind_iov: 410 for (i--; i >= 0 ; i--) 411 free(msg->msg_iov[i].iov_base); 412 return -ENOMEM; 413 } 414 415 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) 416 { 417 int i, j, bytes_cnt = 0; 418 unsigned char k = 0; 419 420 for (i = 0; i < msg->msg_iovlen; i++) { 421 unsigned char *d = msg->msg_iov[i].iov_base; 422 423 for (j = 0; 424 j < msg->msg_iov[i].iov_len && size; j++) { 425 if (d[j] != k++) { 426 fprintf(stderr, 427 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n", 428 i, j, d[j], k - 1, d[j+1], k); 429 return -EIO; 430 } 431 bytes_cnt++; 432 if (bytes_cnt == chunk_sz) { 433 k = 0; 434 bytes_cnt = 0; 435 } 436 size--; 437 } 438 } 439 return 0; 440 } 441 442 static int msg_loop(int fd, int iov_count, int iov_length, int cnt, 443 struct msg_stats *s, bool tx, 444 struct sockmap_options *opt) 445 { 446 struct msghdr msg = {0}, msg_peek = {0}; 447 int err, i, flags = MSG_NOSIGNAL; 448 bool drop = opt->drop_expected; 449 bool data = opt->data_test; 450 451 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx); 452 if (err) 453 goto out_errno; 454 if (peek_flag) { 455 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx); 456 if (err) 457 goto out_errno; 458 } 459 460 if (tx) { 461 clock_gettime(CLOCK_MONOTONIC, &s->start); 462 for (i = 0; i < cnt; i++) { 463 int sent = sendmsg(fd, &msg, flags); 464 465 if (!drop && sent < 0) { 466 perror("send loop error:"); 467 goto out_errno; 468 } else if (drop && sent >= 0) { 469 printf("send loop error expected: %i\n", sent); 470 errno = -EIO; 471 goto out_errno; 472 } 473 if (sent > 0) 474 s->bytes_sent += sent; 475 } 476 clock_gettime(CLOCK_MONOTONIC, &s->end); 477 } else { 478 int slct, recvp = 0, recv, max_fd = fd; 479 float total_bytes, txmsg_pop_total; 480 int fd_flags = O_NONBLOCK; 481 struct timeval timeout; 482 fd_set w; 483 484 fcntl(fd, fd_flags); 485 /* Account for pop bytes noting each iteration of apply will 486 * call msg_pop_data helper so we need to account for this 487 * by calculating the number of apply iterations. Note user 488 * of the tool can create cases where no data is sent by 489 * manipulating pop/push/pull/etc. For example txmsg_apply 1 490 * with txmsg_pop 1 will try to apply 1B at a time but each 491 * iteration will then pop 1B so no data will ever be sent. 492 * This is really only useful for testing edge cases in code 493 * paths. 494 */ 495 total_bytes = (float)iov_count * (float)iov_length * (float)cnt; 496 txmsg_pop_total = txmsg_pop; 497 if (txmsg_apply) 498 txmsg_pop_total *= (total_bytes / txmsg_apply); 499 total_bytes -= txmsg_pop_total; 500 err = clock_gettime(CLOCK_MONOTONIC, &s->start); 501 if (err < 0) 502 perror("recv start time: "); 503 while (s->bytes_recvd < total_bytes) { 504 if (txmsg_cork) { 505 timeout.tv_sec = 0; 506 timeout.tv_usec = 300000; 507 } else { 508 timeout.tv_sec = 3; 509 timeout.tv_usec = 0; 510 } 511 512 /* FD sets */ 513 FD_ZERO(&w); 514 FD_SET(fd, &w); 515 516 slct = select(max_fd + 1, &w, NULL, NULL, &timeout); 517 if (slct == -1) { 518 perror("select()"); 519 clock_gettime(CLOCK_MONOTONIC, &s->end); 520 goto out_errno; 521 } else if (!slct) { 522 if (opt->verbose) 523 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total); 524 errno = -EIO; 525 clock_gettime(CLOCK_MONOTONIC, &s->end); 526 goto out_errno; 527 } 528 529 errno = 0; 530 if (peek_flag) { 531 flags |= MSG_PEEK; 532 recvp = recvmsg(fd, &msg_peek, flags); 533 if (recvp < 0) { 534 if (errno != EWOULDBLOCK) { 535 clock_gettime(CLOCK_MONOTONIC, &s->end); 536 goto out_errno; 537 } 538 } 539 flags = 0; 540 } 541 542 recv = recvmsg(fd, &msg, flags); 543 if (recv < 0) { 544 if (errno != EWOULDBLOCK) { 545 clock_gettime(CLOCK_MONOTONIC, &s->end); 546 perror("recv failed()\n"); 547 goto out_errno; 548 } 549 } 550 551 s->bytes_recvd += recv; 552 553 if (data) { 554 int chunk_sz = opt->sendpage ? 555 iov_length * cnt : 556 iov_length * iov_count; 557 558 errno = msg_verify_data(&msg, recv, chunk_sz); 559 if (errno) { 560 perror("data verify msg failed\n"); 561 goto out_errno; 562 } 563 if (recvp) { 564 errno = msg_verify_data(&msg_peek, 565 recvp, 566 chunk_sz); 567 if (errno) { 568 perror("data verify msg_peek failed\n"); 569 goto out_errno; 570 } 571 } 572 } 573 } 574 clock_gettime(CLOCK_MONOTONIC, &s->end); 575 } 576 577 msg_free_iov(&msg); 578 msg_free_iov(&msg_peek); 579 return err; 580 out_errno: 581 msg_free_iov(&msg); 582 msg_free_iov(&msg_peek); 583 return errno; 584 } 585 586 static float giga = 1000000000; 587 588 static inline float sentBps(struct msg_stats s) 589 { 590 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec); 591 } 592 593 static inline float recvdBps(struct msg_stats s) 594 { 595 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec); 596 } 597 598 static int sendmsg_test(struct sockmap_options *opt) 599 { 600 float sent_Bps = 0, recvd_Bps = 0; 601 int rx_fd, txpid, rxpid, err = 0; 602 struct msg_stats s = {0}; 603 int iov_count = opt->iov_count; 604 int iov_buf = opt->iov_length; 605 int rx_status, tx_status; 606 int cnt = opt->rate; 607 608 errno = 0; 609 610 if (opt->base) 611 rx_fd = p1; 612 else 613 rx_fd = p2; 614 615 if (ktls) { 616 /* Redirecting into non-TLS socket which sends into a TLS 617 * socket is not a valid test. So in this case lets not 618 * enable kTLS but still run the test. 619 */ 620 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) { 621 err = sockmap_init_ktls(opt->verbose, rx_fd); 622 if (err) 623 return err; 624 } 625 err = sockmap_init_ktls(opt->verbose, c1); 626 if (err) 627 return err; 628 } 629 630 rxpid = fork(); 631 if (rxpid == 0) { 632 if (opt->drop_expected) 633 exit(0); 634 635 if (opt->sendpage) 636 iov_count = 1; 637 err = msg_loop(rx_fd, iov_count, iov_buf, 638 cnt, &s, false, opt); 639 if (opt->verbose) 640 fprintf(stderr, 641 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n", 642 iov_count, iov_buf, cnt, err); 643 if (s.end.tv_sec - s.start.tv_sec) { 644 sent_Bps = sentBps(s); 645 recvd_Bps = recvdBps(s); 646 } 647 if (opt->verbose) 648 fprintf(stdout, 649 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n", 650 s.bytes_sent, sent_Bps, sent_Bps/giga, 651 s.bytes_recvd, recvd_Bps, recvd_Bps/giga, 652 peek_flag ? "(peek_msg)" : ""); 653 if (err && txmsg_cork) 654 err = 0; 655 exit(err ? 1 : 0); 656 } else if (rxpid == -1) { 657 perror("msg_loop_rx: "); 658 return errno; 659 } 660 661 txpid = fork(); 662 if (txpid == 0) { 663 if (opt->sendpage) 664 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt); 665 else 666 err = msg_loop(c1, iov_count, iov_buf, 667 cnt, &s, true, opt); 668 669 if (err) 670 fprintf(stderr, 671 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n", 672 iov_count, iov_buf, cnt, err); 673 if (s.end.tv_sec - s.start.tv_sec) { 674 sent_Bps = sentBps(s); 675 recvd_Bps = recvdBps(s); 676 } 677 if (opt->verbose) 678 fprintf(stdout, 679 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n", 680 s.bytes_sent, sent_Bps, sent_Bps/giga, 681 s.bytes_recvd, recvd_Bps, recvd_Bps/giga); 682 exit(err ? 1 : 0); 683 } else if (txpid == -1) { 684 perror("msg_loop_tx: "); 685 return errno; 686 } 687 688 assert(waitpid(rxpid, &rx_status, 0) == rxpid); 689 assert(waitpid(txpid, &tx_status, 0) == txpid); 690 if (WIFEXITED(rx_status)) { 691 err = WEXITSTATUS(rx_status); 692 if (err) { 693 fprintf(stderr, "rx thread exited with err %d. ", err); 694 goto out; 695 } 696 } 697 if (WIFEXITED(tx_status)) { 698 err = WEXITSTATUS(tx_status); 699 if (err) 700 fprintf(stderr, "tx thread exited with err %d. ", err); 701 } 702 out: 703 return err; 704 } 705 706 static int forever_ping_pong(int rate, struct sockmap_options *opt) 707 { 708 struct timeval timeout; 709 char buf[1024] = {0}; 710 int sc; 711 712 timeout.tv_sec = 10; 713 timeout.tv_usec = 0; 714 715 /* Ping/Pong data from client to server */ 716 sc = send(c1, buf, sizeof(buf), 0); 717 if (sc < 0) { 718 perror("send failed()\n"); 719 return sc; 720 } 721 722 do { 723 int s, rc, i, max_fd = p2; 724 fd_set w; 725 726 /* FD sets */ 727 FD_ZERO(&w); 728 FD_SET(c1, &w); 729 FD_SET(c2, &w); 730 FD_SET(p1, &w); 731 FD_SET(p2, &w); 732 733 s = select(max_fd + 1, &w, NULL, NULL, &timeout); 734 if (s == -1) { 735 perror("select()"); 736 break; 737 } else if (!s) { 738 fprintf(stderr, "unexpected timeout\n"); 739 break; 740 } 741 742 for (i = 0; i <= max_fd && s > 0; ++i) { 743 if (!FD_ISSET(i, &w)) 744 continue; 745 746 s--; 747 748 rc = recv(i, buf, sizeof(buf), 0); 749 if (rc < 0) { 750 if (errno != EWOULDBLOCK) { 751 perror("recv failed()\n"); 752 return rc; 753 } 754 } 755 756 if (rc == 0) { 757 close(i); 758 break; 759 } 760 761 sc = send(i, buf, rc, 0); 762 if (sc < 0) { 763 perror("send failed()\n"); 764 return sc; 765 } 766 } 767 768 if (rate) 769 sleep(rate); 770 771 if (opt->verbose) { 772 printf("."); 773 fflush(stdout); 774 775 } 776 } while (running); 777 778 return 0; 779 } 780 781 enum { 782 PING_PONG, 783 SENDMSG, 784 BASE, 785 BASE_SENDPAGE, 786 SENDPAGE, 787 }; 788 789 static int run_options(struct sockmap_options *options, int cg_fd, int test) 790 { 791 int i, key, next_key, err, tx_prog_fd = -1, zero = 0; 792 793 /* If base test skip BPF setup */ 794 if (test == BASE || test == BASE_SENDPAGE) 795 goto run; 796 797 /* Attach programs to sockmap */ 798 err = bpf_prog_attach(prog_fd[0], map_fd[0], 799 BPF_SK_SKB_STREAM_PARSER, 0); 800 if (err) { 801 fprintf(stderr, 802 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n", 803 prog_fd[0], map_fd[0], err, strerror(errno)); 804 return err; 805 } 806 807 err = bpf_prog_attach(prog_fd[1], map_fd[0], 808 BPF_SK_SKB_STREAM_VERDICT, 0); 809 if (err) { 810 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n", 811 err, strerror(errno)); 812 return err; 813 } 814 815 /* Attach to cgroups */ 816 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0); 817 if (err) { 818 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n", 819 err, strerror(errno)); 820 return err; 821 } 822 823 run: 824 err = sockmap_init_sockets(options->verbose); 825 if (err) { 826 fprintf(stderr, "ERROR: test socket failed: %d\n", err); 827 goto out; 828 } 829 830 /* Attach txmsg program to sockmap */ 831 if (txmsg_pass) 832 tx_prog_fd = prog_fd[3]; 833 else if (txmsg_noisy) 834 tx_prog_fd = prog_fd[4]; 835 else if (txmsg_redir) 836 tx_prog_fd = prog_fd[5]; 837 else if (txmsg_redir_noisy) 838 tx_prog_fd = prog_fd[6]; 839 else if (txmsg_drop) 840 tx_prog_fd = prog_fd[9]; 841 /* apply and cork must be last */ 842 else if (txmsg_apply) 843 tx_prog_fd = prog_fd[7]; 844 else if (txmsg_cork) 845 tx_prog_fd = prog_fd[8]; 846 else 847 tx_prog_fd = 0; 848 849 if (tx_prog_fd) { 850 int redir_fd, i = 0; 851 852 err = bpf_prog_attach(tx_prog_fd, 853 map_fd[1], BPF_SK_MSG_VERDICT, 0); 854 if (err) { 855 fprintf(stderr, 856 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n", 857 err, strerror(errno)); 858 goto out; 859 } 860 861 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY); 862 if (err) { 863 fprintf(stderr, 864 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 865 err, strerror(errno)); 866 goto out; 867 } 868 869 if (txmsg_redir || txmsg_redir_noisy) 870 redir_fd = c2; 871 else 872 redir_fd = c1; 873 874 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY); 875 if (err) { 876 fprintf(stderr, 877 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n", 878 err, strerror(errno)); 879 goto out; 880 } 881 882 if (txmsg_apply) { 883 err = bpf_map_update_elem(map_fd[3], 884 &i, &txmsg_apply, BPF_ANY); 885 if (err) { 886 fprintf(stderr, 887 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n", 888 err, strerror(errno)); 889 goto out; 890 } 891 } 892 893 if (txmsg_cork) { 894 err = bpf_map_update_elem(map_fd[4], 895 &i, &txmsg_cork, BPF_ANY); 896 if (err) { 897 fprintf(stderr, 898 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n", 899 err, strerror(errno)); 900 goto out; 901 } 902 } 903 904 if (txmsg_start) { 905 err = bpf_map_update_elem(map_fd[5], 906 &i, &txmsg_start, BPF_ANY); 907 if (err) { 908 fprintf(stderr, 909 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n", 910 err, strerror(errno)); 911 goto out; 912 } 913 } 914 915 if (txmsg_end) { 916 i = 1; 917 err = bpf_map_update_elem(map_fd[5], 918 &i, &txmsg_end, BPF_ANY); 919 if (err) { 920 fprintf(stderr, 921 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n", 922 err, strerror(errno)); 923 goto out; 924 } 925 } 926 927 if (txmsg_start_push) { 928 i = 2; 929 err = bpf_map_update_elem(map_fd[5], 930 &i, &txmsg_start_push, BPF_ANY); 931 if (err) { 932 fprintf(stderr, 933 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n", 934 err, strerror(errno)); 935 goto out; 936 } 937 } 938 939 if (txmsg_end_push) { 940 i = 3; 941 err = bpf_map_update_elem(map_fd[5], 942 &i, &txmsg_end_push, BPF_ANY); 943 if (err) { 944 fprintf(stderr, 945 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n", 946 txmsg_end_push, i, err, strerror(errno)); 947 goto out; 948 } 949 } 950 951 if (txmsg_start_pop) { 952 i = 4; 953 err = bpf_map_update_elem(map_fd[5], 954 &i, &txmsg_start_pop, BPF_ANY); 955 if (err) { 956 fprintf(stderr, 957 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n", 958 txmsg_start_pop, i, err, strerror(errno)); 959 goto out; 960 } 961 } else { 962 i = 4; 963 bpf_map_update_elem(map_fd[5], 964 &i, &txmsg_start_pop, BPF_ANY); 965 } 966 967 if (txmsg_pop) { 968 i = 5; 969 err = bpf_map_update_elem(map_fd[5], 970 &i, &txmsg_pop, BPF_ANY); 971 if (err) { 972 fprintf(stderr, 973 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n", 974 txmsg_pop, i, err, strerror(errno)); 975 goto out; 976 } 977 } else { 978 i = 5; 979 bpf_map_update_elem(map_fd[5], 980 &i, &txmsg_pop, BPF_ANY); 981 982 } 983 984 if (txmsg_ingress) { 985 int in = BPF_F_INGRESS; 986 987 i = 0; 988 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY); 989 if (err) { 990 fprintf(stderr, 991 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 992 err, strerror(errno)); 993 } 994 i = 1; 995 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY); 996 if (err) { 997 fprintf(stderr, 998 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n", 999 err, strerror(errno)); 1000 } 1001 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY); 1002 if (err) { 1003 fprintf(stderr, 1004 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n", 1005 err, strerror(errno)); 1006 } 1007 1008 i = 2; 1009 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY); 1010 if (err) { 1011 fprintf(stderr, 1012 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n", 1013 err, strerror(errno)); 1014 } 1015 } 1016 1017 if (txmsg_skb) { 1018 int skb_fd = (test == SENDMSG || test == SENDPAGE) ? 1019 p2 : p1; 1020 int ingress = BPF_F_INGRESS; 1021 1022 i = 0; 1023 err = bpf_map_update_elem(map_fd[7], 1024 &i, &ingress, BPF_ANY); 1025 if (err) { 1026 fprintf(stderr, 1027 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n", 1028 err, strerror(errno)); 1029 } 1030 1031 i = 3; 1032 err = bpf_map_update_elem(map_fd[0], 1033 &i, &skb_fd, BPF_ANY); 1034 if (err) { 1035 fprintf(stderr, 1036 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n", 1037 err, strerror(errno)); 1038 } 1039 } 1040 } 1041 1042 if (txmsg_drop) 1043 options->drop_expected = true; 1044 1045 if (test == PING_PONG) 1046 err = forever_ping_pong(options->rate, options); 1047 else if (test == SENDMSG) { 1048 options->base = false; 1049 options->sendpage = false; 1050 err = sendmsg_test(options); 1051 } else if (test == SENDPAGE) { 1052 options->base = false; 1053 options->sendpage = true; 1054 err = sendmsg_test(options); 1055 } else if (test == BASE) { 1056 options->base = true; 1057 options->sendpage = false; 1058 err = sendmsg_test(options); 1059 } else if (test == BASE_SENDPAGE) { 1060 options->base = true; 1061 options->sendpage = true; 1062 err = sendmsg_test(options); 1063 } else 1064 fprintf(stderr, "unknown test\n"); 1065 out: 1066 /* Detatch and zero all the maps */ 1067 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS); 1068 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER); 1069 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT); 1070 if (tx_prog_fd >= 0) 1071 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT); 1072 1073 for (i = 0; i < 8; i++) { 1074 key = next_key = 0; 1075 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1076 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) { 1077 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY); 1078 key = next_key; 1079 } 1080 } 1081 1082 close(s1); 1083 close(s2); 1084 close(p1); 1085 close(p2); 1086 close(c1); 1087 close(c2); 1088 return err; 1089 } 1090 1091 static char *test_to_str(int test) 1092 { 1093 switch (test) { 1094 case SENDMSG: 1095 return "sendmsg"; 1096 case SENDPAGE: 1097 return "sendpage"; 1098 } 1099 return "unknown"; 1100 } 1101 1102 #define OPTSTRING 60 1103 static void test_options(char *options) 1104 { 1105 char tstr[OPTSTRING]; 1106 1107 memset(options, 0, OPTSTRING); 1108 1109 if (txmsg_pass) 1110 strncat(options, "pass,", OPTSTRING); 1111 if (txmsg_noisy) 1112 strncat(options, "pass_noisy,", OPTSTRING); 1113 if (txmsg_redir) 1114 strncat(options, "redir,", OPTSTRING); 1115 if (txmsg_redir_noisy) 1116 strncat(options, "redir_noisy,", OPTSTRING); 1117 if (txmsg_drop) 1118 strncat(options, "drop,", OPTSTRING); 1119 if (txmsg_apply) { 1120 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply); 1121 strncat(options, tstr, OPTSTRING); 1122 } 1123 if (txmsg_cork) { 1124 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork); 1125 strncat(options, tstr, OPTSTRING); 1126 } 1127 if (txmsg_start) { 1128 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start); 1129 strncat(options, tstr, OPTSTRING); 1130 } 1131 if (txmsg_end) { 1132 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end); 1133 strncat(options, tstr, OPTSTRING); 1134 } 1135 if (txmsg_start_pop) { 1136 snprintf(tstr, OPTSTRING, "pop (%d,%d),", 1137 txmsg_start_pop, txmsg_start_pop + txmsg_pop); 1138 strncat(options, tstr, OPTSTRING); 1139 } 1140 if (txmsg_ingress) 1141 strncat(options, "ingress,", OPTSTRING); 1142 if (txmsg_skb) 1143 strncat(options, "skb,", OPTSTRING); 1144 if (ktls) 1145 strncat(options, "ktls,", OPTSTRING); 1146 if (peek_flag) 1147 strncat(options, "peek,", OPTSTRING); 1148 } 1149 1150 static int __test_exec(int cgrp, int test, struct sockmap_options *opt) 1151 { 1152 char *options = calloc(OPTSTRING, sizeof(char)); 1153 int err; 1154 1155 if (test == SENDPAGE) 1156 opt->sendpage = true; 1157 else 1158 opt->sendpage = false; 1159 1160 if (txmsg_drop) 1161 opt->drop_expected = true; 1162 else 1163 opt->drop_expected = false; 1164 1165 test_options(options); 1166 1167 fprintf(stdout, 1168 "[TEST %i]: (%i, %i, %i, %s, %s): ", 1169 test_cnt, opt->rate, opt->iov_count, opt->iov_length, 1170 test_to_str(test), options); 1171 fflush(stdout); 1172 err = run_options(opt, cgrp, test); 1173 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED"); 1174 test_cnt++; 1175 !err ? passed++ : failed++; 1176 free(options); 1177 return err; 1178 } 1179 1180 static int test_exec(int cgrp, struct sockmap_options *opt) 1181 { 1182 int err = __test_exec(cgrp, SENDMSG, opt); 1183 1184 if (err) 1185 goto out; 1186 1187 err = __test_exec(cgrp, SENDPAGE, opt); 1188 out: 1189 return err; 1190 } 1191 1192 static int test_loop(int cgrp) 1193 { 1194 struct sockmap_options opt; 1195 1196 int err, i, l, r; 1197 1198 opt.verbose = 0; 1199 opt.base = false; 1200 opt.sendpage = false; 1201 opt.data_test = false; 1202 opt.drop_expected = false; 1203 opt.iov_count = 0; 1204 opt.iov_length = 0; 1205 opt.rate = 0; 1206 1207 r = 1; 1208 for (i = 1; i < 100; i += 33) { 1209 for (l = 1; l < 100; l += 33) { 1210 opt.rate = r; 1211 opt.iov_count = i; 1212 opt.iov_length = l; 1213 err = test_exec(cgrp, &opt); 1214 if (err) 1215 goto out; 1216 } 1217 } 1218 sched_yield(); 1219 out: 1220 return err; 1221 } 1222 1223 static int test_txmsg(int cgrp) 1224 { 1225 int err; 1226 1227 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1228 txmsg_apply = txmsg_cork = 0; 1229 txmsg_ingress = txmsg_skb = 0; 1230 1231 txmsg_pass = 1; 1232 err = test_loop(cgrp); 1233 txmsg_pass = 0; 1234 if (err) 1235 goto out; 1236 1237 txmsg_redir = 1; 1238 err = test_loop(cgrp); 1239 txmsg_redir = 0; 1240 if (err) 1241 goto out; 1242 1243 txmsg_drop = 1; 1244 err = test_loop(cgrp); 1245 txmsg_drop = 0; 1246 if (err) 1247 goto out; 1248 1249 txmsg_redir = 1; 1250 txmsg_ingress = 1; 1251 err = test_loop(cgrp); 1252 txmsg_redir = 0; 1253 txmsg_ingress = 0; 1254 if (err) 1255 goto out; 1256 out: 1257 txmsg_pass = 0; 1258 txmsg_redir = 0; 1259 txmsg_drop = 0; 1260 return err; 1261 } 1262 1263 static int test_send(struct sockmap_options *opt, int cgrp) 1264 { 1265 int err; 1266 1267 opt->iov_length = 1; 1268 opt->iov_count = 1; 1269 opt->rate = 1; 1270 err = test_exec(cgrp, opt); 1271 if (err) 1272 goto out; 1273 1274 opt->iov_length = 1; 1275 opt->iov_count = 1024; 1276 opt->rate = 1; 1277 err = test_exec(cgrp, opt); 1278 if (err) 1279 goto out; 1280 1281 opt->iov_length = 1024; 1282 opt->iov_count = 1; 1283 opt->rate = 1; 1284 err = test_exec(cgrp, opt); 1285 if (err) 1286 goto out; 1287 1288 opt->iov_length = 1; 1289 opt->iov_count = 1; 1290 opt->rate = 512; 1291 err = test_exec(cgrp, opt); 1292 if (err) 1293 goto out; 1294 1295 opt->iov_length = 256; 1296 opt->iov_count = 1024; 1297 opt->rate = 2; 1298 err = test_exec(cgrp, opt); 1299 if (err) 1300 goto out; 1301 1302 opt->rate = 100; 1303 opt->iov_count = 1; 1304 opt->iov_length = 5; 1305 err = test_exec(cgrp, opt); 1306 if (err) 1307 goto out; 1308 out: 1309 sched_yield(); 1310 return err; 1311 } 1312 1313 static int test_mixed(int cgrp) 1314 { 1315 struct sockmap_options opt = {0}; 1316 int err; 1317 1318 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0; 1319 txmsg_apply = txmsg_cork = 0; 1320 txmsg_start = txmsg_end = 0; 1321 txmsg_start_push = txmsg_end_push = 0; 1322 txmsg_start_pop = txmsg_pop = 0; 1323 1324 /* Test small and large iov_count values with pass/redir/apply/cork */ 1325 txmsg_pass = 1; 1326 txmsg_redir = 0; 1327 txmsg_apply = 1; 1328 txmsg_cork = 0; 1329 err = test_send(&opt, cgrp); 1330 if (err) 1331 goto out; 1332 1333 txmsg_pass = 1; 1334 txmsg_redir = 0; 1335 txmsg_apply = 0; 1336 txmsg_cork = 1; 1337 err = test_send(&opt, cgrp); 1338 if (err) 1339 goto out; 1340 1341 txmsg_pass = 1; 1342 txmsg_redir = 0; 1343 txmsg_apply = 1; 1344 txmsg_cork = 1; 1345 err = test_send(&opt, cgrp); 1346 if (err) 1347 goto out; 1348 1349 txmsg_pass = 1; 1350 txmsg_redir = 0; 1351 txmsg_apply = 1024; 1352 txmsg_cork = 0; 1353 err = test_send(&opt, cgrp); 1354 if (err) 1355 goto out; 1356 1357 txmsg_pass = 1; 1358 txmsg_redir = 0; 1359 txmsg_apply = 0; 1360 txmsg_cork = 1024; 1361 err = test_send(&opt, cgrp); 1362 if (err) 1363 goto out; 1364 1365 txmsg_pass = 1; 1366 txmsg_redir = 0; 1367 txmsg_apply = 1024; 1368 txmsg_cork = 1024; 1369 err = test_send(&opt, cgrp); 1370 if (err) 1371 goto out; 1372 1373 txmsg_pass = 1; 1374 txmsg_redir = 0; 1375 txmsg_cork = 4096; 1376 txmsg_apply = 4096; 1377 err = test_send(&opt, cgrp); 1378 if (err) 1379 goto out; 1380 1381 txmsg_pass = 0; 1382 txmsg_redir = 1; 1383 txmsg_apply = 1; 1384 txmsg_cork = 0; 1385 err = test_send(&opt, cgrp); 1386 if (err) 1387 goto out; 1388 1389 txmsg_pass = 0; 1390 txmsg_redir = 1; 1391 txmsg_apply = 0; 1392 txmsg_cork = 1; 1393 err = test_send(&opt, cgrp); 1394 if (err) 1395 goto out; 1396 1397 txmsg_pass = 0; 1398 txmsg_redir = 1; 1399 txmsg_apply = 1024; 1400 txmsg_cork = 0; 1401 err = test_send(&opt, cgrp); 1402 if (err) 1403 goto out; 1404 1405 txmsg_pass = 0; 1406 txmsg_redir = 1; 1407 txmsg_apply = 0; 1408 txmsg_cork = 1024; 1409 err = test_send(&opt, cgrp); 1410 if (err) 1411 goto out; 1412 1413 txmsg_pass = 0; 1414 txmsg_redir = 1; 1415 txmsg_apply = 1024; 1416 txmsg_cork = 1024; 1417 err = test_send(&opt, cgrp); 1418 if (err) 1419 goto out; 1420 1421 txmsg_pass = 0; 1422 txmsg_redir = 1; 1423 txmsg_cork = 4096; 1424 txmsg_apply = 4096; 1425 err = test_send(&opt, cgrp); 1426 if (err) 1427 goto out; 1428 out: 1429 return err; 1430 } 1431 1432 static int test_start_end(int cgrp) 1433 { 1434 struct sockmap_options opt = {0}; 1435 int err, i; 1436 1437 /* Test basic start/end with lots of iov_count and iov_lengths */ 1438 txmsg_start = 1; 1439 txmsg_end = 2; 1440 txmsg_start_push = 1; 1441 txmsg_end_push = 2; 1442 txmsg_start_pop = 1; 1443 txmsg_pop = 1; 1444 err = test_txmsg(cgrp); 1445 if (err) 1446 goto out; 1447 1448 /* Cut a byte of pushed data but leave reamining in place */ 1449 txmsg_start = 1; 1450 txmsg_end = 2; 1451 txmsg_start_push = 1; 1452 txmsg_end_push = 3; 1453 txmsg_start_pop = 1; 1454 txmsg_pop = 1; 1455 err = test_txmsg(cgrp); 1456 if (err) 1457 goto out; 1458 1459 /* Test start/end with cork */ 1460 opt.rate = 16; 1461 opt.iov_count = 1; 1462 opt.iov_length = 100; 1463 txmsg_cork = 1600; 1464 1465 txmsg_start_pop = 0; 1466 txmsg_pop = 0; 1467 1468 for (i = 99; i <= 1600; i += 500) { 1469 txmsg_start = 0; 1470 txmsg_end = i; 1471 txmsg_start_push = 0; 1472 txmsg_end_push = i; 1473 err = test_exec(cgrp, &opt); 1474 if (err) 1475 goto out; 1476 } 1477 1478 /* Test pop data in middle of cork */ 1479 for (i = 99; i <= 1600; i += 500) { 1480 txmsg_start_pop = 10; 1481 txmsg_pop = i; 1482 err = test_exec(cgrp, &opt); 1483 if (err) 1484 goto out; 1485 } 1486 txmsg_start_pop = 0; 1487 txmsg_pop = 0; 1488 1489 /* Test start/end with cork but pull data in middle */ 1490 for (i = 199; i <= 1600; i += 500) { 1491 txmsg_start = 100; 1492 txmsg_end = i; 1493 txmsg_start_push = 100; 1494 txmsg_end_push = i; 1495 err = test_exec(cgrp, &opt); 1496 if (err) 1497 goto out; 1498 } 1499 1500 /* Test start/end with cork pulling last sg entry */ 1501 txmsg_start = 1500; 1502 txmsg_end = 1600; 1503 txmsg_start_push = 1500; 1504 txmsg_end_push = 1600; 1505 err = test_exec(cgrp, &opt); 1506 if (err) 1507 goto out; 1508 1509 /* Test pop with cork pulling last sg entry */ 1510 txmsg_start_pop = 1500; 1511 txmsg_pop = 1600; 1512 err = test_exec(cgrp, &opt); 1513 if (err) 1514 goto out; 1515 txmsg_start_pop = 0; 1516 txmsg_pop = 0; 1517 1518 /* Test start/end pull of single byte in last page */ 1519 txmsg_start = 1111; 1520 txmsg_end = 1112; 1521 txmsg_start_push = 1111; 1522 txmsg_end_push = 1112; 1523 err = test_exec(cgrp, &opt); 1524 if (err) 1525 goto out; 1526 1527 /* Test pop of single byte in last page */ 1528 txmsg_start_pop = 1111; 1529 txmsg_pop = 1112; 1530 err = test_exec(cgrp, &opt); 1531 if (err) 1532 goto out; 1533 1534 /* Test start/end with end < start */ 1535 txmsg_start = 1111; 1536 txmsg_end = 0; 1537 txmsg_start_push = 1111; 1538 txmsg_end_push = 0; 1539 err = test_exec(cgrp, &opt); 1540 if (err) 1541 goto out; 1542 1543 /* Test start/end with end > data */ 1544 txmsg_start = 0; 1545 txmsg_end = 1601; 1546 txmsg_start_push = 0; 1547 txmsg_end_push = 1601; 1548 err = test_exec(cgrp, &opt); 1549 if (err) 1550 goto out; 1551 1552 /* Test start/end with start > data */ 1553 txmsg_start = 1601; 1554 txmsg_end = 1600; 1555 txmsg_start_push = 1601; 1556 txmsg_end_push = 1600; 1557 err = test_exec(cgrp, &opt); 1558 if (err) 1559 goto out; 1560 1561 /* Test pop with start > data */ 1562 txmsg_start_pop = 1601; 1563 txmsg_pop = 1; 1564 err = test_exec(cgrp, &opt); 1565 if (err) 1566 goto out; 1567 1568 /* Test pop with pop range > data */ 1569 txmsg_start_pop = 1599; 1570 txmsg_pop = 10; 1571 err = test_exec(cgrp, &opt); 1572 out: 1573 txmsg_start = 0; 1574 txmsg_end = 0; 1575 sched_yield(); 1576 return err; 1577 } 1578 1579 char *map_names[] = { 1580 "sock_map", 1581 "sock_map_txmsg", 1582 "sock_map_redir", 1583 "sock_apply_bytes", 1584 "sock_cork_bytes", 1585 "sock_bytes", 1586 "sock_redir_flags", 1587 "sock_skb_opts", 1588 }; 1589 1590 int prog_attach_type[] = { 1591 BPF_SK_SKB_STREAM_PARSER, 1592 BPF_SK_SKB_STREAM_VERDICT, 1593 BPF_CGROUP_SOCK_OPS, 1594 BPF_SK_MSG_VERDICT, 1595 BPF_SK_MSG_VERDICT, 1596 BPF_SK_MSG_VERDICT, 1597 BPF_SK_MSG_VERDICT, 1598 BPF_SK_MSG_VERDICT, 1599 BPF_SK_MSG_VERDICT, 1600 BPF_SK_MSG_VERDICT, 1601 }; 1602 1603 int prog_type[] = { 1604 BPF_PROG_TYPE_SK_SKB, 1605 BPF_PROG_TYPE_SK_SKB, 1606 BPF_PROG_TYPE_SOCK_OPS, 1607 BPF_PROG_TYPE_SK_MSG, 1608 BPF_PROG_TYPE_SK_MSG, 1609 BPF_PROG_TYPE_SK_MSG, 1610 BPF_PROG_TYPE_SK_MSG, 1611 BPF_PROG_TYPE_SK_MSG, 1612 BPF_PROG_TYPE_SK_MSG, 1613 BPF_PROG_TYPE_SK_MSG, 1614 }; 1615 1616 static int populate_progs(char *bpf_file) 1617 { 1618 struct bpf_program *prog; 1619 struct bpf_object *obj; 1620 int i = 0; 1621 long err; 1622 1623 obj = bpf_object__open(bpf_file); 1624 err = libbpf_get_error(obj); 1625 if (err) { 1626 char err_buf[256]; 1627 1628 libbpf_strerror(err, err_buf, sizeof(err_buf)); 1629 printf("Unable to load eBPF objects in file '%s' : %s\n", 1630 bpf_file, err_buf); 1631 return -1; 1632 } 1633 1634 bpf_object__for_each_program(prog, obj) { 1635 bpf_program__set_type(prog, prog_type[i]); 1636 bpf_program__set_expected_attach_type(prog, 1637 prog_attach_type[i]); 1638 i++; 1639 } 1640 1641 i = bpf_object__load(obj); 1642 i = 0; 1643 bpf_object__for_each_program(prog, obj) { 1644 prog_fd[i] = bpf_program__fd(prog); 1645 i++; 1646 } 1647 1648 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) { 1649 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]); 1650 map_fd[i] = bpf_map__fd(maps[i]); 1651 if (map_fd[i] < 0) { 1652 fprintf(stderr, "load_bpf_file: (%i) %s\n", 1653 map_fd[i], strerror(errno)); 1654 return -1; 1655 } 1656 } 1657 1658 return 0; 1659 } 1660 1661 static int __test_suite(int cg_fd, char *bpf_file) 1662 { 1663 int err, cleanup = cg_fd; 1664 1665 err = populate_progs(bpf_file); 1666 if (err < 0) { 1667 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err); 1668 return err; 1669 } 1670 1671 if (cg_fd < 0) { 1672 if (setup_cgroup_environment()) { 1673 fprintf(stderr, "ERROR: cgroup env failed\n"); 1674 return -EINVAL; 1675 } 1676 1677 cg_fd = create_and_get_cgroup(CG_PATH); 1678 if (cg_fd < 0) { 1679 fprintf(stderr, 1680 "ERROR: (%i) open cg path failed: %s\n", 1681 cg_fd, optarg); 1682 return cg_fd; 1683 } 1684 1685 if (join_cgroup(CG_PATH)) { 1686 fprintf(stderr, "ERROR: failed to join cgroup\n"); 1687 return -EINVAL; 1688 } 1689 } 1690 1691 /* Tests basic commands and APIs with range of iov values */ 1692 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0; 1693 err = test_txmsg(cg_fd); 1694 if (err) 1695 goto out; 1696 1697 /* Tests interesting combinations of APIs used together */ 1698 err = test_mixed(cg_fd); 1699 if (err) 1700 goto out; 1701 1702 /* Tests pull_data API using start/end API */ 1703 err = test_start_end(cg_fd); 1704 if (err) 1705 goto out; 1706 1707 out: 1708 printf("Summary: %i PASSED %i FAILED\n", passed, failed); 1709 if (cleanup < 0) { 1710 cleanup_cgroup_environment(); 1711 close(cg_fd); 1712 } 1713 return err; 1714 } 1715 1716 static int test_suite(int cg_fd) 1717 { 1718 int err; 1719 1720 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME); 1721 if (err) 1722 goto out; 1723 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME); 1724 out: 1725 if (cg_fd > -1) 1726 close(cg_fd); 1727 return err; 1728 } 1729 1730 int main(int argc, char **argv) 1731 { 1732 int iov_count = 1, length = 1024, rate = 1; 1733 struct sockmap_options options = {0}; 1734 int opt, longindex, err, cg_fd = 0; 1735 char *bpf_file = BPF_SOCKMAP_FILENAME; 1736 int test = PING_PONG; 1737 1738 if (argc < 2) 1739 return test_suite(-1); 1740 1741 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:", 1742 long_options, &longindex)) != -1) { 1743 switch (opt) { 1744 case 's': 1745 txmsg_start = atoi(optarg); 1746 break; 1747 case 'e': 1748 txmsg_end = atoi(optarg); 1749 break; 1750 case 'p': 1751 txmsg_start_push = atoi(optarg); 1752 break; 1753 case 'q': 1754 txmsg_end_push = atoi(optarg); 1755 break; 1756 case 'w': 1757 txmsg_start_pop = atoi(optarg); 1758 break; 1759 case 'x': 1760 txmsg_pop = atoi(optarg); 1761 break; 1762 case 'a': 1763 txmsg_apply = atoi(optarg); 1764 break; 1765 case 'k': 1766 txmsg_cork = atoi(optarg); 1767 break; 1768 case 'c': 1769 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY); 1770 if (cg_fd < 0) { 1771 fprintf(stderr, 1772 "ERROR: (%i) open cg path failed: %s\n", 1773 cg_fd, optarg); 1774 return cg_fd; 1775 } 1776 break; 1777 case 'r': 1778 rate = atoi(optarg); 1779 break; 1780 case 'v': 1781 options.verbose = 1; 1782 break; 1783 case 'i': 1784 iov_count = atoi(optarg); 1785 break; 1786 case 'l': 1787 length = atoi(optarg); 1788 break; 1789 case 'd': 1790 options.data_test = true; 1791 break; 1792 case 't': 1793 if (strcmp(optarg, "ping") == 0) { 1794 test = PING_PONG; 1795 } else if (strcmp(optarg, "sendmsg") == 0) { 1796 test = SENDMSG; 1797 } else if (strcmp(optarg, "base") == 0) { 1798 test = BASE; 1799 } else if (strcmp(optarg, "base_sendpage") == 0) { 1800 test = BASE_SENDPAGE; 1801 } else if (strcmp(optarg, "sendpage") == 0) { 1802 test = SENDPAGE; 1803 } else { 1804 usage(argv); 1805 return -1; 1806 } 1807 break; 1808 case 0: 1809 break; 1810 case 'h': 1811 default: 1812 usage(argv); 1813 return -1; 1814 } 1815 } 1816 1817 if (argc <= 3 && cg_fd) 1818 return test_suite(cg_fd); 1819 1820 if (!cg_fd) { 1821 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n", 1822 argv[0]); 1823 return -1; 1824 } 1825 1826 err = populate_progs(bpf_file); 1827 if (err) { 1828 fprintf(stderr, "populate program: (%s) %s\n", 1829 bpf_file, strerror(errno)); 1830 return 1; 1831 } 1832 running = 1; 1833 1834 /* catch SIGINT */ 1835 signal(SIGINT, running_handler); 1836 1837 options.iov_count = iov_count; 1838 options.iov_length = length; 1839 options.rate = rate; 1840 1841 err = run_options(&options, cg_fd, test); 1842 close(cg_fd); 1843 return err; 1844 } 1845 1846 void running_handler(int a) 1847 { 1848 running = 0; 1849 } 1850