1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2020 Cloudflare 3 /* 4 * Test suite for SOCKMAP/SOCKHASH holding listening sockets. 5 * Covers: 6 * 1. BPF map operations - bpf_map_{update,lookup delete}_elem 7 * 2. BPF redirect helpers - bpf_{sk,msg}_redirect_map 8 * 3. BPF reuseport helper - bpf_sk_select_reuseport 9 */ 10 11 #include <linux/compiler.h> 12 #include <errno.h> 13 #include <error.h> 14 #include <limits.h> 15 #include <netinet/in.h> 16 #include <pthread.h> 17 #include <stdlib.h> 18 #include <string.h> 19 #include <sys/select.h> 20 #include <unistd.h> 21 22 #include <bpf/bpf.h> 23 #include <bpf/libbpf.h> 24 25 #include "bpf_util.h" 26 #include "test_progs.h" 27 #include "test_sockmap_listen.skel.h" 28 29 #define IO_TIMEOUT_SEC 30 30 #define MAX_STRERR_LEN 256 31 #define MAX_TEST_NAME 80 32 33 #define _FAIL(errnum, fmt...) \ 34 ({ \ 35 error_at_line(0, (errnum), __func__, __LINE__, fmt); \ 36 CHECK_FAIL(true); \ 37 }) 38 #define FAIL(fmt...) _FAIL(0, fmt) 39 #define FAIL_ERRNO(fmt...) _FAIL(errno, fmt) 40 #define FAIL_LIBBPF(err, msg) \ 41 ({ \ 42 char __buf[MAX_STRERR_LEN]; \ 43 libbpf_strerror((err), __buf, sizeof(__buf)); \ 44 FAIL("%s: %s", (msg), __buf); \ 45 }) 46 47 /* Wrappers that fail the test on error and report it. */ 48 49 #define xaccept_nonblock(fd, addr, len) \ 50 ({ \ 51 int __ret = \ 52 accept_timeout((fd), (addr), (len), IO_TIMEOUT_SEC); \ 53 if (__ret == -1) \ 54 FAIL_ERRNO("accept"); \ 55 __ret; \ 56 }) 57 58 #define xbind(fd, addr, len) \ 59 ({ \ 60 int __ret = bind((fd), (addr), (len)); \ 61 if (__ret == -1) \ 62 FAIL_ERRNO("bind"); \ 63 __ret; \ 64 }) 65 66 #define xclose(fd) \ 67 ({ \ 68 int __ret = close((fd)); \ 69 if (__ret == -1) \ 70 FAIL_ERRNO("close"); \ 71 __ret; \ 72 }) 73 74 #define xconnect(fd, addr, len) \ 75 ({ \ 76 int __ret = connect((fd), (addr), (len)); \ 77 if (__ret == -1) \ 78 FAIL_ERRNO("connect"); \ 79 __ret; \ 80 }) 81 82 #define xgetsockname(fd, addr, len) \ 83 ({ \ 84 int __ret = getsockname((fd), (addr), (len)); \ 85 if (__ret == -1) \ 86 FAIL_ERRNO("getsockname"); \ 87 __ret; \ 88 }) 89 90 #define xgetsockopt(fd, level, name, val, len) \ 91 ({ \ 92 int __ret = getsockopt((fd), (level), (name), (val), (len)); \ 93 if (__ret == -1) \ 94 FAIL_ERRNO("getsockopt(" #name ")"); \ 95 __ret; \ 96 }) 97 98 #define xlisten(fd, backlog) \ 99 ({ \ 100 int __ret = listen((fd), (backlog)); \ 101 if (__ret == -1) \ 102 FAIL_ERRNO("listen"); \ 103 __ret; \ 104 }) 105 106 #define xsetsockopt(fd, level, name, val, len) \ 107 ({ \ 108 int __ret = setsockopt((fd), (level), (name), (val), (len)); \ 109 if (__ret == -1) \ 110 FAIL_ERRNO("setsockopt(" #name ")"); \ 111 __ret; \ 112 }) 113 114 #define xsend(fd, buf, len, flags) \ 115 ({ \ 116 ssize_t __ret = send((fd), (buf), (len), (flags)); \ 117 if (__ret == -1) \ 118 FAIL_ERRNO("send"); \ 119 __ret; \ 120 }) 121 122 #define xrecv_nonblock(fd, buf, len, flags) \ 123 ({ \ 124 ssize_t __ret = recv_timeout((fd), (buf), (len), (flags), \ 125 IO_TIMEOUT_SEC); \ 126 if (__ret == -1) \ 127 FAIL_ERRNO("recv"); \ 128 __ret; \ 129 }) 130 131 #define xsocket(family, sotype, flags) \ 132 ({ \ 133 int __ret = socket(family, sotype, flags); \ 134 if (__ret == -1) \ 135 FAIL_ERRNO("socket"); \ 136 __ret; \ 137 }) 138 139 #define xbpf_map_delete_elem(fd, key) \ 140 ({ \ 141 int __ret = bpf_map_delete_elem((fd), (key)); \ 142 if (__ret == -1) \ 143 FAIL_ERRNO("map_delete"); \ 144 __ret; \ 145 }) 146 147 #define xbpf_map_lookup_elem(fd, key, val) \ 148 ({ \ 149 int __ret = bpf_map_lookup_elem((fd), (key), (val)); \ 150 if (__ret == -1) \ 151 FAIL_ERRNO("map_lookup"); \ 152 __ret; \ 153 }) 154 155 #define xbpf_map_update_elem(fd, key, val, flags) \ 156 ({ \ 157 int __ret = bpf_map_update_elem((fd), (key), (val), (flags)); \ 158 if (__ret == -1) \ 159 FAIL_ERRNO("map_update"); \ 160 __ret; \ 161 }) 162 163 #define xbpf_prog_attach(prog, target, type, flags) \ 164 ({ \ 165 int __ret = \ 166 bpf_prog_attach((prog), (target), (type), (flags)); \ 167 if (__ret == -1) \ 168 FAIL_ERRNO("prog_attach(" #type ")"); \ 169 __ret; \ 170 }) 171 172 #define xbpf_prog_detach2(prog, target, type) \ 173 ({ \ 174 int __ret = bpf_prog_detach2((prog), (target), (type)); \ 175 if (__ret == -1) \ 176 FAIL_ERRNO("prog_detach2(" #type ")"); \ 177 __ret; \ 178 }) 179 180 #define xpthread_create(thread, attr, func, arg) \ 181 ({ \ 182 int __ret = pthread_create((thread), (attr), (func), (arg)); \ 183 errno = __ret; \ 184 if (__ret) \ 185 FAIL_ERRNO("pthread_create"); \ 186 __ret; \ 187 }) 188 189 #define xpthread_join(thread, retval) \ 190 ({ \ 191 int __ret = pthread_join((thread), (retval)); \ 192 errno = __ret; \ 193 if (__ret) \ 194 FAIL_ERRNO("pthread_join"); \ 195 __ret; \ 196 }) 197 198 static int poll_read(int fd, unsigned int timeout_sec) 199 { 200 struct timeval timeout = { .tv_sec = timeout_sec }; 201 fd_set rfds; 202 int r; 203 204 FD_ZERO(&rfds); 205 FD_SET(fd, &rfds); 206 207 r = select(fd + 1, &rfds, NULL, NULL, &timeout); 208 if (r == 0) 209 errno = ETIME; 210 211 return r == 1 ? 0 : -1; 212 } 213 214 static int accept_timeout(int fd, struct sockaddr *addr, socklen_t *len, 215 unsigned int timeout_sec) 216 { 217 if (poll_read(fd, timeout_sec)) 218 return -1; 219 220 return accept(fd, addr, len); 221 } 222 223 static int recv_timeout(int fd, void *buf, size_t len, int flags, 224 unsigned int timeout_sec) 225 { 226 if (poll_read(fd, timeout_sec)) 227 return -1; 228 229 return recv(fd, buf, len, flags); 230 } 231 232 static void init_addr_loopback4(struct sockaddr_storage *ss, socklen_t *len) 233 { 234 struct sockaddr_in *addr4 = memset(ss, 0, sizeof(*ss)); 235 236 addr4->sin_family = AF_INET; 237 addr4->sin_port = 0; 238 addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 239 *len = sizeof(*addr4); 240 } 241 242 static void init_addr_loopback6(struct sockaddr_storage *ss, socklen_t *len) 243 { 244 struct sockaddr_in6 *addr6 = memset(ss, 0, sizeof(*ss)); 245 246 addr6->sin6_family = AF_INET6; 247 addr6->sin6_port = 0; 248 addr6->sin6_addr = in6addr_loopback; 249 *len = sizeof(*addr6); 250 } 251 252 static void init_addr_loopback(int family, struct sockaddr_storage *ss, 253 socklen_t *len) 254 { 255 switch (family) { 256 case AF_INET: 257 init_addr_loopback4(ss, len); 258 return; 259 case AF_INET6: 260 init_addr_loopback6(ss, len); 261 return; 262 default: 263 FAIL("unsupported address family %d", family); 264 } 265 } 266 267 static inline struct sockaddr *sockaddr(struct sockaddr_storage *ss) 268 { 269 return (struct sockaddr *)ss; 270 } 271 272 static int enable_reuseport(int s, int progfd) 273 { 274 int err, one = 1; 275 276 err = xsetsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); 277 if (err) 278 return -1; 279 err = xsetsockopt(s, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &progfd, 280 sizeof(progfd)); 281 if (err) 282 return -1; 283 284 return 0; 285 } 286 287 static int socket_loopback_reuseport(int family, int sotype, int progfd) 288 { 289 struct sockaddr_storage addr; 290 socklen_t len; 291 int err, s; 292 293 init_addr_loopback(family, &addr, &len); 294 295 s = xsocket(family, sotype, 0); 296 if (s == -1) 297 return -1; 298 299 if (progfd >= 0) 300 enable_reuseport(s, progfd); 301 302 err = xbind(s, sockaddr(&addr), len); 303 if (err) 304 goto close; 305 306 if (sotype & SOCK_DGRAM) 307 return s; 308 309 err = xlisten(s, SOMAXCONN); 310 if (err) 311 goto close; 312 313 return s; 314 close: 315 xclose(s); 316 return -1; 317 } 318 319 static int socket_loopback(int family, int sotype) 320 { 321 return socket_loopback_reuseport(family, sotype, -1); 322 } 323 324 static void test_insert_invalid(int family, int sotype, int mapfd) 325 { 326 u32 key = 0; 327 u64 value; 328 int err; 329 330 value = -1; 331 err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 332 if (!err || errno != EINVAL) 333 FAIL_ERRNO("map_update: expected EINVAL"); 334 335 value = INT_MAX; 336 err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 337 if (!err || errno != EBADF) 338 FAIL_ERRNO("map_update: expected EBADF"); 339 } 340 341 static void test_insert_opened(int family, int sotype, int mapfd) 342 { 343 u32 key = 0; 344 u64 value; 345 int err, s; 346 347 s = xsocket(family, sotype, 0); 348 if (s == -1) 349 return; 350 351 errno = 0; 352 value = s; 353 err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 354 if (!err || errno != EOPNOTSUPP) 355 FAIL_ERRNO("map_update: expected EOPNOTSUPP"); 356 357 xclose(s); 358 } 359 360 static void test_insert_bound(int family, int sotype, int mapfd) 361 { 362 struct sockaddr_storage addr; 363 socklen_t len; 364 u32 key = 0; 365 u64 value; 366 int err, s; 367 368 init_addr_loopback(family, &addr, &len); 369 370 s = xsocket(family, sotype, 0); 371 if (s == -1) 372 return; 373 374 err = xbind(s, sockaddr(&addr), len); 375 if (err) 376 goto close; 377 378 errno = 0; 379 value = s; 380 err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 381 if (!err || errno != EOPNOTSUPP) 382 FAIL_ERRNO("map_update: expected EOPNOTSUPP"); 383 close: 384 xclose(s); 385 } 386 387 static void test_insert(int family, int sotype, int mapfd) 388 { 389 u64 value; 390 u32 key; 391 int s; 392 393 s = socket_loopback(family, sotype); 394 if (s < 0) 395 return; 396 397 key = 0; 398 value = s; 399 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 400 xclose(s); 401 } 402 403 static void test_delete_after_insert(int family, int sotype, int mapfd) 404 { 405 u64 value; 406 u32 key; 407 int s; 408 409 s = socket_loopback(family, sotype); 410 if (s < 0) 411 return; 412 413 key = 0; 414 value = s; 415 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 416 xbpf_map_delete_elem(mapfd, &key); 417 xclose(s); 418 } 419 420 static void test_delete_after_close(int family, int sotype, int mapfd) 421 { 422 int err, s; 423 u64 value; 424 u32 key; 425 426 s = socket_loopback(family, sotype); 427 if (s < 0) 428 return; 429 430 key = 0; 431 value = s; 432 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 433 434 xclose(s); 435 436 errno = 0; 437 err = bpf_map_delete_elem(mapfd, &key); 438 if (!err || (errno != EINVAL && errno != ENOENT)) 439 /* SOCKMAP and SOCKHASH return different error codes */ 440 FAIL_ERRNO("map_delete: expected EINVAL/EINVAL"); 441 } 442 443 static void test_lookup_after_insert(int family, int sotype, int mapfd) 444 { 445 u64 cookie, value; 446 socklen_t len; 447 u32 key; 448 int s; 449 450 s = socket_loopback(family, sotype); 451 if (s < 0) 452 return; 453 454 key = 0; 455 value = s; 456 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 457 458 len = sizeof(cookie); 459 xgetsockopt(s, SOL_SOCKET, SO_COOKIE, &cookie, &len); 460 461 xbpf_map_lookup_elem(mapfd, &key, &value); 462 463 if (value != cookie) { 464 FAIL("map_lookup: have %#llx, want %#llx", 465 (unsigned long long)value, (unsigned long long)cookie); 466 } 467 468 xclose(s); 469 } 470 471 static void test_lookup_after_delete(int family, int sotype, int mapfd) 472 { 473 int err, s; 474 u64 value; 475 u32 key; 476 477 s = socket_loopback(family, sotype); 478 if (s < 0) 479 return; 480 481 key = 0; 482 value = s; 483 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 484 xbpf_map_delete_elem(mapfd, &key); 485 486 errno = 0; 487 err = bpf_map_lookup_elem(mapfd, &key, &value); 488 if (!err || errno != ENOENT) 489 FAIL_ERRNO("map_lookup: expected ENOENT"); 490 491 xclose(s); 492 } 493 494 static void test_lookup_32_bit_value(int family, int sotype, int mapfd) 495 { 496 u32 key, value32; 497 int err, s; 498 499 s = socket_loopback(family, sotype); 500 if (s < 0) 501 return; 502 503 mapfd = bpf_create_map(BPF_MAP_TYPE_SOCKMAP, sizeof(key), 504 sizeof(value32), 1, 0); 505 if (mapfd < 0) { 506 FAIL_ERRNO("map_create"); 507 goto close; 508 } 509 510 key = 0; 511 value32 = s; 512 xbpf_map_update_elem(mapfd, &key, &value32, BPF_NOEXIST); 513 514 errno = 0; 515 err = bpf_map_lookup_elem(mapfd, &key, &value32); 516 if (!err || errno != ENOSPC) 517 FAIL_ERRNO("map_lookup: expected ENOSPC"); 518 519 xclose(mapfd); 520 close: 521 xclose(s); 522 } 523 524 static void test_update_existing(int family, int sotype, int mapfd) 525 { 526 int s1, s2; 527 u64 value; 528 u32 key; 529 530 s1 = socket_loopback(family, sotype); 531 if (s1 < 0) 532 return; 533 534 s2 = socket_loopback(family, sotype); 535 if (s2 < 0) 536 goto close_s1; 537 538 key = 0; 539 value = s1; 540 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 541 542 value = s2; 543 xbpf_map_update_elem(mapfd, &key, &value, BPF_EXIST); 544 xclose(s2); 545 close_s1: 546 xclose(s1); 547 } 548 549 /* Exercise the code path where we destroy child sockets that never 550 * got accept()'ed, aka orphans, when parent socket gets closed. 551 */ 552 static void test_destroy_orphan_child(int family, int sotype, int mapfd) 553 { 554 struct sockaddr_storage addr; 555 socklen_t len; 556 int err, s, c; 557 u64 value; 558 u32 key; 559 560 s = socket_loopback(family, sotype); 561 if (s < 0) 562 return; 563 564 len = sizeof(addr); 565 err = xgetsockname(s, sockaddr(&addr), &len); 566 if (err) 567 goto close_srv; 568 569 key = 0; 570 value = s; 571 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 572 573 c = xsocket(family, sotype, 0); 574 if (c == -1) 575 goto close_srv; 576 577 xconnect(c, sockaddr(&addr), len); 578 xclose(c); 579 close_srv: 580 xclose(s); 581 } 582 583 /* Perform a passive open after removing listening socket from SOCKMAP 584 * to ensure that callbacks get restored properly. 585 */ 586 static void test_clone_after_delete(int family, int sotype, int mapfd) 587 { 588 struct sockaddr_storage addr; 589 socklen_t len; 590 int err, s, c; 591 u64 value; 592 u32 key; 593 594 s = socket_loopback(family, sotype); 595 if (s < 0) 596 return; 597 598 len = sizeof(addr); 599 err = xgetsockname(s, sockaddr(&addr), &len); 600 if (err) 601 goto close_srv; 602 603 key = 0; 604 value = s; 605 xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); 606 xbpf_map_delete_elem(mapfd, &key); 607 608 c = xsocket(family, sotype, 0); 609 if (c < 0) 610 goto close_srv; 611 612 xconnect(c, sockaddr(&addr), len); 613 xclose(c); 614 close_srv: 615 xclose(s); 616 } 617 618 /* Check that child socket that got created while parent was in a 619 * SOCKMAP, but got accept()'ed only after the parent has been removed 620 * from SOCKMAP, gets cloned without parent psock state or callbacks. 621 */ 622 static void test_accept_after_delete(int family, int sotype, int mapfd) 623 { 624 struct sockaddr_storage addr; 625 const u32 zero = 0; 626 int err, s, c, p; 627 socklen_t len; 628 u64 value; 629 630 s = socket_loopback(family, sotype | SOCK_NONBLOCK); 631 if (s == -1) 632 return; 633 634 len = sizeof(addr); 635 err = xgetsockname(s, sockaddr(&addr), &len); 636 if (err) 637 goto close_srv; 638 639 value = s; 640 err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST); 641 if (err) 642 goto close_srv; 643 644 c = xsocket(family, sotype, 0); 645 if (c == -1) 646 goto close_srv; 647 648 /* Create child while parent is in sockmap */ 649 err = xconnect(c, sockaddr(&addr), len); 650 if (err) 651 goto close_cli; 652 653 /* Remove parent from sockmap */ 654 err = xbpf_map_delete_elem(mapfd, &zero); 655 if (err) 656 goto close_cli; 657 658 p = xaccept_nonblock(s, NULL, NULL); 659 if (p == -1) 660 goto close_cli; 661 662 /* Check that child sk_user_data is not set */ 663 value = p; 664 xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST); 665 666 xclose(p); 667 close_cli: 668 xclose(c); 669 close_srv: 670 xclose(s); 671 } 672 673 /* Check that child socket that got created and accepted while parent 674 * was in a SOCKMAP is cloned without parent psock state or callbacks. 675 */ 676 static void test_accept_before_delete(int family, int sotype, int mapfd) 677 { 678 struct sockaddr_storage addr; 679 const u32 zero = 0, one = 1; 680 int err, s, c, p; 681 socklen_t len; 682 u64 value; 683 684 s = socket_loopback(family, sotype | SOCK_NONBLOCK); 685 if (s == -1) 686 return; 687 688 len = sizeof(addr); 689 err = xgetsockname(s, sockaddr(&addr), &len); 690 if (err) 691 goto close_srv; 692 693 value = s; 694 err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST); 695 if (err) 696 goto close_srv; 697 698 c = xsocket(family, sotype, 0); 699 if (c == -1) 700 goto close_srv; 701 702 /* Create & accept child while parent is in sockmap */ 703 err = xconnect(c, sockaddr(&addr), len); 704 if (err) 705 goto close_cli; 706 707 p = xaccept_nonblock(s, NULL, NULL); 708 if (p == -1) 709 goto close_cli; 710 711 /* Check that child sk_user_data is not set */ 712 value = p; 713 xbpf_map_update_elem(mapfd, &one, &value, BPF_NOEXIST); 714 715 xclose(p); 716 close_cli: 717 xclose(c); 718 close_srv: 719 xclose(s); 720 } 721 722 struct connect_accept_ctx { 723 int sockfd; 724 unsigned int done; 725 unsigned int nr_iter; 726 }; 727 728 static bool is_thread_done(struct connect_accept_ctx *ctx) 729 { 730 return READ_ONCE(ctx->done); 731 } 732 733 static void *connect_accept_thread(void *arg) 734 { 735 struct connect_accept_ctx *ctx = arg; 736 struct sockaddr_storage addr; 737 int family, socktype; 738 socklen_t len; 739 int err, i, s; 740 741 s = ctx->sockfd; 742 743 len = sizeof(addr); 744 err = xgetsockname(s, sockaddr(&addr), &len); 745 if (err) 746 goto done; 747 748 len = sizeof(family); 749 err = xgetsockopt(s, SOL_SOCKET, SO_DOMAIN, &family, &len); 750 if (err) 751 goto done; 752 753 len = sizeof(socktype); 754 err = xgetsockopt(s, SOL_SOCKET, SO_TYPE, &socktype, &len); 755 if (err) 756 goto done; 757 758 for (i = 0; i < ctx->nr_iter; i++) { 759 int c, p; 760 761 c = xsocket(family, socktype, 0); 762 if (c < 0) 763 break; 764 765 err = xconnect(c, (struct sockaddr *)&addr, sizeof(addr)); 766 if (err) { 767 xclose(c); 768 break; 769 } 770 771 p = xaccept_nonblock(s, NULL, NULL); 772 if (p < 0) { 773 xclose(c); 774 break; 775 } 776 777 xclose(p); 778 xclose(c); 779 } 780 done: 781 WRITE_ONCE(ctx->done, 1); 782 return NULL; 783 } 784 785 static void test_syn_recv_insert_delete(int family, int sotype, int mapfd) 786 { 787 struct connect_accept_ctx ctx = { 0 }; 788 struct sockaddr_storage addr; 789 socklen_t len; 790 u32 zero = 0; 791 pthread_t t; 792 int err, s; 793 u64 value; 794 795 s = socket_loopback(family, sotype | SOCK_NONBLOCK); 796 if (s < 0) 797 return; 798 799 len = sizeof(addr); 800 err = xgetsockname(s, sockaddr(&addr), &len); 801 if (err) 802 goto close; 803 804 ctx.sockfd = s; 805 ctx.nr_iter = 1000; 806 807 err = xpthread_create(&t, NULL, connect_accept_thread, &ctx); 808 if (err) 809 goto close; 810 811 value = s; 812 while (!is_thread_done(&ctx)) { 813 err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST); 814 if (err) 815 break; 816 817 err = xbpf_map_delete_elem(mapfd, &zero); 818 if (err) 819 break; 820 } 821 822 xpthread_join(t, NULL); 823 close: 824 xclose(s); 825 } 826 827 static void *listen_thread(void *arg) 828 { 829 struct sockaddr unspec = { AF_UNSPEC }; 830 struct connect_accept_ctx *ctx = arg; 831 int err, i, s; 832 833 s = ctx->sockfd; 834 835 for (i = 0; i < ctx->nr_iter; i++) { 836 err = xlisten(s, 1); 837 if (err) 838 break; 839 err = xconnect(s, &unspec, sizeof(unspec)); 840 if (err) 841 break; 842 } 843 844 WRITE_ONCE(ctx->done, 1); 845 return NULL; 846 } 847 848 static void test_race_insert_listen(int family, int socktype, int mapfd) 849 { 850 struct connect_accept_ctx ctx = { 0 }; 851 const u32 zero = 0; 852 const int one = 1; 853 pthread_t t; 854 int err, s; 855 u64 value; 856 857 s = xsocket(family, socktype, 0); 858 if (s < 0) 859 return; 860 861 err = xsetsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); 862 if (err) 863 goto close; 864 865 ctx.sockfd = s; 866 ctx.nr_iter = 10000; 867 868 err = pthread_create(&t, NULL, listen_thread, &ctx); 869 if (err) 870 goto close; 871 872 value = s; 873 while (!is_thread_done(&ctx)) { 874 err = bpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST); 875 /* Expecting EOPNOTSUPP before listen() */ 876 if (err && errno != EOPNOTSUPP) { 877 FAIL_ERRNO("map_update"); 878 break; 879 } 880 881 err = bpf_map_delete_elem(mapfd, &zero); 882 /* Expecting no entry after unhash on connect(AF_UNSPEC) */ 883 if (err && errno != EINVAL && errno != ENOENT) { 884 FAIL_ERRNO("map_delete"); 885 break; 886 } 887 } 888 889 xpthread_join(t, NULL); 890 close: 891 xclose(s); 892 } 893 894 static void zero_verdict_count(int mapfd) 895 { 896 unsigned int zero = 0; 897 int key; 898 899 key = SK_DROP; 900 xbpf_map_update_elem(mapfd, &key, &zero, BPF_ANY); 901 key = SK_PASS; 902 xbpf_map_update_elem(mapfd, &key, &zero, BPF_ANY); 903 } 904 905 enum redir_mode { 906 REDIR_INGRESS, 907 REDIR_EGRESS, 908 }; 909 910 static const char *redir_mode_str(enum redir_mode mode) 911 { 912 switch (mode) { 913 case REDIR_INGRESS: 914 return "ingress"; 915 case REDIR_EGRESS: 916 return "egress"; 917 default: 918 return "unknown"; 919 } 920 } 921 922 static void redir_to_connected(int family, int sotype, int sock_mapfd, 923 int verd_mapfd, enum redir_mode mode) 924 { 925 const char *log_prefix = redir_mode_str(mode); 926 struct sockaddr_storage addr; 927 int s, c0, c1, p0, p1; 928 unsigned int pass; 929 socklen_t len; 930 int err, n; 931 u64 value; 932 u32 key; 933 char b; 934 935 zero_verdict_count(verd_mapfd); 936 937 s = socket_loopback(family, sotype | SOCK_NONBLOCK); 938 if (s < 0) 939 return; 940 941 len = sizeof(addr); 942 err = xgetsockname(s, sockaddr(&addr), &len); 943 if (err) 944 goto close_srv; 945 946 c0 = xsocket(family, sotype, 0); 947 if (c0 < 0) 948 goto close_srv; 949 err = xconnect(c0, sockaddr(&addr), len); 950 if (err) 951 goto close_cli0; 952 953 p0 = xaccept_nonblock(s, NULL, NULL); 954 if (p0 < 0) 955 goto close_cli0; 956 957 c1 = xsocket(family, sotype, 0); 958 if (c1 < 0) 959 goto close_peer0; 960 err = xconnect(c1, sockaddr(&addr), len); 961 if (err) 962 goto close_cli1; 963 964 p1 = xaccept_nonblock(s, NULL, NULL); 965 if (p1 < 0) 966 goto close_cli1; 967 968 key = 0; 969 value = p0; 970 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 971 if (err) 972 goto close_peer1; 973 974 key = 1; 975 value = p1; 976 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 977 if (err) 978 goto close_peer1; 979 980 n = write(mode == REDIR_INGRESS ? c1 : p1, "a", 1); 981 if (n < 0) 982 FAIL_ERRNO("%s: write", log_prefix); 983 if (n == 0) 984 FAIL("%s: incomplete write", log_prefix); 985 if (n < 1) 986 goto close_peer1; 987 988 key = SK_PASS; 989 err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass); 990 if (err) 991 goto close_peer1; 992 if (pass != 1) 993 FAIL("%s: want pass count 1, have %d", log_prefix, pass); 994 995 n = read(c0, &b, 1); 996 if (n < 0) 997 FAIL_ERRNO("%s: read", log_prefix); 998 if (n == 0) 999 FAIL("%s: incomplete read", log_prefix); 1000 1001 close_peer1: 1002 xclose(p1); 1003 close_cli1: 1004 xclose(c1); 1005 close_peer0: 1006 xclose(p0); 1007 close_cli0: 1008 xclose(c0); 1009 close_srv: 1010 xclose(s); 1011 } 1012 1013 static void test_skb_redir_to_connected(struct test_sockmap_listen *skel, 1014 struct bpf_map *inner_map, int family, 1015 int sotype) 1016 { 1017 int verdict = bpf_program__fd(skel->progs.prog_stream_verdict); 1018 int parser = bpf_program__fd(skel->progs.prog_stream_parser); 1019 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1020 int sock_map = bpf_map__fd(inner_map); 1021 int err; 1022 1023 err = xbpf_prog_attach(parser, sock_map, BPF_SK_SKB_STREAM_PARSER, 0); 1024 if (err) 1025 return; 1026 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT, 0); 1027 if (err) 1028 goto detach; 1029 1030 redir_to_connected(family, sotype, sock_map, verdict_map, 1031 REDIR_INGRESS); 1032 1033 xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT); 1034 detach: 1035 xbpf_prog_detach2(parser, sock_map, BPF_SK_SKB_STREAM_PARSER); 1036 } 1037 1038 static void test_msg_redir_to_connected(struct test_sockmap_listen *skel, 1039 struct bpf_map *inner_map, int family, 1040 int sotype) 1041 { 1042 int verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 1043 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1044 int sock_map = bpf_map__fd(inner_map); 1045 int err; 1046 1047 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_MSG_VERDICT, 0); 1048 if (err) 1049 return; 1050 1051 redir_to_connected(family, sotype, sock_map, verdict_map, REDIR_EGRESS); 1052 1053 xbpf_prog_detach2(verdict, sock_map, BPF_SK_MSG_VERDICT); 1054 } 1055 1056 static void redir_to_listening(int family, int sotype, int sock_mapfd, 1057 int verd_mapfd, enum redir_mode mode) 1058 { 1059 const char *log_prefix = redir_mode_str(mode); 1060 struct sockaddr_storage addr; 1061 int s, c, p, err, n; 1062 unsigned int drop; 1063 socklen_t len; 1064 u64 value; 1065 u32 key; 1066 1067 zero_verdict_count(verd_mapfd); 1068 1069 s = socket_loopback(family, sotype | SOCK_NONBLOCK); 1070 if (s < 0) 1071 return; 1072 1073 len = sizeof(addr); 1074 err = xgetsockname(s, sockaddr(&addr), &len); 1075 if (err) 1076 goto close_srv; 1077 1078 c = xsocket(family, sotype, 0); 1079 if (c < 0) 1080 goto close_srv; 1081 err = xconnect(c, sockaddr(&addr), len); 1082 if (err) 1083 goto close_cli; 1084 1085 p = xaccept_nonblock(s, NULL, NULL); 1086 if (p < 0) 1087 goto close_cli; 1088 1089 key = 0; 1090 value = s; 1091 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 1092 if (err) 1093 goto close_peer; 1094 1095 key = 1; 1096 value = p; 1097 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 1098 if (err) 1099 goto close_peer; 1100 1101 n = write(mode == REDIR_INGRESS ? c : p, "a", 1); 1102 if (n < 0 && errno != EACCES) 1103 FAIL_ERRNO("%s: write", log_prefix); 1104 if (n == 0) 1105 FAIL("%s: incomplete write", log_prefix); 1106 if (n < 1) 1107 goto close_peer; 1108 1109 key = SK_DROP; 1110 err = xbpf_map_lookup_elem(verd_mapfd, &key, &drop); 1111 if (err) 1112 goto close_peer; 1113 if (drop != 1) 1114 FAIL("%s: want drop count 1, have %d", log_prefix, drop); 1115 1116 close_peer: 1117 xclose(p); 1118 close_cli: 1119 xclose(c); 1120 close_srv: 1121 xclose(s); 1122 } 1123 1124 static void test_skb_redir_to_listening(struct test_sockmap_listen *skel, 1125 struct bpf_map *inner_map, int family, 1126 int sotype) 1127 { 1128 int verdict = bpf_program__fd(skel->progs.prog_stream_verdict); 1129 int parser = bpf_program__fd(skel->progs.prog_stream_parser); 1130 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1131 int sock_map = bpf_map__fd(inner_map); 1132 int err; 1133 1134 err = xbpf_prog_attach(parser, sock_map, BPF_SK_SKB_STREAM_PARSER, 0); 1135 if (err) 1136 return; 1137 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT, 0); 1138 if (err) 1139 goto detach; 1140 1141 redir_to_listening(family, sotype, sock_map, verdict_map, 1142 REDIR_INGRESS); 1143 1144 xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT); 1145 detach: 1146 xbpf_prog_detach2(parser, sock_map, BPF_SK_SKB_STREAM_PARSER); 1147 } 1148 1149 static void test_msg_redir_to_listening(struct test_sockmap_listen *skel, 1150 struct bpf_map *inner_map, int family, 1151 int sotype) 1152 { 1153 int verdict = bpf_program__fd(skel->progs.prog_msg_verdict); 1154 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1155 int sock_map = bpf_map__fd(inner_map); 1156 int err; 1157 1158 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_MSG_VERDICT, 0); 1159 if (err) 1160 return; 1161 1162 redir_to_listening(family, sotype, sock_map, verdict_map, REDIR_EGRESS); 1163 1164 xbpf_prog_detach2(verdict, sock_map, BPF_SK_MSG_VERDICT); 1165 } 1166 1167 static void test_reuseport_select_listening(int family, int sotype, 1168 int sock_map, int verd_map, 1169 int reuseport_prog) 1170 { 1171 struct sockaddr_storage addr; 1172 unsigned int pass; 1173 int s, c, err; 1174 socklen_t len; 1175 u64 value; 1176 u32 key; 1177 1178 zero_verdict_count(verd_map); 1179 1180 s = socket_loopback_reuseport(family, sotype | SOCK_NONBLOCK, 1181 reuseport_prog); 1182 if (s < 0) 1183 return; 1184 1185 len = sizeof(addr); 1186 err = xgetsockname(s, sockaddr(&addr), &len); 1187 if (err) 1188 goto close_srv; 1189 1190 key = 0; 1191 value = s; 1192 err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST); 1193 if (err) 1194 goto close_srv; 1195 1196 c = xsocket(family, sotype, 0); 1197 if (c < 0) 1198 goto close_srv; 1199 err = xconnect(c, sockaddr(&addr), len); 1200 if (err) 1201 goto close_cli; 1202 1203 if (sotype == SOCK_STREAM) { 1204 int p; 1205 1206 p = xaccept_nonblock(s, NULL, NULL); 1207 if (p < 0) 1208 goto close_cli; 1209 xclose(p); 1210 } else { 1211 char b = 'a'; 1212 ssize_t n; 1213 1214 n = xsend(c, &b, sizeof(b), 0); 1215 if (n == -1) 1216 goto close_cli; 1217 1218 n = xrecv_nonblock(s, &b, sizeof(b), 0); 1219 if (n == -1) 1220 goto close_cli; 1221 } 1222 1223 key = SK_PASS; 1224 err = xbpf_map_lookup_elem(verd_map, &key, &pass); 1225 if (err) 1226 goto close_cli; 1227 if (pass != 1) 1228 FAIL("want pass count 1, have %d", pass); 1229 1230 close_cli: 1231 xclose(c); 1232 close_srv: 1233 xclose(s); 1234 } 1235 1236 static void test_reuseport_select_connected(int family, int sotype, 1237 int sock_map, int verd_map, 1238 int reuseport_prog) 1239 { 1240 struct sockaddr_storage addr; 1241 int s, c0, c1, p0, err; 1242 unsigned int drop; 1243 socklen_t len; 1244 u64 value; 1245 u32 key; 1246 1247 zero_verdict_count(verd_map); 1248 1249 s = socket_loopback_reuseport(family, sotype, reuseport_prog); 1250 if (s < 0) 1251 return; 1252 1253 /* Populate sock_map[0] to avoid ENOENT on first connection */ 1254 key = 0; 1255 value = s; 1256 err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST); 1257 if (err) 1258 goto close_srv; 1259 1260 len = sizeof(addr); 1261 err = xgetsockname(s, sockaddr(&addr), &len); 1262 if (err) 1263 goto close_srv; 1264 1265 c0 = xsocket(family, sotype, 0); 1266 if (c0 < 0) 1267 goto close_srv; 1268 1269 err = xconnect(c0, sockaddr(&addr), len); 1270 if (err) 1271 goto close_cli0; 1272 1273 if (sotype == SOCK_STREAM) { 1274 p0 = xaccept_nonblock(s, NULL, NULL); 1275 if (p0 < 0) 1276 goto close_cli0; 1277 } else { 1278 p0 = xsocket(family, sotype, 0); 1279 if (p0 < 0) 1280 goto close_cli0; 1281 1282 len = sizeof(addr); 1283 err = xgetsockname(c0, sockaddr(&addr), &len); 1284 if (err) 1285 goto close_cli0; 1286 1287 err = xconnect(p0, sockaddr(&addr), len); 1288 if (err) 1289 goto close_cli0; 1290 } 1291 1292 /* Update sock_map[0] to redirect to a connected socket */ 1293 key = 0; 1294 value = p0; 1295 err = xbpf_map_update_elem(sock_map, &key, &value, BPF_EXIST); 1296 if (err) 1297 goto close_peer0; 1298 1299 c1 = xsocket(family, sotype, 0); 1300 if (c1 < 0) 1301 goto close_peer0; 1302 1303 len = sizeof(addr); 1304 err = xgetsockname(s, sockaddr(&addr), &len); 1305 if (err) 1306 goto close_srv; 1307 1308 errno = 0; 1309 err = connect(c1, sockaddr(&addr), len); 1310 if (sotype == SOCK_DGRAM) { 1311 char b = 'a'; 1312 ssize_t n; 1313 1314 n = xsend(c1, &b, sizeof(b), 0); 1315 if (n == -1) 1316 goto close_cli1; 1317 1318 n = recv_timeout(c1, &b, sizeof(b), 0, IO_TIMEOUT_SEC); 1319 err = n == -1; 1320 } 1321 if (!err || errno != ECONNREFUSED) 1322 FAIL_ERRNO("connect: expected ECONNREFUSED"); 1323 1324 key = SK_DROP; 1325 err = xbpf_map_lookup_elem(verd_map, &key, &drop); 1326 if (err) 1327 goto close_cli1; 1328 if (drop != 1) 1329 FAIL("want drop count 1, have %d", drop); 1330 1331 close_cli1: 1332 xclose(c1); 1333 close_peer0: 1334 xclose(p0); 1335 close_cli0: 1336 xclose(c0); 1337 close_srv: 1338 xclose(s); 1339 } 1340 1341 /* Check that redirecting across reuseport groups is not allowed. */ 1342 static void test_reuseport_mixed_groups(int family, int sotype, int sock_map, 1343 int verd_map, int reuseport_prog) 1344 { 1345 struct sockaddr_storage addr; 1346 int s1, s2, c, err; 1347 unsigned int drop; 1348 socklen_t len; 1349 u64 value; 1350 u32 key; 1351 1352 zero_verdict_count(verd_map); 1353 1354 /* Create two listeners, each in its own reuseport group */ 1355 s1 = socket_loopback_reuseport(family, sotype, reuseport_prog); 1356 if (s1 < 0) 1357 return; 1358 1359 s2 = socket_loopback_reuseport(family, sotype, reuseport_prog); 1360 if (s2 < 0) 1361 goto close_srv1; 1362 1363 key = 0; 1364 value = s1; 1365 err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST); 1366 if (err) 1367 goto close_srv2; 1368 1369 key = 1; 1370 value = s2; 1371 err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST); 1372 1373 /* Connect to s2, reuseport BPF selects s1 via sock_map[0] */ 1374 len = sizeof(addr); 1375 err = xgetsockname(s2, sockaddr(&addr), &len); 1376 if (err) 1377 goto close_srv2; 1378 1379 c = xsocket(family, sotype, 0); 1380 if (c < 0) 1381 goto close_srv2; 1382 1383 err = connect(c, sockaddr(&addr), len); 1384 if (sotype == SOCK_DGRAM) { 1385 char b = 'a'; 1386 ssize_t n; 1387 1388 n = xsend(c, &b, sizeof(b), 0); 1389 if (n == -1) 1390 goto close_cli; 1391 1392 n = recv_timeout(c, &b, sizeof(b), 0, IO_TIMEOUT_SEC); 1393 err = n == -1; 1394 } 1395 if (!err || errno != ECONNREFUSED) { 1396 FAIL_ERRNO("connect: expected ECONNREFUSED"); 1397 goto close_cli; 1398 } 1399 1400 /* Expect drop, can't redirect outside of reuseport group */ 1401 key = SK_DROP; 1402 err = xbpf_map_lookup_elem(verd_map, &key, &drop); 1403 if (err) 1404 goto close_cli; 1405 if (drop != 1) 1406 FAIL("want drop count 1, have %d", drop); 1407 1408 close_cli: 1409 xclose(c); 1410 close_srv2: 1411 xclose(s2); 1412 close_srv1: 1413 xclose(s1); 1414 } 1415 1416 #define TEST(fn, ...) \ 1417 { \ 1418 fn, #fn, __VA_ARGS__ \ 1419 } 1420 1421 static void test_ops_cleanup(const struct bpf_map *map) 1422 { 1423 const struct bpf_map_def *def; 1424 int err, mapfd; 1425 u32 key; 1426 1427 def = bpf_map__def(map); 1428 mapfd = bpf_map__fd(map); 1429 1430 for (key = 0; key < def->max_entries; key++) { 1431 err = bpf_map_delete_elem(mapfd, &key); 1432 if (err && errno != EINVAL && errno != ENOENT) 1433 FAIL_ERRNO("map_delete: expected EINVAL/ENOENT"); 1434 } 1435 } 1436 1437 static const char *family_str(sa_family_t family) 1438 { 1439 switch (family) { 1440 case AF_INET: 1441 return "IPv4"; 1442 case AF_INET6: 1443 return "IPv6"; 1444 default: 1445 return "unknown"; 1446 } 1447 } 1448 1449 static const char *map_type_str(const struct bpf_map *map) 1450 { 1451 const struct bpf_map_def *def; 1452 1453 def = bpf_map__def(map); 1454 if (IS_ERR(def)) 1455 return "invalid"; 1456 1457 switch (def->type) { 1458 case BPF_MAP_TYPE_SOCKMAP: 1459 return "sockmap"; 1460 case BPF_MAP_TYPE_SOCKHASH: 1461 return "sockhash"; 1462 default: 1463 return "unknown"; 1464 } 1465 } 1466 1467 static const char *sotype_str(int sotype) 1468 { 1469 switch (sotype) { 1470 case SOCK_DGRAM: 1471 return "UDP"; 1472 case SOCK_STREAM: 1473 return "TCP"; 1474 default: 1475 return "unknown"; 1476 } 1477 } 1478 1479 static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, 1480 int family, int sotype) 1481 { 1482 const struct op_test { 1483 void (*fn)(int family, int sotype, int mapfd); 1484 const char *name; 1485 int sotype; 1486 } tests[] = { 1487 /* insert */ 1488 TEST(test_insert_invalid), 1489 TEST(test_insert_opened), 1490 TEST(test_insert_bound, SOCK_STREAM), 1491 TEST(test_insert), 1492 /* delete */ 1493 TEST(test_delete_after_insert), 1494 TEST(test_delete_after_close), 1495 /* lookup */ 1496 TEST(test_lookup_after_insert), 1497 TEST(test_lookup_after_delete), 1498 TEST(test_lookup_32_bit_value), 1499 /* update */ 1500 TEST(test_update_existing), 1501 /* races with insert/delete */ 1502 TEST(test_destroy_orphan_child, SOCK_STREAM), 1503 TEST(test_syn_recv_insert_delete, SOCK_STREAM), 1504 TEST(test_race_insert_listen, SOCK_STREAM), 1505 /* child clone */ 1506 TEST(test_clone_after_delete, SOCK_STREAM), 1507 TEST(test_accept_after_delete, SOCK_STREAM), 1508 TEST(test_accept_before_delete, SOCK_STREAM), 1509 }; 1510 const char *family_name, *map_name, *sotype_name; 1511 const struct op_test *t; 1512 char s[MAX_TEST_NAME]; 1513 int map_fd; 1514 1515 family_name = family_str(family); 1516 map_name = map_type_str(map); 1517 sotype_name = sotype_str(sotype); 1518 map_fd = bpf_map__fd(map); 1519 1520 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 1521 snprintf(s, sizeof(s), "%s %s %s %s", map_name, family_name, 1522 sotype_name, t->name); 1523 1524 if (t->sotype != 0 && t->sotype != sotype) 1525 continue; 1526 1527 if (!test__start_subtest(s)) 1528 continue; 1529 1530 t->fn(family, sotype, map_fd); 1531 test_ops_cleanup(map); 1532 } 1533 } 1534 1535 static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map, 1536 int family, int sotype) 1537 { 1538 const struct redir_test { 1539 void (*fn)(struct test_sockmap_listen *skel, 1540 struct bpf_map *map, int family, int sotype); 1541 const char *name; 1542 } tests[] = { 1543 TEST(test_skb_redir_to_connected), 1544 TEST(test_skb_redir_to_listening), 1545 TEST(test_msg_redir_to_connected), 1546 TEST(test_msg_redir_to_listening), 1547 }; 1548 const char *family_name, *map_name; 1549 const struct redir_test *t; 1550 char s[MAX_TEST_NAME]; 1551 1552 family_name = family_str(family); 1553 map_name = map_type_str(map); 1554 1555 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 1556 snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, 1557 t->name); 1558 1559 if (!test__start_subtest(s)) 1560 continue; 1561 1562 t->fn(skel, map, family, sotype); 1563 } 1564 } 1565 1566 static void test_reuseport(struct test_sockmap_listen *skel, 1567 struct bpf_map *map, int family, int sotype) 1568 { 1569 const struct reuseport_test { 1570 void (*fn)(int family, int sotype, int socket_map, 1571 int verdict_map, int reuseport_prog); 1572 const char *name; 1573 int sotype; 1574 } tests[] = { 1575 TEST(test_reuseport_select_listening), 1576 TEST(test_reuseport_select_connected), 1577 TEST(test_reuseport_mixed_groups), 1578 }; 1579 int socket_map, verdict_map, reuseport_prog; 1580 const char *family_name, *map_name, *sotype_name; 1581 const struct reuseport_test *t; 1582 char s[MAX_TEST_NAME]; 1583 1584 family_name = family_str(family); 1585 map_name = map_type_str(map); 1586 sotype_name = sotype_str(sotype); 1587 1588 socket_map = bpf_map__fd(map); 1589 verdict_map = bpf_map__fd(skel->maps.verdict_map); 1590 reuseport_prog = bpf_program__fd(skel->progs.prog_reuseport); 1591 1592 for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { 1593 snprintf(s, sizeof(s), "%s %s %s %s", map_name, family_name, 1594 sotype_name, t->name); 1595 1596 if (t->sotype != 0 && t->sotype != sotype) 1597 continue; 1598 1599 if (!test__start_subtest(s)) 1600 continue; 1601 1602 t->fn(family, sotype, socket_map, verdict_map, reuseport_prog); 1603 } 1604 } 1605 1606 static void udp_redir_to_connected(int family, int sotype, int sock_mapfd, 1607 int verd_mapfd, enum redir_mode mode) 1608 { 1609 const char *log_prefix = redir_mode_str(mode); 1610 struct sockaddr_storage addr; 1611 int c0, c1, p0, p1; 1612 unsigned int pass; 1613 socklen_t len; 1614 int err, n; 1615 u64 value; 1616 u32 key; 1617 char b; 1618 1619 zero_verdict_count(verd_mapfd); 1620 1621 p0 = socket_loopback(family, sotype | SOCK_NONBLOCK); 1622 if (p0 < 0) 1623 return; 1624 len = sizeof(addr); 1625 err = xgetsockname(p0, sockaddr(&addr), &len); 1626 if (err) 1627 goto close_peer0; 1628 1629 c0 = xsocket(family, sotype | SOCK_NONBLOCK, 0); 1630 if (c0 < 0) 1631 goto close_peer0; 1632 err = xconnect(c0, sockaddr(&addr), len); 1633 if (err) 1634 goto close_cli0; 1635 err = xgetsockname(c0, sockaddr(&addr), &len); 1636 if (err) 1637 goto close_cli0; 1638 err = xconnect(p0, sockaddr(&addr), len); 1639 if (err) 1640 goto close_cli0; 1641 1642 p1 = socket_loopback(family, sotype | SOCK_NONBLOCK); 1643 if (p1 < 0) 1644 goto close_cli0; 1645 err = xgetsockname(p1, sockaddr(&addr), &len); 1646 if (err) 1647 goto close_cli0; 1648 1649 c1 = xsocket(family, sotype | SOCK_NONBLOCK, 0); 1650 if (c1 < 0) 1651 goto close_peer1; 1652 err = xconnect(c1, sockaddr(&addr), len); 1653 if (err) 1654 goto close_cli1; 1655 err = xgetsockname(c1, sockaddr(&addr), &len); 1656 if (err) 1657 goto close_cli1; 1658 err = xconnect(p1, sockaddr(&addr), len); 1659 if (err) 1660 goto close_cli1; 1661 1662 key = 0; 1663 value = p0; 1664 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 1665 if (err) 1666 goto close_cli1; 1667 1668 key = 1; 1669 value = p1; 1670 err = xbpf_map_update_elem(sock_mapfd, &key, &value, BPF_NOEXIST); 1671 if (err) 1672 goto close_cli1; 1673 1674 n = write(c1, "a", 1); 1675 if (n < 0) 1676 FAIL_ERRNO("%s: write", log_prefix); 1677 if (n == 0) 1678 FAIL("%s: incomplete write", log_prefix); 1679 if (n < 1) 1680 goto close_cli1; 1681 1682 key = SK_PASS; 1683 err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass); 1684 if (err) 1685 goto close_cli1; 1686 if (pass != 1) 1687 FAIL("%s: want pass count 1, have %d", log_prefix, pass); 1688 1689 n = read(mode == REDIR_INGRESS ? p0 : c0, &b, 1); 1690 if (n < 0) 1691 FAIL_ERRNO("%s: read", log_prefix); 1692 if (n == 0) 1693 FAIL("%s: incomplete read", log_prefix); 1694 1695 close_cli1: 1696 xclose(c1); 1697 close_peer1: 1698 xclose(p1); 1699 close_cli0: 1700 xclose(c0); 1701 close_peer0: 1702 xclose(p0); 1703 } 1704 1705 static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel, 1706 struct bpf_map *inner_map, int family) 1707 { 1708 int verdict = bpf_program__fd(skel->progs.prog_skb_verdict); 1709 int verdict_map = bpf_map__fd(skel->maps.verdict_map); 1710 int sock_map = bpf_map__fd(inner_map); 1711 int err; 1712 1713 err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0); 1714 if (err) 1715 return; 1716 1717 skel->bss->test_ingress = false; 1718 udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, 1719 REDIR_EGRESS); 1720 skel->bss->test_ingress = true; 1721 udp_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, 1722 REDIR_INGRESS); 1723 1724 xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT); 1725 } 1726 1727 static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map, 1728 int family) 1729 { 1730 const char *family_name, *map_name; 1731 char s[MAX_TEST_NAME]; 1732 1733 family_name = family_str(family); 1734 map_name = map_type_str(map); 1735 snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__); 1736 if (!test__start_subtest(s)) 1737 return; 1738 udp_skb_redir_to_connected(skel, map, family); 1739 } 1740 1741 static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map, 1742 int family) 1743 { 1744 test_ops(skel, map, family, SOCK_STREAM); 1745 test_ops(skel, map, family, SOCK_DGRAM); 1746 test_redir(skel, map, family, SOCK_STREAM); 1747 test_reuseport(skel, map, family, SOCK_STREAM); 1748 test_reuseport(skel, map, family, SOCK_DGRAM); 1749 test_udp_redir(skel, map, family); 1750 } 1751 1752 void test_sockmap_listen(void) 1753 { 1754 struct test_sockmap_listen *skel; 1755 1756 skel = test_sockmap_listen__open_and_load(); 1757 if (!skel) { 1758 FAIL("skeleton open/load failed"); 1759 return; 1760 } 1761 1762 skel->bss->test_sockmap = true; 1763 run_tests(skel, skel->maps.sock_map, AF_INET); 1764 run_tests(skel, skel->maps.sock_map, AF_INET6); 1765 1766 skel->bss->test_sockmap = false; 1767 run_tests(skel, skel->maps.sock_hash, AF_INET); 1768 run_tests(skel, skel->maps.sock_hash, AF_INET6); 1769 1770 test_sockmap_listen__destroy(skel); 1771 } 1772