1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * 53 * Total tests: 12 54 * 55 * Flow: 56 * ----- 57 * - Single process spawns two threads: Tx and Rx 58 * - Each of these two threads attach to a veth interface within their assigned 59 * namespaces 60 * - Each thread Creates one AF_XDP socket connected to a unique umem for each 61 * veth interface 62 * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy> 63 * - Rx thread verifies if all 10k packets were received and delivered in-order, 64 * and have the right content 65 * 66 * Enable/disable packet dump mode: 67 * -------------------------- 68 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 69 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 70 */ 71 72 #define _GNU_SOURCE 73 #include <fcntl.h> 74 #include <errno.h> 75 #include <getopt.h> 76 #include <asm/barrier.h> 77 #include <linux/if_link.h> 78 #include <linux/if_ether.h> 79 #include <linux/ip.h> 80 #include <linux/udp.h> 81 #include <arpa/inet.h> 82 #include <net/if.h> 83 #include <locale.h> 84 #include <poll.h> 85 #include <pthread.h> 86 #include <signal.h> 87 #include <stdbool.h> 88 #include <stdio.h> 89 #include <stdlib.h> 90 #include <string.h> 91 #include <stddef.h> 92 #include <sys/mman.h> 93 #include <sys/socket.h> 94 #include <sys/time.h> 95 #include <sys/types.h> 96 #include <sys/queue.h> 97 #include <time.h> 98 #include <unistd.h> 99 #include <stdatomic.h> 100 #include "xsk.h" 101 #include "xskxceiver.h" 102 #include "../kselftest.h" 103 104 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf. 105 * Until xskxceiver is either moved or re-writed into libxdp, suppress 106 * deprecation warnings in this file 107 */ 108 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 109 110 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62"; 111 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61"; 112 static const char *IP1 = "192.168.100.162"; 113 static const char *IP2 = "192.168.100.161"; 114 static const u16 UDP_PORT1 = 2020; 115 static const u16 UDP_PORT2 = 2121; 116 117 static void __exit_with_error(int error, const char *file, const char *func, int line) 118 { 119 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, 120 strerror(error)); 121 ksft_exit_xfail(); 122 } 123 124 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 125 126 #define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV" 127 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 128 129 static void report_failure(struct test_spec *test) 130 { 131 if (test->fail) 132 return; 133 134 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 135 test->name); 136 test->fail = true; 137 } 138 139 static void memset32_htonl(void *dest, u32 val, u32 size) 140 { 141 u32 *ptr = (u32 *)dest; 142 int i; 143 144 val = htonl(val); 145 146 for (i = 0; i < (size & (~0x3)); i += 4) 147 ptr[i >> 2] = val; 148 } 149 150 /* 151 * Fold a partial checksum 152 * This function code has been taken from 153 * Linux kernel include/asm-generic/checksum.h 154 */ 155 static __u16 csum_fold(__u32 csum) 156 { 157 u32 sum = (__force u32)csum; 158 159 sum = (sum & 0xffff) + (sum >> 16); 160 sum = (sum & 0xffff) + (sum >> 16); 161 return (__force __u16)~sum; 162 } 163 164 /* 165 * This function code has been taken from 166 * Linux kernel lib/checksum.c 167 */ 168 static u32 from64to32(u64 x) 169 { 170 /* add up 32-bit and 32-bit for 32+c bit */ 171 x = (x & 0xffffffff) + (x >> 32); 172 /* add up carry.. */ 173 x = (x & 0xffffffff) + (x >> 32); 174 return (u32)x; 175 } 176 177 /* 178 * This function code has been taken from 179 * Linux kernel lib/checksum.c 180 */ 181 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 182 { 183 unsigned long long s = (__force u32)sum; 184 185 s += (__force u32)saddr; 186 s += (__force u32)daddr; 187 #ifdef __BIG_ENDIAN__ 188 s += proto + len; 189 #else 190 s += (proto + len) << 8; 191 #endif 192 return (__force __u32)from64to32(s); 193 } 194 195 /* 196 * This function has been taken from 197 * Linux kernel include/asm-generic/checksum.h 198 */ 199 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 200 { 201 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); 202 } 203 204 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt) 205 { 206 u32 csum = 0; 207 u32 cnt = 0; 208 209 /* udp hdr and data */ 210 for (; cnt < len; cnt += 2) 211 csum += udp_pkt[cnt >> 1]; 212 213 return csum_tcpudp_magic(saddr, daddr, len, proto, csum); 214 } 215 216 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr) 217 { 218 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN); 219 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN); 220 eth_hdr->h_proto = htons(ETH_P_IP); 221 } 222 223 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr) 224 { 225 ip_hdr->version = IP_PKT_VER; 226 ip_hdr->ihl = 0x5; 227 ip_hdr->tos = IP_PKT_TOS; 228 ip_hdr->tot_len = htons(IP_PKT_SIZE); 229 ip_hdr->id = 0; 230 ip_hdr->frag_off = 0; 231 ip_hdr->ttl = IPDEFTTL; 232 ip_hdr->protocol = IPPROTO_UDP; 233 ip_hdr->saddr = ifobject->src_ip; 234 ip_hdr->daddr = ifobject->dst_ip; 235 ip_hdr->check = 0; 236 } 237 238 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject, 239 struct udphdr *udp_hdr) 240 { 241 udp_hdr->source = htons(ifobject->src_port); 242 udp_hdr->dest = htons(ifobject->dst_port); 243 udp_hdr->len = htons(UDP_PKT_SIZE); 244 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE); 245 } 246 247 static bool is_umem_valid(struct ifobject *ifobj) 248 { 249 return !!ifobj->umem->umem; 250 } 251 252 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr) 253 { 254 udp_hdr->check = 0; 255 udp_hdr->check = 256 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr); 257 } 258 259 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size) 260 { 261 struct xsk_umem_config cfg = { 262 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 263 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 264 .frame_size = umem->frame_size, 265 .frame_headroom = umem->frame_headroom, 266 .flags = XSK_UMEM__DEFAULT_FLAGS 267 }; 268 int ret; 269 270 if (umem->unaligned_mode) 271 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; 272 273 ret = xsk_umem__create(&umem->umem, buffer, size, 274 &umem->fq, &umem->cq, &cfg); 275 if (ret) 276 return ret; 277 278 umem->buffer = buffer; 279 return 0; 280 } 281 282 static void enable_busy_poll(struct xsk_socket_info *xsk) 283 { 284 int sock_opt; 285 286 sock_opt = 1; 287 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, 288 (void *)&sock_opt, sizeof(sock_opt)) < 0) 289 exit_with_error(errno); 290 291 sock_opt = 20; 292 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, 293 (void *)&sock_opt, sizeof(sock_opt)) < 0) 294 exit_with_error(errno); 295 296 sock_opt = BATCH_SIZE; 297 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, 298 (void *)&sock_opt, sizeof(sock_opt)) < 0) 299 exit_with_error(errno); 300 } 301 302 static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 303 struct ifobject *ifobject, bool shared) 304 { 305 struct xsk_socket_config cfg = {}; 306 struct xsk_ring_cons *rxr; 307 struct xsk_ring_prod *txr; 308 309 xsk->umem = umem; 310 cfg.rx_size = xsk->rxqsize; 311 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 312 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; 313 cfg.xdp_flags = ifobject->xdp_flags; 314 cfg.bind_flags = ifobject->bind_flags; 315 if (shared) 316 cfg.bind_flags |= XDP_SHARED_UMEM; 317 318 txr = ifobject->tx_on ? &xsk->tx : NULL; 319 rxr = ifobject->rx_on ? &xsk->rx : NULL; 320 return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg); 321 } 322 323 static struct option long_options[] = { 324 {"interface", required_argument, 0, 'i'}, 325 {"busy-poll", no_argument, 0, 'b'}, 326 {"dump-pkts", no_argument, 0, 'D'}, 327 {"verbose", no_argument, 0, 'v'}, 328 {0, 0, 0, 0} 329 }; 330 331 static void usage(const char *prog) 332 { 333 const char *str = 334 " Usage: %s [OPTIONS]\n" 335 " Options:\n" 336 " -i, --interface Use interface\n" 337 " -D, --dump-pkts Dump packets L2 - L5\n" 338 " -v, --verbose Verbose output\n" 339 " -b, --busy-poll Enable busy poll\n"; 340 341 ksft_print_msg(str, prog); 342 } 343 344 static int switch_namespace(const char *nsname) 345 { 346 char fqns[26] = "/var/run/netns/"; 347 int nsfd; 348 349 if (!nsname || strlen(nsname) == 0) 350 return -1; 351 352 strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1); 353 nsfd = open(fqns, O_RDONLY); 354 355 if (nsfd == -1) 356 exit_with_error(errno); 357 358 if (setns(nsfd, 0) == -1) 359 exit_with_error(errno); 360 361 print_verbose("NS switched: %s\n", nsname); 362 363 return nsfd; 364 } 365 366 static bool validate_interface(struct ifobject *ifobj) 367 { 368 if (!strcmp(ifobj->ifname, "")) 369 return false; 370 return true; 371 } 372 373 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 374 char **argv) 375 { 376 struct ifobject *ifobj; 377 u32 interface_nb = 0; 378 int option_index, c; 379 380 opterr = 0; 381 382 for (;;) { 383 char *sptr, *token; 384 385 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index); 386 if (c == -1) 387 break; 388 389 switch (c) { 390 case 'i': 391 if (interface_nb == 0) 392 ifobj = ifobj_tx; 393 else if (interface_nb == 1) 394 ifobj = ifobj_rx; 395 else 396 break; 397 398 sptr = strndupa(optarg, strlen(optarg)); 399 memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS); 400 token = strsep(&sptr, ","); 401 if (token) 402 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS); 403 interface_nb++; 404 break; 405 case 'D': 406 opt_pkt_dump = true; 407 break; 408 case 'v': 409 opt_verbose = true; 410 break; 411 case 'b': 412 ifobj_tx->busy_poll = true; 413 ifobj_rx->busy_poll = true; 414 break; 415 default: 416 usage(basename(argv[0])); 417 ksft_exit_xfail(); 418 } 419 } 420 } 421 422 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 423 struct ifobject *ifobj_rx) 424 { 425 u32 i, j; 426 427 for (i = 0; i < MAX_INTERFACES; i++) { 428 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 429 430 ifobj->xsk = &ifobj->xsk_arr[0]; 431 ifobj->use_poll = false; 432 ifobj->use_fill_ring = true; 433 ifobj->release_rx = true; 434 ifobj->pkt_stream = test->pkt_stream_default; 435 ifobj->validation_func = NULL; 436 437 if (i == 0) { 438 ifobj->rx_on = false; 439 ifobj->tx_on = true; 440 } else { 441 ifobj->rx_on = true; 442 ifobj->tx_on = false; 443 } 444 445 memset(ifobj->umem, 0, sizeof(*ifobj->umem)); 446 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; 447 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 448 449 for (j = 0; j < MAX_SOCKETS; j++) { 450 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); 451 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 452 } 453 } 454 455 test->ifobj_tx = ifobj_tx; 456 test->ifobj_rx = ifobj_rx; 457 test->current_step = 0; 458 test->total_steps = 1; 459 test->nb_sockets = 1; 460 test->fail = false; 461 } 462 463 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 464 struct ifobject *ifobj_rx, enum test_mode mode) 465 { 466 struct pkt_stream *pkt_stream; 467 u32 i; 468 469 pkt_stream = test->pkt_stream_default; 470 memset(test, 0, sizeof(*test)); 471 test->pkt_stream_default = pkt_stream; 472 473 for (i = 0; i < MAX_INTERFACES; i++) { 474 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 475 476 ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 477 if (mode == TEST_MODE_SKB) 478 ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE; 479 else 480 ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE; 481 482 ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY; 483 } 484 485 __test_spec_init(test, ifobj_tx, ifobj_rx); 486 } 487 488 static void test_spec_reset(struct test_spec *test) 489 { 490 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 491 } 492 493 static void test_spec_set_name(struct test_spec *test, const char *name) 494 { 495 strncpy(test->name, name, MAX_TEST_NAME_SIZE); 496 } 497 498 static void pkt_stream_reset(struct pkt_stream *pkt_stream) 499 { 500 if (pkt_stream) 501 pkt_stream->rx_pkt_nb = 0; 502 } 503 504 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb) 505 { 506 if (pkt_nb >= pkt_stream->nb_pkts) 507 return NULL; 508 509 return &pkt_stream->pkts[pkt_nb]; 510 } 511 512 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) 513 { 514 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) { 515 (*pkts_sent)++; 516 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid) 517 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++]; 518 pkt_stream->rx_pkt_nb++; 519 } 520 return NULL; 521 } 522 523 static void pkt_stream_delete(struct pkt_stream *pkt_stream) 524 { 525 free(pkt_stream->pkts); 526 free(pkt_stream); 527 } 528 529 static void pkt_stream_restore_default(struct test_spec *test) 530 { 531 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream; 532 533 if (tx_pkt_stream != test->pkt_stream_default) { 534 pkt_stream_delete(test->ifobj_tx->pkt_stream); 535 test->ifobj_tx->pkt_stream = test->pkt_stream_default; 536 } 537 538 if (test->ifobj_rx->pkt_stream != test->pkt_stream_default && 539 test->ifobj_rx->pkt_stream != tx_pkt_stream) 540 pkt_stream_delete(test->ifobj_rx->pkt_stream); 541 test->ifobj_rx->pkt_stream = test->pkt_stream_default; 542 } 543 544 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) 545 { 546 struct pkt_stream *pkt_stream; 547 548 pkt_stream = calloc(1, sizeof(*pkt_stream)); 549 if (!pkt_stream) 550 return NULL; 551 552 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); 553 if (!pkt_stream->pkts) { 554 free(pkt_stream); 555 return NULL; 556 } 557 558 pkt_stream->nb_pkts = nb_pkts; 559 return pkt_stream; 560 } 561 562 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len) 563 { 564 pkt->addr = addr; 565 pkt->len = len; 566 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom) 567 pkt->valid = false; 568 else 569 pkt->valid = true; 570 } 571 572 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len) 573 { 574 struct pkt_stream *pkt_stream; 575 u32 i; 576 577 pkt_stream = __pkt_stream_alloc(nb_pkts); 578 if (!pkt_stream) 579 exit_with_error(ENOMEM); 580 581 pkt_stream->nb_pkts = nb_pkts; 582 for (i = 0; i < nb_pkts; i++) { 583 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size, 584 pkt_len); 585 pkt_stream->pkts[i].payload = i; 586 } 587 588 return pkt_stream; 589 } 590 591 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem, 592 struct pkt_stream *pkt_stream) 593 { 594 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len); 595 } 596 597 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) 598 { 599 struct pkt_stream *pkt_stream; 600 601 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len); 602 test->ifobj_tx->pkt_stream = pkt_stream; 603 test->ifobj_rx->pkt_stream = pkt_stream; 604 } 605 606 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) 607 { 608 struct xsk_umem_info *umem = test->ifobj_tx->umem; 609 struct pkt_stream *pkt_stream; 610 u32 i; 611 612 pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default); 613 for (i = 1; i < test->pkt_stream_default->nb_pkts; i += 2) 614 pkt_set(umem, &pkt_stream->pkts[i], 615 (i % umem->num_frames) * umem->frame_size + offset, pkt_len); 616 617 test->ifobj_tx->pkt_stream = pkt_stream; 618 test->ifobj_rx->pkt_stream = pkt_stream; 619 } 620 621 static void pkt_stream_receive_half(struct test_spec *test) 622 { 623 struct xsk_umem_info *umem = test->ifobj_rx->umem; 624 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream; 625 u32 i; 626 627 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts, 628 pkt_stream->pkts[0].len); 629 pkt_stream = test->ifobj_rx->pkt_stream; 630 for (i = 1; i < pkt_stream->nb_pkts; i += 2) 631 pkt_stream->pkts[i].valid = false; 632 } 633 634 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) 635 { 636 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb); 637 struct udphdr *udp_hdr; 638 struct ethhdr *eth_hdr; 639 struct iphdr *ip_hdr; 640 void *data; 641 642 if (!pkt) 643 return NULL; 644 if (!pkt->valid || pkt->len < MIN_PKT_SIZE) 645 return pkt; 646 647 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr); 648 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr)); 649 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 650 eth_hdr = (struct ethhdr *)data; 651 652 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr); 653 gen_ip_hdr(ifobject, ip_hdr); 654 gen_udp_csum(udp_hdr, ip_hdr); 655 gen_eth_hdr(ifobject, eth_hdr); 656 657 return pkt; 658 } 659 660 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) 661 { 662 struct pkt_stream *pkt_stream; 663 u32 i; 664 665 pkt_stream = __pkt_stream_alloc(nb_pkts); 666 if (!pkt_stream) 667 exit_with_error(ENOMEM); 668 669 test->ifobj_tx->pkt_stream = pkt_stream; 670 test->ifobj_rx->pkt_stream = pkt_stream; 671 672 for (i = 0; i < nb_pkts; i++) { 673 pkt_stream->pkts[i].addr = pkts[i].addr; 674 pkt_stream->pkts[i].len = pkts[i].len; 675 pkt_stream->pkts[i].payload = i; 676 pkt_stream->pkts[i].valid = pkts[i].valid; 677 } 678 } 679 680 static void pkt_dump(void *pkt, u32 len) 681 { 682 char s[INET_ADDRSTRLEN]; 683 struct ethhdr *ethhdr; 684 struct udphdr *udphdr; 685 struct iphdr *iphdr; 686 int payload, i; 687 688 ethhdr = pkt; 689 iphdr = pkt + sizeof(*ethhdr); 690 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr); 691 692 /*extract L2 frame */ 693 fprintf(stdout, "DEBUG>> L2: dst mac: "); 694 for (i = 0; i < ETH_ALEN; i++) 695 fprintf(stdout, "%02X", ethhdr->h_dest[i]); 696 697 fprintf(stdout, "\nDEBUG>> L2: src mac: "); 698 for (i = 0; i < ETH_ALEN; i++) 699 fprintf(stdout, "%02X", ethhdr->h_source[i]); 700 701 /*extract L3 frame */ 702 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl); 703 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n", 704 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s))); 705 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n", 706 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s))); 707 /*extract L4 frame */ 708 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source)); 709 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest)); 710 /*extract L5 frame */ 711 payload = *((uint32_t *)(pkt + PKT_HDR_SIZE)); 712 713 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload); 714 fprintf(stdout, "---------------------------------------\n"); 715 } 716 717 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr, 718 u64 pkt_stream_addr) 719 { 720 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; 721 u32 offset = addr % umem->frame_size, expected_offset = 0; 722 723 if (!pkt_stream->use_addr_for_fill) 724 pkt_stream_addr = 0; 725 726 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; 727 728 if (offset == expected_offset) 729 return true; 730 731 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); 732 return false; 733 } 734 735 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) 736 { 737 void *data = xsk_umem__get_data(buffer, addr); 738 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 739 740 if (!pkt) { 741 ksft_print_msg("[%s] too many packets received\n", __func__); 742 return false; 743 } 744 745 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) { 746 /* Do not try to verify packets that are smaller than minimum size. */ 747 return true; 748 } 749 750 if (pkt->len != len) { 751 ksft_print_msg("[%s] expected length [%d], got length [%d]\n", 752 __func__, pkt->len, len); 753 return false; 754 } 755 756 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) { 757 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE))); 758 759 if (opt_pkt_dump) 760 pkt_dump(data, PKT_SIZE); 761 762 if (pkt->payload != seqnum) { 763 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n", 764 __func__, pkt->payload, seqnum); 765 return false; 766 } 767 } else { 768 ksft_print_msg("Invalid frame received: "); 769 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version, 770 iphdr->tos); 771 return false; 772 } 773 774 return true; 775 } 776 777 static void kick_tx(struct xsk_socket_info *xsk) 778 { 779 int ret; 780 781 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 782 if (ret >= 0) 783 return; 784 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 785 usleep(100); 786 return; 787 } 788 exit_with_error(errno); 789 } 790 791 static void kick_rx(struct xsk_socket_info *xsk) 792 { 793 int ret; 794 795 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 796 if (ret < 0) 797 exit_with_error(errno); 798 } 799 800 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 801 { 802 unsigned int rcvd; 803 u32 idx; 804 805 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) 806 kick_tx(xsk); 807 808 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 809 if (rcvd) { 810 if (rcvd > xsk->outstanding_tx) { 811 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); 812 813 ksft_print_msg("[%s] Too many packets completed\n", __func__); 814 ksft_print_msg("Last completion address: %llx\n", addr); 815 return TEST_FAILURE; 816 } 817 818 xsk_ring_cons__release(&xsk->umem->cq, rcvd); 819 xsk->outstanding_tx -= rcvd; 820 } 821 822 return TEST_PASS; 823 } 824 825 static int receive_pkts(struct test_spec *test, struct pollfd *fds) 826 { 827 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 828 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream; 829 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0; 830 struct xsk_socket_info *xsk = test->ifobj_rx->xsk; 831 struct ifobject *ifobj = test->ifobj_rx; 832 struct xsk_umem_info *umem = xsk->umem; 833 struct pkt *pkt; 834 int ret; 835 836 ret = gettimeofday(&tv_now, NULL); 837 if (ret) 838 exit_with_error(errno); 839 timeradd(&tv_now, &tv_timeout, &tv_end); 840 841 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 842 while (pkt) { 843 ret = gettimeofday(&tv_now, NULL); 844 if (ret) 845 exit_with_error(errno); 846 if (timercmp(&tv_now, &tv_end, >)) { 847 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); 848 return TEST_FAILURE; 849 } 850 851 kick_rx(xsk); 852 if (ifobj->use_poll) { 853 ret = poll(fds, 1, POLL_TMOUT); 854 if (ret < 0) 855 exit_with_error(-ret); 856 857 if (!ret) { 858 if (!is_umem_valid(test->ifobj_tx)) 859 return TEST_PASS; 860 861 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); 862 return TEST_FAILURE; 863 864 } 865 866 if (!(fds->revents & POLLIN)) 867 continue; 868 } 869 870 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); 871 if (!rcvd) 872 continue; 873 874 if (ifobj->use_fill_ring) { 875 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 876 while (ret != rcvd) { 877 if (ret < 0) 878 exit_with_error(-ret); 879 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 880 ret = poll(fds, 1, POLL_TMOUT); 881 if (ret < 0) 882 exit_with_error(-ret); 883 } 884 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 885 } 886 } 887 888 for (i = 0; i < rcvd; i++) { 889 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); 890 u64 addr = desc->addr, orig; 891 892 orig = xsk_umem__extract_addr(addr); 893 addr = xsk_umem__add_offset_to_addr(addr); 894 895 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) || 896 !is_offset_correct(umem, pkt_stream, addr, pkt->addr)) 897 return TEST_FAILURE; 898 899 if (ifobj->use_fill_ring) 900 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; 901 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 902 } 903 904 if (ifobj->use_fill_ring) 905 xsk_ring_prod__submit(&umem->fq, rcvd); 906 if (ifobj->release_rx) 907 xsk_ring_cons__release(&xsk->rx, rcvd); 908 909 pthread_mutex_lock(&pacing_mutex); 910 pkts_in_flight -= pkts_sent; 911 if (pkts_in_flight < umem->num_frames) 912 pthread_cond_signal(&pacing_cond); 913 pthread_mutex_unlock(&pacing_mutex); 914 pkts_sent = 0; 915 } 916 917 return TEST_PASS; 918 } 919 920 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds, 921 bool timeout) 922 { 923 struct xsk_socket_info *xsk = ifobject->xsk; 924 bool use_poll = ifobject->use_poll; 925 u32 i, idx = 0, ret, valid_pkts = 0; 926 927 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) { 928 if (use_poll) { 929 ret = poll(fds, 1, POLL_TMOUT); 930 if (timeout) { 931 if (ret < 0) { 932 ksft_print_msg("ERROR: [%s] Poll error %d\n", 933 __func__, ret); 934 return TEST_FAILURE; 935 } 936 if (ret == 0) 937 return TEST_PASS; 938 break; 939 } 940 if (ret <= 0) { 941 ksft_print_msg("ERROR: [%s] Poll error %d\n", 942 __func__, ret); 943 return TEST_FAILURE; 944 } 945 } 946 947 complete_pkts(xsk, BATCH_SIZE); 948 } 949 950 for (i = 0; i < BATCH_SIZE; i++) { 951 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 952 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb); 953 954 if (!pkt) 955 break; 956 957 tx_desc->addr = pkt->addr; 958 tx_desc->len = pkt->len; 959 (*pkt_nb)++; 960 if (pkt->valid) 961 valid_pkts++; 962 } 963 964 pthread_mutex_lock(&pacing_mutex); 965 pkts_in_flight += valid_pkts; 966 /* pkts_in_flight might be negative if many invalid packets are sent */ 967 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) { 968 kick_tx(xsk); 969 pthread_cond_wait(&pacing_cond, &pacing_mutex); 970 } 971 pthread_mutex_unlock(&pacing_mutex); 972 973 xsk_ring_prod__submit(&xsk->tx, i); 974 xsk->outstanding_tx += valid_pkts; 975 976 if (use_poll) { 977 ret = poll(fds, 1, POLL_TMOUT); 978 if (ret <= 0) { 979 if (ret == 0 && timeout) 980 return TEST_PASS; 981 982 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); 983 return TEST_FAILURE; 984 } 985 } 986 987 if (!timeout) { 988 if (complete_pkts(xsk, i)) 989 return TEST_FAILURE; 990 991 usleep(10); 992 return TEST_PASS; 993 } 994 995 return TEST_CONTINUE; 996 } 997 998 static void wait_for_tx_completion(struct xsk_socket_info *xsk) 999 { 1000 while (xsk->outstanding_tx) 1001 complete_pkts(xsk, BATCH_SIZE); 1002 } 1003 1004 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) 1005 { 1006 bool timeout = !is_umem_valid(test->ifobj_rx); 1007 struct pollfd fds = { }; 1008 u32 pkt_cnt = 0, ret; 1009 1010 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1011 fds.events = POLLOUT; 1012 1013 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) { 1014 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout); 1015 if ((ret || test->fail) && !timeout) 1016 return TEST_FAILURE; 1017 else if (ret == TEST_PASS && timeout) 1018 return ret; 1019 } 1020 1021 wait_for_tx_completion(ifobject->xsk); 1022 return TEST_PASS; 1023 } 1024 1025 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) 1026 { 1027 int fd = xsk_socket__fd(xsk), err; 1028 socklen_t optlen, expected_len; 1029 1030 optlen = sizeof(*stats); 1031 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); 1032 if (err) { 1033 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1034 __func__, -err, strerror(-err)); 1035 return TEST_FAILURE; 1036 } 1037 1038 expected_len = sizeof(struct xdp_statistics); 1039 if (optlen != expected_len) { 1040 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", 1041 __func__, expected_len, optlen); 1042 return TEST_FAILURE; 1043 } 1044 1045 return TEST_PASS; 1046 } 1047 1048 static int validate_rx_dropped(struct ifobject *ifobject) 1049 { 1050 struct xsk_socket *xsk = ifobject->xsk->xsk; 1051 struct xdp_statistics stats; 1052 int err; 1053 1054 kick_rx(ifobject->xsk); 1055 1056 err = get_xsk_stats(xsk, &stats); 1057 if (err) 1058 return TEST_FAILURE; 1059 1060 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2) 1061 return TEST_PASS; 1062 1063 return TEST_FAILURE; 1064 } 1065 1066 static int validate_rx_full(struct ifobject *ifobject) 1067 { 1068 struct xsk_socket *xsk = ifobject->xsk->xsk; 1069 struct xdp_statistics stats; 1070 int err; 1071 1072 usleep(1000); 1073 kick_rx(ifobject->xsk); 1074 1075 err = get_xsk_stats(xsk, &stats); 1076 if (err) 1077 return TEST_FAILURE; 1078 1079 if (stats.rx_ring_full) 1080 return TEST_PASS; 1081 1082 return TEST_FAILURE; 1083 } 1084 1085 static int validate_fill_empty(struct ifobject *ifobject) 1086 { 1087 struct xsk_socket *xsk = ifobject->xsk->xsk; 1088 struct xdp_statistics stats; 1089 int err; 1090 1091 usleep(1000); 1092 kick_rx(ifobject->xsk); 1093 1094 err = get_xsk_stats(xsk, &stats); 1095 if (err) 1096 return TEST_FAILURE; 1097 1098 if (stats.rx_fill_ring_empty_descs) 1099 return TEST_PASS; 1100 1101 return TEST_FAILURE; 1102 } 1103 1104 static int validate_tx_invalid_descs(struct ifobject *ifobject) 1105 { 1106 struct xsk_socket *xsk = ifobject->xsk->xsk; 1107 int fd = xsk_socket__fd(xsk); 1108 struct xdp_statistics stats; 1109 socklen_t optlen; 1110 int err; 1111 1112 optlen = sizeof(stats); 1113 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 1114 if (err) { 1115 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1116 __func__, -err, strerror(-err)); 1117 return TEST_FAILURE; 1118 } 1119 1120 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) { 1121 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n", 1122 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts); 1123 return TEST_FAILURE; 1124 } 1125 1126 return TEST_PASS; 1127 } 1128 1129 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) 1130 { 1131 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; 1132 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 1133 LIBBPF_OPTS(bpf_xdp_query_opts, opts); 1134 int ret, ifindex; 1135 void *bufs; 1136 u32 i; 1137 1138 ifobject->ns_fd = switch_namespace(ifobject->nsname); 1139 1140 if (ifobject->umem->unaligned_mode) 1141 mmap_flags |= MAP_HUGETLB; 1142 1143 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 1144 if (bufs == MAP_FAILED) 1145 exit_with_error(errno); 1146 1147 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz); 1148 if (ret) 1149 exit_with_error(-ret); 1150 1151 for (i = 0; i < test->nb_sockets; i++) { 1152 u32 ctr = 0; 1153 1154 while (ctr++ < SOCK_RECONF_CTR) { 1155 ret = xsk_configure_socket(&ifobject->xsk_arr[i], ifobject->umem, 1156 ifobject, !!i); 1157 if (!ret) 1158 break; 1159 1160 /* Retry if it fails as xsk_socket__create() is asynchronous */ 1161 if (ctr >= SOCK_RECONF_CTR) 1162 exit_with_error(-ret); 1163 usleep(USLEEP_MAX); 1164 } 1165 1166 if (ifobject->busy_poll) 1167 enable_busy_poll(&ifobject->xsk_arr[i]); 1168 } 1169 1170 ifobject->xsk = &ifobject->xsk_arr[0]; 1171 1172 if (!ifobject->rx_on) 1173 return; 1174 1175 ifindex = if_nametoindex(ifobject->ifname); 1176 if (!ifindex) 1177 exit_with_error(errno); 1178 1179 ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd); 1180 if (ret) 1181 exit_with_error(-ret); 1182 1183 ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts); 1184 if (ret) 1185 exit_with_error(-ret); 1186 1187 if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) { 1188 if (opts.attach_mode != XDP_ATTACHED_SKB) { 1189 ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n"); 1190 exit_with_error(-EINVAL); 1191 } 1192 } else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) { 1193 if (opts.attach_mode != XDP_ATTACHED_DRV) { 1194 ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n"); 1195 exit_with_error(-EINVAL); 1196 } 1197 } 1198 1199 ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd); 1200 if (ret) 1201 exit_with_error(-ret); 1202 } 1203 1204 static void testapp_cleanup_xsk_res(struct ifobject *ifobj) 1205 { 1206 print_verbose("Destroying socket\n"); 1207 xsk_socket__delete(ifobj->xsk->xsk); 1208 munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size); 1209 xsk_umem__delete(ifobj->umem->umem); 1210 } 1211 1212 static void *worker_testapp_validate_tx(void *arg) 1213 { 1214 struct test_spec *test = (struct test_spec *)arg; 1215 struct ifobject *ifobject = test->ifobj_tx; 1216 int err; 1217 1218 if (test->current_step == 1) 1219 thread_common_ops(test, ifobject); 1220 1221 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts, 1222 ifobject->ifname); 1223 err = send_pkts(test, ifobject); 1224 1225 if (!err && ifobject->validation_func) 1226 err = ifobject->validation_func(ifobject); 1227 if (err) 1228 report_failure(test); 1229 1230 if (test->total_steps == test->current_step || err) 1231 testapp_cleanup_xsk_res(ifobject); 1232 pthread_exit(NULL); 1233 } 1234 1235 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream) 1236 { 1237 u32 idx = 0, i, buffers_to_fill; 1238 int ret; 1239 1240 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) 1241 buffers_to_fill = umem->num_frames; 1242 else 1243 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; 1244 1245 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); 1246 if (ret != buffers_to_fill) 1247 exit_with_error(ENOSPC); 1248 for (i = 0; i < buffers_to_fill; i++) { 1249 u64 addr; 1250 1251 if (pkt_stream->use_addr_for_fill) { 1252 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i); 1253 1254 if (!pkt) 1255 break; 1256 addr = pkt->addr; 1257 } else { 1258 addr = i * umem->frame_size; 1259 } 1260 1261 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; 1262 } 1263 xsk_ring_prod__submit(&umem->fq, buffers_to_fill); 1264 } 1265 1266 static void *worker_testapp_validate_rx(void *arg) 1267 { 1268 struct test_spec *test = (struct test_spec *)arg; 1269 struct ifobject *ifobject = test->ifobj_rx; 1270 struct pollfd fds = { }; 1271 int err; 1272 1273 if (test->current_step == 1) 1274 thread_common_ops(test, ifobject); 1275 1276 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream); 1277 1278 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1279 fds.events = POLLIN; 1280 1281 pthread_barrier_wait(&barr); 1282 1283 err = receive_pkts(test, &fds); 1284 1285 if (!err && ifobject->validation_func) 1286 err = ifobject->validation_func(ifobject); 1287 if (err) { 1288 report_failure(test); 1289 pthread_mutex_lock(&pacing_mutex); 1290 pthread_cond_signal(&pacing_cond); 1291 pthread_mutex_unlock(&pacing_mutex); 1292 } 1293 1294 if (test->total_steps == test->current_step || err) 1295 testapp_cleanup_xsk_res(ifobject); 1296 pthread_exit(NULL); 1297 } 1298 1299 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj, 1300 enum test_type type) 1301 { 1302 pthread_t t0; 1303 1304 if (pthread_barrier_init(&barr, NULL, 2)) 1305 exit_with_error(errno); 1306 1307 test->current_step++; 1308 if (type == TEST_TYPE_POLL_RXQ_TMOUT) 1309 pkt_stream_reset(ifobj->pkt_stream); 1310 pkts_in_flight = 0; 1311 1312 /*Spawn thread */ 1313 pthread_create(&t0, NULL, ifobj->func_ptr, test); 1314 1315 if (type != TEST_TYPE_POLL_TXQ_TMOUT) 1316 pthread_barrier_wait(&barr); 1317 1318 if (pthread_barrier_destroy(&barr)) 1319 exit_with_error(errno); 1320 1321 pthread_join(t0, NULL); 1322 1323 return !!test->fail; 1324 } 1325 1326 static int testapp_validate_traffic(struct test_spec *test) 1327 { 1328 struct ifobject *ifobj_tx = test->ifobj_tx; 1329 struct ifobject *ifobj_rx = test->ifobj_rx; 1330 pthread_t t0, t1; 1331 1332 if (pthread_barrier_init(&barr, NULL, 2)) 1333 exit_with_error(errno); 1334 1335 test->current_step++; 1336 pkt_stream_reset(ifobj_rx->pkt_stream); 1337 pkts_in_flight = 0; 1338 1339 /*Spawn RX thread */ 1340 pthread_create(&t0, NULL, ifobj_rx->func_ptr, test); 1341 1342 pthread_barrier_wait(&barr); 1343 if (pthread_barrier_destroy(&barr)) 1344 exit_with_error(errno); 1345 1346 /*Spawn TX thread */ 1347 pthread_create(&t1, NULL, ifobj_tx->func_ptr, test); 1348 1349 pthread_join(t1, NULL); 1350 pthread_join(t0, NULL); 1351 1352 return !!test->fail; 1353 } 1354 1355 static void testapp_teardown(struct test_spec *test) 1356 { 1357 int i; 1358 1359 test_spec_set_name(test, "TEARDOWN"); 1360 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 1361 if (testapp_validate_traffic(test)) 1362 return; 1363 test_spec_reset(test); 1364 } 1365 } 1366 1367 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) 1368 { 1369 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; 1370 struct ifobject *tmp_ifobj = (*ifobj1); 1371 1372 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; 1373 (*ifobj2)->func_ptr = tmp_func_ptr; 1374 1375 *ifobj1 = *ifobj2; 1376 *ifobj2 = tmp_ifobj; 1377 } 1378 1379 static void testapp_bidi(struct test_spec *test) 1380 { 1381 test_spec_set_name(test, "BIDIRECTIONAL"); 1382 test->ifobj_tx->rx_on = true; 1383 test->ifobj_rx->tx_on = true; 1384 test->total_steps = 2; 1385 if (testapp_validate_traffic(test)) 1386 return; 1387 1388 print_verbose("Switching Tx/Rx vectors\n"); 1389 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1390 testapp_validate_traffic(test); 1391 1392 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1393 } 1394 1395 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx) 1396 { 1397 int ret; 1398 1399 xsk_socket__delete(ifobj_tx->xsk->xsk); 1400 xsk_socket__delete(ifobj_rx->xsk->xsk); 1401 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1]; 1402 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1]; 1403 1404 ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd); 1405 if (ret) 1406 exit_with_error(-ret); 1407 } 1408 1409 static void testapp_bpf_res(struct test_spec *test) 1410 { 1411 test_spec_set_name(test, "BPF_RES"); 1412 test->total_steps = 2; 1413 test->nb_sockets = 2; 1414 if (testapp_validate_traffic(test)) 1415 return; 1416 1417 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx); 1418 testapp_validate_traffic(test); 1419 } 1420 1421 static void testapp_headroom(struct test_spec *test) 1422 { 1423 test_spec_set_name(test, "UMEM_HEADROOM"); 1424 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 1425 testapp_validate_traffic(test); 1426 } 1427 1428 static void testapp_stats_rx_dropped(struct test_spec *test) 1429 { 1430 test_spec_set_name(test, "STAT_RX_DROPPED"); 1431 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - 1432 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; 1433 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); 1434 pkt_stream_receive_half(test); 1435 test->ifobj_rx->validation_func = validate_rx_dropped; 1436 testapp_validate_traffic(test); 1437 } 1438 1439 static void testapp_stats_tx_invalid_descs(struct test_spec *test) 1440 { 1441 test_spec_set_name(test, "STAT_TX_INVALID"); 1442 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 1443 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 1444 testapp_validate_traffic(test); 1445 1446 pkt_stream_restore_default(test); 1447 } 1448 1449 static void testapp_stats_rx_full(struct test_spec *test) 1450 { 1451 test_spec_set_name(test, "STAT_RX_FULL"); 1452 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1453 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1454 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1455 if (!test->ifobj_rx->pkt_stream) 1456 exit_with_error(ENOMEM); 1457 1458 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; 1459 test->ifobj_rx->release_rx = false; 1460 test->ifobj_rx->validation_func = validate_rx_full; 1461 testapp_validate_traffic(test); 1462 1463 pkt_stream_restore_default(test); 1464 } 1465 1466 static void testapp_stats_fill_empty(struct test_spec *test) 1467 { 1468 test_spec_set_name(test, "STAT_RX_FILL_EMPTY"); 1469 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1470 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1471 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1472 if (!test->ifobj_rx->pkt_stream) 1473 exit_with_error(ENOMEM); 1474 1475 test->ifobj_rx->use_fill_ring = false; 1476 test->ifobj_rx->validation_func = validate_fill_empty; 1477 testapp_validate_traffic(test); 1478 1479 pkt_stream_restore_default(test); 1480 } 1481 1482 /* Simple test */ 1483 static bool hugepages_present(struct ifobject *ifobject) 1484 { 1485 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size; 1486 void *bufs; 1487 1488 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1489 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 1490 if (bufs == MAP_FAILED) 1491 return false; 1492 1493 munmap(bufs, mmap_sz); 1494 return true; 1495 } 1496 1497 static bool testapp_unaligned(struct test_spec *test) 1498 { 1499 if (!hugepages_present(test->ifobj_tx)) { 1500 ksft_test_result_skip("No 2M huge pages present.\n"); 1501 return false; 1502 } 1503 1504 test_spec_set_name(test, "UNALIGNED_MODE"); 1505 test->ifobj_tx->umem->unaligned_mode = true; 1506 test->ifobj_rx->umem->unaligned_mode = true; 1507 /* Let half of the packets straddle a buffer boundrary */ 1508 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2); 1509 test->ifobj_rx->pkt_stream->use_addr_for_fill = true; 1510 testapp_validate_traffic(test); 1511 1512 pkt_stream_restore_default(test); 1513 return true; 1514 } 1515 1516 static void testapp_single_pkt(struct test_spec *test) 1517 { 1518 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}}; 1519 1520 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1521 testapp_validate_traffic(test); 1522 pkt_stream_restore_default(test); 1523 } 1524 1525 static void testapp_invalid_desc(struct test_spec *test) 1526 { 1527 struct pkt pkts[] = { 1528 /* Zero packet address allowed */ 1529 {0, PKT_SIZE, 0, true}, 1530 /* Allowed packet */ 1531 {0x1000, PKT_SIZE, 0, true}, 1532 /* Straddling the start of umem */ 1533 {-2, PKT_SIZE, 0, false}, 1534 /* Packet too large */ 1535 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, 1536 /* After umem ends */ 1537 {UMEM_SIZE, PKT_SIZE, 0, false}, 1538 /* Straddle the end of umem */ 1539 {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1540 /* Straddle a page boundrary */ 1541 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1542 /* Straddle a 2K boundrary */ 1543 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true}, 1544 /* Valid packet for synch so that something is received */ 1545 {0x4000, PKT_SIZE, 0, true}}; 1546 1547 if (test->ifobj_tx->umem->unaligned_mode) { 1548 /* Crossing a page boundrary allowed */ 1549 pkts[6].valid = true; 1550 } 1551 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { 1552 /* Crossing a 2K frame size boundrary not allowed */ 1553 pkts[7].valid = false; 1554 } 1555 1556 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1557 testapp_validate_traffic(test); 1558 pkt_stream_restore_default(test); 1559 } 1560 1561 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac, 1562 const char *dst_ip, const char *src_ip, const u16 dst_port, 1563 const u16 src_port, thread_func_t func_ptr) 1564 { 1565 struct in_addr ip; 1566 1567 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN); 1568 memcpy(ifobj->src_mac, src_mac, ETH_ALEN); 1569 1570 inet_aton(dst_ip, &ip); 1571 ifobj->dst_ip = ip.s_addr; 1572 1573 inet_aton(src_ip, &ip); 1574 ifobj->src_ip = ip.s_addr; 1575 1576 ifobj->dst_port = dst_port; 1577 ifobj->src_port = src_port; 1578 1579 ifobj->func_ptr = func_ptr; 1580 } 1581 1582 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type) 1583 { 1584 switch (type) { 1585 case TEST_TYPE_STATS_RX_DROPPED: 1586 testapp_stats_rx_dropped(test); 1587 break; 1588 case TEST_TYPE_STATS_TX_INVALID_DESCS: 1589 testapp_stats_tx_invalid_descs(test); 1590 break; 1591 case TEST_TYPE_STATS_RX_FULL: 1592 testapp_stats_rx_full(test); 1593 break; 1594 case TEST_TYPE_STATS_FILL_EMPTY: 1595 testapp_stats_fill_empty(test); 1596 break; 1597 case TEST_TYPE_TEARDOWN: 1598 testapp_teardown(test); 1599 break; 1600 case TEST_TYPE_BIDI: 1601 testapp_bidi(test); 1602 break; 1603 case TEST_TYPE_BPF_RES: 1604 testapp_bpf_res(test); 1605 break; 1606 case TEST_TYPE_RUN_TO_COMPLETION: 1607 test_spec_set_name(test, "RUN_TO_COMPLETION"); 1608 testapp_validate_traffic(test); 1609 break; 1610 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT: 1611 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT"); 1612 testapp_single_pkt(test); 1613 break; 1614 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME: 1615 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE"); 1616 test->ifobj_tx->umem->frame_size = 2048; 1617 test->ifobj_rx->umem->frame_size = 2048; 1618 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE); 1619 testapp_validate_traffic(test); 1620 1621 pkt_stream_restore_default(test); 1622 break; 1623 case TEST_TYPE_RX_POLL: 1624 test->ifobj_rx->use_poll = true; 1625 test_spec_set_name(test, "POLL_RX"); 1626 testapp_validate_traffic(test); 1627 break; 1628 case TEST_TYPE_TX_POLL: 1629 test->ifobj_tx->use_poll = true; 1630 test_spec_set_name(test, "POLL_TX"); 1631 testapp_validate_traffic(test); 1632 break; 1633 case TEST_TYPE_POLL_TXQ_TMOUT: 1634 test_spec_set_name(test, "POLL_TXQ_FULL"); 1635 test->ifobj_tx->use_poll = true; 1636 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 1637 test->ifobj_tx->umem->frame_size = 2048; 1638 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); 1639 testapp_validate_traffic_single_thread(test, test->ifobj_tx, type); 1640 pkt_stream_restore_default(test); 1641 break; 1642 case TEST_TYPE_POLL_RXQ_TMOUT: 1643 test_spec_set_name(test, "POLL_RXQ_EMPTY"); 1644 test->ifobj_rx->use_poll = true; 1645 testapp_validate_traffic_single_thread(test, test->ifobj_rx, type); 1646 break; 1647 case TEST_TYPE_ALIGNED_INV_DESC: 1648 test_spec_set_name(test, "ALIGNED_INV_DESC"); 1649 testapp_invalid_desc(test); 1650 break; 1651 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME: 1652 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE"); 1653 test->ifobj_tx->umem->frame_size = 2048; 1654 test->ifobj_rx->umem->frame_size = 2048; 1655 testapp_invalid_desc(test); 1656 break; 1657 case TEST_TYPE_UNALIGNED_INV_DESC: 1658 if (!hugepages_present(test->ifobj_tx)) { 1659 ksft_test_result_skip("No 2M huge pages present.\n"); 1660 return; 1661 } 1662 test_spec_set_name(test, "UNALIGNED_INV_DESC"); 1663 test->ifobj_tx->umem->unaligned_mode = true; 1664 test->ifobj_rx->umem->unaligned_mode = true; 1665 testapp_invalid_desc(test); 1666 break; 1667 case TEST_TYPE_UNALIGNED: 1668 if (!testapp_unaligned(test)) 1669 return; 1670 break; 1671 case TEST_TYPE_HEADROOM: 1672 testapp_headroom(test); 1673 break; 1674 default: 1675 break; 1676 } 1677 1678 if (!test->fail) 1679 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 1680 test->name); 1681 } 1682 1683 static struct ifobject *ifobject_create(void) 1684 { 1685 struct ifobject *ifobj; 1686 1687 ifobj = calloc(1, sizeof(struct ifobject)); 1688 if (!ifobj) 1689 return NULL; 1690 1691 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); 1692 if (!ifobj->xsk_arr) 1693 goto out_xsk_arr; 1694 1695 ifobj->umem = calloc(1, sizeof(*ifobj->umem)); 1696 if (!ifobj->umem) 1697 goto out_umem; 1698 1699 return ifobj; 1700 1701 out_umem: 1702 free(ifobj->xsk_arr); 1703 out_xsk_arr: 1704 free(ifobj); 1705 return NULL; 1706 } 1707 1708 static void ifobject_delete(struct ifobject *ifobj) 1709 { 1710 free(ifobj->umem); 1711 free(ifobj->xsk_arr); 1712 free(ifobj); 1713 } 1714 1715 int main(int argc, char **argv) 1716 { 1717 struct pkt_stream *pkt_stream_default; 1718 struct ifobject *ifobj_tx, *ifobj_rx; 1719 u32 i, j, failed_tests = 0; 1720 struct test_spec test; 1721 1722 /* Use libbpf 1.0 API mode */ 1723 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 1724 1725 ifobj_tx = ifobject_create(); 1726 if (!ifobj_tx) 1727 exit_with_error(ENOMEM); 1728 ifobj_rx = ifobject_create(); 1729 if (!ifobj_rx) 1730 exit_with_error(ENOMEM); 1731 1732 setlocale(LC_ALL, ""); 1733 1734 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 1735 1736 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) { 1737 usage(basename(argv[0])); 1738 ksft_exit_xfail(); 1739 } 1740 1741 init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, 1742 worker_testapp_validate_tx); 1743 init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, 1744 worker_testapp_validate_rx); 1745 1746 test_spec_init(&test, ifobj_tx, ifobj_rx, 0); 1747 pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE); 1748 if (!pkt_stream_default) 1749 exit_with_error(ENOMEM); 1750 test.pkt_stream_default = pkt_stream_default; 1751 1752 ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX); 1753 1754 for (i = 0; i < TEST_MODE_MAX; i++) 1755 for (j = 0; j < TEST_TYPE_MAX; j++) { 1756 test_spec_init(&test, ifobj_tx, ifobj_rx, i); 1757 run_pkt_test(&test, i, j); 1758 usleep(USLEEP_MAX); 1759 1760 if (test.fail) 1761 failed_tests++; 1762 } 1763 1764 pkt_stream_delete(pkt_stream_default); 1765 ifobject_delete(ifobj_tx); 1766 ifobject_delete(ifobj_rx); 1767 1768 if (failed_tests) 1769 ksft_exit_fail(); 1770 else 1771 ksft_exit_pass(); 1772 } 1773