1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 /* 5 * Some functions in this program are taken from 6 * Linux kernel samples/bpf/xdpsock* and modified 7 * for use. 8 * 9 * See test_xsk.sh for detailed information on test topology 10 * and prerequisite network setup. 11 * 12 * This test program contains two threads, each thread is single socket with 13 * a unique UMEM. It validates in-order packet delivery and packet content 14 * by sending packets to each other. 15 * 16 * Tests Information: 17 * ------------------ 18 * These selftests test AF_XDP SKB and Native/DRV modes using veth 19 * Virtual Ethernet interfaces. 20 * 21 * For each mode, the following tests are run: 22 * a. nopoll - soft-irq processing in run-to-completion mode 23 * b. poll - using poll() syscall 24 * c. Socket Teardown 25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy 26 * both sockets, then repeat multiple times. Only nopoll mode is used 27 * d. Bi-directional sockets 28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and 29 * completion rings on each socket, tx/rx in both directions. Only nopoll 30 * mode is used 31 * e. Statistics 32 * Trigger some error conditions and ensure that the appropriate statistics 33 * are incremented. Within this test, the following statistics are tested: 34 * i. rx dropped 35 * Increase the UMEM frame headroom to a value which results in 36 * insufficient space in the rx buffer for both the packet and the headroom. 37 * ii. tx invalid 38 * Set the 'len' field of tx descriptors to an invalid value (umem frame 39 * size + 1). 40 * iii. rx ring full 41 * Reduce the size of the RX ring to a fraction of the fill ring size. 42 * iv. fill queue empty 43 * Do not populate the fill queue and then try to receive pkts. 44 * f. bpf_link resource persistence 45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, 46 * then remove xsk sockets from queue 0 on both veth interfaces and 47 * finally run a traffic on queues ids 1 48 * g. unaligned mode 49 * h. tests for invalid and corner case Tx descriptors so that the correct ones 50 * are discarded and let through, respectively. 51 * i. 2K frame size tests 52 * 53 * Total tests: 12 54 * 55 * Flow: 56 * ----- 57 * - Single process spawns two threads: Tx and Rx 58 * - Each of these two threads attach to a veth interface within their assigned 59 * namespaces 60 * - Each thread Creates one AF_XDP socket connected to a unique umem for each 61 * veth interface 62 * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy> 63 * - Rx thread verifies if all 10k packets were received and delivered in-order, 64 * and have the right content 65 * 66 * Enable/disable packet dump mode: 67 * -------------------------- 68 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add 69 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") 70 */ 71 72 #define _GNU_SOURCE 73 #include <fcntl.h> 74 #include <errno.h> 75 #include <getopt.h> 76 #include <asm/barrier.h> 77 #include <linux/if_link.h> 78 #include <linux/if_ether.h> 79 #include <linux/ip.h> 80 #include <linux/udp.h> 81 #include <arpa/inet.h> 82 #include <net/if.h> 83 #include <locale.h> 84 #include <poll.h> 85 #include <pthread.h> 86 #include <signal.h> 87 #include <stdbool.h> 88 #include <stdio.h> 89 #include <stdlib.h> 90 #include <string.h> 91 #include <stddef.h> 92 #include <sys/mman.h> 93 #include <sys/socket.h> 94 #include <sys/time.h> 95 #include <sys/types.h> 96 #include <sys/queue.h> 97 #include <time.h> 98 #include <unistd.h> 99 #include <stdatomic.h> 100 #include "xsk.h" 101 #include "xskxceiver.h" 102 #include <bpf/bpf.h> 103 #include <linux/filter.h> 104 #include "../kselftest.h" 105 106 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf. 107 * Until xskxceiver is either moved or re-writed into libxdp, suppress 108 * deprecation warnings in this file 109 */ 110 #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 111 112 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62"; 113 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61"; 114 static const char *IP1 = "192.168.100.162"; 115 static const char *IP2 = "192.168.100.161"; 116 static const u16 UDP_PORT1 = 2020; 117 static const u16 UDP_PORT2 = 2121; 118 119 static void __exit_with_error(int error, const char *file, const char *func, int line) 120 { 121 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, 122 strerror(error)); 123 ksft_exit_xfail(); 124 } 125 126 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) 127 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 128 static char *mode_string(struct test_spec *test) 129 { 130 switch (test->mode) { 131 case TEST_MODE_SKB: 132 return "SKB"; 133 case TEST_MODE_DRV: 134 return "DRV"; 135 case TEST_MODE_ZC: 136 return "ZC"; 137 default: 138 return "BOGUS"; 139 } 140 } 141 142 static void report_failure(struct test_spec *test) 143 { 144 if (test->fail) 145 return; 146 147 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), 148 test->name); 149 test->fail = true; 150 } 151 152 static void memset32_htonl(void *dest, u32 val, u32 size) 153 { 154 u32 *ptr = (u32 *)dest; 155 int i; 156 157 val = htonl(val); 158 159 for (i = 0; i < (size & (~0x3)); i += 4) 160 ptr[i >> 2] = val; 161 } 162 163 /* 164 * Fold a partial checksum 165 * This function code has been taken from 166 * Linux kernel include/asm-generic/checksum.h 167 */ 168 static __u16 csum_fold(__u32 csum) 169 { 170 u32 sum = (__force u32)csum; 171 172 sum = (sum & 0xffff) + (sum >> 16); 173 sum = (sum & 0xffff) + (sum >> 16); 174 return (__force __u16)~sum; 175 } 176 177 /* 178 * This function code has been taken from 179 * Linux kernel lib/checksum.c 180 */ 181 static u32 from64to32(u64 x) 182 { 183 /* add up 32-bit and 32-bit for 32+c bit */ 184 x = (x & 0xffffffff) + (x >> 32); 185 /* add up carry.. */ 186 x = (x & 0xffffffff) + (x >> 32); 187 return (u32)x; 188 } 189 190 /* 191 * This function code has been taken from 192 * Linux kernel lib/checksum.c 193 */ 194 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 195 { 196 unsigned long long s = (__force u32)sum; 197 198 s += (__force u32)saddr; 199 s += (__force u32)daddr; 200 #ifdef __BIG_ENDIAN__ 201 s += proto + len; 202 #else 203 s += (proto + len) << 8; 204 #endif 205 return (__force __u32)from64to32(s); 206 } 207 208 /* 209 * This function has been taken from 210 * Linux kernel include/asm-generic/checksum.h 211 */ 212 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum) 213 { 214 return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); 215 } 216 217 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt) 218 { 219 u32 csum = 0; 220 u32 cnt = 0; 221 222 /* udp hdr and data */ 223 for (; cnt < len; cnt += 2) 224 csum += udp_pkt[cnt >> 1]; 225 226 return csum_tcpudp_magic(saddr, daddr, len, proto, csum); 227 } 228 229 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr) 230 { 231 memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN); 232 memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN); 233 eth_hdr->h_proto = htons(ETH_P_IP); 234 } 235 236 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr) 237 { 238 ip_hdr->version = IP_PKT_VER; 239 ip_hdr->ihl = 0x5; 240 ip_hdr->tos = IP_PKT_TOS; 241 ip_hdr->tot_len = htons(IP_PKT_SIZE); 242 ip_hdr->id = 0; 243 ip_hdr->frag_off = 0; 244 ip_hdr->ttl = IPDEFTTL; 245 ip_hdr->protocol = IPPROTO_UDP; 246 ip_hdr->saddr = ifobject->src_ip; 247 ip_hdr->daddr = ifobject->dst_ip; 248 ip_hdr->check = 0; 249 } 250 251 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject, 252 struct udphdr *udp_hdr) 253 { 254 udp_hdr->source = htons(ifobject->src_port); 255 udp_hdr->dest = htons(ifobject->dst_port); 256 udp_hdr->len = htons(UDP_PKT_SIZE); 257 memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE); 258 } 259 260 static bool is_umem_valid(struct ifobject *ifobj) 261 { 262 return !!ifobj->umem->umem; 263 } 264 265 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr) 266 { 267 udp_hdr->check = 0; 268 udp_hdr->check = 269 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr); 270 } 271 272 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size) 273 { 274 struct xsk_umem_config cfg = { 275 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, 276 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, 277 .frame_size = umem->frame_size, 278 .frame_headroom = umem->frame_headroom, 279 .flags = XSK_UMEM__DEFAULT_FLAGS 280 }; 281 int ret; 282 283 if (umem->unaligned_mode) 284 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; 285 286 ret = xsk_umem__create(&umem->umem, buffer, size, 287 &umem->fq, &umem->cq, &cfg); 288 if (ret) 289 return ret; 290 291 umem->buffer = buffer; 292 return 0; 293 } 294 295 static void enable_busy_poll(struct xsk_socket_info *xsk) 296 { 297 int sock_opt; 298 299 sock_opt = 1; 300 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, 301 (void *)&sock_opt, sizeof(sock_opt)) < 0) 302 exit_with_error(errno); 303 304 sock_opt = 20; 305 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, 306 (void *)&sock_opt, sizeof(sock_opt)) < 0) 307 exit_with_error(errno); 308 309 sock_opt = BATCH_SIZE; 310 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, 311 (void *)&sock_opt, sizeof(sock_opt)) < 0) 312 exit_with_error(errno); 313 } 314 315 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 316 struct ifobject *ifobject, bool shared) 317 { 318 struct xsk_socket_config cfg = {}; 319 struct xsk_ring_cons *rxr; 320 struct xsk_ring_prod *txr; 321 322 xsk->umem = umem; 323 cfg.rx_size = xsk->rxqsize; 324 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; 325 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; 326 cfg.xdp_flags = ifobject->xdp_flags; 327 cfg.bind_flags = ifobject->bind_flags; 328 if (shared) 329 cfg.bind_flags |= XDP_SHARED_UMEM; 330 331 txr = ifobject->tx_on ? &xsk->tx : NULL; 332 rxr = ifobject->rx_on ? &xsk->rx : NULL; 333 return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg); 334 } 335 336 static bool ifobj_zc_avail(struct ifobject *ifobject) 337 { 338 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; 339 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 340 struct xsk_socket_info *xsk; 341 struct xsk_umem_info *umem; 342 bool zc_avail = false; 343 void *bufs; 344 int ret; 345 346 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 347 if (bufs == MAP_FAILED) 348 exit_with_error(errno); 349 350 umem = calloc(1, sizeof(struct xsk_umem_info)); 351 if (!umem) { 352 munmap(bufs, umem_sz); 353 exit_with_error(-ENOMEM); 354 } 355 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 356 ret = xsk_configure_umem(umem, bufs, umem_sz); 357 if (ret) 358 exit_with_error(-ret); 359 360 xsk = calloc(1, sizeof(struct xsk_socket_info)); 361 if (!xsk) 362 goto out; 363 ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 364 ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE; 365 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; 366 ifobject->rx_on = true; 367 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 368 ret = __xsk_configure_socket(xsk, umem, ifobject, false); 369 if (!ret) 370 zc_avail = true; 371 372 xsk_socket__delete(xsk->xsk); 373 free(xsk); 374 out: 375 munmap(umem->buffer, umem_sz); 376 xsk_umem__delete(umem->umem); 377 free(umem); 378 return zc_avail; 379 } 380 381 static struct option long_options[] = { 382 {"interface", required_argument, 0, 'i'}, 383 {"busy-poll", no_argument, 0, 'b'}, 384 {"dump-pkts", no_argument, 0, 'D'}, 385 {"verbose", no_argument, 0, 'v'}, 386 {0, 0, 0, 0} 387 }; 388 389 static void usage(const char *prog) 390 { 391 const char *str = 392 " Usage: %s [OPTIONS]\n" 393 " Options:\n" 394 " -i, --interface Use interface\n" 395 " -D, --dump-pkts Dump packets L2 - L5\n" 396 " -v, --verbose Verbose output\n" 397 " -b, --busy-poll Enable busy poll\n"; 398 399 ksft_print_msg(str, prog); 400 } 401 402 static int switch_namespace(const char *nsname) 403 { 404 char fqns[26] = "/var/run/netns/"; 405 int nsfd; 406 407 if (!nsname || strlen(nsname) == 0) 408 return -1; 409 410 strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1); 411 nsfd = open(fqns, O_RDONLY); 412 413 if (nsfd == -1) 414 exit_with_error(errno); 415 416 if (setns(nsfd, 0) == -1) 417 exit_with_error(errno); 418 419 print_verbose("NS switched: %s\n", nsname); 420 421 return nsfd; 422 } 423 424 static bool validate_interface(struct ifobject *ifobj) 425 { 426 if (!strcmp(ifobj->ifname, "")) 427 return false; 428 return true; 429 } 430 431 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, 432 char **argv) 433 { 434 struct ifobject *ifobj; 435 u32 interface_nb = 0; 436 int option_index, c; 437 438 opterr = 0; 439 440 for (;;) { 441 char *sptr, *token; 442 443 c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index); 444 if (c == -1) 445 break; 446 447 switch (c) { 448 case 'i': 449 if (interface_nb == 0) 450 ifobj = ifobj_tx; 451 else if (interface_nb == 1) 452 ifobj = ifobj_rx; 453 else 454 break; 455 456 sptr = strndupa(optarg, strlen(optarg)); 457 memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS); 458 token = strsep(&sptr, ","); 459 if (token) 460 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS); 461 interface_nb++; 462 break; 463 case 'D': 464 opt_pkt_dump = true; 465 break; 466 case 'v': 467 opt_verbose = true; 468 break; 469 case 'b': 470 ifobj_tx->busy_poll = true; 471 ifobj_rx->busy_poll = true; 472 break; 473 default: 474 usage(basename(argv[0])); 475 ksft_exit_xfail(); 476 } 477 } 478 } 479 480 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 481 struct ifobject *ifobj_rx) 482 { 483 u32 i, j; 484 485 for (i = 0; i < MAX_INTERFACES; i++) { 486 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 487 488 ifobj->xsk = &ifobj->xsk_arr[0]; 489 ifobj->use_poll = false; 490 ifobj->use_fill_ring = true; 491 ifobj->release_rx = true; 492 ifobj->validation_func = NULL; 493 494 if (i == 0) { 495 ifobj->rx_on = false; 496 ifobj->tx_on = true; 497 ifobj->pkt_stream = test->tx_pkt_stream_default; 498 } else { 499 ifobj->rx_on = true; 500 ifobj->tx_on = false; 501 ifobj->pkt_stream = test->rx_pkt_stream_default; 502 } 503 504 memset(ifobj->umem, 0, sizeof(*ifobj->umem)); 505 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; 506 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; 507 if (ifobj->shared_umem && ifobj->rx_on) 508 ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS * 509 XSK_UMEM__DEFAULT_FRAME_SIZE; 510 511 for (j = 0; j < MAX_SOCKETS; j++) { 512 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); 513 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; 514 } 515 } 516 517 test->ifobj_tx = ifobj_tx; 518 test->ifobj_rx = ifobj_rx; 519 test->current_step = 0; 520 test->total_steps = 1; 521 test->nb_sockets = 1; 522 test->fail = false; 523 } 524 525 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, 526 struct ifobject *ifobj_rx, enum test_mode mode) 527 { 528 struct pkt_stream *tx_pkt_stream; 529 struct pkt_stream *rx_pkt_stream; 530 u32 i; 531 532 tx_pkt_stream = test->tx_pkt_stream_default; 533 rx_pkt_stream = test->rx_pkt_stream_default; 534 memset(test, 0, sizeof(*test)); 535 test->tx_pkt_stream_default = tx_pkt_stream; 536 test->rx_pkt_stream_default = rx_pkt_stream; 537 538 for (i = 0; i < MAX_INTERFACES; i++) { 539 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; 540 541 ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; 542 if (mode == TEST_MODE_SKB) 543 ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE; 544 else 545 ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE; 546 547 ifobj->bind_flags = XDP_USE_NEED_WAKEUP; 548 if (mode == TEST_MODE_ZC) 549 ifobj->bind_flags |= XDP_ZEROCOPY; 550 else 551 ifobj->bind_flags |= XDP_COPY; 552 } 553 554 test->mode = mode; 555 __test_spec_init(test, ifobj_tx, ifobj_rx); 556 } 557 558 static void test_spec_reset(struct test_spec *test) 559 { 560 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); 561 } 562 563 static void test_spec_set_name(struct test_spec *test, const char *name) 564 { 565 strncpy(test->name, name, MAX_TEST_NAME_SIZE); 566 } 567 568 static void pkt_stream_reset(struct pkt_stream *pkt_stream) 569 { 570 if (pkt_stream) 571 pkt_stream->rx_pkt_nb = 0; 572 } 573 574 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb) 575 { 576 if (pkt_nb >= pkt_stream->nb_pkts) 577 return NULL; 578 579 return &pkt_stream->pkts[pkt_nb]; 580 } 581 582 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) 583 { 584 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) { 585 (*pkts_sent)++; 586 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid) 587 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++]; 588 pkt_stream->rx_pkt_nb++; 589 } 590 return NULL; 591 } 592 593 static void pkt_stream_delete(struct pkt_stream *pkt_stream) 594 { 595 free(pkt_stream->pkts); 596 free(pkt_stream); 597 } 598 599 static void pkt_stream_restore_default(struct test_spec *test) 600 { 601 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream; 602 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream; 603 604 if (tx_pkt_stream != test->tx_pkt_stream_default) { 605 pkt_stream_delete(test->ifobj_tx->pkt_stream); 606 test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default; 607 } 608 609 if (rx_pkt_stream != test->rx_pkt_stream_default) { 610 pkt_stream_delete(test->ifobj_rx->pkt_stream); 611 test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default; 612 } 613 } 614 615 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) 616 { 617 struct pkt_stream *pkt_stream; 618 619 pkt_stream = calloc(1, sizeof(*pkt_stream)); 620 if (!pkt_stream) 621 return NULL; 622 623 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); 624 if (!pkt_stream->pkts) { 625 free(pkt_stream); 626 return NULL; 627 } 628 629 pkt_stream->nb_pkts = nb_pkts; 630 return pkt_stream; 631 } 632 633 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len) 634 { 635 pkt->addr = addr + umem->base_addr; 636 pkt->len = len; 637 if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom) 638 pkt->valid = false; 639 else 640 pkt->valid = true; 641 } 642 643 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len) 644 { 645 struct pkt_stream *pkt_stream; 646 u32 i; 647 648 pkt_stream = __pkt_stream_alloc(nb_pkts); 649 if (!pkt_stream) 650 exit_with_error(ENOMEM); 651 652 pkt_stream->nb_pkts = nb_pkts; 653 for (i = 0; i < nb_pkts; i++) { 654 pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size, 655 pkt_len); 656 pkt_stream->pkts[i].payload = i; 657 } 658 659 return pkt_stream; 660 } 661 662 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem, 663 struct pkt_stream *pkt_stream) 664 { 665 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len); 666 } 667 668 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) 669 { 670 struct pkt_stream *pkt_stream; 671 672 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len); 673 test->ifobj_tx->pkt_stream = pkt_stream; 674 pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len); 675 test->ifobj_rx->pkt_stream = pkt_stream; 676 } 677 678 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, 679 int offset) 680 { 681 struct xsk_umem_info *umem = ifobj->umem; 682 struct pkt_stream *pkt_stream; 683 u32 i; 684 685 pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream); 686 for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2) 687 pkt_set(umem, &pkt_stream->pkts[i], 688 (i % umem->num_frames) * umem->frame_size + offset, pkt_len); 689 690 ifobj->pkt_stream = pkt_stream; 691 } 692 693 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) 694 { 695 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); 696 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); 697 } 698 699 static void pkt_stream_receive_half(struct test_spec *test) 700 { 701 struct xsk_umem_info *umem = test->ifobj_rx->umem; 702 struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream; 703 u32 i; 704 705 test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts, 706 pkt_stream->pkts[0].len); 707 pkt_stream = test->ifobj_rx->pkt_stream; 708 for (i = 1; i < pkt_stream->nb_pkts; i += 2) 709 pkt_stream->pkts[i].valid = false; 710 } 711 712 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) 713 { 714 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb); 715 struct udphdr *udp_hdr; 716 struct ethhdr *eth_hdr; 717 struct iphdr *ip_hdr; 718 void *data; 719 720 if (!pkt) 721 return NULL; 722 if (!pkt->valid || pkt->len < MIN_PKT_SIZE) 723 return pkt; 724 725 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr); 726 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr)); 727 ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 728 eth_hdr = (struct ethhdr *)data; 729 730 gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr); 731 gen_ip_hdr(ifobject, ip_hdr); 732 gen_udp_csum(udp_hdr, ip_hdr); 733 gen_eth_hdr(ifobject, eth_hdr); 734 735 return pkt; 736 } 737 738 static void __pkt_stream_generate_custom(struct ifobject *ifobj, 739 struct pkt *pkts, u32 nb_pkts) 740 { 741 struct pkt_stream *pkt_stream; 742 u32 i; 743 744 pkt_stream = __pkt_stream_alloc(nb_pkts); 745 if (!pkt_stream) 746 exit_with_error(ENOMEM); 747 748 for (i = 0; i < nb_pkts; i++) { 749 pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr; 750 pkt_stream->pkts[i].len = pkts[i].len; 751 pkt_stream->pkts[i].payload = i; 752 pkt_stream->pkts[i].valid = pkts[i].valid; 753 } 754 755 ifobj->pkt_stream = pkt_stream; 756 } 757 758 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) 759 { 760 __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts); 761 __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts); 762 } 763 764 static void pkt_dump(void *pkt, u32 len) 765 { 766 char s[INET_ADDRSTRLEN]; 767 struct ethhdr *ethhdr; 768 struct udphdr *udphdr; 769 struct iphdr *iphdr; 770 int payload, i; 771 772 ethhdr = pkt; 773 iphdr = pkt + sizeof(*ethhdr); 774 udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr); 775 776 /*extract L2 frame */ 777 fprintf(stdout, "DEBUG>> L2: dst mac: "); 778 for (i = 0; i < ETH_ALEN; i++) 779 fprintf(stdout, "%02X", ethhdr->h_dest[i]); 780 781 fprintf(stdout, "\nDEBUG>> L2: src mac: "); 782 for (i = 0; i < ETH_ALEN; i++) 783 fprintf(stdout, "%02X", ethhdr->h_source[i]); 784 785 /*extract L3 frame */ 786 fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl); 787 fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n", 788 inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s))); 789 fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n", 790 inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s))); 791 /*extract L4 frame */ 792 fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source)); 793 fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest)); 794 /*extract L5 frame */ 795 payload = *((uint32_t *)(pkt + PKT_HDR_SIZE)); 796 797 fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload); 798 fprintf(stdout, "---------------------------------------\n"); 799 } 800 801 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr, 802 u64 pkt_stream_addr) 803 { 804 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; 805 u32 offset = addr % umem->frame_size, expected_offset = 0; 806 807 if (!pkt_stream->use_addr_for_fill) 808 pkt_stream_addr = 0; 809 810 expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; 811 812 if (offset == expected_offset) 813 return true; 814 815 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); 816 return false; 817 } 818 819 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) 820 { 821 void *data = xsk_umem__get_data(buffer, addr); 822 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr)); 823 824 if (!pkt) { 825 ksft_print_msg("[%s] too many packets received\n", __func__); 826 return false; 827 } 828 829 if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) { 830 /* Do not try to verify packets that are smaller than minimum size. */ 831 return true; 832 } 833 834 if (pkt->len != len) { 835 ksft_print_msg("[%s] expected length [%d], got length [%d]\n", 836 __func__, pkt->len, len); 837 return false; 838 } 839 840 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) { 841 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE))); 842 843 if (opt_pkt_dump) 844 pkt_dump(data, PKT_SIZE); 845 846 if (pkt->payload != seqnum) { 847 ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n", 848 __func__, pkt->payload, seqnum); 849 return false; 850 } 851 } else { 852 ksft_print_msg("Invalid frame received: "); 853 ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version, 854 iphdr->tos); 855 return false; 856 } 857 858 return true; 859 } 860 861 static void kick_tx(struct xsk_socket_info *xsk) 862 { 863 int ret; 864 865 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); 866 if (ret >= 0) 867 return; 868 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { 869 usleep(100); 870 return; 871 } 872 exit_with_error(errno); 873 } 874 875 static void kick_rx(struct xsk_socket_info *xsk) 876 { 877 int ret; 878 879 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); 880 if (ret < 0) 881 exit_with_error(errno); 882 } 883 884 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) 885 { 886 unsigned int rcvd; 887 u32 idx; 888 889 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) 890 kick_tx(xsk); 891 892 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); 893 if (rcvd) { 894 if (rcvd > xsk->outstanding_tx) { 895 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); 896 897 ksft_print_msg("[%s] Too many packets completed\n", __func__); 898 ksft_print_msg("Last completion address: %llx\n", addr); 899 return TEST_FAILURE; 900 } 901 902 xsk_ring_cons__release(&xsk->umem->cq, rcvd); 903 xsk->outstanding_tx -= rcvd; 904 } 905 906 return TEST_PASS; 907 } 908 909 static int receive_pkts(struct test_spec *test, struct pollfd *fds) 910 { 911 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; 912 struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream; 913 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0; 914 struct xsk_socket_info *xsk = test->ifobj_rx->xsk; 915 struct ifobject *ifobj = test->ifobj_rx; 916 struct xsk_umem_info *umem = xsk->umem; 917 struct pkt *pkt; 918 int ret; 919 920 ret = gettimeofday(&tv_now, NULL); 921 if (ret) 922 exit_with_error(errno); 923 timeradd(&tv_now, &tv_timeout, &tv_end); 924 925 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 926 while (pkt) { 927 ret = gettimeofday(&tv_now, NULL); 928 if (ret) 929 exit_with_error(errno); 930 if (timercmp(&tv_now, &tv_end, >)) { 931 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); 932 return TEST_FAILURE; 933 } 934 935 kick_rx(xsk); 936 if (ifobj->use_poll) { 937 ret = poll(fds, 1, POLL_TMOUT); 938 if (ret < 0) 939 exit_with_error(-ret); 940 941 if (!ret) { 942 if (!is_umem_valid(test->ifobj_tx)) 943 return TEST_PASS; 944 945 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); 946 return TEST_FAILURE; 947 948 } 949 950 if (!(fds->revents & POLLIN)) 951 continue; 952 } 953 954 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); 955 if (!rcvd) 956 continue; 957 958 if (ifobj->use_fill_ring) { 959 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 960 while (ret != rcvd) { 961 if (ret < 0) 962 exit_with_error(-ret); 963 if (xsk_ring_prod__needs_wakeup(&umem->fq)) { 964 ret = poll(fds, 1, POLL_TMOUT); 965 if (ret < 0) 966 exit_with_error(-ret); 967 } 968 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); 969 } 970 } 971 972 for (i = 0; i < rcvd; i++) { 973 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); 974 u64 addr = desc->addr, orig; 975 976 orig = xsk_umem__extract_addr(addr); 977 addr = xsk_umem__add_offset_to_addr(addr); 978 979 if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) || 980 !is_offset_correct(umem, pkt_stream, addr, pkt->addr)) 981 return TEST_FAILURE; 982 983 if (ifobj->use_fill_ring) 984 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; 985 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); 986 } 987 988 if (ifobj->use_fill_ring) 989 xsk_ring_prod__submit(&umem->fq, rcvd); 990 if (ifobj->release_rx) 991 xsk_ring_cons__release(&xsk->rx, rcvd); 992 993 pthread_mutex_lock(&pacing_mutex); 994 pkts_in_flight -= pkts_sent; 995 if (pkts_in_flight < umem->num_frames) 996 pthread_cond_signal(&pacing_cond); 997 pthread_mutex_unlock(&pacing_mutex); 998 pkts_sent = 0; 999 } 1000 1001 return TEST_PASS; 1002 } 1003 1004 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds, 1005 bool timeout) 1006 { 1007 struct xsk_socket_info *xsk = ifobject->xsk; 1008 bool use_poll = ifobject->use_poll; 1009 u32 i, idx = 0, valid_pkts = 0; 1010 int ret; 1011 1012 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) { 1013 if (use_poll) { 1014 ret = poll(fds, 1, POLL_TMOUT); 1015 if (timeout) { 1016 if (ret < 0) { 1017 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1018 __func__, ret); 1019 return TEST_FAILURE; 1020 } 1021 if (ret == 0) 1022 return TEST_PASS; 1023 break; 1024 } 1025 if (ret <= 0) { 1026 ksft_print_msg("ERROR: [%s] Poll error %d\n", 1027 __func__, ret); 1028 return TEST_FAILURE; 1029 } 1030 } 1031 1032 complete_pkts(xsk, BATCH_SIZE); 1033 } 1034 1035 for (i = 0; i < BATCH_SIZE; i++) { 1036 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); 1037 struct pkt *pkt = pkt_generate(ifobject, *pkt_nb); 1038 1039 if (!pkt) 1040 break; 1041 1042 tx_desc->addr = pkt->addr; 1043 tx_desc->len = pkt->len; 1044 (*pkt_nb)++; 1045 if (pkt->valid) 1046 valid_pkts++; 1047 } 1048 1049 pthread_mutex_lock(&pacing_mutex); 1050 pkts_in_flight += valid_pkts; 1051 /* pkts_in_flight might be negative if many invalid packets are sent */ 1052 if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) { 1053 kick_tx(xsk); 1054 pthread_cond_wait(&pacing_cond, &pacing_mutex); 1055 } 1056 pthread_mutex_unlock(&pacing_mutex); 1057 1058 xsk_ring_prod__submit(&xsk->tx, i); 1059 xsk->outstanding_tx += valid_pkts; 1060 1061 if (use_poll) { 1062 ret = poll(fds, 1, POLL_TMOUT); 1063 if (ret <= 0) { 1064 if (ret == 0 && timeout) 1065 return TEST_PASS; 1066 1067 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); 1068 return TEST_FAILURE; 1069 } 1070 } 1071 1072 if (!timeout) { 1073 if (complete_pkts(xsk, i)) 1074 return TEST_FAILURE; 1075 1076 usleep(10); 1077 return TEST_PASS; 1078 } 1079 1080 return TEST_CONTINUE; 1081 } 1082 1083 static void wait_for_tx_completion(struct xsk_socket_info *xsk) 1084 { 1085 while (xsk->outstanding_tx) 1086 complete_pkts(xsk, BATCH_SIZE); 1087 } 1088 1089 static int send_pkts(struct test_spec *test, struct ifobject *ifobject) 1090 { 1091 bool timeout = !is_umem_valid(test->ifobj_rx); 1092 struct pollfd fds = { }; 1093 u32 pkt_cnt = 0, ret; 1094 1095 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1096 fds.events = POLLOUT; 1097 1098 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) { 1099 ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout); 1100 if ((ret || test->fail) && !timeout) 1101 return TEST_FAILURE; 1102 else if (ret == TEST_PASS && timeout) 1103 return ret; 1104 } 1105 1106 wait_for_tx_completion(ifobject->xsk); 1107 return TEST_PASS; 1108 } 1109 1110 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) 1111 { 1112 int fd = xsk_socket__fd(xsk), err; 1113 socklen_t optlen, expected_len; 1114 1115 optlen = sizeof(*stats); 1116 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); 1117 if (err) { 1118 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1119 __func__, -err, strerror(-err)); 1120 return TEST_FAILURE; 1121 } 1122 1123 expected_len = sizeof(struct xdp_statistics); 1124 if (optlen != expected_len) { 1125 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", 1126 __func__, expected_len, optlen); 1127 return TEST_FAILURE; 1128 } 1129 1130 return TEST_PASS; 1131 } 1132 1133 static int validate_rx_dropped(struct ifobject *ifobject) 1134 { 1135 struct xsk_socket *xsk = ifobject->xsk->xsk; 1136 struct xdp_statistics stats; 1137 int err; 1138 1139 kick_rx(ifobject->xsk); 1140 1141 err = get_xsk_stats(xsk, &stats); 1142 if (err) 1143 return TEST_FAILURE; 1144 1145 if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2) 1146 return TEST_PASS; 1147 1148 return TEST_FAILURE; 1149 } 1150 1151 static int validate_rx_full(struct ifobject *ifobject) 1152 { 1153 struct xsk_socket *xsk = ifobject->xsk->xsk; 1154 struct xdp_statistics stats; 1155 int err; 1156 1157 usleep(1000); 1158 kick_rx(ifobject->xsk); 1159 1160 err = get_xsk_stats(xsk, &stats); 1161 if (err) 1162 return TEST_FAILURE; 1163 1164 if (stats.rx_ring_full) 1165 return TEST_PASS; 1166 1167 return TEST_FAILURE; 1168 } 1169 1170 static int validate_fill_empty(struct ifobject *ifobject) 1171 { 1172 struct xsk_socket *xsk = ifobject->xsk->xsk; 1173 struct xdp_statistics stats; 1174 int err; 1175 1176 usleep(1000); 1177 kick_rx(ifobject->xsk); 1178 1179 err = get_xsk_stats(xsk, &stats); 1180 if (err) 1181 return TEST_FAILURE; 1182 1183 if (stats.rx_fill_ring_empty_descs) 1184 return TEST_PASS; 1185 1186 return TEST_FAILURE; 1187 } 1188 1189 static int validate_tx_invalid_descs(struct ifobject *ifobject) 1190 { 1191 struct xsk_socket *xsk = ifobject->xsk->xsk; 1192 int fd = xsk_socket__fd(xsk); 1193 struct xdp_statistics stats; 1194 socklen_t optlen; 1195 int err; 1196 1197 optlen = sizeof(stats); 1198 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); 1199 if (err) { 1200 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", 1201 __func__, -err, strerror(-err)); 1202 return TEST_FAILURE; 1203 } 1204 1205 if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) { 1206 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n", 1207 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts); 1208 return TEST_FAILURE; 1209 } 1210 1211 return TEST_PASS; 1212 } 1213 1214 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, 1215 struct xsk_umem_info *umem, bool tx) 1216 { 1217 int i, ret; 1218 1219 for (i = 0; i < test->nb_sockets; i++) { 1220 bool shared = (ifobject->shared_umem && tx) ? true : !!i; 1221 u32 ctr = 0; 1222 1223 while (ctr++ < SOCK_RECONF_CTR) { 1224 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, 1225 ifobject, shared); 1226 if (!ret) 1227 break; 1228 1229 /* Retry if it fails as xsk_socket__create() is asynchronous */ 1230 if (ctr >= SOCK_RECONF_CTR) 1231 exit_with_error(-ret); 1232 usleep(USLEEP_MAX); 1233 } 1234 if (ifobject->busy_poll) 1235 enable_busy_poll(&ifobject->xsk_arr[i]); 1236 } 1237 } 1238 1239 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) 1240 { 1241 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); 1242 ifobject->xsk = &ifobject->xsk_arr[0]; 1243 ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd; 1244 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); 1245 } 1246 1247 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream) 1248 { 1249 u32 idx = 0, i, buffers_to_fill; 1250 int ret; 1251 1252 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) 1253 buffers_to_fill = umem->num_frames; 1254 else 1255 buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; 1256 1257 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); 1258 if (ret != buffers_to_fill) 1259 exit_with_error(ENOSPC); 1260 for (i = 0; i < buffers_to_fill; i++) { 1261 u64 addr; 1262 1263 if (pkt_stream->use_addr_for_fill) { 1264 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i); 1265 1266 if (!pkt) 1267 break; 1268 addr = pkt->addr; 1269 } else { 1270 addr = i * umem->frame_size; 1271 } 1272 1273 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; 1274 } 1275 xsk_ring_prod__submit(&umem->fq, buffers_to_fill); 1276 } 1277 1278 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) 1279 { 1280 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; 1281 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; 1282 LIBBPF_OPTS(bpf_xdp_query_opts, opts); 1283 int ret, ifindex; 1284 void *bufs; 1285 1286 ifobject->ns_fd = switch_namespace(ifobject->nsname); 1287 1288 if (ifobject->umem->unaligned_mode) 1289 mmap_flags |= MAP_HUGETLB; 1290 1291 if (ifobject->shared_umem) 1292 umem_sz *= 2; 1293 1294 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); 1295 if (bufs == MAP_FAILED) 1296 exit_with_error(errno); 1297 1298 ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz); 1299 if (ret) 1300 exit_with_error(-ret); 1301 1302 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream); 1303 1304 xsk_configure_socket(test, ifobject, ifobject->umem, false); 1305 1306 ifobject->xsk = &ifobject->xsk_arr[0]; 1307 1308 if (!ifobject->rx_on) 1309 return; 1310 1311 ifindex = if_nametoindex(ifobject->ifname); 1312 if (!ifindex) 1313 exit_with_error(errno); 1314 1315 ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd); 1316 if (ret) 1317 exit_with_error(-ret); 1318 1319 ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts); 1320 if (ret) 1321 exit_with_error(-ret); 1322 1323 if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) { 1324 if (opts.attach_mode != XDP_ATTACHED_SKB) { 1325 ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n"); 1326 exit_with_error(-EINVAL); 1327 } 1328 } else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) { 1329 if (opts.attach_mode != XDP_ATTACHED_DRV) { 1330 ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n"); 1331 exit_with_error(-EINVAL); 1332 } 1333 } 1334 1335 ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd); 1336 if (ret) 1337 exit_with_error(-ret); 1338 } 1339 1340 static void *worker_testapp_validate_tx(void *arg) 1341 { 1342 struct test_spec *test = (struct test_spec *)arg; 1343 struct ifobject *ifobject = test->ifobj_tx; 1344 int err; 1345 1346 if (test->current_step == 1) { 1347 if (!ifobject->shared_umem) 1348 thread_common_ops(test, ifobject); 1349 else 1350 thread_common_ops_tx(test, ifobject); 1351 } 1352 1353 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts, 1354 ifobject->ifname); 1355 err = send_pkts(test, ifobject); 1356 1357 if (!err && ifobject->validation_func) 1358 err = ifobject->validation_func(ifobject); 1359 if (err) 1360 report_failure(test); 1361 1362 pthread_exit(NULL); 1363 } 1364 1365 static void *worker_testapp_validate_rx(void *arg) 1366 { 1367 struct test_spec *test = (struct test_spec *)arg; 1368 struct ifobject *ifobject = test->ifobj_rx; 1369 struct pollfd fds = { }; 1370 int id = 0; 1371 int err; 1372 1373 if (test->current_step == 1) { 1374 thread_common_ops(test, ifobject); 1375 } else { 1376 bpf_map_delete_elem(ifobject->xsk_map_fd, &id); 1377 xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd); 1378 } 1379 1380 fds.fd = xsk_socket__fd(ifobject->xsk->xsk); 1381 fds.events = POLLIN; 1382 1383 pthread_barrier_wait(&barr); 1384 1385 err = receive_pkts(test, &fds); 1386 1387 if (!err && ifobject->validation_func) 1388 err = ifobject->validation_func(ifobject); 1389 if (err) { 1390 report_failure(test); 1391 pthread_mutex_lock(&pacing_mutex); 1392 pthread_cond_signal(&pacing_cond); 1393 pthread_mutex_unlock(&pacing_mutex); 1394 } 1395 1396 pthread_exit(NULL); 1397 } 1398 1399 static void testapp_clean_xsk_umem(struct ifobject *ifobj) 1400 { 1401 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; 1402 1403 if (ifobj->shared_umem) 1404 umem_sz *= 2; 1405 1406 xsk_umem__delete(ifobj->umem->umem); 1407 munmap(ifobj->umem->buffer, umem_sz); 1408 } 1409 1410 static void handler(int signum) 1411 { 1412 pthread_exit(NULL); 1413 } 1414 1415 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj, 1416 enum test_type type) 1417 { 1418 bool old_shared_umem = ifobj->shared_umem; 1419 pthread_t t0; 1420 1421 if (pthread_barrier_init(&barr, NULL, 2)) 1422 exit_with_error(errno); 1423 1424 test->current_step++; 1425 if (type == TEST_TYPE_POLL_RXQ_TMOUT) 1426 pkt_stream_reset(ifobj->pkt_stream); 1427 pkts_in_flight = 0; 1428 1429 test->ifobj_rx->shared_umem = false; 1430 test->ifobj_tx->shared_umem = false; 1431 1432 signal(SIGUSR1, handler); 1433 /* Spawn thread */ 1434 pthread_create(&t0, NULL, ifobj->func_ptr, test); 1435 1436 if (type != TEST_TYPE_POLL_TXQ_TMOUT) 1437 pthread_barrier_wait(&barr); 1438 1439 if (pthread_barrier_destroy(&barr)) 1440 exit_with_error(errno); 1441 1442 pthread_kill(t0, SIGUSR1); 1443 pthread_join(t0, NULL); 1444 1445 if (test->total_steps == test->current_step || test->fail) { 1446 xsk_socket__delete(ifobj->xsk->xsk); 1447 testapp_clean_xsk_umem(ifobj); 1448 } 1449 1450 test->ifobj_rx->shared_umem = old_shared_umem; 1451 test->ifobj_tx->shared_umem = old_shared_umem; 1452 1453 return !!test->fail; 1454 } 1455 1456 static int testapp_validate_traffic(struct test_spec *test) 1457 { 1458 struct ifobject *ifobj_tx = test->ifobj_tx; 1459 struct ifobject *ifobj_rx = test->ifobj_rx; 1460 pthread_t t0, t1; 1461 1462 if (pthread_barrier_init(&barr, NULL, 2)) 1463 exit_with_error(errno); 1464 1465 test->current_step++; 1466 pkt_stream_reset(ifobj_rx->pkt_stream); 1467 pkts_in_flight = 0; 1468 1469 /*Spawn RX thread */ 1470 pthread_create(&t0, NULL, ifobj_rx->func_ptr, test); 1471 1472 pthread_barrier_wait(&barr); 1473 if (pthread_barrier_destroy(&barr)) 1474 exit_with_error(errno); 1475 1476 /*Spawn TX thread */ 1477 pthread_create(&t1, NULL, ifobj_tx->func_ptr, test); 1478 1479 pthread_join(t1, NULL); 1480 pthread_join(t0, NULL); 1481 1482 if (test->total_steps == test->current_step || test->fail) { 1483 xsk_socket__delete(ifobj_tx->xsk->xsk); 1484 xsk_socket__delete(ifobj_rx->xsk->xsk); 1485 testapp_clean_xsk_umem(ifobj_rx); 1486 if (!ifobj_tx->shared_umem) 1487 testapp_clean_xsk_umem(ifobj_tx); 1488 } 1489 1490 return !!test->fail; 1491 } 1492 1493 static void testapp_teardown(struct test_spec *test) 1494 { 1495 int i; 1496 1497 test_spec_set_name(test, "TEARDOWN"); 1498 for (i = 0; i < MAX_TEARDOWN_ITER; i++) { 1499 if (testapp_validate_traffic(test)) 1500 return; 1501 test_spec_reset(test); 1502 } 1503 } 1504 1505 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) 1506 { 1507 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; 1508 struct ifobject *tmp_ifobj = (*ifobj1); 1509 1510 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; 1511 (*ifobj2)->func_ptr = tmp_func_ptr; 1512 1513 *ifobj1 = *ifobj2; 1514 *ifobj2 = tmp_ifobj; 1515 } 1516 1517 static void testapp_bidi(struct test_spec *test) 1518 { 1519 test_spec_set_name(test, "BIDIRECTIONAL"); 1520 test->ifobj_tx->rx_on = true; 1521 test->ifobj_rx->tx_on = true; 1522 test->total_steps = 2; 1523 if (testapp_validate_traffic(test)) 1524 return; 1525 1526 print_verbose("Switching Tx/Rx vectors\n"); 1527 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1528 testapp_validate_traffic(test); 1529 1530 swap_directions(&test->ifobj_rx, &test->ifobj_tx); 1531 } 1532 1533 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx) 1534 { 1535 int ret; 1536 1537 xsk_socket__delete(ifobj_tx->xsk->xsk); 1538 xsk_socket__delete(ifobj_rx->xsk->xsk); 1539 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1]; 1540 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1]; 1541 1542 ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd); 1543 if (ret) 1544 exit_with_error(-ret); 1545 } 1546 1547 static void testapp_bpf_res(struct test_spec *test) 1548 { 1549 test_spec_set_name(test, "BPF_RES"); 1550 test->total_steps = 2; 1551 test->nb_sockets = 2; 1552 if (testapp_validate_traffic(test)) 1553 return; 1554 1555 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx); 1556 testapp_validate_traffic(test); 1557 } 1558 1559 static void testapp_headroom(struct test_spec *test) 1560 { 1561 test_spec_set_name(test, "UMEM_HEADROOM"); 1562 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; 1563 testapp_validate_traffic(test); 1564 } 1565 1566 static void testapp_stats_rx_dropped(struct test_spec *test) 1567 { 1568 test_spec_set_name(test, "STAT_RX_DROPPED"); 1569 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); 1570 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - 1571 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; 1572 pkt_stream_receive_half(test); 1573 test->ifobj_rx->validation_func = validate_rx_dropped; 1574 testapp_validate_traffic(test); 1575 } 1576 1577 static void testapp_stats_tx_invalid_descs(struct test_spec *test) 1578 { 1579 test_spec_set_name(test, "STAT_TX_INVALID"); 1580 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); 1581 test->ifobj_tx->validation_func = validate_tx_invalid_descs; 1582 testapp_validate_traffic(test); 1583 1584 pkt_stream_restore_default(test); 1585 } 1586 1587 static void testapp_stats_rx_full(struct test_spec *test) 1588 { 1589 test_spec_set_name(test, "STAT_RX_FULL"); 1590 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1591 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1592 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1593 if (!test->ifobj_rx->pkt_stream) 1594 exit_with_error(ENOMEM); 1595 1596 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; 1597 test->ifobj_rx->release_rx = false; 1598 test->ifobj_rx->validation_func = validate_rx_full; 1599 testapp_validate_traffic(test); 1600 1601 pkt_stream_restore_default(test); 1602 } 1603 1604 static void testapp_stats_fill_empty(struct test_spec *test) 1605 { 1606 test_spec_set_name(test, "STAT_RX_FILL_EMPTY"); 1607 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE); 1608 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 1609 DEFAULT_UMEM_BUFFERS, PKT_SIZE); 1610 if (!test->ifobj_rx->pkt_stream) 1611 exit_with_error(ENOMEM); 1612 1613 test->ifobj_rx->use_fill_ring = false; 1614 test->ifobj_rx->validation_func = validate_fill_empty; 1615 testapp_validate_traffic(test); 1616 1617 pkt_stream_restore_default(test); 1618 } 1619 1620 /* Simple test */ 1621 static bool hugepages_present(struct ifobject *ifobject) 1622 { 1623 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size; 1624 void *bufs; 1625 1626 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1627 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); 1628 if (bufs == MAP_FAILED) 1629 return false; 1630 1631 munmap(bufs, mmap_sz); 1632 return true; 1633 } 1634 1635 static bool testapp_unaligned(struct test_spec *test) 1636 { 1637 if (!hugepages_present(test->ifobj_tx)) { 1638 ksft_test_result_skip("No 2M huge pages present.\n"); 1639 return false; 1640 } 1641 1642 test_spec_set_name(test, "UNALIGNED_MODE"); 1643 test->ifobj_tx->umem->unaligned_mode = true; 1644 test->ifobj_rx->umem->unaligned_mode = true; 1645 /* Let half of the packets straddle a buffer boundrary */ 1646 pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2); 1647 test->ifobj_rx->pkt_stream->use_addr_for_fill = true; 1648 testapp_validate_traffic(test); 1649 1650 pkt_stream_restore_default(test); 1651 return true; 1652 } 1653 1654 static void testapp_single_pkt(struct test_spec *test) 1655 { 1656 struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}}; 1657 1658 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1659 testapp_validate_traffic(test); 1660 pkt_stream_restore_default(test); 1661 } 1662 1663 static void testapp_invalid_desc(struct test_spec *test) 1664 { 1665 struct pkt pkts[] = { 1666 /* Zero packet address allowed */ 1667 {0, PKT_SIZE, 0, true}, 1668 /* Allowed packet */ 1669 {0x1000, PKT_SIZE, 0, true}, 1670 /* Straddling the start of umem */ 1671 {-2, PKT_SIZE, 0, false}, 1672 /* Packet too large */ 1673 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, 1674 /* After umem ends */ 1675 {UMEM_SIZE, PKT_SIZE, 0, false}, 1676 /* Straddle the end of umem */ 1677 {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1678 /* Straddle a page boundrary */ 1679 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false}, 1680 /* Straddle a 2K boundrary */ 1681 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true}, 1682 /* Valid packet for synch so that something is received */ 1683 {0x4000, PKT_SIZE, 0, true}}; 1684 1685 if (test->ifobj_tx->umem->unaligned_mode) { 1686 /* Crossing a page boundrary allowed */ 1687 pkts[6].valid = true; 1688 } 1689 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { 1690 /* Crossing a 2K frame size boundrary not allowed */ 1691 pkts[7].valid = false; 1692 } 1693 1694 if (test->ifobj_tx->shared_umem) { 1695 pkts[4].addr += UMEM_SIZE; 1696 pkts[5].addr += UMEM_SIZE; 1697 } 1698 1699 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); 1700 testapp_validate_traffic(test); 1701 pkt_stream_restore_default(test); 1702 } 1703 1704 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac, 1705 const char *dst_ip, const char *src_ip, const u16 dst_port, 1706 const u16 src_port, thread_func_t func_ptr) 1707 { 1708 struct in_addr ip; 1709 1710 memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN); 1711 memcpy(ifobj->src_mac, src_mac, ETH_ALEN); 1712 1713 inet_aton(dst_ip, &ip); 1714 ifobj->dst_ip = ip.s_addr; 1715 1716 inet_aton(src_ip, &ip); 1717 ifobj->src_ip = ip.s_addr; 1718 1719 ifobj->dst_port = dst_port; 1720 ifobj->src_port = src_port; 1721 1722 ifobj->func_ptr = func_ptr; 1723 } 1724 1725 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type) 1726 { 1727 switch (type) { 1728 case TEST_TYPE_STATS_RX_DROPPED: 1729 if (mode == TEST_MODE_ZC) { 1730 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); 1731 return; 1732 } 1733 testapp_stats_rx_dropped(test); 1734 break; 1735 case TEST_TYPE_STATS_TX_INVALID_DESCS: 1736 testapp_stats_tx_invalid_descs(test); 1737 break; 1738 case TEST_TYPE_STATS_RX_FULL: 1739 testapp_stats_rx_full(test); 1740 break; 1741 case TEST_TYPE_STATS_FILL_EMPTY: 1742 testapp_stats_fill_empty(test); 1743 break; 1744 case TEST_TYPE_TEARDOWN: 1745 testapp_teardown(test); 1746 break; 1747 case TEST_TYPE_BIDI: 1748 testapp_bidi(test); 1749 break; 1750 case TEST_TYPE_BPF_RES: 1751 testapp_bpf_res(test); 1752 break; 1753 case TEST_TYPE_RUN_TO_COMPLETION: 1754 test_spec_set_name(test, "RUN_TO_COMPLETION"); 1755 testapp_validate_traffic(test); 1756 break; 1757 case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT: 1758 test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT"); 1759 testapp_single_pkt(test); 1760 break; 1761 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME: 1762 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE"); 1763 test->ifobj_tx->umem->frame_size = 2048; 1764 test->ifobj_rx->umem->frame_size = 2048; 1765 pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE); 1766 testapp_validate_traffic(test); 1767 1768 pkt_stream_restore_default(test); 1769 break; 1770 case TEST_TYPE_RX_POLL: 1771 test->ifobj_rx->use_poll = true; 1772 test_spec_set_name(test, "POLL_RX"); 1773 testapp_validate_traffic(test); 1774 break; 1775 case TEST_TYPE_TX_POLL: 1776 test->ifobj_tx->use_poll = true; 1777 test_spec_set_name(test, "POLL_TX"); 1778 testapp_validate_traffic(test); 1779 break; 1780 case TEST_TYPE_POLL_TXQ_TMOUT: 1781 test_spec_set_name(test, "POLL_TXQ_FULL"); 1782 test->ifobj_tx->use_poll = true; 1783 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ 1784 test->ifobj_tx->umem->frame_size = 2048; 1785 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); 1786 testapp_validate_traffic_single_thread(test, test->ifobj_tx, type); 1787 pkt_stream_restore_default(test); 1788 break; 1789 case TEST_TYPE_POLL_RXQ_TMOUT: 1790 test_spec_set_name(test, "POLL_RXQ_EMPTY"); 1791 test->ifobj_rx->use_poll = true; 1792 testapp_validate_traffic_single_thread(test, test->ifobj_rx, type); 1793 break; 1794 case TEST_TYPE_ALIGNED_INV_DESC: 1795 test_spec_set_name(test, "ALIGNED_INV_DESC"); 1796 testapp_invalid_desc(test); 1797 break; 1798 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME: 1799 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE"); 1800 test->ifobj_tx->umem->frame_size = 2048; 1801 test->ifobj_rx->umem->frame_size = 2048; 1802 testapp_invalid_desc(test); 1803 break; 1804 case TEST_TYPE_UNALIGNED_INV_DESC: 1805 if (!hugepages_present(test->ifobj_tx)) { 1806 ksft_test_result_skip("No 2M huge pages present.\n"); 1807 return; 1808 } 1809 test_spec_set_name(test, "UNALIGNED_INV_DESC"); 1810 test->ifobj_tx->umem->unaligned_mode = true; 1811 test->ifobj_rx->umem->unaligned_mode = true; 1812 testapp_invalid_desc(test); 1813 break; 1814 case TEST_TYPE_UNALIGNED: 1815 if (!testapp_unaligned(test)) 1816 return; 1817 break; 1818 case TEST_TYPE_HEADROOM: 1819 testapp_headroom(test); 1820 break; 1821 default: 1822 break; 1823 } 1824 1825 if (!test->fail) 1826 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), 1827 test->name); 1828 } 1829 1830 static struct ifobject *ifobject_create(void) 1831 { 1832 struct ifobject *ifobj; 1833 1834 ifobj = calloc(1, sizeof(struct ifobject)); 1835 if (!ifobj) 1836 return NULL; 1837 1838 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); 1839 if (!ifobj->xsk_arr) 1840 goto out_xsk_arr; 1841 1842 ifobj->umem = calloc(1, sizeof(*ifobj->umem)); 1843 if (!ifobj->umem) 1844 goto out_umem; 1845 1846 ifobj->ns_fd = -1; 1847 1848 return ifobj; 1849 1850 out_umem: 1851 free(ifobj->xsk_arr); 1852 out_xsk_arr: 1853 free(ifobj); 1854 return NULL; 1855 } 1856 1857 static void ifobject_delete(struct ifobject *ifobj) 1858 { 1859 if (ifobj->ns_fd != -1) 1860 close(ifobj->ns_fd); 1861 free(ifobj->umem); 1862 free(ifobj->xsk_arr); 1863 free(ifobj); 1864 } 1865 1866 static bool is_xdp_supported(struct ifobject *ifobject) 1867 { 1868 int flags = XDP_FLAGS_DRV_MODE; 1869 1870 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); 1871 struct bpf_insn insns[2] = { 1872 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), 1873 BPF_EXIT_INSN() 1874 }; 1875 int ifindex = if_nametoindex(ifobject->ifname); 1876 int prog_fd, insn_cnt = ARRAY_SIZE(insns); 1877 int err; 1878 1879 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); 1880 if (prog_fd < 0) 1881 return false; 1882 1883 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); 1884 if (err) { 1885 close(prog_fd); 1886 return false; 1887 } 1888 1889 bpf_xdp_detach(ifindex, flags, NULL); 1890 close(prog_fd); 1891 1892 return true; 1893 } 1894 1895 int main(int argc, char **argv) 1896 { 1897 struct pkt_stream *rx_pkt_stream_default; 1898 struct pkt_stream *tx_pkt_stream_default; 1899 struct ifobject *ifobj_tx, *ifobj_rx; 1900 int modes = TEST_MODE_SKB + 1; 1901 u32 i, j, failed_tests = 0; 1902 struct test_spec test; 1903 bool shared_umem; 1904 1905 /* Use libbpf 1.0 API mode */ 1906 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 1907 1908 ifobj_tx = ifobject_create(); 1909 if (!ifobj_tx) 1910 exit_with_error(ENOMEM); 1911 ifobj_rx = ifobject_create(); 1912 if (!ifobj_rx) 1913 exit_with_error(ENOMEM); 1914 1915 setlocale(LC_ALL, ""); 1916 1917 parse_command_line(ifobj_tx, ifobj_rx, argc, argv); 1918 shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname); 1919 1920 ifobj_tx->shared_umem = shared_umem; 1921 ifobj_rx->shared_umem = shared_umem; 1922 1923 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) { 1924 usage(basename(argv[0])); 1925 ksft_exit_xfail(); 1926 } 1927 1928 init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, 1929 worker_testapp_validate_tx); 1930 init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, 1931 worker_testapp_validate_rx); 1932 1933 if (is_xdp_supported(ifobj_tx)) { 1934 modes++; 1935 if (ifobj_zc_avail(ifobj_tx)) 1936 modes++; 1937 } 1938 1939 test_spec_init(&test, ifobj_tx, ifobj_rx, 0); 1940 tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE); 1941 rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE); 1942 if (!tx_pkt_stream_default || !rx_pkt_stream_default) 1943 exit_with_error(ENOMEM); 1944 test.tx_pkt_stream_default = tx_pkt_stream_default; 1945 test.rx_pkt_stream_default = rx_pkt_stream_default; 1946 1947 ksft_set_plan(modes * TEST_TYPE_MAX); 1948 1949 for (i = 0; i < modes; i++) 1950 for (j = 0; j < TEST_TYPE_MAX; j++) { 1951 test_spec_init(&test, ifobj_tx, ifobj_rx, i); 1952 run_pkt_test(&test, i, j); 1953 usleep(USLEEP_MAX); 1954 1955 if (test.fail) 1956 failed_tests++; 1957 } 1958 1959 pkt_stream_delete(tx_pkt_stream_default); 1960 pkt_stream_delete(rx_pkt_stream_default); 1961 ifobject_delete(ifobj_tx); 1962 ifobject_delete(ifobj_rx); 1963 1964 if (failed_tests) 1965 ksft_exit_fail(); 1966 else 1967 ksft_exit_pass(); 1968 } 1969