1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2020 Intel Corporation. */
3 
4 /*
5  * Some functions in this program are taken from
6  * Linux kernel samples/bpf/xdpsock* and modified
7  * for use.
8  *
9  * See test_xsk.sh for detailed information on test topology
10  * and prerequisite network setup.
11  *
12  * This test program contains two threads, each thread is single socket with
13  * a unique UMEM. It validates in-order packet delivery and packet content
14  * by sending packets to each other.
15  *
16  * Tests Information:
17  * ------------------
18  * These selftests test AF_XDP SKB and Native/DRV modes using veth
19  * Virtual Ethernet interfaces.
20  *
21  * For each mode, the following tests are run:
22  *    a. nopoll - soft-irq processing in run-to-completion mode
23  *    b. poll - using poll() syscall
24  *    c. Socket Teardown
25  *       Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26  *       both sockets, then repeat multiple times. Only nopoll mode is used
27  *    d. Bi-directional sockets
28  *       Configure sockets as bi-directional tx/rx sockets, sets up fill and
29  *       completion rings on each socket, tx/rx in both directions. Only nopoll
30  *       mode is used
31  *    e. Statistics
32  *       Trigger some error conditions and ensure that the appropriate statistics
33  *       are incremented. Within this test, the following statistics are tested:
34  *       i.   rx dropped
35  *            Increase the UMEM frame headroom to a value which results in
36  *            insufficient space in the rx buffer for both the packet and the headroom.
37  *       ii.  tx invalid
38  *            Set the 'len' field of tx descriptors to an invalid value (umem frame
39  *            size + 1).
40  *       iii. rx ring full
41  *            Reduce the size of the RX ring to a fraction of the fill ring size.
42  *       iv.  fill queue empty
43  *            Do not populate the fill queue and then try to receive pkts.
44  *    f. bpf_link resource persistence
45  *       Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46  *       then remove xsk sockets from queue 0 on both veth interfaces and
47  *       finally run a traffic on queues ids 1
48  *    g. unaligned mode
49  *    h. tests for invalid and corner case Tx descriptors so that the correct ones
50  *       are discarded and let through, respectively.
51  *    i. 2K frame size tests
52  *
53  * Total tests: 12
54  *
55  * Flow:
56  * -----
57  * - Single process spawns two threads: Tx and Rx
58  * - Each of these two threads attach to a veth interface within their assigned
59  *   namespaces
60  * - Each thread Creates one AF_XDP socket connected to a unique umem for each
61  *   veth interface
62  * - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
63  * - Rx thread verifies if all 10k packets were received and delivered in-order,
64  *   and have the right content
65  *
66  * Enable/disable packet dump mode:
67  * --------------------------
68  * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
69  * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
70  */
71 
72 #define _GNU_SOURCE
73 #include <fcntl.h>
74 #include <errno.h>
75 #include <getopt.h>
76 #include <asm/barrier.h>
77 #include <linux/if_link.h>
78 #include <linux/if_ether.h>
79 #include <linux/ip.h>
80 #include <linux/udp.h>
81 #include <arpa/inet.h>
82 #include <net/if.h>
83 #include <locale.h>
84 #include <poll.h>
85 #include <pthread.h>
86 #include <signal.h>
87 #include <stdbool.h>
88 #include <stdio.h>
89 #include <stdlib.h>
90 #include <string.h>
91 #include <stddef.h>
92 #include <sys/mman.h>
93 #include <sys/socket.h>
94 #include <sys/time.h>
95 #include <sys/types.h>
96 #include <sys/queue.h>
97 #include <time.h>
98 #include <unistd.h>
99 #include <stdatomic.h>
100 #include "xsk.h"
101 #include "xskxceiver.h"
102 #include <bpf/bpf.h>
103 #include <linux/filter.h>
104 #include "../kselftest.h"
105 
106 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
107  * Until xskxceiver is either moved or re-writed into libxdp, suppress
108  * deprecation warnings in this file
109  */
110 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
111 
112 static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
113 static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
114 static const char *IP1 = "192.168.100.162";
115 static const char *IP2 = "192.168.100.161";
116 static const u16 UDP_PORT1 = 2020;
117 static const u16 UDP_PORT2 = 2121;
118 
119 static void __exit_with_error(int error, const char *file, const char *func, int line)
120 {
121 	ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
122 			      strerror(error));
123 	ksft_exit_xfail();
124 }
125 
126 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
127 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
128 static char *mode_string(struct test_spec *test)
129 {
130 	switch (test->mode) {
131 	case TEST_MODE_SKB:
132 		return "SKB";
133 	case TEST_MODE_DRV:
134 		return "DRV";
135 	case TEST_MODE_ZC:
136 		return "ZC";
137 	default:
138 		return "BOGUS";
139 	}
140 }
141 
142 static void report_failure(struct test_spec *test)
143 {
144 	if (test->fail)
145 		return;
146 
147 	ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
148 			      test->name);
149 	test->fail = true;
150 }
151 
152 static void memset32_htonl(void *dest, u32 val, u32 size)
153 {
154 	u32 *ptr = (u32 *)dest;
155 	int i;
156 
157 	val = htonl(val);
158 
159 	for (i = 0; i < (size & (~0x3)); i += 4)
160 		ptr[i >> 2] = val;
161 }
162 
163 /*
164  * Fold a partial checksum
165  * This function code has been taken from
166  * Linux kernel include/asm-generic/checksum.h
167  */
168 static __u16 csum_fold(__u32 csum)
169 {
170 	u32 sum = (__force u32)csum;
171 
172 	sum = (sum & 0xffff) + (sum >> 16);
173 	sum = (sum & 0xffff) + (sum >> 16);
174 	return (__force __u16)~sum;
175 }
176 
177 /*
178  * This function code has been taken from
179  * Linux kernel lib/checksum.c
180  */
181 static u32 from64to32(u64 x)
182 {
183 	/* add up 32-bit and 32-bit for 32+c bit */
184 	x = (x & 0xffffffff) + (x >> 32);
185 	/* add up carry.. */
186 	x = (x & 0xffffffff) + (x >> 32);
187 	return (u32)x;
188 }
189 
190 /*
191  * This function code has been taken from
192  * Linux kernel lib/checksum.c
193  */
194 static __u32 csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
195 {
196 	unsigned long long s = (__force u32)sum;
197 
198 	s += (__force u32)saddr;
199 	s += (__force u32)daddr;
200 #ifdef __BIG_ENDIAN__
201 	s += proto + len;
202 #else
203 	s += (proto + len) << 8;
204 #endif
205 	return (__force __u32)from64to32(s);
206 }
207 
208 /*
209  * This function has been taken from
210  * Linux kernel include/asm-generic/checksum.h
211  */
212 static __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __u32 sum)
213 {
214 	return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum));
215 }
216 
217 static u16 udp_csum(u32 saddr, u32 daddr, u32 len, u8 proto, u16 *udp_pkt)
218 {
219 	u32 csum = 0;
220 	u32 cnt = 0;
221 
222 	/* udp hdr and data */
223 	for (; cnt < len; cnt += 2)
224 		csum += udp_pkt[cnt >> 1];
225 
226 	return csum_tcpudp_magic(saddr, daddr, len, proto, csum);
227 }
228 
229 static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
230 {
231 	memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
232 	memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
233 	eth_hdr->h_proto = htons(ETH_P_IP);
234 }
235 
236 static void gen_ip_hdr(struct ifobject *ifobject, struct iphdr *ip_hdr)
237 {
238 	ip_hdr->version = IP_PKT_VER;
239 	ip_hdr->ihl = 0x5;
240 	ip_hdr->tos = IP_PKT_TOS;
241 	ip_hdr->tot_len = htons(IP_PKT_SIZE);
242 	ip_hdr->id = 0;
243 	ip_hdr->frag_off = 0;
244 	ip_hdr->ttl = IPDEFTTL;
245 	ip_hdr->protocol = IPPROTO_UDP;
246 	ip_hdr->saddr = ifobject->src_ip;
247 	ip_hdr->daddr = ifobject->dst_ip;
248 	ip_hdr->check = 0;
249 }
250 
251 static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
252 			struct udphdr *udp_hdr)
253 {
254 	udp_hdr->source = htons(ifobject->src_port);
255 	udp_hdr->dest = htons(ifobject->dst_port);
256 	udp_hdr->len = htons(UDP_PKT_SIZE);
257 	memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
258 }
259 
260 static bool is_umem_valid(struct ifobject *ifobj)
261 {
262 	return !!ifobj->umem->umem;
263 }
264 
265 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
266 {
267 	udp_hdr->check = 0;
268 	udp_hdr->check =
269 	    udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
270 }
271 
272 static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
273 {
274 	struct xsk_umem_config cfg = {
275 		.fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
276 		.comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
277 		.frame_size = umem->frame_size,
278 		.frame_headroom = umem->frame_headroom,
279 		.flags = XSK_UMEM__DEFAULT_FLAGS
280 	};
281 	int ret;
282 
283 	if (umem->unaligned_mode)
284 		cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
285 
286 	ret = xsk_umem__create(&umem->umem, buffer, size,
287 			       &umem->fq, &umem->cq, &cfg);
288 	if (ret)
289 		return ret;
290 
291 	umem->buffer = buffer;
292 	return 0;
293 }
294 
295 static void enable_busy_poll(struct xsk_socket_info *xsk)
296 {
297 	int sock_opt;
298 
299 	sock_opt = 1;
300 	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
301 		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
302 		exit_with_error(errno);
303 
304 	sock_opt = 20;
305 	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
306 		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
307 		exit_with_error(errno);
308 
309 	sock_opt = BATCH_SIZE;
310 	if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
311 		       (void *)&sock_opt, sizeof(sock_opt)) < 0)
312 		exit_with_error(errno);
313 }
314 
315 static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
316 				  struct ifobject *ifobject, bool shared)
317 {
318 	struct xsk_socket_config cfg = {};
319 	struct xsk_ring_cons *rxr;
320 	struct xsk_ring_prod *txr;
321 
322 	xsk->umem = umem;
323 	cfg.rx_size = xsk->rxqsize;
324 	cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
325 	cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
326 	cfg.xdp_flags = ifobject->xdp_flags;
327 	cfg.bind_flags = ifobject->bind_flags;
328 	if (shared)
329 		cfg.bind_flags |= XDP_SHARED_UMEM;
330 
331 	txr = ifobject->tx_on ? &xsk->tx : NULL;
332 	rxr = ifobject->rx_on ? &xsk->rx : NULL;
333 	return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
334 }
335 
336 static bool ifobj_zc_avail(struct ifobject *ifobject)
337 {
338 	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
339 	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
340 	struct xsk_socket_info *xsk;
341 	struct xsk_umem_info *umem;
342 	bool zc_avail = false;
343 	void *bufs;
344 	int ret;
345 
346 	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
347 	if (bufs == MAP_FAILED)
348 		exit_with_error(errno);
349 
350 	umem = calloc(1, sizeof(struct xsk_umem_info));
351 	if (!umem) {
352 		munmap(bufs, umem_sz);
353 		exit_with_error(-ENOMEM);
354 	}
355 	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
356 	ret = xsk_configure_umem(umem, bufs, umem_sz);
357 	if (ret)
358 		exit_with_error(-ret);
359 
360 	xsk = calloc(1, sizeof(struct xsk_socket_info));
361 	if (!xsk)
362 		goto out;
363 	ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
364 	ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE;
365 	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
366 	ifobject->rx_on = true;
367 	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
368 	ret = __xsk_configure_socket(xsk, umem, ifobject, false);
369 	if (!ret)
370 		zc_avail = true;
371 
372 	xsk_socket__delete(xsk->xsk);
373 	free(xsk);
374 out:
375 	munmap(umem->buffer, umem_sz);
376 	xsk_umem__delete(umem->umem);
377 	free(umem);
378 	return zc_avail;
379 }
380 
381 static struct option long_options[] = {
382 	{"interface", required_argument, 0, 'i'},
383 	{"busy-poll", no_argument, 0, 'b'},
384 	{"dump-pkts", no_argument, 0, 'D'},
385 	{"verbose", no_argument, 0, 'v'},
386 	{0, 0, 0, 0}
387 };
388 
389 static void usage(const char *prog)
390 {
391 	const char *str =
392 		"  Usage: %s [OPTIONS]\n"
393 		"  Options:\n"
394 		"  -i, --interface      Use interface\n"
395 		"  -D, --dump-pkts      Dump packets L2 - L5\n"
396 		"  -v, --verbose        Verbose output\n"
397 		"  -b, --busy-poll      Enable busy poll\n";
398 
399 	ksft_print_msg(str, prog);
400 }
401 
402 static int switch_namespace(const char *nsname)
403 {
404 	char fqns[26] = "/var/run/netns/";
405 	int nsfd;
406 
407 	if (!nsname || strlen(nsname) == 0)
408 		return -1;
409 
410 	strncat(fqns, nsname, sizeof(fqns) - strlen(fqns) - 1);
411 	nsfd = open(fqns, O_RDONLY);
412 
413 	if (nsfd == -1)
414 		exit_with_error(errno);
415 
416 	if (setns(nsfd, 0) == -1)
417 		exit_with_error(errno);
418 
419 	print_verbose("NS switched: %s\n", nsname);
420 
421 	return nsfd;
422 }
423 
424 static bool validate_interface(struct ifobject *ifobj)
425 {
426 	if (!strcmp(ifobj->ifname, ""))
427 		return false;
428 	return true;
429 }
430 
431 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
432 			       char **argv)
433 {
434 	struct ifobject *ifobj;
435 	u32 interface_nb = 0;
436 	int option_index, c;
437 
438 	opterr = 0;
439 
440 	for (;;) {
441 		char *sptr, *token;
442 
443 		c = getopt_long(argc, argv, "i:Dvb", long_options, &option_index);
444 		if (c == -1)
445 			break;
446 
447 		switch (c) {
448 		case 'i':
449 			if (interface_nb == 0)
450 				ifobj = ifobj_tx;
451 			else if (interface_nb == 1)
452 				ifobj = ifobj_rx;
453 			else
454 				break;
455 
456 			sptr = strndupa(optarg, strlen(optarg));
457 			memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
458 			token = strsep(&sptr, ",");
459 			if (token)
460 				memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS);
461 			interface_nb++;
462 			break;
463 		case 'D':
464 			opt_pkt_dump = true;
465 			break;
466 		case 'v':
467 			opt_verbose = true;
468 			break;
469 		case 'b':
470 			ifobj_tx->busy_poll = true;
471 			ifobj_rx->busy_poll = true;
472 			break;
473 		default:
474 			usage(basename(argv[0]));
475 			ksft_exit_xfail();
476 		}
477 	}
478 }
479 
480 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
481 			     struct ifobject *ifobj_rx)
482 {
483 	u32 i, j;
484 
485 	for (i = 0; i < MAX_INTERFACES; i++) {
486 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
487 
488 		ifobj->xsk = &ifobj->xsk_arr[0];
489 		ifobj->use_poll = false;
490 		ifobj->use_fill_ring = true;
491 		ifobj->release_rx = true;
492 		ifobj->validation_func = NULL;
493 
494 		if (i == 0) {
495 			ifobj->rx_on = false;
496 			ifobj->tx_on = true;
497 			ifobj->pkt_stream = test->tx_pkt_stream_default;
498 		} else {
499 			ifobj->rx_on = true;
500 			ifobj->tx_on = false;
501 			ifobj->pkt_stream = test->rx_pkt_stream_default;
502 		}
503 
504 		memset(ifobj->umem, 0, sizeof(*ifobj->umem));
505 		ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
506 		ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
507 		if (ifobj->shared_umem && ifobj->rx_on)
508 			ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
509 				XSK_UMEM__DEFAULT_FRAME_SIZE;
510 
511 		for (j = 0; j < MAX_SOCKETS; j++) {
512 			memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
513 			ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
514 		}
515 	}
516 
517 	test->ifobj_tx = ifobj_tx;
518 	test->ifobj_rx = ifobj_rx;
519 	test->current_step = 0;
520 	test->total_steps = 1;
521 	test->nb_sockets = 1;
522 	test->fail = false;
523 }
524 
525 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
526 			   struct ifobject *ifobj_rx, enum test_mode mode)
527 {
528 	struct pkt_stream *tx_pkt_stream;
529 	struct pkt_stream *rx_pkt_stream;
530 	u32 i;
531 
532 	tx_pkt_stream = test->tx_pkt_stream_default;
533 	rx_pkt_stream = test->rx_pkt_stream_default;
534 	memset(test, 0, sizeof(*test));
535 	test->tx_pkt_stream_default = tx_pkt_stream;
536 	test->rx_pkt_stream_default = rx_pkt_stream;
537 
538 	for (i = 0; i < MAX_INTERFACES; i++) {
539 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
540 
541 		ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
542 		if (mode == TEST_MODE_SKB)
543 			ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE;
544 		else
545 			ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
546 
547 		ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
548 		if (mode == TEST_MODE_ZC)
549 			ifobj->bind_flags |= XDP_ZEROCOPY;
550 		else
551 			ifobj->bind_flags |= XDP_COPY;
552 	}
553 
554 	test->mode = mode;
555 	__test_spec_init(test, ifobj_tx, ifobj_rx);
556 }
557 
558 static void test_spec_reset(struct test_spec *test)
559 {
560 	__test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
561 }
562 
563 static void test_spec_set_name(struct test_spec *test, const char *name)
564 {
565 	strncpy(test->name, name, MAX_TEST_NAME_SIZE);
566 }
567 
568 static void pkt_stream_reset(struct pkt_stream *pkt_stream)
569 {
570 	if (pkt_stream)
571 		pkt_stream->rx_pkt_nb = 0;
572 }
573 
574 static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
575 {
576 	if (pkt_nb >= pkt_stream->nb_pkts)
577 		return NULL;
578 
579 	return &pkt_stream->pkts[pkt_nb];
580 }
581 
582 static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
583 {
584 	while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
585 		(*pkts_sent)++;
586 		if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
587 			return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
588 		pkt_stream->rx_pkt_nb++;
589 	}
590 	return NULL;
591 }
592 
593 static void pkt_stream_delete(struct pkt_stream *pkt_stream)
594 {
595 	free(pkt_stream->pkts);
596 	free(pkt_stream);
597 }
598 
599 static void pkt_stream_restore_default(struct test_spec *test)
600 {
601 	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
602 	struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
603 
604 	if (tx_pkt_stream != test->tx_pkt_stream_default) {
605 		pkt_stream_delete(test->ifobj_tx->pkt_stream);
606 		test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
607 	}
608 
609 	if (rx_pkt_stream != test->rx_pkt_stream_default) {
610 		pkt_stream_delete(test->ifobj_rx->pkt_stream);
611 		test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
612 	}
613 }
614 
615 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
616 {
617 	struct pkt_stream *pkt_stream;
618 
619 	pkt_stream = calloc(1, sizeof(*pkt_stream));
620 	if (!pkt_stream)
621 		return NULL;
622 
623 	pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
624 	if (!pkt_stream->pkts) {
625 		free(pkt_stream);
626 		return NULL;
627 	}
628 
629 	pkt_stream->nb_pkts = nb_pkts;
630 	return pkt_stream;
631 }
632 
633 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
634 {
635 	pkt->addr = addr + umem->base_addr;
636 	pkt->len = len;
637 	if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
638 		pkt->valid = false;
639 	else
640 		pkt->valid = true;
641 }
642 
643 static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
644 {
645 	struct pkt_stream *pkt_stream;
646 	u32 i;
647 
648 	pkt_stream = __pkt_stream_alloc(nb_pkts);
649 	if (!pkt_stream)
650 		exit_with_error(ENOMEM);
651 
652 	pkt_stream->nb_pkts = nb_pkts;
653 	for (i = 0; i < nb_pkts; i++) {
654 		pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size,
655 			pkt_len);
656 		pkt_stream->pkts[i].payload = i;
657 	}
658 
659 	return pkt_stream;
660 }
661 
662 static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
663 					   struct pkt_stream *pkt_stream)
664 {
665 	return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
666 }
667 
668 static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
669 {
670 	struct pkt_stream *pkt_stream;
671 
672 	pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
673 	test->ifobj_tx->pkt_stream = pkt_stream;
674 	pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
675 	test->ifobj_rx->pkt_stream = pkt_stream;
676 }
677 
678 static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
679 				      int offset)
680 {
681 	struct xsk_umem_info *umem = ifobj->umem;
682 	struct pkt_stream *pkt_stream;
683 	u32 i;
684 
685 	pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
686 	for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
687 		pkt_set(umem, &pkt_stream->pkts[i],
688 			(i % umem->num_frames) * umem->frame_size + offset, pkt_len);
689 
690 	ifobj->pkt_stream = pkt_stream;
691 }
692 
693 static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
694 {
695 	__pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
696 	__pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
697 }
698 
699 static void pkt_stream_receive_half(struct test_spec *test)
700 {
701 	struct xsk_umem_info *umem = test->ifobj_rx->umem;
702 	struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
703 	u32 i;
704 
705 	test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
706 							 pkt_stream->pkts[0].len);
707 	pkt_stream = test->ifobj_rx->pkt_stream;
708 	for (i = 1; i < pkt_stream->nb_pkts; i += 2)
709 		pkt_stream->pkts[i].valid = false;
710 }
711 
712 static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
713 {
714 	struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
715 	struct udphdr *udp_hdr;
716 	struct ethhdr *eth_hdr;
717 	struct iphdr *ip_hdr;
718 	void *data;
719 
720 	if (!pkt)
721 		return NULL;
722 	if (!pkt->valid || pkt->len < MIN_PKT_SIZE)
723 		return pkt;
724 
725 	data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
726 	udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
727 	ip_hdr = (struct iphdr *)(data + sizeof(struct ethhdr));
728 	eth_hdr = (struct ethhdr *)data;
729 
730 	gen_udp_hdr(pkt_nb, data, ifobject, udp_hdr);
731 	gen_ip_hdr(ifobject, ip_hdr);
732 	gen_udp_csum(udp_hdr, ip_hdr);
733 	gen_eth_hdr(ifobject, eth_hdr);
734 
735 	return pkt;
736 }
737 
738 static void __pkt_stream_generate_custom(struct ifobject *ifobj,
739 					 struct pkt *pkts, u32 nb_pkts)
740 {
741 	struct pkt_stream *pkt_stream;
742 	u32 i;
743 
744 	pkt_stream = __pkt_stream_alloc(nb_pkts);
745 	if (!pkt_stream)
746 		exit_with_error(ENOMEM);
747 
748 	for (i = 0; i < nb_pkts; i++) {
749 		pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
750 		pkt_stream->pkts[i].len = pkts[i].len;
751 		pkt_stream->pkts[i].payload = i;
752 		pkt_stream->pkts[i].valid = pkts[i].valid;
753 	}
754 
755 	ifobj->pkt_stream = pkt_stream;
756 }
757 
758 static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
759 {
760 	__pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
761 	__pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
762 }
763 
764 static void pkt_dump(void *pkt, u32 len)
765 {
766 	char s[INET_ADDRSTRLEN];
767 	struct ethhdr *ethhdr;
768 	struct udphdr *udphdr;
769 	struct iphdr *iphdr;
770 	int payload, i;
771 
772 	ethhdr = pkt;
773 	iphdr = pkt + sizeof(*ethhdr);
774 	udphdr = pkt + sizeof(*ethhdr) + sizeof(*iphdr);
775 
776 	/*extract L2 frame */
777 	fprintf(stdout, "DEBUG>> L2: dst mac: ");
778 	for (i = 0; i < ETH_ALEN; i++)
779 		fprintf(stdout, "%02X", ethhdr->h_dest[i]);
780 
781 	fprintf(stdout, "\nDEBUG>> L2: src mac: ");
782 	for (i = 0; i < ETH_ALEN; i++)
783 		fprintf(stdout, "%02X", ethhdr->h_source[i]);
784 
785 	/*extract L3 frame */
786 	fprintf(stdout, "\nDEBUG>> L3: ip_hdr->ihl: %02X\n", iphdr->ihl);
787 	fprintf(stdout, "DEBUG>> L3: ip_hdr->saddr: %s\n",
788 		inet_ntop(AF_INET, &iphdr->saddr, s, sizeof(s)));
789 	fprintf(stdout, "DEBUG>> L3: ip_hdr->daddr: %s\n",
790 		inet_ntop(AF_INET, &iphdr->daddr, s, sizeof(s)));
791 	/*extract L4 frame */
792 	fprintf(stdout, "DEBUG>> L4: udp_hdr->src: %d\n", ntohs(udphdr->source));
793 	fprintf(stdout, "DEBUG>> L4: udp_hdr->dst: %d\n", ntohs(udphdr->dest));
794 	/*extract L5 frame */
795 	payload = *((uint32_t *)(pkt + PKT_HDR_SIZE));
796 
797 	fprintf(stdout, "DEBUG>> L5: payload: %d\n", payload);
798 	fprintf(stdout, "---------------------------------------\n");
799 }
800 
801 static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, u64 addr,
802 			      u64 pkt_stream_addr)
803 {
804 	u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
805 	u32 offset = addr % umem->frame_size, expected_offset = 0;
806 
807 	if (!pkt_stream->use_addr_for_fill)
808 		pkt_stream_addr = 0;
809 
810 	expected_offset += (pkt_stream_addr + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
811 
812 	if (offset == expected_offset)
813 		return true;
814 
815 	ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
816 	return false;
817 }
818 
819 static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
820 {
821 	void *data = xsk_umem__get_data(buffer, addr);
822 	struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
823 
824 	if (!pkt) {
825 		ksft_print_msg("[%s] too many packets received\n", __func__);
826 		return false;
827 	}
828 
829 	if (len < MIN_PKT_SIZE || pkt->len < MIN_PKT_SIZE) {
830 		/* Do not try to verify packets that are smaller than minimum size. */
831 		return true;
832 	}
833 
834 	if (pkt->len != len) {
835 		ksft_print_msg("[%s] expected length [%d], got length [%d]\n",
836 			       __func__, pkt->len, len);
837 		return false;
838 	}
839 
840 	if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
841 		u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
842 
843 		if (opt_pkt_dump)
844 			pkt_dump(data, PKT_SIZE);
845 
846 		if (pkt->payload != seqnum) {
847 			ksft_print_msg("[%s] expected seqnum [%d], got seqnum [%d]\n",
848 				       __func__, pkt->payload, seqnum);
849 			return false;
850 		}
851 	} else {
852 		ksft_print_msg("Invalid frame received: ");
853 		ksft_print_msg("[IP_PKT_VER: %02X], [IP_PKT_TOS: %02X]\n", iphdr->version,
854 			       iphdr->tos);
855 		return false;
856 	}
857 
858 	return true;
859 }
860 
861 static void kick_tx(struct xsk_socket_info *xsk)
862 {
863 	int ret;
864 
865 	ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
866 	if (ret >= 0)
867 		return;
868 	if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
869 		usleep(100);
870 		return;
871 	}
872 	exit_with_error(errno);
873 }
874 
875 static void kick_rx(struct xsk_socket_info *xsk)
876 {
877 	int ret;
878 
879 	ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
880 	if (ret < 0)
881 		exit_with_error(errno);
882 }
883 
884 static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
885 {
886 	unsigned int rcvd;
887 	u32 idx;
888 
889 	if (xsk_ring_prod__needs_wakeup(&xsk->tx))
890 		kick_tx(xsk);
891 
892 	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
893 	if (rcvd) {
894 		if (rcvd > xsk->outstanding_tx) {
895 			u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
896 
897 			ksft_print_msg("[%s] Too many packets completed\n", __func__);
898 			ksft_print_msg("Last completion address: %llx\n", addr);
899 			return TEST_FAILURE;
900 		}
901 
902 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);
903 		xsk->outstanding_tx -= rcvd;
904 	}
905 
906 	return TEST_PASS;
907 }
908 
909 static int receive_pkts(struct test_spec *test, struct pollfd *fds)
910 {
911 	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
912 	struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
913 	u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
914 	struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
915 	struct ifobject *ifobj = test->ifobj_rx;
916 	struct xsk_umem_info *umem = xsk->umem;
917 	struct pkt *pkt;
918 	int ret;
919 
920 	ret = gettimeofday(&tv_now, NULL);
921 	if (ret)
922 		exit_with_error(errno);
923 	timeradd(&tv_now, &tv_timeout, &tv_end);
924 
925 	pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
926 	while (pkt) {
927 		ret = gettimeofday(&tv_now, NULL);
928 		if (ret)
929 			exit_with_error(errno);
930 		if (timercmp(&tv_now, &tv_end, >)) {
931 			ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
932 			return TEST_FAILURE;
933 		}
934 
935 		kick_rx(xsk);
936 		if (ifobj->use_poll) {
937 			ret = poll(fds, 1, POLL_TMOUT);
938 			if (ret < 0)
939 				exit_with_error(-ret);
940 
941 			if (!ret) {
942 				if (!is_umem_valid(test->ifobj_tx))
943 					return TEST_PASS;
944 
945 				ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
946 				return TEST_FAILURE;
947 
948 			}
949 
950 			if (!(fds->revents & POLLIN))
951 				continue;
952 		}
953 
954 		rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
955 		if (!rcvd)
956 			continue;
957 
958 		if (ifobj->use_fill_ring) {
959 			ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
960 			while (ret != rcvd) {
961 				if (ret < 0)
962 					exit_with_error(-ret);
963 				if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
964 					ret = poll(fds, 1, POLL_TMOUT);
965 					if (ret < 0)
966 						exit_with_error(-ret);
967 				}
968 				ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
969 			}
970 		}
971 
972 		for (i = 0; i < rcvd; i++) {
973 			const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
974 			u64 addr = desc->addr, orig;
975 
976 			orig = xsk_umem__extract_addr(addr);
977 			addr = xsk_umem__add_offset_to_addr(addr);
978 
979 			if (!is_pkt_valid(pkt, umem->buffer, addr, desc->len) ||
980 			    !is_offset_correct(umem, pkt_stream, addr, pkt->addr))
981 				return TEST_FAILURE;
982 
983 			if (ifobj->use_fill_ring)
984 				*xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
985 			pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
986 		}
987 
988 		if (ifobj->use_fill_ring)
989 			xsk_ring_prod__submit(&umem->fq, rcvd);
990 		if (ifobj->release_rx)
991 			xsk_ring_cons__release(&xsk->rx, rcvd);
992 
993 		pthread_mutex_lock(&pacing_mutex);
994 		pkts_in_flight -= pkts_sent;
995 		if (pkts_in_flight < umem->num_frames)
996 			pthread_cond_signal(&pacing_cond);
997 		pthread_mutex_unlock(&pacing_mutex);
998 		pkts_sent = 0;
999 	}
1000 
1001 	return TEST_PASS;
1002 }
1003 
1004 static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,
1005 		       bool timeout)
1006 {
1007 	struct xsk_socket_info *xsk = ifobject->xsk;
1008 	bool use_poll = ifobject->use_poll;
1009 	u32 i, idx = 0, ret, valid_pkts = 0;
1010 
1011 	while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
1012 		if (use_poll) {
1013 			ret = poll(fds, 1, POLL_TMOUT);
1014 			if (timeout) {
1015 				if (ret < 0) {
1016 					ksft_print_msg("ERROR: [%s] Poll error %d\n",
1017 						       __func__, ret);
1018 					return TEST_FAILURE;
1019 				}
1020 				if (ret == 0)
1021 					return TEST_PASS;
1022 				break;
1023 			}
1024 			if (ret <= 0) {
1025 				ksft_print_msg("ERROR: [%s] Poll error %d\n",
1026 					       __func__, ret);
1027 				return TEST_FAILURE;
1028 			}
1029 		}
1030 
1031 		complete_pkts(xsk, BATCH_SIZE);
1032 	}
1033 
1034 	for (i = 0; i < BATCH_SIZE; i++) {
1035 		struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1036 		struct pkt *pkt = pkt_generate(ifobject, *pkt_nb);
1037 
1038 		if (!pkt)
1039 			break;
1040 
1041 		tx_desc->addr = pkt->addr;
1042 		tx_desc->len = pkt->len;
1043 		(*pkt_nb)++;
1044 		if (pkt->valid)
1045 			valid_pkts++;
1046 	}
1047 
1048 	pthread_mutex_lock(&pacing_mutex);
1049 	pkts_in_flight += valid_pkts;
1050 	/* pkts_in_flight might be negative if many invalid packets are sent */
1051 	if (pkts_in_flight >= (int)(ifobject->umem->num_frames - BATCH_SIZE)) {
1052 		kick_tx(xsk);
1053 		pthread_cond_wait(&pacing_cond, &pacing_mutex);
1054 	}
1055 	pthread_mutex_unlock(&pacing_mutex);
1056 
1057 	xsk_ring_prod__submit(&xsk->tx, i);
1058 	xsk->outstanding_tx += valid_pkts;
1059 
1060 	if (use_poll) {
1061 		ret = poll(fds, 1, POLL_TMOUT);
1062 		if (ret <= 0) {
1063 			if (ret == 0 && timeout)
1064 				return TEST_PASS;
1065 
1066 			ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1067 			return TEST_FAILURE;
1068 		}
1069 	}
1070 
1071 	if (!timeout) {
1072 		if (complete_pkts(xsk, i))
1073 			return TEST_FAILURE;
1074 
1075 		usleep(10);
1076 		return TEST_PASS;
1077 	}
1078 
1079 	return TEST_CONTINUE;
1080 }
1081 
1082 static void wait_for_tx_completion(struct xsk_socket_info *xsk)
1083 {
1084 	while (xsk->outstanding_tx)
1085 		complete_pkts(xsk, BATCH_SIZE);
1086 }
1087 
1088 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1089 {
1090 	bool timeout = !is_umem_valid(test->ifobj_rx);
1091 	struct pollfd fds = { };
1092 	u32 pkt_cnt = 0, ret;
1093 
1094 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1095 	fds.events = POLLOUT;
1096 
1097 	while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
1098 		ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);
1099 		if ((ret || test->fail) && !timeout)
1100 			return TEST_FAILURE;
1101 		else if (ret == TEST_PASS && timeout)
1102 			return ret;
1103 	}
1104 
1105 	wait_for_tx_completion(ifobject->xsk);
1106 	return TEST_PASS;
1107 }
1108 
1109 static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1110 {
1111 	int fd = xsk_socket__fd(xsk), err;
1112 	socklen_t optlen, expected_len;
1113 
1114 	optlen = sizeof(*stats);
1115 	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1116 	if (err) {
1117 		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1118 			       __func__, -err, strerror(-err));
1119 		return TEST_FAILURE;
1120 	}
1121 
1122 	expected_len = sizeof(struct xdp_statistics);
1123 	if (optlen != expected_len) {
1124 		ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1125 			       __func__, expected_len, optlen);
1126 		return TEST_FAILURE;
1127 	}
1128 
1129 	return TEST_PASS;
1130 }
1131 
1132 static int validate_rx_dropped(struct ifobject *ifobject)
1133 {
1134 	struct xsk_socket *xsk = ifobject->xsk->xsk;
1135 	struct xdp_statistics stats;
1136 	int err;
1137 
1138 	kick_rx(ifobject->xsk);
1139 
1140 	err = get_xsk_stats(xsk, &stats);
1141 	if (err)
1142 		return TEST_FAILURE;
1143 
1144 	if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2)
1145 		return TEST_PASS;
1146 
1147 	return TEST_FAILURE;
1148 }
1149 
1150 static int validate_rx_full(struct ifobject *ifobject)
1151 {
1152 	struct xsk_socket *xsk = ifobject->xsk->xsk;
1153 	struct xdp_statistics stats;
1154 	int err;
1155 
1156 	usleep(1000);
1157 	kick_rx(ifobject->xsk);
1158 
1159 	err = get_xsk_stats(xsk, &stats);
1160 	if (err)
1161 		return TEST_FAILURE;
1162 
1163 	if (stats.rx_ring_full)
1164 		return TEST_PASS;
1165 
1166 	return TEST_FAILURE;
1167 }
1168 
1169 static int validate_fill_empty(struct ifobject *ifobject)
1170 {
1171 	struct xsk_socket *xsk = ifobject->xsk->xsk;
1172 	struct xdp_statistics stats;
1173 	int err;
1174 
1175 	usleep(1000);
1176 	kick_rx(ifobject->xsk);
1177 
1178 	err = get_xsk_stats(xsk, &stats);
1179 	if (err)
1180 		return TEST_FAILURE;
1181 
1182 	if (stats.rx_fill_ring_empty_descs)
1183 		return TEST_PASS;
1184 
1185 	return TEST_FAILURE;
1186 }
1187 
1188 static int validate_tx_invalid_descs(struct ifobject *ifobject)
1189 {
1190 	struct xsk_socket *xsk = ifobject->xsk->xsk;
1191 	int fd = xsk_socket__fd(xsk);
1192 	struct xdp_statistics stats;
1193 	socklen_t optlen;
1194 	int err;
1195 
1196 	optlen = sizeof(stats);
1197 	err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1198 	if (err) {
1199 		ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1200 			       __func__, -err, strerror(-err));
1201 		return TEST_FAILURE;
1202 	}
1203 
1204 	if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1205 		ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1206 			       __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1207 		return TEST_FAILURE;
1208 	}
1209 
1210 	return TEST_PASS;
1211 }
1212 
1213 static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1214 				 struct xsk_umem_info *umem, bool tx)
1215 {
1216 	int i, ret;
1217 
1218 	for (i = 0; i < test->nb_sockets; i++) {
1219 		bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1220 		u32 ctr = 0;
1221 
1222 		while (ctr++ < SOCK_RECONF_CTR) {
1223 			ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1224 						     ifobject, shared);
1225 			if (!ret)
1226 				break;
1227 
1228 			/* Retry if it fails as xsk_socket__create() is asynchronous */
1229 			if (ctr >= SOCK_RECONF_CTR)
1230 				exit_with_error(-ret);
1231 			usleep(USLEEP_MAX);
1232 		}
1233 		if (ifobject->busy_poll)
1234 			enable_busy_poll(&ifobject->xsk_arr[i]);
1235 	}
1236 }
1237 
1238 static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1239 {
1240 	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1241 	ifobject->xsk = &ifobject->xsk_arr[0];
1242 	ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;
1243 	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1244 }
1245 
1246 static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
1247 {
1248 	u32 idx = 0, i, buffers_to_fill;
1249 	int ret;
1250 
1251 	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1252 		buffers_to_fill = umem->num_frames;
1253 	else
1254 		buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1255 
1256 	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1257 	if (ret != buffers_to_fill)
1258 		exit_with_error(ENOSPC);
1259 	for (i = 0; i < buffers_to_fill; i++) {
1260 		u64 addr;
1261 
1262 		if (pkt_stream->use_addr_for_fill) {
1263 			struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1264 
1265 			if (!pkt)
1266 				break;
1267 			addr = pkt->addr;
1268 		} else {
1269 			addr = i * umem->frame_size;
1270 		}
1271 
1272 		*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1273 	}
1274 	xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
1275 }
1276 
1277 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1278 {
1279 	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1280 	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1281 	LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1282 	int ret, ifindex;
1283 	void *bufs;
1284 
1285 	ifobject->ns_fd = switch_namespace(ifobject->nsname);
1286 
1287 	if (ifobject->umem->unaligned_mode)
1288 		mmap_flags |= MAP_HUGETLB;
1289 
1290 	if (ifobject->shared_umem)
1291 		umem_sz *= 2;
1292 
1293 	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1294 	if (bufs == MAP_FAILED)
1295 		exit_with_error(errno);
1296 
1297 	ret = xsk_configure_umem(ifobject->umem, bufs, umem_sz);
1298 	if (ret)
1299 		exit_with_error(-ret);
1300 
1301 	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1302 
1303 	xsk_configure_socket(test, ifobject, ifobject->umem, false);
1304 
1305 	ifobject->xsk = &ifobject->xsk_arr[0];
1306 
1307 	if (!ifobject->rx_on)
1308 		return;
1309 
1310 	ifindex = if_nametoindex(ifobject->ifname);
1311 	if (!ifindex)
1312 		exit_with_error(errno);
1313 
1314 	ret = xsk_setup_xdp_prog_xsk(ifobject->xsk->xsk, &ifobject->xsk_map_fd);
1315 	if (ret)
1316 		exit_with_error(-ret);
1317 
1318 	ret = bpf_xdp_query(ifindex, ifobject->xdp_flags, &opts);
1319 	if (ret)
1320 		exit_with_error(-ret);
1321 
1322 	if (ifobject->xdp_flags & XDP_FLAGS_SKB_MODE) {
1323 		if (opts.attach_mode != XDP_ATTACHED_SKB) {
1324 			ksft_print_msg("ERROR: [%s] XDP prog not in SKB mode\n");
1325 			exit_with_error(-EINVAL);
1326 		}
1327 	} else if (ifobject->xdp_flags & XDP_FLAGS_DRV_MODE) {
1328 		if (opts.attach_mode != XDP_ATTACHED_DRV) {
1329 			ksft_print_msg("ERROR: [%s] XDP prog not in DRV mode\n");
1330 			exit_with_error(-EINVAL);
1331 		}
1332 	}
1333 
1334 	ret = xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1335 	if (ret)
1336 		exit_with_error(-ret);
1337 }
1338 
1339 static void *worker_testapp_validate_tx(void *arg)
1340 {
1341 	struct test_spec *test = (struct test_spec *)arg;
1342 	struct ifobject *ifobject = test->ifobj_tx;
1343 	int err;
1344 
1345 	if (test->current_step == 1) {
1346 		if (!ifobject->shared_umem)
1347 			thread_common_ops(test, ifobject);
1348 		else
1349 			thread_common_ops_tx(test, ifobject);
1350 	}
1351 
1352 	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1353 		      ifobject->ifname);
1354 	err = send_pkts(test, ifobject);
1355 
1356 	if (!err && ifobject->validation_func)
1357 		err = ifobject->validation_func(ifobject);
1358 	if (err)
1359 		report_failure(test);
1360 
1361 	pthread_exit(NULL);
1362 }
1363 
1364 static void *worker_testapp_validate_rx(void *arg)
1365 {
1366 	struct test_spec *test = (struct test_spec *)arg;
1367 	struct ifobject *ifobject = test->ifobj_rx;
1368 	struct pollfd fds = { };
1369 	int id = 0;
1370 	int err;
1371 
1372 	if (test->current_step == 1) {
1373 		thread_common_ops(test, ifobject);
1374 	} else {
1375 		bpf_map_delete_elem(ifobject->xsk_map_fd, &id);
1376 		xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
1377 	}
1378 
1379 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1380 	fds.events = POLLIN;
1381 
1382 	pthread_barrier_wait(&barr);
1383 
1384 	err = receive_pkts(test, &fds);
1385 
1386 	if (!err && ifobject->validation_func)
1387 		err = ifobject->validation_func(ifobject);
1388 	if (err) {
1389 		report_failure(test);
1390 		pthread_mutex_lock(&pacing_mutex);
1391 		pthread_cond_signal(&pacing_cond);
1392 		pthread_mutex_unlock(&pacing_mutex);
1393 	}
1394 
1395 	pthread_exit(NULL);
1396 }
1397 
1398 static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1399 {
1400 	u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1401 
1402 	if (ifobj->shared_umem)
1403 		umem_sz *= 2;
1404 
1405 	xsk_umem__delete(ifobj->umem->umem);
1406 	munmap(ifobj->umem->buffer, umem_sz);
1407 }
1408 
1409 static void handler(int signum)
1410 {
1411 	pthread_exit(NULL);
1412 }
1413 
1414 static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj,
1415 						  enum test_type type)
1416 {
1417 	bool old_shared_umem = ifobj->shared_umem;
1418 	pthread_t t0;
1419 
1420 	if (pthread_barrier_init(&barr, NULL, 2))
1421 		exit_with_error(errno);
1422 
1423 	test->current_step++;
1424 	if (type == TEST_TYPE_POLL_RXQ_TMOUT)
1425 		pkt_stream_reset(ifobj->pkt_stream);
1426 	pkts_in_flight = 0;
1427 
1428 	test->ifobj_rx->shared_umem = false;
1429 	test->ifobj_tx->shared_umem = false;
1430 
1431 	signal(SIGUSR1, handler);
1432 	/* Spawn thread */
1433 	pthread_create(&t0, NULL, ifobj->func_ptr, test);
1434 
1435 	if (type != TEST_TYPE_POLL_TXQ_TMOUT)
1436 		pthread_barrier_wait(&barr);
1437 
1438 	if (pthread_barrier_destroy(&barr))
1439 		exit_with_error(errno);
1440 
1441 	pthread_kill(t0, SIGUSR1);
1442 	pthread_join(t0, NULL);
1443 
1444 	if (test->total_steps == test->current_step || test->fail) {
1445 		xsk_socket__delete(ifobj->xsk->xsk);
1446 		testapp_clean_xsk_umem(ifobj);
1447 	}
1448 
1449 	test->ifobj_rx->shared_umem = old_shared_umem;
1450 	test->ifobj_tx->shared_umem = old_shared_umem;
1451 
1452 	return !!test->fail;
1453 }
1454 
1455 static int testapp_validate_traffic(struct test_spec *test)
1456 {
1457 	struct ifobject *ifobj_tx = test->ifobj_tx;
1458 	struct ifobject *ifobj_rx = test->ifobj_rx;
1459 	pthread_t t0, t1;
1460 
1461 	if (pthread_barrier_init(&barr, NULL, 2))
1462 		exit_with_error(errno);
1463 
1464 	test->current_step++;
1465 	pkt_stream_reset(ifobj_rx->pkt_stream);
1466 	pkts_in_flight = 0;
1467 
1468 	/*Spawn RX thread */
1469 	pthread_create(&t0, NULL, ifobj_rx->func_ptr, test);
1470 
1471 	pthread_barrier_wait(&barr);
1472 	if (pthread_barrier_destroy(&barr))
1473 		exit_with_error(errno);
1474 
1475 	/*Spawn TX thread */
1476 	pthread_create(&t1, NULL, ifobj_tx->func_ptr, test);
1477 
1478 	pthread_join(t1, NULL);
1479 	pthread_join(t0, NULL);
1480 
1481 	if (test->total_steps == test->current_step || test->fail) {
1482 		xsk_socket__delete(ifobj_tx->xsk->xsk);
1483 		xsk_socket__delete(ifobj_rx->xsk->xsk);
1484 		testapp_clean_xsk_umem(ifobj_rx);
1485 		if (!ifobj_tx->shared_umem)
1486 			testapp_clean_xsk_umem(ifobj_tx);
1487 	}
1488 
1489 	return !!test->fail;
1490 }
1491 
1492 static void testapp_teardown(struct test_spec *test)
1493 {
1494 	int i;
1495 
1496 	test_spec_set_name(test, "TEARDOWN");
1497 	for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1498 		if (testapp_validate_traffic(test))
1499 			return;
1500 		test_spec_reset(test);
1501 	}
1502 }
1503 
1504 static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1505 {
1506 	thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1507 	struct ifobject *tmp_ifobj = (*ifobj1);
1508 
1509 	(*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1510 	(*ifobj2)->func_ptr = tmp_func_ptr;
1511 
1512 	*ifobj1 = *ifobj2;
1513 	*ifobj2 = tmp_ifobj;
1514 }
1515 
1516 static void testapp_bidi(struct test_spec *test)
1517 {
1518 	test_spec_set_name(test, "BIDIRECTIONAL");
1519 	test->ifobj_tx->rx_on = true;
1520 	test->ifobj_rx->tx_on = true;
1521 	test->total_steps = 2;
1522 	if (testapp_validate_traffic(test))
1523 		return;
1524 
1525 	print_verbose("Switching Tx/Rx vectors\n");
1526 	swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1527 	testapp_validate_traffic(test);
1528 
1529 	swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1530 }
1531 
1532 static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1533 {
1534 	int ret;
1535 
1536 	xsk_socket__delete(ifobj_tx->xsk->xsk);
1537 	xsk_socket__delete(ifobj_rx->xsk->xsk);
1538 	ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1539 	ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1540 
1541 	ret = xsk_socket__update_xskmap(ifobj_rx->xsk->xsk, ifobj_rx->xsk_map_fd);
1542 	if (ret)
1543 		exit_with_error(-ret);
1544 }
1545 
1546 static void testapp_bpf_res(struct test_spec *test)
1547 {
1548 	test_spec_set_name(test, "BPF_RES");
1549 	test->total_steps = 2;
1550 	test->nb_sockets = 2;
1551 	if (testapp_validate_traffic(test))
1552 		return;
1553 
1554 	swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1555 	testapp_validate_traffic(test);
1556 }
1557 
1558 static void testapp_headroom(struct test_spec *test)
1559 {
1560 	test_spec_set_name(test, "UMEM_HEADROOM");
1561 	test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1562 	testapp_validate_traffic(test);
1563 }
1564 
1565 static void testapp_stats_rx_dropped(struct test_spec *test)
1566 {
1567 	test_spec_set_name(test, "STAT_RX_DROPPED");
1568 	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1569 	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1570 		XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1571 	pkt_stream_receive_half(test);
1572 	test->ifobj_rx->validation_func = validate_rx_dropped;
1573 	testapp_validate_traffic(test);
1574 }
1575 
1576 static void testapp_stats_tx_invalid_descs(struct test_spec *test)
1577 {
1578 	test_spec_set_name(test, "STAT_TX_INVALID");
1579 	pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1580 	test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1581 	testapp_validate_traffic(test);
1582 
1583 	pkt_stream_restore_default(test);
1584 }
1585 
1586 static void testapp_stats_rx_full(struct test_spec *test)
1587 {
1588 	test_spec_set_name(test, "STAT_RX_FULL");
1589 	pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1590 	test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1591 							 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1592 	if (!test->ifobj_rx->pkt_stream)
1593 		exit_with_error(ENOMEM);
1594 
1595 	test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1596 	test->ifobj_rx->release_rx = false;
1597 	test->ifobj_rx->validation_func = validate_rx_full;
1598 	testapp_validate_traffic(test);
1599 
1600 	pkt_stream_restore_default(test);
1601 }
1602 
1603 static void testapp_stats_fill_empty(struct test_spec *test)
1604 {
1605 	test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1606 	pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, PKT_SIZE);
1607 	test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1608 							 DEFAULT_UMEM_BUFFERS, PKT_SIZE);
1609 	if (!test->ifobj_rx->pkt_stream)
1610 		exit_with_error(ENOMEM);
1611 
1612 	test->ifobj_rx->use_fill_ring = false;
1613 	test->ifobj_rx->validation_func = validate_fill_empty;
1614 	testapp_validate_traffic(test);
1615 
1616 	pkt_stream_restore_default(test);
1617 }
1618 
1619 /* Simple test */
1620 static bool hugepages_present(struct ifobject *ifobject)
1621 {
1622 	const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1623 	void *bufs;
1624 
1625 	bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1626 		    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
1627 	if (bufs == MAP_FAILED)
1628 		return false;
1629 
1630 	munmap(bufs, mmap_sz);
1631 	return true;
1632 }
1633 
1634 static bool testapp_unaligned(struct test_spec *test)
1635 {
1636 	if (!hugepages_present(test->ifobj_tx)) {
1637 		ksft_test_result_skip("No 2M huge pages present.\n");
1638 		return false;
1639 	}
1640 
1641 	test_spec_set_name(test, "UNALIGNED_MODE");
1642 	test->ifobj_tx->umem->unaligned_mode = true;
1643 	test->ifobj_rx->umem->unaligned_mode = true;
1644 	/* Let half of the packets straddle a buffer boundrary */
1645 	pkt_stream_replace_half(test, PKT_SIZE, -PKT_SIZE / 2);
1646 	test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1647 	testapp_validate_traffic(test);
1648 
1649 	pkt_stream_restore_default(test);
1650 	return true;
1651 }
1652 
1653 static void testapp_single_pkt(struct test_spec *test)
1654 {
1655 	struct pkt pkts[] = {{0x1000, PKT_SIZE, 0, true}};
1656 
1657 	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1658 	testapp_validate_traffic(test);
1659 	pkt_stream_restore_default(test);
1660 }
1661 
1662 static void testapp_invalid_desc(struct test_spec *test)
1663 {
1664 	struct pkt pkts[] = {
1665 		/* Zero packet address allowed */
1666 		{0, PKT_SIZE, 0, true},
1667 		/* Allowed packet */
1668 		{0x1000, PKT_SIZE, 0, true},
1669 		/* Straddling the start of umem */
1670 		{-2, PKT_SIZE, 0, false},
1671 		/* Packet too large */
1672 		{0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1673 		/* After umem ends */
1674 		{UMEM_SIZE, PKT_SIZE, 0, false},
1675 		/* Straddle the end of umem */
1676 		{UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false},
1677 		/* Straddle a page boundrary */
1678 		{0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1679 		/* Straddle a 2K boundrary */
1680 		{0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1681 		/* Valid packet for synch so that something is received */
1682 		{0x4000, PKT_SIZE, 0, true}};
1683 
1684 	if (test->ifobj_tx->umem->unaligned_mode) {
1685 		/* Crossing a page boundrary allowed */
1686 		pkts[6].valid = true;
1687 	}
1688 	if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1689 		/* Crossing a 2K frame size boundrary not allowed */
1690 		pkts[7].valid = false;
1691 	}
1692 
1693 	if (test->ifobj_tx->shared_umem) {
1694 		pkts[4].addr += UMEM_SIZE;
1695 		pkts[5].addr += UMEM_SIZE;
1696 	}
1697 
1698 	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1699 	testapp_validate_traffic(test);
1700 	pkt_stream_restore_default(test);
1701 }
1702 
1703 static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
1704 		       const char *dst_ip, const char *src_ip, const u16 dst_port,
1705 		       const u16 src_port, thread_func_t func_ptr)
1706 {
1707 	struct in_addr ip;
1708 
1709 	memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
1710 	memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
1711 
1712 	inet_aton(dst_ip, &ip);
1713 	ifobj->dst_ip = ip.s_addr;
1714 
1715 	inet_aton(src_ip, &ip);
1716 	ifobj->src_ip = ip.s_addr;
1717 
1718 	ifobj->dst_port = dst_port;
1719 	ifobj->src_port = src_port;
1720 
1721 	ifobj->func_ptr = func_ptr;
1722 }
1723 
1724 static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1725 {
1726 	switch (type) {
1727 	case TEST_TYPE_STATS_RX_DROPPED:
1728 		if (mode == TEST_MODE_ZC) {
1729 			ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
1730 			return;
1731 		}
1732 		testapp_stats_rx_dropped(test);
1733 		break;
1734 	case TEST_TYPE_STATS_TX_INVALID_DESCS:
1735 		testapp_stats_tx_invalid_descs(test);
1736 		break;
1737 	case TEST_TYPE_STATS_RX_FULL:
1738 		testapp_stats_rx_full(test);
1739 		break;
1740 	case TEST_TYPE_STATS_FILL_EMPTY:
1741 		testapp_stats_fill_empty(test);
1742 		break;
1743 	case TEST_TYPE_TEARDOWN:
1744 		testapp_teardown(test);
1745 		break;
1746 	case TEST_TYPE_BIDI:
1747 		testapp_bidi(test);
1748 		break;
1749 	case TEST_TYPE_BPF_RES:
1750 		testapp_bpf_res(test);
1751 		break;
1752 	case TEST_TYPE_RUN_TO_COMPLETION:
1753 		test_spec_set_name(test, "RUN_TO_COMPLETION");
1754 		testapp_validate_traffic(test);
1755 		break;
1756 	case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
1757 		test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
1758 		testapp_single_pkt(test);
1759 		break;
1760 	case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1761 		test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1762 		test->ifobj_tx->umem->frame_size = 2048;
1763 		test->ifobj_rx->umem->frame_size = 2048;
1764 		pkt_stream_replace(test, DEFAULT_PKT_CNT, PKT_SIZE);
1765 		testapp_validate_traffic(test);
1766 
1767 		pkt_stream_restore_default(test);
1768 		break;
1769 	case TEST_TYPE_RX_POLL:
1770 		test->ifobj_rx->use_poll = true;
1771 		test_spec_set_name(test, "POLL_RX");
1772 		testapp_validate_traffic(test);
1773 		break;
1774 	case TEST_TYPE_TX_POLL:
1775 		test->ifobj_tx->use_poll = true;
1776 		test_spec_set_name(test, "POLL_TX");
1777 		testapp_validate_traffic(test);
1778 		break;
1779 	case TEST_TYPE_POLL_TXQ_TMOUT:
1780 		test_spec_set_name(test, "POLL_TXQ_FULL");
1781 		test->ifobj_tx->use_poll = true;
1782 		/* create invalid frame by set umem frame_size and pkt length equal to 2048 */
1783 		test->ifobj_tx->umem->frame_size = 2048;
1784 		pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
1785 		testapp_validate_traffic_single_thread(test, test->ifobj_tx, type);
1786 		pkt_stream_restore_default(test);
1787 		break;
1788 	case TEST_TYPE_POLL_RXQ_TMOUT:
1789 		test_spec_set_name(test, "POLL_RXQ_EMPTY");
1790 		test->ifobj_rx->use_poll = true;
1791 		testapp_validate_traffic_single_thread(test, test->ifobj_rx, type);
1792 		break;
1793 	case TEST_TYPE_ALIGNED_INV_DESC:
1794 		test_spec_set_name(test, "ALIGNED_INV_DESC");
1795 		testapp_invalid_desc(test);
1796 		break;
1797 	case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1798 		test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1799 		test->ifobj_tx->umem->frame_size = 2048;
1800 		test->ifobj_rx->umem->frame_size = 2048;
1801 		testapp_invalid_desc(test);
1802 		break;
1803 	case TEST_TYPE_UNALIGNED_INV_DESC:
1804 		if (!hugepages_present(test->ifobj_tx)) {
1805 			ksft_test_result_skip("No 2M huge pages present.\n");
1806 			return;
1807 		}
1808 		test_spec_set_name(test, "UNALIGNED_INV_DESC");
1809 		test->ifobj_tx->umem->unaligned_mode = true;
1810 		test->ifobj_rx->umem->unaligned_mode = true;
1811 		testapp_invalid_desc(test);
1812 		break;
1813 	case TEST_TYPE_UNALIGNED:
1814 		if (!testapp_unaligned(test))
1815 			return;
1816 		break;
1817 	case TEST_TYPE_HEADROOM:
1818 		testapp_headroom(test);
1819 		break;
1820 	default:
1821 		break;
1822 	}
1823 
1824 	if (!test->fail)
1825 		ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
1826 				      test->name);
1827 }
1828 
1829 static struct ifobject *ifobject_create(void)
1830 {
1831 	struct ifobject *ifobj;
1832 
1833 	ifobj = calloc(1, sizeof(struct ifobject));
1834 	if (!ifobj)
1835 		return NULL;
1836 
1837 	ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1838 	if (!ifobj->xsk_arr)
1839 		goto out_xsk_arr;
1840 
1841 	ifobj->umem = calloc(1, sizeof(*ifobj->umem));
1842 	if (!ifobj->umem)
1843 		goto out_umem;
1844 
1845 	ifobj->ns_fd = -1;
1846 
1847 	return ifobj;
1848 
1849 out_umem:
1850 	free(ifobj->xsk_arr);
1851 out_xsk_arr:
1852 	free(ifobj);
1853 	return NULL;
1854 }
1855 
1856 static void ifobject_delete(struct ifobject *ifobj)
1857 {
1858 	if (ifobj->ns_fd != -1)
1859 		close(ifobj->ns_fd);
1860 	free(ifobj->umem);
1861 	free(ifobj->xsk_arr);
1862 	free(ifobj);
1863 }
1864 
1865 static bool is_xdp_supported(struct ifobject *ifobject)
1866 {
1867 	int flags = XDP_FLAGS_DRV_MODE;
1868 
1869 	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
1870 	struct bpf_insn insns[2] = {
1871 		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
1872 		BPF_EXIT_INSN()
1873 	};
1874 	int ifindex = if_nametoindex(ifobject->ifname);
1875 	int prog_fd, insn_cnt = ARRAY_SIZE(insns);
1876 	int err;
1877 
1878 	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
1879 	if (prog_fd < 0)
1880 		return false;
1881 
1882 	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
1883 	if (err) {
1884 		close(prog_fd);
1885 		return false;
1886 	}
1887 
1888 	bpf_xdp_detach(ifindex, flags, NULL);
1889 	close(prog_fd);
1890 
1891 	return true;
1892 }
1893 
1894 int main(int argc, char **argv)
1895 {
1896 	struct pkt_stream *rx_pkt_stream_default;
1897 	struct pkt_stream *tx_pkt_stream_default;
1898 	struct ifobject *ifobj_tx, *ifobj_rx;
1899 	int modes = TEST_MODE_SKB + 1;
1900 	u32 i, j, failed_tests = 0;
1901 	struct test_spec test;
1902 	bool shared_umem;
1903 
1904 	/* Use libbpf 1.0 API mode */
1905 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1906 
1907 	ifobj_tx = ifobject_create();
1908 	if (!ifobj_tx)
1909 		exit_with_error(ENOMEM);
1910 	ifobj_rx = ifobject_create();
1911 	if (!ifobj_rx)
1912 		exit_with_error(ENOMEM);
1913 
1914 	setlocale(LC_ALL, "");
1915 
1916 	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1917 	shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname);
1918 
1919 	ifobj_tx->shared_umem = shared_umem;
1920 	ifobj_rx->shared_umem = shared_umem;
1921 
1922 	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1923 		usage(basename(argv[0]));
1924 		ksft_exit_xfail();
1925 	}
1926 
1927 	init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1928 		   worker_testapp_validate_tx);
1929 	init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1930 		   worker_testapp_validate_rx);
1931 
1932 	if (is_xdp_supported(ifobj_tx)) {
1933 		modes++;
1934 		if (ifobj_zc_avail(ifobj_tx))
1935 			modes++;
1936 	}
1937 
1938 	test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1939 	tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1940 	rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1941 	if (!tx_pkt_stream_default || !rx_pkt_stream_default)
1942 		exit_with_error(ENOMEM);
1943 	test.tx_pkt_stream_default = tx_pkt_stream_default;
1944 	test.rx_pkt_stream_default = rx_pkt_stream_default;
1945 
1946 	ksft_set_plan(modes * TEST_TYPE_MAX);
1947 
1948 	for (i = 0; i < modes; i++)
1949 		for (j = 0; j < TEST_TYPE_MAX; j++) {
1950 			test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1951 			run_pkt_test(&test, i, j);
1952 			usleep(USLEEP_MAX);
1953 
1954 			if (test.fail)
1955 				failed_tests++;
1956 		}
1957 
1958 	pkt_stream_delete(tx_pkt_stream_default);
1959 	pkt_stream_delete(rx_pkt_stream_default);
1960 	ifobject_delete(ifobj_tx);
1961 	ifobject_delete(ifobj_rx);
1962 
1963 	if (failed_tests)
1964 		ksft_exit_fail();
1965 	else
1966 		ksft_exit_pass();
1967 }
1968