1018a8e75SMaciej Fijalkowski // SPDX-License-Identifier: GPL-2.0
2018a8e75SMaciej Fijalkowski /* Copyright(c) 2020 Intel Corporation. */
3018a8e75SMaciej Fijalkowski
4018a8e75SMaciej Fijalkowski /*
5018a8e75SMaciej Fijalkowski * Some functions in this program are taken from
6018a8e75SMaciej Fijalkowski * Linux kernel samples/bpf/xdpsock* and modified
7018a8e75SMaciej Fijalkowski * for use.
8018a8e75SMaciej Fijalkowski *
9018a8e75SMaciej Fijalkowski * See test_xsk.sh for detailed information on test topology
10018a8e75SMaciej Fijalkowski * and prerequisite network setup.
11018a8e75SMaciej Fijalkowski *
12018a8e75SMaciej Fijalkowski * This test program contains two threads, each thread is single socket with
13018a8e75SMaciej Fijalkowski * a unique UMEM. It validates in-order packet delivery and packet content
14018a8e75SMaciej Fijalkowski * by sending packets to each other.
15018a8e75SMaciej Fijalkowski *
16018a8e75SMaciej Fijalkowski * Tests Information:
17018a8e75SMaciej Fijalkowski * ------------------
18018a8e75SMaciej Fijalkowski * These selftests test AF_XDP SKB and Native/DRV modes using veth
19018a8e75SMaciej Fijalkowski * Virtual Ethernet interfaces.
20018a8e75SMaciej Fijalkowski *
21018a8e75SMaciej Fijalkowski * For each mode, the following tests are run:
22018a8e75SMaciej Fijalkowski * a. nopoll - soft-irq processing in run-to-completion mode
23018a8e75SMaciej Fijalkowski * b. poll - using poll() syscall
24018a8e75SMaciej Fijalkowski * c. Socket Teardown
25018a8e75SMaciej Fijalkowski * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26018a8e75SMaciej Fijalkowski * both sockets, then repeat multiple times. Only nopoll mode is used
27018a8e75SMaciej Fijalkowski * d. Bi-directional sockets
28018a8e75SMaciej Fijalkowski * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29018a8e75SMaciej Fijalkowski * completion rings on each socket, tx/rx in both directions. Only nopoll
30018a8e75SMaciej Fijalkowski * mode is used
31018a8e75SMaciej Fijalkowski * e. Statistics
32018a8e75SMaciej Fijalkowski * Trigger some error conditions and ensure that the appropriate statistics
33018a8e75SMaciej Fijalkowski * are incremented. Within this test, the following statistics are tested:
34018a8e75SMaciej Fijalkowski * i. rx dropped
35018a8e75SMaciej Fijalkowski * Increase the UMEM frame headroom to a value which results in
36018a8e75SMaciej Fijalkowski * insufficient space in the rx buffer for both the packet and the headroom.
37018a8e75SMaciej Fijalkowski * ii. tx invalid
38018a8e75SMaciej Fijalkowski * Set the 'len' field of tx descriptors to an invalid value (umem frame
39018a8e75SMaciej Fijalkowski * size + 1).
40018a8e75SMaciej Fijalkowski * iii. rx ring full
41018a8e75SMaciej Fijalkowski * Reduce the size of the RX ring to a fraction of the fill ring size.
42018a8e75SMaciej Fijalkowski * iv. fill queue empty
43018a8e75SMaciej Fijalkowski * Do not populate the fill queue and then try to receive pkts.
44018a8e75SMaciej Fijalkowski * f. bpf_link resource persistence
45018a8e75SMaciej Fijalkowski * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46018a8e75SMaciej Fijalkowski * then remove xsk sockets from queue 0 on both veth interfaces and
47018a8e75SMaciej Fijalkowski * finally run a traffic on queues ids 1
48018a8e75SMaciej Fijalkowski * g. unaligned mode
49018a8e75SMaciej Fijalkowski * h. tests for invalid and corner case Tx descriptors so that the correct ones
50018a8e75SMaciej Fijalkowski * are discarded and let through, respectively.
51018a8e75SMaciej Fijalkowski * i. 2K frame size tests
52f540d44eSMagnus Karlsson * j. If multi-buffer is supported, send 9k packets divided into 3 frames
531005a226SMagnus Karlsson * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
541005a226SMagnus Karlsson * using unaligned mode
5569760449SMagnus Karlsson * l. If multi-buffer is supported, try various nasty combinations of descriptors to
5669760449SMagnus Karlsson * check if they pass the validation or not
57018a8e75SMaciej Fijalkowski *
58018a8e75SMaciej Fijalkowski * Flow:
59018a8e75SMaciej Fijalkowski * -----
60018a8e75SMaciej Fijalkowski * - Single process spawns two threads: Tx and Rx
6164aef77dSMagnus Karlsson * - Each of these two threads attach to a veth interface
6264aef77dSMagnus Karlsson * - Each thread creates one AF_XDP socket connected to a unique umem for each
63018a8e75SMaciej Fijalkowski * veth interface
6464aef77dSMagnus Karlsson * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
6564aef77dSMagnus Karlsson * - Rx thread verifies if all packets were received and delivered in-order,
66018a8e75SMaciej Fijalkowski * and have the right content
67018a8e75SMaciej Fijalkowski *
68018a8e75SMaciej Fijalkowski * Enable/disable packet dump mode:
69018a8e75SMaciej Fijalkowski * --------------------------
70018a8e75SMaciej Fijalkowski * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71018a8e75SMaciej Fijalkowski * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72018a8e75SMaciej Fijalkowski */
73018a8e75SMaciej Fijalkowski
74018a8e75SMaciej Fijalkowski #define _GNU_SOURCE
75c0801598SKal Conley #include <assert.h>
76018a8e75SMaciej Fijalkowski #include <fcntl.h>
77018a8e75SMaciej Fijalkowski #include <errno.h>
78018a8e75SMaciej Fijalkowski #include <getopt.h>
79018a8e75SMaciej Fijalkowski #include <linux/if_link.h>
80018a8e75SMaciej Fijalkowski #include <linux/if_ether.h>
812ddade32SMagnus Karlsson #include <linux/mman.h>
82f540d44eSMagnus Karlsson #include <linux/netdev.h>
83018a8e75SMaciej Fijalkowski #include <arpa/inet.h>
84018a8e75SMaciej Fijalkowski #include <net/if.h>
85018a8e75SMaciej Fijalkowski #include <locale.h>
86018a8e75SMaciej Fijalkowski #include <poll.h>
87018a8e75SMaciej Fijalkowski #include <pthread.h>
88018a8e75SMaciej Fijalkowski #include <signal.h>
89018a8e75SMaciej Fijalkowski #include <stdio.h>
90018a8e75SMaciej Fijalkowski #include <stdlib.h>
91018a8e75SMaciej Fijalkowski #include <string.h>
92018a8e75SMaciej Fijalkowski #include <stddef.h>
93018a8e75SMaciej Fijalkowski #include <sys/mman.h>
94018a8e75SMaciej Fijalkowski #include <sys/socket.h>
95018a8e75SMaciej Fijalkowski #include <sys/time.h>
96018a8e75SMaciej Fijalkowski #include <sys/types.h>
97018a8e75SMaciej Fijalkowski #include <unistd.h>
987d8319a7SMagnus Karlsson
997d8319a7SMagnus Karlsson #include "xsk_xdp_progs.skel.h"
100018a8e75SMaciej Fijalkowski #include "xsk.h"
101018a8e75SMaciej Fijalkowski #include "xskxceiver.h"
1020d68e6feSMaciej Fijalkowski #include <bpf/bpf.h>
1030d68e6feSMaciej Fijalkowski #include <linux/filter.h>
104018a8e75SMaciej Fijalkowski #include "../kselftest.h"
1059a321fd3STushar Vyavahare #include "xsk_xdp_metadata.h"
106018a8e75SMaciej Fijalkowski
107018a8e75SMaciej Fijalkowski static const char *MAC1 = "\x00\x0A\x56\x9E\xEE\x62";
108018a8e75SMaciej Fijalkowski static const char *MAC2 = "\x00\x0A\x56\x9E\xEE\x61";
109018a8e75SMaciej Fijalkowski
__exit_with_error(int error,const char * file,const char * func,int line)110018a8e75SMaciej Fijalkowski static void __exit_with_error(int error, const char *file, const char *func, int line)
111018a8e75SMaciej Fijalkowski {
112018a8e75SMaciej Fijalkowski ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
113018a8e75SMaciej Fijalkowski strerror(error));
114018a8e75SMaciej Fijalkowski ksft_exit_xfail();
115018a8e75SMaciej Fijalkowski }
116018a8e75SMaciej Fijalkowski
117018a8e75SMaciej Fijalkowski #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
118018a8e75SMaciej Fijalkowski #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
mode_string(struct test_spec * test)119fe2ad08eSMaciej Fijalkowski static char *mode_string(struct test_spec *test)
120fe2ad08eSMaciej Fijalkowski {
121fe2ad08eSMaciej Fijalkowski switch (test->mode) {
122fe2ad08eSMaciej Fijalkowski case TEST_MODE_SKB:
123fe2ad08eSMaciej Fijalkowski return "SKB";
124fe2ad08eSMaciej Fijalkowski case TEST_MODE_DRV:
125fe2ad08eSMaciej Fijalkowski return "DRV";
126fe2ad08eSMaciej Fijalkowski case TEST_MODE_ZC:
127fe2ad08eSMaciej Fijalkowski return "ZC";
128fe2ad08eSMaciej Fijalkowski default:
129fe2ad08eSMaciej Fijalkowski return "BOGUS";
130fe2ad08eSMaciej Fijalkowski }
131fe2ad08eSMaciej Fijalkowski }
132018a8e75SMaciej Fijalkowski
report_failure(struct test_spec * test)133018a8e75SMaciej Fijalkowski static void report_failure(struct test_spec *test)
134018a8e75SMaciej Fijalkowski {
135018a8e75SMaciej Fijalkowski if (test->fail)
136018a8e75SMaciej Fijalkowski return;
137018a8e75SMaciej Fijalkowski
138018a8e75SMaciej Fijalkowski ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
139018a8e75SMaciej Fijalkowski test->name);
140018a8e75SMaciej Fijalkowski test->fail = true;
141018a8e75SMaciej Fijalkowski }
142018a8e75SMaciej Fijalkowski
143feb973a9SMagnus Karlsson /* The payload is a word consisting of a packet sequence number in the upper
144feb973a9SMagnus Karlsson * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's
145feb973a9SMagnus Karlsson * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.
146feb973a9SMagnus Karlsson */
write_payload(void * dest,u32 pkt_nb,u32 start,u32 size)1472f6eae0dSMagnus Karlsson static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
148018a8e75SMaciej Fijalkowski {
149feb973a9SMagnus Karlsson u32 *ptr = (u32 *)dest, i;
150018a8e75SMaciej Fijalkowski
1512f6eae0dSMagnus Karlsson start /= sizeof(*ptr);
1522f6eae0dSMagnus Karlsson size /= sizeof(*ptr);
1532f6eae0dSMagnus Karlsson for (i = 0; i < size; i++)
1542f6eae0dSMagnus Karlsson ptr[i] = htonl(pkt_nb << 16 | (i + start));
155018a8e75SMaciej Fijalkowski }
156018a8e75SMaciej Fijalkowski
gen_eth_hdr(struct ifobject * ifobject,struct ethhdr * eth_hdr)157018a8e75SMaciej Fijalkowski static void gen_eth_hdr(struct ifobject *ifobject, struct ethhdr *eth_hdr)
158018a8e75SMaciej Fijalkowski {
159018a8e75SMaciej Fijalkowski memcpy(eth_hdr->h_dest, ifobject->dst_mac, ETH_ALEN);
160018a8e75SMaciej Fijalkowski memcpy(eth_hdr->h_source, ifobject->src_mac, ETH_ALEN);
161df82d2e8SMagnus Karlsson eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
162018a8e75SMaciej Fijalkowski }
163018a8e75SMaciej Fijalkowski
is_umem_valid(struct ifobject * ifobj)1643143d10bSShibin Koikkara Reeny static bool is_umem_valid(struct ifobject *ifobj)
1653143d10bSShibin Koikkara Reeny {
1663143d10bSShibin Koikkara Reeny return !!ifobj->umem->umem;
1673143d10bSShibin Koikkara Reeny }
1683143d10bSShibin Koikkara Reeny
mode_to_xdp_flags(enum test_mode mode)169aa61d81fSMagnus Karlsson static u32 mode_to_xdp_flags(enum test_mode mode)
170aa61d81fSMagnus Karlsson {
171aa61d81fSMagnus Karlsson return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
172aa61d81fSMagnus Karlsson }
173aa61d81fSMagnus Karlsson
umem_size(struct xsk_umem_info * umem)174d9f6d970SMagnus Karlsson static u64 umem_size(struct xsk_umem_info *umem)
175d9f6d970SMagnus Karlsson {
176d9f6d970SMagnus Karlsson return umem->num_frames * umem->frame_size;
177d9f6d970SMagnus Karlsson }
178d9f6d970SMagnus Karlsson
xsk_configure_umem(struct ifobject * ifobj,struct xsk_umem_info * umem,void * buffer,u64 size)179d9f6d970SMagnus Karlsson static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
180d9f6d970SMagnus Karlsson u64 size)
181018a8e75SMaciej Fijalkowski {
182018a8e75SMaciej Fijalkowski struct xsk_umem_config cfg = {
183018a8e75SMaciej Fijalkowski .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
184018a8e75SMaciej Fijalkowski .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
185018a8e75SMaciej Fijalkowski .frame_size = umem->frame_size,
186018a8e75SMaciej Fijalkowski .frame_headroom = umem->frame_headroom,
187018a8e75SMaciej Fijalkowski .flags = XSK_UMEM__DEFAULT_FLAGS
188018a8e75SMaciej Fijalkowski };
189018a8e75SMaciej Fijalkowski int ret;
190018a8e75SMaciej Fijalkowski
191018a8e75SMaciej Fijalkowski if (umem->unaligned_mode)
192018a8e75SMaciej Fijalkowski cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
193018a8e75SMaciej Fijalkowski
194018a8e75SMaciej Fijalkowski ret = xsk_umem__create(&umem->umem, buffer, size,
195018a8e75SMaciej Fijalkowski &umem->fq, &umem->cq, &cfg);
196018a8e75SMaciej Fijalkowski if (ret)
197018a8e75SMaciej Fijalkowski return ret;
198018a8e75SMaciej Fijalkowski
199018a8e75SMaciej Fijalkowski umem->buffer = buffer;
200d9f6d970SMagnus Karlsson if (ifobj->shared_umem && ifobj->rx_on) {
201d9f6d970SMagnus Karlsson umem->base_addr = umem_size(umem);
202d9f6d970SMagnus Karlsson umem->next_buffer = umem_size(umem);
203d9f6d970SMagnus Karlsson }
204d9f6d970SMagnus Karlsson
205018a8e75SMaciej Fijalkowski return 0;
206018a8e75SMaciej Fijalkowski }
207018a8e75SMaciej Fijalkowski
umem_alloc_buffer(struct xsk_umem_info * umem)208d9f6d970SMagnus Karlsson static u64 umem_alloc_buffer(struct xsk_umem_info *umem)
209d9f6d970SMagnus Karlsson {
210d9f6d970SMagnus Karlsson u64 addr;
211d9f6d970SMagnus Karlsson
212d9f6d970SMagnus Karlsson addr = umem->next_buffer;
213d9f6d970SMagnus Karlsson umem->next_buffer += umem->frame_size;
214d9f6d970SMagnus Karlsson if (umem->next_buffer >= umem->base_addr + umem_size(umem))
215d9f6d970SMagnus Karlsson umem->next_buffer = umem->base_addr;
216d9f6d970SMagnus Karlsson
217d9f6d970SMagnus Karlsson return addr;
218d9f6d970SMagnus Karlsson }
219d9f6d970SMagnus Karlsson
umem_reset_alloc(struct xsk_umem_info * umem)220d9f6d970SMagnus Karlsson static void umem_reset_alloc(struct xsk_umem_info *umem)
221d9f6d970SMagnus Karlsson {
222d9f6d970SMagnus Karlsson umem->next_buffer = 0;
223d9f6d970SMagnus Karlsson }
224d9f6d970SMagnus Karlsson
enable_busy_poll(struct xsk_socket_info * xsk)225018a8e75SMaciej Fijalkowski static void enable_busy_poll(struct xsk_socket_info *xsk)
226018a8e75SMaciej Fijalkowski {
227018a8e75SMaciej Fijalkowski int sock_opt;
228018a8e75SMaciej Fijalkowski
229018a8e75SMaciej Fijalkowski sock_opt = 1;
230018a8e75SMaciej Fijalkowski if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
231018a8e75SMaciej Fijalkowski (void *)&sock_opt, sizeof(sock_opt)) < 0)
232018a8e75SMaciej Fijalkowski exit_with_error(errno);
233018a8e75SMaciej Fijalkowski
234018a8e75SMaciej Fijalkowski sock_opt = 20;
235018a8e75SMaciej Fijalkowski if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
236018a8e75SMaciej Fijalkowski (void *)&sock_opt, sizeof(sock_opt)) < 0)
237018a8e75SMaciej Fijalkowski exit_with_error(errno);
238018a8e75SMaciej Fijalkowski
239018a8e75SMaciej Fijalkowski sock_opt = BATCH_SIZE;
240018a8e75SMaciej Fijalkowski if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
241018a8e75SMaciej Fijalkowski (void *)&sock_opt, sizeof(sock_opt)) < 0)
242018a8e75SMaciej Fijalkowski exit_with_error(errno);
243018a8e75SMaciej Fijalkowski }
244018a8e75SMaciej Fijalkowski
__xsk_configure_socket(struct xsk_socket_info * xsk,struct xsk_umem_info * umem,struct ifobject * ifobject,bool shared)245a693ff3eSMaciej Fijalkowski static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
246018a8e75SMaciej Fijalkowski struct ifobject *ifobject, bool shared)
247018a8e75SMaciej Fijalkowski {
248018a8e75SMaciej Fijalkowski struct xsk_socket_config cfg = {};
249018a8e75SMaciej Fijalkowski struct xsk_ring_cons *rxr;
250018a8e75SMaciej Fijalkowski struct xsk_ring_prod *txr;
251018a8e75SMaciej Fijalkowski
252018a8e75SMaciej Fijalkowski xsk->umem = umem;
253018a8e75SMaciej Fijalkowski cfg.rx_size = xsk->rxqsize;
254018a8e75SMaciej Fijalkowski cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
255018a8e75SMaciej Fijalkowski cfg.bind_flags = ifobject->bind_flags;
256018a8e75SMaciej Fijalkowski if (shared)
257018a8e75SMaciej Fijalkowski cfg.bind_flags |= XDP_SHARED_UMEM;
258f540d44eSMagnus Karlsson if (ifobject->pkt_stream && ifobject->mtu > MAX_ETH_PKT_SIZE)
259f540d44eSMagnus Karlsson cfg.bind_flags |= XDP_USE_SG;
260018a8e75SMaciej Fijalkowski
261018a8e75SMaciej Fijalkowski txr = ifobject->tx_on ? &xsk->tx : NULL;
262018a8e75SMaciej Fijalkowski rxr = ifobject->rx_on ? &xsk->rx : NULL;
263aa61d81fSMagnus Karlsson return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
264018a8e75SMaciej Fijalkowski }
265018a8e75SMaciej Fijalkowski
ifobj_zc_avail(struct ifobject * ifobject)266fe2ad08eSMaciej Fijalkowski static bool ifobj_zc_avail(struct ifobject *ifobject)
267fe2ad08eSMaciej Fijalkowski {
268fe2ad08eSMaciej Fijalkowski size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
269fe2ad08eSMaciej Fijalkowski int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
270fe2ad08eSMaciej Fijalkowski struct xsk_socket_info *xsk;
271fe2ad08eSMaciej Fijalkowski struct xsk_umem_info *umem;
272fe2ad08eSMaciej Fijalkowski bool zc_avail = false;
273fe2ad08eSMaciej Fijalkowski void *bufs;
274fe2ad08eSMaciej Fijalkowski int ret;
275fe2ad08eSMaciej Fijalkowski
276fe2ad08eSMaciej Fijalkowski bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
277fe2ad08eSMaciej Fijalkowski if (bufs == MAP_FAILED)
278fe2ad08eSMaciej Fijalkowski exit_with_error(errno);
279fe2ad08eSMaciej Fijalkowski
280fe2ad08eSMaciej Fijalkowski umem = calloc(1, sizeof(struct xsk_umem_info));
281fe2ad08eSMaciej Fijalkowski if (!umem) {
282fe2ad08eSMaciej Fijalkowski munmap(bufs, umem_sz);
283085dcccfSMagnus Karlsson exit_with_error(ENOMEM);
284fe2ad08eSMaciej Fijalkowski }
285fe2ad08eSMaciej Fijalkowski umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
286d9f6d970SMagnus Karlsson ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
287fe2ad08eSMaciej Fijalkowski if (ret)
288fe2ad08eSMaciej Fijalkowski exit_with_error(-ret);
289fe2ad08eSMaciej Fijalkowski
290fe2ad08eSMaciej Fijalkowski xsk = calloc(1, sizeof(struct xsk_socket_info));
291fe2ad08eSMaciej Fijalkowski if (!xsk)
292fe2ad08eSMaciej Fijalkowski goto out;
293fe2ad08eSMaciej Fijalkowski ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
294fe2ad08eSMaciej Fijalkowski ifobject->rx_on = true;
295fe2ad08eSMaciej Fijalkowski xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
296fe2ad08eSMaciej Fijalkowski ret = __xsk_configure_socket(xsk, umem, ifobject, false);
297fe2ad08eSMaciej Fijalkowski if (!ret)
298fe2ad08eSMaciej Fijalkowski zc_avail = true;
299fe2ad08eSMaciej Fijalkowski
300fe2ad08eSMaciej Fijalkowski xsk_socket__delete(xsk->xsk);
301fe2ad08eSMaciej Fijalkowski free(xsk);
302fe2ad08eSMaciej Fijalkowski out:
303fe2ad08eSMaciej Fijalkowski munmap(umem->buffer, umem_sz);
304fe2ad08eSMaciej Fijalkowski xsk_umem__delete(umem->umem);
305fe2ad08eSMaciej Fijalkowski free(umem);
306fe2ad08eSMaciej Fijalkowski return zc_avail;
307fe2ad08eSMaciej Fijalkowski }
308fe2ad08eSMaciej Fijalkowski
309018a8e75SMaciej Fijalkowski static struct option long_options[] = {
310018a8e75SMaciej Fijalkowski {"interface", required_argument, 0, 'i'},
311018a8e75SMaciej Fijalkowski {"busy-poll", no_argument, 0, 'b'},
312018a8e75SMaciej Fijalkowski {"verbose", no_argument, 0, 'v'},
313018a8e75SMaciej Fijalkowski {0, 0, 0, 0}
314018a8e75SMaciej Fijalkowski };
315018a8e75SMaciej Fijalkowski
usage(const char * prog)316018a8e75SMaciej Fijalkowski static void usage(const char *prog)
317018a8e75SMaciej Fijalkowski {
318018a8e75SMaciej Fijalkowski const char *str =
319018a8e75SMaciej Fijalkowski " Usage: %s [OPTIONS]\n"
320018a8e75SMaciej Fijalkowski " Options:\n"
321018a8e75SMaciej Fijalkowski " -i, --interface Use interface\n"
322018a8e75SMaciej Fijalkowski " -v, --verbose Verbose output\n"
323018a8e75SMaciej Fijalkowski " -b, --busy-poll Enable busy poll\n";
324018a8e75SMaciej Fijalkowski
325018a8e75SMaciej Fijalkowski ksft_print_msg(str, prog);
326018a8e75SMaciej Fijalkowski }
327018a8e75SMaciej Fijalkowski
validate_interface(struct ifobject * ifobj)328018a8e75SMaciej Fijalkowski static bool validate_interface(struct ifobject *ifobj)
329018a8e75SMaciej Fijalkowski {
330018a8e75SMaciej Fijalkowski if (!strcmp(ifobj->ifname, ""))
331018a8e75SMaciej Fijalkowski return false;
332018a8e75SMaciej Fijalkowski return true;
333018a8e75SMaciej Fijalkowski }
334018a8e75SMaciej Fijalkowski
parse_command_line(struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,int argc,char ** argv)335018a8e75SMaciej Fijalkowski static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
336018a8e75SMaciej Fijalkowski char **argv)
337018a8e75SMaciej Fijalkowski {
338018a8e75SMaciej Fijalkowski struct ifobject *ifobj;
339018a8e75SMaciej Fijalkowski u32 interface_nb = 0;
340018a8e75SMaciej Fijalkowski int option_index, c;
341018a8e75SMaciej Fijalkowski
342018a8e75SMaciej Fijalkowski opterr = 0;
343018a8e75SMaciej Fijalkowski
344018a8e75SMaciej Fijalkowski for (;;) {
3457a8a6762SMagnus Karlsson c = getopt_long(argc, argv, "i:vb", long_options, &option_index);
346018a8e75SMaciej Fijalkowski if (c == -1)
347018a8e75SMaciej Fijalkowski break;
348018a8e75SMaciej Fijalkowski
349018a8e75SMaciej Fijalkowski switch (c) {
350018a8e75SMaciej Fijalkowski case 'i':
351018a8e75SMaciej Fijalkowski if (interface_nb == 0)
352018a8e75SMaciej Fijalkowski ifobj = ifobj_tx;
353018a8e75SMaciej Fijalkowski else if (interface_nb == 1)
354018a8e75SMaciej Fijalkowski ifobj = ifobj_rx;
355018a8e75SMaciej Fijalkowski else
356018a8e75SMaciej Fijalkowski break;
357018a8e75SMaciej Fijalkowski
35864aef77dSMagnus Karlsson memcpy(ifobj->ifname, optarg,
35964aef77dSMagnus Karlsson min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
360aa61d81fSMagnus Karlsson
361aa61d81fSMagnus Karlsson ifobj->ifindex = if_nametoindex(ifobj->ifname);
362aa61d81fSMagnus Karlsson if (!ifobj->ifindex)
363aa61d81fSMagnus Karlsson exit_with_error(errno);
364aa61d81fSMagnus Karlsson
365018a8e75SMaciej Fijalkowski interface_nb++;
366018a8e75SMaciej Fijalkowski break;
367018a8e75SMaciej Fijalkowski case 'v':
368018a8e75SMaciej Fijalkowski opt_verbose = true;
369018a8e75SMaciej Fijalkowski break;
370018a8e75SMaciej Fijalkowski case 'b':
371018a8e75SMaciej Fijalkowski ifobj_tx->busy_poll = true;
372018a8e75SMaciej Fijalkowski ifobj_rx->busy_poll = true;
373018a8e75SMaciej Fijalkowski break;
374018a8e75SMaciej Fijalkowski default:
375018a8e75SMaciej Fijalkowski usage(basename(argv[0]));
376018a8e75SMaciej Fijalkowski ksft_exit_xfail();
377018a8e75SMaciej Fijalkowski }
378018a8e75SMaciej Fijalkowski }
379018a8e75SMaciej Fijalkowski }
380018a8e75SMaciej Fijalkowski
__test_spec_init(struct test_spec * test,struct ifobject * ifobj_tx,struct ifobject * ifobj_rx)381018a8e75SMaciej Fijalkowski static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
382018a8e75SMaciej Fijalkowski struct ifobject *ifobj_rx)
383018a8e75SMaciej Fijalkowski {
384018a8e75SMaciej Fijalkowski u32 i, j;
385018a8e75SMaciej Fijalkowski
386018a8e75SMaciej Fijalkowski for (i = 0; i < MAX_INTERFACES; i++) {
387018a8e75SMaciej Fijalkowski struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
388018a8e75SMaciej Fijalkowski
389018a8e75SMaciej Fijalkowski ifobj->xsk = &ifobj->xsk_arr[0];
390018a8e75SMaciej Fijalkowski ifobj->use_poll = false;
391018a8e75SMaciej Fijalkowski ifobj->use_fill_ring = true;
392018a8e75SMaciej Fijalkowski ifobj->release_rx = true;
393018a8e75SMaciej Fijalkowski ifobj->validation_func = NULL;
3949a321fd3STushar Vyavahare ifobj->use_metadata = false;
395018a8e75SMaciej Fijalkowski
396018a8e75SMaciej Fijalkowski if (i == 0) {
397018a8e75SMaciej Fijalkowski ifobj->rx_on = false;
398018a8e75SMaciej Fijalkowski ifobj->tx_on = true;
3991adef064SMaciej Fijalkowski ifobj->pkt_stream = test->tx_pkt_stream_default;
400018a8e75SMaciej Fijalkowski } else {
401018a8e75SMaciej Fijalkowski ifobj->rx_on = true;
402018a8e75SMaciej Fijalkowski ifobj->tx_on = false;
4031adef064SMaciej Fijalkowski ifobj->pkt_stream = test->rx_pkt_stream_default;
404018a8e75SMaciej Fijalkowski }
405018a8e75SMaciej Fijalkowski
406018a8e75SMaciej Fijalkowski memset(ifobj->umem, 0, sizeof(*ifobj->umem));
407018a8e75SMaciej Fijalkowski ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
408018a8e75SMaciej Fijalkowski ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
409018a8e75SMaciej Fijalkowski
410018a8e75SMaciej Fijalkowski for (j = 0; j < MAX_SOCKETS; j++) {
411018a8e75SMaciej Fijalkowski memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
412018a8e75SMaciej Fijalkowski ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
413018a8e75SMaciej Fijalkowski }
414018a8e75SMaciej Fijalkowski }
415018a8e75SMaciej Fijalkowski
416018a8e75SMaciej Fijalkowski test->ifobj_tx = ifobj_tx;
417018a8e75SMaciej Fijalkowski test->ifobj_rx = ifobj_rx;
418018a8e75SMaciej Fijalkowski test->current_step = 0;
419018a8e75SMaciej Fijalkowski test->total_steps = 1;
420018a8e75SMaciej Fijalkowski test->nb_sockets = 1;
421018a8e75SMaciej Fijalkowski test->fail = false;
422f540d44eSMagnus Karlsson test->mtu = MAX_ETH_PKT_SIZE;
4237d8319a7SMagnus Karlsson test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
4247d8319a7SMagnus Karlsson test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
4257d8319a7SMagnus Karlsson test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
4267d8319a7SMagnus Karlsson test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
427018a8e75SMaciej Fijalkowski }
428018a8e75SMaciej Fijalkowski
test_spec_init(struct test_spec * test,struct ifobject * ifobj_tx,struct ifobject * ifobj_rx,enum test_mode mode)429018a8e75SMaciej Fijalkowski static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
430018a8e75SMaciej Fijalkowski struct ifobject *ifobj_rx, enum test_mode mode)
431018a8e75SMaciej Fijalkowski {
4321adef064SMaciej Fijalkowski struct pkt_stream *tx_pkt_stream;
4331adef064SMaciej Fijalkowski struct pkt_stream *rx_pkt_stream;
434018a8e75SMaciej Fijalkowski u32 i;
435018a8e75SMaciej Fijalkowski
4361adef064SMaciej Fijalkowski tx_pkt_stream = test->tx_pkt_stream_default;
4371adef064SMaciej Fijalkowski rx_pkt_stream = test->rx_pkt_stream_default;
438018a8e75SMaciej Fijalkowski memset(test, 0, sizeof(*test));
4391adef064SMaciej Fijalkowski test->tx_pkt_stream_default = tx_pkt_stream;
4401adef064SMaciej Fijalkowski test->rx_pkt_stream_default = rx_pkt_stream;
441018a8e75SMaciej Fijalkowski
442018a8e75SMaciej Fijalkowski for (i = 0; i < MAX_INTERFACES; i++) {
443018a8e75SMaciej Fijalkowski struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
444018a8e75SMaciej Fijalkowski
445fe2ad08eSMaciej Fijalkowski ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
446fe2ad08eSMaciej Fijalkowski if (mode == TEST_MODE_ZC)
447fe2ad08eSMaciej Fijalkowski ifobj->bind_flags |= XDP_ZEROCOPY;
448fe2ad08eSMaciej Fijalkowski else
449fe2ad08eSMaciej Fijalkowski ifobj->bind_flags |= XDP_COPY;
450018a8e75SMaciej Fijalkowski }
451018a8e75SMaciej Fijalkowski
452fe2ad08eSMaciej Fijalkowski test->mode = mode;
453018a8e75SMaciej Fijalkowski __test_spec_init(test, ifobj_tx, ifobj_rx);
454018a8e75SMaciej Fijalkowski }
455018a8e75SMaciej Fijalkowski
test_spec_reset(struct test_spec * test)456018a8e75SMaciej Fijalkowski static void test_spec_reset(struct test_spec *test)
457018a8e75SMaciej Fijalkowski {
458018a8e75SMaciej Fijalkowski __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
459018a8e75SMaciej Fijalkowski }
460018a8e75SMaciej Fijalkowski
test_spec_set_name(struct test_spec * test,const char * name)461018a8e75SMaciej Fijalkowski static void test_spec_set_name(struct test_spec *test, const char *name)
462018a8e75SMaciej Fijalkowski {
463018a8e75SMaciej Fijalkowski strncpy(test->name, name, MAX_TEST_NAME_SIZE);
464018a8e75SMaciej Fijalkowski }
465018a8e75SMaciej Fijalkowski
test_spec_set_xdp_prog(struct test_spec * test,struct bpf_program * xdp_prog_rx,struct bpf_program * xdp_prog_tx,struct bpf_map * xskmap_rx,struct bpf_map * xskmap_tx)4667d8319a7SMagnus Karlsson static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
4677d8319a7SMagnus Karlsson struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
4687d8319a7SMagnus Karlsson struct bpf_map *xskmap_tx)
4697d8319a7SMagnus Karlsson {
4707d8319a7SMagnus Karlsson test->xdp_prog_rx = xdp_prog_rx;
4717d8319a7SMagnus Karlsson test->xdp_prog_tx = xdp_prog_tx;
4727d8319a7SMagnus Karlsson test->xskmap_rx = xskmap_rx;
4737d8319a7SMagnus Karlsson test->xskmap_tx = xskmap_tx;
4747d8319a7SMagnus Karlsson }
4757d8319a7SMagnus Karlsson
test_spec_set_mtu(struct test_spec * test,int mtu)476f540d44eSMagnus Karlsson static int test_spec_set_mtu(struct test_spec *test, int mtu)
477f540d44eSMagnus Karlsson {
478f540d44eSMagnus Karlsson int err;
479f540d44eSMagnus Karlsson
480f540d44eSMagnus Karlsson if (test->ifobj_rx->mtu != mtu) {
481f540d44eSMagnus Karlsson err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);
482f540d44eSMagnus Karlsson if (err)
483f540d44eSMagnus Karlsson return err;
484f540d44eSMagnus Karlsson test->ifobj_rx->mtu = mtu;
485f540d44eSMagnus Karlsson }
486f540d44eSMagnus Karlsson if (test->ifobj_tx->mtu != mtu) {
487f540d44eSMagnus Karlsson err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);
488f540d44eSMagnus Karlsson if (err)
489f540d44eSMagnus Karlsson return err;
490f540d44eSMagnus Karlsson test->ifobj_tx->mtu = mtu;
491f540d44eSMagnus Karlsson }
492f540d44eSMagnus Karlsson
493f540d44eSMagnus Karlsson return 0;
494f540d44eSMagnus Karlsson }
495f540d44eSMagnus Karlsson
pkt_stream_reset(struct pkt_stream * pkt_stream)496018a8e75SMaciej Fijalkowski static void pkt_stream_reset(struct pkt_stream *pkt_stream)
497018a8e75SMaciej Fijalkowski {
498018a8e75SMaciej Fijalkowski if (pkt_stream)
49969fc03d2SMagnus Karlsson pkt_stream->current_pkt_nb = 0;
500018a8e75SMaciej Fijalkowski }
501018a8e75SMaciej Fijalkowski
pkt_stream_get_next_tx_pkt(struct pkt_stream * pkt_stream)50269fc03d2SMagnus Karlsson static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
503018a8e75SMaciej Fijalkowski {
50469fc03d2SMagnus Karlsson if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)
505018a8e75SMaciej Fijalkowski return NULL;
506018a8e75SMaciej Fijalkowski
50769fc03d2SMagnus Karlsson return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
508018a8e75SMaciej Fijalkowski }
509018a8e75SMaciej Fijalkowski
pkt_stream_get_next_rx_pkt(struct pkt_stream * pkt_stream,u32 * pkts_sent)510018a8e75SMaciej Fijalkowski static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
511018a8e75SMaciej Fijalkowski {
51269fc03d2SMagnus Karlsson while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
513018a8e75SMaciej Fijalkowski (*pkts_sent)++;
51469fc03d2SMagnus Karlsson if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)
51569fc03d2SMagnus Karlsson return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
51669fc03d2SMagnus Karlsson pkt_stream->current_pkt_nb++;
517018a8e75SMaciej Fijalkowski }
518018a8e75SMaciej Fijalkowski return NULL;
519018a8e75SMaciej Fijalkowski }
520018a8e75SMaciej Fijalkowski
pkt_stream_delete(struct pkt_stream * pkt_stream)521018a8e75SMaciej Fijalkowski static void pkt_stream_delete(struct pkt_stream *pkt_stream)
522018a8e75SMaciej Fijalkowski {
523018a8e75SMaciej Fijalkowski free(pkt_stream->pkts);
524018a8e75SMaciej Fijalkowski free(pkt_stream);
525018a8e75SMaciej Fijalkowski }
526018a8e75SMaciej Fijalkowski
pkt_stream_restore_default(struct test_spec * test)527018a8e75SMaciej Fijalkowski static void pkt_stream_restore_default(struct test_spec *test)
528018a8e75SMaciej Fijalkowski {
529018a8e75SMaciej Fijalkowski struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
5301adef064SMaciej Fijalkowski struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
531018a8e75SMaciej Fijalkowski
5321adef064SMaciej Fijalkowski if (tx_pkt_stream != test->tx_pkt_stream_default) {
533018a8e75SMaciej Fijalkowski pkt_stream_delete(test->ifobj_tx->pkt_stream);
5341adef064SMaciej Fijalkowski test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
535018a8e75SMaciej Fijalkowski }
536018a8e75SMaciej Fijalkowski
5371adef064SMaciej Fijalkowski if (rx_pkt_stream != test->rx_pkt_stream_default) {
538018a8e75SMaciej Fijalkowski pkt_stream_delete(test->ifobj_rx->pkt_stream);
5391adef064SMaciej Fijalkowski test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
5401adef064SMaciej Fijalkowski }
541018a8e75SMaciej Fijalkowski }
542018a8e75SMaciej Fijalkowski
__pkt_stream_alloc(u32 nb_pkts)543018a8e75SMaciej Fijalkowski static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
544018a8e75SMaciej Fijalkowski {
545018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream;
546018a8e75SMaciej Fijalkowski
547018a8e75SMaciej Fijalkowski pkt_stream = calloc(1, sizeof(*pkt_stream));
548018a8e75SMaciej Fijalkowski if (!pkt_stream)
549018a8e75SMaciej Fijalkowski return NULL;
550018a8e75SMaciej Fijalkowski
551018a8e75SMaciej Fijalkowski pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
552018a8e75SMaciej Fijalkowski if (!pkt_stream->pkts) {
553018a8e75SMaciej Fijalkowski free(pkt_stream);
554018a8e75SMaciej Fijalkowski return NULL;
555018a8e75SMaciej Fijalkowski }
556018a8e75SMaciej Fijalkowski
557018a8e75SMaciej Fijalkowski pkt_stream->nb_pkts = nb_pkts;
558018a8e75SMaciej Fijalkowski return pkt_stream;
559018a8e75SMaciej Fijalkowski }
560018a8e75SMaciej Fijalkowski
pkt_continues(u32 options)56169760449SMagnus Karlsson static bool pkt_continues(u32 options)
56217f1034dSMagnus Karlsson {
56369760449SMagnus Karlsson return options & XDP_PKT_CONTD;
56417f1034dSMagnus Karlsson }
56517f1034dSMagnus Karlsson
ceil_u32(u32 a,u32 b)56686e41755SMagnus Karlsson static u32 ceil_u32(u32 a, u32 b)
56786e41755SMagnus Karlsson {
56886e41755SMagnus Karlsson return (a + b - 1) / b;
56986e41755SMagnus Karlsson }
57086e41755SMagnus Karlsson
pkt_nb_frags(u32 frame_size,struct pkt_stream * pkt_stream,struct pkt * pkt)57169760449SMagnus Karlsson static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)
57286e41755SMagnus Karlsson {
57369760449SMagnus Karlsson u32 nb_frags = 1, next_frag;
57469760449SMagnus Karlsson
57569760449SMagnus Karlsson if (!pkt)
57669760449SMagnus Karlsson return 1;
57769760449SMagnus Karlsson
57869760449SMagnus Karlsson if (!pkt_stream->verbatim) {
57969760449SMagnus Karlsson if (!pkt->valid || !pkt->len)
58086e41755SMagnus Karlsson return 1;
58186e41755SMagnus Karlsson return ceil_u32(pkt->len, frame_size);
58286e41755SMagnus Karlsson }
58386e41755SMagnus Karlsson
58469760449SMagnus Karlsson /* Search for the end of the packet in verbatim mode */
58569760449SMagnus Karlsson if (!pkt_continues(pkt->options))
58669760449SMagnus Karlsson return nb_frags;
58769760449SMagnus Karlsson
58869760449SMagnus Karlsson next_frag = pkt_stream->current_pkt_nb;
58969760449SMagnus Karlsson pkt++;
59069760449SMagnus Karlsson while (next_frag++ < pkt_stream->nb_pkts) {
59169760449SMagnus Karlsson nb_frags++;
59269760449SMagnus Karlsson if (!pkt_continues(pkt->options) || !pkt->valid)
59369760449SMagnus Karlsson break;
59469760449SMagnus Karlsson pkt++;
59569760449SMagnus Karlsson }
59669760449SMagnus Karlsson return nb_frags;
59769760449SMagnus Karlsson }
59869760449SMagnus Karlsson
pkt_set(struct xsk_umem_info * umem,struct pkt * pkt,int offset,u32 len)599d9f6d970SMagnus Karlsson static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, int offset, u32 len)
600018a8e75SMaciej Fijalkowski {
601d9f6d970SMagnus Karlsson pkt->offset = offset;
602018a8e75SMaciej Fijalkowski pkt->len = len;
60317f1034dSMagnus Karlsson if (len > MAX_ETH_JUMBO_SIZE)
604018a8e75SMaciej Fijalkowski pkt->valid = false;
605018a8e75SMaciej Fijalkowski else
606018a8e75SMaciej Fijalkowski pkt->valid = true;
607018a8e75SMaciej Fijalkowski }
608018a8e75SMaciej Fijalkowski
pkt_get_buffer_len(struct xsk_umem_info * umem,u32 len)6097cd6df4fSMagnus Karlsson static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
6107cd6df4fSMagnus Karlsson {
6117cd6df4fSMagnus Karlsson return ceil_u32(len, umem->frame_size) * umem->frame_size;
6127cd6df4fSMagnus Karlsson }
6137cd6df4fSMagnus Karlsson
pkt_stream_generate(struct xsk_umem_info * umem,u32 nb_pkts,u32 pkt_len)614018a8e75SMaciej Fijalkowski static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
615018a8e75SMaciej Fijalkowski {
616018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream;
617018a8e75SMaciej Fijalkowski u32 i;
618018a8e75SMaciej Fijalkowski
619018a8e75SMaciej Fijalkowski pkt_stream = __pkt_stream_alloc(nb_pkts);
620018a8e75SMaciej Fijalkowski if (!pkt_stream)
621018a8e75SMaciej Fijalkowski exit_with_error(ENOMEM);
622018a8e75SMaciej Fijalkowski
6237cd6df4fSMagnus Karlsson pkt_stream->nb_pkts = nb_pkts;
6247cd6df4fSMagnus Karlsson pkt_stream->max_pkt_len = pkt_len;
625018a8e75SMaciej Fijalkowski for (i = 0; i < nb_pkts; i++) {
6262f6eae0dSMagnus Karlsson struct pkt *pkt = &pkt_stream->pkts[i];
6272f6eae0dSMagnus Karlsson
6282f6eae0dSMagnus Karlsson pkt_set(umem, pkt, 0, pkt_len);
6292f6eae0dSMagnus Karlsson pkt->pkt_nb = i;
630018a8e75SMaciej Fijalkowski }
631018a8e75SMaciej Fijalkowski
632018a8e75SMaciej Fijalkowski return pkt_stream;
633018a8e75SMaciej Fijalkowski }
634018a8e75SMaciej Fijalkowski
pkt_stream_clone(struct xsk_umem_info * umem,struct pkt_stream * pkt_stream)635018a8e75SMaciej Fijalkowski static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
636018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream)
637018a8e75SMaciej Fijalkowski {
638018a8e75SMaciej Fijalkowski return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
639018a8e75SMaciej Fijalkowski }
640018a8e75SMaciej Fijalkowski
pkt_stream_replace(struct test_spec * test,u32 nb_pkts,u32 pkt_len)641018a8e75SMaciej Fijalkowski static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
642018a8e75SMaciej Fijalkowski {
643018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream;
644018a8e75SMaciej Fijalkowski
645018a8e75SMaciej Fijalkowski pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
646018a8e75SMaciej Fijalkowski test->ifobj_tx->pkt_stream = pkt_stream;
6471adef064SMaciej Fijalkowski pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
648018a8e75SMaciej Fijalkowski test->ifobj_rx->pkt_stream = pkt_stream;
649018a8e75SMaciej Fijalkowski }
650018a8e75SMaciej Fijalkowski
__pkt_stream_replace_half(struct ifobject * ifobj,u32 pkt_len,int offset)6511adef064SMaciej Fijalkowski static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
6521adef064SMaciej Fijalkowski int offset)
6531adef064SMaciej Fijalkowski {
6541adef064SMaciej Fijalkowski struct xsk_umem_info *umem = ifobj->umem;
6551adef064SMaciej Fijalkowski struct pkt_stream *pkt_stream;
6561adef064SMaciej Fijalkowski u32 i;
6571adef064SMaciej Fijalkowski
6581adef064SMaciej Fijalkowski pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
6591adef064SMaciej Fijalkowski for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
660d9f6d970SMagnus Karlsson pkt_set(umem, &pkt_stream->pkts[i], offset, pkt_len);
6611adef064SMaciej Fijalkowski
6621adef064SMaciej Fijalkowski ifobj->pkt_stream = pkt_stream;
6631adef064SMaciej Fijalkowski }
6641adef064SMaciej Fijalkowski
pkt_stream_replace_half(struct test_spec * test,u32 pkt_len,int offset)665018a8e75SMaciej Fijalkowski static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
666018a8e75SMaciej Fijalkowski {
6671adef064SMaciej Fijalkowski __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
6681adef064SMaciej Fijalkowski __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
669018a8e75SMaciej Fijalkowski }
670018a8e75SMaciej Fijalkowski
pkt_stream_receive_half(struct test_spec * test)671018a8e75SMaciej Fijalkowski static void pkt_stream_receive_half(struct test_spec *test)
672018a8e75SMaciej Fijalkowski {
673018a8e75SMaciej Fijalkowski struct xsk_umem_info *umem = test->ifobj_rx->umem;
674018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream = test->ifobj_tx->pkt_stream;
675018a8e75SMaciej Fijalkowski u32 i;
676018a8e75SMaciej Fijalkowski
677018a8e75SMaciej Fijalkowski test->ifobj_rx->pkt_stream = pkt_stream_generate(umem, pkt_stream->nb_pkts,
678018a8e75SMaciej Fijalkowski pkt_stream->pkts[0].len);
679018a8e75SMaciej Fijalkowski pkt_stream = test->ifobj_rx->pkt_stream;
680018a8e75SMaciej Fijalkowski for (i = 1; i < pkt_stream->nb_pkts; i += 2)
681018a8e75SMaciej Fijalkowski pkt_stream->pkts[i].valid = false;
682018a8e75SMaciej Fijalkowski }
683018a8e75SMaciej Fijalkowski
pkt_get_addr(struct pkt * pkt,struct xsk_umem_info * umem)684d9f6d970SMagnus Karlsson static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
685018a8e75SMaciej Fijalkowski {
686d9f6d970SMagnus Karlsson if (!pkt->valid)
687d9f6d970SMagnus Karlsson return pkt->offset;
688d9f6d970SMagnus Karlsson return pkt->offset + umem_alloc_buffer(umem);
689d9f6d970SMagnus Karlsson }
690d9f6d970SMagnus Karlsson
pkt_stream_cancel(struct pkt_stream * pkt_stream)69117f1034dSMagnus Karlsson static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
69217f1034dSMagnus Karlsson {
69317f1034dSMagnus Karlsson pkt_stream->current_pkt_nb--;
69417f1034dSMagnus Karlsson }
69517f1034dSMagnus Karlsson
pkt_generate(struct ifobject * ifobject,u64 addr,u32 len,u32 pkt_nb,u32 bytes_written)6962f6eae0dSMagnus Karlsson static void pkt_generate(struct ifobject *ifobject, u64 addr, u32 len, u32 pkt_nb,
6972f6eae0dSMagnus Karlsson u32 bytes_written)
698d9f6d970SMagnus Karlsson {
6992f6eae0dSMagnus Karlsson void *data = xsk_umem__get_data(ifobject->umem->buffer, addr);
700018a8e75SMaciej Fijalkowski
7012f6eae0dSMagnus Karlsson if (len < MIN_PKT_SIZE)
702d9f6d970SMagnus Karlsson return;
703018a8e75SMaciej Fijalkowski
7042f6eae0dSMagnus Karlsson if (!bytes_written) {
7052f6eae0dSMagnus Karlsson gen_eth_hdr(ifobject, data);
706018a8e75SMaciej Fijalkowski
7072f6eae0dSMagnus Karlsson len -= PKT_HDR_SIZE;
7082f6eae0dSMagnus Karlsson data += PKT_HDR_SIZE;
7092f6eae0dSMagnus Karlsson } else {
7102f6eae0dSMagnus Karlsson bytes_written -= PKT_HDR_SIZE;
7112f6eae0dSMagnus Karlsson }
7122f6eae0dSMagnus Karlsson
7132f6eae0dSMagnus Karlsson write_payload(data, pkt_nb, bytes_written, len);
714018a8e75SMaciej Fijalkowski }
715018a8e75SMaciej Fijalkowski
__pkt_stream_generate_custom(struct ifobject * ifobj,struct pkt * frames,u32 nb_frames,bool verbatim)71669760449SMagnus Karlsson static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,
71769760449SMagnus Karlsson u32 nb_frames, bool verbatim)
718018a8e75SMaciej Fijalkowski {
71969760449SMagnus Karlsson u32 i, len = 0, pkt_nb = 0, payload = 0;
720018a8e75SMaciej Fijalkowski struct pkt_stream *pkt_stream;
721018a8e75SMaciej Fijalkowski
72269760449SMagnus Karlsson pkt_stream = __pkt_stream_alloc(nb_frames);
723018a8e75SMaciej Fijalkowski if (!pkt_stream)
724018a8e75SMaciej Fijalkowski exit_with_error(ENOMEM);
725018a8e75SMaciej Fijalkowski
72669760449SMagnus Karlsson for (i = 0; i < nb_frames; i++) {
72769760449SMagnus Karlsson struct pkt *pkt = &pkt_stream->pkts[pkt_nb];
72869760449SMagnus Karlsson struct pkt *frame = &frames[i];
7297cd6df4fSMagnus Karlsson
73069760449SMagnus Karlsson pkt->offset = frame->offset;
73169760449SMagnus Karlsson if (verbatim) {
73269760449SMagnus Karlsson *pkt = *frame;
73369760449SMagnus Karlsson pkt->pkt_nb = payload;
73469760449SMagnus Karlsson if (!frame->valid || !pkt_continues(frame->options))
73569760449SMagnus Karlsson payload++;
73669760449SMagnus Karlsson } else {
73769760449SMagnus Karlsson if (frame->valid)
73869760449SMagnus Karlsson len += frame->len;
73969760449SMagnus Karlsson if (frame->valid && pkt_continues(frame->options))
74069760449SMagnus Karlsson continue;
74169760449SMagnus Karlsson
74269760449SMagnus Karlsson pkt->pkt_nb = pkt_nb;
74369760449SMagnus Karlsson pkt->len = len;
74469760449SMagnus Karlsson pkt->valid = frame->valid;
74569760449SMagnus Karlsson pkt->options = 0;
74669760449SMagnus Karlsson
74769760449SMagnus Karlsson len = 0;
748018a8e75SMaciej Fijalkowski }
7491adef064SMaciej Fijalkowski
75069760449SMagnus Karlsson if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
75169760449SMagnus Karlsson pkt_stream->max_pkt_len = pkt->len;
75269760449SMagnus Karlsson pkt_nb++;
75369760449SMagnus Karlsson }
75469760449SMagnus Karlsson
75569760449SMagnus Karlsson pkt_stream->nb_pkts = pkt_nb;
75669760449SMagnus Karlsson pkt_stream->verbatim = verbatim;
75769760449SMagnus Karlsson return pkt_stream;
7581adef064SMaciej Fijalkowski }
7591adef064SMaciej Fijalkowski
pkt_stream_generate_custom(struct test_spec * test,struct pkt * pkts,u32 nb_pkts)7601adef064SMaciej Fijalkowski static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
7611adef064SMaciej Fijalkowski {
76269760449SMagnus Karlsson struct pkt_stream *pkt_stream;
76369760449SMagnus Karlsson
76469760449SMagnus Karlsson pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
76569760449SMagnus Karlsson test->ifobj_tx->pkt_stream = pkt_stream;
76669760449SMagnus Karlsson
76769760449SMagnus Karlsson pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
76869760449SMagnus Karlsson test->ifobj_rx->pkt_stream = pkt_stream;
769018a8e75SMaciej Fijalkowski }
770018a8e75SMaciej Fijalkowski
pkt_print_data(u32 * data,u32 cnt)771feb973a9SMagnus Karlsson static void pkt_print_data(u32 *data, u32 cnt)
772feb973a9SMagnus Karlsson {
773feb973a9SMagnus Karlsson u32 i;
774feb973a9SMagnus Karlsson
775feb973a9SMagnus Karlsson for (i = 0; i < cnt; i++) {
776feb973a9SMagnus Karlsson u32 seqnum, pkt_nb;
777feb973a9SMagnus Karlsson
778feb973a9SMagnus Karlsson seqnum = ntohl(*data) & 0xffff;
779feb973a9SMagnus Karlsson pkt_nb = ntohl(*data) >> 16;
780feb973a9SMagnus Karlsson fprintf(stdout, "%u:%u ", pkt_nb, seqnum);
781feb973a9SMagnus Karlsson data++;
782feb973a9SMagnus Karlsson }
783feb973a9SMagnus Karlsson }
784feb973a9SMagnus Karlsson
pkt_dump(void * pkt,u32 len,bool eth_header)7852f6eae0dSMagnus Karlsson static void pkt_dump(void *pkt, u32 len, bool eth_header)
786018a8e75SMaciej Fijalkowski {
787df82d2e8SMagnus Karlsson struct ethhdr *ethhdr = pkt;
7882f6eae0dSMagnus Karlsson u32 i, *data;
789018a8e75SMaciej Fijalkowski
7902f6eae0dSMagnus Karlsson if (eth_header) {
791018a8e75SMaciej Fijalkowski /*extract L2 frame */
792018a8e75SMaciej Fijalkowski fprintf(stdout, "DEBUG>> L2: dst mac: ");
793018a8e75SMaciej Fijalkowski for (i = 0; i < ETH_ALEN; i++)
794018a8e75SMaciej Fijalkowski fprintf(stdout, "%02X", ethhdr->h_dest[i]);
795018a8e75SMaciej Fijalkowski
796018a8e75SMaciej Fijalkowski fprintf(stdout, "\nDEBUG>> L2: src mac: ");
797018a8e75SMaciej Fijalkowski for (i = 0; i < ETH_ALEN; i++)
798018a8e75SMaciej Fijalkowski fprintf(stdout, "%02X", ethhdr->h_source[i]);
799018a8e75SMaciej Fijalkowski
8002f6eae0dSMagnus Karlsson data = pkt + PKT_HDR_SIZE;
8012f6eae0dSMagnus Karlsson } else {
8022f6eae0dSMagnus Karlsson data = pkt;
8032f6eae0dSMagnus Karlsson }
8042f6eae0dSMagnus Karlsson
805018a8e75SMaciej Fijalkowski /*extract L5 frame */
806feb973a9SMagnus Karlsson fprintf(stdout, "\nDEBUG>> L5: seqnum: ");
8072f6eae0dSMagnus Karlsson pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);
808feb973a9SMagnus Karlsson fprintf(stdout, "....");
809feb973a9SMagnus Karlsson if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {
810feb973a9SMagnus Karlsson fprintf(stdout, "\n.... ");
8112f6eae0dSMagnus Karlsson pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,
812feb973a9SMagnus Karlsson PKT_DUMP_NB_TO_PRINT);
813feb973a9SMagnus Karlsson }
814feb973a9SMagnus Karlsson fprintf(stdout, "\n---------------------------------------\n");
815018a8e75SMaciej Fijalkowski }
816018a8e75SMaciej Fijalkowski
is_offset_correct(struct xsk_umem_info * umem,struct pkt * pkt,u64 addr)817d9f6d970SMagnus Karlsson static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)
818018a8e75SMaciej Fijalkowski {
819018a8e75SMaciej Fijalkowski u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
820d9f6d970SMagnus Karlsson u32 offset = addr % umem->frame_size, expected_offset;
821d9f6d970SMagnus Karlsson int pkt_offset = pkt->valid ? pkt->offset : 0;
822018a8e75SMaciej Fijalkowski
823d9f6d970SMagnus Karlsson if (!umem->unaligned_mode)
824d9f6d970SMagnus Karlsson pkt_offset = 0;
825018a8e75SMaciej Fijalkowski
826d9f6d970SMagnus Karlsson expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
827018a8e75SMaciej Fijalkowski
828018a8e75SMaciej Fijalkowski if (offset == expected_offset)
829018a8e75SMaciej Fijalkowski return true;
830018a8e75SMaciej Fijalkowski
831018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
832018a8e75SMaciej Fijalkowski return false;
833018a8e75SMaciej Fijalkowski }
834018a8e75SMaciej Fijalkowski
is_metadata_correct(struct pkt * pkt,void * buffer,u64 addr)8359a321fd3STushar Vyavahare static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
8369a321fd3STushar Vyavahare {
8379a321fd3STushar Vyavahare void *data = xsk_umem__get_data(buffer, addr);
8389a321fd3STushar Vyavahare struct xdp_info *meta = data - sizeof(struct xdp_info);
8399a321fd3STushar Vyavahare
840feb973a9SMagnus Karlsson if (meta->count != pkt->pkt_nb) {
8419a321fd3STushar Vyavahare ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%d]\n",
842feb973a9SMagnus Karlsson __func__, pkt->pkt_nb, meta->count);
8439a321fd3STushar Vyavahare return false;
8449a321fd3STushar Vyavahare }
8459a321fd3STushar Vyavahare
8469a321fd3STushar Vyavahare return true;
8479a321fd3STushar Vyavahare }
8489a321fd3STushar Vyavahare
is_frag_valid(struct xsk_umem_info * umem,u64 addr,u32 len,u32 expected_pkt_nb,u32 bytes_processed)84917f1034dSMagnus Karlsson static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,
85017f1034dSMagnus Karlsson u32 bytes_processed)
851018a8e75SMaciej Fijalkowski {
85217f1034dSMagnus Karlsson u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;
85317f1034dSMagnus Karlsson void *data = xsk_umem__get_data(umem->buffer, addr);
854018a8e75SMaciej Fijalkowski
85517f1034dSMagnus Karlsson addr -= umem->base_addr;
85617f1034dSMagnus Karlsson
85717f1034dSMagnus Karlsson if (addr >= umem->num_frames * umem->frame_size ||
85817f1034dSMagnus Karlsson addr + len > umem->num_frames * umem->frame_size) {
85917f1034dSMagnus Karlsson ksft_print_msg("Frag invalid addr: %llx len: %u\n", addr, len);
86017f1034dSMagnus Karlsson return false;
86117f1034dSMagnus Karlsson }
86217f1034dSMagnus Karlsson if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {
86317f1034dSMagnus Karlsson ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", addr, len);
86417f1034dSMagnus Karlsson return false;
86517f1034dSMagnus Karlsson }
86617f1034dSMagnus Karlsson
86717f1034dSMagnus Karlsson pkt_data = data;
86817f1034dSMagnus Karlsson if (!bytes_processed) {
86917f1034dSMagnus Karlsson pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);
87017f1034dSMagnus Karlsson len -= PKT_HDR_SIZE;
87117f1034dSMagnus Karlsson } else {
87217f1034dSMagnus Karlsson bytes_processed -= PKT_HDR_SIZE;
87317f1034dSMagnus Karlsson }
87417f1034dSMagnus Karlsson
87517f1034dSMagnus Karlsson expected_seqnum = bytes_processed / sizeof(*pkt_data);
87617f1034dSMagnus Karlsson seqnum = ntohl(*pkt_data) & 0xffff;
87717f1034dSMagnus Karlsson pkt_nb = ntohl(*pkt_data) >> 16;
87817f1034dSMagnus Karlsson
87917f1034dSMagnus Karlsson if (expected_pkt_nb != pkt_nb) {
88017f1034dSMagnus Karlsson ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",
88117f1034dSMagnus Karlsson __func__, expected_pkt_nb, pkt_nb);
88217f1034dSMagnus Karlsson goto error;
88317f1034dSMagnus Karlsson }
88417f1034dSMagnus Karlsson if (expected_seqnum != seqnum) {
88517f1034dSMagnus Karlsson ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",
88617f1034dSMagnus Karlsson __func__, expected_seqnum, seqnum);
8877a8a6762SMagnus Karlsson goto error;
888018a8e75SMaciej Fijalkowski }
889018a8e75SMaciej Fijalkowski
89017f1034dSMagnus Karlsson words_to_end = len / sizeof(*pkt_data) - 1;
89117f1034dSMagnus Karlsson pkt_data += words_to_end;
89217f1034dSMagnus Karlsson seqnum = ntohl(*pkt_data) & 0xffff;
89317f1034dSMagnus Karlsson expected_seqnum += words_to_end;
89417f1034dSMagnus Karlsson if (expected_seqnum != seqnum) {
89517f1034dSMagnus Karlsson ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",
89617f1034dSMagnus Karlsson __func__, expected_seqnum, seqnum);
8977a8a6762SMagnus Karlsson goto error;
898018a8e75SMaciej Fijalkowski }
899018a8e75SMaciej Fijalkowski
900018a8e75SMaciej Fijalkowski return true;
9017a8a6762SMagnus Karlsson
9027a8a6762SMagnus Karlsson error:
90317f1034dSMagnus Karlsson pkt_dump(data, len, !bytes_processed);
9047a8a6762SMagnus Karlsson return false;
905018a8e75SMaciej Fijalkowski }
906018a8e75SMaciej Fijalkowski
is_pkt_valid(struct pkt * pkt,void * buffer,u64 addr,u32 len)90717f1034dSMagnus Karlsson static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
90817f1034dSMagnus Karlsson {
90917f1034dSMagnus Karlsson if (pkt->len != len) {
91017f1034dSMagnus Karlsson ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",
91117f1034dSMagnus Karlsson __func__, pkt->len, len);
91217f1034dSMagnus Karlsson pkt_dump(xsk_umem__get_data(buffer, addr), len, true);
91317f1034dSMagnus Karlsson return false;
91417f1034dSMagnus Karlsson }
91517f1034dSMagnus Karlsson
91617f1034dSMagnus Karlsson return true;
91717f1034dSMagnus Karlsson }
91817f1034dSMagnus Karlsson
kick_tx(struct xsk_socket_info * xsk)919018a8e75SMaciej Fijalkowski static void kick_tx(struct xsk_socket_info *xsk)
920018a8e75SMaciej Fijalkowski {
921018a8e75SMaciej Fijalkowski int ret;
922018a8e75SMaciej Fijalkowski
923018a8e75SMaciej Fijalkowski ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
924018a8e75SMaciej Fijalkowski if (ret >= 0)
925018a8e75SMaciej Fijalkowski return;
926018a8e75SMaciej Fijalkowski if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
927018a8e75SMaciej Fijalkowski usleep(100);
928018a8e75SMaciej Fijalkowski return;
929018a8e75SMaciej Fijalkowski }
930018a8e75SMaciej Fijalkowski exit_with_error(errno);
931018a8e75SMaciej Fijalkowski }
932018a8e75SMaciej Fijalkowski
kick_rx(struct xsk_socket_info * xsk)933018a8e75SMaciej Fijalkowski static void kick_rx(struct xsk_socket_info *xsk)
934018a8e75SMaciej Fijalkowski {
935018a8e75SMaciej Fijalkowski int ret;
936018a8e75SMaciej Fijalkowski
937018a8e75SMaciej Fijalkowski ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
938018a8e75SMaciej Fijalkowski if (ret < 0)
939018a8e75SMaciej Fijalkowski exit_with_error(errno);
940018a8e75SMaciej Fijalkowski }
941018a8e75SMaciej Fijalkowski
complete_pkts(struct xsk_socket_info * xsk,int batch_size)942018a8e75SMaciej Fijalkowski static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
943018a8e75SMaciej Fijalkowski {
944018a8e75SMaciej Fijalkowski unsigned int rcvd;
945018a8e75SMaciej Fijalkowski u32 idx;
946018a8e75SMaciej Fijalkowski
947018a8e75SMaciej Fijalkowski if (xsk_ring_prod__needs_wakeup(&xsk->tx))
948018a8e75SMaciej Fijalkowski kick_tx(xsk);
949018a8e75SMaciej Fijalkowski
950018a8e75SMaciej Fijalkowski rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
951018a8e75SMaciej Fijalkowski if (rcvd) {
952018a8e75SMaciej Fijalkowski if (rcvd > xsk->outstanding_tx) {
953018a8e75SMaciej Fijalkowski u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
954018a8e75SMaciej Fijalkowski
955018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] Too many packets completed\n", __func__);
956018a8e75SMaciej Fijalkowski ksft_print_msg("Last completion address: %llx\n", addr);
957018a8e75SMaciej Fijalkowski return TEST_FAILURE;
958018a8e75SMaciej Fijalkowski }
959018a8e75SMaciej Fijalkowski
960018a8e75SMaciej Fijalkowski xsk_ring_cons__release(&xsk->umem->cq, rcvd);
961018a8e75SMaciej Fijalkowski xsk->outstanding_tx -= rcvd;
962018a8e75SMaciej Fijalkowski }
963018a8e75SMaciej Fijalkowski
964018a8e75SMaciej Fijalkowski return TEST_PASS;
965018a8e75SMaciej Fijalkowski }
966018a8e75SMaciej Fijalkowski
receive_pkts(struct test_spec * test,struct pollfd * fds)9673143d10bSShibin Koikkara Reeny static int receive_pkts(struct test_spec *test, struct pollfd *fds)
968018a8e75SMaciej Fijalkowski {
9693143d10bSShibin Koikkara Reeny struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
9703143d10bSShibin Koikkara Reeny struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
9713143d10bSShibin Koikkara Reeny struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
97217f1034dSMagnus Karlsson u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
9733143d10bSShibin Koikkara Reeny struct ifobject *ifobj = test->ifobj_rx;
974018a8e75SMaciej Fijalkowski struct xsk_umem_info *umem = xsk->umem;
975018a8e75SMaciej Fijalkowski struct pkt *pkt;
976018a8e75SMaciej Fijalkowski int ret;
977018a8e75SMaciej Fijalkowski
978018a8e75SMaciej Fijalkowski ret = gettimeofday(&tv_now, NULL);
979018a8e75SMaciej Fijalkowski if (ret)
980018a8e75SMaciej Fijalkowski exit_with_error(errno);
981018a8e75SMaciej Fijalkowski timeradd(&tv_now, &tv_timeout, &tv_end);
982018a8e75SMaciej Fijalkowski
983018a8e75SMaciej Fijalkowski pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
984018a8e75SMaciej Fijalkowski while (pkt) {
98517f1034dSMagnus Karlsson u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
98617f1034dSMagnus Karlsson u64 first_addr;
98717f1034dSMagnus Karlsson
988018a8e75SMaciej Fijalkowski ret = gettimeofday(&tv_now, NULL);
989018a8e75SMaciej Fijalkowski if (ret)
990018a8e75SMaciej Fijalkowski exit_with_error(errno);
991018a8e75SMaciej Fijalkowski if (timercmp(&tv_now, &tv_end, >)) {
992018a8e75SMaciej Fijalkowski ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
993018a8e75SMaciej Fijalkowski return TEST_FAILURE;
994018a8e75SMaciej Fijalkowski }
995018a8e75SMaciej Fijalkowski
996018a8e75SMaciej Fijalkowski kick_rx(xsk);
9973143d10bSShibin Koikkara Reeny if (ifobj->use_poll) {
998018a8e75SMaciej Fijalkowski ret = poll(fds, 1, POLL_TMOUT);
999018a8e75SMaciej Fijalkowski if (ret < 0)
1000085dcccfSMagnus Karlsson exit_with_error(errno);
10013143d10bSShibin Koikkara Reeny
10023143d10bSShibin Koikkara Reeny if (!ret) {
10033143d10bSShibin Koikkara Reeny if (!is_umem_valid(test->ifobj_tx))
10043143d10bSShibin Koikkara Reeny return TEST_PASS;
10053143d10bSShibin Koikkara Reeny
10063143d10bSShibin Koikkara Reeny ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
10073143d10bSShibin Koikkara Reeny return TEST_FAILURE;
1008018a8e75SMaciej Fijalkowski }
10093143d10bSShibin Koikkara Reeny
10103143d10bSShibin Koikkara Reeny if (!(fds->revents & POLLIN))
1011018a8e75SMaciej Fijalkowski continue;
1012018a8e75SMaciej Fijalkowski }
1013018a8e75SMaciej Fijalkowski
10143143d10bSShibin Koikkara Reeny rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
10153143d10bSShibin Koikkara Reeny if (!rcvd)
10163143d10bSShibin Koikkara Reeny continue;
10173143d10bSShibin Koikkara Reeny
1018018a8e75SMaciej Fijalkowski if (ifobj->use_fill_ring) {
1019018a8e75SMaciej Fijalkowski ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1020018a8e75SMaciej Fijalkowski while (ret != rcvd) {
1021018a8e75SMaciej Fijalkowski if (ret < 0)
1022018a8e75SMaciej Fijalkowski exit_with_error(-ret);
1023018a8e75SMaciej Fijalkowski if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
1024018a8e75SMaciej Fijalkowski ret = poll(fds, 1, POLL_TMOUT);
1025018a8e75SMaciej Fijalkowski if (ret < 0)
1026085dcccfSMagnus Karlsson exit_with_error(errno);
1027018a8e75SMaciej Fijalkowski }
1028018a8e75SMaciej Fijalkowski ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1029018a8e75SMaciej Fijalkowski }
1030018a8e75SMaciej Fijalkowski }
1031018a8e75SMaciej Fijalkowski
103217f1034dSMagnus Karlsson while (frags_processed < rcvd) {
1033018a8e75SMaciej Fijalkowski const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
1034018a8e75SMaciej Fijalkowski u64 addr = desc->addr, orig;
1035018a8e75SMaciej Fijalkowski
1036018a8e75SMaciej Fijalkowski orig = xsk_umem__extract_addr(addr);
1037018a8e75SMaciej Fijalkowski addr = xsk_umem__add_offset_to_addr(addr);
1038018a8e75SMaciej Fijalkowski
103969760449SMagnus Karlsson if (!pkt) {
104069760449SMagnus Karlsson ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
104169760449SMagnus Karlsson __func__, addr, desc->len);
104269760449SMagnus Karlsson return TEST_FAILURE;
104369760449SMagnus Karlsson }
104469760449SMagnus Karlsson
104517f1034dSMagnus Karlsson if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
1046d9f6d970SMagnus Karlsson !is_offset_correct(umem, pkt, addr) ||
10479a321fd3STushar Vyavahare (ifobj->use_metadata && !is_metadata_correct(pkt, umem->buffer, addr)))
1048018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1049018a8e75SMaciej Fijalkowski
105017f1034dSMagnus Karlsson if (!nb_frags++)
105117f1034dSMagnus Karlsson first_addr = addr;
105217f1034dSMagnus Karlsson frags_processed++;
105317f1034dSMagnus Karlsson pkt_len += desc->len;
1054018a8e75SMaciej Fijalkowski if (ifobj->use_fill_ring)
1055018a8e75SMaciej Fijalkowski *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
105617f1034dSMagnus Karlsson
105769760449SMagnus Karlsson if (pkt_continues(desc->options))
105817f1034dSMagnus Karlsson continue;
105917f1034dSMagnus Karlsson
106017f1034dSMagnus Karlsson /* The complete packet has been received */
106117f1034dSMagnus Karlsson if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
106217f1034dSMagnus Karlsson !is_offset_correct(umem, pkt, addr))
106317f1034dSMagnus Karlsson return TEST_FAILURE;
106417f1034dSMagnus Karlsson
1065018a8e75SMaciej Fijalkowski pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
106617f1034dSMagnus Karlsson nb_frags = 0;
106717f1034dSMagnus Karlsson pkt_len = 0;
106817f1034dSMagnus Karlsson }
106917f1034dSMagnus Karlsson
107017f1034dSMagnus Karlsson if (nb_frags) {
107117f1034dSMagnus Karlsson /* In the middle of a packet. Start over from beginning of packet. */
107217f1034dSMagnus Karlsson idx_rx -= nb_frags;
107317f1034dSMagnus Karlsson xsk_ring_cons__cancel(&xsk->rx, nb_frags);
107417f1034dSMagnus Karlsson if (ifobj->use_fill_ring) {
107517f1034dSMagnus Karlsson idx_fq -= nb_frags;
107617f1034dSMagnus Karlsson xsk_ring_prod__cancel(&umem->fq, nb_frags);
107717f1034dSMagnus Karlsson }
107817f1034dSMagnus Karlsson frags_processed -= nb_frags;
1079018a8e75SMaciej Fijalkowski }
1080018a8e75SMaciej Fijalkowski
1081018a8e75SMaciej Fijalkowski if (ifobj->use_fill_ring)
108217f1034dSMagnus Karlsson xsk_ring_prod__submit(&umem->fq, frags_processed);
1083018a8e75SMaciej Fijalkowski if (ifobj->release_rx)
108417f1034dSMagnus Karlsson xsk_ring_cons__release(&xsk->rx, frags_processed);
1085018a8e75SMaciej Fijalkowski
1086018a8e75SMaciej Fijalkowski pthread_mutex_lock(&pacing_mutex);
1087018a8e75SMaciej Fijalkowski pkts_in_flight -= pkts_sent;
1088018a8e75SMaciej Fijalkowski pthread_mutex_unlock(&pacing_mutex);
1089018a8e75SMaciej Fijalkowski pkts_sent = 0;
1090018a8e75SMaciej Fijalkowski }
1091018a8e75SMaciej Fijalkowski
1092018a8e75SMaciej Fijalkowski return TEST_PASS;
1093018a8e75SMaciej Fijalkowski }
1094018a8e75SMaciej Fijalkowski
__send_pkts(struct ifobject * ifobject,struct pollfd * fds,bool timeout)109569fc03d2SMagnus Karlsson static int __send_pkts(struct ifobject *ifobject, struct pollfd *fds, bool timeout)
1096018a8e75SMaciej Fijalkowski {
109717f1034dSMagnus Karlsson u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
109817f1034dSMagnus Karlsson struct pkt_stream *pkt_stream = ifobject->pkt_stream;
1099018a8e75SMaciej Fijalkowski struct xsk_socket_info *xsk = ifobject->xsk;
11007cd6df4fSMagnus Karlsson struct xsk_umem_info *umem = ifobject->umem;
11013143d10bSShibin Koikkara Reeny bool use_poll = ifobject->use_poll;
1102e8f50c4fSKang Minchul int ret;
1103018a8e75SMaciej Fijalkowski
110417f1034dSMagnus Karlsson buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
11057cd6df4fSMagnus Karlsson /* pkts_in_flight might be negative if many invalid packets are sent */
11067cd6df4fSMagnus Karlsson if (pkts_in_flight >= (int)((umem_size(umem) - BATCH_SIZE * buffer_len) / buffer_len)) {
11077cd6df4fSMagnus Karlsson kick_tx(xsk);
11087cd6df4fSMagnus Karlsson return TEST_CONTINUE;
11097cd6df4fSMagnus Karlsson }
11107cd6df4fSMagnus Karlsson
11113143d10bSShibin Koikkara Reeny while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
11123143d10bSShibin Koikkara Reeny if (use_poll) {
11133143d10bSShibin Koikkara Reeny ret = poll(fds, 1, POLL_TMOUT);
11143143d10bSShibin Koikkara Reeny if (timeout) {
11153143d10bSShibin Koikkara Reeny if (ret < 0) {
11163143d10bSShibin Koikkara Reeny ksft_print_msg("ERROR: [%s] Poll error %d\n",
1117085dcccfSMagnus Karlsson __func__, errno);
11183143d10bSShibin Koikkara Reeny return TEST_FAILURE;
11193143d10bSShibin Koikkara Reeny }
11203143d10bSShibin Koikkara Reeny if (ret == 0)
11213143d10bSShibin Koikkara Reeny return TEST_PASS;
11223143d10bSShibin Koikkara Reeny break;
11233143d10bSShibin Koikkara Reeny }
11243143d10bSShibin Koikkara Reeny if (ret <= 0) {
11253143d10bSShibin Koikkara Reeny ksft_print_msg("ERROR: [%s] Poll error %d\n",
1126085dcccfSMagnus Karlsson __func__, errno);
11273143d10bSShibin Koikkara Reeny return TEST_FAILURE;
11283143d10bSShibin Koikkara Reeny }
11293143d10bSShibin Koikkara Reeny }
11303143d10bSShibin Koikkara Reeny
1131018a8e75SMaciej Fijalkowski complete_pkts(xsk, BATCH_SIZE);
11323143d10bSShibin Koikkara Reeny }
1133018a8e75SMaciej Fijalkowski
1134018a8e75SMaciej Fijalkowski for (i = 0; i < BATCH_SIZE; i++) {
113517f1034dSMagnus Karlsson struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
113669760449SMagnus Karlsson u32 nb_frags_left, nb_frags, bytes_written = 0;
1137018a8e75SMaciej Fijalkowski
1138018a8e75SMaciej Fijalkowski if (!pkt)
1139018a8e75SMaciej Fijalkowski break;
1140018a8e75SMaciej Fijalkowski
114169760449SMagnus Karlsson nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);
114217f1034dSMagnus Karlsson if (nb_frags > BATCH_SIZE - i) {
114317f1034dSMagnus Karlsson pkt_stream_cancel(pkt_stream);
114417f1034dSMagnus Karlsson xsk_ring_prod__cancel(&xsk->tx, BATCH_SIZE - i);
114517f1034dSMagnus Karlsson break;
114617f1034dSMagnus Karlsson }
114769760449SMagnus Karlsson nb_frags_left = nb_frags;
114817f1034dSMagnus Karlsson
114969760449SMagnus Karlsson while (nb_frags_left--) {
115017f1034dSMagnus Karlsson struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
115117f1034dSMagnus Karlsson
115217f1034dSMagnus Karlsson tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);
115369760449SMagnus Karlsson if (pkt_stream->verbatim) {
115469760449SMagnus Karlsson tx_desc->len = pkt->len;
115569760449SMagnus Karlsson tx_desc->options = pkt->options;
115669760449SMagnus Karlsson } else if (nb_frags_left) {
115717f1034dSMagnus Karlsson tx_desc->len = umem->frame_size;
115817f1034dSMagnus Karlsson tx_desc->options = XDP_PKT_CONTD;
115917f1034dSMagnus Karlsson } else {
116017f1034dSMagnus Karlsson tx_desc->len = pkt->len - bytes_written;
116117f1034dSMagnus Karlsson tx_desc->options = 0;
116217f1034dSMagnus Karlsson }
116317f1034dSMagnus Karlsson if (pkt->valid)
116417f1034dSMagnus Karlsson pkt_generate(ifobject, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
116517f1034dSMagnus Karlsson bytes_written);
116617f1034dSMagnus Karlsson bytes_written += tx_desc->len;
116769760449SMagnus Karlsson
116869760449SMagnus Karlsson if (nb_frags_left) {
116969760449SMagnus Karlsson i++;
117069760449SMagnus Karlsson if (pkt_stream->verbatim)
117169760449SMagnus Karlsson pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
117269760449SMagnus Karlsson }
117369760449SMagnus Karlsson }
117469760449SMagnus Karlsson
117569760449SMagnus Karlsson if (pkt && pkt->valid) {
117669760449SMagnus Karlsson valid_pkts++;
117769760449SMagnus Karlsson valid_frags += nb_frags;
11782f6eae0dSMagnus Karlsson }
1179018a8e75SMaciej Fijalkowski }
1180018a8e75SMaciej Fijalkowski
1181018a8e75SMaciej Fijalkowski pthread_mutex_lock(&pacing_mutex);
1182018a8e75SMaciej Fijalkowski pkts_in_flight += valid_pkts;
1183018a8e75SMaciej Fijalkowski pthread_mutex_unlock(&pacing_mutex);
1184018a8e75SMaciej Fijalkowski
1185018a8e75SMaciej Fijalkowski xsk_ring_prod__submit(&xsk->tx, i);
118617f1034dSMagnus Karlsson xsk->outstanding_tx += valid_frags;
11873143d10bSShibin Koikkara Reeny
11883143d10bSShibin Koikkara Reeny if (use_poll) {
11893143d10bSShibin Koikkara Reeny ret = poll(fds, 1, POLL_TMOUT);
11903143d10bSShibin Koikkara Reeny if (ret <= 0) {
11913143d10bSShibin Koikkara Reeny if (ret == 0 && timeout)
11923143d10bSShibin Koikkara Reeny return TEST_PASS;
11933143d10bSShibin Koikkara Reeny
11943143d10bSShibin Koikkara Reeny ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
11953143d10bSShibin Koikkara Reeny return TEST_FAILURE;
11963143d10bSShibin Koikkara Reeny }
11973143d10bSShibin Koikkara Reeny }
11983143d10bSShibin Koikkara Reeny
11993143d10bSShibin Koikkara Reeny if (!timeout) {
1200018a8e75SMaciej Fijalkowski if (complete_pkts(xsk, i))
1201018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1202018a8e75SMaciej Fijalkowski
1203018a8e75SMaciej Fijalkowski usleep(10);
1204018a8e75SMaciej Fijalkowski return TEST_PASS;
1205018a8e75SMaciej Fijalkowski }
1206018a8e75SMaciej Fijalkowski
12073143d10bSShibin Koikkara Reeny return TEST_CONTINUE;
12083143d10bSShibin Koikkara Reeny }
12093143d10bSShibin Koikkara Reeny
wait_for_tx_completion(struct xsk_socket_info * xsk)1210018a8e75SMaciej Fijalkowski static void wait_for_tx_completion(struct xsk_socket_info *xsk)
1211018a8e75SMaciej Fijalkowski {
1212018a8e75SMaciej Fijalkowski while (xsk->outstanding_tx)
1213018a8e75SMaciej Fijalkowski complete_pkts(xsk, BATCH_SIZE);
1214018a8e75SMaciej Fijalkowski }
1215018a8e75SMaciej Fijalkowski
send_pkts(struct test_spec * test,struct ifobject * ifobject)1216018a8e75SMaciej Fijalkowski static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
1217018a8e75SMaciej Fijalkowski {
121869fc03d2SMagnus Karlsson struct pkt_stream *pkt_stream = ifobject->pkt_stream;
12193143d10bSShibin Koikkara Reeny bool timeout = !is_umem_valid(test->ifobj_rx);
1220018a8e75SMaciej Fijalkowski struct pollfd fds = { };
122169fc03d2SMagnus Karlsson u32 ret;
1222018a8e75SMaciej Fijalkowski
1223018a8e75SMaciej Fijalkowski fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1224018a8e75SMaciej Fijalkowski fds.events = POLLOUT;
1225018a8e75SMaciej Fijalkowski
122669fc03d2SMagnus Karlsson while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
122769fc03d2SMagnus Karlsson ret = __send_pkts(ifobject, &fds, timeout);
12287cd6df4fSMagnus Karlsson if (ret == TEST_CONTINUE && !test->fail)
12297cd6df4fSMagnus Karlsson continue;
12303143d10bSShibin Koikkara Reeny if ((ret || test->fail) && !timeout)
1231018a8e75SMaciej Fijalkowski return TEST_FAILURE;
12327cd6df4fSMagnus Karlsson if (ret == TEST_PASS && timeout)
12333143d10bSShibin Koikkara Reeny return ret;
1234018a8e75SMaciej Fijalkowski }
1235018a8e75SMaciej Fijalkowski
1236018a8e75SMaciej Fijalkowski wait_for_tx_completion(ifobject->xsk);
1237018a8e75SMaciej Fijalkowski return TEST_PASS;
1238018a8e75SMaciej Fijalkowski }
1239018a8e75SMaciej Fijalkowski
get_xsk_stats(struct xsk_socket * xsk,struct xdp_statistics * stats)1240018a8e75SMaciej Fijalkowski static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1241018a8e75SMaciej Fijalkowski {
1242018a8e75SMaciej Fijalkowski int fd = xsk_socket__fd(xsk), err;
1243018a8e75SMaciej Fijalkowski socklen_t optlen, expected_len;
1244018a8e75SMaciej Fijalkowski
1245018a8e75SMaciej Fijalkowski optlen = sizeof(*stats);
1246018a8e75SMaciej Fijalkowski err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1247018a8e75SMaciej Fijalkowski if (err) {
1248018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1249018a8e75SMaciej Fijalkowski __func__, -err, strerror(-err));
1250018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1251018a8e75SMaciej Fijalkowski }
1252018a8e75SMaciej Fijalkowski
1253018a8e75SMaciej Fijalkowski expected_len = sizeof(struct xdp_statistics);
1254018a8e75SMaciej Fijalkowski if (optlen != expected_len) {
1255018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1256018a8e75SMaciej Fijalkowski __func__, expected_len, optlen);
1257018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1258018a8e75SMaciej Fijalkowski }
1259018a8e75SMaciej Fijalkowski
1260018a8e75SMaciej Fijalkowski return TEST_PASS;
1261018a8e75SMaciej Fijalkowski }
1262018a8e75SMaciej Fijalkowski
validate_rx_dropped(struct ifobject * ifobject)1263018a8e75SMaciej Fijalkowski static int validate_rx_dropped(struct ifobject *ifobject)
1264018a8e75SMaciej Fijalkowski {
1265018a8e75SMaciej Fijalkowski struct xsk_socket *xsk = ifobject->xsk->xsk;
1266018a8e75SMaciej Fijalkowski struct xdp_statistics stats;
1267018a8e75SMaciej Fijalkowski int err;
1268018a8e75SMaciej Fijalkowski
1269018a8e75SMaciej Fijalkowski kick_rx(ifobject->xsk);
1270018a8e75SMaciej Fijalkowski
1271018a8e75SMaciej Fijalkowski err = get_xsk_stats(xsk, &stats);
1272018a8e75SMaciej Fijalkowski if (err)
1273018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1274018a8e75SMaciej Fijalkowski
127568e73221SKal Conley /* The receiver calls getsockopt after receiving the last (valid)
127668e73221SKal Conley * packet which is not the final packet sent in this test (valid and
127768e73221SKal Conley * invalid packets are sent in alternating fashion with the final
127868e73221SKal Conley * packet being invalid). Since the last packet may or may not have
127968e73221SKal Conley * been dropped already, both outcomes must be allowed.
128068e73221SKal Conley */
128168e73221SKal Conley if (stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 ||
128268e73221SKal Conley stats.rx_dropped == ifobject->pkt_stream->nb_pkts / 2 - 1)
1283018a8e75SMaciej Fijalkowski return TEST_PASS;
1284018a8e75SMaciej Fijalkowski
1285018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1286018a8e75SMaciej Fijalkowski }
1287018a8e75SMaciej Fijalkowski
validate_rx_full(struct ifobject * ifobject)1288018a8e75SMaciej Fijalkowski static int validate_rx_full(struct ifobject *ifobject)
1289018a8e75SMaciej Fijalkowski {
1290018a8e75SMaciej Fijalkowski struct xsk_socket *xsk = ifobject->xsk->xsk;
1291018a8e75SMaciej Fijalkowski struct xdp_statistics stats;
1292018a8e75SMaciej Fijalkowski int err;
1293018a8e75SMaciej Fijalkowski
1294018a8e75SMaciej Fijalkowski usleep(1000);
1295018a8e75SMaciej Fijalkowski kick_rx(ifobject->xsk);
1296018a8e75SMaciej Fijalkowski
1297018a8e75SMaciej Fijalkowski err = get_xsk_stats(xsk, &stats);
1298018a8e75SMaciej Fijalkowski if (err)
1299018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1300018a8e75SMaciej Fijalkowski
1301018a8e75SMaciej Fijalkowski if (stats.rx_ring_full)
1302018a8e75SMaciej Fijalkowski return TEST_PASS;
1303018a8e75SMaciej Fijalkowski
1304018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1305018a8e75SMaciej Fijalkowski }
1306018a8e75SMaciej Fijalkowski
validate_fill_empty(struct ifobject * ifobject)1307018a8e75SMaciej Fijalkowski static int validate_fill_empty(struct ifobject *ifobject)
1308018a8e75SMaciej Fijalkowski {
1309018a8e75SMaciej Fijalkowski struct xsk_socket *xsk = ifobject->xsk->xsk;
1310018a8e75SMaciej Fijalkowski struct xdp_statistics stats;
1311018a8e75SMaciej Fijalkowski int err;
1312018a8e75SMaciej Fijalkowski
1313018a8e75SMaciej Fijalkowski usleep(1000);
1314018a8e75SMaciej Fijalkowski kick_rx(ifobject->xsk);
1315018a8e75SMaciej Fijalkowski
1316018a8e75SMaciej Fijalkowski err = get_xsk_stats(xsk, &stats);
1317018a8e75SMaciej Fijalkowski if (err)
1318018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1319018a8e75SMaciej Fijalkowski
1320018a8e75SMaciej Fijalkowski if (stats.rx_fill_ring_empty_descs)
1321018a8e75SMaciej Fijalkowski return TEST_PASS;
1322018a8e75SMaciej Fijalkowski
1323018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1324018a8e75SMaciej Fijalkowski }
1325018a8e75SMaciej Fijalkowski
validate_tx_invalid_descs(struct ifobject * ifobject)1326018a8e75SMaciej Fijalkowski static int validate_tx_invalid_descs(struct ifobject *ifobject)
1327018a8e75SMaciej Fijalkowski {
1328018a8e75SMaciej Fijalkowski struct xsk_socket *xsk = ifobject->xsk->xsk;
1329018a8e75SMaciej Fijalkowski int fd = xsk_socket__fd(xsk);
1330018a8e75SMaciej Fijalkowski struct xdp_statistics stats;
1331018a8e75SMaciej Fijalkowski socklen_t optlen;
1332018a8e75SMaciej Fijalkowski int err;
1333018a8e75SMaciej Fijalkowski
1334018a8e75SMaciej Fijalkowski optlen = sizeof(stats);
1335018a8e75SMaciej Fijalkowski err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1336018a8e75SMaciej Fijalkowski if (err) {
1337018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1338018a8e75SMaciej Fijalkowski __func__, -err, strerror(-err));
1339018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1340018a8e75SMaciej Fijalkowski }
1341018a8e75SMaciej Fijalkowski
1342018a8e75SMaciej Fijalkowski if (stats.tx_invalid_descs != ifobject->pkt_stream->nb_pkts / 2) {
1343018a8e75SMaciej Fijalkowski ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%u] expected [%u]\n",
1344018a8e75SMaciej Fijalkowski __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
1345018a8e75SMaciej Fijalkowski return TEST_FAILURE;
1346018a8e75SMaciej Fijalkowski }
1347018a8e75SMaciej Fijalkowski
1348018a8e75SMaciej Fijalkowski return TEST_PASS;
1349018a8e75SMaciej Fijalkowski }
1350018a8e75SMaciej Fijalkowski
xsk_configure_socket(struct test_spec * test,struct ifobject * ifobject,struct xsk_umem_info * umem,bool tx)1351a693ff3eSMaciej Fijalkowski static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1352a693ff3eSMaciej Fijalkowski struct xsk_umem_info *umem, bool tx)
1353a693ff3eSMaciej Fijalkowski {
1354a693ff3eSMaciej Fijalkowski int i, ret;
1355a693ff3eSMaciej Fijalkowski
1356a693ff3eSMaciej Fijalkowski for (i = 0; i < test->nb_sockets; i++) {
1357a693ff3eSMaciej Fijalkowski bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1358a693ff3eSMaciej Fijalkowski u32 ctr = 0;
1359a693ff3eSMaciej Fijalkowski
1360a693ff3eSMaciej Fijalkowski while (ctr++ < SOCK_RECONF_CTR) {
1361a693ff3eSMaciej Fijalkowski ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1362a693ff3eSMaciej Fijalkowski ifobject, shared);
1363a693ff3eSMaciej Fijalkowski if (!ret)
1364a693ff3eSMaciej Fijalkowski break;
1365a693ff3eSMaciej Fijalkowski
1366a693ff3eSMaciej Fijalkowski /* Retry if it fails as xsk_socket__create() is asynchronous */
1367a693ff3eSMaciej Fijalkowski if (ctr >= SOCK_RECONF_CTR)
1368a693ff3eSMaciej Fijalkowski exit_with_error(-ret);
1369a693ff3eSMaciej Fijalkowski usleep(USLEEP_MAX);
1370a693ff3eSMaciej Fijalkowski }
1371a693ff3eSMaciej Fijalkowski if (ifobject->busy_poll)
1372a693ff3eSMaciej Fijalkowski enable_busy_poll(&ifobject->xsk_arr[i]);
1373a693ff3eSMaciej Fijalkowski }
1374a693ff3eSMaciej Fijalkowski }
1375a693ff3eSMaciej Fijalkowski
thread_common_ops_tx(struct test_spec * test,struct ifobject * ifobject)1376a693ff3eSMaciej Fijalkowski static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1377a693ff3eSMaciej Fijalkowski {
1378a693ff3eSMaciej Fijalkowski xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1379a693ff3eSMaciej Fijalkowski ifobject->xsk = &ifobject->xsk_arr[0];
1380f0a249dfSMagnus Karlsson ifobject->xskmap = test->ifobj_rx->xskmap;
1381a693ff3eSMaciej Fijalkowski memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
1382d9f6d970SMagnus Karlsson ifobject->umem->base_addr = 0;
1383a693ff3eSMaciej Fijalkowski }
1384a693ff3eSMaciej Fijalkowski
xsk_populate_fill_ring(struct xsk_umem_info * umem,struct pkt_stream * pkt_stream,bool fill_up)138586e41755SMagnus Karlsson static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,
138686e41755SMagnus Karlsson bool fill_up)
1387a693ff3eSMaciej Fijalkowski {
138886e41755SMagnus Karlsson u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;
138986e41755SMagnus Karlsson u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;
1390a693ff3eSMaciej Fijalkowski int ret;
1391a693ff3eSMaciej Fijalkowski
1392a693ff3eSMaciej Fijalkowski if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1393a693ff3eSMaciej Fijalkowski buffers_to_fill = umem->num_frames;
1394a693ff3eSMaciej Fijalkowski else
1395a693ff3eSMaciej Fijalkowski buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1396a693ff3eSMaciej Fijalkowski
1397a693ff3eSMaciej Fijalkowski ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1398a693ff3eSMaciej Fijalkowski if (ret != buffers_to_fill)
1399a693ff3eSMaciej Fijalkowski exit_with_error(ENOSPC);
1400d9f6d970SMagnus Karlsson
140186e41755SMagnus Karlsson while (filled < buffers_to_fill) {
1402d9f6d970SMagnus Karlsson struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);
1403a693ff3eSMaciej Fijalkowski u64 addr;
140486e41755SMagnus Karlsson u32 i;
1405a693ff3eSMaciej Fijalkowski
140669760449SMagnus Karlsson for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {
140786e41755SMagnus Karlsson if (!pkt) {
140886e41755SMagnus Karlsson if (!fill_up)
140986e41755SMagnus Karlsson break;
141086e41755SMagnus Karlsson addr = filled * umem->frame_size + umem->base_addr;
141186e41755SMagnus Karlsson } else if (pkt->offset >= 0) {
1412d9f6d970SMagnus Karlsson addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);
141386e41755SMagnus Karlsson } else {
1414d9f6d970SMagnus Karlsson addr = pkt->offset + umem_alloc_buffer(umem);
1415a693ff3eSMaciej Fijalkowski }
141686e41755SMagnus Karlsson
141786e41755SMagnus Karlsson *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
141886e41755SMagnus Karlsson if (++filled >= buffers_to_fill)
141986e41755SMagnus Karlsson break;
142086e41755SMagnus Karlsson }
142186e41755SMagnus Karlsson }
142286e41755SMagnus Karlsson xsk_ring_prod__submit(&umem->fq, filled);
142386e41755SMagnus Karlsson xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);
142469fc03d2SMagnus Karlsson
142569fc03d2SMagnus Karlsson pkt_stream_reset(pkt_stream);
1426d9f6d970SMagnus Karlsson umem_reset_alloc(umem);
1427a693ff3eSMaciej Fijalkowski }
1428a693ff3eSMaciej Fijalkowski
thread_common_ops(struct test_spec * test,struct ifobject * ifobject)1429018a8e75SMaciej Fijalkowski static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
1430018a8e75SMaciej Fijalkowski {
1431018a8e75SMaciej Fijalkowski u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
1432018a8e75SMaciej Fijalkowski int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
1433018a8e75SMaciej Fijalkowski LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1434018a8e75SMaciej Fijalkowski void *bufs;
1435f0a249dfSMagnus Karlsson int ret;
1436018a8e75SMaciej Fijalkowski
143702e93e04SMagnus Karlsson if (ifobject->umem->unaligned_mode)
143802e93e04SMagnus Karlsson mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
1439018a8e75SMaciej Fijalkowski
1440a693ff3eSMaciej Fijalkowski if (ifobject->shared_umem)
1441a693ff3eSMaciej Fijalkowski umem_sz *= 2;
1442a693ff3eSMaciej Fijalkowski
144302e93e04SMagnus Karlsson bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
1444018a8e75SMaciej Fijalkowski if (bufs == MAP_FAILED)
1445018a8e75SMaciej Fijalkowski exit_with_error(errno);
1446018a8e75SMaciej Fijalkowski
1447d9f6d970SMagnus Karlsson ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);
1448018a8e75SMaciej Fijalkowski if (ret)
1449018a8e75SMaciej Fijalkowski exit_with_error(-ret);
1450018a8e75SMaciej Fijalkowski
1451a693ff3eSMaciej Fijalkowski xsk_configure_socket(test, ifobject, ifobject->umem, false);
1452018a8e75SMaciej Fijalkowski
1453018a8e75SMaciej Fijalkowski ifobject->xsk = &ifobject->xsk_arr[0];
1454018a8e75SMaciej Fijalkowski
1455018a8e75SMaciej Fijalkowski if (!ifobject->rx_on)
1456018a8e75SMaciej Fijalkowski return;
1457018a8e75SMaciej Fijalkowski
145886e41755SMagnus Karlsson xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream, ifobject->use_fill_ring);
1459d9f6d970SMagnus Karlsson
1460f0a249dfSMagnus Karlsson ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
1461018a8e75SMaciej Fijalkowski if (ret)
1462085dcccfSMagnus Karlsson exit_with_error(errno);
1463018a8e75SMaciej Fijalkowski }
1464018a8e75SMaciej Fijalkowski
worker_testapp_validate_tx(void * arg)1465018a8e75SMaciej Fijalkowski static void *worker_testapp_validate_tx(void *arg)
1466018a8e75SMaciej Fijalkowski {
1467018a8e75SMaciej Fijalkowski struct test_spec *test = (struct test_spec *)arg;
1468018a8e75SMaciej Fijalkowski struct ifobject *ifobject = test->ifobj_tx;
1469018a8e75SMaciej Fijalkowski int err;
1470018a8e75SMaciej Fijalkowski
1471a693ff3eSMaciej Fijalkowski if (test->current_step == 1) {
1472a693ff3eSMaciej Fijalkowski if (!ifobject->shared_umem)
1473018a8e75SMaciej Fijalkowski thread_common_ops(test, ifobject);
1474a693ff3eSMaciej Fijalkowski else
1475a693ff3eSMaciej Fijalkowski thread_common_ops_tx(test, ifobject);
1476a693ff3eSMaciej Fijalkowski }
1477018a8e75SMaciej Fijalkowski
1478018a8e75SMaciej Fijalkowski print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1479018a8e75SMaciej Fijalkowski ifobject->ifname);
1480018a8e75SMaciej Fijalkowski err = send_pkts(test, ifobject);
1481018a8e75SMaciej Fijalkowski
1482018a8e75SMaciej Fijalkowski if (!err && ifobject->validation_func)
1483018a8e75SMaciej Fijalkowski err = ifobject->validation_func(ifobject);
1484018a8e75SMaciej Fijalkowski if (err)
1485018a8e75SMaciej Fijalkowski report_failure(test);
1486018a8e75SMaciej Fijalkowski
1487018a8e75SMaciej Fijalkowski pthread_exit(NULL);
1488018a8e75SMaciej Fijalkowski }
1489018a8e75SMaciej Fijalkowski
worker_testapp_validate_rx(void * arg)1490018a8e75SMaciej Fijalkowski static void *worker_testapp_validate_rx(void *arg)
1491018a8e75SMaciej Fijalkowski {
1492018a8e75SMaciej Fijalkowski struct test_spec *test = (struct test_spec *)arg;
1493018a8e75SMaciej Fijalkowski struct ifobject *ifobject = test->ifobj_rx;
1494018a8e75SMaciej Fijalkowski struct pollfd fds = { };
1495f0a249dfSMagnus Karlsson int err;
1496018a8e75SMaciej Fijalkowski
1497a693ff3eSMaciej Fijalkowski if (test->current_step == 1) {
1498018a8e75SMaciej Fijalkowski thread_common_ops(test, ifobject);
1499a693ff3eSMaciej Fijalkowski } else {
1500f0a249dfSMagnus Karlsson xsk_clear_xskmap(ifobject->xskmap);
1501f0a249dfSMagnus Karlsson err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk);
1502aa61d81fSMagnus Karlsson if (err) {
1503f0a249dfSMagnus Karlsson printf("Error: Failed to update xskmap, error %s\n", strerror(-err));
1504f0a249dfSMagnus Karlsson exit_with_error(-err);
1505aa61d81fSMagnus Karlsson }
1506a693ff3eSMaciej Fijalkowski }
1507018a8e75SMaciej Fijalkowski
1508018a8e75SMaciej Fijalkowski fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1509018a8e75SMaciej Fijalkowski fds.events = POLLIN;
1510018a8e75SMaciej Fijalkowski
1511018a8e75SMaciej Fijalkowski pthread_barrier_wait(&barr);
1512018a8e75SMaciej Fijalkowski
15133143d10bSShibin Koikkara Reeny err = receive_pkts(test, &fds);
1514018a8e75SMaciej Fijalkowski
1515018a8e75SMaciej Fijalkowski if (!err && ifobject->validation_func)
1516018a8e75SMaciej Fijalkowski err = ifobject->validation_func(ifobject);
15177cd6df4fSMagnus Karlsson if (err)
1518018a8e75SMaciej Fijalkowski report_failure(test);
1519018a8e75SMaciej Fijalkowski
1520018a8e75SMaciej Fijalkowski pthread_exit(NULL);
1521018a8e75SMaciej Fijalkowski }
1522018a8e75SMaciej Fijalkowski
ceil_u64(u64 a,u64 b)15232ddade32SMagnus Karlsson static u64 ceil_u64(u64 a, u64 b)
15242ddade32SMagnus Karlsson {
15252ddade32SMagnus Karlsson return (a + b - 1) / b;
15262ddade32SMagnus Karlsson }
15272ddade32SMagnus Karlsson
testapp_clean_xsk_umem(struct ifobject * ifobj)1528a693ff3eSMaciej Fijalkowski static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1529a693ff3eSMaciej Fijalkowski {
1530a693ff3eSMaciej Fijalkowski u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1531a693ff3eSMaciej Fijalkowski
1532a693ff3eSMaciej Fijalkowski if (ifobj->shared_umem)
1533a693ff3eSMaciej Fijalkowski umem_sz *= 2;
1534a693ff3eSMaciej Fijalkowski
15352ddade32SMagnus Karlsson umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
1536a693ff3eSMaciej Fijalkowski xsk_umem__delete(ifobj->umem->umem);
1537a693ff3eSMaciej Fijalkowski munmap(ifobj->umem->buffer, umem_sz);
1538a693ff3eSMaciej Fijalkowski }
1539a693ff3eSMaciej Fijalkowski
handler(int signum)1540c29fe883SMaciej Fijalkowski static void handler(int signum)
1541c29fe883SMaciej Fijalkowski {
1542c29fe883SMaciej Fijalkowski pthread_exit(NULL);
1543c29fe883SMaciej Fijalkowski }
1544c29fe883SMaciej Fijalkowski
xdp_prog_changed_rx(struct test_spec * test)1545d2e54149SMagnus Karlsson static bool xdp_prog_changed_rx(struct test_spec *test)
15467d8319a7SMagnus Karlsson {
1547d2e54149SMagnus Karlsson struct ifobject *ifobj = test->ifobj_rx;
1548d2e54149SMagnus Karlsson
15497d8319a7SMagnus Karlsson return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
15507d8319a7SMagnus Karlsson }
15517d8319a7SMagnus Karlsson
xdp_prog_changed_tx(struct test_spec * test)1552d2e54149SMagnus Karlsson static bool xdp_prog_changed_tx(struct test_spec *test)
1553d2e54149SMagnus Karlsson {
1554d2e54149SMagnus Karlsson struct ifobject *ifobj = test->ifobj_tx;
1555d2e54149SMagnus Karlsson
1556d2e54149SMagnus Karlsson return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;
1557d2e54149SMagnus Karlsson }
1558d2e54149SMagnus Karlsson
xsk_reattach_xdp(struct ifobject * ifobj,struct bpf_program * xdp_prog,struct bpf_map * xskmap,enum test_mode mode)15597d8319a7SMagnus Karlsson static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
15607d8319a7SMagnus Karlsson struct bpf_map *xskmap, enum test_mode mode)
15617d8319a7SMagnus Karlsson {
15627d8319a7SMagnus Karlsson int err;
15637d8319a7SMagnus Karlsson
15647d8319a7SMagnus Karlsson xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
15657d8319a7SMagnus Karlsson err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
15667d8319a7SMagnus Karlsson if (err) {
15677d8319a7SMagnus Karlsson printf("Error attaching XDP program\n");
15687d8319a7SMagnus Karlsson exit_with_error(-err);
15697d8319a7SMagnus Karlsson }
15707d8319a7SMagnus Karlsson
15717d8319a7SMagnus Karlsson if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
15727d8319a7SMagnus Karlsson if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
15737d8319a7SMagnus Karlsson ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
15747d8319a7SMagnus Karlsson exit_with_error(EINVAL);
15757d8319a7SMagnus Karlsson }
15767d8319a7SMagnus Karlsson
15777d8319a7SMagnus Karlsson ifobj->xdp_prog = xdp_prog;
15787d8319a7SMagnus Karlsson ifobj->xskmap = xskmap;
15797d8319a7SMagnus Karlsson ifobj->mode = mode;
15807d8319a7SMagnus Karlsson }
15817d8319a7SMagnus Karlsson
xsk_attach_xdp_progs(struct test_spec * test,struct ifobject * ifobj_rx,struct ifobject * ifobj_tx)15827d8319a7SMagnus Karlsson static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
15837d8319a7SMagnus Karlsson struct ifobject *ifobj_tx)
15847d8319a7SMagnus Karlsson {
1585d2e54149SMagnus Karlsson if (xdp_prog_changed_rx(test))
15867d8319a7SMagnus Karlsson xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
15877d8319a7SMagnus Karlsson
15887d8319a7SMagnus Karlsson if (!ifobj_tx || ifobj_tx->shared_umem)
15897d8319a7SMagnus Karlsson return;
15907d8319a7SMagnus Karlsson
1591d2e54149SMagnus Karlsson if (xdp_prog_changed_tx(test))
15927d8319a7SMagnus Karlsson xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
15937d8319a7SMagnus Karlsson }
15947d8319a7SMagnus Karlsson
__testapp_validate_traffic(struct test_spec * test,struct ifobject * ifobj1,struct ifobject * ifobj2)15957f881984SMagnus Karlsson static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
15967f881984SMagnus Karlsson struct ifobject *ifobj2)
15973143d10bSShibin Koikkara Reeny {
15987f881984SMagnus Karlsson pthread_t t0, t1;
1599f540d44eSMagnus Karlsson int err;
1600f540d44eSMagnus Karlsson
1601f540d44eSMagnus Karlsson if (test->mtu > MAX_ETH_PKT_SIZE) {
1602f540d44eSMagnus Karlsson if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||
1603f540d44eSMagnus Karlsson (ifobj2 && !ifobj2->multi_buff_zc_supp))) {
1604f540d44eSMagnus Karlsson ksft_test_result_skip("Multi buffer for zero-copy not supported.\n");
1605f540d44eSMagnus Karlsson return TEST_SKIP;
1606f540d44eSMagnus Karlsson }
1607f540d44eSMagnus Karlsson if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||
1608f540d44eSMagnus Karlsson (ifobj2 && !ifobj2->multi_buff_supp))) {
1609f540d44eSMagnus Karlsson ksft_test_result_skip("Multi buffer not supported.\n");
1610f540d44eSMagnus Karlsson return TEST_SKIP;
1611f540d44eSMagnus Karlsson }
1612f540d44eSMagnus Karlsson }
1613f540d44eSMagnus Karlsson err = test_spec_set_mtu(test, test->mtu);
1614f540d44eSMagnus Karlsson if (err) {
1615f540d44eSMagnus Karlsson ksft_print_msg("Error, could not set mtu.\n");
1616f540d44eSMagnus Karlsson exit_with_error(err);
1617f540d44eSMagnus Karlsson }
16183143d10bSShibin Koikkara Reeny
161969fc03d2SMagnus Karlsson if (ifobj2) {
16203143d10bSShibin Koikkara Reeny if (pthread_barrier_init(&barr, NULL, 2))
16213143d10bSShibin Koikkara Reeny exit_with_error(errno);
162269fc03d2SMagnus Karlsson pkt_stream_reset(ifobj2->pkt_stream);
162369fc03d2SMagnus Karlsson }
16243143d10bSShibin Koikkara Reeny
16253143d10bSShibin Koikkara Reeny test->current_step++;
16267f881984SMagnus Karlsson pkt_stream_reset(ifobj1->pkt_stream);
16273143d10bSShibin Koikkara Reeny pkts_in_flight = 0;
16283143d10bSShibin Koikkara Reeny
1629c29fe883SMaciej Fijalkowski signal(SIGUSR1, handler);
16307f881984SMagnus Karlsson /*Spawn RX thread */
16317f881984SMagnus Karlsson pthread_create(&t0, NULL, ifobj1->func_ptr, test);
16323143d10bSShibin Koikkara Reeny
16337f881984SMagnus Karlsson if (ifobj2) {
16343143d10bSShibin Koikkara Reeny pthread_barrier_wait(&barr);
16353143d10bSShibin Koikkara Reeny if (pthread_barrier_destroy(&barr))
16363143d10bSShibin Koikkara Reeny exit_with_error(errno);
16373143d10bSShibin Koikkara Reeny
16387f881984SMagnus Karlsson /*Spawn TX thread */
16397f881984SMagnus Karlsson pthread_create(&t1, NULL, ifobj2->func_ptr, test);
16407f881984SMagnus Karlsson
16417f881984SMagnus Karlsson pthread_join(t1, NULL);
16427f881984SMagnus Karlsson }
16437f881984SMagnus Karlsson
16447f881984SMagnus Karlsson if (!ifobj2)
1645c29fe883SMaciej Fijalkowski pthread_kill(t0, SIGUSR1);
16467f881984SMagnus Karlsson else
16473143d10bSShibin Koikkara Reeny pthread_join(t0, NULL);
16483143d10bSShibin Koikkara Reeny
1649a693ff3eSMaciej Fijalkowski if (test->total_steps == test->current_step || test->fail) {
16507f881984SMagnus Karlsson if (ifobj2)
16517f881984SMagnus Karlsson xsk_socket__delete(ifobj2->xsk->xsk);
16527f881984SMagnus Karlsson xsk_socket__delete(ifobj1->xsk->xsk);
16537f881984SMagnus Karlsson testapp_clean_xsk_umem(ifobj1);
16547f881984SMagnus Karlsson if (ifobj2 && !ifobj2->shared_umem)
16557f881984SMagnus Karlsson testapp_clean_xsk_umem(ifobj2);
1656a693ff3eSMaciej Fijalkowski }
1657a693ff3eSMaciej Fijalkowski
16583143d10bSShibin Koikkara Reeny return !!test->fail;
16593143d10bSShibin Koikkara Reeny }
16603143d10bSShibin Koikkara Reeny
testapp_validate_traffic(struct test_spec * test)1661018a8e75SMaciej Fijalkowski static int testapp_validate_traffic(struct test_spec *test)
1662018a8e75SMaciej Fijalkowski {
16637d8319a7SMagnus Karlsson struct ifobject *ifobj_rx = test->ifobj_rx;
16647d8319a7SMagnus Karlsson struct ifobject *ifobj_tx = test->ifobj_tx;
16657d8319a7SMagnus Karlsson
1666041b68f6SMagnus Karlsson if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||
1667041b68f6SMagnus Karlsson (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {
1668041b68f6SMagnus Karlsson ksft_test_result_skip("No huge pages present.\n");
1669041b68f6SMagnus Karlsson return TEST_SKIP;
1670041b68f6SMagnus Karlsson }
1671041b68f6SMagnus Karlsson
16727d8319a7SMagnus Karlsson xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);
16737d8319a7SMagnus Karlsson return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
1674a693ff3eSMaciej Fijalkowski }
1675a693ff3eSMaciej Fijalkowski
testapp_validate_traffic_single_thread(struct test_spec * test,struct ifobject * ifobj)16767f881984SMagnus Karlsson static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
16777f881984SMagnus Karlsson {
16787f881984SMagnus Karlsson return __testapp_validate_traffic(test, ifobj, NULL);
1679018a8e75SMaciej Fijalkowski }
1680018a8e75SMaciej Fijalkowski
testapp_teardown(struct test_spec * test)1681041b68f6SMagnus Karlsson static int testapp_teardown(struct test_spec *test)
1682018a8e75SMaciej Fijalkowski {
1683018a8e75SMaciej Fijalkowski int i;
1684018a8e75SMaciej Fijalkowski
1685018a8e75SMaciej Fijalkowski test_spec_set_name(test, "TEARDOWN");
1686018a8e75SMaciej Fijalkowski for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
1687018a8e75SMaciej Fijalkowski if (testapp_validate_traffic(test))
1688041b68f6SMagnus Karlsson return TEST_FAILURE;
1689018a8e75SMaciej Fijalkowski test_spec_reset(test);
1690018a8e75SMaciej Fijalkowski }
1691041b68f6SMagnus Karlsson
1692041b68f6SMagnus Karlsson return TEST_PASS;
1693018a8e75SMaciej Fijalkowski }
1694018a8e75SMaciej Fijalkowski
swap_directions(struct ifobject ** ifobj1,struct ifobject ** ifobj2)1695018a8e75SMaciej Fijalkowski static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
1696018a8e75SMaciej Fijalkowski {
1697018a8e75SMaciej Fijalkowski thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
1698018a8e75SMaciej Fijalkowski struct ifobject *tmp_ifobj = (*ifobj1);
1699018a8e75SMaciej Fijalkowski
1700018a8e75SMaciej Fijalkowski (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
1701018a8e75SMaciej Fijalkowski (*ifobj2)->func_ptr = tmp_func_ptr;
1702018a8e75SMaciej Fijalkowski
1703018a8e75SMaciej Fijalkowski *ifobj1 = *ifobj2;
1704018a8e75SMaciej Fijalkowski *ifobj2 = tmp_ifobj;
1705018a8e75SMaciej Fijalkowski }
1706018a8e75SMaciej Fijalkowski
testapp_bidi(struct test_spec * test)1707041b68f6SMagnus Karlsson static int testapp_bidi(struct test_spec *test)
1708018a8e75SMaciej Fijalkowski {
1709041b68f6SMagnus Karlsson int res;
1710041b68f6SMagnus Karlsson
1711018a8e75SMaciej Fijalkowski test_spec_set_name(test, "BIDIRECTIONAL");
1712018a8e75SMaciej Fijalkowski test->ifobj_tx->rx_on = true;
1713018a8e75SMaciej Fijalkowski test->ifobj_rx->tx_on = true;
1714018a8e75SMaciej Fijalkowski test->total_steps = 2;
1715018a8e75SMaciej Fijalkowski if (testapp_validate_traffic(test))
1716041b68f6SMagnus Karlsson return TEST_FAILURE;
1717018a8e75SMaciej Fijalkowski
1718018a8e75SMaciej Fijalkowski print_verbose("Switching Tx/Rx vectors\n");
1719018a8e75SMaciej Fijalkowski swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1720041b68f6SMagnus Karlsson res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
1721018a8e75SMaciej Fijalkowski
1722018a8e75SMaciej Fijalkowski swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1723041b68f6SMagnus Karlsson return res;
1724018a8e75SMaciej Fijalkowski }
1725018a8e75SMaciej Fijalkowski
swap_xsk_resources(struct ifobject * ifobj_tx,struct ifobject * ifobj_rx)1726018a8e75SMaciej Fijalkowski static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
1727018a8e75SMaciej Fijalkowski {
1728f0a249dfSMagnus Karlsson int ret;
1729018a8e75SMaciej Fijalkowski
1730018a8e75SMaciej Fijalkowski xsk_socket__delete(ifobj_tx->xsk->xsk);
1731018a8e75SMaciej Fijalkowski xsk_socket__delete(ifobj_rx->xsk->xsk);
1732018a8e75SMaciej Fijalkowski ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
1733018a8e75SMaciej Fijalkowski ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
1734018a8e75SMaciej Fijalkowski
1735f0a249dfSMagnus Karlsson ret = xsk_update_xskmap(ifobj_rx->xskmap, ifobj_rx->xsk->xsk);
1736018a8e75SMaciej Fijalkowski if (ret)
1737085dcccfSMagnus Karlsson exit_with_error(errno);
1738018a8e75SMaciej Fijalkowski }
1739018a8e75SMaciej Fijalkowski
testapp_bpf_res(struct test_spec * test)1740041b68f6SMagnus Karlsson static int testapp_bpf_res(struct test_spec *test)
1741018a8e75SMaciej Fijalkowski {
1742018a8e75SMaciej Fijalkowski test_spec_set_name(test, "BPF_RES");
1743018a8e75SMaciej Fijalkowski test->total_steps = 2;
1744018a8e75SMaciej Fijalkowski test->nb_sockets = 2;
1745018a8e75SMaciej Fijalkowski if (testapp_validate_traffic(test))
1746041b68f6SMagnus Karlsson return TEST_FAILURE;
1747018a8e75SMaciej Fijalkowski
1748018a8e75SMaciej Fijalkowski swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
1749041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1750018a8e75SMaciej Fijalkowski }
1751018a8e75SMaciej Fijalkowski
testapp_headroom(struct test_spec * test)1752041b68f6SMagnus Karlsson static int testapp_headroom(struct test_spec *test)
1753018a8e75SMaciej Fijalkowski {
1754018a8e75SMaciej Fijalkowski test_spec_set_name(test, "UMEM_HEADROOM");
1755018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
1756041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1757018a8e75SMaciej Fijalkowski }
1758018a8e75SMaciej Fijalkowski
testapp_stats_rx_dropped(struct test_spec * test)1759041b68f6SMagnus Karlsson static int testapp_stats_rx_dropped(struct test_spec *test)
1760018a8e75SMaciej Fijalkowski {
1761018a8e75SMaciej Fijalkowski test_spec_set_name(test, "STAT_RX_DROPPED");
1762041b68f6SMagnus Karlsson if (test->mode == TEST_MODE_ZC) {
1763041b68f6SMagnus Karlsson ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
1764041b68f6SMagnus Karlsson return TEST_SKIP;
1765041b68f6SMagnus Karlsson }
1766041b68f6SMagnus Karlsson
1767a693ff3eSMaciej Fijalkowski pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1768018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1769018a8e75SMaciej Fijalkowski XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1770018a8e75SMaciej Fijalkowski pkt_stream_receive_half(test);
1771018a8e75SMaciej Fijalkowski test->ifobj_rx->validation_func = validate_rx_dropped;
1772041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1773018a8e75SMaciej Fijalkowski }
1774018a8e75SMaciej Fijalkowski
testapp_stats_tx_invalid_descs(struct test_spec * test)1775041b68f6SMagnus Karlsson static int testapp_stats_tx_invalid_descs(struct test_spec *test)
1776018a8e75SMaciej Fijalkowski {
1777018a8e75SMaciej Fijalkowski test_spec_set_name(test, "STAT_TX_INVALID");
1778018a8e75SMaciej Fijalkowski pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
1779018a8e75SMaciej Fijalkowski test->ifobj_tx->validation_func = validate_tx_invalid_descs;
1780041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1781018a8e75SMaciej Fijalkowski }
1782018a8e75SMaciej Fijalkowski
testapp_stats_rx_full(struct test_spec * test)1783041b68f6SMagnus Karlsson static int testapp_stats_rx_full(struct test_spec *test)
1784018a8e75SMaciej Fijalkowski {
1785018a8e75SMaciej Fijalkowski test_spec_set_name(test, "STAT_RX_FULL");
1786df82d2e8SMagnus Karlsson pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
1787018a8e75SMaciej Fijalkowski test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1788df82d2e8SMagnus Karlsson DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
1789018a8e75SMaciej Fijalkowski
1790018a8e75SMaciej Fijalkowski test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
1791018a8e75SMaciej Fijalkowski test->ifobj_rx->release_rx = false;
1792018a8e75SMaciej Fijalkowski test->ifobj_rx->validation_func = validate_rx_full;
1793041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1794018a8e75SMaciej Fijalkowski }
1795018a8e75SMaciej Fijalkowski
testapp_stats_fill_empty(struct test_spec * test)1796041b68f6SMagnus Karlsson static int testapp_stats_fill_empty(struct test_spec *test)
1797018a8e75SMaciej Fijalkowski {
1798018a8e75SMaciej Fijalkowski test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1799df82d2e8SMagnus Karlsson pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
1800018a8e75SMaciej Fijalkowski test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem,
1801df82d2e8SMagnus Karlsson DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
1802018a8e75SMaciej Fijalkowski
1803018a8e75SMaciej Fijalkowski test->ifobj_rx->use_fill_ring = false;
1804018a8e75SMaciej Fijalkowski test->ifobj_rx->validation_func = validate_fill_empty;
1805041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1806018a8e75SMaciej Fijalkowski }
1807018a8e75SMaciej Fijalkowski
testapp_unaligned(struct test_spec * test)1808041b68f6SMagnus Karlsson static int testapp_unaligned(struct test_spec *test)
1809018a8e75SMaciej Fijalkowski {
1810018a8e75SMaciej Fijalkowski test_spec_set_name(test, "UNALIGNED_MODE");
1811018a8e75SMaciej Fijalkowski test->ifobj_tx->umem->unaligned_mode = true;
1812018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->unaligned_mode = true;
1813d9f6d970SMagnus Karlsson /* Let half of the packets straddle a 4K buffer boundary */
1814df82d2e8SMagnus Karlsson pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2);
1815018a8e75SMaciej Fijalkowski
1816041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1817018a8e75SMaciej Fijalkowski }
1818018a8e75SMaciej Fijalkowski
testapp_unaligned_mb(struct test_spec * test)18191005a226SMagnus Karlsson static int testapp_unaligned_mb(struct test_spec *test)
18201005a226SMagnus Karlsson {
18211005a226SMagnus Karlsson test_spec_set_name(test, "UNALIGNED_MODE_9K");
18221005a226SMagnus Karlsson test->mtu = MAX_ETH_JUMBO_SIZE;
18231005a226SMagnus Karlsson test->ifobj_tx->umem->unaligned_mode = true;
18241005a226SMagnus Karlsson test->ifobj_rx->umem->unaligned_mode = true;
18251005a226SMagnus Karlsson pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
18261005a226SMagnus Karlsson return testapp_validate_traffic(test);
18271005a226SMagnus Karlsson }
18281005a226SMagnus Karlsson
testapp_single_pkt(struct test_spec * test)1829041b68f6SMagnus Karlsson static int testapp_single_pkt(struct test_spec *test)
1830018a8e75SMaciej Fijalkowski {
1831d9f6d970SMagnus Karlsson struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};
1832018a8e75SMaciej Fijalkowski
1833018a8e75SMaciej Fijalkowski pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1834041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1835018a8e75SMaciej Fijalkowski }
1836018a8e75SMaciej Fijalkowski
testapp_multi_buffer(struct test_spec * test)1837f540d44eSMagnus Karlsson static int testapp_multi_buffer(struct test_spec *test)
1838f540d44eSMagnus Karlsson {
1839f540d44eSMagnus Karlsson test_spec_set_name(test, "RUN_TO_COMPLETION_9K_PACKETS");
1840f540d44eSMagnus Karlsson test->mtu = MAX_ETH_JUMBO_SIZE;
1841f540d44eSMagnus Karlsson pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
1842f540d44eSMagnus Karlsson
1843f540d44eSMagnus Karlsson return testapp_validate_traffic(test);
1844f540d44eSMagnus Karlsson }
1845f540d44eSMagnus Karlsson
testapp_invalid_desc_mb(struct test_spec * test)184669760449SMagnus Karlsson static int testapp_invalid_desc_mb(struct test_spec *test)
184769760449SMagnus Karlsson {
184869760449SMagnus Karlsson struct xsk_umem_info *umem = test->ifobj_tx->umem;
184969760449SMagnus Karlsson u64 umem_size = umem->num_frames * umem->frame_size;
185069760449SMagnus Karlsson struct pkt pkts[] = {
185169760449SMagnus Karlsson /* Valid packet for synch to start with */
185269760449SMagnus Karlsson {0, MIN_PKT_SIZE, 0, true, 0},
185369760449SMagnus Karlsson /* Zero frame len is not legal */
185469760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
185569760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
185669760449SMagnus Karlsson {0, 0, 0, false, 0},
185769760449SMagnus Karlsson /* Invalid address in the second frame */
185869760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
185969760449SMagnus Karlsson {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
186069760449SMagnus Karlsson /* Invalid len in the middle */
186169760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
186269760449SMagnus Karlsson {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
186369760449SMagnus Karlsson /* Invalid options in the middle */
186469760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
186569760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},
186669760449SMagnus Karlsson /* Transmit 2 frags, receive 3 */
186769760449SMagnus Karlsson {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},
186869760449SMagnus Karlsson {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},
186969760449SMagnus Karlsson /* Middle frame crosses chunk boundary with small length */
187069760449SMagnus Karlsson {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
187169760449SMagnus Karlsson {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},
187269760449SMagnus Karlsson /* Valid packet for synch so that something is received */
187369760449SMagnus Karlsson {0, MIN_PKT_SIZE, 0, true, 0}};
187469760449SMagnus Karlsson
187569760449SMagnus Karlsson if (umem->unaligned_mode) {
187669760449SMagnus Karlsson /* Crossing a chunk boundary allowed */
187769760449SMagnus Karlsson pkts[12].valid = true;
187869760449SMagnus Karlsson pkts[13].valid = true;
187969760449SMagnus Karlsson }
188069760449SMagnus Karlsson
188169760449SMagnus Karlsson test->mtu = MAX_ETH_JUMBO_SIZE;
188269760449SMagnus Karlsson pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
188369760449SMagnus Karlsson return testapp_validate_traffic(test);
188469760449SMagnus Karlsson }
188569760449SMagnus Karlsson
testapp_invalid_desc(struct test_spec * test)1886041b68f6SMagnus Karlsson static int testapp_invalid_desc(struct test_spec *test)
1887018a8e75SMaciej Fijalkowski {
1888d9f6d970SMagnus Karlsson struct xsk_umem_info *umem = test->ifobj_tx->umem;
1889d9f6d970SMagnus Karlsson u64 umem_size = umem->num_frames * umem->frame_size;
1890018a8e75SMaciej Fijalkowski struct pkt pkts[] = {
1891018a8e75SMaciej Fijalkowski /* Zero packet address allowed */
1892df82d2e8SMagnus Karlsson {0, MIN_PKT_SIZE, 0, true},
1893018a8e75SMaciej Fijalkowski /* Allowed packet */
1894d9f6d970SMagnus Karlsson {0, MIN_PKT_SIZE, 0, true},
1895018a8e75SMaciej Fijalkowski /* Straddling the start of umem */
1896df82d2e8SMagnus Karlsson {-2, MIN_PKT_SIZE, 0, false},
1897018a8e75SMaciej Fijalkowski /* Packet too large */
1898d9f6d970SMagnus Karlsson {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1899ccd1b293SKal Conley /* Up to end of umem allowed */
1900d9f6d970SMagnus Karlsson {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},
1901018a8e75SMaciej Fijalkowski /* After umem ends */
1902df82d2e8SMagnus Karlsson {umem_size, MIN_PKT_SIZE, 0, false},
1903018a8e75SMaciej Fijalkowski /* Straddle the end of umem */
1904df82d2e8SMagnus Karlsson {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
1905d9f6d970SMagnus Karlsson /* Straddle a 4K boundary */
1906d9f6d970SMagnus Karlsson {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
1907d9f6d970SMagnus Karlsson /* Straddle a 2K boundary */
1908d9f6d970SMagnus Karlsson {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},
1909018a8e75SMaciej Fijalkowski /* Valid packet for synch so that something is received */
1910d9f6d970SMagnus Karlsson {0, MIN_PKT_SIZE, 0, true}};
1911018a8e75SMaciej Fijalkowski
1912d9f6d970SMagnus Karlsson if (umem->unaligned_mode) {
1913d9f6d970SMagnus Karlsson /* Crossing a page boundary allowed */
1914ccd1b293SKal Conley pkts[7].valid = true;
1915018a8e75SMaciej Fijalkowski }
1916d9f6d970SMagnus Karlsson if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1917d9f6d970SMagnus Karlsson /* Crossing a 2K frame size boundary not allowed */
1918ccd1b293SKal Conley pkts[8].valid = false;
1919018a8e75SMaciej Fijalkowski }
1920018a8e75SMaciej Fijalkowski
1921a693ff3eSMaciej Fijalkowski if (test->ifobj_tx->shared_umem) {
1922d9f6d970SMagnus Karlsson pkts[4].offset += umem_size;
1923d9f6d970SMagnus Karlsson pkts[5].offset += umem_size;
1924d9f6d970SMagnus Karlsson pkts[6].offset += umem_size;
1925a693ff3eSMaciej Fijalkowski }
1926a693ff3eSMaciej Fijalkowski
1927018a8e75SMaciej Fijalkowski pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1928041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
1929018a8e75SMaciej Fijalkowski }
1930018a8e75SMaciej Fijalkowski
testapp_xdp_drop(struct test_spec * test)1931041b68f6SMagnus Karlsson static int testapp_xdp_drop(struct test_spec *test)
193280bea9acSMagnus Karlsson {
19337d8319a7SMagnus Karlsson struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
19347d8319a7SMagnus Karlsson struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
193580bea9acSMagnus Karlsson
193680bea9acSMagnus Karlsson test_spec_set_name(test, "XDP_DROP_HALF");
19377d8319a7SMagnus Karlsson test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
19387d8319a7SMagnus Karlsson skel_rx->maps.xsk, skel_tx->maps.xsk);
193980bea9acSMagnus Karlsson
194080bea9acSMagnus Karlsson pkt_stream_receive_half(test);
1941041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
194280bea9acSMagnus Karlsson }
194380bea9acSMagnus Karlsson
testapp_xdp_metadata_count(struct test_spec * test)1944041b68f6SMagnus Karlsson static int testapp_xdp_metadata_count(struct test_spec *test)
19459a321fd3STushar Vyavahare {
19469a321fd3STushar Vyavahare struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
19479a321fd3STushar Vyavahare struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
19489a321fd3STushar Vyavahare struct bpf_map *data_map;
19499a321fd3STushar Vyavahare int count = 0;
19509a321fd3STushar Vyavahare int key = 0;
19519a321fd3STushar Vyavahare
19529a321fd3STushar Vyavahare test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
19539a321fd3STushar Vyavahare skel_tx->progs.xsk_xdp_populate_metadata,
19549a321fd3STushar Vyavahare skel_rx->maps.xsk, skel_tx->maps.xsk);
19559a321fd3STushar Vyavahare test->ifobj_rx->use_metadata = true;
19569a321fd3STushar Vyavahare
19579a321fd3STushar Vyavahare data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
19589a321fd3STushar Vyavahare if (!data_map || !bpf_map__is_internal(data_map))
19599a321fd3STushar Vyavahare exit_with_error(ENOMEM);
19609a321fd3STushar Vyavahare
19619a321fd3STushar Vyavahare if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY))
19629a321fd3STushar Vyavahare exit_with_error(errno);
19639a321fd3STushar Vyavahare
1964041b68f6SMagnus Karlsson return testapp_validate_traffic(test);
19659a321fd3STushar Vyavahare }
19669a321fd3STushar Vyavahare
testapp_poll_txq_tmout(struct test_spec * test)1967041b68f6SMagnus Karlsson static int testapp_poll_txq_tmout(struct test_spec *test)
19687f881984SMagnus Karlsson {
19697f881984SMagnus Karlsson test_spec_set_name(test, "POLL_TXQ_FULL");
19707f881984SMagnus Karlsson
19717f881984SMagnus Karlsson test->ifobj_tx->use_poll = true;
19727f881984SMagnus Karlsson /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
19737f881984SMagnus Karlsson test->ifobj_tx->umem->frame_size = 2048;
19747f881984SMagnus Karlsson pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
1975041b68f6SMagnus Karlsson return testapp_validate_traffic_single_thread(test, test->ifobj_tx);
19767f881984SMagnus Karlsson }
19777f881984SMagnus Karlsson
testapp_poll_rxq_tmout(struct test_spec * test)1978041b68f6SMagnus Karlsson static int testapp_poll_rxq_tmout(struct test_spec *test)
19797f881984SMagnus Karlsson {
19807f881984SMagnus Karlsson test_spec_set_name(test, "POLL_RXQ_EMPTY");
19817f881984SMagnus Karlsson test->ifobj_rx->use_poll = true;
1982041b68f6SMagnus Karlsson return testapp_validate_traffic_single_thread(test, test->ifobj_rx);
19837f881984SMagnus Karlsson }
19847f881984SMagnus Karlsson
testapp_too_many_frags(struct test_spec * test)1985807bf4daSMagnus Karlsson static int testapp_too_many_frags(struct test_spec *test)
1986807bf4daSMagnus Karlsson {
1987807bf4daSMagnus Karlsson struct pkt pkts[2 * XSK_DESC__MAX_SKB_FRAGS + 2] = {};
1988807bf4daSMagnus Karlsson u32 max_frags, i;
1989807bf4daSMagnus Karlsson
1990807bf4daSMagnus Karlsson test_spec_set_name(test, "TOO_MANY_FRAGS");
1991807bf4daSMagnus Karlsson if (test->mode == TEST_MODE_ZC)
1992807bf4daSMagnus Karlsson max_frags = test->ifobj_tx->xdp_zc_max_segs;
1993807bf4daSMagnus Karlsson else
1994807bf4daSMagnus Karlsson max_frags = XSK_DESC__MAX_SKB_FRAGS;
1995807bf4daSMagnus Karlsson
1996807bf4daSMagnus Karlsson test->mtu = MAX_ETH_JUMBO_SIZE;
1997807bf4daSMagnus Karlsson
1998807bf4daSMagnus Karlsson /* Valid packet for synch */
1999807bf4daSMagnus Karlsson pkts[0].len = MIN_PKT_SIZE;
2000807bf4daSMagnus Karlsson pkts[0].valid = true;
2001807bf4daSMagnus Karlsson
2002807bf4daSMagnus Karlsson /* One valid packet with the max amount of frags */
2003807bf4daSMagnus Karlsson for (i = 1; i < max_frags + 1; i++) {
2004807bf4daSMagnus Karlsson pkts[i].len = MIN_PKT_SIZE;
2005807bf4daSMagnus Karlsson pkts[i].options = XDP_PKT_CONTD;
2006807bf4daSMagnus Karlsson pkts[i].valid = true;
2007807bf4daSMagnus Karlsson }
2008807bf4daSMagnus Karlsson pkts[max_frags].options = 0;
2009807bf4daSMagnus Karlsson
2010807bf4daSMagnus Karlsson /* An invalid packet with the max amount of frags but signals packet
2011807bf4daSMagnus Karlsson * continues on the last frag
2012807bf4daSMagnus Karlsson */
2013807bf4daSMagnus Karlsson for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {
2014807bf4daSMagnus Karlsson pkts[i].len = MIN_PKT_SIZE;
2015807bf4daSMagnus Karlsson pkts[i].options = XDP_PKT_CONTD;
2016807bf4daSMagnus Karlsson pkts[i].valid = false;
2017807bf4daSMagnus Karlsson }
2018807bf4daSMagnus Karlsson
2019807bf4daSMagnus Karlsson /* Valid packet for synch */
2020807bf4daSMagnus Karlsson pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;
2021807bf4daSMagnus Karlsson pkts[2 * max_frags + 1].valid = true;
2022807bf4daSMagnus Karlsson
2023807bf4daSMagnus Karlsson pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2);
2024807bf4daSMagnus Karlsson return testapp_validate_traffic(test);
2025807bf4daSMagnus Karlsson }
2026807bf4daSMagnus Karlsson
xsk_load_xdp_programs(struct ifobject * ifobj)2027f0a249dfSMagnus Karlsson static int xsk_load_xdp_programs(struct ifobject *ifobj)
2028f0a249dfSMagnus Karlsson {
2029f0a249dfSMagnus Karlsson ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
2030f0a249dfSMagnus Karlsson if (libbpf_get_error(ifobj->xdp_progs))
2031f0a249dfSMagnus Karlsson return libbpf_get_error(ifobj->xdp_progs);
2032f0a249dfSMagnus Karlsson
2033f0a249dfSMagnus Karlsson return 0;
2034f0a249dfSMagnus Karlsson }
2035f0a249dfSMagnus Karlsson
xsk_unload_xdp_programs(struct ifobject * ifobj)2036f0a249dfSMagnus Karlsson static void xsk_unload_xdp_programs(struct ifobject *ifobj)
2037f0a249dfSMagnus Karlsson {
2038f0a249dfSMagnus Karlsson xsk_xdp_progs__destroy(ifobj->xdp_progs);
2039f0a249dfSMagnus Karlsson }
2040f0a249dfSMagnus Karlsson
2041041b68f6SMagnus Karlsson /* Simple test */
hugepages_present(void)2042041b68f6SMagnus Karlsson static bool hugepages_present(void)
2043041b68f6SMagnus Karlsson {
2044041b68f6SMagnus Karlsson size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
2045041b68f6SMagnus Karlsson void *bufs;
2046041b68f6SMagnus Karlsson
2047041b68f6SMagnus Karlsson bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2048041b68f6SMagnus Karlsson MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);
2049041b68f6SMagnus Karlsson if (bufs == MAP_FAILED)
2050041b68f6SMagnus Karlsson return false;
2051041b68f6SMagnus Karlsson
2052041b68f6SMagnus Karlsson mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
2053041b68f6SMagnus Karlsson munmap(bufs, mmap_sz);
2054041b68f6SMagnus Karlsson return true;
2055041b68f6SMagnus Karlsson }
2056041b68f6SMagnus Karlsson
init_iface(struct ifobject * ifobj,const char * dst_mac,const char * src_mac,thread_func_t func_ptr)2057018a8e75SMaciej Fijalkowski static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
2058df82d2e8SMagnus Karlsson thread_func_t func_ptr)
2059018a8e75SMaciej Fijalkowski {
2060f540d44eSMagnus Karlsson LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
2061f0a249dfSMagnus Karlsson int err;
2062018a8e75SMaciej Fijalkowski
2063018a8e75SMaciej Fijalkowski memcpy(ifobj->dst_mac, dst_mac, ETH_ALEN);
2064018a8e75SMaciej Fijalkowski memcpy(ifobj->src_mac, src_mac, ETH_ALEN);
2065018a8e75SMaciej Fijalkowski
2066018a8e75SMaciej Fijalkowski ifobj->func_ptr = func_ptr;
2067aa61d81fSMagnus Karlsson
2068f0a249dfSMagnus Karlsson err = xsk_load_xdp_programs(ifobj);
2069aa61d81fSMagnus Karlsson if (err) {
2070aa61d81fSMagnus Karlsson printf("Error loading XDP program\n");
2071aa61d81fSMagnus Karlsson exit_with_error(err);
2072aa61d81fSMagnus Karlsson }
2073041b68f6SMagnus Karlsson
2074041b68f6SMagnus Karlsson if (hugepages_present())
2075041b68f6SMagnus Karlsson ifobj->unaligned_supp = true;
2076f540d44eSMagnus Karlsson
2077f540d44eSMagnus Karlsson err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);
2078f540d44eSMagnus Karlsson if (err) {
2079*13fd5e14SColin Ian King ksft_print_msg("Error querying XDP capabilities\n");
2080f540d44eSMagnus Karlsson exit_with_error(-err);
2081f540d44eSMagnus Karlsson }
2082f540d44eSMagnus Karlsson if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)
2083f540d44eSMagnus Karlsson ifobj->multi_buff_supp = true;
2084807bf4daSMagnus Karlsson if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
2085807bf4daSMagnus Karlsson if (query_opts.xdp_zc_max_segs > 1) {
2086f540d44eSMagnus Karlsson ifobj->multi_buff_zc_supp = true;
2087807bf4daSMagnus Karlsson ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;
2088807bf4daSMagnus Karlsson } else {
2089807bf4daSMagnus Karlsson ifobj->xdp_zc_max_segs = 0;
2090807bf4daSMagnus Karlsson }
2091807bf4daSMagnus Karlsson }
2092018a8e75SMaciej Fijalkowski }
2093018a8e75SMaciej Fijalkowski
run_pkt_test(struct test_spec * test,enum test_mode mode,enum test_type type)2094018a8e75SMaciej Fijalkowski static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
2095018a8e75SMaciej Fijalkowski {
2096041b68f6SMagnus Karlsson int ret = TEST_SKIP;
2097041b68f6SMagnus Karlsson
2098018a8e75SMaciej Fijalkowski switch (type) {
2099018a8e75SMaciej Fijalkowski case TEST_TYPE_STATS_RX_DROPPED:
2100041b68f6SMagnus Karlsson ret = testapp_stats_rx_dropped(test);
2101018a8e75SMaciej Fijalkowski break;
2102018a8e75SMaciej Fijalkowski case TEST_TYPE_STATS_TX_INVALID_DESCS:
2103041b68f6SMagnus Karlsson ret = testapp_stats_tx_invalid_descs(test);
2104018a8e75SMaciej Fijalkowski break;
2105018a8e75SMaciej Fijalkowski case TEST_TYPE_STATS_RX_FULL:
2106041b68f6SMagnus Karlsson ret = testapp_stats_rx_full(test);
2107018a8e75SMaciej Fijalkowski break;
2108018a8e75SMaciej Fijalkowski case TEST_TYPE_STATS_FILL_EMPTY:
2109041b68f6SMagnus Karlsson ret = testapp_stats_fill_empty(test);
2110018a8e75SMaciej Fijalkowski break;
2111018a8e75SMaciej Fijalkowski case TEST_TYPE_TEARDOWN:
2112041b68f6SMagnus Karlsson ret = testapp_teardown(test);
2113018a8e75SMaciej Fijalkowski break;
2114018a8e75SMaciej Fijalkowski case TEST_TYPE_BIDI:
2115041b68f6SMagnus Karlsson ret = testapp_bidi(test);
2116018a8e75SMaciej Fijalkowski break;
2117018a8e75SMaciej Fijalkowski case TEST_TYPE_BPF_RES:
2118041b68f6SMagnus Karlsson ret = testapp_bpf_res(test);
2119018a8e75SMaciej Fijalkowski break;
2120018a8e75SMaciej Fijalkowski case TEST_TYPE_RUN_TO_COMPLETION:
2121018a8e75SMaciej Fijalkowski test_spec_set_name(test, "RUN_TO_COMPLETION");
2122041b68f6SMagnus Karlsson ret = testapp_validate_traffic(test);
2123018a8e75SMaciej Fijalkowski break;
2124f540d44eSMagnus Karlsson case TEST_TYPE_RUN_TO_COMPLETION_MB:
2125f540d44eSMagnus Karlsson ret = testapp_multi_buffer(test);
2126f540d44eSMagnus Karlsson break;
2127018a8e75SMaciej Fijalkowski case TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT:
2128018a8e75SMaciej Fijalkowski test_spec_set_name(test, "RUN_TO_COMPLETION_SINGLE_PKT");
2129041b68f6SMagnus Karlsson ret = testapp_single_pkt(test);
2130018a8e75SMaciej Fijalkowski break;
2131018a8e75SMaciej Fijalkowski case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
2132018a8e75SMaciej Fijalkowski test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
2133018a8e75SMaciej Fijalkowski test->ifobj_tx->umem->frame_size = 2048;
2134018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->frame_size = 2048;
2135df82d2e8SMagnus Karlsson pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2136041b68f6SMagnus Karlsson ret = testapp_validate_traffic(test);
2137018a8e75SMaciej Fijalkowski break;
21383143d10bSShibin Koikkara Reeny case TEST_TYPE_RX_POLL:
2139018a8e75SMaciej Fijalkowski test->ifobj_rx->use_poll = true;
21403143d10bSShibin Koikkara Reeny test_spec_set_name(test, "POLL_RX");
2141041b68f6SMagnus Karlsson ret = testapp_validate_traffic(test);
2142018a8e75SMaciej Fijalkowski break;
21433143d10bSShibin Koikkara Reeny case TEST_TYPE_TX_POLL:
21443143d10bSShibin Koikkara Reeny test->ifobj_tx->use_poll = true;
21453143d10bSShibin Koikkara Reeny test_spec_set_name(test, "POLL_TX");
2146041b68f6SMagnus Karlsson ret = testapp_validate_traffic(test);
21473143d10bSShibin Koikkara Reeny break;
21483143d10bSShibin Koikkara Reeny case TEST_TYPE_POLL_TXQ_TMOUT:
2149041b68f6SMagnus Karlsson ret = testapp_poll_txq_tmout(test);
21503143d10bSShibin Koikkara Reeny break;
21513143d10bSShibin Koikkara Reeny case TEST_TYPE_POLL_RXQ_TMOUT:
2152041b68f6SMagnus Karlsson ret = testapp_poll_rxq_tmout(test);
21533143d10bSShibin Koikkara Reeny break;
2154018a8e75SMaciej Fijalkowski case TEST_TYPE_ALIGNED_INV_DESC:
2155018a8e75SMaciej Fijalkowski test_spec_set_name(test, "ALIGNED_INV_DESC");
2156041b68f6SMagnus Karlsson ret = testapp_invalid_desc(test);
2157018a8e75SMaciej Fijalkowski break;
2158018a8e75SMaciej Fijalkowski case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
2159018a8e75SMaciej Fijalkowski test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
2160018a8e75SMaciej Fijalkowski test->ifobj_tx->umem->frame_size = 2048;
2161018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->frame_size = 2048;
2162041b68f6SMagnus Karlsson ret = testapp_invalid_desc(test);
2163018a8e75SMaciej Fijalkowski break;
2164018a8e75SMaciej Fijalkowski case TEST_TYPE_UNALIGNED_INV_DESC:
2165018a8e75SMaciej Fijalkowski test_spec_set_name(test, "UNALIGNED_INV_DESC");
2166018a8e75SMaciej Fijalkowski test->ifobj_tx->umem->unaligned_mode = true;
2167018a8e75SMaciej Fijalkowski test->ifobj_rx->umem->unaligned_mode = true;
2168041b68f6SMagnus Karlsson ret = testapp_invalid_desc(test);
2169018a8e75SMaciej Fijalkowski break;
2170c0801598SKal Conley case TEST_TYPE_UNALIGNED_INV_DESC_4K1_FRAME: {
2171c0801598SKal Conley u64 page_size, umem_size;
2172c0801598SKal Conley
2173c0801598SKal Conley test_spec_set_name(test, "UNALIGNED_INV_DESC_4K1_FRAME_SIZE");
2174c0801598SKal Conley /* Odd frame size so the UMEM doesn't end near a page boundary. */
2175c0801598SKal Conley test->ifobj_tx->umem->frame_size = 4001;
2176c0801598SKal Conley test->ifobj_rx->umem->frame_size = 4001;
2177c0801598SKal Conley test->ifobj_tx->umem->unaligned_mode = true;
2178c0801598SKal Conley test->ifobj_rx->umem->unaligned_mode = true;
2179c0801598SKal Conley /* This test exists to test descriptors that staddle the end of
2180c0801598SKal Conley * the UMEM but not a page.
2181c0801598SKal Conley */
2182c0801598SKal Conley page_size = sysconf(_SC_PAGESIZE);
2183c0801598SKal Conley umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
2184df82d2e8SMagnus Karlsson assert(umem_size % page_size > MIN_PKT_SIZE);
2185df82d2e8SMagnus Karlsson assert(umem_size % page_size < page_size - MIN_PKT_SIZE);
2186041b68f6SMagnus Karlsson ret = testapp_invalid_desc(test);
2187c0801598SKal Conley break;
2188c0801598SKal Conley }
218969760449SMagnus Karlsson case TEST_TYPE_ALIGNED_INV_DESC_MB:
219069760449SMagnus Karlsson test_spec_set_name(test, "ALIGNED_INV_DESC_MULTI_BUFF");
219169760449SMagnus Karlsson ret = testapp_invalid_desc_mb(test);
219269760449SMagnus Karlsson break;
219369760449SMagnus Karlsson case TEST_TYPE_UNALIGNED_INV_DESC_MB:
219469760449SMagnus Karlsson test_spec_set_name(test, "UNALIGNED_INV_DESC_MULTI_BUFF");
219569760449SMagnus Karlsson test->ifobj_tx->umem->unaligned_mode = true;
219669760449SMagnus Karlsson test->ifobj_rx->umem->unaligned_mode = true;
219769760449SMagnus Karlsson ret = testapp_invalid_desc_mb(test);
219869760449SMagnus Karlsson break;
2199018a8e75SMaciej Fijalkowski case TEST_TYPE_UNALIGNED:
2200041b68f6SMagnus Karlsson ret = testapp_unaligned(test);
2201018a8e75SMaciej Fijalkowski break;
22021005a226SMagnus Karlsson case TEST_TYPE_UNALIGNED_MB:
22031005a226SMagnus Karlsson ret = testapp_unaligned_mb(test);
22041005a226SMagnus Karlsson break;
2205018a8e75SMaciej Fijalkowski case TEST_TYPE_HEADROOM:
2206041b68f6SMagnus Karlsson ret = testapp_headroom(test);
2207018a8e75SMaciej Fijalkowski break;
220880bea9acSMagnus Karlsson case TEST_TYPE_XDP_DROP_HALF:
2209041b68f6SMagnus Karlsson ret = testapp_xdp_drop(test);
221080bea9acSMagnus Karlsson break;
22119a321fd3STushar Vyavahare case TEST_TYPE_XDP_METADATA_COUNT:
2212f80ddbecSMagnus Karlsson test_spec_set_name(test, "XDP_METADATA_COUNT");
2213f80ddbecSMagnus Karlsson ret = testapp_xdp_metadata_count(test);
2214f80ddbecSMagnus Karlsson break;
2215f80ddbecSMagnus Karlsson case TEST_TYPE_XDP_METADATA_COUNT_MB:
2216f80ddbecSMagnus Karlsson test_spec_set_name(test, "XDP_METADATA_COUNT_MULTI_BUFF");
2217f80ddbecSMagnus Karlsson test->mtu = MAX_ETH_JUMBO_SIZE;
2218041b68f6SMagnus Karlsson ret = testapp_xdp_metadata_count(test);
22199a321fd3STushar Vyavahare break;
2220807bf4daSMagnus Karlsson case TEST_TYPE_TOO_MANY_FRAGS:
2221807bf4daSMagnus Karlsson ret = testapp_too_many_frags(test);
2222807bf4daSMagnus Karlsson break;
2223018a8e75SMaciej Fijalkowski default:
2224018a8e75SMaciej Fijalkowski break;
2225018a8e75SMaciej Fijalkowski }
2226018a8e75SMaciej Fijalkowski
2227041b68f6SMagnus Karlsson if (ret == TEST_PASS)
2228018a8e75SMaciej Fijalkowski ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
2229018a8e75SMaciej Fijalkowski test->name);
2230e67b2554SMagnus Karlsson pkt_stream_restore_default(test);
2231018a8e75SMaciej Fijalkowski }
2232018a8e75SMaciej Fijalkowski
ifobject_create(void)2233018a8e75SMaciej Fijalkowski static struct ifobject *ifobject_create(void)
2234018a8e75SMaciej Fijalkowski {
2235018a8e75SMaciej Fijalkowski struct ifobject *ifobj;
2236018a8e75SMaciej Fijalkowski
2237018a8e75SMaciej Fijalkowski ifobj = calloc(1, sizeof(struct ifobject));
2238018a8e75SMaciej Fijalkowski if (!ifobj)
2239018a8e75SMaciej Fijalkowski return NULL;
2240018a8e75SMaciej Fijalkowski
2241018a8e75SMaciej Fijalkowski ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
2242018a8e75SMaciej Fijalkowski if (!ifobj->xsk_arr)
2243018a8e75SMaciej Fijalkowski goto out_xsk_arr;
2244018a8e75SMaciej Fijalkowski
2245018a8e75SMaciej Fijalkowski ifobj->umem = calloc(1, sizeof(*ifobj->umem));
2246018a8e75SMaciej Fijalkowski if (!ifobj->umem)
2247018a8e75SMaciej Fijalkowski goto out_umem;
2248018a8e75SMaciej Fijalkowski
2249018a8e75SMaciej Fijalkowski return ifobj;
2250018a8e75SMaciej Fijalkowski
2251018a8e75SMaciej Fijalkowski out_umem:
2252018a8e75SMaciej Fijalkowski free(ifobj->xsk_arr);
2253018a8e75SMaciej Fijalkowski out_xsk_arr:
2254018a8e75SMaciej Fijalkowski free(ifobj);
2255018a8e75SMaciej Fijalkowski return NULL;
2256018a8e75SMaciej Fijalkowski }
2257018a8e75SMaciej Fijalkowski
ifobject_delete(struct ifobject * ifobj)2258018a8e75SMaciej Fijalkowski static void ifobject_delete(struct ifobject *ifobj)
2259018a8e75SMaciej Fijalkowski {
2260018a8e75SMaciej Fijalkowski free(ifobj->umem);
2261018a8e75SMaciej Fijalkowski free(ifobj->xsk_arr);
2262018a8e75SMaciej Fijalkowski free(ifobj);
2263018a8e75SMaciej Fijalkowski }
2264018a8e75SMaciej Fijalkowski
is_xdp_supported(int ifindex)2265aa61d81fSMagnus Karlsson static bool is_xdp_supported(int ifindex)
22660d68e6feSMaciej Fijalkowski {
22670d68e6feSMaciej Fijalkowski int flags = XDP_FLAGS_DRV_MODE;
22680d68e6feSMaciej Fijalkowski
22690d68e6feSMaciej Fijalkowski LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
22700d68e6feSMaciej Fijalkowski struct bpf_insn insns[2] = {
22710d68e6feSMaciej Fijalkowski BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
22720d68e6feSMaciej Fijalkowski BPF_EXIT_INSN()
22730d68e6feSMaciej Fijalkowski };
22740d68e6feSMaciej Fijalkowski int prog_fd, insn_cnt = ARRAY_SIZE(insns);
22750d68e6feSMaciej Fijalkowski int err;
22760d68e6feSMaciej Fijalkowski
22770d68e6feSMaciej Fijalkowski prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
22780d68e6feSMaciej Fijalkowski if (prog_fd < 0)
22790d68e6feSMaciej Fijalkowski return false;
22800d68e6feSMaciej Fijalkowski
22810d68e6feSMaciej Fijalkowski err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
22820d68e6feSMaciej Fijalkowski if (err) {
22830d68e6feSMaciej Fijalkowski close(prog_fd);
22840d68e6feSMaciej Fijalkowski return false;
22850d68e6feSMaciej Fijalkowski }
22860d68e6feSMaciej Fijalkowski
22870d68e6feSMaciej Fijalkowski bpf_xdp_detach(ifindex, flags, NULL);
22880d68e6feSMaciej Fijalkowski close(prog_fd);
22890d68e6feSMaciej Fijalkowski
22900d68e6feSMaciej Fijalkowski return true;
22910d68e6feSMaciej Fijalkowski }
22920d68e6feSMaciej Fijalkowski
main(int argc,char ** argv)2293018a8e75SMaciej Fijalkowski int main(int argc, char **argv)
2294018a8e75SMaciej Fijalkowski {
22951adef064SMaciej Fijalkowski struct pkt_stream *rx_pkt_stream_default;
22961adef064SMaciej Fijalkowski struct pkt_stream *tx_pkt_stream_default;
2297018a8e75SMaciej Fijalkowski struct ifobject *ifobj_tx, *ifobj_rx;
22980d68e6feSMaciej Fijalkowski int modes = TEST_MODE_SKB + 1;
2299018a8e75SMaciej Fijalkowski u32 i, j, failed_tests = 0;
2300018a8e75SMaciej Fijalkowski struct test_spec test;
2301aa61d81fSMagnus Karlsson bool shared_netdev;
2302018a8e75SMaciej Fijalkowski
2303018a8e75SMaciej Fijalkowski /* Use libbpf 1.0 API mode */
2304018a8e75SMaciej Fijalkowski libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
2305018a8e75SMaciej Fijalkowski
2306018a8e75SMaciej Fijalkowski ifobj_tx = ifobject_create();
2307018a8e75SMaciej Fijalkowski if (!ifobj_tx)
2308018a8e75SMaciej Fijalkowski exit_with_error(ENOMEM);
2309018a8e75SMaciej Fijalkowski ifobj_rx = ifobject_create();
2310018a8e75SMaciej Fijalkowski if (!ifobj_rx)
2311018a8e75SMaciej Fijalkowski exit_with_error(ENOMEM);
2312018a8e75SMaciej Fijalkowski
2313018a8e75SMaciej Fijalkowski setlocale(LC_ALL, "");
2314018a8e75SMaciej Fijalkowski
2315018a8e75SMaciej Fijalkowski parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
2316a693ff3eSMaciej Fijalkowski
2317aa61d81fSMagnus Karlsson shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
2318aa61d81fSMagnus Karlsson ifobj_tx->shared_umem = shared_netdev;
2319aa61d81fSMagnus Karlsson ifobj_rx->shared_umem = shared_netdev;
2320018a8e75SMaciej Fijalkowski
2321018a8e75SMaciej Fijalkowski if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
2322018a8e75SMaciej Fijalkowski usage(basename(argv[0]));
2323018a8e75SMaciej Fijalkowski ksft_exit_xfail();
2324018a8e75SMaciej Fijalkowski }
2325018a8e75SMaciej Fijalkowski
2326aa61d81fSMagnus Karlsson if (is_xdp_supported(ifobj_tx->ifindex)) {
23270d68e6feSMaciej Fijalkowski modes++;
2328fe2ad08eSMaciej Fijalkowski if (ifobj_zc_avail(ifobj_tx))
2329fe2ad08eSMaciej Fijalkowski modes++;
2330fe2ad08eSMaciej Fijalkowski }
23310d68e6feSMaciej Fijalkowski
2332df82d2e8SMagnus Karlsson init_iface(ifobj_rx, MAC1, MAC2, worker_testapp_validate_rx);
2333df82d2e8SMagnus Karlsson init_iface(ifobj_tx, MAC2, MAC1, worker_testapp_validate_tx);
2334aa61d81fSMagnus Karlsson
2335018a8e75SMaciej Fijalkowski test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
2336df82d2e8SMagnus Karlsson tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2337df82d2e8SMagnus Karlsson rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
23381adef064SMaciej Fijalkowski if (!tx_pkt_stream_default || !rx_pkt_stream_default)
2339018a8e75SMaciej Fijalkowski exit_with_error(ENOMEM);
23401adef064SMaciej Fijalkowski test.tx_pkt_stream_default = tx_pkt_stream_default;
23411adef064SMaciej Fijalkowski test.rx_pkt_stream_default = rx_pkt_stream_default;
2342018a8e75SMaciej Fijalkowski
23430d68e6feSMaciej Fijalkowski ksft_set_plan(modes * TEST_TYPE_MAX);
2344018a8e75SMaciej Fijalkowski
2345aa61d81fSMagnus Karlsson for (i = 0; i < modes; i++) {
2346018a8e75SMaciej Fijalkowski for (j = 0; j < TEST_TYPE_MAX; j++) {
2347018a8e75SMaciej Fijalkowski test_spec_init(&test, ifobj_tx, ifobj_rx, i);
2348018a8e75SMaciej Fijalkowski run_pkt_test(&test, i, j);
2349018a8e75SMaciej Fijalkowski usleep(USLEEP_MAX);
2350018a8e75SMaciej Fijalkowski
2351018a8e75SMaciej Fijalkowski if (test.fail)
2352018a8e75SMaciej Fijalkowski failed_tests++;
2353018a8e75SMaciej Fijalkowski }
2354aa61d81fSMagnus Karlsson }
2355018a8e75SMaciej Fijalkowski
23561adef064SMaciej Fijalkowski pkt_stream_delete(tx_pkt_stream_default);
23571adef064SMaciej Fijalkowski pkt_stream_delete(rx_pkt_stream_default);
2358f0a249dfSMagnus Karlsson xsk_unload_xdp_programs(ifobj_tx);
2359f0a249dfSMagnus Karlsson xsk_unload_xdp_programs(ifobj_rx);
2360018a8e75SMaciej Fijalkowski ifobject_delete(ifobj_tx);
2361018a8e75SMaciej Fijalkowski ifobject_delete(ifobj_rx);
2362018a8e75SMaciej Fijalkowski
2363018a8e75SMaciej Fijalkowski if (failed_tests)
2364018a8e75SMaciej Fijalkowski ksft_exit_fail();
2365018a8e75SMaciej Fijalkowski else
2366018a8e75SMaciej Fijalkowski ksft_exit_pass();
2367018a8e75SMaciej Fijalkowski }
2368