1f3660063SAndrii Nakryiko // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2f3660063SAndrii Nakryiko
3f3660063SAndrii Nakryiko /*
4f3660063SAndrii Nakryiko * AF_XDP user-space access library.
5f3660063SAndrii Nakryiko *
6f3660063SAndrii Nakryiko * Copyright(c) 2018 - 2019 Intel Corporation.
7f3660063SAndrii Nakryiko *
8f3660063SAndrii Nakryiko * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
9f3660063SAndrii Nakryiko */
10f3660063SAndrii Nakryiko
11f3660063SAndrii Nakryiko #include <errno.h>
12f3660063SAndrii Nakryiko #include <stdlib.h>
13f3660063SAndrii Nakryiko #include <string.h>
14f3660063SAndrii Nakryiko #include <unistd.h>
15f3660063SAndrii Nakryiko #include <arpa/inet.h>
16f3660063SAndrii Nakryiko #include <asm/barrier.h>
17f3660063SAndrii Nakryiko #include <linux/compiler.h>
18f3660063SAndrii Nakryiko #include <linux/ethtool.h>
19f3660063SAndrii Nakryiko #include <linux/filter.h>
20f3660063SAndrii Nakryiko #include <linux/if_ether.h>
21*f540d44eSMagnus Karlsson #include <linux/if_link.h>
22f3660063SAndrii Nakryiko #include <linux/if_packet.h>
23f3660063SAndrii Nakryiko #include <linux/if_xdp.h>
24f3660063SAndrii Nakryiko #include <linux/kernel.h>
25f3660063SAndrii Nakryiko #include <linux/list.h>
26*f540d44eSMagnus Karlsson #include <linux/netlink.h>
27*f540d44eSMagnus Karlsson #include <linux/rtnetlink.h>
28f3660063SAndrii Nakryiko #include <linux/sockios.h>
29f3660063SAndrii Nakryiko #include <net/if.h>
30f3660063SAndrii Nakryiko #include <sys/ioctl.h>
31f3660063SAndrii Nakryiko #include <sys/mman.h>
32f3660063SAndrii Nakryiko #include <sys/socket.h>
33f3660063SAndrii Nakryiko #include <sys/types.h>
34f3660063SAndrii Nakryiko
35f3660063SAndrii Nakryiko #include <bpf/bpf.h>
36f3660063SAndrii Nakryiko #include <bpf/libbpf.h>
37f3660063SAndrii Nakryiko #include "xsk.h"
38b3c09fdcSRong Tao #include "bpf_util.h"
39f3660063SAndrii Nakryiko
40f3660063SAndrii Nakryiko #ifndef SOL_XDP
41f3660063SAndrii Nakryiko #define SOL_XDP 283
42f3660063SAndrii Nakryiko #endif
43f3660063SAndrii Nakryiko
44f3660063SAndrii Nakryiko #ifndef AF_XDP
45f3660063SAndrii Nakryiko #define AF_XDP 44
46f3660063SAndrii Nakryiko #endif
47f3660063SAndrii Nakryiko
48f3660063SAndrii Nakryiko #ifndef PF_XDP
49f3660063SAndrii Nakryiko #define PF_XDP AF_XDP
50f3660063SAndrii Nakryiko #endif
51f3660063SAndrii Nakryiko
52f3660063SAndrii Nakryiko #define pr_warn(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__)
53f3660063SAndrii Nakryiko
54aa61d81fSMagnus Karlsson #define XSKMAP_SIZE 1
55aa61d81fSMagnus Karlsson
56f3660063SAndrii Nakryiko struct xsk_umem {
57f3660063SAndrii Nakryiko struct xsk_ring_prod *fill_save;
58f3660063SAndrii Nakryiko struct xsk_ring_cons *comp_save;
59f3660063SAndrii Nakryiko char *umem_area;
60f3660063SAndrii Nakryiko struct xsk_umem_config config;
61f3660063SAndrii Nakryiko int fd;
62f3660063SAndrii Nakryiko int refcount;
63f3660063SAndrii Nakryiko struct list_head ctx_list;
64f3660063SAndrii Nakryiko bool rx_ring_setup_done;
65f3660063SAndrii Nakryiko bool tx_ring_setup_done;
66f3660063SAndrii Nakryiko };
67f3660063SAndrii Nakryiko
68f3660063SAndrii Nakryiko struct xsk_ctx {
69f3660063SAndrii Nakryiko struct xsk_ring_prod *fill;
70f3660063SAndrii Nakryiko struct xsk_ring_cons *comp;
71f3660063SAndrii Nakryiko __u32 queue_id;
72f3660063SAndrii Nakryiko struct xsk_umem *umem;
73f3660063SAndrii Nakryiko int refcount;
74f3660063SAndrii Nakryiko int ifindex;
75f3660063SAndrii Nakryiko struct list_head list;
76f3660063SAndrii Nakryiko };
77f3660063SAndrii Nakryiko
78f3660063SAndrii Nakryiko struct xsk_socket {
79f3660063SAndrii Nakryiko struct xsk_ring_cons *rx;
80f3660063SAndrii Nakryiko struct xsk_ring_prod *tx;
81f3660063SAndrii Nakryiko struct xsk_ctx *ctx;
82f3660063SAndrii Nakryiko struct xsk_socket_config config;
83f3660063SAndrii Nakryiko int fd;
84f3660063SAndrii Nakryiko };
85f3660063SAndrii Nakryiko
86*f540d44eSMagnus Karlsson struct nl_mtu_req {
87*f540d44eSMagnus Karlsson struct nlmsghdr nh;
88*f540d44eSMagnus Karlsson struct ifinfomsg msg;
89*f540d44eSMagnus Karlsson char buf[512];
90*f540d44eSMagnus Karlsson };
91*f540d44eSMagnus Karlsson
xsk_umem__fd(const struct xsk_umem * umem)92f3660063SAndrii Nakryiko int xsk_umem__fd(const struct xsk_umem *umem)
93f3660063SAndrii Nakryiko {
94f3660063SAndrii Nakryiko return umem ? umem->fd : -EINVAL;
95f3660063SAndrii Nakryiko }
96f3660063SAndrii Nakryiko
xsk_socket__fd(const struct xsk_socket * xsk)97f3660063SAndrii Nakryiko int xsk_socket__fd(const struct xsk_socket *xsk)
98f3660063SAndrii Nakryiko {
99f3660063SAndrii Nakryiko return xsk ? xsk->fd : -EINVAL;
100f3660063SAndrii Nakryiko }
101f3660063SAndrii Nakryiko
xsk_page_aligned(void * buffer)102f3660063SAndrii Nakryiko static bool xsk_page_aligned(void *buffer)
103f3660063SAndrii Nakryiko {
104f3660063SAndrii Nakryiko unsigned long addr = (unsigned long)buffer;
105f3660063SAndrii Nakryiko
106f3660063SAndrii Nakryiko return !(addr & (getpagesize() - 1));
107f3660063SAndrii Nakryiko }
108f3660063SAndrii Nakryiko
xsk_set_umem_config(struct xsk_umem_config * cfg,const struct xsk_umem_config * usr_cfg)109f3660063SAndrii Nakryiko static void xsk_set_umem_config(struct xsk_umem_config *cfg,
110f3660063SAndrii Nakryiko const struct xsk_umem_config *usr_cfg)
111f3660063SAndrii Nakryiko {
112f3660063SAndrii Nakryiko if (!usr_cfg) {
113f3660063SAndrii Nakryiko cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
114f3660063SAndrii Nakryiko cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
115f3660063SAndrii Nakryiko cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
116f3660063SAndrii Nakryiko cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
117f3660063SAndrii Nakryiko cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
118f3660063SAndrii Nakryiko return;
119f3660063SAndrii Nakryiko }
120f3660063SAndrii Nakryiko
121f3660063SAndrii Nakryiko cfg->fill_size = usr_cfg->fill_size;
122f3660063SAndrii Nakryiko cfg->comp_size = usr_cfg->comp_size;
123f3660063SAndrii Nakryiko cfg->frame_size = usr_cfg->frame_size;
124f3660063SAndrii Nakryiko cfg->frame_headroom = usr_cfg->frame_headroom;
125f3660063SAndrii Nakryiko cfg->flags = usr_cfg->flags;
126f3660063SAndrii Nakryiko }
127f3660063SAndrii Nakryiko
xsk_set_xdp_socket_config(struct xsk_socket_config * cfg,const struct xsk_socket_config * usr_cfg)128f3660063SAndrii Nakryiko static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
129f3660063SAndrii Nakryiko const struct xsk_socket_config *usr_cfg)
130f3660063SAndrii Nakryiko {
131f3660063SAndrii Nakryiko if (!usr_cfg) {
132f3660063SAndrii Nakryiko cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
133f3660063SAndrii Nakryiko cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
134f3660063SAndrii Nakryiko cfg->bind_flags = 0;
135f3660063SAndrii Nakryiko return 0;
136f3660063SAndrii Nakryiko }
137f3660063SAndrii Nakryiko
138f3660063SAndrii Nakryiko cfg->rx_size = usr_cfg->rx_size;
139f3660063SAndrii Nakryiko cfg->tx_size = usr_cfg->tx_size;
140f3660063SAndrii Nakryiko cfg->bind_flags = usr_cfg->bind_flags;
141f3660063SAndrii Nakryiko
142f3660063SAndrii Nakryiko return 0;
143f3660063SAndrii Nakryiko }
144f3660063SAndrii Nakryiko
xsk_get_mmap_offsets(int fd,struct xdp_mmap_offsets * off)145f3660063SAndrii Nakryiko static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
146f3660063SAndrii Nakryiko {
147f3660063SAndrii Nakryiko socklen_t optlen;
148f3660063SAndrii Nakryiko int err;
149f3660063SAndrii Nakryiko
150f3660063SAndrii Nakryiko optlen = sizeof(*off);
151f3660063SAndrii Nakryiko err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
152f3660063SAndrii Nakryiko if (err)
153f3660063SAndrii Nakryiko return err;
154f3660063SAndrii Nakryiko
155f3660063SAndrii Nakryiko if (optlen == sizeof(*off))
156f3660063SAndrii Nakryiko return 0;
157f3660063SAndrii Nakryiko
158f3660063SAndrii Nakryiko return -EINVAL;
159f3660063SAndrii Nakryiko }
160f3660063SAndrii Nakryiko
xsk_create_umem_rings(struct xsk_umem * umem,int fd,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)161f3660063SAndrii Nakryiko static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
162f3660063SAndrii Nakryiko struct xsk_ring_prod *fill,
163f3660063SAndrii Nakryiko struct xsk_ring_cons *comp)
164f3660063SAndrii Nakryiko {
165f3660063SAndrii Nakryiko struct xdp_mmap_offsets off;
166f3660063SAndrii Nakryiko void *map;
167f3660063SAndrii Nakryiko int err;
168f3660063SAndrii Nakryiko
169f3660063SAndrii Nakryiko err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
170f3660063SAndrii Nakryiko &umem->config.fill_size,
171f3660063SAndrii Nakryiko sizeof(umem->config.fill_size));
172f3660063SAndrii Nakryiko if (err)
173f3660063SAndrii Nakryiko return -errno;
174f3660063SAndrii Nakryiko
175f3660063SAndrii Nakryiko err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
176f3660063SAndrii Nakryiko &umem->config.comp_size,
177f3660063SAndrii Nakryiko sizeof(umem->config.comp_size));
178f3660063SAndrii Nakryiko if (err)
179f3660063SAndrii Nakryiko return -errno;
180f3660063SAndrii Nakryiko
181f3660063SAndrii Nakryiko err = xsk_get_mmap_offsets(fd, &off);
182f3660063SAndrii Nakryiko if (err)
183f3660063SAndrii Nakryiko return -errno;
184f3660063SAndrii Nakryiko
185f3660063SAndrii Nakryiko map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
186f3660063SAndrii Nakryiko PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
187f3660063SAndrii Nakryiko XDP_UMEM_PGOFF_FILL_RING);
188f3660063SAndrii Nakryiko if (map == MAP_FAILED)
189f3660063SAndrii Nakryiko return -errno;
190f3660063SAndrii Nakryiko
191f3660063SAndrii Nakryiko fill->mask = umem->config.fill_size - 1;
192f3660063SAndrii Nakryiko fill->size = umem->config.fill_size;
193f3660063SAndrii Nakryiko fill->producer = map + off.fr.producer;
194f3660063SAndrii Nakryiko fill->consumer = map + off.fr.consumer;
195f3660063SAndrii Nakryiko fill->flags = map + off.fr.flags;
196f3660063SAndrii Nakryiko fill->ring = map + off.fr.desc;
197f3660063SAndrii Nakryiko fill->cached_cons = umem->config.fill_size;
198f3660063SAndrii Nakryiko
199f3660063SAndrii Nakryiko map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
200f3660063SAndrii Nakryiko PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
201f3660063SAndrii Nakryiko XDP_UMEM_PGOFF_COMPLETION_RING);
202f3660063SAndrii Nakryiko if (map == MAP_FAILED) {
203f3660063SAndrii Nakryiko err = -errno;
204f3660063SAndrii Nakryiko goto out_mmap;
205f3660063SAndrii Nakryiko }
206f3660063SAndrii Nakryiko
207f3660063SAndrii Nakryiko comp->mask = umem->config.comp_size - 1;
208f3660063SAndrii Nakryiko comp->size = umem->config.comp_size;
209f3660063SAndrii Nakryiko comp->producer = map + off.cr.producer;
210f3660063SAndrii Nakryiko comp->consumer = map + off.cr.consumer;
211f3660063SAndrii Nakryiko comp->flags = map + off.cr.flags;
212f3660063SAndrii Nakryiko comp->ring = map + off.cr.desc;
213f3660063SAndrii Nakryiko
214f3660063SAndrii Nakryiko return 0;
215f3660063SAndrii Nakryiko
216f3660063SAndrii Nakryiko out_mmap:
217f3660063SAndrii Nakryiko munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
218f3660063SAndrii Nakryiko return err;
219f3660063SAndrii Nakryiko }
220f3660063SAndrii Nakryiko
xsk_umem__create(struct xsk_umem ** umem_ptr,void * umem_area,__u64 size,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_umem_config * usr_config)221f3660063SAndrii Nakryiko int xsk_umem__create(struct xsk_umem **umem_ptr, void *umem_area,
222f3660063SAndrii Nakryiko __u64 size, struct xsk_ring_prod *fill,
223f3660063SAndrii Nakryiko struct xsk_ring_cons *comp,
224f3660063SAndrii Nakryiko const struct xsk_umem_config *usr_config)
225f3660063SAndrii Nakryiko {
226f3660063SAndrii Nakryiko struct xdp_umem_reg mr;
227f3660063SAndrii Nakryiko struct xsk_umem *umem;
228f3660063SAndrii Nakryiko int err;
229f3660063SAndrii Nakryiko
230f3660063SAndrii Nakryiko if (!umem_area || !umem_ptr || !fill || !comp)
231f3660063SAndrii Nakryiko return -EFAULT;
232f3660063SAndrii Nakryiko if (!size && !xsk_page_aligned(umem_area))
233f3660063SAndrii Nakryiko return -EINVAL;
234f3660063SAndrii Nakryiko
235f3660063SAndrii Nakryiko umem = calloc(1, sizeof(*umem));
236f3660063SAndrii Nakryiko if (!umem)
237f3660063SAndrii Nakryiko return -ENOMEM;
238f3660063SAndrii Nakryiko
239f3660063SAndrii Nakryiko umem->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
240f3660063SAndrii Nakryiko if (umem->fd < 0) {
241f3660063SAndrii Nakryiko err = -errno;
242f3660063SAndrii Nakryiko goto out_umem_alloc;
243f3660063SAndrii Nakryiko }
244f3660063SAndrii Nakryiko
245f3660063SAndrii Nakryiko umem->umem_area = umem_area;
246f3660063SAndrii Nakryiko INIT_LIST_HEAD(&umem->ctx_list);
247f3660063SAndrii Nakryiko xsk_set_umem_config(&umem->config, usr_config);
248f3660063SAndrii Nakryiko
249f3660063SAndrii Nakryiko memset(&mr, 0, sizeof(mr));
250f3660063SAndrii Nakryiko mr.addr = (uintptr_t)umem_area;
251f3660063SAndrii Nakryiko mr.len = size;
252f3660063SAndrii Nakryiko mr.chunk_size = umem->config.frame_size;
253f3660063SAndrii Nakryiko mr.headroom = umem->config.frame_headroom;
254f3660063SAndrii Nakryiko mr.flags = umem->config.flags;
255f3660063SAndrii Nakryiko
256f3660063SAndrii Nakryiko err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
257f3660063SAndrii Nakryiko if (err) {
258f3660063SAndrii Nakryiko err = -errno;
259f3660063SAndrii Nakryiko goto out_socket;
260f3660063SAndrii Nakryiko }
261f3660063SAndrii Nakryiko
262f3660063SAndrii Nakryiko err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
263f3660063SAndrii Nakryiko if (err)
264f3660063SAndrii Nakryiko goto out_socket;
265f3660063SAndrii Nakryiko
266f3660063SAndrii Nakryiko umem->fill_save = fill;
267f3660063SAndrii Nakryiko umem->comp_save = comp;
268f3660063SAndrii Nakryiko *umem_ptr = umem;
269f3660063SAndrii Nakryiko return 0;
270f3660063SAndrii Nakryiko
271f3660063SAndrii Nakryiko out_socket:
272f3660063SAndrii Nakryiko close(umem->fd);
273f3660063SAndrii Nakryiko out_umem_alloc:
274f3660063SAndrii Nakryiko free(umem);
275f3660063SAndrii Nakryiko return err;
276f3660063SAndrii Nakryiko }
277f3660063SAndrii Nakryiko
xsk_is_in_mode(u32 ifindex,int mode)2787d8319a7SMagnus Karlsson bool xsk_is_in_mode(u32 ifindex, int mode)
2797d8319a7SMagnus Karlsson {
2807d8319a7SMagnus Karlsson LIBBPF_OPTS(bpf_xdp_query_opts, opts);
2817d8319a7SMagnus Karlsson int ret;
2827d8319a7SMagnus Karlsson
2837d8319a7SMagnus Karlsson ret = bpf_xdp_query(ifindex, mode, &opts);
2847d8319a7SMagnus Karlsson if (ret) {
2857d8319a7SMagnus Karlsson printf("XDP mode query returned error %s\n", strerror(errno));
2867d8319a7SMagnus Karlsson return false;
2877d8319a7SMagnus Karlsson }
2887d8319a7SMagnus Karlsson
2897d8319a7SMagnus Karlsson if (mode == XDP_FLAGS_DRV_MODE)
2907d8319a7SMagnus Karlsson return opts.attach_mode == XDP_ATTACHED_DRV;
2917d8319a7SMagnus Karlsson else if (mode == XDP_FLAGS_SKB_MODE)
2927d8319a7SMagnus Karlsson return opts.attach_mode == XDP_ATTACHED_SKB;
2937d8319a7SMagnus Karlsson
2947d8319a7SMagnus Karlsson return false;
2957d8319a7SMagnus Karlsson }
2967d8319a7SMagnus Karlsson
297*f540d44eSMagnus Karlsson /* Lifted from netlink.c in tools/lib/bpf */
netlink_recvmsg(int sock,struct msghdr * mhdr,int flags)298*f540d44eSMagnus Karlsson static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags)
299*f540d44eSMagnus Karlsson {
300*f540d44eSMagnus Karlsson int len;
301*f540d44eSMagnus Karlsson
302*f540d44eSMagnus Karlsson do {
303*f540d44eSMagnus Karlsson len = recvmsg(sock, mhdr, flags);
304*f540d44eSMagnus Karlsson } while (len < 0 && (errno == EINTR || errno == EAGAIN));
305*f540d44eSMagnus Karlsson
306*f540d44eSMagnus Karlsson if (len < 0)
307*f540d44eSMagnus Karlsson return -errno;
308*f540d44eSMagnus Karlsson return len;
309*f540d44eSMagnus Karlsson }
310*f540d44eSMagnus Karlsson
311*f540d44eSMagnus Karlsson /* Lifted from netlink.c in tools/lib/bpf */
alloc_iov(struct iovec * iov,int len)312*f540d44eSMagnus Karlsson static int alloc_iov(struct iovec *iov, int len)
313*f540d44eSMagnus Karlsson {
314*f540d44eSMagnus Karlsson void *nbuf;
315*f540d44eSMagnus Karlsson
316*f540d44eSMagnus Karlsson nbuf = realloc(iov->iov_base, len);
317*f540d44eSMagnus Karlsson if (!nbuf)
318*f540d44eSMagnus Karlsson return -ENOMEM;
319*f540d44eSMagnus Karlsson
320*f540d44eSMagnus Karlsson iov->iov_base = nbuf;
321*f540d44eSMagnus Karlsson iov->iov_len = len;
322*f540d44eSMagnus Karlsson return 0;
323*f540d44eSMagnus Karlsson }
324*f540d44eSMagnus Karlsson
325*f540d44eSMagnus Karlsson /* Original version lifted from netlink.c in tools/lib/bpf */
netlink_recv(int sock)326*f540d44eSMagnus Karlsson static int netlink_recv(int sock)
327*f540d44eSMagnus Karlsson {
328*f540d44eSMagnus Karlsson struct iovec iov = {};
329*f540d44eSMagnus Karlsson struct msghdr mhdr = {
330*f540d44eSMagnus Karlsson .msg_iov = &iov,
331*f540d44eSMagnus Karlsson .msg_iovlen = 1,
332*f540d44eSMagnus Karlsson };
333*f540d44eSMagnus Karlsson bool multipart = true;
334*f540d44eSMagnus Karlsson struct nlmsgerr *err;
335*f540d44eSMagnus Karlsson struct nlmsghdr *nh;
336*f540d44eSMagnus Karlsson int len, ret;
337*f540d44eSMagnus Karlsson
338*f540d44eSMagnus Karlsson ret = alloc_iov(&iov, 4096);
339*f540d44eSMagnus Karlsson if (ret)
340*f540d44eSMagnus Karlsson goto done;
341*f540d44eSMagnus Karlsson
342*f540d44eSMagnus Karlsson while (multipart) {
343*f540d44eSMagnus Karlsson multipart = false;
344*f540d44eSMagnus Karlsson len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC);
345*f540d44eSMagnus Karlsson if (len < 0) {
346*f540d44eSMagnus Karlsson ret = len;
347*f540d44eSMagnus Karlsson goto done;
348*f540d44eSMagnus Karlsson }
349*f540d44eSMagnus Karlsson
350*f540d44eSMagnus Karlsson if (len > iov.iov_len) {
351*f540d44eSMagnus Karlsson ret = alloc_iov(&iov, len);
352*f540d44eSMagnus Karlsson if (ret)
353*f540d44eSMagnus Karlsson goto done;
354*f540d44eSMagnus Karlsson }
355*f540d44eSMagnus Karlsson
356*f540d44eSMagnus Karlsson len = netlink_recvmsg(sock, &mhdr, 0);
357*f540d44eSMagnus Karlsson if (len < 0) {
358*f540d44eSMagnus Karlsson ret = len;
359*f540d44eSMagnus Karlsson goto done;
360*f540d44eSMagnus Karlsson }
361*f540d44eSMagnus Karlsson
362*f540d44eSMagnus Karlsson if (len == 0)
363*f540d44eSMagnus Karlsson break;
364*f540d44eSMagnus Karlsson
365*f540d44eSMagnus Karlsson for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len);
366*f540d44eSMagnus Karlsson nh = NLMSG_NEXT(nh, len)) {
367*f540d44eSMagnus Karlsson if (nh->nlmsg_flags & NLM_F_MULTI)
368*f540d44eSMagnus Karlsson multipart = true;
369*f540d44eSMagnus Karlsson switch (nh->nlmsg_type) {
370*f540d44eSMagnus Karlsson case NLMSG_ERROR:
371*f540d44eSMagnus Karlsson err = (struct nlmsgerr *)NLMSG_DATA(nh);
372*f540d44eSMagnus Karlsson if (!err->error)
373*f540d44eSMagnus Karlsson continue;
374*f540d44eSMagnus Karlsson ret = err->error;
375*f540d44eSMagnus Karlsson goto done;
376*f540d44eSMagnus Karlsson case NLMSG_DONE:
377*f540d44eSMagnus Karlsson ret = 0;
378*f540d44eSMagnus Karlsson goto done;
379*f540d44eSMagnus Karlsson default:
380*f540d44eSMagnus Karlsson break;
381*f540d44eSMagnus Karlsson }
382*f540d44eSMagnus Karlsson }
383*f540d44eSMagnus Karlsson }
384*f540d44eSMagnus Karlsson ret = 0;
385*f540d44eSMagnus Karlsson done:
386*f540d44eSMagnus Karlsson free(iov.iov_base);
387*f540d44eSMagnus Karlsson return ret;
388*f540d44eSMagnus Karlsson }
389*f540d44eSMagnus Karlsson
xsk_set_mtu(int ifindex,int mtu)390*f540d44eSMagnus Karlsson int xsk_set_mtu(int ifindex, int mtu)
391*f540d44eSMagnus Karlsson {
392*f540d44eSMagnus Karlsson struct nl_mtu_req req;
393*f540d44eSMagnus Karlsson struct rtattr *rta;
394*f540d44eSMagnus Karlsson int fd, ret;
395*f540d44eSMagnus Karlsson
396*f540d44eSMagnus Karlsson fd = socket(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE);
397*f540d44eSMagnus Karlsson if (fd < 0)
398*f540d44eSMagnus Karlsson return fd;
399*f540d44eSMagnus Karlsson
400*f540d44eSMagnus Karlsson memset(&req, 0, sizeof(req));
401*f540d44eSMagnus Karlsson req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
402*f540d44eSMagnus Karlsson req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
403*f540d44eSMagnus Karlsson req.nh.nlmsg_type = RTM_NEWLINK;
404*f540d44eSMagnus Karlsson req.msg.ifi_family = AF_UNSPEC;
405*f540d44eSMagnus Karlsson req.msg.ifi_index = ifindex;
406*f540d44eSMagnus Karlsson rta = (struct rtattr *)(((char *)&req) + NLMSG_ALIGN(req.nh.nlmsg_len));
407*f540d44eSMagnus Karlsson rta->rta_type = IFLA_MTU;
408*f540d44eSMagnus Karlsson rta->rta_len = RTA_LENGTH(sizeof(unsigned int));
409*f540d44eSMagnus Karlsson req.nh.nlmsg_len = NLMSG_ALIGN(req.nh.nlmsg_len) + RTA_LENGTH(sizeof(mtu));
410*f540d44eSMagnus Karlsson memcpy(RTA_DATA(rta), &mtu, sizeof(mtu));
411*f540d44eSMagnus Karlsson
412*f540d44eSMagnus Karlsson ret = send(fd, &req, req.nh.nlmsg_len, 0);
413*f540d44eSMagnus Karlsson if (ret < 0) {
414*f540d44eSMagnus Karlsson close(fd);
415*f540d44eSMagnus Karlsson return errno;
416*f540d44eSMagnus Karlsson }
417*f540d44eSMagnus Karlsson
418*f540d44eSMagnus Karlsson ret = netlink_recv(fd);
419*f540d44eSMagnus Karlsson close(fd);
420*f540d44eSMagnus Karlsson return ret;
421*f540d44eSMagnus Karlsson }
422*f540d44eSMagnus Karlsson
xsk_attach_xdp_program(struct bpf_program * prog,int ifindex,u32 xdp_flags)423f0a249dfSMagnus Karlsson int xsk_attach_xdp_program(struct bpf_program *prog, int ifindex, u32 xdp_flags)
424f3660063SAndrii Nakryiko {
425f3660063SAndrii Nakryiko int prog_fd;
426f3660063SAndrii Nakryiko
427f0a249dfSMagnus Karlsson prog_fd = bpf_program__fd(prog);
428f0a249dfSMagnus Karlsson return bpf_xdp_attach(ifindex, prog_fd, xdp_flags, NULL);
429f3660063SAndrii Nakryiko }
430f3660063SAndrii Nakryiko
xsk_detach_xdp_program(int ifindex,u32 xdp_flags)431f0a249dfSMagnus Karlsson void xsk_detach_xdp_program(int ifindex, u32 xdp_flags)
432aa61d81fSMagnus Karlsson {
433f0a249dfSMagnus Karlsson bpf_xdp_detach(ifindex, xdp_flags, NULL);
434aa61d81fSMagnus Karlsson }
435aa61d81fSMagnus Karlsson
xsk_clear_xskmap(struct bpf_map * map)436f0a249dfSMagnus Karlsson void xsk_clear_xskmap(struct bpf_map *map)
437aa61d81fSMagnus Karlsson {
438f0a249dfSMagnus Karlsson u32 index = 0;
439f0a249dfSMagnus Karlsson int map_fd;
440aa61d81fSMagnus Karlsson
441f0a249dfSMagnus Karlsson map_fd = bpf_map__fd(map);
442f0a249dfSMagnus Karlsson bpf_map_delete_elem(map_fd, &index);
443aa61d81fSMagnus Karlsson }
444aa61d81fSMagnus Karlsson
xsk_update_xskmap(struct bpf_map * map,struct xsk_socket * xsk)445f0a249dfSMagnus Karlsson int xsk_update_xskmap(struct bpf_map *map, struct xsk_socket *xsk)
446f0a249dfSMagnus Karlsson {
447f0a249dfSMagnus Karlsson int map_fd, sock_fd;
448f0a249dfSMagnus Karlsson u32 index = 0;
449f0a249dfSMagnus Karlsson
450f0a249dfSMagnus Karlsson map_fd = bpf_map__fd(map);
451f0a249dfSMagnus Karlsson sock_fd = xsk_socket__fd(xsk);
452f0a249dfSMagnus Karlsson
453f0a249dfSMagnus Karlsson return bpf_map_update_elem(map_fd, &index, &sock_fd, 0);
454aa61d81fSMagnus Karlsson }
455aa61d81fSMagnus Karlsson
xsk_get_ctx(struct xsk_umem * umem,int ifindex,__u32 queue_id)456f3660063SAndrii Nakryiko static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
457f3660063SAndrii Nakryiko __u32 queue_id)
458f3660063SAndrii Nakryiko {
459f3660063SAndrii Nakryiko struct xsk_ctx *ctx;
460f3660063SAndrii Nakryiko
461f3660063SAndrii Nakryiko if (list_empty(&umem->ctx_list))
462f3660063SAndrii Nakryiko return NULL;
463f3660063SAndrii Nakryiko
464f3660063SAndrii Nakryiko list_for_each_entry(ctx, &umem->ctx_list, list) {
465f3660063SAndrii Nakryiko if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
466f3660063SAndrii Nakryiko ctx->refcount++;
467f3660063SAndrii Nakryiko return ctx;
468f3660063SAndrii Nakryiko }
469f3660063SAndrii Nakryiko }
470f3660063SAndrii Nakryiko
471f3660063SAndrii Nakryiko return NULL;
472f3660063SAndrii Nakryiko }
473f3660063SAndrii Nakryiko
xsk_put_ctx(struct xsk_ctx * ctx,bool unmap)474f3660063SAndrii Nakryiko static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
475f3660063SAndrii Nakryiko {
476f3660063SAndrii Nakryiko struct xsk_umem *umem = ctx->umem;
477f3660063SAndrii Nakryiko struct xdp_mmap_offsets off;
478f3660063SAndrii Nakryiko int err;
479f3660063SAndrii Nakryiko
480f3660063SAndrii Nakryiko if (--ctx->refcount)
481f3660063SAndrii Nakryiko return;
482f3660063SAndrii Nakryiko
483f3660063SAndrii Nakryiko if (!unmap)
484f3660063SAndrii Nakryiko goto out_free;
485f3660063SAndrii Nakryiko
486f3660063SAndrii Nakryiko err = xsk_get_mmap_offsets(umem->fd, &off);
487f3660063SAndrii Nakryiko if (err)
488f3660063SAndrii Nakryiko goto out_free;
489f3660063SAndrii Nakryiko
490f3660063SAndrii Nakryiko munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
491f3660063SAndrii Nakryiko sizeof(__u64));
492f3660063SAndrii Nakryiko munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
493f3660063SAndrii Nakryiko sizeof(__u64));
494f3660063SAndrii Nakryiko
495f3660063SAndrii Nakryiko out_free:
496f3660063SAndrii Nakryiko list_del(&ctx->list);
497f3660063SAndrii Nakryiko free(ctx);
498f3660063SAndrii Nakryiko }
499f3660063SAndrii Nakryiko
xsk_create_ctx(struct xsk_socket * xsk,struct xsk_umem * umem,int ifindex,__u32 queue_id,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp)500f3660063SAndrii Nakryiko static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
501f3660063SAndrii Nakryiko struct xsk_umem *umem, int ifindex,
502aa61d81fSMagnus Karlsson __u32 queue_id,
503f3660063SAndrii Nakryiko struct xsk_ring_prod *fill,
504f3660063SAndrii Nakryiko struct xsk_ring_cons *comp)
505f3660063SAndrii Nakryiko {
506f3660063SAndrii Nakryiko struct xsk_ctx *ctx;
507f3660063SAndrii Nakryiko int err;
508f3660063SAndrii Nakryiko
509f3660063SAndrii Nakryiko ctx = calloc(1, sizeof(*ctx));
510f3660063SAndrii Nakryiko if (!ctx)
511f3660063SAndrii Nakryiko return NULL;
512f3660063SAndrii Nakryiko
513f3660063SAndrii Nakryiko if (!umem->fill_save) {
514f3660063SAndrii Nakryiko err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
515f3660063SAndrii Nakryiko if (err) {
516f3660063SAndrii Nakryiko free(ctx);
517f3660063SAndrii Nakryiko return NULL;
518f3660063SAndrii Nakryiko }
519f3660063SAndrii Nakryiko } else if (umem->fill_save != fill || umem->comp_save != comp) {
520f3660063SAndrii Nakryiko /* Copy over rings to new structs. */
521f3660063SAndrii Nakryiko memcpy(fill, umem->fill_save, sizeof(*fill));
522f3660063SAndrii Nakryiko memcpy(comp, umem->comp_save, sizeof(*comp));
523f3660063SAndrii Nakryiko }
524f3660063SAndrii Nakryiko
525f3660063SAndrii Nakryiko ctx->ifindex = ifindex;
526f3660063SAndrii Nakryiko ctx->refcount = 1;
527f3660063SAndrii Nakryiko ctx->umem = umem;
528f3660063SAndrii Nakryiko ctx->queue_id = queue_id;
529f3660063SAndrii Nakryiko
530f3660063SAndrii Nakryiko ctx->fill = fill;
531f3660063SAndrii Nakryiko ctx->comp = comp;
532f3660063SAndrii Nakryiko list_add(&ctx->list, &umem->ctx_list);
533f3660063SAndrii Nakryiko return ctx;
534f3660063SAndrii Nakryiko }
535f3660063SAndrii Nakryiko
xsk_socket__create_shared(struct xsk_socket ** xsk_ptr,int ifindex,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,struct xsk_ring_prod * fill,struct xsk_ring_cons * comp,const struct xsk_socket_config * usr_config)536f3660063SAndrii Nakryiko int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
537aa61d81fSMagnus Karlsson int ifindex,
538f3660063SAndrii Nakryiko __u32 queue_id, struct xsk_umem *umem,
539f3660063SAndrii Nakryiko struct xsk_ring_cons *rx,
540f3660063SAndrii Nakryiko struct xsk_ring_prod *tx,
541f3660063SAndrii Nakryiko struct xsk_ring_prod *fill,
542f3660063SAndrii Nakryiko struct xsk_ring_cons *comp,
543f3660063SAndrii Nakryiko const struct xsk_socket_config *usr_config)
544f3660063SAndrii Nakryiko {
545f3660063SAndrii Nakryiko bool unmap, rx_setup_done = false, tx_setup_done = false;
546f3660063SAndrii Nakryiko void *rx_map = NULL, *tx_map = NULL;
547f3660063SAndrii Nakryiko struct sockaddr_xdp sxdp = {};
548f3660063SAndrii Nakryiko struct xdp_mmap_offsets off;
549f3660063SAndrii Nakryiko struct xsk_socket *xsk;
550f3660063SAndrii Nakryiko struct xsk_ctx *ctx;
551aa61d81fSMagnus Karlsson int err;
552f3660063SAndrii Nakryiko
553f3660063SAndrii Nakryiko if (!umem || !xsk_ptr || !(rx || tx))
554f3660063SAndrii Nakryiko return -EFAULT;
555f3660063SAndrii Nakryiko
556f3660063SAndrii Nakryiko unmap = umem->fill_save != fill;
557f3660063SAndrii Nakryiko
558f3660063SAndrii Nakryiko xsk = calloc(1, sizeof(*xsk));
559f3660063SAndrii Nakryiko if (!xsk)
560f3660063SAndrii Nakryiko return -ENOMEM;
561f3660063SAndrii Nakryiko
562f3660063SAndrii Nakryiko err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
563f3660063SAndrii Nakryiko if (err)
564f3660063SAndrii Nakryiko goto out_xsk_alloc;
565f3660063SAndrii Nakryiko
566f3660063SAndrii Nakryiko if (umem->refcount++ > 0) {
567f3660063SAndrii Nakryiko xsk->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
568f3660063SAndrii Nakryiko if (xsk->fd < 0) {
569f3660063SAndrii Nakryiko err = -errno;
570f3660063SAndrii Nakryiko goto out_xsk_alloc;
571f3660063SAndrii Nakryiko }
572f3660063SAndrii Nakryiko } else {
573f3660063SAndrii Nakryiko xsk->fd = umem->fd;
574f3660063SAndrii Nakryiko rx_setup_done = umem->rx_ring_setup_done;
575f3660063SAndrii Nakryiko tx_setup_done = umem->tx_ring_setup_done;
576f3660063SAndrii Nakryiko }
577f3660063SAndrii Nakryiko
578f3660063SAndrii Nakryiko ctx = xsk_get_ctx(umem, ifindex, queue_id);
579f3660063SAndrii Nakryiko if (!ctx) {
580f3660063SAndrii Nakryiko if (!fill || !comp) {
581f3660063SAndrii Nakryiko err = -EFAULT;
582f3660063SAndrii Nakryiko goto out_socket;
583f3660063SAndrii Nakryiko }
584f3660063SAndrii Nakryiko
585aa61d81fSMagnus Karlsson ctx = xsk_create_ctx(xsk, umem, ifindex, queue_id, fill, comp);
586f3660063SAndrii Nakryiko if (!ctx) {
587f3660063SAndrii Nakryiko err = -ENOMEM;
588f3660063SAndrii Nakryiko goto out_socket;
589f3660063SAndrii Nakryiko }
590f3660063SAndrii Nakryiko }
591f3660063SAndrii Nakryiko xsk->ctx = ctx;
592f3660063SAndrii Nakryiko
593f3660063SAndrii Nakryiko if (rx && !rx_setup_done) {
594f3660063SAndrii Nakryiko err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
595f3660063SAndrii Nakryiko &xsk->config.rx_size,
596f3660063SAndrii Nakryiko sizeof(xsk->config.rx_size));
597f3660063SAndrii Nakryiko if (err) {
598f3660063SAndrii Nakryiko err = -errno;
599f3660063SAndrii Nakryiko goto out_put_ctx;
600f3660063SAndrii Nakryiko }
601f3660063SAndrii Nakryiko if (xsk->fd == umem->fd)
602f3660063SAndrii Nakryiko umem->rx_ring_setup_done = true;
603f3660063SAndrii Nakryiko }
604f3660063SAndrii Nakryiko if (tx && !tx_setup_done) {
605f3660063SAndrii Nakryiko err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
606f3660063SAndrii Nakryiko &xsk->config.tx_size,
607f3660063SAndrii Nakryiko sizeof(xsk->config.tx_size));
608f3660063SAndrii Nakryiko if (err) {
609f3660063SAndrii Nakryiko err = -errno;
610f3660063SAndrii Nakryiko goto out_put_ctx;
611f3660063SAndrii Nakryiko }
612f3660063SAndrii Nakryiko if (xsk->fd == umem->fd)
613f3660063SAndrii Nakryiko umem->tx_ring_setup_done = true;
614f3660063SAndrii Nakryiko }
615f3660063SAndrii Nakryiko
616f3660063SAndrii Nakryiko err = xsk_get_mmap_offsets(xsk->fd, &off);
617f3660063SAndrii Nakryiko if (err) {
618f3660063SAndrii Nakryiko err = -errno;
619f3660063SAndrii Nakryiko goto out_put_ctx;
620f3660063SAndrii Nakryiko }
621f3660063SAndrii Nakryiko
622f3660063SAndrii Nakryiko if (rx) {
623f3660063SAndrii Nakryiko rx_map = mmap(NULL, off.rx.desc +
624f3660063SAndrii Nakryiko xsk->config.rx_size * sizeof(struct xdp_desc),
625f3660063SAndrii Nakryiko PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
626f3660063SAndrii Nakryiko xsk->fd, XDP_PGOFF_RX_RING);
627f3660063SAndrii Nakryiko if (rx_map == MAP_FAILED) {
628f3660063SAndrii Nakryiko err = -errno;
629f3660063SAndrii Nakryiko goto out_put_ctx;
630f3660063SAndrii Nakryiko }
631f3660063SAndrii Nakryiko
632f3660063SAndrii Nakryiko rx->mask = xsk->config.rx_size - 1;
633f3660063SAndrii Nakryiko rx->size = xsk->config.rx_size;
634f3660063SAndrii Nakryiko rx->producer = rx_map + off.rx.producer;
635f3660063SAndrii Nakryiko rx->consumer = rx_map + off.rx.consumer;
636f3660063SAndrii Nakryiko rx->flags = rx_map + off.rx.flags;
637f3660063SAndrii Nakryiko rx->ring = rx_map + off.rx.desc;
638f3660063SAndrii Nakryiko rx->cached_prod = *rx->producer;
639f3660063SAndrii Nakryiko rx->cached_cons = *rx->consumer;
640f3660063SAndrii Nakryiko }
641f3660063SAndrii Nakryiko xsk->rx = rx;
642f3660063SAndrii Nakryiko
643f3660063SAndrii Nakryiko if (tx) {
644f3660063SAndrii Nakryiko tx_map = mmap(NULL, off.tx.desc +
645f3660063SAndrii Nakryiko xsk->config.tx_size * sizeof(struct xdp_desc),
646f3660063SAndrii Nakryiko PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
647f3660063SAndrii Nakryiko xsk->fd, XDP_PGOFF_TX_RING);
648f3660063SAndrii Nakryiko if (tx_map == MAP_FAILED) {
649f3660063SAndrii Nakryiko err = -errno;
650f3660063SAndrii Nakryiko goto out_mmap_rx;
651f3660063SAndrii Nakryiko }
652f3660063SAndrii Nakryiko
653f3660063SAndrii Nakryiko tx->mask = xsk->config.tx_size - 1;
654f3660063SAndrii Nakryiko tx->size = xsk->config.tx_size;
655f3660063SAndrii Nakryiko tx->producer = tx_map + off.tx.producer;
656f3660063SAndrii Nakryiko tx->consumer = tx_map + off.tx.consumer;
657f3660063SAndrii Nakryiko tx->flags = tx_map + off.tx.flags;
658f3660063SAndrii Nakryiko tx->ring = tx_map + off.tx.desc;
659f3660063SAndrii Nakryiko tx->cached_prod = *tx->producer;
660f3660063SAndrii Nakryiko /* cached_cons is r->size bigger than the real consumer pointer
661f3660063SAndrii Nakryiko * See xsk_prod_nb_free
662f3660063SAndrii Nakryiko */
663f3660063SAndrii Nakryiko tx->cached_cons = *tx->consumer + xsk->config.tx_size;
664f3660063SAndrii Nakryiko }
665f3660063SAndrii Nakryiko xsk->tx = tx;
666f3660063SAndrii Nakryiko
667f3660063SAndrii Nakryiko sxdp.sxdp_family = PF_XDP;
668f3660063SAndrii Nakryiko sxdp.sxdp_ifindex = ctx->ifindex;
669f3660063SAndrii Nakryiko sxdp.sxdp_queue_id = ctx->queue_id;
670f3660063SAndrii Nakryiko if (umem->refcount > 1) {
671f3660063SAndrii Nakryiko sxdp.sxdp_flags |= XDP_SHARED_UMEM;
672f3660063SAndrii Nakryiko sxdp.sxdp_shared_umem_fd = umem->fd;
673f3660063SAndrii Nakryiko } else {
674f3660063SAndrii Nakryiko sxdp.sxdp_flags = xsk->config.bind_flags;
675f3660063SAndrii Nakryiko }
676f3660063SAndrii Nakryiko
677f3660063SAndrii Nakryiko err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
678f3660063SAndrii Nakryiko if (err) {
679f3660063SAndrii Nakryiko err = -errno;
680f3660063SAndrii Nakryiko goto out_mmap_tx;
681f3660063SAndrii Nakryiko }
682f3660063SAndrii Nakryiko
683f3660063SAndrii Nakryiko *xsk_ptr = xsk;
684f3660063SAndrii Nakryiko umem->fill_save = NULL;
685f3660063SAndrii Nakryiko umem->comp_save = NULL;
686f3660063SAndrii Nakryiko return 0;
687f3660063SAndrii Nakryiko
688f3660063SAndrii Nakryiko out_mmap_tx:
689f3660063SAndrii Nakryiko if (tx)
690f3660063SAndrii Nakryiko munmap(tx_map, off.tx.desc +
691f3660063SAndrii Nakryiko xsk->config.tx_size * sizeof(struct xdp_desc));
692f3660063SAndrii Nakryiko out_mmap_rx:
693f3660063SAndrii Nakryiko if (rx)
694f3660063SAndrii Nakryiko munmap(rx_map, off.rx.desc +
695f3660063SAndrii Nakryiko xsk->config.rx_size * sizeof(struct xdp_desc));
696f3660063SAndrii Nakryiko out_put_ctx:
697f3660063SAndrii Nakryiko xsk_put_ctx(ctx, unmap);
698f3660063SAndrii Nakryiko out_socket:
699f3660063SAndrii Nakryiko if (--umem->refcount)
700f3660063SAndrii Nakryiko close(xsk->fd);
701f3660063SAndrii Nakryiko out_xsk_alloc:
702f3660063SAndrii Nakryiko free(xsk);
703f3660063SAndrii Nakryiko return err;
704f3660063SAndrii Nakryiko }
705f3660063SAndrii Nakryiko
xsk_socket__create(struct xsk_socket ** xsk_ptr,int ifindex,__u32 queue_id,struct xsk_umem * umem,struct xsk_ring_cons * rx,struct xsk_ring_prod * tx,const struct xsk_socket_config * usr_config)706aa61d81fSMagnus Karlsson int xsk_socket__create(struct xsk_socket **xsk_ptr, int ifindex,
707f3660063SAndrii Nakryiko __u32 queue_id, struct xsk_umem *umem,
708f3660063SAndrii Nakryiko struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
709f3660063SAndrii Nakryiko const struct xsk_socket_config *usr_config)
710f3660063SAndrii Nakryiko {
711f3660063SAndrii Nakryiko if (!umem)
712f3660063SAndrii Nakryiko return -EFAULT;
713f3660063SAndrii Nakryiko
714aa61d81fSMagnus Karlsson return xsk_socket__create_shared(xsk_ptr, ifindex, queue_id, umem,
715f3660063SAndrii Nakryiko rx, tx, umem->fill_save,
716f3660063SAndrii Nakryiko umem->comp_save, usr_config);
717f3660063SAndrii Nakryiko }
718f3660063SAndrii Nakryiko
xsk_umem__delete(struct xsk_umem * umem)719f3660063SAndrii Nakryiko int xsk_umem__delete(struct xsk_umem *umem)
720f3660063SAndrii Nakryiko {
721f3660063SAndrii Nakryiko struct xdp_mmap_offsets off;
722f3660063SAndrii Nakryiko int err;
723f3660063SAndrii Nakryiko
724f3660063SAndrii Nakryiko if (!umem)
725f3660063SAndrii Nakryiko return 0;
726f3660063SAndrii Nakryiko
727f3660063SAndrii Nakryiko if (umem->refcount)
728f3660063SAndrii Nakryiko return -EBUSY;
729f3660063SAndrii Nakryiko
730f3660063SAndrii Nakryiko err = xsk_get_mmap_offsets(umem->fd, &off);
731f3660063SAndrii Nakryiko if (!err && umem->fill_save && umem->comp_save) {
732f3660063SAndrii Nakryiko munmap(umem->fill_save->ring - off.fr.desc,
733f3660063SAndrii Nakryiko off.fr.desc + umem->config.fill_size * sizeof(__u64));
734f3660063SAndrii Nakryiko munmap(umem->comp_save->ring - off.cr.desc,
735f3660063SAndrii Nakryiko off.cr.desc + umem->config.comp_size * sizeof(__u64));
736f3660063SAndrii Nakryiko }
737f3660063SAndrii Nakryiko
738f3660063SAndrii Nakryiko close(umem->fd);
739f3660063SAndrii Nakryiko free(umem);
740f3660063SAndrii Nakryiko
741f3660063SAndrii Nakryiko return 0;
742f3660063SAndrii Nakryiko }
743f3660063SAndrii Nakryiko
xsk_socket__delete(struct xsk_socket * xsk)744f3660063SAndrii Nakryiko void xsk_socket__delete(struct xsk_socket *xsk)
745f3660063SAndrii Nakryiko {
746f3660063SAndrii Nakryiko size_t desc_sz = sizeof(struct xdp_desc);
747f3660063SAndrii Nakryiko struct xdp_mmap_offsets off;
748f3660063SAndrii Nakryiko struct xsk_umem *umem;
749f3660063SAndrii Nakryiko struct xsk_ctx *ctx;
750f3660063SAndrii Nakryiko int err;
751f3660063SAndrii Nakryiko
752f3660063SAndrii Nakryiko if (!xsk)
753f3660063SAndrii Nakryiko return;
754f3660063SAndrii Nakryiko
755f3660063SAndrii Nakryiko ctx = xsk->ctx;
756f3660063SAndrii Nakryiko umem = ctx->umem;
75739e940d4SMaciej Fijalkowski
758af515a55SIan Rogers xsk_put_ctx(ctx, true);
759af515a55SIan Rogers
760f3660063SAndrii Nakryiko err = xsk_get_mmap_offsets(xsk->fd, &off);
761f3660063SAndrii Nakryiko if (!err) {
762f3660063SAndrii Nakryiko if (xsk->rx) {
763f3660063SAndrii Nakryiko munmap(xsk->rx->ring - off.rx.desc,
764f3660063SAndrii Nakryiko off.rx.desc + xsk->config.rx_size * desc_sz);
765f3660063SAndrii Nakryiko }
766f3660063SAndrii Nakryiko if (xsk->tx) {
767f3660063SAndrii Nakryiko munmap(xsk->tx->ring - off.tx.desc,
768f3660063SAndrii Nakryiko off.tx.desc + xsk->config.tx_size * desc_sz);
769f3660063SAndrii Nakryiko }
770f3660063SAndrii Nakryiko }
771f3660063SAndrii Nakryiko
772f3660063SAndrii Nakryiko umem->refcount--;
773f3660063SAndrii Nakryiko /* Do not close an fd that also has an associated umem connected
774f3660063SAndrii Nakryiko * to it.
775f3660063SAndrii Nakryiko */
776f3660063SAndrii Nakryiko if (xsk->fd != umem->fd)
777f3660063SAndrii Nakryiko close(xsk->fd);
778f3660063SAndrii Nakryiko free(xsk);
779f3660063SAndrii Nakryiko }
780