1bf99c936SAndrii Nakryiko // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2bf99c936SAndrii Nakryiko /*
3bf99c936SAndrii Nakryiko * Ring buffer operations.
4bf99c936SAndrii Nakryiko *
5bf99c936SAndrii Nakryiko * Copyright (C) 2020 Facebook, Inc.
6bf99c936SAndrii Nakryiko */
7febeb6dfSAndrii Nakryiko #ifndef _GNU_SOURCE
8febeb6dfSAndrii Nakryiko #define _GNU_SOURCE
9febeb6dfSAndrii Nakryiko #endif
10bf99c936SAndrii Nakryiko #include <stdlib.h>
11bf99c936SAndrii Nakryiko #include <stdio.h>
12bf99c936SAndrii Nakryiko #include <errno.h>
13bf99c936SAndrii Nakryiko #include <unistd.h>
14bf99c936SAndrii Nakryiko #include <linux/err.h>
15bf99c936SAndrii Nakryiko #include <linux/bpf.h>
16bf99c936SAndrii Nakryiko #include <asm/barrier.h>
17bf99c936SAndrii Nakryiko #include <sys/mman.h>
18bf99c936SAndrii Nakryiko #include <sys/epoll.h>
19b66ccae0SDavid Vernet #include <time.h>
20bf99c936SAndrii Nakryiko
21bf99c936SAndrii Nakryiko #include "libbpf.h"
22bf99c936SAndrii Nakryiko #include "libbpf_internal.h"
23bf99c936SAndrii Nakryiko #include "bpf.h"
24bf99c936SAndrii Nakryiko
25bf99c936SAndrii Nakryiko struct ring {
26bf99c936SAndrii Nakryiko ring_buffer_sample_fn sample_cb;
27bf99c936SAndrii Nakryiko void *ctx;
28bf99c936SAndrii Nakryiko void *data;
29bf99c936SAndrii Nakryiko unsigned long *consumer_pos;
30bf99c936SAndrii Nakryiko unsigned long *producer_pos;
31bf99c936SAndrii Nakryiko unsigned long mask;
32bf99c936SAndrii Nakryiko int map_fd;
33bf99c936SAndrii Nakryiko };
34bf99c936SAndrii Nakryiko
35bf99c936SAndrii Nakryiko struct ring_buffer {
36bf99c936SAndrii Nakryiko struct epoll_event *events;
37bf99c936SAndrii Nakryiko struct ring *rings;
38bf99c936SAndrii Nakryiko size_t page_size;
39bf99c936SAndrii Nakryiko int epoll_fd;
40bf99c936SAndrii Nakryiko int ring_cnt;
41bf99c936SAndrii Nakryiko };
42bf99c936SAndrii Nakryiko
43b66ccae0SDavid Vernet struct user_ring_buffer {
44b66ccae0SDavid Vernet struct epoll_event event;
45b66ccae0SDavid Vernet unsigned long *consumer_pos;
46b66ccae0SDavid Vernet unsigned long *producer_pos;
47b66ccae0SDavid Vernet void *data;
48b66ccae0SDavid Vernet unsigned long mask;
49b66ccae0SDavid Vernet size_t page_size;
50b66ccae0SDavid Vernet int map_fd;
51b66ccae0SDavid Vernet int epoll_fd;
52b66ccae0SDavid Vernet };
53b66ccae0SDavid Vernet
54b66ccae0SDavid Vernet /* 8-byte ring buffer header structure */
55b66ccae0SDavid Vernet struct ringbuf_hdr {
56b66ccae0SDavid Vernet __u32 len;
57b66ccae0SDavid Vernet __u32 pad;
58b66ccae0SDavid Vernet };
59b66ccae0SDavid Vernet
ringbuf_unmap_ring(struct ring_buffer * rb,struct ring * r)60bf99c936SAndrii Nakryiko static void ringbuf_unmap_ring(struct ring_buffer *rb, struct ring *r)
61bf99c936SAndrii Nakryiko {
62bf99c936SAndrii Nakryiko if (r->consumer_pos) {
63bf99c936SAndrii Nakryiko munmap(r->consumer_pos, rb->page_size);
64bf99c936SAndrii Nakryiko r->consumer_pos = NULL;
65bf99c936SAndrii Nakryiko }
66bf99c936SAndrii Nakryiko if (r->producer_pos) {
67bf99c936SAndrii Nakryiko munmap(r->producer_pos, rb->page_size + 2 * (r->mask + 1));
68bf99c936SAndrii Nakryiko r->producer_pos = NULL;
69bf99c936SAndrii Nakryiko }
70bf99c936SAndrii Nakryiko }
71bf99c936SAndrii Nakryiko
72bf99c936SAndrii Nakryiko /* Add extra RINGBUF maps to this ring buffer manager */
ring_buffer__add(struct ring_buffer * rb,int map_fd,ring_buffer_sample_fn sample_cb,void * ctx)73bf99c936SAndrii Nakryiko int ring_buffer__add(struct ring_buffer *rb, int map_fd,
74bf99c936SAndrii Nakryiko ring_buffer_sample_fn sample_cb, void *ctx)
75bf99c936SAndrii Nakryiko {
76bf99c936SAndrii Nakryiko struct bpf_map_info info;
77bf99c936SAndrii Nakryiko __u32 len = sizeof(info);
78bf99c936SAndrii Nakryiko struct epoll_event *e;
79bf99c936SAndrii Nakryiko struct ring *r;
80927cbb47SHou Tao __u64 mmap_sz;
81bf99c936SAndrii Nakryiko void *tmp;
82bf99c936SAndrii Nakryiko int err;
83bf99c936SAndrii Nakryiko
84bf99c936SAndrii Nakryiko memset(&info, 0, sizeof(info));
85bf99c936SAndrii Nakryiko
86*629dfc66SIlya Leoshkevich err = bpf_map_get_info_by_fd(map_fd, &info, &len);
87bf99c936SAndrii Nakryiko if (err) {
88bf99c936SAndrii Nakryiko err = -errno;
89bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to get map info for fd=%d: %d\n",
90bf99c936SAndrii Nakryiko map_fd, err);
91e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
92bf99c936SAndrii Nakryiko }
93bf99c936SAndrii Nakryiko
94bf99c936SAndrii Nakryiko if (info.type != BPF_MAP_TYPE_RINGBUF) {
95bf99c936SAndrii Nakryiko pr_warn("ringbuf: map fd=%d is not BPF_MAP_TYPE_RINGBUF\n",
96bf99c936SAndrii Nakryiko map_fd);
97e9fc3ce9SAndrii Nakryiko return libbpf_err(-EINVAL);
98bf99c936SAndrii Nakryiko }
99bf99c936SAndrii Nakryiko
100029258d7SAndrii Nakryiko tmp = libbpf_reallocarray(rb->rings, rb->ring_cnt + 1, sizeof(*rb->rings));
101bf99c936SAndrii Nakryiko if (!tmp)
102e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM);
103bf99c936SAndrii Nakryiko rb->rings = tmp;
104bf99c936SAndrii Nakryiko
105029258d7SAndrii Nakryiko tmp = libbpf_reallocarray(rb->events, rb->ring_cnt + 1, sizeof(*rb->events));
106bf99c936SAndrii Nakryiko if (!tmp)
107e9fc3ce9SAndrii Nakryiko return libbpf_err(-ENOMEM);
108bf99c936SAndrii Nakryiko rb->events = tmp;
109bf99c936SAndrii Nakryiko
110bf99c936SAndrii Nakryiko r = &rb->rings[rb->ring_cnt];
111bf99c936SAndrii Nakryiko memset(r, 0, sizeof(*r));
112bf99c936SAndrii Nakryiko
113bf99c936SAndrii Nakryiko r->map_fd = map_fd;
114bf99c936SAndrii Nakryiko r->sample_cb = sample_cb;
115bf99c936SAndrii Nakryiko r->ctx = ctx;
116bf99c936SAndrii Nakryiko r->mask = info.max_entries - 1;
117bf99c936SAndrii Nakryiko
118bf99c936SAndrii Nakryiko /* Map writable consumer page */
119927cbb47SHou Tao tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);
120bf99c936SAndrii Nakryiko if (tmp == MAP_FAILED) {
121bf99c936SAndrii Nakryiko err = -errno;
122bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %d\n",
123bf99c936SAndrii Nakryiko map_fd, err);
124e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
125bf99c936SAndrii Nakryiko }
126bf99c936SAndrii Nakryiko r->consumer_pos = tmp;
127bf99c936SAndrii Nakryiko
128bf99c936SAndrii Nakryiko /* Map read-only producer page and data pages. We map twice as big
129bf99c936SAndrii Nakryiko * data size to allow simple reading of samples that wrap around the
130bf99c936SAndrii Nakryiko * end of a ring buffer. See kernel implementation for details.
131b486d19aSKang Minchul */
132927cbb47SHou Tao mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;
133927cbb47SHou Tao if (mmap_sz != (__u64)(size_t)mmap_sz) {
134927cbb47SHou Tao pr_warn("ringbuf: ring buffer size (%u) is too big\n", info.max_entries);
135927cbb47SHou Tao return libbpf_err(-E2BIG);
136927cbb47SHou Tao }
137927cbb47SHou Tao tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);
138bf99c936SAndrii Nakryiko if (tmp == MAP_FAILED) {
139bf99c936SAndrii Nakryiko err = -errno;
140bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, r);
141bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %d\n",
142bf99c936SAndrii Nakryiko map_fd, err);
143e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
144bf99c936SAndrii Nakryiko }
145bf99c936SAndrii Nakryiko r->producer_pos = tmp;
146bf99c936SAndrii Nakryiko r->data = tmp + rb->page_size;
147bf99c936SAndrii Nakryiko
148bf99c936SAndrii Nakryiko e = &rb->events[rb->ring_cnt];
149bf99c936SAndrii Nakryiko memset(e, 0, sizeof(*e));
150bf99c936SAndrii Nakryiko
151bf99c936SAndrii Nakryiko e->events = EPOLLIN;
152bf99c936SAndrii Nakryiko e->data.fd = rb->ring_cnt;
153bf99c936SAndrii Nakryiko if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {
154bf99c936SAndrii Nakryiko err = -errno;
155bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, r);
156bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to epoll add map fd=%d: %d\n",
157bf99c936SAndrii Nakryiko map_fd, err);
158e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
159bf99c936SAndrii Nakryiko }
160bf99c936SAndrii Nakryiko
161bf99c936SAndrii Nakryiko rb->ring_cnt++;
162bf99c936SAndrii Nakryiko return 0;
163bf99c936SAndrii Nakryiko }
164bf99c936SAndrii Nakryiko
ring_buffer__free(struct ring_buffer * rb)165bf99c936SAndrii Nakryiko void ring_buffer__free(struct ring_buffer *rb)
166bf99c936SAndrii Nakryiko {
167bf99c936SAndrii Nakryiko int i;
168bf99c936SAndrii Nakryiko
169bf99c936SAndrii Nakryiko if (!rb)
170bf99c936SAndrii Nakryiko return;
171bf99c936SAndrii Nakryiko
172bf99c936SAndrii Nakryiko for (i = 0; i < rb->ring_cnt; ++i)
173bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, &rb->rings[i]);
174bf99c936SAndrii Nakryiko if (rb->epoll_fd >= 0)
175bf99c936SAndrii Nakryiko close(rb->epoll_fd);
176bf99c936SAndrii Nakryiko
177bf99c936SAndrii Nakryiko free(rb->events);
178bf99c936SAndrii Nakryiko free(rb->rings);
179bf99c936SAndrii Nakryiko free(rb);
180bf99c936SAndrii Nakryiko }
181bf99c936SAndrii Nakryiko
182bf99c936SAndrii Nakryiko struct ring_buffer *
ring_buffer__new(int map_fd,ring_buffer_sample_fn sample_cb,void * ctx,const struct ring_buffer_opts * opts)183bf99c936SAndrii Nakryiko ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx,
184bf99c936SAndrii Nakryiko const struct ring_buffer_opts *opts)
185bf99c936SAndrii Nakryiko {
186bf99c936SAndrii Nakryiko struct ring_buffer *rb;
187bf99c936SAndrii Nakryiko int err;
188bf99c936SAndrii Nakryiko
189bf99c936SAndrii Nakryiko if (!OPTS_VALID(opts, ring_buffer_opts))
190e9fc3ce9SAndrii Nakryiko return errno = EINVAL, NULL;
191bf99c936SAndrii Nakryiko
192bf99c936SAndrii Nakryiko rb = calloc(1, sizeof(*rb));
193bf99c936SAndrii Nakryiko if (!rb)
194e9fc3ce9SAndrii Nakryiko return errno = ENOMEM, NULL;
195bf99c936SAndrii Nakryiko
196bf99c936SAndrii Nakryiko rb->page_size = getpagesize();
197bf99c936SAndrii Nakryiko
198bf99c936SAndrii Nakryiko rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
199bf99c936SAndrii Nakryiko if (rb->epoll_fd < 0) {
200bf99c936SAndrii Nakryiko err = -errno;
201bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to create epoll instance: %d\n", err);
202bf99c936SAndrii Nakryiko goto err_out;
203bf99c936SAndrii Nakryiko }
204bf99c936SAndrii Nakryiko
205bf99c936SAndrii Nakryiko err = ring_buffer__add(rb, map_fd, sample_cb, ctx);
206bf99c936SAndrii Nakryiko if (err)
207bf99c936SAndrii Nakryiko goto err_out;
208bf99c936SAndrii Nakryiko
209bf99c936SAndrii Nakryiko return rb;
210bf99c936SAndrii Nakryiko
211bf99c936SAndrii Nakryiko err_out:
212bf99c936SAndrii Nakryiko ring_buffer__free(rb);
213e9fc3ce9SAndrii Nakryiko return errno = -err, NULL;
214bf99c936SAndrii Nakryiko }
215bf99c936SAndrii Nakryiko
roundup_len(__u32 len)216bf99c936SAndrii Nakryiko static inline int roundup_len(__u32 len)
217bf99c936SAndrii Nakryiko {
218bf99c936SAndrii Nakryiko /* clear out top 2 bits (discard and busy, if set) */
219bf99c936SAndrii Nakryiko len <<= 2;
220bf99c936SAndrii Nakryiko len >>= 2;
221bf99c936SAndrii Nakryiko /* add length prefix */
222bf99c936SAndrii Nakryiko len += BPF_RINGBUF_HDR_SZ;
223bf99c936SAndrii Nakryiko /* round up to 8 byte alignment */
224bf99c936SAndrii Nakryiko return (len + 7) / 8 * 8;
225bf99c936SAndrii Nakryiko }
226bf99c936SAndrii Nakryiko
ringbuf_process_ring(struct ring * r)2272a30f944SBrendan Jackman static int64_t ringbuf_process_ring(struct ring *r)
228bf99c936SAndrii Nakryiko {
2292a30f944SBrendan Jackman int *len_ptr, len, err;
2302a30f944SBrendan Jackman /* 64-bit to avoid overflow in case of extreme application behavior */
2312a30f944SBrendan Jackman int64_t cnt = 0;
232bf99c936SAndrii Nakryiko unsigned long cons_pos, prod_pos;
233bf99c936SAndrii Nakryiko bool got_new_data;
234bf99c936SAndrii Nakryiko void *sample;
235bf99c936SAndrii Nakryiko
236bf99c936SAndrii Nakryiko cons_pos = smp_load_acquire(r->consumer_pos);
237bf99c936SAndrii Nakryiko do {
238bf99c936SAndrii Nakryiko got_new_data = false;
239bf99c936SAndrii Nakryiko prod_pos = smp_load_acquire(r->producer_pos);
240bf99c936SAndrii Nakryiko while (cons_pos < prod_pos) {
241bf99c936SAndrii Nakryiko len_ptr = r->data + (cons_pos & r->mask);
242bf99c936SAndrii Nakryiko len = smp_load_acquire(len_ptr);
243bf99c936SAndrii Nakryiko
244bf99c936SAndrii Nakryiko /* sample not committed yet, bail out for now */
245bf99c936SAndrii Nakryiko if (len & BPF_RINGBUF_BUSY_BIT)
246bf99c936SAndrii Nakryiko goto done;
247bf99c936SAndrii Nakryiko
248bf99c936SAndrii Nakryiko got_new_data = true;
249bf99c936SAndrii Nakryiko cons_pos += roundup_len(len);
250bf99c936SAndrii Nakryiko
251bf99c936SAndrii Nakryiko if ((len & BPF_RINGBUF_DISCARD_BIT) == 0) {
252bf99c936SAndrii Nakryiko sample = (void *)len_ptr + BPF_RINGBUF_HDR_SZ;
253bf99c936SAndrii Nakryiko err = r->sample_cb(r->ctx, sample, len);
2546032ebb5SPedro Tammela if (err < 0) {
255bf99c936SAndrii Nakryiko /* update consumer pos and bail out */
256bf99c936SAndrii Nakryiko smp_store_release(r->consumer_pos,
257bf99c936SAndrii Nakryiko cons_pos);
258bf99c936SAndrii Nakryiko return err;
259bf99c936SAndrii Nakryiko }
260bf99c936SAndrii Nakryiko cnt++;
261bf99c936SAndrii Nakryiko }
262bf99c936SAndrii Nakryiko
263bf99c936SAndrii Nakryiko smp_store_release(r->consumer_pos, cons_pos);
264bf99c936SAndrii Nakryiko }
265bf99c936SAndrii Nakryiko } while (got_new_data);
266bf99c936SAndrii Nakryiko done:
267bf99c936SAndrii Nakryiko return cnt;
268bf99c936SAndrii Nakryiko }
269bf99c936SAndrii Nakryiko
270bf99c936SAndrii Nakryiko /* Consume available ring buffer(s) data without event polling.
2712a30f944SBrendan Jackman * Returns number of records consumed across all registered ring buffers (or
2722a30f944SBrendan Jackman * INT_MAX, whichever is less), or negative number if any of the callbacks
2732a30f944SBrendan Jackman * return error.
274bf99c936SAndrii Nakryiko */
ring_buffer__consume(struct ring_buffer * rb)275bf99c936SAndrii Nakryiko int ring_buffer__consume(struct ring_buffer *rb)
276bf99c936SAndrii Nakryiko {
2772a30f944SBrendan Jackman int64_t err, res = 0;
2782a30f944SBrendan Jackman int i;
279bf99c936SAndrii Nakryiko
280bf99c936SAndrii Nakryiko for (i = 0; i < rb->ring_cnt; i++) {
281bf99c936SAndrii Nakryiko struct ring *ring = &rb->rings[i];
282bf99c936SAndrii Nakryiko
283bf99c936SAndrii Nakryiko err = ringbuf_process_ring(ring);
284bf99c936SAndrii Nakryiko if (err < 0)
285e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
286bf99c936SAndrii Nakryiko res += err;
287bf99c936SAndrii Nakryiko }
2882a30f944SBrendan Jackman if (res > INT_MAX)
2892a30f944SBrendan Jackman return INT_MAX;
290bf99c936SAndrii Nakryiko return res;
291bf99c936SAndrii Nakryiko }
292bf99c936SAndrii Nakryiko
293bf99c936SAndrii Nakryiko /* Poll for available data and consume records, if any are available.
2942a30f944SBrendan Jackman * Returns number of records consumed (or INT_MAX, whichever is less), or
2952a30f944SBrendan Jackman * negative number, if any of the registered callbacks returned error.
296bf99c936SAndrii Nakryiko */
ring_buffer__poll(struct ring_buffer * rb,int timeout_ms)297bf99c936SAndrii Nakryiko int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms)
298bf99c936SAndrii Nakryiko {
2992a30f944SBrendan Jackman int i, cnt;
3002a30f944SBrendan Jackman int64_t err, res = 0;
301bf99c936SAndrii Nakryiko
302bf99c936SAndrii Nakryiko cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms);
3032a30f944SBrendan Jackman if (cnt < 0)
304e9fc3ce9SAndrii Nakryiko return libbpf_err(-errno);
3052a30f944SBrendan Jackman
306bf99c936SAndrii Nakryiko for (i = 0; i < cnt; i++) {
307bf99c936SAndrii Nakryiko __u32 ring_id = rb->events[i].data.fd;
308bf99c936SAndrii Nakryiko struct ring *ring = &rb->rings[ring_id];
309bf99c936SAndrii Nakryiko
310bf99c936SAndrii Nakryiko err = ringbuf_process_ring(ring);
311bf99c936SAndrii Nakryiko if (err < 0)
312e9fc3ce9SAndrii Nakryiko return libbpf_err(err);
313f6a8250eSAndrii Nakryiko res += err;
314bf99c936SAndrii Nakryiko }
3152a30f944SBrendan Jackman if (res > INT_MAX)
3162a30f944SBrendan Jackman return INT_MAX;
3172a30f944SBrendan Jackman return res;
318bf99c936SAndrii Nakryiko }
319a4d2a7adSBrendan Jackman
320a4d2a7adSBrendan Jackman /* Get an fd that can be used to sleep until data is available in the ring(s) */
ring_buffer__epoll_fd(const struct ring_buffer * rb)321a4d2a7adSBrendan Jackman int ring_buffer__epoll_fd(const struct ring_buffer *rb)
322a4d2a7adSBrendan Jackman {
323a4d2a7adSBrendan Jackman return rb->epoll_fd;
324a4d2a7adSBrendan Jackman }
325b66ccae0SDavid Vernet
user_ringbuf_unmap_ring(struct user_ring_buffer * rb)326b66ccae0SDavid Vernet static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb)
327b66ccae0SDavid Vernet {
328b66ccae0SDavid Vernet if (rb->consumer_pos) {
329b66ccae0SDavid Vernet munmap(rb->consumer_pos, rb->page_size);
330b66ccae0SDavid Vernet rb->consumer_pos = NULL;
331b66ccae0SDavid Vernet }
332b66ccae0SDavid Vernet if (rb->producer_pos) {
333b66ccae0SDavid Vernet munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1));
334b66ccae0SDavid Vernet rb->producer_pos = NULL;
335b66ccae0SDavid Vernet }
336b66ccae0SDavid Vernet }
337b66ccae0SDavid Vernet
user_ring_buffer__free(struct user_ring_buffer * rb)338b66ccae0SDavid Vernet void user_ring_buffer__free(struct user_ring_buffer *rb)
339b66ccae0SDavid Vernet {
340b66ccae0SDavid Vernet if (!rb)
341b66ccae0SDavid Vernet return;
342b66ccae0SDavid Vernet
343b66ccae0SDavid Vernet user_ringbuf_unmap_ring(rb);
344b66ccae0SDavid Vernet
345b66ccae0SDavid Vernet if (rb->epoll_fd >= 0)
346b66ccae0SDavid Vernet close(rb->epoll_fd);
347b66ccae0SDavid Vernet
348b66ccae0SDavid Vernet free(rb);
349b66ccae0SDavid Vernet }
350b66ccae0SDavid Vernet
user_ringbuf_map(struct user_ring_buffer * rb,int map_fd)351b66ccae0SDavid Vernet static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd)
352b66ccae0SDavid Vernet {
353b66ccae0SDavid Vernet struct bpf_map_info info;
354b66ccae0SDavid Vernet __u32 len = sizeof(info);
35564176bffSHou Tao __u64 mmap_sz;
356b66ccae0SDavid Vernet void *tmp;
357b66ccae0SDavid Vernet struct epoll_event *rb_epoll;
358b66ccae0SDavid Vernet int err;
359b66ccae0SDavid Vernet
360b66ccae0SDavid Vernet memset(&info, 0, sizeof(info));
361b66ccae0SDavid Vernet
362*629dfc66SIlya Leoshkevich err = bpf_map_get_info_by_fd(map_fd, &info, &len);
363b66ccae0SDavid Vernet if (err) {
364b66ccae0SDavid Vernet err = -errno;
365b66ccae0SDavid Vernet pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err);
366b66ccae0SDavid Vernet return err;
367b66ccae0SDavid Vernet }
368b66ccae0SDavid Vernet
369b66ccae0SDavid Vernet if (info.type != BPF_MAP_TYPE_USER_RINGBUF) {
370b66ccae0SDavid Vernet pr_warn("user ringbuf: map fd=%d is not BPF_MAP_TYPE_USER_RINGBUF\n", map_fd);
371b66ccae0SDavid Vernet return -EINVAL;
372b66ccae0SDavid Vernet }
373b66ccae0SDavid Vernet
374b66ccae0SDavid Vernet rb->map_fd = map_fd;
375b66ccae0SDavid Vernet rb->mask = info.max_entries - 1;
376b66ccae0SDavid Vernet
377b66ccae0SDavid Vernet /* Map read-only consumer page */
378b66ccae0SDavid Vernet tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);
379b66ccae0SDavid Vernet if (tmp == MAP_FAILED) {
380b66ccae0SDavid Vernet err = -errno;
381b66ccae0SDavid Vernet pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n",
382b66ccae0SDavid Vernet map_fd, err);
383b66ccae0SDavid Vernet return err;
384b66ccae0SDavid Vernet }
385b66ccae0SDavid Vernet rb->consumer_pos = tmp;
386b66ccae0SDavid Vernet
387b66ccae0SDavid Vernet /* Map read-write the producer page and data pages. We map the data
388b66ccae0SDavid Vernet * region as twice the total size of the ring buffer to allow the
389b66ccae0SDavid Vernet * simple reading and writing of samples that wrap around the end of
390b66ccae0SDavid Vernet * the buffer. See the kernel implementation for details.
391b66ccae0SDavid Vernet */
39264176bffSHou Tao mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;
39364176bffSHou Tao if (mmap_sz != (__u64)(size_t)mmap_sz) {
39464176bffSHou Tao pr_warn("user ringbuf: ring buf size (%u) is too big\n", info.max_entries);
39564176bffSHou Tao return -E2BIG;
39664176bffSHou Tao }
39764176bffSHou Tao tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ | PROT_WRITE, MAP_SHARED,
39864176bffSHou Tao map_fd, rb->page_size);
399b66ccae0SDavid Vernet if (tmp == MAP_FAILED) {
400b66ccae0SDavid Vernet err = -errno;
401b66ccae0SDavid Vernet pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n",
402b66ccae0SDavid Vernet map_fd, err);
403b66ccae0SDavid Vernet return err;
404b66ccae0SDavid Vernet }
405b66ccae0SDavid Vernet
406b66ccae0SDavid Vernet rb->producer_pos = tmp;
407b66ccae0SDavid Vernet rb->data = tmp + rb->page_size;
408b66ccae0SDavid Vernet
409b66ccae0SDavid Vernet rb_epoll = &rb->event;
410b66ccae0SDavid Vernet rb_epoll->events = EPOLLOUT;
411b66ccae0SDavid Vernet if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {
412b66ccae0SDavid Vernet err = -errno;
413b66ccae0SDavid Vernet pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err);
414b66ccae0SDavid Vernet return err;
415b66ccae0SDavid Vernet }
416b66ccae0SDavid Vernet
417b66ccae0SDavid Vernet return 0;
418b66ccae0SDavid Vernet }
419b66ccae0SDavid Vernet
420b66ccae0SDavid Vernet struct user_ring_buffer *
user_ring_buffer__new(int map_fd,const struct user_ring_buffer_opts * opts)421b66ccae0SDavid Vernet user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts)
422b66ccae0SDavid Vernet {
423b66ccae0SDavid Vernet struct user_ring_buffer *rb;
424b66ccae0SDavid Vernet int err;
425b66ccae0SDavid Vernet
426b66ccae0SDavid Vernet if (!OPTS_VALID(opts, user_ring_buffer_opts))
427b66ccae0SDavid Vernet return errno = EINVAL, NULL;
428b66ccae0SDavid Vernet
429b66ccae0SDavid Vernet rb = calloc(1, sizeof(*rb));
430b66ccae0SDavid Vernet if (!rb)
431b66ccae0SDavid Vernet return errno = ENOMEM, NULL;
432b66ccae0SDavid Vernet
433b66ccae0SDavid Vernet rb->page_size = getpagesize();
434b66ccae0SDavid Vernet
435b66ccae0SDavid Vernet rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
436b66ccae0SDavid Vernet if (rb->epoll_fd < 0) {
437b66ccae0SDavid Vernet err = -errno;
438b66ccae0SDavid Vernet pr_warn("user ringbuf: failed to create epoll instance: %d\n", err);
439b66ccae0SDavid Vernet goto err_out;
440b66ccae0SDavid Vernet }
441b66ccae0SDavid Vernet
442b66ccae0SDavid Vernet err = user_ringbuf_map(rb, map_fd);
443b66ccae0SDavid Vernet if (err)
444b66ccae0SDavid Vernet goto err_out;
445b66ccae0SDavid Vernet
446b66ccae0SDavid Vernet return rb;
447b66ccae0SDavid Vernet
448b66ccae0SDavid Vernet err_out:
449b66ccae0SDavid Vernet user_ring_buffer__free(rb);
450b66ccae0SDavid Vernet return errno = -err, NULL;
451b66ccae0SDavid Vernet }
452b66ccae0SDavid Vernet
user_ringbuf_commit(struct user_ring_buffer * rb,void * sample,bool discard)453b66ccae0SDavid Vernet static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard)
454b66ccae0SDavid Vernet {
455b66ccae0SDavid Vernet __u32 new_len;
456b66ccae0SDavid Vernet struct ringbuf_hdr *hdr;
457b66ccae0SDavid Vernet uintptr_t hdr_offset;
458b66ccae0SDavid Vernet
459b66ccae0SDavid Vernet hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ;
460b66ccae0SDavid Vernet hdr = rb->data + (hdr_offset & rb->mask);
461b66ccae0SDavid Vernet
462b66ccae0SDavid Vernet new_len = hdr->len & ~BPF_RINGBUF_BUSY_BIT;
463b66ccae0SDavid Vernet if (discard)
464b66ccae0SDavid Vernet new_len |= BPF_RINGBUF_DISCARD_BIT;
465b66ccae0SDavid Vernet
466b66ccae0SDavid Vernet /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in
467b66ccae0SDavid Vernet * the kernel.
468b66ccae0SDavid Vernet */
469b66ccae0SDavid Vernet __atomic_exchange_n(&hdr->len, new_len, __ATOMIC_ACQ_REL);
470b66ccae0SDavid Vernet }
471b66ccae0SDavid Vernet
user_ring_buffer__discard(struct user_ring_buffer * rb,void * sample)472b66ccae0SDavid Vernet void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample)
473b66ccae0SDavid Vernet {
474b66ccae0SDavid Vernet user_ringbuf_commit(rb, sample, true);
475b66ccae0SDavid Vernet }
476b66ccae0SDavid Vernet
user_ring_buffer__submit(struct user_ring_buffer * rb,void * sample)477b66ccae0SDavid Vernet void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample)
478b66ccae0SDavid Vernet {
479b66ccae0SDavid Vernet user_ringbuf_commit(rb, sample, false);
480b66ccae0SDavid Vernet }
481b66ccae0SDavid Vernet
user_ring_buffer__reserve(struct user_ring_buffer * rb,__u32 size)482b66ccae0SDavid Vernet void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size)
483b66ccae0SDavid Vernet {
484b66ccae0SDavid Vernet __u32 avail_size, total_size, max_size;
485b66ccae0SDavid Vernet /* 64-bit to avoid overflow in case of extreme application behavior */
486b66ccae0SDavid Vernet __u64 cons_pos, prod_pos;
487b66ccae0SDavid Vernet struct ringbuf_hdr *hdr;
488b66ccae0SDavid Vernet
48905c1558bSHou Tao /* The top two bits are used as special flags */
49005c1558bSHou Tao if (size & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT))
49105c1558bSHou Tao return errno = E2BIG, NULL;
49205c1558bSHou Tao
493b66ccae0SDavid Vernet /* Synchronizes with smp_store_release() in __bpf_user_ringbuf_peek() in
494b66ccae0SDavid Vernet * the kernel.
495b66ccae0SDavid Vernet */
496b66ccae0SDavid Vernet cons_pos = smp_load_acquire(rb->consumer_pos);
497b66ccae0SDavid Vernet /* Synchronizes with smp_store_release() in user_ringbuf_commit() */
498b66ccae0SDavid Vernet prod_pos = smp_load_acquire(rb->producer_pos);
499b66ccae0SDavid Vernet
500b66ccae0SDavid Vernet max_size = rb->mask + 1;
501b66ccae0SDavid Vernet avail_size = max_size - (prod_pos - cons_pos);
502b66ccae0SDavid Vernet /* Round up total size to a multiple of 8. */
503b66ccae0SDavid Vernet total_size = (size + BPF_RINGBUF_HDR_SZ + 7) / 8 * 8;
504b66ccae0SDavid Vernet
505b66ccae0SDavid Vernet if (total_size > max_size)
506b66ccae0SDavid Vernet return errno = E2BIG, NULL;
507b66ccae0SDavid Vernet
508b66ccae0SDavid Vernet if (avail_size < total_size)
509b66ccae0SDavid Vernet return errno = ENOSPC, NULL;
510b66ccae0SDavid Vernet
511b66ccae0SDavid Vernet hdr = rb->data + (prod_pos & rb->mask);
512b66ccae0SDavid Vernet hdr->len = size | BPF_RINGBUF_BUSY_BIT;
513b66ccae0SDavid Vernet hdr->pad = 0;
514b66ccae0SDavid Vernet
515b66ccae0SDavid Vernet /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in
516b66ccae0SDavid Vernet * the kernel.
517b66ccae0SDavid Vernet */
518b66ccae0SDavid Vernet smp_store_release(rb->producer_pos, prod_pos + total_size);
519b66ccae0SDavid Vernet
520b66ccae0SDavid Vernet return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask);
521b66ccae0SDavid Vernet }
522b66ccae0SDavid Vernet
ns_elapsed_timespec(const struct timespec * start,const struct timespec * end)523b66ccae0SDavid Vernet static __u64 ns_elapsed_timespec(const struct timespec *start, const struct timespec *end)
524b66ccae0SDavid Vernet {
525b66ccae0SDavid Vernet __u64 start_ns, end_ns, ns_per_s = 1000000000;
526b66ccae0SDavid Vernet
527b66ccae0SDavid Vernet start_ns = (__u64)start->tv_sec * ns_per_s + start->tv_nsec;
528b66ccae0SDavid Vernet end_ns = (__u64)end->tv_sec * ns_per_s + end->tv_nsec;
529b66ccae0SDavid Vernet
530b66ccae0SDavid Vernet return end_ns - start_ns;
531b66ccae0SDavid Vernet }
532b66ccae0SDavid Vernet
user_ring_buffer__reserve_blocking(struct user_ring_buffer * rb,__u32 size,int timeout_ms)533b66ccae0SDavid Vernet void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms)
534b66ccae0SDavid Vernet {
535b66ccae0SDavid Vernet void *sample;
536b66ccae0SDavid Vernet int err, ms_remaining = timeout_ms;
537b66ccae0SDavid Vernet struct timespec start;
538b66ccae0SDavid Vernet
539b66ccae0SDavid Vernet if (timeout_ms < 0 && timeout_ms != -1)
540b66ccae0SDavid Vernet return errno = EINVAL, NULL;
541b66ccae0SDavid Vernet
542b66ccae0SDavid Vernet if (timeout_ms != -1) {
543b66ccae0SDavid Vernet err = clock_gettime(CLOCK_MONOTONIC, &start);
544b66ccae0SDavid Vernet if (err)
545b66ccae0SDavid Vernet return NULL;
546b66ccae0SDavid Vernet }
547b66ccae0SDavid Vernet
548b66ccae0SDavid Vernet do {
549b66ccae0SDavid Vernet int cnt, ms_elapsed;
550b66ccae0SDavid Vernet struct timespec curr;
551b66ccae0SDavid Vernet __u64 ns_per_ms = 1000000;
552b66ccae0SDavid Vernet
553b66ccae0SDavid Vernet sample = user_ring_buffer__reserve(rb, size);
554b66ccae0SDavid Vernet if (sample)
555b66ccae0SDavid Vernet return sample;
556b66ccae0SDavid Vernet else if (errno != ENOSPC)
557b66ccae0SDavid Vernet return NULL;
558b66ccae0SDavid Vernet
559b66ccae0SDavid Vernet /* The kernel guarantees at least one event notification
560b66ccae0SDavid Vernet * delivery whenever at least one sample is drained from the
561b66ccae0SDavid Vernet * ring buffer in an invocation to bpf_ringbuf_drain(). Other
562b66ccae0SDavid Vernet * additional events may be delivered at any time, but only one
563b66ccae0SDavid Vernet * event is guaranteed per bpf_ringbuf_drain() invocation,
564b66ccae0SDavid Vernet * provided that a sample is drained, and the BPF program did
565b66ccae0SDavid Vernet * not pass BPF_RB_NO_WAKEUP to bpf_ringbuf_drain(). If
566b66ccae0SDavid Vernet * BPF_RB_FORCE_WAKEUP is passed to bpf_ringbuf_drain(), a
567b66ccae0SDavid Vernet * wakeup event will be delivered even if no samples are
568b66ccae0SDavid Vernet * drained.
569b66ccae0SDavid Vernet */
570b66ccae0SDavid Vernet cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining);
571b66ccae0SDavid Vernet if (cnt < 0)
572b66ccae0SDavid Vernet return NULL;
573b66ccae0SDavid Vernet
574b66ccae0SDavid Vernet if (timeout_ms == -1)
575b66ccae0SDavid Vernet continue;
576b66ccae0SDavid Vernet
577b66ccae0SDavid Vernet err = clock_gettime(CLOCK_MONOTONIC, &curr);
578b66ccae0SDavid Vernet if (err)
579b66ccae0SDavid Vernet return NULL;
580b66ccae0SDavid Vernet
581b66ccae0SDavid Vernet ms_elapsed = ns_elapsed_timespec(&start, &curr) / ns_per_ms;
582b66ccae0SDavid Vernet ms_remaining = timeout_ms - ms_elapsed;
583b66ccae0SDavid Vernet } while (ms_remaining > 0);
584b66ccae0SDavid Vernet
585b66ccae0SDavid Vernet /* Try one more time to reserve a sample after the specified timeout has elapsed. */
586b66ccae0SDavid Vernet return user_ring_buffer__reserve(rb, size);
587b66ccae0SDavid Vernet }
588