1bf99c936SAndrii Nakryiko // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2bf99c936SAndrii Nakryiko /* 3bf99c936SAndrii Nakryiko * Ring buffer operations. 4bf99c936SAndrii Nakryiko * 5bf99c936SAndrii Nakryiko * Copyright (C) 2020 Facebook, Inc. 6bf99c936SAndrii Nakryiko */ 7febeb6dfSAndrii Nakryiko #ifndef _GNU_SOURCE 8febeb6dfSAndrii Nakryiko #define _GNU_SOURCE 9febeb6dfSAndrii Nakryiko #endif 10bf99c936SAndrii Nakryiko #include <stdlib.h> 11bf99c936SAndrii Nakryiko #include <stdio.h> 12bf99c936SAndrii Nakryiko #include <errno.h> 13bf99c936SAndrii Nakryiko #include <unistd.h> 14bf99c936SAndrii Nakryiko #include <linux/err.h> 15bf99c936SAndrii Nakryiko #include <linux/bpf.h> 16bf99c936SAndrii Nakryiko #include <asm/barrier.h> 17bf99c936SAndrii Nakryiko #include <sys/mman.h> 18bf99c936SAndrii Nakryiko #include <sys/epoll.h> 19bf99c936SAndrii Nakryiko #include <tools/libc_compat.h> 20bf99c936SAndrii Nakryiko 21bf99c936SAndrii Nakryiko #include "libbpf.h" 22bf99c936SAndrii Nakryiko #include "libbpf_internal.h" 23bf99c936SAndrii Nakryiko #include "bpf.h" 24bf99c936SAndrii Nakryiko 25bf99c936SAndrii Nakryiko /* make sure libbpf doesn't use kernel-only integer typedefs */ 26bf99c936SAndrii Nakryiko #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 27bf99c936SAndrii Nakryiko 28bf99c936SAndrii Nakryiko struct ring { 29bf99c936SAndrii Nakryiko ring_buffer_sample_fn sample_cb; 30bf99c936SAndrii Nakryiko void *ctx; 31bf99c936SAndrii Nakryiko void *data; 32bf99c936SAndrii Nakryiko unsigned long *consumer_pos; 33bf99c936SAndrii Nakryiko unsigned long *producer_pos; 34bf99c936SAndrii Nakryiko unsigned long mask; 35bf99c936SAndrii Nakryiko int map_fd; 36bf99c936SAndrii Nakryiko }; 37bf99c936SAndrii Nakryiko 38bf99c936SAndrii Nakryiko struct ring_buffer { 39bf99c936SAndrii Nakryiko struct epoll_event *events; 40bf99c936SAndrii Nakryiko struct ring *rings; 41bf99c936SAndrii Nakryiko size_t page_size; 42bf99c936SAndrii Nakryiko int epoll_fd; 43bf99c936SAndrii Nakryiko int ring_cnt; 44bf99c936SAndrii Nakryiko }; 45bf99c936SAndrii Nakryiko 46bf99c936SAndrii Nakryiko static void ringbuf_unmap_ring(struct ring_buffer *rb, struct ring *r) 47bf99c936SAndrii Nakryiko { 48bf99c936SAndrii Nakryiko if (r->consumer_pos) { 49bf99c936SAndrii Nakryiko munmap(r->consumer_pos, rb->page_size); 50bf99c936SAndrii Nakryiko r->consumer_pos = NULL; 51bf99c936SAndrii Nakryiko } 52bf99c936SAndrii Nakryiko if (r->producer_pos) { 53bf99c936SAndrii Nakryiko munmap(r->producer_pos, rb->page_size + 2 * (r->mask + 1)); 54bf99c936SAndrii Nakryiko r->producer_pos = NULL; 55bf99c936SAndrii Nakryiko } 56bf99c936SAndrii Nakryiko } 57bf99c936SAndrii Nakryiko 58bf99c936SAndrii Nakryiko /* Add extra RINGBUF maps to this ring buffer manager */ 59bf99c936SAndrii Nakryiko int ring_buffer__add(struct ring_buffer *rb, int map_fd, 60bf99c936SAndrii Nakryiko ring_buffer_sample_fn sample_cb, void *ctx) 61bf99c936SAndrii Nakryiko { 62bf99c936SAndrii Nakryiko struct bpf_map_info info; 63bf99c936SAndrii Nakryiko __u32 len = sizeof(info); 64bf99c936SAndrii Nakryiko struct epoll_event *e; 65bf99c936SAndrii Nakryiko struct ring *r; 66bf99c936SAndrii Nakryiko void *tmp; 67bf99c936SAndrii Nakryiko int err; 68bf99c936SAndrii Nakryiko 69bf99c936SAndrii Nakryiko memset(&info, 0, sizeof(info)); 70bf99c936SAndrii Nakryiko 71bf99c936SAndrii Nakryiko err = bpf_obj_get_info_by_fd(map_fd, &info, &len); 72bf99c936SAndrii Nakryiko if (err) { 73bf99c936SAndrii Nakryiko err = -errno; 74bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to get map info for fd=%d: %d\n", 75bf99c936SAndrii Nakryiko map_fd, err); 76bf99c936SAndrii Nakryiko return err; 77bf99c936SAndrii Nakryiko } 78bf99c936SAndrii Nakryiko 79bf99c936SAndrii Nakryiko if (info.type != BPF_MAP_TYPE_RINGBUF) { 80bf99c936SAndrii Nakryiko pr_warn("ringbuf: map fd=%d is not BPF_MAP_TYPE_RINGBUF\n", 81bf99c936SAndrii Nakryiko map_fd); 82bf99c936SAndrii Nakryiko return -EINVAL; 83bf99c936SAndrii Nakryiko } 84bf99c936SAndrii Nakryiko 85bf99c936SAndrii Nakryiko tmp = reallocarray(rb->rings, rb->ring_cnt + 1, sizeof(*rb->rings)); 86bf99c936SAndrii Nakryiko if (!tmp) 87bf99c936SAndrii Nakryiko return -ENOMEM; 88bf99c936SAndrii Nakryiko rb->rings = tmp; 89bf99c936SAndrii Nakryiko 90bf99c936SAndrii Nakryiko tmp = reallocarray(rb->events, rb->ring_cnt + 1, sizeof(*rb->events)); 91bf99c936SAndrii Nakryiko if (!tmp) 92bf99c936SAndrii Nakryiko return -ENOMEM; 93bf99c936SAndrii Nakryiko rb->events = tmp; 94bf99c936SAndrii Nakryiko 95bf99c936SAndrii Nakryiko r = &rb->rings[rb->ring_cnt]; 96bf99c936SAndrii Nakryiko memset(r, 0, sizeof(*r)); 97bf99c936SAndrii Nakryiko 98bf99c936SAndrii Nakryiko r->map_fd = map_fd; 99bf99c936SAndrii Nakryiko r->sample_cb = sample_cb; 100bf99c936SAndrii Nakryiko r->ctx = ctx; 101bf99c936SAndrii Nakryiko r->mask = info.max_entries - 1; 102bf99c936SAndrii Nakryiko 103bf99c936SAndrii Nakryiko /* Map writable consumer page */ 104bf99c936SAndrii Nakryiko tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, 105bf99c936SAndrii Nakryiko map_fd, 0); 106bf99c936SAndrii Nakryiko if (tmp == MAP_FAILED) { 107bf99c936SAndrii Nakryiko err = -errno; 108bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %d\n", 109bf99c936SAndrii Nakryiko map_fd, err); 110bf99c936SAndrii Nakryiko return err; 111bf99c936SAndrii Nakryiko } 112bf99c936SAndrii Nakryiko r->consumer_pos = tmp; 113bf99c936SAndrii Nakryiko 114bf99c936SAndrii Nakryiko /* Map read-only producer page and data pages. We map twice as big 115bf99c936SAndrii Nakryiko * data size to allow simple reading of samples that wrap around the 116bf99c936SAndrii Nakryiko * end of a ring buffer. See kernel implementation for details. 117bf99c936SAndrii Nakryiko * */ 118bf99c936SAndrii Nakryiko tmp = mmap(NULL, rb->page_size + 2 * info.max_entries, PROT_READ, 119bf99c936SAndrii Nakryiko MAP_SHARED, map_fd, rb->page_size); 120bf99c936SAndrii Nakryiko if (tmp == MAP_FAILED) { 121bf99c936SAndrii Nakryiko err = -errno; 122bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, r); 123bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %d\n", 124bf99c936SAndrii Nakryiko map_fd, err); 125bf99c936SAndrii Nakryiko return err; 126bf99c936SAndrii Nakryiko } 127bf99c936SAndrii Nakryiko r->producer_pos = tmp; 128bf99c936SAndrii Nakryiko r->data = tmp + rb->page_size; 129bf99c936SAndrii Nakryiko 130bf99c936SAndrii Nakryiko e = &rb->events[rb->ring_cnt]; 131bf99c936SAndrii Nakryiko memset(e, 0, sizeof(*e)); 132bf99c936SAndrii Nakryiko 133bf99c936SAndrii Nakryiko e->events = EPOLLIN; 134bf99c936SAndrii Nakryiko e->data.fd = rb->ring_cnt; 135bf99c936SAndrii Nakryiko if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) { 136bf99c936SAndrii Nakryiko err = -errno; 137bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, r); 138bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to epoll add map fd=%d: %d\n", 139bf99c936SAndrii Nakryiko map_fd, err); 140bf99c936SAndrii Nakryiko return err; 141bf99c936SAndrii Nakryiko } 142bf99c936SAndrii Nakryiko 143bf99c936SAndrii Nakryiko rb->ring_cnt++; 144bf99c936SAndrii Nakryiko return 0; 145bf99c936SAndrii Nakryiko } 146bf99c936SAndrii Nakryiko 147bf99c936SAndrii Nakryiko void ring_buffer__free(struct ring_buffer *rb) 148bf99c936SAndrii Nakryiko { 149bf99c936SAndrii Nakryiko int i; 150bf99c936SAndrii Nakryiko 151bf99c936SAndrii Nakryiko if (!rb) 152bf99c936SAndrii Nakryiko return; 153bf99c936SAndrii Nakryiko 154bf99c936SAndrii Nakryiko for (i = 0; i < rb->ring_cnt; ++i) 155bf99c936SAndrii Nakryiko ringbuf_unmap_ring(rb, &rb->rings[i]); 156bf99c936SAndrii Nakryiko if (rb->epoll_fd >= 0) 157bf99c936SAndrii Nakryiko close(rb->epoll_fd); 158bf99c936SAndrii Nakryiko 159bf99c936SAndrii Nakryiko free(rb->events); 160bf99c936SAndrii Nakryiko free(rb->rings); 161bf99c936SAndrii Nakryiko free(rb); 162bf99c936SAndrii Nakryiko } 163bf99c936SAndrii Nakryiko 164bf99c936SAndrii Nakryiko struct ring_buffer * 165bf99c936SAndrii Nakryiko ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, 166bf99c936SAndrii Nakryiko const struct ring_buffer_opts *opts) 167bf99c936SAndrii Nakryiko { 168bf99c936SAndrii Nakryiko struct ring_buffer *rb; 169bf99c936SAndrii Nakryiko int err; 170bf99c936SAndrii Nakryiko 171bf99c936SAndrii Nakryiko if (!OPTS_VALID(opts, ring_buffer_opts)) 172bf99c936SAndrii Nakryiko return NULL; 173bf99c936SAndrii Nakryiko 174bf99c936SAndrii Nakryiko rb = calloc(1, sizeof(*rb)); 175bf99c936SAndrii Nakryiko if (!rb) 176bf99c936SAndrii Nakryiko return NULL; 177bf99c936SAndrii Nakryiko 178bf99c936SAndrii Nakryiko rb->page_size = getpagesize(); 179bf99c936SAndrii Nakryiko 180bf99c936SAndrii Nakryiko rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 181bf99c936SAndrii Nakryiko if (rb->epoll_fd < 0) { 182bf99c936SAndrii Nakryiko err = -errno; 183bf99c936SAndrii Nakryiko pr_warn("ringbuf: failed to create epoll instance: %d\n", err); 184bf99c936SAndrii Nakryiko goto err_out; 185bf99c936SAndrii Nakryiko } 186bf99c936SAndrii Nakryiko 187bf99c936SAndrii Nakryiko err = ring_buffer__add(rb, map_fd, sample_cb, ctx); 188bf99c936SAndrii Nakryiko if (err) 189bf99c936SAndrii Nakryiko goto err_out; 190bf99c936SAndrii Nakryiko 191bf99c936SAndrii Nakryiko return rb; 192bf99c936SAndrii Nakryiko 193bf99c936SAndrii Nakryiko err_out: 194bf99c936SAndrii Nakryiko ring_buffer__free(rb); 195bf99c936SAndrii Nakryiko return NULL; 196bf99c936SAndrii Nakryiko } 197bf99c936SAndrii Nakryiko 198bf99c936SAndrii Nakryiko static inline int roundup_len(__u32 len) 199bf99c936SAndrii Nakryiko { 200bf99c936SAndrii Nakryiko /* clear out top 2 bits (discard and busy, if set) */ 201bf99c936SAndrii Nakryiko len <<= 2; 202bf99c936SAndrii Nakryiko len >>= 2; 203bf99c936SAndrii Nakryiko /* add length prefix */ 204bf99c936SAndrii Nakryiko len += BPF_RINGBUF_HDR_SZ; 205bf99c936SAndrii Nakryiko /* round up to 8 byte alignment */ 206bf99c936SAndrii Nakryiko return (len + 7) / 8 * 8; 207bf99c936SAndrii Nakryiko } 208bf99c936SAndrii Nakryiko 209bf99c936SAndrii Nakryiko static int ringbuf_process_ring(struct ring* r) 210bf99c936SAndrii Nakryiko { 211bf99c936SAndrii Nakryiko int *len_ptr, len, err, cnt = 0; 212bf99c936SAndrii Nakryiko unsigned long cons_pos, prod_pos; 213bf99c936SAndrii Nakryiko bool got_new_data; 214bf99c936SAndrii Nakryiko void *sample; 215bf99c936SAndrii Nakryiko 216bf99c936SAndrii Nakryiko cons_pos = smp_load_acquire(r->consumer_pos); 217bf99c936SAndrii Nakryiko do { 218bf99c936SAndrii Nakryiko got_new_data = false; 219bf99c936SAndrii Nakryiko prod_pos = smp_load_acquire(r->producer_pos); 220bf99c936SAndrii Nakryiko while (cons_pos < prod_pos) { 221bf99c936SAndrii Nakryiko len_ptr = r->data + (cons_pos & r->mask); 222bf99c936SAndrii Nakryiko len = smp_load_acquire(len_ptr); 223bf99c936SAndrii Nakryiko 224bf99c936SAndrii Nakryiko /* sample not committed yet, bail out for now */ 225bf99c936SAndrii Nakryiko if (len & BPF_RINGBUF_BUSY_BIT) 226bf99c936SAndrii Nakryiko goto done; 227bf99c936SAndrii Nakryiko 228bf99c936SAndrii Nakryiko got_new_data = true; 229bf99c936SAndrii Nakryiko cons_pos += roundup_len(len); 230bf99c936SAndrii Nakryiko 231bf99c936SAndrii Nakryiko if ((len & BPF_RINGBUF_DISCARD_BIT) == 0) { 232bf99c936SAndrii Nakryiko sample = (void *)len_ptr + BPF_RINGBUF_HDR_SZ; 233bf99c936SAndrii Nakryiko err = r->sample_cb(r->ctx, sample, len); 234bf99c936SAndrii Nakryiko if (err) { 235bf99c936SAndrii Nakryiko /* update consumer pos and bail out */ 236bf99c936SAndrii Nakryiko smp_store_release(r->consumer_pos, 237bf99c936SAndrii Nakryiko cons_pos); 238bf99c936SAndrii Nakryiko return err; 239bf99c936SAndrii Nakryiko } 240bf99c936SAndrii Nakryiko cnt++; 241bf99c936SAndrii Nakryiko } 242bf99c936SAndrii Nakryiko 243bf99c936SAndrii Nakryiko smp_store_release(r->consumer_pos, cons_pos); 244bf99c936SAndrii Nakryiko } 245bf99c936SAndrii Nakryiko } while (got_new_data); 246bf99c936SAndrii Nakryiko done: 247bf99c936SAndrii Nakryiko return cnt; 248bf99c936SAndrii Nakryiko } 249bf99c936SAndrii Nakryiko 250bf99c936SAndrii Nakryiko /* Consume available ring buffer(s) data without event polling. 251bf99c936SAndrii Nakryiko * Returns number of records consumed across all registered ring buffers, or 252bf99c936SAndrii Nakryiko * negative number if any of the callbacks return error. 253bf99c936SAndrii Nakryiko */ 254bf99c936SAndrii Nakryiko int ring_buffer__consume(struct ring_buffer *rb) 255bf99c936SAndrii Nakryiko { 256bf99c936SAndrii Nakryiko int i, err, res = 0; 257bf99c936SAndrii Nakryiko 258bf99c936SAndrii Nakryiko for (i = 0; i < rb->ring_cnt; i++) { 259bf99c936SAndrii Nakryiko struct ring *ring = &rb->rings[i]; 260bf99c936SAndrii Nakryiko 261bf99c936SAndrii Nakryiko err = ringbuf_process_ring(ring); 262bf99c936SAndrii Nakryiko if (err < 0) 263bf99c936SAndrii Nakryiko return err; 264bf99c936SAndrii Nakryiko res += err; 265bf99c936SAndrii Nakryiko } 266bf99c936SAndrii Nakryiko return res; 267bf99c936SAndrii Nakryiko } 268bf99c936SAndrii Nakryiko 269bf99c936SAndrii Nakryiko /* Poll for available data and consume records, if any are available. 270bf99c936SAndrii Nakryiko * Returns number of records consumed, or negative number, if any of the 271bf99c936SAndrii Nakryiko * registered callbacks returned error. 272bf99c936SAndrii Nakryiko */ 273bf99c936SAndrii Nakryiko int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms) 274bf99c936SAndrii Nakryiko { 275bf99c936SAndrii Nakryiko int i, cnt, err, res = 0; 276bf99c936SAndrii Nakryiko 277bf99c936SAndrii Nakryiko cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms); 278bf99c936SAndrii Nakryiko for (i = 0; i < cnt; i++) { 279bf99c936SAndrii Nakryiko __u32 ring_id = rb->events[i].data.fd; 280bf99c936SAndrii Nakryiko struct ring *ring = &rb->rings[ring_id]; 281bf99c936SAndrii Nakryiko 282bf99c936SAndrii Nakryiko err = ringbuf_process_ring(ring); 283bf99c936SAndrii Nakryiko if (err < 0) 284bf99c936SAndrii Nakryiko return err; 285bf99c936SAndrii Nakryiko res += cnt; 286bf99c936SAndrii Nakryiko } 287bf99c936SAndrii Nakryiko return cnt < 0 ? -errno : res; 288bf99c936SAndrii Nakryiko } 289