1457f4436SAndrii Nakryiko #include <linux/bpf.h> 2457f4436SAndrii Nakryiko #include <linux/btf.h> 3457f4436SAndrii Nakryiko #include <linux/err.h> 4457f4436SAndrii Nakryiko #include <linux/irq_work.h> 5457f4436SAndrii Nakryiko #include <linux/slab.h> 6457f4436SAndrii Nakryiko #include <linux/filter.h> 7457f4436SAndrii Nakryiko #include <linux/mm.h> 8457f4436SAndrii Nakryiko #include <linux/vmalloc.h> 9457f4436SAndrii Nakryiko #include <linux/wait.h> 10457f4436SAndrii Nakryiko #include <linux/poll.h> 11ccff81e1SRustam Kovhaev #include <linux/kmemleak.h> 12457f4436SAndrii Nakryiko #include <uapi/linux/btf.h> 13c317ab71SMenglong Dong #include <linux/btf_ids.h> 14457f4436SAndrii Nakryiko 15457f4436SAndrii Nakryiko #define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE) 16457f4436SAndrii Nakryiko 17457f4436SAndrii Nakryiko /* non-mmap()'able part of bpf_ringbuf (everything up to consumer page) */ 18457f4436SAndrii Nakryiko #define RINGBUF_PGOFF \ 19457f4436SAndrii Nakryiko (offsetof(struct bpf_ringbuf, consumer_pos) >> PAGE_SHIFT) 20457f4436SAndrii Nakryiko /* consumer page and producer page */ 21457f4436SAndrii Nakryiko #define RINGBUF_POS_PAGES 2 22457f4436SAndrii Nakryiko 23457f4436SAndrii Nakryiko #define RINGBUF_MAX_RECORD_SZ (UINT_MAX/4) 24457f4436SAndrii Nakryiko 25457f4436SAndrii Nakryiko /* Maximum size of ring buffer area is limited by 32-bit page offset within 26457f4436SAndrii Nakryiko * record header, counted in pages. Reserve 8 bits for extensibility, and take 27457f4436SAndrii Nakryiko * into account few extra pages for consumer/producer pages and 28457f4436SAndrii Nakryiko * non-mmap()'able parts. This gives 64GB limit, which seems plenty for single 29457f4436SAndrii Nakryiko * ring buffer. 30457f4436SAndrii Nakryiko */ 31457f4436SAndrii Nakryiko #define RINGBUF_MAX_DATA_SZ \ 32457f4436SAndrii Nakryiko (((1ULL << 24) - RINGBUF_POS_PAGES - RINGBUF_PGOFF) * PAGE_SIZE) 33457f4436SAndrii Nakryiko 34457f4436SAndrii Nakryiko struct bpf_ringbuf { 35457f4436SAndrii Nakryiko wait_queue_head_t waitq; 36457f4436SAndrii Nakryiko struct irq_work work; 37457f4436SAndrii Nakryiko u64 mask; 38457f4436SAndrii Nakryiko struct page **pages; 39457f4436SAndrii Nakryiko int nr_pages; 40457f4436SAndrii Nakryiko spinlock_t spinlock ____cacheline_aligned_in_smp; 41*583c1f42SDavid Vernet /* Consumer and producer counters are put into separate pages to 42*583c1f42SDavid Vernet * allow each position to be mapped with different permissions. 43*583c1f42SDavid Vernet * This prevents a user-space application from modifying the 44*583c1f42SDavid Vernet * position and ruining in-kernel tracking. The permissions of the 45*583c1f42SDavid Vernet * pages depend on who is producing samples: user-space or the 46*583c1f42SDavid Vernet * kernel. 47*583c1f42SDavid Vernet * 48*583c1f42SDavid Vernet * Kernel-producer 49*583c1f42SDavid Vernet * --------------- 50*583c1f42SDavid Vernet * The producer position and data pages are mapped as r/o in 51*583c1f42SDavid Vernet * userspace. For this approach, bits in the header of samples are 52*583c1f42SDavid Vernet * used to signal to user-space, and to other producers, whether a 53*583c1f42SDavid Vernet * sample is currently being written. 54*583c1f42SDavid Vernet * 55*583c1f42SDavid Vernet * User-space producer 56*583c1f42SDavid Vernet * ------------------- 57*583c1f42SDavid Vernet * Only the page containing the consumer position is mapped r/o in 58*583c1f42SDavid Vernet * user-space. User-space producers also use bits of the header to 59*583c1f42SDavid Vernet * communicate to the kernel, but the kernel must carefully check and 60*583c1f42SDavid Vernet * validate each sample to ensure that they're correctly formatted, and 61*583c1f42SDavid Vernet * fully contained within the ring buffer. 62457f4436SAndrii Nakryiko */ 63457f4436SAndrii Nakryiko unsigned long consumer_pos __aligned(PAGE_SIZE); 64457f4436SAndrii Nakryiko unsigned long producer_pos __aligned(PAGE_SIZE); 65457f4436SAndrii Nakryiko char data[] __aligned(PAGE_SIZE); 66457f4436SAndrii Nakryiko }; 67457f4436SAndrii Nakryiko 68457f4436SAndrii Nakryiko struct bpf_ringbuf_map { 69457f4436SAndrii Nakryiko struct bpf_map map; 70457f4436SAndrii Nakryiko struct bpf_ringbuf *rb; 71457f4436SAndrii Nakryiko }; 72457f4436SAndrii Nakryiko 73457f4436SAndrii Nakryiko /* 8-byte ring buffer record header structure */ 74457f4436SAndrii Nakryiko struct bpf_ringbuf_hdr { 75457f4436SAndrii Nakryiko u32 len; 76457f4436SAndrii Nakryiko u32 pg_off; 77457f4436SAndrii Nakryiko }; 78457f4436SAndrii Nakryiko 79457f4436SAndrii Nakryiko static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node) 80457f4436SAndrii Nakryiko { 81be4035c7SRoman Gushchin const gfp_t flags = GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL | 82be4035c7SRoman Gushchin __GFP_NOWARN | __GFP_ZERO; 83457f4436SAndrii Nakryiko int nr_meta_pages = RINGBUF_PGOFF + RINGBUF_POS_PAGES; 84457f4436SAndrii Nakryiko int nr_data_pages = data_sz >> PAGE_SHIFT; 85457f4436SAndrii Nakryiko int nr_pages = nr_meta_pages + nr_data_pages; 86457f4436SAndrii Nakryiko struct page **pages, *page; 87457f4436SAndrii Nakryiko struct bpf_ringbuf *rb; 88457f4436SAndrii Nakryiko size_t array_size; 89457f4436SAndrii Nakryiko int i; 90457f4436SAndrii Nakryiko 91457f4436SAndrii Nakryiko /* Each data page is mapped twice to allow "virtual" 92457f4436SAndrii Nakryiko * continuous read of samples wrapping around the end of ring 93457f4436SAndrii Nakryiko * buffer area: 94457f4436SAndrii Nakryiko * ------------------------------------------------------ 95457f4436SAndrii Nakryiko * | meta pages | real data pages | same data pages | 96457f4436SAndrii Nakryiko * ------------------------------------------------------ 97457f4436SAndrii Nakryiko * | | 1 2 3 4 5 6 7 8 9 | 1 2 3 4 5 6 7 8 9 | 98457f4436SAndrii Nakryiko * ------------------------------------------------------ 99457f4436SAndrii Nakryiko * | | TA DA | TA DA | 100457f4436SAndrii Nakryiko * ------------------------------------------------------ 101457f4436SAndrii Nakryiko * ^^^^^^^ 102457f4436SAndrii Nakryiko * | 103457f4436SAndrii Nakryiko * Here, no need to worry about special handling of wrapped-around 104457f4436SAndrii Nakryiko * data due to double-mapped data pages. This works both in kernel and 105457f4436SAndrii Nakryiko * when mmap()'ed in user-space, simplifying both kernel and 106457f4436SAndrii Nakryiko * user-space implementations significantly. 107457f4436SAndrii Nakryiko */ 108457f4436SAndrii Nakryiko array_size = (nr_meta_pages + 2 * nr_data_pages) * sizeof(*pages); 109be4035c7SRoman Gushchin pages = bpf_map_area_alloc(array_size, numa_node); 110457f4436SAndrii Nakryiko if (!pages) 111457f4436SAndrii Nakryiko return NULL; 112457f4436SAndrii Nakryiko 113457f4436SAndrii Nakryiko for (i = 0; i < nr_pages; i++) { 114457f4436SAndrii Nakryiko page = alloc_pages_node(numa_node, flags, 0); 115457f4436SAndrii Nakryiko if (!page) { 116457f4436SAndrii Nakryiko nr_pages = i; 117457f4436SAndrii Nakryiko goto err_free_pages; 118457f4436SAndrii Nakryiko } 119457f4436SAndrii Nakryiko pages[i] = page; 120457f4436SAndrii Nakryiko if (i >= nr_meta_pages) 121457f4436SAndrii Nakryiko pages[nr_data_pages + i] = page; 122457f4436SAndrii Nakryiko } 123457f4436SAndrii Nakryiko 124457f4436SAndrii Nakryiko rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages, 125b293dcc4SHou Tao VM_MAP | VM_USERMAP, PAGE_KERNEL); 126457f4436SAndrii Nakryiko if (rb) { 127ccff81e1SRustam Kovhaev kmemleak_not_leak(pages); 128457f4436SAndrii Nakryiko rb->pages = pages; 129457f4436SAndrii Nakryiko rb->nr_pages = nr_pages; 130457f4436SAndrii Nakryiko return rb; 131457f4436SAndrii Nakryiko } 132457f4436SAndrii Nakryiko 133457f4436SAndrii Nakryiko err_free_pages: 134457f4436SAndrii Nakryiko for (i = 0; i < nr_pages; i++) 135457f4436SAndrii Nakryiko __free_page(pages[i]); 1368f58ee54SYafang Shao bpf_map_area_free(pages); 137457f4436SAndrii Nakryiko return NULL; 138457f4436SAndrii Nakryiko } 139457f4436SAndrii Nakryiko 140457f4436SAndrii Nakryiko static void bpf_ringbuf_notify(struct irq_work *work) 141457f4436SAndrii Nakryiko { 142457f4436SAndrii Nakryiko struct bpf_ringbuf *rb = container_of(work, struct bpf_ringbuf, work); 143457f4436SAndrii Nakryiko 144457f4436SAndrii Nakryiko wake_up_all(&rb->waitq); 145457f4436SAndrii Nakryiko } 146457f4436SAndrii Nakryiko 147457f4436SAndrii Nakryiko static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node) 148457f4436SAndrii Nakryiko { 149457f4436SAndrii Nakryiko struct bpf_ringbuf *rb; 150457f4436SAndrii Nakryiko 151457f4436SAndrii Nakryiko rb = bpf_ringbuf_area_alloc(data_sz, numa_node); 152457f4436SAndrii Nakryiko if (!rb) 153abbdd081SRoman Gushchin return NULL; 154457f4436SAndrii Nakryiko 155457f4436SAndrii Nakryiko spin_lock_init(&rb->spinlock); 156457f4436SAndrii Nakryiko init_waitqueue_head(&rb->waitq); 157457f4436SAndrii Nakryiko init_irq_work(&rb->work, bpf_ringbuf_notify); 158457f4436SAndrii Nakryiko 159457f4436SAndrii Nakryiko rb->mask = data_sz - 1; 160457f4436SAndrii Nakryiko rb->consumer_pos = 0; 161457f4436SAndrii Nakryiko rb->producer_pos = 0; 162457f4436SAndrii Nakryiko 163457f4436SAndrii Nakryiko return rb; 164457f4436SAndrii Nakryiko } 165457f4436SAndrii Nakryiko 166457f4436SAndrii Nakryiko static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr) 167457f4436SAndrii Nakryiko { 168457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 169457f4436SAndrii Nakryiko 170457f4436SAndrii Nakryiko if (attr->map_flags & ~RINGBUF_CREATE_FLAG_MASK) 171457f4436SAndrii Nakryiko return ERR_PTR(-EINVAL); 172457f4436SAndrii Nakryiko 173457f4436SAndrii Nakryiko if (attr->key_size || attr->value_size || 174517bbe19SAndrii Nakryiko !is_power_of_2(attr->max_entries) || 175517bbe19SAndrii Nakryiko !PAGE_ALIGNED(attr->max_entries)) 176457f4436SAndrii Nakryiko return ERR_PTR(-EINVAL); 177457f4436SAndrii Nakryiko 178517bbe19SAndrii Nakryiko #ifdef CONFIG_64BIT 179517bbe19SAndrii Nakryiko /* on 32-bit arch, it's impossible to overflow record's hdr->pgoff */ 180517bbe19SAndrii Nakryiko if (attr->max_entries > RINGBUF_MAX_DATA_SZ) 181517bbe19SAndrii Nakryiko return ERR_PTR(-E2BIG); 182517bbe19SAndrii Nakryiko #endif 183517bbe19SAndrii Nakryiko 18473cf09a3SYafang Shao rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE); 185457f4436SAndrii Nakryiko if (!rb_map) 186457f4436SAndrii Nakryiko return ERR_PTR(-ENOMEM); 187457f4436SAndrii Nakryiko 188457f4436SAndrii Nakryiko bpf_map_init_from_attr(&rb_map->map, attr); 189457f4436SAndrii Nakryiko 190457f4436SAndrii Nakryiko rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node); 191abbdd081SRoman Gushchin if (!rb_map->rb) { 19273cf09a3SYafang Shao bpf_map_area_free(rb_map); 193abbdd081SRoman Gushchin return ERR_PTR(-ENOMEM); 194457f4436SAndrii Nakryiko } 195457f4436SAndrii Nakryiko 196457f4436SAndrii Nakryiko return &rb_map->map; 197457f4436SAndrii Nakryiko } 198457f4436SAndrii Nakryiko 199457f4436SAndrii Nakryiko static void bpf_ringbuf_free(struct bpf_ringbuf *rb) 200457f4436SAndrii Nakryiko { 201457f4436SAndrii Nakryiko /* copy pages pointer and nr_pages to local variable, as we are going 202457f4436SAndrii Nakryiko * to unmap rb itself with vunmap() below 203457f4436SAndrii Nakryiko */ 204457f4436SAndrii Nakryiko struct page **pages = rb->pages; 205457f4436SAndrii Nakryiko int i, nr_pages = rb->nr_pages; 206457f4436SAndrii Nakryiko 207457f4436SAndrii Nakryiko vunmap(rb); 208457f4436SAndrii Nakryiko for (i = 0; i < nr_pages; i++) 209457f4436SAndrii Nakryiko __free_page(pages[i]); 2108f58ee54SYafang Shao bpf_map_area_free(pages); 211457f4436SAndrii Nakryiko } 212457f4436SAndrii Nakryiko 213457f4436SAndrii Nakryiko static void ringbuf_map_free(struct bpf_map *map) 214457f4436SAndrii Nakryiko { 215457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 216457f4436SAndrii Nakryiko 217457f4436SAndrii Nakryiko rb_map = container_of(map, struct bpf_ringbuf_map, map); 218457f4436SAndrii Nakryiko bpf_ringbuf_free(rb_map->rb); 21973cf09a3SYafang Shao bpf_map_area_free(rb_map); 220457f4436SAndrii Nakryiko } 221457f4436SAndrii Nakryiko 222457f4436SAndrii Nakryiko static void *ringbuf_map_lookup_elem(struct bpf_map *map, void *key) 223457f4436SAndrii Nakryiko { 224457f4436SAndrii Nakryiko return ERR_PTR(-ENOTSUPP); 225457f4436SAndrii Nakryiko } 226457f4436SAndrii Nakryiko 227457f4436SAndrii Nakryiko static int ringbuf_map_update_elem(struct bpf_map *map, void *key, void *value, 228457f4436SAndrii Nakryiko u64 flags) 229457f4436SAndrii Nakryiko { 230457f4436SAndrii Nakryiko return -ENOTSUPP; 231457f4436SAndrii Nakryiko } 232457f4436SAndrii Nakryiko 233457f4436SAndrii Nakryiko static int ringbuf_map_delete_elem(struct bpf_map *map, void *key) 234457f4436SAndrii Nakryiko { 235457f4436SAndrii Nakryiko return -ENOTSUPP; 236457f4436SAndrii Nakryiko } 237457f4436SAndrii Nakryiko 238457f4436SAndrii Nakryiko static int ringbuf_map_get_next_key(struct bpf_map *map, void *key, 239457f4436SAndrii Nakryiko void *next_key) 240457f4436SAndrii Nakryiko { 241457f4436SAndrii Nakryiko return -ENOTSUPP; 242457f4436SAndrii Nakryiko } 243457f4436SAndrii Nakryiko 244*583c1f42SDavid Vernet static int ringbuf_map_mmap_kern(struct bpf_map *map, struct vm_area_struct *vma) 245457f4436SAndrii Nakryiko { 246457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 247457f4436SAndrii Nakryiko 248457f4436SAndrii Nakryiko rb_map = container_of(map, struct bpf_ringbuf_map, map); 249457f4436SAndrii Nakryiko 25004ea3086SAndrii Nakryiko if (vma->vm_flags & VM_WRITE) { 25104ea3086SAndrii Nakryiko /* allow writable mapping for the consumer_pos only */ 25204ea3086SAndrii Nakryiko if (vma->vm_pgoff != 0 || vma->vm_end - vma->vm_start != PAGE_SIZE) 25304ea3086SAndrii Nakryiko return -EPERM; 25404ea3086SAndrii Nakryiko } else { 25504ea3086SAndrii Nakryiko vma->vm_flags &= ~VM_MAYWRITE; 25604ea3086SAndrii Nakryiko } 25704ea3086SAndrii Nakryiko /* remap_vmalloc_range() checks size and offset constraints */ 258457f4436SAndrii Nakryiko return remap_vmalloc_range(vma, rb_map->rb, 259457f4436SAndrii Nakryiko vma->vm_pgoff + RINGBUF_PGOFF); 260457f4436SAndrii Nakryiko } 261457f4436SAndrii Nakryiko 262*583c1f42SDavid Vernet static int ringbuf_map_mmap_user(struct bpf_map *map, struct vm_area_struct *vma) 263*583c1f42SDavid Vernet { 264*583c1f42SDavid Vernet struct bpf_ringbuf_map *rb_map; 265*583c1f42SDavid Vernet 266*583c1f42SDavid Vernet rb_map = container_of(map, struct bpf_ringbuf_map, map); 267*583c1f42SDavid Vernet 268*583c1f42SDavid Vernet if (vma->vm_flags & VM_WRITE) { 269*583c1f42SDavid Vernet if (vma->vm_pgoff == 0) 270*583c1f42SDavid Vernet /* Disallow writable mappings to the consumer pointer, 271*583c1f42SDavid Vernet * and allow writable mappings to both the producer 272*583c1f42SDavid Vernet * position, and the ring buffer data itself. 273*583c1f42SDavid Vernet */ 274*583c1f42SDavid Vernet return -EPERM; 275*583c1f42SDavid Vernet } else { 276*583c1f42SDavid Vernet vma->vm_flags &= ~VM_MAYWRITE; 277*583c1f42SDavid Vernet } 278*583c1f42SDavid Vernet /* remap_vmalloc_range() checks size and offset constraints */ 279*583c1f42SDavid Vernet return remap_vmalloc_range(vma, rb_map->rb, vma->vm_pgoff + RINGBUF_PGOFF); 280*583c1f42SDavid Vernet } 281*583c1f42SDavid Vernet 282457f4436SAndrii Nakryiko static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) 283457f4436SAndrii Nakryiko { 284457f4436SAndrii Nakryiko unsigned long cons_pos, prod_pos; 285457f4436SAndrii Nakryiko 286457f4436SAndrii Nakryiko cons_pos = smp_load_acquire(&rb->consumer_pos); 287457f4436SAndrii Nakryiko prod_pos = smp_load_acquire(&rb->producer_pos); 288457f4436SAndrii Nakryiko return prod_pos - cons_pos; 289457f4436SAndrii Nakryiko } 290457f4436SAndrii Nakryiko 291457f4436SAndrii Nakryiko static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp, 292457f4436SAndrii Nakryiko struct poll_table_struct *pts) 293457f4436SAndrii Nakryiko { 294457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 295457f4436SAndrii Nakryiko 296457f4436SAndrii Nakryiko rb_map = container_of(map, struct bpf_ringbuf_map, map); 297457f4436SAndrii Nakryiko poll_wait(filp, &rb_map->rb->waitq, pts); 298457f4436SAndrii Nakryiko 299457f4436SAndrii Nakryiko if (ringbuf_avail_data_sz(rb_map->rb)) 300457f4436SAndrii Nakryiko return EPOLLIN | EPOLLRDNORM; 301457f4436SAndrii Nakryiko return 0; 302457f4436SAndrii Nakryiko } 303457f4436SAndrii Nakryiko 304c317ab71SMenglong Dong BTF_ID_LIST_SINGLE(ringbuf_map_btf_ids, struct, bpf_ringbuf_map) 305457f4436SAndrii Nakryiko const struct bpf_map_ops ringbuf_map_ops = { 306f4d05259SMartin KaFai Lau .map_meta_equal = bpf_map_meta_equal, 307457f4436SAndrii Nakryiko .map_alloc = ringbuf_map_alloc, 308457f4436SAndrii Nakryiko .map_free = ringbuf_map_free, 309*583c1f42SDavid Vernet .map_mmap = ringbuf_map_mmap_kern, 310457f4436SAndrii Nakryiko .map_poll = ringbuf_map_poll, 311457f4436SAndrii Nakryiko .map_lookup_elem = ringbuf_map_lookup_elem, 312457f4436SAndrii Nakryiko .map_update_elem = ringbuf_map_update_elem, 313457f4436SAndrii Nakryiko .map_delete_elem = ringbuf_map_delete_elem, 314457f4436SAndrii Nakryiko .map_get_next_key = ringbuf_map_get_next_key, 315c317ab71SMenglong Dong .map_btf_id = &ringbuf_map_btf_ids[0], 316457f4436SAndrii Nakryiko }; 317457f4436SAndrii Nakryiko 318*583c1f42SDavid Vernet BTF_ID_LIST_SINGLE(user_ringbuf_map_btf_ids, struct, bpf_ringbuf_map) 319*583c1f42SDavid Vernet const struct bpf_map_ops user_ringbuf_map_ops = { 320*583c1f42SDavid Vernet .map_meta_equal = bpf_map_meta_equal, 321*583c1f42SDavid Vernet .map_alloc = ringbuf_map_alloc, 322*583c1f42SDavid Vernet .map_free = ringbuf_map_free, 323*583c1f42SDavid Vernet .map_mmap = ringbuf_map_mmap_user, 324*583c1f42SDavid Vernet .map_lookup_elem = ringbuf_map_lookup_elem, 325*583c1f42SDavid Vernet .map_update_elem = ringbuf_map_update_elem, 326*583c1f42SDavid Vernet .map_delete_elem = ringbuf_map_delete_elem, 327*583c1f42SDavid Vernet .map_get_next_key = ringbuf_map_get_next_key, 328*583c1f42SDavid Vernet .map_btf_id = &user_ringbuf_map_btf_ids[0], 329*583c1f42SDavid Vernet }; 330*583c1f42SDavid Vernet 331457f4436SAndrii Nakryiko /* Given pointer to ring buffer record metadata and struct bpf_ringbuf itself, 332457f4436SAndrii Nakryiko * calculate offset from record metadata to ring buffer in pages, rounded 333457f4436SAndrii Nakryiko * down. This page offset is stored as part of record metadata and allows to 334457f4436SAndrii Nakryiko * restore struct bpf_ringbuf * from record pointer. This page offset is 335457f4436SAndrii Nakryiko * stored at offset 4 of record metadata header. 336457f4436SAndrii Nakryiko */ 337457f4436SAndrii Nakryiko static size_t bpf_ringbuf_rec_pg_off(struct bpf_ringbuf *rb, 338457f4436SAndrii Nakryiko struct bpf_ringbuf_hdr *hdr) 339457f4436SAndrii Nakryiko { 340457f4436SAndrii Nakryiko return ((void *)hdr - (void *)rb) >> PAGE_SHIFT; 341457f4436SAndrii Nakryiko } 342457f4436SAndrii Nakryiko 343457f4436SAndrii Nakryiko /* Given pointer to ring buffer record header, restore pointer to struct 344457f4436SAndrii Nakryiko * bpf_ringbuf itself by using page offset stored at offset 4 345457f4436SAndrii Nakryiko */ 346457f4436SAndrii Nakryiko static struct bpf_ringbuf * 347457f4436SAndrii Nakryiko bpf_ringbuf_restore_from_rec(struct bpf_ringbuf_hdr *hdr) 348457f4436SAndrii Nakryiko { 349457f4436SAndrii Nakryiko unsigned long addr = (unsigned long)(void *)hdr; 350457f4436SAndrii Nakryiko unsigned long off = (unsigned long)hdr->pg_off << PAGE_SHIFT; 351457f4436SAndrii Nakryiko 352457f4436SAndrii Nakryiko return (void*)((addr & PAGE_MASK) - off); 353457f4436SAndrii Nakryiko } 354457f4436SAndrii Nakryiko 355457f4436SAndrii Nakryiko static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) 356457f4436SAndrii Nakryiko { 357457f4436SAndrii Nakryiko unsigned long cons_pos, prod_pos, new_prod_pos, flags; 358457f4436SAndrii Nakryiko u32 len, pg_off; 359457f4436SAndrii Nakryiko struct bpf_ringbuf_hdr *hdr; 360457f4436SAndrii Nakryiko 361457f4436SAndrii Nakryiko if (unlikely(size > RINGBUF_MAX_RECORD_SZ)) 362457f4436SAndrii Nakryiko return NULL; 363457f4436SAndrii Nakryiko 364457f4436SAndrii Nakryiko len = round_up(size + BPF_RINGBUF_HDR_SZ, 8); 3654b81ccebSThadeu Lima de Souza Cascardo if (len > rb->mask + 1) 3664b81ccebSThadeu Lima de Souza Cascardo return NULL; 3674b81ccebSThadeu Lima de Souza Cascardo 368457f4436SAndrii Nakryiko cons_pos = smp_load_acquire(&rb->consumer_pos); 369457f4436SAndrii Nakryiko 370457f4436SAndrii Nakryiko if (in_nmi()) { 371457f4436SAndrii Nakryiko if (!spin_trylock_irqsave(&rb->spinlock, flags)) 372457f4436SAndrii Nakryiko return NULL; 373457f4436SAndrii Nakryiko } else { 374457f4436SAndrii Nakryiko spin_lock_irqsave(&rb->spinlock, flags); 375457f4436SAndrii Nakryiko } 376457f4436SAndrii Nakryiko 377457f4436SAndrii Nakryiko prod_pos = rb->producer_pos; 378457f4436SAndrii Nakryiko new_prod_pos = prod_pos + len; 379457f4436SAndrii Nakryiko 380457f4436SAndrii Nakryiko /* check for out of ringbuf space by ensuring producer position 381457f4436SAndrii Nakryiko * doesn't advance more than (ringbuf_size - 1) ahead 382457f4436SAndrii Nakryiko */ 383457f4436SAndrii Nakryiko if (new_prod_pos - cons_pos > rb->mask) { 384457f4436SAndrii Nakryiko spin_unlock_irqrestore(&rb->spinlock, flags); 385457f4436SAndrii Nakryiko return NULL; 386457f4436SAndrii Nakryiko } 387457f4436SAndrii Nakryiko 388457f4436SAndrii Nakryiko hdr = (void *)rb->data + (prod_pos & rb->mask); 389457f4436SAndrii Nakryiko pg_off = bpf_ringbuf_rec_pg_off(rb, hdr); 390457f4436SAndrii Nakryiko hdr->len = size | BPF_RINGBUF_BUSY_BIT; 391457f4436SAndrii Nakryiko hdr->pg_off = pg_off; 392457f4436SAndrii Nakryiko 393457f4436SAndrii Nakryiko /* pairs with consumer's smp_load_acquire() */ 394457f4436SAndrii Nakryiko smp_store_release(&rb->producer_pos, new_prod_pos); 395457f4436SAndrii Nakryiko 396457f4436SAndrii Nakryiko spin_unlock_irqrestore(&rb->spinlock, flags); 397457f4436SAndrii Nakryiko 398457f4436SAndrii Nakryiko return (void *)hdr + BPF_RINGBUF_HDR_SZ; 399457f4436SAndrii Nakryiko } 400457f4436SAndrii Nakryiko 401457f4436SAndrii Nakryiko BPF_CALL_3(bpf_ringbuf_reserve, struct bpf_map *, map, u64, size, u64, flags) 402457f4436SAndrii Nakryiko { 403457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 404457f4436SAndrii Nakryiko 405457f4436SAndrii Nakryiko if (unlikely(flags)) 406457f4436SAndrii Nakryiko return 0; 407457f4436SAndrii Nakryiko 408457f4436SAndrii Nakryiko rb_map = container_of(map, struct bpf_ringbuf_map, map); 409457f4436SAndrii Nakryiko return (unsigned long)__bpf_ringbuf_reserve(rb_map->rb, size); 410457f4436SAndrii Nakryiko } 411457f4436SAndrii Nakryiko 412457f4436SAndrii Nakryiko const struct bpf_func_proto bpf_ringbuf_reserve_proto = { 413457f4436SAndrii Nakryiko .func = bpf_ringbuf_reserve, 414457f4436SAndrii Nakryiko .ret_type = RET_PTR_TO_ALLOC_MEM_OR_NULL, 415457f4436SAndrii Nakryiko .arg1_type = ARG_CONST_MAP_PTR, 416457f4436SAndrii Nakryiko .arg2_type = ARG_CONST_ALLOC_SIZE_OR_ZERO, 417457f4436SAndrii Nakryiko .arg3_type = ARG_ANYTHING, 418457f4436SAndrii Nakryiko }; 419457f4436SAndrii Nakryiko 420457f4436SAndrii Nakryiko static void bpf_ringbuf_commit(void *sample, u64 flags, bool discard) 421457f4436SAndrii Nakryiko { 422457f4436SAndrii Nakryiko unsigned long rec_pos, cons_pos; 423457f4436SAndrii Nakryiko struct bpf_ringbuf_hdr *hdr; 424457f4436SAndrii Nakryiko struct bpf_ringbuf *rb; 425457f4436SAndrii Nakryiko u32 new_len; 426457f4436SAndrii Nakryiko 427457f4436SAndrii Nakryiko hdr = sample - BPF_RINGBUF_HDR_SZ; 428457f4436SAndrii Nakryiko rb = bpf_ringbuf_restore_from_rec(hdr); 429457f4436SAndrii Nakryiko new_len = hdr->len ^ BPF_RINGBUF_BUSY_BIT; 430457f4436SAndrii Nakryiko if (discard) 431457f4436SAndrii Nakryiko new_len |= BPF_RINGBUF_DISCARD_BIT; 432457f4436SAndrii Nakryiko 433457f4436SAndrii Nakryiko /* update record header with correct final size prefix */ 434457f4436SAndrii Nakryiko xchg(&hdr->len, new_len); 435457f4436SAndrii Nakryiko 436457f4436SAndrii Nakryiko /* if consumer caught up and is waiting for our record, notify about 437457f4436SAndrii Nakryiko * new data availability 438457f4436SAndrii Nakryiko */ 439457f4436SAndrii Nakryiko rec_pos = (void *)hdr - (void *)rb->data; 440457f4436SAndrii Nakryiko cons_pos = smp_load_acquire(&rb->consumer_pos) & rb->mask; 441457f4436SAndrii Nakryiko 442457f4436SAndrii Nakryiko if (flags & BPF_RB_FORCE_WAKEUP) 443457f4436SAndrii Nakryiko irq_work_queue(&rb->work); 444457f4436SAndrii Nakryiko else if (cons_pos == rec_pos && !(flags & BPF_RB_NO_WAKEUP)) 445457f4436SAndrii Nakryiko irq_work_queue(&rb->work); 446457f4436SAndrii Nakryiko } 447457f4436SAndrii Nakryiko 448457f4436SAndrii Nakryiko BPF_CALL_2(bpf_ringbuf_submit, void *, sample, u64, flags) 449457f4436SAndrii Nakryiko { 450457f4436SAndrii Nakryiko bpf_ringbuf_commit(sample, flags, false /* discard */); 451457f4436SAndrii Nakryiko return 0; 452457f4436SAndrii Nakryiko } 453457f4436SAndrii Nakryiko 454457f4436SAndrii Nakryiko const struct bpf_func_proto bpf_ringbuf_submit_proto = { 455457f4436SAndrii Nakryiko .func = bpf_ringbuf_submit, 456457f4436SAndrii Nakryiko .ret_type = RET_VOID, 4578f14852eSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_ALLOC_MEM | OBJ_RELEASE, 458457f4436SAndrii Nakryiko .arg2_type = ARG_ANYTHING, 459457f4436SAndrii Nakryiko }; 460457f4436SAndrii Nakryiko 461457f4436SAndrii Nakryiko BPF_CALL_2(bpf_ringbuf_discard, void *, sample, u64, flags) 462457f4436SAndrii Nakryiko { 463457f4436SAndrii Nakryiko bpf_ringbuf_commit(sample, flags, true /* discard */); 464457f4436SAndrii Nakryiko return 0; 465457f4436SAndrii Nakryiko } 466457f4436SAndrii Nakryiko 467457f4436SAndrii Nakryiko const struct bpf_func_proto bpf_ringbuf_discard_proto = { 468457f4436SAndrii Nakryiko .func = bpf_ringbuf_discard, 469457f4436SAndrii Nakryiko .ret_type = RET_VOID, 4708f14852eSKumar Kartikeya Dwivedi .arg1_type = ARG_PTR_TO_ALLOC_MEM | OBJ_RELEASE, 471457f4436SAndrii Nakryiko .arg2_type = ARG_ANYTHING, 472457f4436SAndrii Nakryiko }; 473457f4436SAndrii Nakryiko 474457f4436SAndrii Nakryiko BPF_CALL_4(bpf_ringbuf_output, struct bpf_map *, map, void *, data, u64, size, 475457f4436SAndrii Nakryiko u64, flags) 476457f4436SAndrii Nakryiko { 477457f4436SAndrii Nakryiko struct bpf_ringbuf_map *rb_map; 478457f4436SAndrii Nakryiko void *rec; 479457f4436SAndrii Nakryiko 480457f4436SAndrii Nakryiko if (unlikely(flags & ~(BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP))) 481457f4436SAndrii Nakryiko return -EINVAL; 482457f4436SAndrii Nakryiko 483457f4436SAndrii Nakryiko rb_map = container_of(map, struct bpf_ringbuf_map, map); 484457f4436SAndrii Nakryiko rec = __bpf_ringbuf_reserve(rb_map->rb, size); 485457f4436SAndrii Nakryiko if (!rec) 486457f4436SAndrii Nakryiko return -EAGAIN; 487457f4436SAndrii Nakryiko 488457f4436SAndrii Nakryiko memcpy(rec, data, size); 489457f4436SAndrii Nakryiko bpf_ringbuf_commit(rec, flags, false /* discard */); 490457f4436SAndrii Nakryiko return 0; 491457f4436SAndrii Nakryiko } 492457f4436SAndrii Nakryiko 493457f4436SAndrii Nakryiko const struct bpf_func_proto bpf_ringbuf_output_proto = { 494457f4436SAndrii Nakryiko .func = bpf_ringbuf_output, 495457f4436SAndrii Nakryiko .ret_type = RET_INTEGER, 496457f4436SAndrii Nakryiko .arg1_type = ARG_CONST_MAP_PTR, 497216e3cd2SHao Luo .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 498457f4436SAndrii Nakryiko .arg3_type = ARG_CONST_SIZE_OR_ZERO, 499457f4436SAndrii Nakryiko .arg4_type = ARG_ANYTHING, 500457f4436SAndrii Nakryiko }; 501457f4436SAndrii Nakryiko 502457f4436SAndrii Nakryiko BPF_CALL_2(bpf_ringbuf_query, struct bpf_map *, map, u64, flags) 503457f4436SAndrii Nakryiko { 504457f4436SAndrii Nakryiko struct bpf_ringbuf *rb; 505457f4436SAndrii Nakryiko 506457f4436SAndrii Nakryiko rb = container_of(map, struct bpf_ringbuf_map, map)->rb; 507457f4436SAndrii Nakryiko 508457f4436SAndrii Nakryiko switch (flags) { 509457f4436SAndrii Nakryiko case BPF_RB_AVAIL_DATA: 510457f4436SAndrii Nakryiko return ringbuf_avail_data_sz(rb); 511457f4436SAndrii Nakryiko case BPF_RB_RING_SIZE: 512457f4436SAndrii Nakryiko return rb->mask + 1; 513457f4436SAndrii Nakryiko case BPF_RB_CONS_POS: 514457f4436SAndrii Nakryiko return smp_load_acquire(&rb->consumer_pos); 515457f4436SAndrii Nakryiko case BPF_RB_PROD_POS: 516457f4436SAndrii Nakryiko return smp_load_acquire(&rb->producer_pos); 517457f4436SAndrii Nakryiko default: 518457f4436SAndrii Nakryiko return 0; 519457f4436SAndrii Nakryiko } 520457f4436SAndrii Nakryiko } 521457f4436SAndrii Nakryiko 522457f4436SAndrii Nakryiko const struct bpf_func_proto bpf_ringbuf_query_proto = { 523457f4436SAndrii Nakryiko .func = bpf_ringbuf_query, 524457f4436SAndrii Nakryiko .ret_type = RET_INTEGER, 525457f4436SAndrii Nakryiko .arg1_type = ARG_CONST_MAP_PTR, 526457f4436SAndrii Nakryiko .arg2_type = ARG_ANYTHING, 527457f4436SAndrii Nakryiko }; 528bc34dee6SJoanne Koong 529bc34dee6SJoanne Koong BPF_CALL_4(bpf_ringbuf_reserve_dynptr, struct bpf_map *, map, u32, size, u64, flags, 530bc34dee6SJoanne Koong struct bpf_dynptr_kern *, ptr) 531bc34dee6SJoanne Koong { 532bc34dee6SJoanne Koong struct bpf_ringbuf_map *rb_map; 533bc34dee6SJoanne Koong void *sample; 534bc34dee6SJoanne Koong int err; 535bc34dee6SJoanne Koong 536bc34dee6SJoanne Koong if (unlikely(flags)) { 537bc34dee6SJoanne Koong bpf_dynptr_set_null(ptr); 538bc34dee6SJoanne Koong return -EINVAL; 539bc34dee6SJoanne Koong } 540bc34dee6SJoanne Koong 541bc34dee6SJoanne Koong err = bpf_dynptr_check_size(size); 542bc34dee6SJoanne Koong if (err) { 543bc34dee6SJoanne Koong bpf_dynptr_set_null(ptr); 544bc34dee6SJoanne Koong return err; 545bc34dee6SJoanne Koong } 546bc34dee6SJoanne Koong 547bc34dee6SJoanne Koong rb_map = container_of(map, struct bpf_ringbuf_map, map); 548bc34dee6SJoanne Koong 549bc34dee6SJoanne Koong sample = __bpf_ringbuf_reserve(rb_map->rb, size); 550bc34dee6SJoanne Koong if (!sample) { 551bc34dee6SJoanne Koong bpf_dynptr_set_null(ptr); 552bc34dee6SJoanne Koong return -EINVAL; 553bc34dee6SJoanne Koong } 554bc34dee6SJoanne Koong 555bc34dee6SJoanne Koong bpf_dynptr_init(ptr, sample, BPF_DYNPTR_TYPE_RINGBUF, 0, size); 556bc34dee6SJoanne Koong 557bc34dee6SJoanne Koong return 0; 558bc34dee6SJoanne Koong } 559bc34dee6SJoanne Koong 560bc34dee6SJoanne Koong const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto = { 561bc34dee6SJoanne Koong .func = bpf_ringbuf_reserve_dynptr, 562bc34dee6SJoanne Koong .ret_type = RET_INTEGER, 563bc34dee6SJoanne Koong .arg1_type = ARG_CONST_MAP_PTR, 564bc34dee6SJoanne Koong .arg2_type = ARG_ANYTHING, 565bc34dee6SJoanne Koong .arg3_type = ARG_ANYTHING, 566bc34dee6SJoanne Koong .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | MEM_UNINIT, 567bc34dee6SJoanne Koong }; 568bc34dee6SJoanne Koong 569bc34dee6SJoanne Koong BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) 570bc34dee6SJoanne Koong { 571bc34dee6SJoanne Koong if (!ptr->data) 572bc34dee6SJoanne Koong return 0; 573bc34dee6SJoanne Koong 574bc34dee6SJoanne Koong bpf_ringbuf_commit(ptr->data, flags, false /* discard */); 575bc34dee6SJoanne Koong 576bc34dee6SJoanne Koong bpf_dynptr_set_null(ptr); 577bc34dee6SJoanne Koong 578bc34dee6SJoanne Koong return 0; 579bc34dee6SJoanne Koong } 580bc34dee6SJoanne Koong 581bc34dee6SJoanne Koong const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = { 582bc34dee6SJoanne Koong .func = bpf_ringbuf_submit_dynptr, 583bc34dee6SJoanne Koong .ret_type = RET_VOID, 584bc34dee6SJoanne Koong .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, 585bc34dee6SJoanne Koong .arg2_type = ARG_ANYTHING, 586bc34dee6SJoanne Koong }; 587bc34dee6SJoanne Koong 588bc34dee6SJoanne Koong BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags) 589bc34dee6SJoanne Koong { 590bc34dee6SJoanne Koong if (!ptr->data) 591bc34dee6SJoanne Koong return 0; 592bc34dee6SJoanne Koong 593bc34dee6SJoanne Koong bpf_ringbuf_commit(ptr->data, flags, true /* discard */); 594bc34dee6SJoanne Koong 595bc34dee6SJoanne Koong bpf_dynptr_set_null(ptr); 596bc34dee6SJoanne Koong 597bc34dee6SJoanne Koong return 0; 598bc34dee6SJoanne Koong } 599bc34dee6SJoanne Koong 600bc34dee6SJoanne Koong const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { 601bc34dee6SJoanne Koong .func = bpf_ringbuf_discard_dynptr, 602bc34dee6SJoanne Koong .ret_type = RET_VOID, 603bc34dee6SJoanne Koong .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, 604bc34dee6SJoanne Koong .arg2_type = ARG_ANYTHING, 605bc34dee6SJoanne Koong }; 606