1e5a9df51SDavid Vernet // SPDX-License-Identifier: GPL-2.0
2e5a9df51SDavid Vernet /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3e5a9df51SDavid Vernet
4e5a9df51SDavid Vernet #define _GNU_SOURCE
5e5a9df51SDavid Vernet #include <linux/compiler.h>
6e5a9df51SDavid Vernet #include <linux/ring_buffer.h>
7e5a9df51SDavid Vernet #include <pthread.h>
8e5a9df51SDavid Vernet #include <stdio.h>
9e5a9df51SDavid Vernet #include <stdlib.h>
10e5a9df51SDavid Vernet #include <sys/mman.h>
11e5a9df51SDavid Vernet #include <sys/syscall.h>
12e5a9df51SDavid Vernet #include <sys/sysinfo.h>
13e5a9df51SDavid Vernet #include <test_progs.h>
14e5a9df51SDavid Vernet #include <uapi/linux/bpf.h>
15e5a9df51SDavid Vernet #include <unistd.h>
16e5a9df51SDavid Vernet
17e5a9df51SDavid Vernet #include "user_ringbuf_fail.skel.h"
18e5a9df51SDavid Vernet #include "user_ringbuf_success.skel.h"
19e5a9df51SDavid Vernet
20e5a9df51SDavid Vernet #include "../progs/test_user_ringbuf.h"
21e5a9df51SDavid Vernet
22e5a9df51SDavid Vernet static const long c_sample_size = sizeof(struct sample) + BPF_RINGBUF_HDR_SZ;
23e5a9df51SDavid Vernet static const long c_ringbuf_size = 1 << 12; /* 1 small page */
24e5a9df51SDavid Vernet static const long c_max_entries = c_ringbuf_size / c_sample_size;
25e5a9df51SDavid Vernet
drain_current_samples(void)26e5a9df51SDavid Vernet static void drain_current_samples(void)
27e5a9df51SDavid Vernet {
28e5a9df51SDavid Vernet syscall(__NR_getpgid);
29e5a9df51SDavid Vernet }
30e5a9df51SDavid Vernet
write_samples(struct user_ring_buffer * ringbuf,uint32_t num_samples)31e5a9df51SDavid Vernet static int write_samples(struct user_ring_buffer *ringbuf, uint32_t num_samples)
32e5a9df51SDavid Vernet {
33e5a9df51SDavid Vernet int i, err = 0;
34e5a9df51SDavid Vernet
35e5a9df51SDavid Vernet /* Write some number of samples to the ring buffer. */
36e5a9df51SDavid Vernet for (i = 0; i < num_samples; i++) {
37e5a9df51SDavid Vernet struct sample *entry;
38e5a9df51SDavid Vernet int read;
39e5a9df51SDavid Vernet
40e5a9df51SDavid Vernet entry = user_ring_buffer__reserve(ringbuf, sizeof(*entry));
41e5a9df51SDavid Vernet if (!entry) {
42e5a9df51SDavid Vernet err = -errno;
43e5a9df51SDavid Vernet goto done;
44e5a9df51SDavid Vernet }
45e5a9df51SDavid Vernet
46e5a9df51SDavid Vernet entry->pid = getpid();
47e5a9df51SDavid Vernet entry->seq = i;
48e5a9df51SDavid Vernet entry->value = i * i;
49e5a9df51SDavid Vernet
50e5a9df51SDavid Vernet read = snprintf(entry->comm, sizeof(entry->comm), "%u", i);
51e5a9df51SDavid Vernet if (read <= 0) {
52e5a9df51SDavid Vernet /* Assert on the error path to avoid spamming logs with
53e5a9df51SDavid Vernet * mostly success messages.
54e5a9df51SDavid Vernet */
55e5a9df51SDavid Vernet ASSERT_GT(read, 0, "snprintf_comm");
56e5a9df51SDavid Vernet err = read;
57e5a9df51SDavid Vernet user_ring_buffer__discard(ringbuf, entry);
58e5a9df51SDavid Vernet goto done;
59e5a9df51SDavid Vernet }
60e5a9df51SDavid Vernet
61e5a9df51SDavid Vernet user_ring_buffer__submit(ringbuf, entry);
62e5a9df51SDavid Vernet }
63e5a9df51SDavid Vernet
64e5a9df51SDavid Vernet done:
65e5a9df51SDavid Vernet drain_current_samples();
66e5a9df51SDavid Vernet
67e5a9df51SDavid Vernet return err;
68e5a9df51SDavid Vernet }
69e5a9df51SDavid Vernet
open_load_ringbuf_skel(void)70e5a9df51SDavid Vernet static struct user_ringbuf_success *open_load_ringbuf_skel(void)
71e5a9df51SDavid Vernet {
72e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
73e5a9df51SDavid Vernet int err;
74e5a9df51SDavid Vernet
75e5a9df51SDavid Vernet skel = user_ringbuf_success__open();
76e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(skel, "skel_open"))
77e5a9df51SDavid Vernet return NULL;
78e5a9df51SDavid Vernet
79e5a9df51SDavid Vernet err = bpf_map__set_max_entries(skel->maps.user_ringbuf, c_ringbuf_size);
80e5a9df51SDavid Vernet if (!ASSERT_OK(err, "set_max_entries"))
81e5a9df51SDavid Vernet goto cleanup;
82e5a9df51SDavid Vernet
83e5a9df51SDavid Vernet err = bpf_map__set_max_entries(skel->maps.kernel_ringbuf, c_ringbuf_size);
84e5a9df51SDavid Vernet if (!ASSERT_OK(err, "set_max_entries"))
85e5a9df51SDavid Vernet goto cleanup;
86e5a9df51SDavid Vernet
87e5a9df51SDavid Vernet err = user_ringbuf_success__load(skel);
88e5a9df51SDavid Vernet if (!ASSERT_OK(err, "skel_load"))
89e5a9df51SDavid Vernet goto cleanup;
90e5a9df51SDavid Vernet
91e5a9df51SDavid Vernet return skel;
92e5a9df51SDavid Vernet
93e5a9df51SDavid Vernet cleanup:
94e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
95e5a9df51SDavid Vernet return NULL;
96e5a9df51SDavid Vernet }
97e5a9df51SDavid Vernet
test_user_ringbuf_mappings(void)98e5a9df51SDavid Vernet static void test_user_ringbuf_mappings(void)
99e5a9df51SDavid Vernet {
100e5a9df51SDavid Vernet int err, rb_fd;
101e5a9df51SDavid Vernet int page_size = getpagesize();
102e5a9df51SDavid Vernet void *mmap_ptr;
103e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
104e5a9df51SDavid Vernet
105e5a9df51SDavid Vernet skel = open_load_ringbuf_skel();
106e5a9df51SDavid Vernet if (!skel)
107e5a9df51SDavid Vernet return;
108e5a9df51SDavid Vernet
109e5a9df51SDavid Vernet rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
110e5a9df51SDavid Vernet /* cons_pos can be mapped R/O, can't add +X with mprotect. */
111e5a9df51SDavid Vernet mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, 0);
112e5a9df51SDavid Vernet ASSERT_OK_PTR(mmap_ptr, "ro_cons_pos");
113e5a9df51SDavid Vernet ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_cons_pos_protect");
114e5a9df51SDavid Vernet ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect");
115e5a9df51SDavid Vernet ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "wr_prod_pos");
116e5a9df51SDavid Vernet err = -errno;
117e5a9df51SDavid Vernet ASSERT_ERR(err, "wr_prod_pos_err");
118e5a9df51SDavid Vernet ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro_cons");
119e5a9df51SDavid Vernet
120e5a9df51SDavid Vernet /* prod_pos can be mapped RW, can't add +X with mprotect. */
121e5a9df51SDavid Vernet mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
122e5a9df51SDavid Vernet rb_fd, page_size);
123e5a9df51SDavid Vernet ASSERT_OK_PTR(mmap_ptr, "rw_prod_pos");
124e5a9df51SDavid Vernet ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_prod_pos_protect");
125e5a9df51SDavid Vernet err = -errno;
126e5a9df51SDavid Vernet ASSERT_ERR(err, "wr_prod_pos_err");
127e5a9df51SDavid Vernet ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_prod");
128e5a9df51SDavid Vernet
129e5a9df51SDavid Vernet /* data pages can be mapped RW, can't add +X with mprotect. */
130e5a9df51SDavid Vernet mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd,
131e5a9df51SDavid Vernet 2 * page_size);
132e5a9df51SDavid Vernet ASSERT_OK_PTR(mmap_ptr, "rw_data");
133e5a9df51SDavid Vernet ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_data_protect");
134e5a9df51SDavid Vernet err = -errno;
135e5a9df51SDavid Vernet ASSERT_ERR(err, "exec_data_err");
136e5a9df51SDavid Vernet ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_data");
137e5a9df51SDavid Vernet
138e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
139e5a9df51SDavid Vernet }
140e5a9df51SDavid Vernet
load_skel_create_ringbufs(struct user_ringbuf_success ** skel_out,struct ring_buffer ** kern_ringbuf_out,ring_buffer_sample_fn callback,struct user_ring_buffer ** user_ringbuf_out)141e5a9df51SDavid Vernet static int load_skel_create_ringbufs(struct user_ringbuf_success **skel_out,
142e5a9df51SDavid Vernet struct ring_buffer **kern_ringbuf_out,
143e5a9df51SDavid Vernet ring_buffer_sample_fn callback,
144e5a9df51SDavid Vernet struct user_ring_buffer **user_ringbuf_out)
145e5a9df51SDavid Vernet {
146e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
147e5a9df51SDavid Vernet struct ring_buffer *kern_ringbuf = NULL;
148e5a9df51SDavid Vernet struct user_ring_buffer *user_ringbuf = NULL;
149e5a9df51SDavid Vernet int err = -ENOMEM, rb_fd;
150e5a9df51SDavid Vernet
151e5a9df51SDavid Vernet skel = open_load_ringbuf_skel();
152e5a9df51SDavid Vernet if (!skel)
153e5a9df51SDavid Vernet return err;
154e5a9df51SDavid Vernet
155e5a9df51SDavid Vernet /* only trigger BPF program for current process */
156e5a9df51SDavid Vernet skel->bss->pid = getpid();
157e5a9df51SDavid Vernet
158e5a9df51SDavid Vernet if (kern_ringbuf_out) {
159e5a9df51SDavid Vernet rb_fd = bpf_map__fd(skel->maps.kernel_ringbuf);
160e5a9df51SDavid Vernet kern_ringbuf = ring_buffer__new(rb_fd, callback, skel, NULL);
161e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(kern_ringbuf, "kern_ringbuf_create"))
162e5a9df51SDavid Vernet goto cleanup;
163e5a9df51SDavid Vernet
164e5a9df51SDavid Vernet *kern_ringbuf_out = kern_ringbuf;
165e5a9df51SDavid Vernet }
166e5a9df51SDavid Vernet
167e5a9df51SDavid Vernet if (user_ringbuf_out) {
168e5a9df51SDavid Vernet rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
169e5a9df51SDavid Vernet user_ringbuf = user_ring_buffer__new(rb_fd, NULL);
170e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(user_ringbuf, "user_ringbuf_create"))
171e5a9df51SDavid Vernet goto cleanup;
172e5a9df51SDavid Vernet
173e5a9df51SDavid Vernet *user_ringbuf_out = user_ringbuf;
174e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "no_reads_after_load");
175e5a9df51SDavid Vernet }
176e5a9df51SDavid Vernet
177e5a9df51SDavid Vernet err = user_ringbuf_success__attach(skel);
178e5a9df51SDavid Vernet if (!ASSERT_OK(err, "skel_attach"))
179e5a9df51SDavid Vernet goto cleanup;
180e5a9df51SDavid Vernet
181e5a9df51SDavid Vernet *skel_out = skel;
182e5a9df51SDavid Vernet return 0;
183e5a9df51SDavid Vernet
184e5a9df51SDavid Vernet cleanup:
185e5a9df51SDavid Vernet if (kern_ringbuf_out)
186e5a9df51SDavid Vernet *kern_ringbuf_out = NULL;
187e5a9df51SDavid Vernet if (user_ringbuf_out)
188e5a9df51SDavid Vernet *user_ringbuf_out = NULL;
189e5a9df51SDavid Vernet ring_buffer__free(kern_ringbuf);
190e5a9df51SDavid Vernet user_ring_buffer__free(user_ringbuf);
191e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
192e5a9df51SDavid Vernet return err;
193e5a9df51SDavid Vernet }
194e5a9df51SDavid Vernet
load_skel_create_user_ringbuf(struct user_ringbuf_success ** skel_out,struct user_ring_buffer ** ringbuf_out)195e5a9df51SDavid Vernet static int load_skel_create_user_ringbuf(struct user_ringbuf_success **skel_out,
196e5a9df51SDavid Vernet struct user_ring_buffer **ringbuf_out)
197e5a9df51SDavid Vernet {
198e5a9df51SDavid Vernet return load_skel_create_ringbufs(skel_out, NULL, NULL, ringbuf_out);
199e5a9df51SDavid Vernet }
200e5a9df51SDavid Vernet
manually_write_test_invalid_sample(struct user_ringbuf_success * skel,__u32 size,__u64 producer_pos,int err)201e5a9df51SDavid Vernet static void manually_write_test_invalid_sample(struct user_ringbuf_success *skel,
202e5a9df51SDavid Vernet __u32 size, __u64 producer_pos, int err)
203e5a9df51SDavid Vernet {
204e5a9df51SDavid Vernet void *data_ptr;
205e5a9df51SDavid Vernet __u64 *producer_pos_ptr;
206e5a9df51SDavid Vernet int rb_fd, page_size = getpagesize();
207e5a9df51SDavid Vernet
208e5a9df51SDavid Vernet rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
209e5a9df51SDavid Vernet
210e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_before_bad_sample");
211e5a9df51SDavid Vernet
212e5a9df51SDavid Vernet /* Map the producer_pos as RW. */
213e5a9df51SDavid Vernet producer_pos_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE,
214e5a9df51SDavid Vernet MAP_SHARED, rb_fd, page_size);
215e5a9df51SDavid Vernet ASSERT_OK_PTR(producer_pos_ptr, "producer_pos_ptr");
216e5a9df51SDavid Vernet
217e5a9df51SDavid Vernet /* Map the data pages as RW. */
218e5a9df51SDavid Vernet data_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);
219e5a9df51SDavid Vernet ASSERT_OK_PTR(data_ptr, "rw_data");
220e5a9df51SDavid Vernet
221e5a9df51SDavid Vernet memset(data_ptr, 0, BPF_RINGBUF_HDR_SZ);
222e5a9df51SDavid Vernet *(__u32 *)data_ptr = size;
223e5a9df51SDavid Vernet
224e5a9df51SDavid Vernet /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in the kernel. */
225e5a9df51SDavid Vernet smp_store_release(producer_pos_ptr, producer_pos + BPF_RINGBUF_HDR_SZ);
226e5a9df51SDavid Vernet
227e5a9df51SDavid Vernet drain_current_samples();
228e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_after_bad_sample");
229e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->err, err, "err_after_bad_sample");
230e5a9df51SDavid Vernet
231e5a9df51SDavid Vernet ASSERT_OK(munmap(producer_pos_ptr, page_size), "unmap_producer_pos");
232e5a9df51SDavid Vernet ASSERT_OK(munmap(data_ptr, page_size), "unmap_data_ptr");
233e5a9df51SDavid Vernet }
234e5a9df51SDavid Vernet
test_user_ringbuf_post_misaligned(void)235e5a9df51SDavid Vernet static void test_user_ringbuf_post_misaligned(void)
236e5a9df51SDavid Vernet {
237e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
238e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
239e5a9df51SDavid Vernet int err;
240e5a9df51SDavid Vernet __u32 size = (1 << 5) + 7;
241e5a9df51SDavid Vernet
242e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
243e5a9df51SDavid Vernet if (!ASSERT_OK(err, "misaligned_skel"))
244e5a9df51SDavid Vernet return;
245e5a9df51SDavid Vernet
246e5a9df51SDavid Vernet manually_write_test_invalid_sample(skel, size, size, -EINVAL);
247e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
248e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
249e5a9df51SDavid Vernet }
250e5a9df51SDavid Vernet
test_user_ringbuf_post_producer_wrong_offset(void)251e5a9df51SDavid Vernet static void test_user_ringbuf_post_producer_wrong_offset(void)
252e5a9df51SDavid Vernet {
253e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
254e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
255e5a9df51SDavid Vernet int err;
256e5a9df51SDavid Vernet __u32 size = (1 << 5);
257e5a9df51SDavid Vernet
258e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
259e5a9df51SDavid Vernet if (!ASSERT_OK(err, "wrong_offset_skel"))
260e5a9df51SDavid Vernet return;
261e5a9df51SDavid Vernet
262e5a9df51SDavid Vernet manually_write_test_invalid_sample(skel, size, size - 8, -EINVAL);
263e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
264e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
265e5a9df51SDavid Vernet }
266e5a9df51SDavid Vernet
test_user_ringbuf_post_larger_than_ringbuf_sz(void)267e5a9df51SDavid Vernet static void test_user_ringbuf_post_larger_than_ringbuf_sz(void)
268e5a9df51SDavid Vernet {
269e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
270e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
271e5a9df51SDavid Vernet int err;
272e5a9df51SDavid Vernet __u32 size = c_ringbuf_size;
273e5a9df51SDavid Vernet
274e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
275e5a9df51SDavid Vernet if (!ASSERT_OK(err, "huge_sample_skel"))
276e5a9df51SDavid Vernet return;
277e5a9df51SDavid Vernet
278e5a9df51SDavid Vernet manually_write_test_invalid_sample(skel, size, size, -E2BIG);
279e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
280e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
281e5a9df51SDavid Vernet }
282e5a9df51SDavid Vernet
test_user_ringbuf_basic(void)283e5a9df51SDavid Vernet static void test_user_ringbuf_basic(void)
284e5a9df51SDavid Vernet {
285e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
286e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
287e5a9df51SDavid Vernet int err;
288e5a9df51SDavid Vernet
289e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
290e5a9df51SDavid Vernet if (!ASSERT_OK(err, "ringbuf_basic_skel"))
291e5a9df51SDavid Vernet return;
292e5a9df51SDavid Vernet
293e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
294e5a9df51SDavid Vernet
295e5a9df51SDavid Vernet err = write_samples(ringbuf, 2);
296e5a9df51SDavid Vernet if (!ASSERT_OK(err, "write_samples"))
297e5a9df51SDavid Vernet goto cleanup;
298e5a9df51SDavid Vernet
299e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 2, "num_samples_read_after");
300e5a9df51SDavid Vernet
301e5a9df51SDavid Vernet cleanup:
302e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
303e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
304e5a9df51SDavid Vernet }
305e5a9df51SDavid Vernet
test_user_ringbuf_sample_full_ring_buffer(void)306e5a9df51SDavid Vernet static void test_user_ringbuf_sample_full_ring_buffer(void)
307e5a9df51SDavid Vernet {
308e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
309e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
310e5a9df51SDavid Vernet int err;
311e5a9df51SDavid Vernet void *sample;
312e5a9df51SDavid Vernet
313e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
314e5a9df51SDavid Vernet if (!ASSERT_OK(err, "ringbuf_full_sample_skel"))
315e5a9df51SDavid Vernet return;
316e5a9df51SDavid Vernet
317e5a9df51SDavid Vernet sample = user_ring_buffer__reserve(ringbuf, c_ringbuf_size - BPF_RINGBUF_HDR_SZ);
318e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(sample, "full_sample"))
319e5a9df51SDavid Vernet goto cleanup;
320e5a9df51SDavid Vernet
321e5a9df51SDavid Vernet user_ring_buffer__submit(ringbuf, sample);
322e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
323e5a9df51SDavid Vernet drain_current_samples();
324e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");
325e5a9df51SDavid Vernet
326e5a9df51SDavid Vernet cleanup:
327e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
328e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
329e5a9df51SDavid Vernet }
330e5a9df51SDavid Vernet
test_user_ringbuf_post_alignment_autoadjust(void)331e5a9df51SDavid Vernet static void test_user_ringbuf_post_alignment_autoadjust(void)
332e5a9df51SDavid Vernet {
333e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
334e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
335e5a9df51SDavid Vernet struct sample *sample;
336e5a9df51SDavid Vernet int err;
337e5a9df51SDavid Vernet
338e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
339e5a9df51SDavid Vernet if (!ASSERT_OK(err, "ringbuf_align_autoadjust_skel"))
340e5a9df51SDavid Vernet return;
341e5a9df51SDavid Vernet
342e5a9df51SDavid Vernet /* libbpf should automatically round any sample up to an 8-byte alignment. */
343e5a9df51SDavid Vernet sample = user_ring_buffer__reserve(ringbuf, sizeof(*sample) + 1);
344e5a9df51SDavid Vernet ASSERT_OK_PTR(sample, "reserve_autoaligned");
345e5a9df51SDavid Vernet user_ring_buffer__submit(ringbuf, sample);
346e5a9df51SDavid Vernet
347e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
348e5a9df51SDavid Vernet drain_current_samples();
349e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");
350e5a9df51SDavid Vernet
351e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
352e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
353e5a9df51SDavid Vernet }
354e5a9df51SDavid Vernet
test_user_ringbuf_overfill(void)355e5a9df51SDavid Vernet static void test_user_ringbuf_overfill(void)
356e5a9df51SDavid Vernet {
357e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
358e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
359e5a9df51SDavid Vernet int err;
360e5a9df51SDavid Vernet
361e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
362e5a9df51SDavid Vernet if (err)
363e5a9df51SDavid Vernet return;
364e5a9df51SDavid Vernet
365e5a9df51SDavid Vernet err = write_samples(ringbuf, c_max_entries * 5);
366e5a9df51SDavid Vernet ASSERT_ERR(err, "write_samples");
367e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, c_max_entries, "max_entries");
368e5a9df51SDavid Vernet
369e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
370e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
371e5a9df51SDavid Vernet }
372e5a9df51SDavid Vernet
test_user_ringbuf_discards_properly_ignored(void)373e5a9df51SDavid Vernet static void test_user_ringbuf_discards_properly_ignored(void)
374e5a9df51SDavid Vernet {
375e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
376e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
377e5a9df51SDavid Vernet int err, num_discarded = 0;
378e5a9df51SDavid Vernet __u64 *token;
379e5a9df51SDavid Vernet
380e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
381e5a9df51SDavid Vernet if (err)
382e5a9df51SDavid Vernet return;
383e5a9df51SDavid Vernet
384e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
385e5a9df51SDavid Vernet
386e5a9df51SDavid Vernet while (1) {
387e5a9df51SDavid Vernet /* Write samples until the buffer is full. */
388e5a9df51SDavid Vernet token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
389e5a9df51SDavid Vernet if (!token)
390e5a9df51SDavid Vernet break;
391e5a9df51SDavid Vernet
392e5a9df51SDavid Vernet user_ring_buffer__discard(ringbuf, token);
393e5a9df51SDavid Vernet num_discarded++;
394e5a9df51SDavid Vernet }
395e5a9df51SDavid Vernet
396e5a9df51SDavid Vernet if (!ASSERT_GE(num_discarded, 0, "num_discarded"))
397e5a9df51SDavid Vernet goto cleanup;
398e5a9df51SDavid Vernet
399e5a9df51SDavid Vernet /* Should not read any samples, as they are all discarded. */
400e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");
401e5a9df51SDavid Vernet drain_current_samples();
402e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_post_kick");
403e5a9df51SDavid Vernet
404e5a9df51SDavid Vernet /* Now that the ring buffer has been drained, we should be able to
405e5a9df51SDavid Vernet * reserve another token.
406e5a9df51SDavid Vernet */
407e5a9df51SDavid Vernet token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
408e5a9df51SDavid Vernet
409e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(token, "new_token"))
410e5a9df51SDavid Vernet goto cleanup;
411e5a9df51SDavid Vernet
412e5a9df51SDavid Vernet user_ring_buffer__discard(ringbuf, token);
413e5a9df51SDavid Vernet cleanup:
414e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
415e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
416e5a9df51SDavid Vernet }
417e5a9df51SDavid Vernet
test_user_ringbuf_loop(void)418e5a9df51SDavid Vernet static void test_user_ringbuf_loop(void)
419e5a9df51SDavid Vernet {
420e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
421e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
422e5a9df51SDavid Vernet uint32_t total_samples = 8192;
423e5a9df51SDavid Vernet uint32_t remaining_samples = total_samples;
424e5a9df51SDavid Vernet int err;
425e5a9df51SDavid Vernet
426e5a9df51SDavid Vernet BUILD_BUG_ON(total_samples <= c_max_entries);
427e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
428e5a9df51SDavid Vernet if (err)
429e5a9df51SDavid Vernet return;
430e5a9df51SDavid Vernet
431e5a9df51SDavid Vernet do {
432e5a9df51SDavid Vernet uint32_t curr_samples;
433e5a9df51SDavid Vernet
434e5a9df51SDavid Vernet curr_samples = remaining_samples > c_max_entries
435e5a9df51SDavid Vernet ? c_max_entries : remaining_samples;
436e5a9df51SDavid Vernet err = write_samples(ringbuf, curr_samples);
437e5a9df51SDavid Vernet if (err != 0) {
438e5a9df51SDavid Vernet /* Assert inside of if statement to avoid flooding logs
439e5a9df51SDavid Vernet * on the success path.
440e5a9df51SDavid Vernet */
441e5a9df51SDavid Vernet ASSERT_OK(err, "write_samples");
442e5a9df51SDavid Vernet goto cleanup;
443e5a9df51SDavid Vernet }
444e5a9df51SDavid Vernet
445e5a9df51SDavid Vernet remaining_samples -= curr_samples;
446e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, total_samples - remaining_samples,
447e5a9df51SDavid Vernet "current_batched_entries");
448e5a9df51SDavid Vernet } while (remaining_samples > 0);
449e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, total_samples, "total_batched_entries");
450e5a9df51SDavid Vernet
451e5a9df51SDavid Vernet cleanup:
452e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
453e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
454e5a9df51SDavid Vernet }
455e5a9df51SDavid Vernet
send_test_message(struct user_ring_buffer * ringbuf,enum test_msg_op op,s64 operand_64,s32 operand_32)456e5a9df51SDavid Vernet static int send_test_message(struct user_ring_buffer *ringbuf,
457e5a9df51SDavid Vernet enum test_msg_op op, s64 operand_64,
458e5a9df51SDavid Vernet s32 operand_32)
459e5a9df51SDavid Vernet {
460e5a9df51SDavid Vernet struct test_msg *msg;
461e5a9df51SDavid Vernet
462e5a9df51SDavid Vernet msg = user_ring_buffer__reserve(ringbuf, sizeof(*msg));
463e5a9df51SDavid Vernet if (!msg) {
464e5a9df51SDavid Vernet /* Assert on the error path to avoid spamming logs with mostly
465e5a9df51SDavid Vernet * success messages.
466e5a9df51SDavid Vernet */
467e5a9df51SDavid Vernet ASSERT_OK_PTR(msg, "reserve_msg");
468e5a9df51SDavid Vernet return -ENOMEM;
469e5a9df51SDavid Vernet }
470e5a9df51SDavid Vernet
471e5a9df51SDavid Vernet msg->msg_op = op;
472e5a9df51SDavid Vernet
473e5a9df51SDavid Vernet switch (op) {
474e5a9df51SDavid Vernet case TEST_MSG_OP_INC64:
475e5a9df51SDavid Vernet case TEST_MSG_OP_MUL64:
476e5a9df51SDavid Vernet msg->operand_64 = operand_64;
477e5a9df51SDavid Vernet break;
478e5a9df51SDavid Vernet case TEST_MSG_OP_INC32:
479e5a9df51SDavid Vernet case TEST_MSG_OP_MUL32:
480e5a9df51SDavid Vernet msg->operand_32 = operand_32;
481e5a9df51SDavid Vernet break;
482e5a9df51SDavid Vernet default:
483e5a9df51SDavid Vernet PRINT_FAIL("Invalid operand %d\n", op);
484e5a9df51SDavid Vernet user_ring_buffer__discard(ringbuf, msg);
485e5a9df51SDavid Vernet return -EINVAL;
486e5a9df51SDavid Vernet }
487e5a9df51SDavid Vernet
488e5a9df51SDavid Vernet user_ring_buffer__submit(ringbuf, msg);
489e5a9df51SDavid Vernet
490e5a9df51SDavid Vernet return 0;
491e5a9df51SDavid Vernet }
492e5a9df51SDavid Vernet
kick_kernel_read_messages(void)493e5a9df51SDavid Vernet static void kick_kernel_read_messages(void)
494e5a9df51SDavid Vernet {
495e5a9df51SDavid Vernet syscall(__NR_prctl);
496e5a9df51SDavid Vernet }
497e5a9df51SDavid Vernet
handle_kernel_msg(void * ctx,void * data,size_t len)498e5a9df51SDavid Vernet static int handle_kernel_msg(void *ctx, void *data, size_t len)
499e5a9df51SDavid Vernet {
500e5a9df51SDavid Vernet struct user_ringbuf_success *skel = ctx;
501e5a9df51SDavid Vernet struct test_msg *msg = data;
502e5a9df51SDavid Vernet
503e5a9df51SDavid Vernet switch (msg->msg_op) {
504e5a9df51SDavid Vernet case TEST_MSG_OP_INC64:
505e5a9df51SDavid Vernet skel->bss->user_mutated += msg->operand_64;
506e5a9df51SDavid Vernet return 0;
507e5a9df51SDavid Vernet case TEST_MSG_OP_INC32:
508e5a9df51SDavid Vernet skel->bss->user_mutated += msg->operand_32;
509e5a9df51SDavid Vernet return 0;
510e5a9df51SDavid Vernet case TEST_MSG_OP_MUL64:
511e5a9df51SDavid Vernet skel->bss->user_mutated *= msg->operand_64;
512e5a9df51SDavid Vernet return 0;
513e5a9df51SDavid Vernet case TEST_MSG_OP_MUL32:
514e5a9df51SDavid Vernet skel->bss->user_mutated *= msg->operand_32;
515e5a9df51SDavid Vernet return 0;
516e5a9df51SDavid Vernet default:
517e5a9df51SDavid Vernet fprintf(stderr, "Invalid operand %d\n", msg->msg_op);
518e5a9df51SDavid Vernet return -EINVAL;
519e5a9df51SDavid Vernet }
520e5a9df51SDavid Vernet }
521e5a9df51SDavid Vernet
drain_kernel_messages_buffer(struct ring_buffer * kern_ringbuf,struct user_ringbuf_success * skel)522e5a9df51SDavid Vernet static void drain_kernel_messages_buffer(struct ring_buffer *kern_ringbuf,
523e5a9df51SDavid Vernet struct user_ringbuf_success *skel)
524e5a9df51SDavid Vernet {
525e5a9df51SDavid Vernet int cnt;
526e5a9df51SDavid Vernet
527e5a9df51SDavid Vernet cnt = ring_buffer__consume(kern_ringbuf);
528e5a9df51SDavid Vernet ASSERT_EQ(cnt, 8, "consume_kern_ringbuf");
529e5a9df51SDavid Vernet ASSERT_OK(skel->bss->err, "consume_kern_ringbuf_err");
530e5a9df51SDavid Vernet }
531e5a9df51SDavid Vernet
test_user_ringbuf_msg_protocol(void)532e5a9df51SDavid Vernet static void test_user_ringbuf_msg_protocol(void)
533e5a9df51SDavid Vernet {
534e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
535e5a9df51SDavid Vernet struct user_ring_buffer *user_ringbuf;
536e5a9df51SDavid Vernet struct ring_buffer *kern_ringbuf;
537e5a9df51SDavid Vernet int err, i;
538e5a9df51SDavid Vernet __u64 expected_kern = 0;
539e5a9df51SDavid Vernet
540e5a9df51SDavid Vernet err = load_skel_create_ringbufs(&skel, &kern_ringbuf, handle_kernel_msg, &user_ringbuf);
541e5a9df51SDavid Vernet if (!ASSERT_OK(err, "create_ringbufs"))
542e5a9df51SDavid Vernet return;
543e5a9df51SDavid Vernet
544e5a9df51SDavid Vernet for (i = 0; i < 64; i++) {
545e5a9df51SDavid Vernet enum test_msg_op op = i % TEST_MSG_OP_NUM_OPS;
546e5a9df51SDavid Vernet __u64 operand_64 = TEST_OP_64;
547e5a9df51SDavid Vernet __u32 operand_32 = TEST_OP_32;
548e5a9df51SDavid Vernet
549e5a9df51SDavid Vernet err = send_test_message(user_ringbuf, op, operand_64, operand_32);
550e5a9df51SDavid Vernet if (err) {
551e5a9df51SDavid Vernet /* Only assert on a failure to avoid spamming success logs. */
552e5a9df51SDavid Vernet ASSERT_OK(err, "send_test_message");
553e5a9df51SDavid Vernet goto cleanup;
554e5a9df51SDavid Vernet }
555e5a9df51SDavid Vernet
556e5a9df51SDavid Vernet switch (op) {
557e5a9df51SDavid Vernet case TEST_MSG_OP_INC64:
558e5a9df51SDavid Vernet expected_kern += operand_64;
559e5a9df51SDavid Vernet break;
560e5a9df51SDavid Vernet case TEST_MSG_OP_INC32:
561e5a9df51SDavid Vernet expected_kern += operand_32;
562e5a9df51SDavid Vernet break;
563e5a9df51SDavid Vernet case TEST_MSG_OP_MUL64:
564e5a9df51SDavid Vernet expected_kern *= operand_64;
565e5a9df51SDavid Vernet break;
566e5a9df51SDavid Vernet case TEST_MSG_OP_MUL32:
567e5a9df51SDavid Vernet expected_kern *= operand_32;
568e5a9df51SDavid Vernet break;
569e5a9df51SDavid Vernet default:
570e5a9df51SDavid Vernet PRINT_FAIL("Unexpected op %d\n", op);
571e5a9df51SDavid Vernet goto cleanup;
572e5a9df51SDavid Vernet }
573e5a9df51SDavid Vernet
574e5a9df51SDavid Vernet if (i % 8 == 0) {
575e5a9df51SDavid Vernet kick_kernel_read_messages();
576e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->kern_mutated, expected_kern, "expected_kern");
577e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->err, 0, "bpf_prog_err");
578e5a9df51SDavid Vernet drain_kernel_messages_buffer(kern_ringbuf, skel);
579e5a9df51SDavid Vernet }
580e5a9df51SDavid Vernet }
581e5a9df51SDavid Vernet
582e5a9df51SDavid Vernet cleanup:
583e5a9df51SDavid Vernet ring_buffer__free(kern_ringbuf);
584e5a9df51SDavid Vernet user_ring_buffer__free(user_ringbuf);
585e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
586e5a9df51SDavid Vernet }
587e5a9df51SDavid Vernet
kick_kernel_cb(void * arg)588e5a9df51SDavid Vernet static void *kick_kernel_cb(void *arg)
589e5a9df51SDavid Vernet {
590e5a9df51SDavid Vernet /* Kick the kernel, causing it to drain the ring buffer and then wake
591e5a9df51SDavid Vernet * up the test thread waiting on epoll.
592e5a9df51SDavid Vernet */
593*84c22fa8STiezhu Yang syscall(__NR_prlimit64);
594e5a9df51SDavid Vernet
595e5a9df51SDavid Vernet return NULL;
596e5a9df51SDavid Vernet }
597e5a9df51SDavid Vernet
spawn_kick_thread_for_poll(void)598e5a9df51SDavid Vernet static int spawn_kick_thread_for_poll(void)
599e5a9df51SDavid Vernet {
600e5a9df51SDavid Vernet pthread_t thread;
601e5a9df51SDavid Vernet
602e5a9df51SDavid Vernet return pthread_create(&thread, NULL, kick_kernel_cb, NULL);
603e5a9df51SDavid Vernet }
604e5a9df51SDavid Vernet
test_user_ringbuf_blocking_reserve(void)605e5a9df51SDavid Vernet static void test_user_ringbuf_blocking_reserve(void)
606e5a9df51SDavid Vernet {
607e5a9df51SDavid Vernet struct user_ringbuf_success *skel;
608e5a9df51SDavid Vernet struct user_ring_buffer *ringbuf;
609e5a9df51SDavid Vernet int err, num_written = 0;
610e5a9df51SDavid Vernet __u64 *token;
611e5a9df51SDavid Vernet
612e5a9df51SDavid Vernet err = load_skel_create_user_ringbuf(&skel, &ringbuf);
613e5a9df51SDavid Vernet if (err)
614e5a9df51SDavid Vernet return;
615e5a9df51SDavid Vernet
616e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
617e5a9df51SDavid Vernet
618e5a9df51SDavid Vernet while (1) {
619e5a9df51SDavid Vernet /* Write samples until the buffer is full. */
620e5a9df51SDavid Vernet token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
621e5a9df51SDavid Vernet if (!token)
622e5a9df51SDavid Vernet break;
623e5a9df51SDavid Vernet
624e5a9df51SDavid Vernet *token = 0xdeadbeef;
625e5a9df51SDavid Vernet
626e5a9df51SDavid Vernet user_ring_buffer__submit(ringbuf, token);
627e5a9df51SDavid Vernet num_written++;
628e5a9df51SDavid Vernet }
629e5a9df51SDavid Vernet
630e5a9df51SDavid Vernet if (!ASSERT_GE(num_written, 0, "num_written"))
631e5a9df51SDavid Vernet goto cleanup;
632e5a9df51SDavid Vernet
633e5a9df51SDavid Vernet /* Should not have read any samples until the kernel is kicked. */
634e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");
635e5a9df51SDavid Vernet
636e5a9df51SDavid Vernet /* We correctly time out after 1 second, without a sample. */
637e5a9df51SDavid Vernet token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 1000);
638e5a9df51SDavid Vernet if (!ASSERT_EQ(token, NULL, "pre_kick_timeout_token"))
639e5a9df51SDavid Vernet goto cleanup;
640e5a9df51SDavid Vernet
641e5a9df51SDavid Vernet err = spawn_kick_thread_for_poll();
642e5a9df51SDavid Vernet if (!ASSERT_EQ(err, 0, "deferred_kick_thread\n"))
643e5a9df51SDavid Vernet goto cleanup;
644e5a9df51SDavid Vernet
645e5a9df51SDavid Vernet /* After spawning another thread that asychronously kicks the kernel to
646e5a9df51SDavid Vernet * drain the messages, we're able to block and successfully get a
647e5a9df51SDavid Vernet * sample once we receive an event notification.
648e5a9df51SDavid Vernet */
649e5a9df51SDavid Vernet token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 10000);
650e5a9df51SDavid Vernet
651e5a9df51SDavid Vernet if (!ASSERT_OK_PTR(token, "block_token"))
652e5a9df51SDavid Vernet goto cleanup;
653e5a9df51SDavid Vernet
654e5a9df51SDavid Vernet ASSERT_GT(skel->bss->read, 0, "num_post_kill");
655e5a9df51SDavid Vernet ASSERT_LE(skel->bss->read, num_written, "num_post_kill");
656e5a9df51SDavid Vernet ASSERT_EQ(skel->bss->err, 0, "err_post_poll");
657e5a9df51SDavid Vernet user_ring_buffer__discard(ringbuf, token);
658e5a9df51SDavid Vernet
659e5a9df51SDavid Vernet cleanup:
660e5a9df51SDavid Vernet user_ring_buffer__free(ringbuf);
661e5a9df51SDavid Vernet user_ringbuf_success__destroy(skel);
662e5a9df51SDavid Vernet }
663e5a9df51SDavid Vernet
664e5a9df51SDavid Vernet #define SUCCESS_TEST(_func) { _func, #_func }
665e5a9df51SDavid Vernet
666e5a9df51SDavid Vernet static struct {
667e5a9df51SDavid Vernet void (*test_callback)(void);
668e5a9df51SDavid Vernet const char *test_name;
669e5a9df51SDavid Vernet } success_tests[] = {
670e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_mappings),
671e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_post_misaligned),
672e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_post_producer_wrong_offset),
673e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_post_larger_than_ringbuf_sz),
674e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_basic),
675e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_sample_full_ring_buffer),
676e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_post_alignment_autoadjust),
677e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_overfill),
678e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_discards_properly_ignored),
679e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_loop),
680e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_msg_protocol),
681e5a9df51SDavid Vernet SUCCESS_TEST(test_user_ringbuf_blocking_reserve),
682e5a9df51SDavid Vernet };
683e5a9df51SDavid Vernet
test_user_ringbuf(void)684e5a9df51SDavid Vernet void test_user_ringbuf(void)
685e5a9df51SDavid Vernet {
686e5a9df51SDavid Vernet int i;
687e5a9df51SDavid Vernet
688e5a9df51SDavid Vernet for (i = 0; i < ARRAY_SIZE(success_tests); i++) {
689e5a9df51SDavid Vernet if (!test__start_subtest(success_tests[i].test_name))
690e5a9df51SDavid Vernet continue;
691e5a9df51SDavid Vernet
692e5a9df51SDavid Vernet success_tests[i].test_callback();
693e5a9df51SDavid Vernet }
694e5a9df51SDavid Vernet
6958032cad1SJoanne Koong RUN_TESTS(user_ringbuf_fail);
696e5a9df51SDavid Vernet }
697