1c5d420c3SAndrii Nakryiko // SPDX-License-Identifier: GPL-2.0
2c5d420c3SAndrii Nakryiko /* Copyright (c) 2020 Facebook */
3c5d420c3SAndrii Nakryiko #include "bench.h"
4c5d420c3SAndrii Nakryiko #include "trigger_bench.skel.h"
5d41bc48bSAndrii Nakryiko #include "trace_helpers.h"
6c5d420c3SAndrii Nakryiko
7c5d420c3SAndrii Nakryiko /* BPF triggering benchmarks */
8c5d420c3SAndrii Nakryiko static struct trigger_ctx {
9c5d420c3SAndrii Nakryiko struct trigger_bench *skel;
10c5d420c3SAndrii Nakryiko } ctx;
11c5d420c3SAndrii Nakryiko
12c5d420c3SAndrii Nakryiko static struct counter base_hits;
13c5d420c3SAndrii Nakryiko
trigger_validate(void)149a93bf3fSHou Tao static void trigger_validate(void)
15c5d420c3SAndrii Nakryiko {
16*970308a7SHou Tao if (env.consumer_cnt != 0) {
17*970308a7SHou Tao fprintf(stderr, "benchmark doesn't support consumer!\n");
18c5d420c3SAndrii Nakryiko exit(1);
19c5d420c3SAndrii Nakryiko }
20c5d420c3SAndrii Nakryiko }
21c5d420c3SAndrii Nakryiko
trigger_base_producer(void * input)22c5d420c3SAndrii Nakryiko static void *trigger_base_producer(void *input)
23c5d420c3SAndrii Nakryiko {
24c5d420c3SAndrii Nakryiko while (true) {
25c5d420c3SAndrii Nakryiko (void)syscall(__NR_getpgid);
26c5d420c3SAndrii Nakryiko atomic_inc(&base_hits.value);
27c5d420c3SAndrii Nakryiko }
28c5d420c3SAndrii Nakryiko return NULL;
29c5d420c3SAndrii Nakryiko }
30c5d420c3SAndrii Nakryiko
trigger_base_measure(struct bench_res * res)31c5d420c3SAndrii Nakryiko static void trigger_base_measure(struct bench_res *res)
32c5d420c3SAndrii Nakryiko {
33c5d420c3SAndrii Nakryiko res->hits = atomic_swap(&base_hits.value, 0);
34c5d420c3SAndrii Nakryiko }
35c5d420c3SAndrii Nakryiko
trigger_producer(void * input)36c5d420c3SAndrii Nakryiko static void *trigger_producer(void *input)
37c5d420c3SAndrii Nakryiko {
38c5d420c3SAndrii Nakryiko while (true)
39c5d420c3SAndrii Nakryiko (void)syscall(__NR_getpgid);
40c5d420c3SAndrii Nakryiko return NULL;
41c5d420c3SAndrii Nakryiko }
42c5d420c3SAndrii Nakryiko
trigger_measure(struct bench_res * res)43c5d420c3SAndrii Nakryiko static void trigger_measure(struct bench_res *res)
44c5d420c3SAndrii Nakryiko {
45c5d420c3SAndrii Nakryiko res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
46c5d420c3SAndrii Nakryiko }
47c5d420c3SAndrii Nakryiko
setup_ctx(void)489a93bf3fSHou Tao static void setup_ctx(void)
49c5d420c3SAndrii Nakryiko {
50c5d420c3SAndrii Nakryiko setup_libbpf();
51c5d420c3SAndrii Nakryiko
52c5d420c3SAndrii Nakryiko ctx.skel = trigger_bench__open_and_load();
53c5d420c3SAndrii Nakryiko if (!ctx.skel) {
54c5d420c3SAndrii Nakryiko fprintf(stderr, "failed to open skeleton\n");
55c5d420c3SAndrii Nakryiko exit(1);
56c5d420c3SAndrii Nakryiko }
57c5d420c3SAndrii Nakryiko }
58c5d420c3SAndrii Nakryiko
attach_bpf(struct bpf_program * prog)59c5d420c3SAndrii Nakryiko static void attach_bpf(struct bpf_program *prog)
60c5d420c3SAndrii Nakryiko {
61c5d420c3SAndrii Nakryiko struct bpf_link *link;
62c5d420c3SAndrii Nakryiko
63c5d420c3SAndrii Nakryiko link = bpf_program__attach(prog);
64bad2e478SAndrii Nakryiko if (!link) {
65c5d420c3SAndrii Nakryiko fprintf(stderr, "failed to attach program!\n");
66c5d420c3SAndrii Nakryiko exit(1);
67c5d420c3SAndrii Nakryiko }
68c5d420c3SAndrii Nakryiko }
69c5d420c3SAndrii Nakryiko
trigger_tp_setup(void)709a93bf3fSHou Tao static void trigger_tp_setup(void)
71c5d420c3SAndrii Nakryiko {
72c5d420c3SAndrii Nakryiko setup_ctx();
73c5d420c3SAndrii Nakryiko attach_bpf(ctx.skel->progs.bench_trigger_tp);
74c5d420c3SAndrii Nakryiko }
75c5d420c3SAndrii Nakryiko
trigger_rawtp_setup(void)769a93bf3fSHou Tao static void trigger_rawtp_setup(void)
77c5d420c3SAndrii Nakryiko {
78c5d420c3SAndrii Nakryiko setup_ctx();
79c5d420c3SAndrii Nakryiko attach_bpf(ctx.skel->progs.bench_trigger_raw_tp);
80c5d420c3SAndrii Nakryiko }
81c5d420c3SAndrii Nakryiko
trigger_kprobe_setup(void)829a93bf3fSHou Tao static void trigger_kprobe_setup(void)
83c5d420c3SAndrii Nakryiko {
84c5d420c3SAndrii Nakryiko setup_ctx();
85c5d420c3SAndrii Nakryiko attach_bpf(ctx.skel->progs.bench_trigger_kprobe);
86c5d420c3SAndrii Nakryiko }
87c5d420c3SAndrii Nakryiko
trigger_fentry_setup(void)889a93bf3fSHou Tao static void trigger_fentry_setup(void)
89c5d420c3SAndrii Nakryiko {
90c5d420c3SAndrii Nakryiko setup_ctx();
91c5d420c3SAndrii Nakryiko attach_bpf(ctx.skel->progs.bench_trigger_fentry);
92c5d420c3SAndrii Nakryiko }
93c5d420c3SAndrii Nakryiko
trigger_fentry_sleep_setup(void)949a93bf3fSHou Tao static void trigger_fentry_sleep_setup(void)
95e68a1445SAlexei Starovoitov {
96e68a1445SAlexei Starovoitov setup_ctx();
97e68a1445SAlexei Starovoitov attach_bpf(ctx.skel->progs.bench_trigger_fentry_sleep);
98e68a1445SAlexei Starovoitov }
99e68a1445SAlexei Starovoitov
trigger_fmodret_setup(void)1009a93bf3fSHou Tao static void trigger_fmodret_setup(void)
101c5d420c3SAndrii Nakryiko {
102c5d420c3SAndrii Nakryiko setup_ctx();
103c5d420c3SAndrii Nakryiko attach_bpf(ctx.skel->progs.bench_trigger_fmodret);
104c5d420c3SAndrii Nakryiko }
105c5d420c3SAndrii Nakryiko
106d41bc48bSAndrii Nakryiko /* make sure call is not inlined and not avoided by compiler, so __weak and
107d41bc48bSAndrii Nakryiko * inline asm volatile in the body of the function
108d41bc48bSAndrii Nakryiko *
109d41bc48bSAndrii Nakryiko * There is a performance difference between uprobing at nop location vs other
110d41bc48bSAndrii Nakryiko * instructions. So use two different targets, one of which starts with nop
111d41bc48bSAndrii Nakryiko * and another doesn't.
112d41bc48bSAndrii Nakryiko *
113d41bc48bSAndrii Nakryiko * GCC doesn't generate stack setup preample for these functions due to them
114d41bc48bSAndrii Nakryiko * having no input arguments and doing nothing in the body.
115d41bc48bSAndrii Nakryiko */
uprobe_target_with_nop(void)116d41bc48bSAndrii Nakryiko __weak void uprobe_target_with_nop(void)
117d41bc48bSAndrii Nakryiko {
118d41bc48bSAndrii Nakryiko asm volatile ("nop");
119d41bc48bSAndrii Nakryiko }
120d41bc48bSAndrii Nakryiko
uprobe_target_without_nop(void)121d41bc48bSAndrii Nakryiko __weak void uprobe_target_without_nop(void)
122d41bc48bSAndrii Nakryiko {
123d41bc48bSAndrii Nakryiko asm volatile ("");
124d41bc48bSAndrii Nakryiko }
125d41bc48bSAndrii Nakryiko
uprobe_base_producer(void * input)126d41bc48bSAndrii Nakryiko static void *uprobe_base_producer(void *input)
127d41bc48bSAndrii Nakryiko {
128d41bc48bSAndrii Nakryiko while (true) {
129d41bc48bSAndrii Nakryiko uprobe_target_with_nop();
130d41bc48bSAndrii Nakryiko atomic_inc(&base_hits.value);
131d41bc48bSAndrii Nakryiko }
132d41bc48bSAndrii Nakryiko return NULL;
133d41bc48bSAndrii Nakryiko }
134d41bc48bSAndrii Nakryiko
uprobe_producer_with_nop(void * input)135d41bc48bSAndrii Nakryiko static void *uprobe_producer_with_nop(void *input)
136d41bc48bSAndrii Nakryiko {
137d41bc48bSAndrii Nakryiko while (true)
138d41bc48bSAndrii Nakryiko uprobe_target_with_nop();
139d41bc48bSAndrii Nakryiko return NULL;
140d41bc48bSAndrii Nakryiko }
141d41bc48bSAndrii Nakryiko
uprobe_producer_without_nop(void * input)142d41bc48bSAndrii Nakryiko static void *uprobe_producer_without_nop(void *input)
143d41bc48bSAndrii Nakryiko {
144d41bc48bSAndrii Nakryiko while (true)
145d41bc48bSAndrii Nakryiko uprobe_target_without_nop();
146d41bc48bSAndrii Nakryiko return NULL;
147d41bc48bSAndrii Nakryiko }
148d41bc48bSAndrii Nakryiko
usetup(bool use_retprobe,bool use_nop)149d41bc48bSAndrii Nakryiko static void usetup(bool use_retprobe, bool use_nop)
150d41bc48bSAndrii Nakryiko {
151d41bc48bSAndrii Nakryiko size_t uprobe_offset;
152d41bc48bSAndrii Nakryiko struct bpf_link *link;
153d41bc48bSAndrii Nakryiko
154d41bc48bSAndrii Nakryiko setup_libbpf();
155d41bc48bSAndrii Nakryiko
156d41bc48bSAndrii Nakryiko ctx.skel = trigger_bench__open_and_load();
157d41bc48bSAndrii Nakryiko if (!ctx.skel) {
158d41bc48bSAndrii Nakryiko fprintf(stderr, "failed to open skeleton\n");
159d41bc48bSAndrii Nakryiko exit(1);
160d41bc48bSAndrii Nakryiko }
161d41bc48bSAndrii Nakryiko
162d41bc48bSAndrii Nakryiko if (use_nop)
163ff943683SAndrii Nakryiko uprobe_offset = get_uprobe_offset(&uprobe_target_with_nop);
164d41bc48bSAndrii Nakryiko else
165ff943683SAndrii Nakryiko uprobe_offset = get_uprobe_offset(&uprobe_target_without_nop);
166d41bc48bSAndrii Nakryiko
167d41bc48bSAndrii Nakryiko link = bpf_program__attach_uprobe(ctx.skel->progs.bench_trigger_uprobe,
168d41bc48bSAndrii Nakryiko use_retprobe,
169d41bc48bSAndrii Nakryiko -1 /* all PIDs */,
170d41bc48bSAndrii Nakryiko "/proc/self/exe",
171d41bc48bSAndrii Nakryiko uprobe_offset);
172d41bc48bSAndrii Nakryiko if (!link) {
173d41bc48bSAndrii Nakryiko fprintf(stderr, "failed to attach uprobe!\n");
174d41bc48bSAndrii Nakryiko exit(1);
175d41bc48bSAndrii Nakryiko }
176d41bc48bSAndrii Nakryiko ctx.skel->links.bench_trigger_uprobe = link;
177d41bc48bSAndrii Nakryiko }
178d41bc48bSAndrii Nakryiko
uprobe_setup_with_nop(void)1799a93bf3fSHou Tao static void uprobe_setup_with_nop(void)
180d41bc48bSAndrii Nakryiko {
181d41bc48bSAndrii Nakryiko usetup(false, true);
182d41bc48bSAndrii Nakryiko }
183d41bc48bSAndrii Nakryiko
uretprobe_setup_with_nop(void)1849a93bf3fSHou Tao static void uretprobe_setup_with_nop(void)
185d41bc48bSAndrii Nakryiko {
186d41bc48bSAndrii Nakryiko usetup(true, true);
187d41bc48bSAndrii Nakryiko }
188d41bc48bSAndrii Nakryiko
uprobe_setup_without_nop(void)1899a93bf3fSHou Tao static void uprobe_setup_without_nop(void)
190d41bc48bSAndrii Nakryiko {
191d41bc48bSAndrii Nakryiko usetup(false, false);
192d41bc48bSAndrii Nakryiko }
193d41bc48bSAndrii Nakryiko
uretprobe_setup_without_nop(void)1949a93bf3fSHou Tao static void uretprobe_setup_without_nop(void)
195d41bc48bSAndrii Nakryiko {
196d41bc48bSAndrii Nakryiko usetup(true, false);
197d41bc48bSAndrii Nakryiko }
198d41bc48bSAndrii Nakryiko
199c5d420c3SAndrii Nakryiko const struct bench bench_trig_base = {
200c5d420c3SAndrii Nakryiko .name = "trig-base",
201c5d420c3SAndrii Nakryiko .validate = trigger_validate,
202c5d420c3SAndrii Nakryiko .producer_thread = trigger_base_producer,
203c5d420c3SAndrii Nakryiko .measure = trigger_base_measure,
204c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
205c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
206c5d420c3SAndrii Nakryiko };
207c5d420c3SAndrii Nakryiko
208c5d420c3SAndrii Nakryiko const struct bench bench_trig_tp = {
209c5d420c3SAndrii Nakryiko .name = "trig-tp",
210c5d420c3SAndrii Nakryiko .validate = trigger_validate,
211c5d420c3SAndrii Nakryiko .setup = trigger_tp_setup,
212c5d420c3SAndrii Nakryiko .producer_thread = trigger_producer,
213c5d420c3SAndrii Nakryiko .measure = trigger_measure,
214c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
215c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
216c5d420c3SAndrii Nakryiko };
217c5d420c3SAndrii Nakryiko
218c5d420c3SAndrii Nakryiko const struct bench bench_trig_rawtp = {
219c5d420c3SAndrii Nakryiko .name = "trig-rawtp",
220c5d420c3SAndrii Nakryiko .validate = trigger_validate,
221c5d420c3SAndrii Nakryiko .setup = trigger_rawtp_setup,
222c5d420c3SAndrii Nakryiko .producer_thread = trigger_producer,
223c5d420c3SAndrii Nakryiko .measure = trigger_measure,
224c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
225c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
226c5d420c3SAndrii Nakryiko };
227c5d420c3SAndrii Nakryiko
228c5d420c3SAndrii Nakryiko const struct bench bench_trig_kprobe = {
229c5d420c3SAndrii Nakryiko .name = "trig-kprobe",
230c5d420c3SAndrii Nakryiko .validate = trigger_validate,
231c5d420c3SAndrii Nakryiko .setup = trigger_kprobe_setup,
232c5d420c3SAndrii Nakryiko .producer_thread = trigger_producer,
233c5d420c3SAndrii Nakryiko .measure = trigger_measure,
234c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
235c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
236c5d420c3SAndrii Nakryiko };
237c5d420c3SAndrii Nakryiko
238c5d420c3SAndrii Nakryiko const struct bench bench_trig_fentry = {
239c5d420c3SAndrii Nakryiko .name = "trig-fentry",
240c5d420c3SAndrii Nakryiko .validate = trigger_validate,
241c5d420c3SAndrii Nakryiko .setup = trigger_fentry_setup,
242c5d420c3SAndrii Nakryiko .producer_thread = trigger_producer,
243c5d420c3SAndrii Nakryiko .measure = trigger_measure,
244c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
245c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
246c5d420c3SAndrii Nakryiko };
247c5d420c3SAndrii Nakryiko
248e68a1445SAlexei Starovoitov const struct bench bench_trig_fentry_sleep = {
249e68a1445SAlexei Starovoitov .name = "trig-fentry-sleep",
250e68a1445SAlexei Starovoitov .validate = trigger_validate,
251e68a1445SAlexei Starovoitov .setup = trigger_fentry_sleep_setup,
252e68a1445SAlexei Starovoitov .producer_thread = trigger_producer,
253e68a1445SAlexei Starovoitov .measure = trigger_measure,
254e68a1445SAlexei Starovoitov .report_progress = hits_drops_report_progress,
255e68a1445SAlexei Starovoitov .report_final = hits_drops_report_final,
256e68a1445SAlexei Starovoitov };
257e68a1445SAlexei Starovoitov
258c5d420c3SAndrii Nakryiko const struct bench bench_trig_fmodret = {
259c5d420c3SAndrii Nakryiko .name = "trig-fmodret",
260c5d420c3SAndrii Nakryiko .validate = trigger_validate,
261c5d420c3SAndrii Nakryiko .setup = trigger_fmodret_setup,
262c5d420c3SAndrii Nakryiko .producer_thread = trigger_producer,
263c5d420c3SAndrii Nakryiko .measure = trigger_measure,
264c5d420c3SAndrii Nakryiko .report_progress = hits_drops_report_progress,
265c5d420c3SAndrii Nakryiko .report_final = hits_drops_report_final,
266c5d420c3SAndrii Nakryiko };
267d41bc48bSAndrii Nakryiko
268d41bc48bSAndrii Nakryiko const struct bench bench_trig_uprobe_base = {
269d41bc48bSAndrii Nakryiko .name = "trig-uprobe-base",
270d41bc48bSAndrii Nakryiko .setup = NULL, /* no uprobe/uretprobe is attached */
271d41bc48bSAndrii Nakryiko .producer_thread = uprobe_base_producer,
272d41bc48bSAndrii Nakryiko .measure = trigger_base_measure,
273d41bc48bSAndrii Nakryiko .report_progress = hits_drops_report_progress,
274d41bc48bSAndrii Nakryiko .report_final = hits_drops_report_final,
275d41bc48bSAndrii Nakryiko };
276d41bc48bSAndrii Nakryiko
277d41bc48bSAndrii Nakryiko const struct bench bench_trig_uprobe_with_nop = {
278d41bc48bSAndrii Nakryiko .name = "trig-uprobe-with-nop",
279d41bc48bSAndrii Nakryiko .setup = uprobe_setup_with_nop,
280d41bc48bSAndrii Nakryiko .producer_thread = uprobe_producer_with_nop,
281d41bc48bSAndrii Nakryiko .measure = trigger_measure,
282d41bc48bSAndrii Nakryiko .report_progress = hits_drops_report_progress,
283d41bc48bSAndrii Nakryiko .report_final = hits_drops_report_final,
284d41bc48bSAndrii Nakryiko };
285d41bc48bSAndrii Nakryiko
286d41bc48bSAndrii Nakryiko const struct bench bench_trig_uretprobe_with_nop = {
287d41bc48bSAndrii Nakryiko .name = "trig-uretprobe-with-nop",
288d41bc48bSAndrii Nakryiko .setup = uretprobe_setup_with_nop,
289d41bc48bSAndrii Nakryiko .producer_thread = uprobe_producer_with_nop,
290d41bc48bSAndrii Nakryiko .measure = trigger_measure,
291d41bc48bSAndrii Nakryiko .report_progress = hits_drops_report_progress,
292d41bc48bSAndrii Nakryiko .report_final = hits_drops_report_final,
293d41bc48bSAndrii Nakryiko };
294d41bc48bSAndrii Nakryiko
295d41bc48bSAndrii Nakryiko const struct bench bench_trig_uprobe_without_nop = {
296d41bc48bSAndrii Nakryiko .name = "trig-uprobe-without-nop",
297d41bc48bSAndrii Nakryiko .setup = uprobe_setup_without_nop,
298d41bc48bSAndrii Nakryiko .producer_thread = uprobe_producer_without_nop,
299d41bc48bSAndrii Nakryiko .measure = trigger_measure,
300d41bc48bSAndrii Nakryiko .report_progress = hits_drops_report_progress,
301d41bc48bSAndrii Nakryiko .report_final = hits_drops_report_final,
302d41bc48bSAndrii Nakryiko };
303d41bc48bSAndrii Nakryiko
304d41bc48bSAndrii Nakryiko const struct bench bench_trig_uretprobe_without_nop = {
305d41bc48bSAndrii Nakryiko .name = "trig-uretprobe-without-nop",
306d41bc48bSAndrii Nakryiko .setup = uretprobe_setup_without_nop,
307d41bc48bSAndrii Nakryiko .producer_thread = uprobe_producer_without_nop,
308d41bc48bSAndrii Nakryiko .measure = trigger_measure,
309d41bc48bSAndrii Nakryiko .report_progress = hits_drops_report_progress,
310d41bc48bSAndrii Nakryiko .report_final = hits_drops_report_final,
311d41bc48bSAndrii Nakryiko };
312