1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "kprobe_multi.skel.h"
4 #include "trace_helpers.h"
5 #include "kprobe_multi_empty.skel.h"
6 #include "kprobe_multi_override.skel.h"
7 #include "bpf/libbpf_internal.h"
8 #include "bpf/hashmap.h"
9
kprobe_multi_test_run(struct kprobe_multi * skel,bool test_return)10 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
11 {
12 LIBBPF_OPTS(bpf_test_run_opts, topts);
13 int err, prog_fd;
14
15 prog_fd = bpf_program__fd(skel->progs.trigger);
16 err = bpf_prog_test_run_opts(prog_fd, &topts);
17 ASSERT_OK(err, "test_run");
18 ASSERT_EQ(topts.retval, 0, "test_run");
19
20 ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
21 ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
22 ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
23 ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
24 ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
25 ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
26 ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
27 ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
28
29 if (test_return) {
30 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
31 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
32 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
33 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
34 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
35 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
36 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
37 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
38 }
39 }
40
test_skel_api(void)41 static void test_skel_api(void)
42 {
43 struct kprobe_multi *skel = NULL;
44 int err;
45
46 skel = kprobe_multi__open_and_load();
47 if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
48 goto cleanup;
49
50 skel->bss->pid = getpid();
51 err = kprobe_multi__attach(skel);
52 if (!ASSERT_OK(err, "kprobe_multi__attach"))
53 goto cleanup;
54
55 kprobe_multi_test_run(skel, true);
56
57 cleanup:
58 kprobe_multi__destroy(skel);
59 }
60
test_link_api(struct bpf_link_create_opts * opts)61 static void test_link_api(struct bpf_link_create_opts *opts)
62 {
63 int prog_fd, link1_fd = -1, link2_fd = -1;
64 struct kprobe_multi *skel = NULL;
65
66 skel = kprobe_multi__open_and_load();
67 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
68 goto cleanup;
69
70 skel->bss->pid = getpid();
71 prog_fd = bpf_program__fd(skel->progs.test_kprobe);
72 link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
73 if (!ASSERT_GE(link1_fd, 0, "link_fd"))
74 goto cleanup;
75
76 opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
77 prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
78 link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
79 if (!ASSERT_GE(link2_fd, 0, "link_fd"))
80 goto cleanup;
81
82 kprobe_multi_test_run(skel, true);
83
84 cleanup:
85 if (link1_fd != -1)
86 close(link1_fd);
87 if (link2_fd != -1)
88 close(link2_fd);
89 kprobe_multi__destroy(skel);
90 }
91
92 #define GET_ADDR(__sym, __addr) ({ \
93 __addr = ksym_get_addr(__sym); \
94 if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
95 return; \
96 })
97
test_link_api_addrs(void)98 static void test_link_api_addrs(void)
99 {
100 LIBBPF_OPTS(bpf_link_create_opts, opts);
101 unsigned long long addrs[8];
102
103 GET_ADDR("bpf_fentry_test1", addrs[0]);
104 GET_ADDR("bpf_fentry_test2", addrs[1]);
105 GET_ADDR("bpf_fentry_test3", addrs[2]);
106 GET_ADDR("bpf_fentry_test4", addrs[3]);
107 GET_ADDR("bpf_fentry_test5", addrs[4]);
108 GET_ADDR("bpf_fentry_test6", addrs[5]);
109 GET_ADDR("bpf_fentry_test7", addrs[6]);
110 GET_ADDR("bpf_fentry_test8", addrs[7]);
111
112 opts.kprobe_multi.addrs = (const unsigned long*) addrs;
113 opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
114 test_link_api(&opts);
115 }
116
test_link_api_syms(void)117 static void test_link_api_syms(void)
118 {
119 LIBBPF_OPTS(bpf_link_create_opts, opts);
120 const char *syms[8] = {
121 "bpf_fentry_test1",
122 "bpf_fentry_test2",
123 "bpf_fentry_test3",
124 "bpf_fentry_test4",
125 "bpf_fentry_test5",
126 "bpf_fentry_test6",
127 "bpf_fentry_test7",
128 "bpf_fentry_test8",
129 };
130
131 opts.kprobe_multi.syms = syms;
132 opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
133 test_link_api(&opts);
134 }
135
136 static void
test_attach_api(const char * pattern,struct bpf_kprobe_multi_opts * opts)137 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
138 {
139 struct bpf_link *link1 = NULL, *link2 = NULL;
140 struct kprobe_multi *skel = NULL;
141
142 skel = kprobe_multi__open_and_load();
143 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
144 goto cleanup;
145
146 skel->bss->pid = getpid();
147 link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
148 pattern, opts);
149 if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
150 goto cleanup;
151
152 if (opts) {
153 opts->retprobe = true;
154 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
155 pattern, opts);
156 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
157 goto cleanup;
158 }
159
160 kprobe_multi_test_run(skel, !!opts);
161
162 cleanup:
163 bpf_link__destroy(link2);
164 bpf_link__destroy(link1);
165 kprobe_multi__destroy(skel);
166 }
167
test_attach_api_pattern(void)168 static void test_attach_api_pattern(void)
169 {
170 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
171
172 test_attach_api("bpf_fentry_test*", &opts);
173 test_attach_api("bpf_fentry_test?", NULL);
174 }
175
test_attach_api_addrs(void)176 static void test_attach_api_addrs(void)
177 {
178 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
179 unsigned long long addrs[8];
180
181 GET_ADDR("bpf_fentry_test1", addrs[0]);
182 GET_ADDR("bpf_fentry_test2", addrs[1]);
183 GET_ADDR("bpf_fentry_test3", addrs[2]);
184 GET_ADDR("bpf_fentry_test4", addrs[3]);
185 GET_ADDR("bpf_fentry_test5", addrs[4]);
186 GET_ADDR("bpf_fentry_test6", addrs[5]);
187 GET_ADDR("bpf_fentry_test7", addrs[6]);
188 GET_ADDR("bpf_fentry_test8", addrs[7]);
189
190 opts.addrs = (const unsigned long *) addrs;
191 opts.cnt = ARRAY_SIZE(addrs);
192 test_attach_api(NULL, &opts);
193 }
194
test_attach_api_syms(void)195 static void test_attach_api_syms(void)
196 {
197 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
198 const char *syms[8] = {
199 "bpf_fentry_test1",
200 "bpf_fentry_test2",
201 "bpf_fentry_test3",
202 "bpf_fentry_test4",
203 "bpf_fentry_test5",
204 "bpf_fentry_test6",
205 "bpf_fentry_test7",
206 "bpf_fentry_test8",
207 };
208
209 opts.syms = syms;
210 opts.cnt = ARRAY_SIZE(syms);
211 test_attach_api(NULL, &opts);
212 }
213
test_attach_api_fails(void)214 static void test_attach_api_fails(void)
215 {
216 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
217 struct kprobe_multi *skel = NULL;
218 struct bpf_link *link = NULL;
219 unsigned long long addrs[2];
220 const char *syms[2] = {
221 "bpf_fentry_test1",
222 "bpf_fentry_test2",
223 };
224 __u64 cookies[2];
225
226 addrs[0] = ksym_get_addr("bpf_fentry_test1");
227 addrs[1] = ksym_get_addr("bpf_fentry_test2");
228
229 if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
230 goto cleanup;
231
232 skel = kprobe_multi__open_and_load();
233 if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
234 goto cleanup;
235
236 skel->bss->pid = getpid();
237
238 /* fail_1 - pattern and opts NULL */
239 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
240 NULL, NULL);
241 if (!ASSERT_ERR_PTR(link, "fail_1"))
242 goto cleanup;
243
244 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
245 goto cleanup;
246
247 /* fail_2 - both addrs and syms set */
248 opts.addrs = (const unsigned long *) addrs;
249 opts.syms = syms;
250 opts.cnt = ARRAY_SIZE(syms);
251 opts.cookies = NULL;
252
253 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
254 NULL, &opts);
255 if (!ASSERT_ERR_PTR(link, "fail_2"))
256 goto cleanup;
257
258 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
259 goto cleanup;
260
261 /* fail_3 - pattern and addrs set */
262 opts.addrs = (const unsigned long *) addrs;
263 opts.syms = NULL;
264 opts.cnt = ARRAY_SIZE(syms);
265 opts.cookies = NULL;
266
267 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
268 "ksys_*", &opts);
269 if (!ASSERT_ERR_PTR(link, "fail_3"))
270 goto cleanup;
271
272 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
273 goto cleanup;
274
275 /* fail_4 - pattern and cnt set */
276 opts.addrs = NULL;
277 opts.syms = NULL;
278 opts.cnt = ARRAY_SIZE(syms);
279 opts.cookies = NULL;
280
281 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
282 "ksys_*", &opts);
283 if (!ASSERT_ERR_PTR(link, "fail_4"))
284 goto cleanup;
285
286 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
287 goto cleanup;
288
289 /* fail_5 - pattern and cookies */
290 opts.addrs = NULL;
291 opts.syms = NULL;
292 opts.cnt = 0;
293 opts.cookies = cookies;
294
295 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
296 "ksys_*", &opts);
297 if (!ASSERT_ERR_PTR(link, "fail_5"))
298 goto cleanup;
299
300 if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
301 goto cleanup;
302
303 cleanup:
304 bpf_link__destroy(link);
305 kprobe_multi__destroy(skel);
306 }
307
symbol_hash(long key,void * ctx __maybe_unused)308 static size_t symbol_hash(long key, void *ctx __maybe_unused)
309 {
310 return str_hash((const char *) key);
311 }
312
symbol_equal(long key1,long key2,void * ctx __maybe_unused)313 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
314 {
315 return strcmp((const char *) key1, (const char *) key2) == 0;
316 }
317
get_syms(char *** symsp,size_t * cntp,bool kernel)318 static int get_syms(char ***symsp, size_t *cntp, bool kernel)
319 {
320 size_t cap = 0, cnt = 0, i;
321 char *name = NULL, **syms = NULL;
322 struct hashmap *map;
323 char buf[256];
324 FILE *f;
325 int err = 0;
326
327 /*
328 * The available_filter_functions contains many duplicates,
329 * but other than that all symbols are usable in kprobe multi
330 * interface.
331 * Filtering out duplicates by using hashmap__add, which won't
332 * add existing entry.
333 */
334
335 if (access("/sys/kernel/tracing/trace", F_OK) == 0)
336 f = fopen("/sys/kernel/tracing/available_filter_functions", "r");
337 else
338 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
339
340 if (!f)
341 return -EINVAL;
342
343 map = hashmap__new(symbol_hash, symbol_equal, NULL);
344 if (IS_ERR(map)) {
345 err = libbpf_get_error(map);
346 goto error;
347 }
348
349 while (fgets(buf, sizeof(buf), f)) {
350 if (kernel && strchr(buf, '['))
351 continue;
352 if (!kernel && !strchr(buf, '['))
353 continue;
354
355 free(name);
356 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
357 continue;
358 /*
359 * We attach to almost all kernel functions and some of them
360 * will cause 'suspicious RCU usage' when fprobe is attached
361 * to them. Filter out the current culprits - arch_cpu_idle
362 * default_idle and rcu_* functions.
363 */
364 if (!strcmp(name, "arch_cpu_idle"))
365 continue;
366 if (!strcmp(name, "default_idle"))
367 continue;
368 if (!strncmp(name, "rcu_", 4))
369 continue;
370 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
371 continue;
372 if (!strncmp(name, "__ftrace_invalid_address__",
373 sizeof("__ftrace_invalid_address__") - 1))
374 continue;
375
376 err = hashmap__add(map, name, 0);
377 if (err == -EEXIST) {
378 err = 0;
379 continue;
380 }
381 if (err)
382 goto error;
383
384 err = libbpf_ensure_mem((void **) &syms, &cap,
385 sizeof(*syms), cnt + 1);
386 if (err)
387 goto error;
388
389 syms[cnt++] = name;
390 name = NULL;
391 }
392
393 *symsp = syms;
394 *cntp = cnt;
395
396 error:
397 free(name);
398 fclose(f);
399 hashmap__free(map);
400 if (err) {
401 for (i = 0; i < cnt; i++)
402 free(syms[i]);
403 free(syms);
404 }
405 return err;
406 }
407
test_kprobe_multi_bench_attach(bool kernel)408 static void test_kprobe_multi_bench_attach(bool kernel)
409 {
410 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
411 struct kprobe_multi_empty *skel = NULL;
412 long attach_start_ns, attach_end_ns;
413 long detach_start_ns, detach_end_ns;
414 double attach_delta, detach_delta;
415 struct bpf_link *link = NULL;
416 char **syms = NULL;
417 size_t cnt = 0, i;
418
419 if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
420 return;
421
422 skel = kprobe_multi_empty__open_and_load();
423 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
424 goto cleanup;
425
426 opts.syms = (const char **) syms;
427 opts.cnt = cnt;
428
429 attach_start_ns = get_time_ns();
430 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
431 NULL, &opts);
432 attach_end_ns = get_time_ns();
433
434 if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
435 goto cleanup;
436
437 detach_start_ns = get_time_ns();
438 bpf_link__destroy(link);
439 detach_end_ns = get_time_ns();
440
441 attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
442 detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
443
444 printf("%s: found %lu functions\n", __func__, cnt);
445 printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
446 printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
447
448 cleanup:
449 kprobe_multi_empty__destroy(skel);
450 if (syms) {
451 for (i = 0; i < cnt; i++)
452 free(syms[i]);
453 free(syms);
454 }
455 }
456
test_attach_override(void)457 static void test_attach_override(void)
458 {
459 struct kprobe_multi_override *skel = NULL;
460 struct bpf_link *link = NULL;
461
462 skel = kprobe_multi_override__open_and_load();
463 if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
464 goto cleanup;
465
466 /* The test_override calls bpf_override_return so it should fail
467 * to attach to bpf_fentry_test1 function, which is not on error
468 * injection list.
469 */
470 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
471 "bpf_fentry_test1", NULL);
472 if (!ASSERT_ERR_PTR(link, "override_attached_bpf_fentry_test1")) {
473 bpf_link__destroy(link);
474 goto cleanup;
475 }
476
477 /* The should_fail_bio function is on error injection list,
478 * attach should succeed.
479 */
480 link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
481 "should_fail_bio", NULL);
482 if (!ASSERT_OK_PTR(link, "override_attached_should_fail_bio"))
483 goto cleanup;
484
485 bpf_link__destroy(link);
486
487 cleanup:
488 kprobe_multi_override__destroy(skel);
489 }
490
serial_test_kprobe_multi_bench_attach(void)491 void serial_test_kprobe_multi_bench_attach(void)
492 {
493 if (test__start_subtest("kernel"))
494 test_kprobe_multi_bench_attach(true);
495 if (test__start_subtest("modules"))
496 test_kprobe_multi_bench_attach(false);
497 }
498
test_kprobe_multi_test(void)499 void test_kprobe_multi_test(void)
500 {
501 if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
502 return;
503
504 if (test__start_subtest("skel_api"))
505 test_skel_api();
506 if (test__start_subtest("link_api_addrs"))
507 test_link_api_syms();
508 if (test__start_subtest("link_api_syms"))
509 test_link_api_addrs();
510 if (test__start_subtest("attach_api_pattern"))
511 test_attach_api_pattern();
512 if (test__start_subtest("attach_api_addrs"))
513 test_attach_api_addrs();
514 if (test__start_subtest("attach_api_syms"))
515 test_attach_api_syms();
516 if (test__start_subtest("attach_api_fails"))
517 test_attach_api_fails();
518 if (test__start_subtest("attach_override"))
519 test_attach_override();
520 }
521