1 // SPDX-License-Identifier: GPL-2.0
2 #include <test_progs.h>
3 #include "kprobe_multi.skel.h"
4 #include "trace_helpers.h"
5 #include "kprobe_multi_empty.skel.h"
6 #include "bpf/libbpf_internal.h"
7 #include "bpf/hashmap.h"
8 
9 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
10 {
11 	LIBBPF_OPTS(bpf_test_run_opts, topts);
12 	int err, prog_fd;
13 
14 	prog_fd = bpf_program__fd(skel->progs.trigger);
15 	err = bpf_prog_test_run_opts(prog_fd, &topts);
16 	ASSERT_OK(err, "test_run");
17 	ASSERT_EQ(topts.retval, 0, "test_run");
18 
19 	ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
20 	ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
21 	ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
22 	ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
23 	ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
24 	ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
25 	ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
26 	ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
27 
28 	if (test_return) {
29 		ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
30 		ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
31 		ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
32 		ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
33 		ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
34 		ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
35 		ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
36 		ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
37 	}
38 }
39 
40 static void test_skel_api(void)
41 {
42 	struct kprobe_multi *skel = NULL;
43 	int err;
44 
45 	skel = kprobe_multi__open_and_load();
46 	if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
47 		goto cleanup;
48 
49 	skel->bss->pid = getpid();
50 	err = kprobe_multi__attach(skel);
51 	if (!ASSERT_OK(err, "kprobe_multi__attach"))
52 		goto cleanup;
53 
54 	kprobe_multi_test_run(skel, true);
55 
56 cleanup:
57 	kprobe_multi__destroy(skel);
58 }
59 
60 static void test_link_api(struct bpf_link_create_opts *opts)
61 {
62 	int prog_fd, link1_fd = -1, link2_fd = -1;
63 	struct kprobe_multi *skel = NULL;
64 
65 	skel = kprobe_multi__open_and_load();
66 	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
67 		goto cleanup;
68 
69 	skel->bss->pid = getpid();
70 	prog_fd = bpf_program__fd(skel->progs.test_kprobe);
71 	link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
72 	if (!ASSERT_GE(link1_fd, 0, "link_fd"))
73 		goto cleanup;
74 
75 	opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
76 	prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
77 	link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
78 	if (!ASSERT_GE(link2_fd, 0, "link_fd"))
79 		goto cleanup;
80 
81 	kprobe_multi_test_run(skel, true);
82 
83 cleanup:
84 	if (link1_fd != -1)
85 		close(link1_fd);
86 	if (link2_fd != -1)
87 		close(link2_fd);
88 	kprobe_multi__destroy(skel);
89 }
90 
91 #define GET_ADDR(__sym, __addr) ({					\
92 	__addr = ksym_get_addr(__sym);					\
93 	if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym))	\
94 		return;							\
95 })
96 
97 static void test_link_api_addrs(void)
98 {
99 	LIBBPF_OPTS(bpf_link_create_opts, opts);
100 	unsigned long long addrs[8];
101 
102 	GET_ADDR("bpf_fentry_test1", addrs[0]);
103 	GET_ADDR("bpf_fentry_test2", addrs[1]);
104 	GET_ADDR("bpf_fentry_test3", addrs[2]);
105 	GET_ADDR("bpf_fentry_test4", addrs[3]);
106 	GET_ADDR("bpf_fentry_test5", addrs[4]);
107 	GET_ADDR("bpf_fentry_test6", addrs[5]);
108 	GET_ADDR("bpf_fentry_test7", addrs[6]);
109 	GET_ADDR("bpf_fentry_test8", addrs[7]);
110 
111 	opts.kprobe_multi.addrs = (const unsigned long*) addrs;
112 	opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
113 	test_link_api(&opts);
114 }
115 
116 static void test_link_api_syms(void)
117 {
118 	LIBBPF_OPTS(bpf_link_create_opts, opts);
119 	const char *syms[8] = {
120 		"bpf_fentry_test1",
121 		"bpf_fentry_test2",
122 		"bpf_fentry_test3",
123 		"bpf_fentry_test4",
124 		"bpf_fentry_test5",
125 		"bpf_fentry_test6",
126 		"bpf_fentry_test7",
127 		"bpf_fentry_test8",
128 	};
129 
130 	opts.kprobe_multi.syms = syms;
131 	opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
132 	test_link_api(&opts);
133 }
134 
135 static void
136 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
137 {
138 	struct bpf_link *link1 = NULL, *link2 = NULL;
139 	struct kprobe_multi *skel = NULL;
140 
141 	skel = kprobe_multi__open_and_load();
142 	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
143 		goto cleanup;
144 
145 	skel->bss->pid = getpid();
146 	link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
147 						      pattern, opts);
148 	if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
149 		goto cleanup;
150 
151 	if (opts) {
152 		opts->retprobe = true;
153 		link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
154 							      pattern, opts);
155 		if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
156 			goto cleanup;
157 	}
158 
159 	kprobe_multi_test_run(skel, !!opts);
160 
161 cleanup:
162 	bpf_link__destroy(link2);
163 	bpf_link__destroy(link1);
164 	kprobe_multi__destroy(skel);
165 }
166 
167 static void test_attach_api_pattern(void)
168 {
169 	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
170 
171 	test_attach_api("bpf_fentry_test*", &opts);
172 	test_attach_api("bpf_fentry_test?", NULL);
173 }
174 
175 static void test_attach_api_addrs(void)
176 {
177 	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
178 	unsigned long long addrs[8];
179 
180 	GET_ADDR("bpf_fentry_test1", addrs[0]);
181 	GET_ADDR("bpf_fentry_test2", addrs[1]);
182 	GET_ADDR("bpf_fentry_test3", addrs[2]);
183 	GET_ADDR("bpf_fentry_test4", addrs[3]);
184 	GET_ADDR("bpf_fentry_test5", addrs[4]);
185 	GET_ADDR("bpf_fentry_test6", addrs[5]);
186 	GET_ADDR("bpf_fentry_test7", addrs[6]);
187 	GET_ADDR("bpf_fentry_test8", addrs[7]);
188 
189 	opts.addrs = (const unsigned long *) addrs;
190 	opts.cnt = ARRAY_SIZE(addrs);
191 	test_attach_api(NULL, &opts);
192 }
193 
194 static void test_attach_api_syms(void)
195 {
196 	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
197 	const char *syms[8] = {
198 		"bpf_fentry_test1",
199 		"bpf_fentry_test2",
200 		"bpf_fentry_test3",
201 		"bpf_fentry_test4",
202 		"bpf_fentry_test5",
203 		"bpf_fentry_test6",
204 		"bpf_fentry_test7",
205 		"bpf_fentry_test8",
206 	};
207 
208 	opts.syms = syms;
209 	opts.cnt = ARRAY_SIZE(syms);
210 	test_attach_api(NULL, &opts);
211 }
212 
213 static void test_attach_api_fails(void)
214 {
215 	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
216 	struct kprobe_multi *skel = NULL;
217 	struct bpf_link *link = NULL;
218 	unsigned long long addrs[2];
219 	const char *syms[2] = {
220 		"bpf_fentry_test1",
221 		"bpf_fentry_test2",
222 	};
223 	__u64 cookies[2];
224 
225 	addrs[0] = ksym_get_addr("bpf_fentry_test1");
226 	addrs[1] = ksym_get_addr("bpf_fentry_test2");
227 
228 	if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
229 		goto cleanup;
230 
231 	skel = kprobe_multi__open_and_load();
232 	if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
233 		goto cleanup;
234 
235 	skel->bss->pid = getpid();
236 
237 	/* fail_1 - pattern and opts NULL */
238 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
239 						     NULL, NULL);
240 	if (!ASSERT_ERR_PTR(link, "fail_1"))
241 		goto cleanup;
242 
243 	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
244 		goto cleanup;
245 
246 	/* fail_2 - both addrs and syms set */
247 	opts.addrs = (const unsigned long *) addrs;
248 	opts.syms = syms;
249 	opts.cnt = ARRAY_SIZE(syms);
250 	opts.cookies = NULL;
251 
252 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
253 						     NULL, &opts);
254 	if (!ASSERT_ERR_PTR(link, "fail_2"))
255 		goto cleanup;
256 
257 	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
258 		goto cleanup;
259 
260 	/* fail_3 - pattern and addrs set */
261 	opts.addrs = (const unsigned long *) addrs;
262 	opts.syms = NULL;
263 	opts.cnt = ARRAY_SIZE(syms);
264 	opts.cookies = NULL;
265 
266 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
267 						     "ksys_*", &opts);
268 	if (!ASSERT_ERR_PTR(link, "fail_3"))
269 		goto cleanup;
270 
271 	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
272 		goto cleanup;
273 
274 	/* fail_4 - pattern and cnt set */
275 	opts.addrs = NULL;
276 	opts.syms = NULL;
277 	opts.cnt = ARRAY_SIZE(syms);
278 	opts.cookies = NULL;
279 
280 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
281 						     "ksys_*", &opts);
282 	if (!ASSERT_ERR_PTR(link, "fail_4"))
283 		goto cleanup;
284 
285 	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
286 		goto cleanup;
287 
288 	/* fail_5 - pattern and cookies */
289 	opts.addrs = NULL;
290 	opts.syms = NULL;
291 	opts.cnt = 0;
292 	opts.cookies = cookies;
293 
294 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
295 						     "ksys_*", &opts);
296 	if (!ASSERT_ERR_PTR(link, "fail_5"))
297 		goto cleanup;
298 
299 	if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
300 		goto cleanup;
301 
302 cleanup:
303 	bpf_link__destroy(link);
304 	kprobe_multi__destroy(skel);
305 }
306 
307 static inline __u64 get_time_ns(void)
308 {
309 	struct timespec t;
310 
311 	clock_gettime(CLOCK_MONOTONIC, &t);
312 	return (__u64) t.tv_sec * 1000000000 + t.tv_nsec;
313 }
314 
315 static size_t symbol_hash(long key, void *ctx __maybe_unused)
316 {
317 	return str_hash((const char *) key);
318 }
319 
320 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
321 {
322 	return strcmp((const char *) key1, (const char *) key2) == 0;
323 }
324 
325 static int get_syms(char ***symsp, size_t *cntp, bool kernel)
326 {
327 	size_t cap = 0, cnt = 0, i;
328 	char *name = NULL, **syms = NULL;
329 	struct hashmap *map;
330 	char buf[256];
331 	FILE *f;
332 	int err = 0;
333 
334 	/*
335 	 * The available_filter_functions contains many duplicates,
336 	 * but other than that all symbols are usable in kprobe multi
337 	 * interface.
338 	 * Filtering out duplicates by using hashmap__add, which won't
339 	 * add existing entry.
340 	 */
341 	f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
342 	if (!f)
343 		return -EINVAL;
344 
345 	map = hashmap__new(symbol_hash, symbol_equal, NULL);
346 	if (IS_ERR(map)) {
347 		err = libbpf_get_error(map);
348 		goto error;
349 	}
350 
351 	while (fgets(buf, sizeof(buf), f)) {
352 		if (kernel && strchr(buf, '['))
353 			continue;
354 		if (!kernel && !strchr(buf, '['))
355 			continue;
356 
357 		free(name);
358 		if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
359 			continue;
360 		/*
361 		 * We attach to almost all kernel functions and some of them
362 		 * will cause 'suspicious RCU usage' when fprobe is attached
363 		 * to them. Filter out the current culprits - arch_cpu_idle
364 		 * default_idle and rcu_* functions.
365 		 */
366 		if (!strcmp(name, "arch_cpu_idle"))
367 			continue;
368 		if (!strcmp(name, "default_idle"))
369 			continue;
370 		if (!strncmp(name, "rcu_", 4))
371 			continue;
372 		if (!strcmp(name, "bpf_dispatcher_xdp_func"))
373 			continue;
374 		if (!strncmp(name, "__ftrace_invalid_address__",
375 			     sizeof("__ftrace_invalid_address__") - 1))
376 			continue;
377 
378 		err = hashmap__add(map, name, 0);
379 		if (err == -EEXIST)
380 			continue;
381 		if (err)
382 			goto error;
383 
384 		err = libbpf_ensure_mem((void **) &syms, &cap,
385 					sizeof(*syms), cnt + 1);
386 		if (err)
387 			goto error;
388 
389 		syms[cnt++] = name;
390 		name = NULL;
391 	}
392 
393 	*symsp = syms;
394 	*cntp = cnt;
395 
396 error:
397 	free(name);
398 	fclose(f);
399 	hashmap__free(map);
400 	if (err) {
401 		for (i = 0; i < cnt; i++)
402 			free(syms[i]);
403 		free(syms);
404 	}
405 	return err;
406 }
407 
408 static void test_kprobe_multi_bench_attach(bool kernel)
409 {
410 	LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
411 	struct kprobe_multi_empty *skel = NULL;
412 	long attach_start_ns, attach_end_ns;
413 	long detach_start_ns, detach_end_ns;
414 	double attach_delta, detach_delta;
415 	struct bpf_link *link = NULL;
416 	char **syms = NULL;
417 	size_t cnt = 0, i;
418 
419 	if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
420 		return;
421 
422 	skel = kprobe_multi_empty__open_and_load();
423 	if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
424 		goto cleanup;
425 
426 	opts.syms = (const char **) syms;
427 	opts.cnt = cnt;
428 
429 	attach_start_ns = get_time_ns();
430 	link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
431 						     NULL, &opts);
432 	attach_end_ns = get_time_ns();
433 
434 	if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
435 		goto cleanup;
436 
437 	detach_start_ns = get_time_ns();
438 	bpf_link__destroy(link);
439 	detach_end_ns = get_time_ns();
440 
441 	attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
442 	detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
443 
444 	printf("%s: found %lu functions\n", __func__, cnt);
445 	printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
446 	printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
447 
448 cleanup:
449 	kprobe_multi_empty__destroy(skel);
450 	if (syms) {
451 		for (i = 0; i < cnt; i++)
452 			free(syms[i]);
453 		free(syms);
454 	}
455 }
456 
457 void serial_test_kprobe_multi_bench_attach(void)
458 {
459 	if (test__start_subtest("kernel"))
460 		test_kprobe_multi_bench_attach(true);
461 	if (test__start_subtest("modules"))
462 		test_kprobe_multi_bench_attach(false);
463 }
464 
465 void test_kprobe_multi_test(void)
466 {
467 	if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
468 		return;
469 
470 	if (test__start_subtest("skel_api"))
471 		test_skel_api();
472 	if (test__start_subtest("link_api_addrs"))
473 		test_link_api_syms();
474 	if (test__start_subtest("link_api_syms"))
475 		test_link_api_addrs();
476 	if (test__start_subtest("attach_api_pattern"))
477 		test_attach_api_pattern();
478 	if (test__start_subtest("attach_api_addrs"))
479 		test_attach_api_addrs();
480 	if (test__start_subtest("attach_api_syms"))
481 		test_attach_api_syms();
482 	if (test__start_subtest("attach_api_fails"))
483 		test_attach_api_fails();
484 }
485