1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2020 Facebook */
3 #include <linux/btf.h>
4 #include <linux/btf_ids.h>
5 #include <linux/delay.h>
6 #include <linux/error-injection.h>
7 #include <linux/init.h>
8 #include <linux/module.h>
9 #include <linux/percpu-defs.h>
10 #include <linux/sysfs.h>
11 #include <linux/tracepoint.h>
12 #include "bpf_testmod.h"
13 #include "bpf_testmod_kfunc.h"
14 
15 #define CREATE_TRACE_POINTS
16 #include "bpf_testmod-events.h"
17 
18 typedef int (*func_proto_typedef)(long);
19 typedef int (*func_proto_typedef_nested1)(func_proto_typedef);
20 typedef int (*func_proto_typedef_nested2)(func_proto_typedef_nested1);
21 
22 DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123;
23 long bpf_testmod_test_struct_arg_result;
24 
25 struct bpf_testmod_struct_arg_1 {
26 	int a;
27 };
28 struct bpf_testmod_struct_arg_2 {
29 	long a;
30 	long b;
31 };
32 
33 struct bpf_testmod_struct_arg_3 {
34 	int a;
35 	int b[];
36 };
37 
38 struct bpf_testmod_struct_arg_4 {
39 	u64 a;
40 	int b;
41 };
42 
43 __diag_push();
44 __diag_ignore_all("-Wmissing-prototypes",
45 		  "Global functions as their definitions will be in bpf_testmod.ko BTF");
46 
47 noinline int
48 bpf_testmod_test_struct_arg_1(struct bpf_testmod_struct_arg_2 a, int b, int c) {
49 	bpf_testmod_test_struct_arg_result = a.a + a.b  + b + c;
50 	return bpf_testmod_test_struct_arg_result;
51 }
52 
53 noinline int
54 bpf_testmod_test_struct_arg_2(int a, struct bpf_testmod_struct_arg_2 b, int c) {
55 	bpf_testmod_test_struct_arg_result = a + b.a + b.b + c;
56 	return bpf_testmod_test_struct_arg_result;
57 }
58 
59 noinline int
60 bpf_testmod_test_struct_arg_3(int a, int b, struct bpf_testmod_struct_arg_2 c) {
61 	bpf_testmod_test_struct_arg_result = a + b + c.a + c.b;
62 	return bpf_testmod_test_struct_arg_result;
63 }
64 
65 noinline int
66 bpf_testmod_test_struct_arg_4(struct bpf_testmod_struct_arg_1 a, int b,
67 			      int c, int d, struct bpf_testmod_struct_arg_2 e) {
68 	bpf_testmod_test_struct_arg_result = a.a + b + c + d + e.a + e.b;
69 	return bpf_testmod_test_struct_arg_result;
70 }
71 
72 noinline int
73 bpf_testmod_test_struct_arg_5(void) {
74 	bpf_testmod_test_struct_arg_result = 1;
75 	return bpf_testmod_test_struct_arg_result;
76 }
77 
78 noinline int
79 bpf_testmod_test_struct_arg_6(struct bpf_testmod_struct_arg_3 *a) {
80 	bpf_testmod_test_struct_arg_result = a->b[0];
81 	return bpf_testmod_test_struct_arg_result;
82 }
83 
84 noinline int
85 bpf_testmod_test_struct_arg_7(u64 a, void *b, short c, int d, void *e,
86 			      struct bpf_testmod_struct_arg_4 f)
87 {
88 	bpf_testmod_test_struct_arg_result = a + (long)b + c + d +
89 		(long)e + f.a + f.b;
90 	return bpf_testmod_test_struct_arg_result;
91 }
92 
93 noinline int
94 bpf_testmod_test_struct_arg_8(u64 a, void *b, short c, int d, void *e,
95 			      struct bpf_testmod_struct_arg_4 f, int g)
96 {
97 	bpf_testmod_test_struct_arg_result = a + (long)b + c + d +
98 		(long)e + f.a + f.b + g;
99 	return bpf_testmod_test_struct_arg_result;
100 }
101 
102 noinline int
103 bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) {
104 	bpf_testmod_test_struct_arg_result = a->a;
105 	return bpf_testmod_test_struct_arg_result;
106 }
107 
108 __bpf_kfunc void
109 bpf_testmod_test_mod_kfunc(int i)
110 {
111 	*(int *)this_cpu_ptr(&bpf_testmod_ksym_percpu) = i;
112 }
113 
114 __bpf_kfunc int bpf_iter_testmod_seq_new(struct bpf_iter_testmod_seq *it, s64 value, int cnt)
115 {
116 	if (cnt < 0) {
117 		it->cnt = 0;
118 		return -EINVAL;
119 	}
120 
121 	it->value = value;
122 	it->cnt = cnt;
123 
124 	return 0;
125 }
126 
127 __bpf_kfunc s64 *bpf_iter_testmod_seq_next(struct bpf_iter_testmod_seq* it)
128 {
129 	if (it->cnt <= 0)
130 		return NULL;
131 
132 	it->cnt--;
133 
134 	return &it->value;
135 }
136 
137 __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it)
138 {
139 	it->cnt = 0;
140 }
141 
142 struct bpf_testmod_btf_type_tag_1 {
143 	int a;
144 };
145 
146 struct bpf_testmod_btf_type_tag_2 {
147 	struct bpf_testmod_btf_type_tag_1 __user *p;
148 };
149 
150 struct bpf_testmod_btf_type_tag_3 {
151 	struct bpf_testmod_btf_type_tag_1 __percpu *p;
152 };
153 
154 noinline int
155 bpf_testmod_test_btf_type_tag_user_1(struct bpf_testmod_btf_type_tag_1 __user *arg) {
156 	BTF_TYPE_EMIT(func_proto_typedef);
157 	BTF_TYPE_EMIT(func_proto_typedef_nested1);
158 	BTF_TYPE_EMIT(func_proto_typedef_nested2);
159 	return arg->a;
160 }
161 
162 noinline int
163 bpf_testmod_test_btf_type_tag_user_2(struct bpf_testmod_btf_type_tag_2 *arg) {
164 	return arg->p->a;
165 }
166 
167 noinline int
168 bpf_testmod_test_btf_type_tag_percpu_1(struct bpf_testmod_btf_type_tag_1 __percpu *arg) {
169 	return arg->a;
170 }
171 
172 noinline int
173 bpf_testmod_test_btf_type_tag_percpu_2(struct bpf_testmod_btf_type_tag_3 *arg) {
174 	return arg->p->a;
175 }
176 
177 noinline int bpf_testmod_loop_test(int n)
178 {
179 	/* Make sum volatile, so smart compilers, such as clang, will not
180 	 * optimize the code by removing the loop.
181 	 */
182 	volatile int sum = 0;
183 	int i;
184 
185 	/* the primary goal of this test is to test LBR. Create a lot of
186 	 * branches in the function, so we can catch it easily.
187 	 */
188 	for (i = 0; i < n; i++)
189 		sum += i;
190 	return sum;
191 }
192 
193 __weak noinline struct file *bpf_testmod_return_ptr(int arg)
194 {
195 	static struct file f = {};
196 
197 	switch (arg) {
198 	case 1: return (void *)EINVAL;		/* user addr */
199 	case 2: return (void *)0xcafe4a11;	/* user addr */
200 	case 3: return (void *)-EINVAL;		/* canonical, but invalid */
201 	case 4: return (void *)(1ull << 60);	/* non-canonical and invalid */
202 	case 5: return (void *)~(1ull << 30);	/* trigger extable */
203 	case 6: return &f;			/* valid addr */
204 	case 7: return (void *)((long)&f | 1);	/* kernel tricks */
205 	default: return NULL;
206 	}
207 }
208 
209 noinline int bpf_testmod_fentry_test1(int a)
210 {
211 	return a + 1;
212 }
213 
214 noinline int bpf_testmod_fentry_test2(int a, u64 b)
215 {
216 	return a + b;
217 }
218 
219 noinline int bpf_testmod_fentry_test3(char a, int b, u64 c)
220 {
221 	return a + b + c;
222 }
223 
224 noinline int bpf_testmod_fentry_test7(u64 a, void *b, short c, int d,
225 				      void *e, char f, int g)
226 {
227 	return a + (long)b + c + d + (long)e + f + g;
228 }
229 
230 noinline int bpf_testmod_fentry_test11(u64 a, void *b, short c, int d,
231 				       void *e, char f, int g,
232 				       unsigned int h, long i, __u64 j,
233 				       unsigned long k)
234 {
235 	return a + (long)b + c + d + (long)e + f + g + h + i + j + k;
236 }
237 
238 int bpf_testmod_fentry_ok;
239 
240 noinline ssize_t
241 bpf_testmod_test_read(struct file *file, struct kobject *kobj,
242 		      struct bin_attribute *bin_attr,
243 		      char *buf, loff_t off, size_t len)
244 {
245 	struct bpf_testmod_test_read_ctx ctx = {
246 		.buf = buf,
247 		.off = off,
248 		.len = len,
249 	};
250 	struct bpf_testmod_struct_arg_1 struct_arg1 = {10}, struct_arg1_2 = {-1};
251 	struct bpf_testmod_struct_arg_2 struct_arg2 = {2, 3};
252 	struct bpf_testmod_struct_arg_3 *struct_arg3;
253 	struct bpf_testmod_struct_arg_4 struct_arg4 = {21, 22};
254 	int i = 1;
255 
256 	while (bpf_testmod_return_ptr(i))
257 		i++;
258 
259 	(void)bpf_testmod_test_struct_arg_1(struct_arg2, 1, 4);
260 	(void)bpf_testmod_test_struct_arg_2(1, struct_arg2, 4);
261 	(void)bpf_testmod_test_struct_arg_3(1, 4, struct_arg2);
262 	(void)bpf_testmod_test_struct_arg_4(struct_arg1, 1, 2, 3, struct_arg2);
263 	(void)bpf_testmod_test_struct_arg_5();
264 	(void)bpf_testmod_test_struct_arg_7(16, (void *)17, 18, 19,
265 					    (void *)20, struct_arg4);
266 	(void)bpf_testmod_test_struct_arg_8(16, (void *)17, 18, 19,
267 					    (void *)20, struct_arg4, 23);
268 
269 	(void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2);
270 
271 	struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) +
272 				sizeof(int)), GFP_KERNEL);
273 	if (struct_arg3 != NULL) {
274 		struct_arg3->b[0] = 1;
275 		(void)bpf_testmod_test_struct_arg_6(struct_arg3);
276 		kfree(struct_arg3);
277 	}
278 
279 	/* This is always true. Use the check to make sure the compiler
280 	 * doesn't remove bpf_testmod_loop_test.
281 	 */
282 	if (bpf_testmod_loop_test(101) > 100)
283 		trace_bpf_testmod_test_read(current, &ctx);
284 
285 	/* Magic number to enable writable tp */
286 	if (len == 64) {
287 		struct bpf_testmod_test_writable_ctx writable = {
288 			.val = 1024,
289 		};
290 		trace_bpf_testmod_test_writable_bare(&writable);
291 		if (writable.early_ret)
292 			return snprintf(buf, len, "%d\n", writable.val);
293 	}
294 
295 	if (bpf_testmod_fentry_test1(1) != 2 ||
296 	    bpf_testmod_fentry_test2(2, 3) != 5 ||
297 	    bpf_testmod_fentry_test3(4, 5, 6) != 15 ||
298 	    bpf_testmod_fentry_test7(16, (void *)17, 18, 19, (void *)20,
299 			21, 22) != 133 ||
300 	    bpf_testmod_fentry_test11(16, (void *)17, 18, 19, (void *)20,
301 			21, 22, 23, 24, 25, 26) != 231)
302 		goto out;
303 
304 	bpf_testmod_fentry_ok = 1;
305 out:
306 	return -EIO; /* always fail */
307 }
308 EXPORT_SYMBOL(bpf_testmod_test_read);
309 ALLOW_ERROR_INJECTION(bpf_testmod_test_read, ERRNO);
310 
311 noinline ssize_t
312 bpf_testmod_test_write(struct file *file, struct kobject *kobj,
313 		      struct bin_attribute *bin_attr,
314 		      char *buf, loff_t off, size_t len)
315 {
316 	struct bpf_testmod_test_write_ctx ctx = {
317 		.buf = buf,
318 		.off = off,
319 		.len = len,
320 	};
321 
322 	trace_bpf_testmod_test_write_bare(current, &ctx);
323 
324 	return -EIO; /* always fail */
325 }
326 EXPORT_SYMBOL(bpf_testmod_test_write);
327 ALLOW_ERROR_INJECTION(bpf_testmod_test_write, ERRNO);
328 
329 noinline int bpf_fentry_shadow_test(int a)
330 {
331 	return a + 2;
332 }
333 EXPORT_SYMBOL_GPL(bpf_fentry_shadow_test);
334 
335 __diag_pop();
336 
337 static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {
338 	.attr = { .name = "bpf_testmod", .mode = 0666, },
339 	.read = bpf_testmod_test_read,
340 	.write = bpf_testmod_test_write,
341 };
342 
343 BTF_SET8_START(bpf_testmod_common_kfunc_ids)
344 BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW)
345 BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL)
346 BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY)
347 BTF_SET8_END(bpf_testmod_common_kfunc_ids)
348 
349 static const struct btf_kfunc_id_set bpf_testmod_common_kfunc_set = {
350 	.owner = THIS_MODULE,
351 	.set   = &bpf_testmod_common_kfunc_ids,
352 };
353 
354 __bpf_kfunc u64 bpf_kfunc_call_test1(struct sock *sk, u32 a, u64 b, u32 c, u64 d)
355 {
356 	return a + b + c + d;
357 }
358 
359 __bpf_kfunc int bpf_kfunc_call_test2(struct sock *sk, u32 a, u32 b)
360 {
361 	return a + b;
362 }
363 
364 __bpf_kfunc struct sock *bpf_kfunc_call_test3(struct sock *sk)
365 {
366 	return sk;
367 }
368 
369 __bpf_kfunc long noinline bpf_kfunc_call_test4(signed char a, short b, int c, long d)
370 {
371 	/* Provoke the compiler to assume that the caller has sign-extended a,
372 	 * b and c on platforms where this is required (e.g. s390x).
373 	 */
374 	return (long)a + (long)b + (long)c + d;
375 }
376 
377 static struct prog_test_ref_kfunc prog_test_struct = {
378 	.a = 42,
379 	.b = 108,
380 	.next = &prog_test_struct,
381 	.cnt = REFCOUNT_INIT(1),
382 };
383 
384 __bpf_kfunc struct prog_test_ref_kfunc *
385 bpf_kfunc_call_test_acquire(unsigned long *scalar_ptr)
386 {
387 	refcount_inc(&prog_test_struct.cnt);
388 	return &prog_test_struct;
389 }
390 
391 __bpf_kfunc void bpf_kfunc_call_test_offset(struct prog_test_ref_kfunc *p)
392 {
393 	WARN_ON_ONCE(1);
394 }
395 
396 __bpf_kfunc struct prog_test_member *
397 bpf_kfunc_call_memb_acquire(void)
398 {
399 	WARN_ON_ONCE(1);
400 	return NULL;
401 }
402 
403 __bpf_kfunc void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p)
404 {
405 	WARN_ON_ONCE(1);
406 }
407 
408 static int *__bpf_kfunc_call_test_get_mem(struct prog_test_ref_kfunc *p, const int size)
409 {
410 	if (size > 2 * sizeof(int))
411 		return NULL;
412 
413 	return (int *)p;
414 }
415 
416 __bpf_kfunc int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p,
417 						  const int rdwr_buf_size)
418 {
419 	return __bpf_kfunc_call_test_get_mem(p, rdwr_buf_size);
420 }
421 
422 __bpf_kfunc int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p,
423 						    const int rdonly_buf_size)
424 {
425 	return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size);
426 }
427 
428 /* the next 2 ones can't be really used for testing expect to ensure
429  * that the verifier rejects the call.
430  * Acquire functions must return struct pointers, so these ones are
431  * failing.
432  */
433 __bpf_kfunc int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p,
434 						    const int rdonly_buf_size)
435 {
436 	return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size);
437 }
438 
439 __bpf_kfunc void bpf_kfunc_call_int_mem_release(int *p)
440 {
441 }
442 
443 __bpf_kfunc void bpf_kfunc_call_test_pass_ctx(struct __sk_buff *skb)
444 {
445 }
446 
447 __bpf_kfunc void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p)
448 {
449 }
450 
451 __bpf_kfunc void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p)
452 {
453 }
454 
455 __bpf_kfunc void bpf_kfunc_call_test_fail1(struct prog_test_fail1 *p)
456 {
457 }
458 
459 __bpf_kfunc void bpf_kfunc_call_test_fail2(struct prog_test_fail2 *p)
460 {
461 }
462 
463 __bpf_kfunc void bpf_kfunc_call_test_fail3(struct prog_test_fail3 *p)
464 {
465 }
466 
467 __bpf_kfunc void bpf_kfunc_call_test_mem_len_pass1(void *mem, int mem__sz)
468 {
469 }
470 
471 __bpf_kfunc void bpf_kfunc_call_test_mem_len_fail1(void *mem, int len)
472 {
473 }
474 
475 __bpf_kfunc void bpf_kfunc_call_test_mem_len_fail2(u64 *mem, int len)
476 {
477 }
478 
479 __bpf_kfunc void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p)
480 {
481 	/* p != NULL, but p->cnt could be 0 */
482 }
483 
484 __bpf_kfunc void bpf_kfunc_call_test_destructive(void)
485 {
486 }
487 
488 __bpf_kfunc static u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused)
489 {
490 	return arg;
491 }
492 
493 BTF_SET8_START(bpf_testmod_check_kfunc_ids)
494 BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc)
495 BTF_ID_FLAGS(func, bpf_kfunc_call_test1)
496 BTF_ID_FLAGS(func, bpf_kfunc_call_test2)
497 BTF_ID_FLAGS(func, bpf_kfunc_call_test3)
498 BTF_ID_FLAGS(func, bpf_kfunc_call_test4)
499 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1)
500 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1)
501 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2)
502 BTF_ID_FLAGS(func, bpf_kfunc_call_test_acquire, KF_ACQUIRE | KF_RET_NULL)
503 BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL)
504 BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE)
505 BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdwr_mem, KF_RET_NULL)
506 BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdonly_mem, KF_RET_NULL)
507 BTF_ID_FLAGS(func, bpf_kfunc_call_test_acq_rdonly_mem, KF_ACQUIRE | KF_RET_NULL)
508 BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE)
509 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx)
510 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1)
511 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass2)
512 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail1)
513 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail2)
514 BTF_ID_FLAGS(func, bpf_kfunc_call_test_fail3)
515 BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS | KF_RCU)
516 BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE)
517 BTF_ID_FLAGS(func, bpf_kfunc_call_test_static_unused_arg)
518 BTF_ID_FLAGS(func, bpf_kfunc_call_test_offset)
519 BTF_SET8_END(bpf_testmod_check_kfunc_ids)
520 
521 static const struct btf_kfunc_id_set bpf_testmod_kfunc_set = {
522 	.owner = THIS_MODULE,
523 	.set   = &bpf_testmod_check_kfunc_ids,
524 };
525 
526 extern int bpf_fentry_test1(int a);
527 
528 static int bpf_testmod_init(void)
529 {
530 	int ret;
531 
532 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_testmod_common_kfunc_set);
533 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_testmod_kfunc_set);
534 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_testmod_kfunc_set);
535 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_testmod_kfunc_set);
536 	if (ret < 0)
537 		return ret;
538 	if (bpf_fentry_test1(0) < 0)
539 		return -EINVAL;
540 	return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
541 }
542 
543 static void bpf_testmod_exit(void)
544 {
545         /* Need to wait for all references to be dropped because
546          * bpf_kfunc_call_test_release() which currently resides in kernel can
547          * be called after bpf_testmod is unloaded. Once release function is
548          * moved into the module this wait can be removed.
549          */
550 	while (refcount_read(&prog_test_struct.cnt) > 1)
551 		msleep(20);
552 
553 	return sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
554 }
555 
556 module_init(bpf_testmod_init);
557 module_exit(bpf_testmod_exit);
558 
559 MODULE_AUTHOR("Andrii Nakryiko");
560 MODULE_DESCRIPTION("BPF selftests module");
561 MODULE_LICENSE("Dual BSD/GPL");
562