1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2022 Facebook */
3 
4 #include <errno.h>
5 #include <string.h>
6 #include <linux/bpf.h>
7 #include <bpf/bpf_helpers.h>
8 #include <linux/if_ether.h>
9 #include "bpf_misc.h"
10 #include "bpf_kfuncs.h"
11 
12 char _license[] SEC("license") = "GPL";
13 
14 struct test_info {
15 	int x;
16 	struct bpf_dynptr ptr;
17 };
18 
19 struct {
20 	__uint(type, BPF_MAP_TYPE_ARRAY);
21 	__uint(max_entries, 1);
22 	__type(key, __u32);
23 	__type(value, struct bpf_dynptr);
24 } array_map1 SEC(".maps");
25 
26 struct {
27 	__uint(type, BPF_MAP_TYPE_ARRAY);
28 	__uint(max_entries, 1);
29 	__type(key, __u32);
30 	__type(value, struct test_info);
31 } array_map2 SEC(".maps");
32 
33 struct {
34 	__uint(type, BPF_MAP_TYPE_ARRAY);
35 	__uint(max_entries, 1);
36 	__type(key, __u32);
37 	__type(value, __u32);
38 } array_map3 SEC(".maps");
39 
40 struct {
41 	__uint(type, BPF_MAP_TYPE_ARRAY);
42 	__uint(max_entries, 1);
43 	__type(key, __u32);
44 	__type(value, __u64);
45 } array_map4 SEC(".maps");
46 
47 struct sample {
48 	int pid;
49 	long value;
50 	char comm[16];
51 };
52 
53 struct {
54 	__uint(type, BPF_MAP_TYPE_RINGBUF);
55 	__uint(max_entries, 4096);
56 } ringbuf SEC(".maps");
57 
58 int err, val;
59 
60 static int get_map_val_dynptr(struct bpf_dynptr *ptr)
61 {
62 	__u32 key = 0, *map_val;
63 
64 	bpf_map_update_elem(&array_map3, &key, &val, 0);
65 
66 	map_val = bpf_map_lookup_elem(&array_map3, &key);
67 	if (!map_val)
68 		return -ENOENT;
69 
70 	bpf_dynptr_from_mem(map_val, sizeof(*map_val), 0, ptr);
71 
72 	return 0;
73 }
74 
75 /* Every bpf_ringbuf_reserve_dynptr call must have a corresponding
76  * bpf_ringbuf_submit/discard_dynptr call
77  */
78 SEC("?raw_tp")
79 __failure __msg("Unreleased reference id=2")
80 int ringbuf_missing_release1(void *ctx)
81 {
82 	struct bpf_dynptr ptr;
83 
84 	bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr);
85 
86 	/* missing a call to bpf_ringbuf_discard/submit_dynptr */
87 
88 	return 0;
89 }
90 
91 SEC("?raw_tp")
92 __failure __msg("Unreleased reference id=4")
93 int ringbuf_missing_release2(void *ctx)
94 {
95 	struct bpf_dynptr ptr1, ptr2;
96 	struct sample *sample;
97 
98 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr1);
99 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr2);
100 
101 	sample = bpf_dynptr_data(&ptr1, 0, sizeof(*sample));
102 	if (!sample) {
103 		bpf_ringbuf_discard_dynptr(&ptr1, 0);
104 		bpf_ringbuf_discard_dynptr(&ptr2, 0);
105 		return 0;
106 	}
107 
108 	bpf_ringbuf_submit_dynptr(&ptr1, 0);
109 
110 	/* missing a call to bpf_ringbuf_discard/submit_dynptr on ptr2 */
111 
112 	return 0;
113 }
114 
115 static int missing_release_callback_fn(__u32 index, void *data)
116 {
117 	struct bpf_dynptr ptr;
118 
119 	bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr);
120 
121 	/* missing a call to bpf_ringbuf_discard/submit_dynptr */
122 
123 	return 0;
124 }
125 
126 /* Any dynptr initialized within a callback must have bpf_dynptr_put called */
127 SEC("?raw_tp")
128 __failure __msg("Unreleased reference id")
129 int ringbuf_missing_release_callback(void *ctx)
130 {
131 	bpf_loop(10, missing_release_callback_fn, NULL, 0);
132 	return 0;
133 }
134 
135 /* Can't call bpf_ringbuf_submit/discard_dynptr on a non-initialized dynptr */
136 SEC("?raw_tp")
137 __failure __msg("arg 1 is an unacquired reference")
138 int ringbuf_release_uninit_dynptr(void *ctx)
139 {
140 	struct bpf_dynptr ptr;
141 
142 	/* this should fail */
143 	bpf_ringbuf_submit_dynptr(&ptr, 0);
144 
145 	return 0;
146 }
147 
148 /* A dynptr can't be used after it has been invalidated */
149 SEC("?raw_tp")
150 __failure __msg("Expected an initialized dynptr as arg #3")
151 int use_after_invalid(void *ctx)
152 {
153 	struct bpf_dynptr ptr;
154 	char read_data[64];
155 
156 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(read_data), 0, &ptr);
157 
158 	bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0);
159 
160 	bpf_ringbuf_submit_dynptr(&ptr, 0);
161 
162 	/* this should fail */
163 	bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0);
164 
165 	return 0;
166 }
167 
168 /* Can't call non-dynptr ringbuf APIs on a dynptr ringbuf sample */
169 SEC("?raw_tp")
170 __failure __msg("type=mem expected=ringbuf_mem")
171 int ringbuf_invalid_api(void *ctx)
172 {
173 	struct bpf_dynptr ptr;
174 	struct sample *sample;
175 
176 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr);
177 	sample = bpf_dynptr_data(&ptr, 0, sizeof(*sample));
178 	if (!sample)
179 		goto done;
180 
181 	sample->pid = 123;
182 
183 	/* invalid API use. need to use dynptr API to submit/discard */
184 	bpf_ringbuf_submit(sample, 0);
185 
186 done:
187 	bpf_ringbuf_discard_dynptr(&ptr, 0);
188 	return 0;
189 }
190 
191 /* Can't add a dynptr to a map */
192 SEC("?raw_tp")
193 __failure __msg("invalid indirect read from stack")
194 int add_dynptr_to_map1(void *ctx)
195 {
196 	struct bpf_dynptr ptr;
197 	int key = 0;
198 
199 	bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &ptr);
200 
201 	/* this should fail */
202 	bpf_map_update_elem(&array_map1, &key, &ptr, 0);
203 
204 	bpf_ringbuf_submit_dynptr(&ptr, 0);
205 
206 	return 0;
207 }
208 
209 /* Can't add a struct with an embedded dynptr to a map */
210 SEC("?raw_tp")
211 __failure __msg("invalid indirect read from stack")
212 int add_dynptr_to_map2(void *ctx)
213 {
214 	struct test_info x;
215 	int key = 0;
216 
217 	bpf_ringbuf_reserve_dynptr(&ringbuf, val, 0, &x.ptr);
218 
219 	/* this should fail */
220 	bpf_map_update_elem(&array_map2, &key, &x, 0);
221 
222 	bpf_ringbuf_submit_dynptr(&x.ptr, 0);
223 
224 	return 0;
225 }
226 
227 /* A data slice can't be accessed out of bounds */
228 SEC("?raw_tp")
229 __failure __msg("value is outside of the allowed memory range")
230 int data_slice_out_of_bounds_ringbuf(void *ctx)
231 {
232 	struct bpf_dynptr ptr;
233 	void *data;
234 
235 	bpf_ringbuf_reserve_dynptr(&ringbuf, 8, 0, &ptr);
236 
237 	data  = bpf_dynptr_data(&ptr, 0, 8);
238 	if (!data)
239 		goto done;
240 
241 	/* can't index out of bounds of the data slice */
242 	val = *((char *)data + 8);
243 
244 done:
245 	bpf_ringbuf_submit_dynptr(&ptr, 0);
246 	return 0;
247 }
248 
249 /* A data slice can't be accessed out of bounds */
250 SEC("?tc")
251 __failure __msg("value is outside of the allowed memory range")
252 int data_slice_out_of_bounds_skb(struct __sk_buff *skb)
253 {
254 	struct bpf_dynptr ptr;
255 	struct ethhdr *hdr;
256 	char buffer[sizeof(*hdr)] = {};
257 
258 	bpf_dynptr_from_skb(skb, 0, &ptr);
259 
260 	hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer));
261 	if (!hdr)
262 		return SK_DROP;
263 
264 	/* this should fail */
265 	*(__u8*)(hdr + 1) = 1;
266 
267 	return SK_PASS;
268 }
269 
270 SEC("?raw_tp")
271 __failure __msg("value is outside of the allowed memory range")
272 int data_slice_out_of_bounds_map_value(void *ctx)
273 {
274 	__u32 map_val;
275 	struct bpf_dynptr ptr;
276 	void *data;
277 
278 	get_map_val_dynptr(&ptr);
279 
280 	data  = bpf_dynptr_data(&ptr, 0, sizeof(map_val));
281 	if (!data)
282 		return 0;
283 
284 	/* can't index out of bounds of the data slice */
285 	val = *((char *)data + (sizeof(map_val) + 1));
286 
287 	return 0;
288 }
289 
290 /* A data slice can't be used after it has been released */
291 SEC("?raw_tp")
292 __failure __msg("invalid mem access 'scalar'")
293 int data_slice_use_after_release1(void *ctx)
294 {
295 	struct bpf_dynptr ptr;
296 	struct sample *sample;
297 
298 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr);
299 	sample = bpf_dynptr_data(&ptr, 0, sizeof(*sample));
300 	if (!sample)
301 		goto done;
302 
303 	sample->pid = 123;
304 
305 	bpf_ringbuf_submit_dynptr(&ptr, 0);
306 
307 	/* this should fail */
308 	val = sample->pid;
309 
310 	return 0;
311 
312 done:
313 	bpf_ringbuf_discard_dynptr(&ptr, 0);
314 	return 0;
315 }
316 
317 /* A data slice can't be used after it has been released.
318  *
319  * This tests the case where the data slice tracks a dynptr (ptr2)
320  * that is at a non-zero offset from the frame pointer (ptr1 is at fp,
321  * ptr2 is at fp - 16).
322  */
323 SEC("?raw_tp")
324 __failure __msg("invalid mem access 'scalar'")
325 int data_slice_use_after_release2(void *ctx)
326 {
327 	struct bpf_dynptr ptr1, ptr2;
328 	struct sample *sample;
329 
330 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr1);
331 	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr2);
332 
333 	sample = bpf_dynptr_data(&ptr2, 0, sizeof(*sample));
334 	if (!sample)
335 		goto done;
336 
337 	sample->pid = 23;
338 
339 	bpf_ringbuf_submit_dynptr(&ptr2, 0);
340 
341 	/* this should fail */
342 	sample->pid = 23;
343 
344 	bpf_ringbuf_submit_dynptr(&ptr1, 0);
345 
346 	return 0;
347 
348 done:
349 	bpf_ringbuf_discard_dynptr(&ptr2, 0);
350 	bpf_ringbuf_discard_dynptr(&ptr1, 0);
351 	return 0;
352 }
353 
354 /* A data slice must be first checked for NULL */
355 SEC("?raw_tp")
356 __failure __msg("invalid mem access 'mem_or_null'")
357 int data_slice_missing_null_check1(void *ctx)
358 {
359 	struct bpf_dynptr ptr;
360 	void *data;
361 
362 	bpf_ringbuf_reserve_dynptr(&ringbuf, 8, 0, &ptr);
363 
364 	data  = bpf_dynptr_data(&ptr, 0, 8);
365 
366 	/* missing if (!data) check */
367 
368 	/* this should fail */
369 	*(__u8 *)data = 3;
370 
371 	bpf_ringbuf_submit_dynptr(&ptr, 0);
372 	return 0;
373 }
374 
375 /* A data slice can't be dereferenced if it wasn't checked for null */
376 SEC("?raw_tp")
377 __failure __msg("invalid mem access 'mem_or_null'")
378 int data_slice_missing_null_check2(void *ctx)
379 {
380 	struct bpf_dynptr ptr;
381 	__u64 *data1, *data2;
382 
383 	bpf_ringbuf_reserve_dynptr(&ringbuf, 16, 0, &ptr);
384 
385 	data1 = bpf_dynptr_data(&ptr, 0, 8);
386 	data2 = bpf_dynptr_data(&ptr, 0, 8);
387 	if (data1)
388 		/* this should fail */
389 		*data2 = 3;
390 
391 	bpf_ringbuf_discard_dynptr(&ptr, 0);
392 	return 0;
393 }
394 
395 /* Can't pass in a dynptr as an arg to a helper function that doesn't take in a
396  * dynptr argument
397  */
398 SEC("?raw_tp")
399 __failure __msg("invalid indirect read from stack")
400 int invalid_helper1(void *ctx)
401 {
402 	struct bpf_dynptr ptr;
403 
404 	get_map_val_dynptr(&ptr);
405 
406 	/* this should fail */
407 	bpf_strncmp((const char *)&ptr, sizeof(ptr), "hello!");
408 
409 	return 0;
410 }
411 
412 /* A dynptr can't be passed into a helper function at a non-zero offset */
413 SEC("?raw_tp")
414 __failure __msg("cannot pass in dynptr at an offset=-8")
415 int invalid_helper2(void *ctx)
416 {
417 	struct bpf_dynptr ptr;
418 	char read_data[64];
419 
420 	get_map_val_dynptr(&ptr);
421 
422 	/* this should fail */
423 	bpf_dynptr_read(read_data, sizeof(read_data), (void *)&ptr + 8, 0, 0);
424 	return 0;
425 }
426 
427 /* A bpf_dynptr is invalidated if it's been written into */
428 SEC("?raw_tp")
429 __failure __msg("Expected an initialized dynptr as arg #1")
430 int invalid_write1(void *ctx)
431 {
432 	struct bpf_dynptr ptr;
433 	void *data;
434 	__u8 x = 0;
435 
436 	get_map_val_dynptr(&ptr);
437 
438 	memcpy(&ptr, &x, sizeof(x));
439 
440 	/* this should fail */
441 	data = bpf_dynptr_data(&ptr, 0, 1);
442 	__sink(data);
443 
444 	return 0;
445 }
446 
447 /*
448  * A bpf_dynptr can't be used as a dynptr if it has been written into at a fixed
449  * offset
450  */
451 SEC("?raw_tp")
452 __failure __msg("cannot overwrite referenced dynptr")
453 int invalid_write2(void *ctx)
454 {
455 	struct bpf_dynptr ptr;
456 	char read_data[64];
457 	__u8 x = 0;
458 
459 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr);
460 
461 	memcpy((void *)&ptr + 8, &x, sizeof(x));
462 
463 	/* this should fail */
464 	bpf_dynptr_read(read_data, sizeof(read_data), &ptr, 0, 0);
465 
466 	bpf_ringbuf_submit_dynptr(&ptr, 0);
467 
468 	return 0;
469 }
470 
471 /*
472  * A bpf_dynptr can't be used as a dynptr if it has been written into at a
473  * non-const offset
474  */
475 SEC("?raw_tp")
476 __failure __msg("cannot overwrite referenced dynptr")
477 int invalid_write3(void *ctx)
478 {
479 	struct bpf_dynptr ptr;
480 	char stack_buf[16];
481 	unsigned long len;
482 	__u8 x = 0;
483 
484 	bpf_ringbuf_reserve_dynptr(&ringbuf, 8, 0, &ptr);
485 
486 	memcpy(stack_buf, &val, sizeof(val));
487 	len = stack_buf[0] & 0xf;
488 
489 	memcpy((void *)&ptr + len, &x, sizeof(x));
490 
491 	/* this should fail */
492 	bpf_ringbuf_submit_dynptr(&ptr, 0);
493 
494 	return 0;
495 }
496 
497 static int invalid_write4_callback(__u32 index, void *data)
498 {
499 	*(__u32 *)data = 123;
500 
501 	return 0;
502 }
503 
504 /* If the dynptr is written into in a callback function, it should
505  * be invalidated as a dynptr
506  */
507 SEC("?raw_tp")
508 __failure __msg("cannot overwrite referenced dynptr")
509 int invalid_write4(void *ctx)
510 {
511 	struct bpf_dynptr ptr;
512 
513 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr);
514 
515 	bpf_loop(10, invalid_write4_callback, &ptr, 0);
516 
517 	/* this should fail */
518 	bpf_ringbuf_submit_dynptr(&ptr, 0);
519 
520 	return 0;
521 }
522 
523 /* A globally-defined bpf_dynptr can't be used (it must reside as a stack frame) */
524 struct bpf_dynptr global_dynptr;
525 
526 SEC("?raw_tp")
527 __failure __msg("type=map_value expected=fp")
528 int global(void *ctx)
529 {
530 	/* this should fail */
531 	bpf_ringbuf_reserve_dynptr(&ringbuf, 16, 0, &global_dynptr);
532 
533 	bpf_ringbuf_discard_dynptr(&global_dynptr, 0);
534 
535 	return 0;
536 }
537 
538 /* A direct read should fail */
539 SEC("?raw_tp")
540 __failure __msg("invalid read from stack")
541 int invalid_read1(void *ctx)
542 {
543 	struct bpf_dynptr ptr;
544 
545 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr);
546 
547 	/* this should fail */
548 	val = *(int *)&ptr;
549 
550 	bpf_ringbuf_discard_dynptr(&ptr, 0);
551 
552 	return 0;
553 }
554 
555 /* A direct read at an offset should fail */
556 SEC("?raw_tp")
557 __failure __msg("cannot pass in dynptr at an offset")
558 int invalid_read2(void *ctx)
559 {
560 	struct bpf_dynptr ptr;
561 	char read_data[64];
562 
563 	get_map_val_dynptr(&ptr);
564 
565 	/* this should fail */
566 	bpf_dynptr_read(read_data, sizeof(read_data), (void *)&ptr + 1, 0, 0);
567 
568 	return 0;
569 }
570 
571 /* A direct read at an offset into the lower stack slot should fail */
572 SEC("?raw_tp")
573 __failure __msg("invalid read from stack")
574 int invalid_read3(void *ctx)
575 {
576 	struct bpf_dynptr ptr1, ptr2;
577 
578 	bpf_ringbuf_reserve_dynptr(&ringbuf, 16, 0, &ptr1);
579 	bpf_ringbuf_reserve_dynptr(&ringbuf, 16, 0, &ptr2);
580 
581 	/* this should fail */
582 	memcpy(&val, (void *)&ptr1 + 8, sizeof(val));
583 
584 	bpf_ringbuf_discard_dynptr(&ptr1, 0);
585 	bpf_ringbuf_discard_dynptr(&ptr2, 0);
586 
587 	return 0;
588 }
589 
590 static int invalid_read4_callback(__u32 index, void *data)
591 {
592 	/* this should fail */
593 	val = *(__u32 *)data;
594 
595 	return 0;
596 }
597 
598 /* A direct read within a callback function should fail */
599 SEC("?raw_tp")
600 __failure __msg("invalid read from stack")
601 int invalid_read4(void *ctx)
602 {
603 	struct bpf_dynptr ptr;
604 
605 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr);
606 
607 	bpf_loop(10, invalid_read4_callback, &ptr, 0);
608 
609 	bpf_ringbuf_submit_dynptr(&ptr, 0);
610 
611 	return 0;
612 }
613 
614 /* Initializing a dynptr on an offset should fail */
615 SEC("?raw_tp")
616 __failure __msg("cannot pass in dynptr at an offset=0")
617 int invalid_offset(void *ctx)
618 {
619 	struct bpf_dynptr ptr;
620 
621 	/* this should fail */
622 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr + 1);
623 
624 	bpf_ringbuf_discard_dynptr(&ptr, 0);
625 
626 	return 0;
627 }
628 
629 /* Can't release a dynptr twice */
630 SEC("?raw_tp")
631 __failure __msg("arg 1 is an unacquired reference")
632 int release_twice(void *ctx)
633 {
634 	struct bpf_dynptr ptr;
635 
636 	bpf_ringbuf_reserve_dynptr(&ringbuf, 16, 0, &ptr);
637 
638 	bpf_ringbuf_discard_dynptr(&ptr, 0);
639 
640 	/* this second release should fail */
641 	bpf_ringbuf_discard_dynptr(&ptr, 0);
642 
643 	return 0;
644 }
645 
646 static int release_twice_callback_fn(__u32 index, void *data)
647 {
648 	/* this should fail */
649 	bpf_ringbuf_discard_dynptr(data, 0);
650 
651 	return 0;
652 }
653 
654 /* Test that releasing a dynptr twice, where one of the releases happens
655  * within a callback function, fails
656  */
657 SEC("?raw_tp")
658 __failure __msg("arg 1 is an unacquired reference")
659 int release_twice_callback(void *ctx)
660 {
661 	struct bpf_dynptr ptr;
662 
663 	bpf_ringbuf_reserve_dynptr(&ringbuf, 32, 0, &ptr);
664 
665 	bpf_ringbuf_discard_dynptr(&ptr, 0);
666 
667 	bpf_loop(10, release_twice_callback_fn, &ptr, 0);
668 
669 	return 0;
670 }
671 
672 /* Reject unsupported local mem types for dynptr_from_mem API */
673 SEC("?raw_tp")
674 __failure __msg("Unsupported reg type fp for bpf_dynptr_from_mem data")
675 int dynptr_from_mem_invalid_api(void *ctx)
676 {
677 	struct bpf_dynptr ptr;
678 	int x = 0;
679 
680 	/* this should fail */
681 	bpf_dynptr_from_mem(&x, sizeof(x), 0, &ptr);
682 
683 	return 0;
684 }
685 
686 SEC("?tc")
687 __failure __msg("cannot overwrite referenced dynptr") __log_level(2)
688 int dynptr_pruning_overwrite(struct __sk_buff *ctx)
689 {
690 	asm volatile (
691 		"r9 = 0xeB9F;				\
692 		 r6 = %[ringbuf] ll;			\
693 		 r1 = r6;				\
694 		 r2 = 8;				\
695 		 r3 = 0;				\
696 		 r4 = r10;				\
697 		 r4 += -16;				\
698 		 call %[bpf_ringbuf_reserve_dynptr];	\
699 		 if r0 == 0 goto pjmp1;			\
700 		 goto pjmp2;				\
701 	pjmp1:						\
702 		 *(u64 *)(r10 - 16) = r9;		\
703 	pjmp2:						\
704 		 r1 = r10;				\
705 		 r1 += -16;				\
706 		 r2 = 0;				\
707 		 call %[bpf_ringbuf_discard_dynptr];	"
708 		:
709 		: __imm(bpf_ringbuf_reserve_dynptr),
710 		  __imm(bpf_ringbuf_discard_dynptr),
711 		  __imm_addr(ringbuf)
712 		: __clobber_all
713 	);
714 	return 0;
715 }
716 
717 SEC("?tc")
718 __success __msg("12: safe") __log_level(2)
719 int dynptr_pruning_stacksafe(struct __sk_buff *ctx)
720 {
721 	asm volatile (
722 		"r9 = 0xeB9F;				\
723 		 r6 = %[ringbuf] ll;			\
724 		 r1 = r6;				\
725 		 r2 = 8;				\
726 		 r3 = 0;				\
727 		 r4 = r10;				\
728 		 r4 += -16;				\
729 		 call %[bpf_ringbuf_reserve_dynptr];	\
730 		 if r0 == 0 goto stjmp1;		\
731 		 goto stjmp2;				\
732 	stjmp1:						\
733 		 r9 = r9;				\
734 	stjmp2:						\
735 		 r1 = r10;				\
736 		 r1 += -16;				\
737 		 r2 = 0;				\
738 		 call %[bpf_ringbuf_discard_dynptr];	"
739 		:
740 		: __imm(bpf_ringbuf_reserve_dynptr),
741 		  __imm(bpf_ringbuf_discard_dynptr),
742 		  __imm_addr(ringbuf)
743 		: __clobber_all
744 	);
745 	return 0;
746 }
747 
748 SEC("?tc")
749 __failure __msg("cannot overwrite referenced dynptr") __log_level(2)
750 int dynptr_pruning_type_confusion(struct __sk_buff *ctx)
751 {
752 	asm volatile (
753 		"r6 = %[array_map4] ll;			\
754 		 r7 = %[ringbuf] ll;			\
755 		 r1 = r6;				\
756 		 r2 = r10;				\
757 		 r2 += -8;				\
758 		 r9 = 0;				\
759 		 *(u64 *)(r2 + 0) = r9;			\
760 		 r3 = r10;				\
761 		 r3 += -24;				\
762 		 r9 = 0xeB9FeB9F;			\
763 		 *(u64 *)(r10 - 16) = r9;		\
764 		 *(u64 *)(r10 - 24) = r9;		\
765 		 r9 = 0;				\
766 		 r4 = 0;				\
767 		 r8 = r2;				\
768 		 call %[bpf_map_update_elem];		\
769 		 r1 = r6;				\
770 		 r2 = r8;				\
771 		 call %[bpf_map_lookup_elem];		\
772 		 if r0 != 0 goto tjmp1;			\
773 		 exit;					\
774 	tjmp1:						\
775 		 r8 = r0;				\
776 		 r1 = r7;				\
777 		 r2 = 8;				\
778 		 r3 = 0;				\
779 		 r4 = r10;				\
780 		 r4 += -16;				\
781 		 r0 = *(u64 *)(r0 + 0);			\
782 		 call %[bpf_ringbuf_reserve_dynptr];	\
783 		 if r0 == 0 goto tjmp2;			\
784 		 r8 = r8;				\
785 		 r8 = r8;				\
786 		 r8 = r8;				\
787 		 r8 = r8;				\
788 		 r8 = r8;				\
789 		 r8 = r8;				\
790 		 r8 = r8;				\
791 		 goto tjmp3;				\
792 	tjmp2:						\
793 		 *(u64 *)(r10 - 8) = r9;		\
794 		 *(u64 *)(r10 - 16) = r9;		\
795 		 r1 = r8;				\
796 		 r1 += 8;				\
797 		 r2 = 0;				\
798 		 r3 = 0;				\
799 		 r4 = r10;				\
800 		 r4 += -16;				\
801 		 call %[bpf_dynptr_from_mem];		\
802 	tjmp3:						\
803 		 r1 = r10;				\
804 		 r1 += -16;				\
805 		 r2 = 0;				\
806 		 call %[bpf_ringbuf_discard_dynptr];	"
807 		:
808 		: __imm(bpf_map_update_elem),
809 		  __imm(bpf_map_lookup_elem),
810 		  __imm(bpf_ringbuf_reserve_dynptr),
811 		  __imm(bpf_dynptr_from_mem),
812 		  __imm(bpf_ringbuf_discard_dynptr),
813 		  __imm_addr(array_map4),
814 		  __imm_addr(ringbuf)
815 		: __clobber_all
816 	);
817 	return 0;
818 }
819 
820 SEC("?tc")
821 __failure __msg("dynptr has to be at a constant offset") __log_level(2)
822 int dynptr_var_off_overwrite(struct __sk_buff *ctx)
823 {
824 	asm volatile (
825 		"r9 = 16;				\
826 		 *(u32 *)(r10 - 4) = r9;		\
827 		 r8 = *(u32 *)(r10 - 4);		\
828 		 if r8 >= 0 goto vjmp1;			\
829 		 r0 = 1;				\
830 		 exit;					\
831 	vjmp1:						\
832 		 if r8 <= 16 goto vjmp2;		\
833 		 r0 = 1;				\
834 		 exit;					\
835 	vjmp2:						\
836 		 r8 &= 16;				\
837 		 r1 = %[ringbuf] ll;			\
838 		 r2 = 8;				\
839 		 r3 = 0;				\
840 		 r4 = r10;				\
841 		 r4 += -32;				\
842 		 r4 += r8;				\
843 		 call %[bpf_ringbuf_reserve_dynptr];	\
844 		 r9 = 0xeB9F;				\
845 		 *(u64 *)(r10 - 16) = r9;		\
846 		 r1 = r10;				\
847 		 r1 += -32;				\
848 		 r1 += r8;				\
849 		 r2 = 0;				\
850 		 call %[bpf_ringbuf_discard_dynptr];	"
851 		:
852 		: __imm(bpf_ringbuf_reserve_dynptr),
853 		  __imm(bpf_ringbuf_discard_dynptr),
854 		  __imm_addr(ringbuf)
855 		: __clobber_all
856 	);
857 	return 0;
858 }
859 
860 SEC("?tc")
861 __failure __msg("cannot overwrite referenced dynptr") __log_level(2)
862 int dynptr_partial_slot_invalidate(struct __sk_buff *ctx)
863 {
864 	asm volatile (
865 		"r6 = %[ringbuf] ll;			\
866 		 r7 = %[array_map4] ll;			\
867 		 r1 = r7;				\
868 		 r2 = r10;				\
869 		 r2 += -8;				\
870 		 r9 = 0;				\
871 		 *(u64 *)(r2 + 0) = r9;			\
872 		 r3 = r2;				\
873 		 r4 = 0;				\
874 		 r8 = r2;				\
875 		 call %[bpf_map_update_elem];		\
876 		 r1 = r7;				\
877 		 r2 = r8;				\
878 		 call %[bpf_map_lookup_elem];		\
879 		 if r0 != 0 goto sjmp1;			\
880 		 exit;					\
881 	sjmp1:						\
882 		 r7 = r0;				\
883 		 r1 = r6;				\
884 		 r2 = 8;				\
885 		 r3 = 0;				\
886 		 r4 = r10;				\
887 		 r4 += -24;				\
888 		 call %[bpf_ringbuf_reserve_dynptr];	\
889 		 *(u64 *)(r10 - 16) = r9;		\
890 		 r1 = r7;				\
891 		 r2 = 8;				\
892 		 r3 = 0;				\
893 		 r4 = r10;				\
894 		 r4 += -16;				\
895 		 call %[bpf_dynptr_from_mem];		\
896 		 r1 = r10;				\
897 		 r1 += -512;				\
898 		 r2 = 488;				\
899 		 r3 = r10;				\
900 		 r3 += -24;				\
901 		 r4 = 0;				\
902 		 r5 = 0;				\
903 		 call %[bpf_dynptr_read];		\
904 		 r8 = 1;				\
905 		 if r0 != 0 goto sjmp2;			\
906 		 r8 = 0;				\
907 	sjmp2:						\
908 		 r1 = r10;				\
909 		 r1 += -24;				\
910 		 r2 = 0;				\
911 		 call %[bpf_ringbuf_discard_dynptr];	"
912 		:
913 		: __imm(bpf_map_update_elem),
914 		  __imm(bpf_map_lookup_elem),
915 		  __imm(bpf_ringbuf_reserve_dynptr),
916 		  __imm(bpf_ringbuf_discard_dynptr),
917 		  __imm(bpf_dynptr_from_mem),
918 		  __imm(bpf_dynptr_read),
919 		  __imm_addr(ringbuf),
920 		  __imm_addr(array_map4)
921 		: __clobber_all
922 	);
923 	return 0;
924 }
925 
926 /* Test that it is allowed to overwrite unreferenced dynptr. */
927 SEC("?raw_tp")
928 __success
929 int dynptr_overwrite_unref(void *ctx)
930 {
931 	struct bpf_dynptr ptr;
932 
933 	if (get_map_val_dynptr(&ptr))
934 		return 0;
935 	if (get_map_val_dynptr(&ptr))
936 		return 0;
937 	if (get_map_val_dynptr(&ptr))
938 		return 0;
939 
940 	return 0;
941 }
942 
943 /* Test that slices are invalidated on reinitializing a dynptr. */
944 SEC("?raw_tp")
945 __failure __msg("invalid mem access 'scalar'")
946 int dynptr_invalidate_slice_reinit(void *ctx)
947 {
948 	struct bpf_dynptr ptr;
949 	__u8 *p;
950 
951 	if (get_map_val_dynptr(&ptr))
952 		return 0;
953 	p = bpf_dynptr_data(&ptr, 0, 1);
954 	if (!p)
955 		return 0;
956 	if (get_map_val_dynptr(&ptr))
957 		return 0;
958 	/* this should fail */
959 	return *p;
960 }
961 
962 /* Invalidation of dynptr slices on destruction of dynptr should not miss
963  * mem_or_null pointers.
964  */
965 SEC("?raw_tp")
966 __failure __msg("R1 type=scalar expected=percpu_ptr_")
967 int dynptr_invalidate_slice_or_null(void *ctx)
968 {
969 	struct bpf_dynptr ptr;
970 	__u8 *p;
971 
972 	if (get_map_val_dynptr(&ptr))
973 		return 0;
974 
975 	p = bpf_dynptr_data(&ptr, 0, 1);
976 	*(__u8 *)&ptr = 0;
977 	/* this should fail */
978 	bpf_this_cpu_ptr(p);
979 	return 0;
980 }
981 
982 /* Destruction of dynptr should also any slices obtained from it */
983 SEC("?raw_tp")
984 __failure __msg("R7 invalid mem access 'scalar'")
985 int dynptr_invalidate_slice_failure(void *ctx)
986 {
987 	struct bpf_dynptr ptr1;
988 	struct bpf_dynptr ptr2;
989 	__u8 *p1, *p2;
990 
991 	if (get_map_val_dynptr(&ptr1))
992 		return 0;
993 	if (get_map_val_dynptr(&ptr2))
994 		return 0;
995 
996 	p1 = bpf_dynptr_data(&ptr1, 0, 1);
997 	if (!p1)
998 		return 0;
999 	p2 = bpf_dynptr_data(&ptr2, 0, 1);
1000 	if (!p2)
1001 		return 0;
1002 
1003 	*(__u8 *)&ptr1 = 0;
1004 	/* this should fail */
1005 	return *p1;
1006 }
1007 
1008 /* Invalidation of slices should be scoped and should not prevent dereferencing
1009  * slices of another dynptr after destroying unrelated dynptr
1010  */
1011 SEC("?raw_tp")
1012 __success
1013 int dynptr_invalidate_slice_success(void *ctx)
1014 {
1015 	struct bpf_dynptr ptr1;
1016 	struct bpf_dynptr ptr2;
1017 	__u8 *p1, *p2;
1018 
1019 	if (get_map_val_dynptr(&ptr1))
1020 		return 1;
1021 	if (get_map_val_dynptr(&ptr2))
1022 		return 1;
1023 
1024 	p1 = bpf_dynptr_data(&ptr1, 0, 1);
1025 	if (!p1)
1026 		return 1;
1027 	p2 = bpf_dynptr_data(&ptr2, 0, 1);
1028 	if (!p2)
1029 		return 1;
1030 
1031 	*(__u8 *)&ptr1 = 0;
1032 	return *p2;
1033 }
1034 
1035 /* Overwriting referenced dynptr should be rejected */
1036 SEC("?raw_tp")
1037 __failure __msg("cannot overwrite referenced dynptr")
1038 int dynptr_overwrite_ref(void *ctx)
1039 {
1040 	struct bpf_dynptr ptr;
1041 
1042 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr);
1043 	/* this should fail */
1044 	if (get_map_val_dynptr(&ptr))
1045 		bpf_ringbuf_discard_dynptr(&ptr, 0);
1046 	return 0;
1047 }
1048 
1049 /* Reject writes to dynptr slot from bpf_dynptr_read */
1050 SEC("?raw_tp")
1051 __failure __msg("potential write to dynptr at off=-16")
1052 int dynptr_read_into_slot(void *ctx)
1053 {
1054 	union {
1055 		struct {
1056 			char _pad[48];
1057 			struct bpf_dynptr ptr;
1058 		};
1059 		char buf[64];
1060 	} data;
1061 
1062 	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &data.ptr);
1063 	/* this should fail */
1064 	bpf_dynptr_read(data.buf, sizeof(data.buf), &data.ptr, 0, 0);
1065 
1066 	return 0;
1067 }
1068 
1069 /* bpf_dynptr_slice()s are read-only and cannot be written to */
1070 SEC("?tc")
1071 __failure __msg("R0 cannot write into rdonly_mem")
1072 int skb_invalid_slice_write(struct __sk_buff *skb)
1073 {
1074 	struct bpf_dynptr ptr;
1075 	struct ethhdr *hdr;
1076 	char buffer[sizeof(*hdr)] = {};
1077 
1078 	bpf_dynptr_from_skb(skb, 0, &ptr);
1079 
1080 	hdr = bpf_dynptr_slice(&ptr, 0, buffer, sizeof(buffer));
1081 	if (!hdr)
1082 		return SK_DROP;
1083 
1084 	/* this should fail */
1085 	hdr->h_proto = 1;
1086 
1087 	return SK_PASS;
1088 }
1089 
1090 /* The read-only data slice is invalidated whenever a helper changes packet data */
1091 SEC("?tc")
1092 __failure __msg("invalid mem access 'scalar'")
1093 int skb_invalid_data_slice1(struct __sk_buff *skb)
1094 {
1095 	struct bpf_dynptr ptr;
1096 	struct ethhdr *hdr;
1097 	char buffer[sizeof(*hdr)] = {};
1098 
1099 	bpf_dynptr_from_skb(skb, 0, &ptr);
1100 
1101 	hdr = bpf_dynptr_slice(&ptr, 0, buffer, sizeof(buffer));
1102 	if (!hdr)
1103 		return SK_DROP;
1104 
1105 	val = hdr->h_proto;
1106 
1107 	if (bpf_skb_pull_data(skb, skb->len))
1108 		return SK_DROP;
1109 
1110 	/* this should fail */
1111 	val = hdr->h_proto;
1112 
1113 	return SK_PASS;
1114 }
1115 
1116 /* The read-write data slice is invalidated whenever a helper changes packet data */
1117 SEC("?tc")
1118 __failure __msg("invalid mem access 'scalar'")
1119 int skb_invalid_data_slice2(struct __sk_buff *skb)
1120 {
1121 	struct bpf_dynptr ptr;
1122 	struct ethhdr *hdr;
1123 	char buffer[sizeof(*hdr)] = {};
1124 
1125 	bpf_dynptr_from_skb(skb, 0, &ptr);
1126 
1127 	hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer));
1128 	if (!hdr)
1129 		return SK_DROP;
1130 
1131 	hdr->h_proto = 123;
1132 
1133 	if (bpf_skb_pull_data(skb, skb->len))
1134 		return SK_DROP;
1135 
1136 	/* this should fail */
1137 	hdr->h_proto = 1;
1138 
1139 	return SK_PASS;
1140 }
1141 
1142 /* The read-only data slice is invalidated whenever bpf_dynptr_write() is called */
1143 SEC("?tc")
1144 __failure __msg("invalid mem access 'scalar'")
1145 int skb_invalid_data_slice3(struct __sk_buff *skb)
1146 {
1147 	char write_data[64] = "hello there, world!!";
1148 	struct bpf_dynptr ptr;
1149 	struct ethhdr *hdr;
1150 	char buffer[sizeof(*hdr)] = {};
1151 
1152 	bpf_dynptr_from_skb(skb, 0, &ptr);
1153 
1154 	hdr = bpf_dynptr_slice(&ptr, 0, buffer, sizeof(buffer));
1155 	if (!hdr)
1156 		return SK_DROP;
1157 
1158 	val = hdr->h_proto;
1159 
1160 	bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
1161 
1162 	/* this should fail */
1163 	val = hdr->h_proto;
1164 
1165 	return SK_PASS;
1166 }
1167 
1168 /* The read-write data slice is invalidated whenever bpf_dynptr_write() is called */
1169 SEC("?tc")
1170 __failure __msg("invalid mem access 'scalar'")
1171 int skb_invalid_data_slice4(struct __sk_buff *skb)
1172 {
1173 	char write_data[64] = "hello there, world!!";
1174 	struct bpf_dynptr ptr;
1175 	struct ethhdr *hdr;
1176 	char buffer[sizeof(*hdr)] = {};
1177 
1178 	bpf_dynptr_from_skb(skb, 0, &ptr);
1179 	hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer));
1180 	if (!hdr)
1181 		return SK_DROP;
1182 
1183 	hdr->h_proto = 123;
1184 
1185 	bpf_dynptr_write(&ptr, 0, write_data, sizeof(write_data), 0);
1186 
1187 	/* this should fail */
1188 	hdr->h_proto = 1;
1189 
1190 	return SK_PASS;
1191 }
1192 
1193 /* The read-only data slice is invalidated whenever a helper changes packet data */
1194 SEC("?xdp")
1195 __failure __msg("invalid mem access 'scalar'")
1196 int xdp_invalid_data_slice1(struct xdp_md *xdp)
1197 {
1198 	struct bpf_dynptr ptr;
1199 	struct ethhdr *hdr;
1200 	char buffer[sizeof(*hdr)] = {};
1201 
1202 	bpf_dynptr_from_xdp(xdp, 0, &ptr);
1203 	hdr = bpf_dynptr_slice(&ptr, 0, buffer, sizeof(buffer));
1204 	if (!hdr)
1205 		return SK_DROP;
1206 
1207 	val = hdr->h_proto;
1208 
1209 	if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(*hdr)))
1210 		return XDP_DROP;
1211 
1212 	/* this should fail */
1213 	val = hdr->h_proto;
1214 
1215 	return XDP_PASS;
1216 }
1217 
1218 /* The read-write data slice is invalidated whenever a helper changes packet data */
1219 SEC("?xdp")
1220 __failure __msg("invalid mem access 'scalar'")
1221 int xdp_invalid_data_slice2(struct xdp_md *xdp)
1222 {
1223 	struct bpf_dynptr ptr;
1224 	struct ethhdr *hdr;
1225 	char buffer[sizeof(*hdr)] = {};
1226 
1227 	bpf_dynptr_from_xdp(xdp, 0, &ptr);
1228 	hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer));
1229 	if (!hdr)
1230 		return SK_DROP;
1231 
1232 	hdr->h_proto = 9;
1233 
1234 	if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(*hdr)))
1235 		return XDP_DROP;
1236 
1237 	/* this should fail */
1238 	hdr->h_proto = 1;
1239 
1240 	return XDP_PASS;
1241 }
1242 
1243 /* Only supported prog type can create skb-type dynptrs */
1244 SEC("?raw_tp")
1245 __failure __msg("calling kernel function bpf_dynptr_from_skb is not allowed")
1246 int skb_invalid_ctx(void *ctx)
1247 {
1248 	struct bpf_dynptr ptr;
1249 
1250 	/* this should fail */
1251 	bpf_dynptr_from_skb(ctx, 0, &ptr);
1252 
1253 	return 0;
1254 }
1255 
1256 /* Reject writes to dynptr slot for uninit arg */
1257 SEC("?raw_tp")
1258 __failure __msg("potential write to dynptr at off=-16")
1259 int uninit_write_into_slot(void *ctx)
1260 {
1261 	struct {
1262 		char buf[64];
1263 		struct bpf_dynptr ptr;
1264 	} data;
1265 
1266 	bpf_ringbuf_reserve_dynptr(&ringbuf, 80, 0, &data.ptr);
1267 	/* this should fail */
1268 	bpf_get_current_comm(data.buf, 80);
1269 
1270 	return 0;
1271 }
1272 
1273 /* Only supported prog type can create xdp-type dynptrs */
1274 SEC("?raw_tp")
1275 __failure __msg("calling kernel function bpf_dynptr_from_xdp is not allowed")
1276 int xdp_invalid_ctx(void *ctx)
1277 {
1278 	struct bpf_dynptr ptr;
1279 
1280 	/* this should fail */
1281 	bpf_dynptr_from_xdp(ctx, 0, &ptr);
1282 
1283 	return 0;
1284 }
1285 
1286 __u32 hdr_size = sizeof(struct ethhdr);
1287 /* Can't pass in variable-sized len to bpf_dynptr_slice */
1288 SEC("?tc")
1289 __failure __msg("unbounded memory access")
1290 int dynptr_slice_var_len1(struct __sk_buff *skb)
1291 {
1292 	struct bpf_dynptr ptr;
1293 	struct ethhdr *hdr;
1294 	char buffer[sizeof(*hdr)] = {};
1295 
1296 	bpf_dynptr_from_skb(skb, 0, &ptr);
1297 
1298 	/* this should fail */
1299 	hdr = bpf_dynptr_slice(&ptr, 0, buffer, hdr_size);
1300 	if (!hdr)
1301 		return SK_DROP;
1302 
1303 	return SK_PASS;
1304 }
1305 
1306 /* Can't pass in variable-sized len to bpf_dynptr_slice */
1307 SEC("?tc")
1308 __failure __msg("must be a known constant")
1309 int dynptr_slice_var_len2(struct __sk_buff *skb)
1310 {
1311 	char buffer[sizeof(struct ethhdr)] = {};
1312 	struct bpf_dynptr ptr;
1313 	struct ethhdr *hdr;
1314 
1315 	bpf_dynptr_from_skb(skb, 0, &ptr);
1316 
1317 	if (hdr_size <= sizeof(buffer)) {
1318 		/* this should fail */
1319 		hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, hdr_size);
1320 		if (!hdr)
1321 			return SK_DROP;
1322 		hdr->h_proto = 12;
1323 	}
1324 
1325 	return SK_PASS;
1326 }
1327 
1328 static int callback(__u32 index, void *data)
1329 {
1330         *(__u32 *)data = 123;
1331 
1332         return 0;
1333 }
1334 
1335 /* If the dynptr is written into in a callback function, its data
1336  * slices should be invalidated as well.
1337  */
1338 SEC("?raw_tp")
1339 __failure __msg("invalid mem access 'scalar'")
1340 int invalid_data_slices(void *ctx)
1341 {
1342 	struct bpf_dynptr ptr;
1343 	__u32 *slice;
1344 
1345 	if (get_map_val_dynptr(&ptr))
1346 		return 0;
1347 
1348 	slice = bpf_dynptr_data(&ptr, 0, sizeof(__u32));
1349 	if (!slice)
1350 		return 0;
1351 
1352 	bpf_loop(10, callback, &ptr, 0);
1353 
1354 	/* this should fail */
1355 	*slice = 1;
1356 
1357 	return 0;
1358 }
1359 
1360 /* Program types that don't allow writes to packet data should fail if
1361  * bpf_dynptr_slice_rdwr is called
1362  */
1363 SEC("cgroup_skb/ingress")
1364 __failure __msg("the prog does not allow writes to packet data")
1365 int invalid_slice_rdwr_rdonly(struct __sk_buff *skb)
1366 {
1367 	char buffer[sizeof(struct ethhdr)] = {};
1368 	struct bpf_dynptr ptr;
1369 	struct ethhdr *hdr;
1370 
1371 	bpf_dynptr_from_skb(skb, 0, &ptr);
1372 
1373 	/* this should fail since cgroup_skb doesn't allow
1374 	 * changing packet data
1375 	 */
1376 	hdr = bpf_dynptr_slice_rdwr(&ptr, 0, buffer, sizeof(buffer));
1377 	__sink(hdr);
1378 
1379 	return 0;
1380 }
1381