1 /*
2  * Testsuite for eBPF verifier
3  *
4  * Copyright (c) 2014 PLUMgrid, http://plumgrid.com
5  * Copyright (c) 2017 Facebook
6  * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of version 2 of the GNU General Public
10  * License as published by the Free Software Foundation.
11  */
12 
13 #include <endian.h>
14 #include <asm/types.h>
15 #include <linux/types.h>
16 #include <stdint.h>
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <unistd.h>
20 #include <errno.h>
21 #include <string.h>
22 #include <stddef.h>
23 #include <stdbool.h>
24 #include <sched.h>
25 #include <limits.h>
26 #include <assert.h>
27 
28 #include <sys/capability.h>
29 
30 #include <linux/unistd.h>
31 #include <linux/filter.h>
32 #include <linux/bpf_perf_event.h>
33 #include <linux/bpf.h>
34 #include <linux/if_ether.h>
35 #include <linux/btf.h>
36 
37 #include <bpf/bpf.h>
38 #include <bpf/libbpf.h>
39 
40 #ifdef HAVE_GENHDR
41 # include "autoconf.h"
42 #else
43 # if defined(__i386) || defined(__x86_64) || defined(__s390x__) || defined(__aarch64__)
44 #  define CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS 1
45 # endif
46 #endif
47 #include "bpf_rlimit.h"
48 #include "bpf_rand.h"
49 #include "bpf_util.h"
50 #include "test_btf.h"
51 #include "../../../include/linux/filter.h"
52 
53 #define MAX_INSNS	BPF_MAXINSNS
54 #define MAX_TEST_INSNS	1000000
55 #define MAX_FIXUPS	8
56 #define MAX_NR_MAPS	18
57 #define MAX_TEST_RUNS	8
58 #define POINTER_VALUE	0xcafe4all
59 #define TEST_DATA_LEN	64
60 
61 #define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS	(1 << 0)
62 #define F_LOAD_WITH_STRICT_ALIGNMENT		(1 << 1)
63 
64 #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled"
65 static bool unpriv_disabled = false;
66 static int skips;
67 
68 struct bpf_test {
69 	const char *descr;
70 	struct bpf_insn	insns[MAX_INSNS];
71 	struct bpf_insn	*fill_insns;
72 	int fixup_map_hash_8b[MAX_FIXUPS];
73 	int fixup_map_hash_48b[MAX_FIXUPS];
74 	int fixup_map_hash_16b[MAX_FIXUPS];
75 	int fixup_map_array_48b[MAX_FIXUPS];
76 	int fixup_map_sockmap[MAX_FIXUPS];
77 	int fixup_map_sockhash[MAX_FIXUPS];
78 	int fixup_map_xskmap[MAX_FIXUPS];
79 	int fixup_map_stacktrace[MAX_FIXUPS];
80 	int fixup_prog1[MAX_FIXUPS];
81 	int fixup_prog2[MAX_FIXUPS];
82 	int fixup_map_in_map[MAX_FIXUPS];
83 	int fixup_cgroup_storage[MAX_FIXUPS];
84 	int fixup_percpu_cgroup_storage[MAX_FIXUPS];
85 	int fixup_map_spin_lock[MAX_FIXUPS];
86 	int fixup_map_array_ro[MAX_FIXUPS];
87 	int fixup_map_array_wo[MAX_FIXUPS];
88 	int fixup_map_array_small[MAX_FIXUPS];
89 	int fixup_sk_storage_map[MAX_FIXUPS];
90 	const char *errstr;
91 	const char *errstr_unpriv;
92 	uint32_t retval, retval_unpriv, insn_processed;
93 	int prog_len;
94 	enum {
95 		UNDEF,
96 		ACCEPT,
97 		REJECT
98 	} result, result_unpriv;
99 	enum bpf_prog_type prog_type;
100 	uint8_t flags;
101 	__u8 data[TEST_DATA_LEN];
102 	void (*fill_helper)(struct bpf_test *self);
103 	uint8_t runs;
104 	struct {
105 		uint32_t retval, retval_unpriv;
106 		union {
107 			__u8 data[TEST_DATA_LEN];
108 			__u64 data64[TEST_DATA_LEN / 8];
109 		};
110 	} retvals[MAX_TEST_RUNS];
111 };
112 
113 /* Note we want this to be 64 bit aligned so that the end of our array is
114  * actually the end of the structure.
115  */
116 #define MAX_ENTRIES 11
117 
118 struct test_val {
119 	unsigned int index;
120 	int foo[MAX_ENTRIES];
121 };
122 
123 struct other_val {
124 	long long foo;
125 	long long bar;
126 };
127 
128 static void bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
129 {
130 	/* test: {skb->data[0], vlan_push} x 51 + {skb->data[0], vlan_pop} x 51 */
131 #define PUSH_CNT 51
132 	/* jump range is limited to 16 bit. PUSH_CNT of ld_abs needs room */
133 	unsigned int len = (1 << 15) - PUSH_CNT * 2 * 5 * 6;
134 	struct bpf_insn *insn = self->fill_insns;
135 	int i = 0, j, k = 0;
136 
137 	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
138 loop:
139 	for (j = 0; j < PUSH_CNT; j++) {
140 		insn[i++] = BPF_LD_ABS(BPF_B, 0);
141 		insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
142 		i++;
143 		insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
144 		insn[i++] = BPF_MOV64_IMM(BPF_REG_2, 1);
145 		insn[i++] = BPF_MOV64_IMM(BPF_REG_3, 2);
146 		insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
147 					 BPF_FUNC_skb_vlan_push),
148 		insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
149 		i++;
150 	}
151 
152 	for (j = 0; j < PUSH_CNT; j++) {
153 		insn[i++] = BPF_LD_ABS(BPF_B, 0);
154 		insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
155 		i++;
156 		insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
157 		insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
158 					 BPF_FUNC_skb_vlan_pop),
159 		insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
160 		i++;
161 	}
162 	if (++k < 5)
163 		goto loop;
164 
165 	for (; i < len - 1; i++)
166 		insn[i] = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, 0xbef);
167 	insn[len - 1] = BPF_EXIT_INSN();
168 	self->prog_len = len;
169 }
170 
171 static void bpf_fill_jump_around_ld_abs(struct bpf_test *self)
172 {
173 	struct bpf_insn *insn = self->fill_insns;
174 	/* jump range is limited to 16 bit. every ld_abs is replaced by 6 insns */
175 	unsigned int len = (1 << 15) / 6;
176 	int i = 0;
177 
178 	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
179 	insn[i++] = BPF_LD_ABS(BPF_B, 0);
180 	insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 10, len - i - 2);
181 	i++;
182 	while (i < len - 1)
183 		insn[i++] = BPF_LD_ABS(BPF_B, 1);
184 	insn[i] = BPF_EXIT_INSN();
185 	self->prog_len = i + 1;
186 }
187 
188 static void bpf_fill_rand_ld_dw(struct bpf_test *self)
189 {
190 	struct bpf_insn *insn = self->fill_insns;
191 	uint64_t res = 0;
192 	int i = 0;
193 
194 	insn[i++] = BPF_MOV32_IMM(BPF_REG_0, 0);
195 	while (i < self->retval) {
196 		uint64_t val = bpf_semi_rand_get();
197 		struct bpf_insn tmp[2] = { BPF_LD_IMM64(BPF_REG_1, val) };
198 
199 		res ^= val;
200 		insn[i++] = tmp[0];
201 		insn[i++] = tmp[1];
202 		insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1);
203 	}
204 	insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
205 	insn[i++] = BPF_ALU64_IMM(BPF_RSH, BPF_REG_1, 32);
206 	insn[i++] = BPF_ALU64_REG(BPF_XOR, BPF_REG_0, BPF_REG_1);
207 	insn[i] = BPF_EXIT_INSN();
208 	self->prog_len = i + 1;
209 	res ^= (res >> 32);
210 	self->retval = (uint32_t)res;
211 }
212 
213 /* test the sequence of 1k jumps */
214 static void bpf_fill_scale1(struct bpf_test *self)
215 {
216 	struct bpf_insn *insn = self->fill_insns;
217 	int i = 0, k = 0;
218 
219 	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
220 	/* test to check that the sequence of 1024 jumps is acceptable */
221 	while (k++ < 1024) {
222 		insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
223 					 BPF_FUNC_get_prandom_u32);
224 		insn[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, bpf_semi_rand_get(), 2);
225 		insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10);
226 		insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
227 					-8 * (k % 64 + 1));
228 	}
229 	/* every jump adds 1024 steps to insn_processed, so to stay exactly
230 	 * within 1m limit add MAX_TEST_INSNS - 1025 MOVs and 1 EXIT
231 	 */
232 	while (i < MAX_TEST_INSNS - 1025)
233 		insn[i++] = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, 42);
234 	insn[i] = BPF_EXIT_INSN();
235 	self->prog_len = i + 1;
236 	self->retval = 42;
237 }
238 
239 /* test the sequence of 1k jumps in inner most function (function depth 8)*/
240 static void bpf_fill_scale2(struct bpf_test *self)
241 {
242 	struct bpf_insn *insn = self->fill_insns;
243 	int i = 0, k = 0;
244 
245 #define FUNC_NEST 7
246 	for (k = 0; k < FUNC_NEST; k++) {
247 		insn[i++] = BPF_CALL_REL(1);
248 		insn[i++] = BPF_EXIT_INSN();
249 	}
250 	insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
251 	/* test to check that the sequence of 1024 jumps is acceptable */
252 	while (k++ < 1024) {
253 		insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
254 					 BPF_FUNC_get_prandom_u32);
255 		insn[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, bpf_semi_rand_get(), 2);
256 		insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_10);
257 		insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
258 					-8 * (k % (64 - 4 * FUNC_NEST) + 1));
259 	}
260 	/* every jump adds 1024 steps to insn_processed, so to stay exactly
261 	 * within 1m limit add MAX_TEST_INSNS - 1025 MOVs and 1 EXIT
262 	 */
263 	while (i < MAX_TEST_INSNS - 1025)
264 		insn[i++] = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, 42);
265 	insn[i] = BPF_EXIT_INSN();
266 	self->prog_len = i + 1;
267 	self->retval = 42;
268 }
269 
270 static void bpf_fill_scale(struct bpf_test *self)
271 {
272 	switch (self->retval) {
273 	case 1:
274 		return bpf_fill_scale1(self);
275 	case 2:
276 		return bpf_fill_scale2(self);
277 	default:
278 		self->prog_len = 0;
279 		break;
280 	}
281 }
282 
283 /* BPF_SK_LOOKUP contains 13 instructions, if you need to fix up maps */
284 #define BPF_SK_LOOKUP(func)						\
285 	/* struct bpf_sock_tuple tuple = {} */				\
286 	BPF_MOV64_IMM(BPF_REG_2, 0),					\
287 	BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8),			\
288 	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -16),		\
289 	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -24),		\
290 	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -32),		\
291 	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -40),		\
292 	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -48),		\
293 	/* sk = func(ctx, &tuple, sizeof tuple, 0, 0) */		\
294 	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),				\
295 	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -48),				\
296 	BPF_MOV64_IMM(BPF_REG_3, sizeof(struct bpf_sock_tuple)),	\
297 	BPF_MOV64_IMM(BPF_REG_4, 0),					\
298 	BPF_MOV64_IMM(BPF_REG_5, 0),					\
299 	BPF_EMIT_CALL(BPF_FUNC_ ## func)
300 
301 /* BPF_DIRECT_PKT_R2 contains 7 instructions, it initializes default return
302  * value into 0 and does necessary preparation for direct packet access
303  * through r2. The allowed access range is 8 bytes.
304  */
305 #define BPF_DIRECT_PKT_R2						\
306 	BPF_MOV64_IMM(BPF_REG_0, 0),					\
307 	BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,			\
308 		    offsetof(struct __sk_buff, data)),			\
309 	BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,			\
310 		    offsetof(struct __sk_buff, data_end)),		\
311 	BPF_MOV64_REG(BPF_REG_4, BPF_REG_2),				\
312 	BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 8),				\
313 	BPF_JMP_REG(BPF_JLE, BPF_REG_4, BPF_REG_3, 1),			\
314 	BPF_EXIT_INSN()
315 
316 /* BPF_RAND_UEXT_R7 contains 4 instructions, it initializes R7 into a random
317  * positive u32, and zero-extend it into 64-bit.
318  */
319 #define BPF_RAND_UEXT_R7						\
320 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,			\
321 		     BPF_FUNC_get_prandom_u32),				\
322 	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),				\
323 	BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 33),				\
324 	BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 33)
325 
326 /* BPF_RAND_SEXT_R7 contains 5 instructions, it initializes R7 into a random
327  * negative u32, and sign-extend it into 64-bit.
328  */
329 #define BPF_RAND_SEXT_R7						\
330 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,			\
331 		     BPF_FUNC_get_prandom_u32),				\
332 	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),				\
333 	BPF_ALU64_IMM(BPF_OR, BPF_REG_7, 0x80000000),			\
334 	BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 32),				\
335 	BPF_ALU64_IMM(BPF_ARSH, BPF_REG_7, 32)
336 
337 static struct bpf_test tests[] = {
338 #define FILL_ARRAY
339 #include <verifier/tests.h>
340 #undef FILL_ARRAY
341 };
342 
343 static int probe_filter_length(const struct bpf_insn *fp)
344 {
345 	int len;
346 
347 	for (len = MAX_INSNS - 1; len > 0; --len)
348 		if (fp[len].code != 0 || fp[len].imm != 0)
349 			break;
350 	return len + 1;
351 }
352 
353 static bool skip_unsupported_map(enum bpf_map_type map_type)
354 {
355 	if (!bpf_probe_map_type(map_type, 0)) {
356 		printf("SKIP (unsupported map type %d)\n", map_type);
357 		skips++;
358 		return true;
359 	}
360 	return false;
361 }
362 
363 static int __create_map(uint32_t type, uint32_t size_key,
364 			uint32_t size_value, uint32_t max_elem,
365 			uint32_t extra_flags)
366 {
367 	int fd;
368 
369 	fd = bpf_create_map(type, size_key, size_value, max_elem,
370 			    (type == BPF_MAP_TYPE_HASH ?
371 			     BPF_F_NO_PREALLOC : 0) | extra_flags);
372 	if (fd < 0) {
373 		if (skip_unsupported_map(type))
374 			return -1;
375 		printf("Failed to create hash map '%s'!\n", strerror(errno));
376 	}
377 
378 	return fd;
379 }
380 
381 static int create_map(uint32_t type, uint32_t size_key,
382 		      uint32_t size_value, uint32_t max_elem)
383 {
384 	return __create_map(type, size_key, size_value, max_elem, 0);
385 }
386 
387 static void update_map(int fd, int index)
388 {
389 	struct test_val value = {
390 		.index = (6 + 1) * sizeof(int),
391 		.foo[6] = 0xabcdef12,
392 	};
393 
394 	assert(!bpf_map_update_elem(fd, &index, &value, 0));
395 }
396 
397 static int create_prog_dummy1(enum bpf_prog_type prog_type)
398 {
399 	struct bpf_insn prog[] = {
400 		BPF_MOV64_IMM(BPF_REG_0, 42),
401 		BPF_EXIT_INSN(),
402 	};
403 
404 	return bpf_load_program(prog_type, prog,
405 				ARRAY_SIZE(prog), "GPL", 0, NULL, 0);
406 }
407 
408 static int create_prog_dummy2(enum bpf_prog_type prog_type, int mfd, int idx)
409 {
410 	struct bpf_insn prog[] = {
411 		BPF_MOV64_IMM(BPF_REG_3, idx),
412 		BPF_LD_MAP_FD(BPF_REG_2, mfd),
413 		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
414 			     BPF_FUNC_tail_call),
415 		BPF_MOV64_IMM(BPF_REG_0, 41),
416 		BPF_EXIT_INSN(),
417 	};
418 
419 	return bpf_load_program(prog_type, prog,
420 				ARRAY_SIZE(prog), "GPL", 0, NULL, 0);
421 }
422 
423 static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem,
424 			     int p1key)
425 {
426 	int p2key = 1;
427 	int mfd, p1fd, p2fd;
428 
429 	mfd = bpf_create_map(BPF_MAP_TYPE_PROG_ARRAY, sizeof(int),
430 			     sizeof(int), max_elem, 0);
431 	if (mfd < 0) {
432 		if (skip_unsupported_map(BPF_MAP_TYPE_PROG_ARRAY))
433 			return -1;
434 		printf("Failed to create prog array '%s'!\n", strerror(errno));
435 		return -1;
436 	}
437 
438 	p1fd = create_prog_dummy1(prog_type);
439 	p2fd = create_prog_dummy2(prog_type, mfd, p2key);
440 	if (p1fd < 0 || p2fd < 0)
441 		goto out;
442 	if (bpf_map_update_elem(mfd, &p1key, &p1fd, BPF_ANY) < 0)
443 		goto out;
444 	if (bpf_map_update_elem(mfd, &p2key, &p2fd, BPF_ANY) < 0)
445 		goto out;
446 	close(p2fd);
447 	close(p1fd);
448 
449 	return mfd;
450 out:
451 	close(p2fd);
452 	close(p1fd);
453 	close(mfd);
454 	return -1;
455 }
456 
457 static int create_map_in_map(void)
458 {
459 	int inner_map_fd, outer_map_fd;
460 
461 	inner_map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(int),
462 				      sizeof(int), 1, 0);
463 	if (inner_map_fd < 0) {
464 		if (skip_unsupported_map(BPF_MAP_TYPE_ARRAY))
465 			return -1;
466 		printf("Failed to create array '%s'!\n", strerror(errno));
467 		return inner_map_fd;
468 	}
469 
470 	outer_map_fd = bpf_create_map_in_map(BPF_MAP_TYPE_ARRAY_OF_MAPS, NULL,
471 					     sizeof(int), inner_map_fd, 1, 0);
472 	if (outer_map_fd < 0) {
473 		if (skip_unsupported_map(BPF_MAP_TYPE_ARRAY_OF_MAPS))
474 			return -1;
475 		printf("Failed to create array of maps '%s'!\n",
476 		       strerror(errno));
477 	}
478 
479 	close(inner_map_fd);
480 
481 	return outer_map_fd;
482 }
483 
484 static int create_cgroup_storage(bool percpu)
485 {
486 	enum bpf_map_type type = percpu ? BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE :
487 		BPF_MAP_TYPE_CGROUP_STORAGE;
488 	int fd;
489 
490 	fd = bpf_create_map(type, sizeof(struct bpf_cgroup_storage_key),
491 			    TEST_DATA_LEN, 0, 0);
492 	if (fd < 0) {
493 		if (skip_unsupported_map(type))
494 			return -1;
495 		printf("Failed to create cgroup storage '%s'!\n",
496 		       strerror(errno));
497 	}
498 
499 	return fd;
500 }
501 
502 /* struct bpf_spin_lock {
503  *   int val;
504  * };
505  * struct val {
506  *   int cnt;
507  *   struct bpf_spin_lock l;
508  * };
509  */
510 static const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";
511 static __u32 btf_raw_types[] = {
512 	/* int */
513 	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
514 	/* struct bpf_spin_lock */                      /* [2] */
515 	BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
516 	BTF_MEMBER_ENC(15, 1, 0), /* int val; */
517 	/* struct val */                                /* [3] */
518 	BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
519 	BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */
520 	BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */
521 };
522 
523 static int load_btf(void)
524 {
525 	struct btf_header hdr = {
526 		.magic = BTF_MAGIC,
527 		.version = BTF_VERSION,
528 		.hdr_len = sizeof(struct btf_header),
529 		.type_len = sizeof(btf_raw_types),
530 		.str_off = sizeof(btf_raw_types),
531 		.str_len = sizeof(btf_str_sec),
532 	};
533 	void *ptr, *raw_btf;
534 	int btf_fd;
535 
536 	ptr = raw_btf = malloc(sizeof(hdr) + sizeof(btf_raw_types) +
537 			       sizeof(btf_str_sec));
538 
539 	memcpy(ptr, &hdr, sizeof(hdr));
540 	ptr += sizeof(hdr);
541 	memcpy(ptr, btf_raw_types, hdr.type_len);
542 	ptr += hdr.type_len;
543 	memcpy(ptr, btf_str_sec, hdr.str_len);
544 	ptr += hdr.str_len;
545 
546 	btf_fd = bpf_load_btf(raw_btf, ptr - raw_btf, 0, 0, 0);
547 	free(raw_btf);
548 	if (btf_fd < 0)
549 		return -1;
550 	return btf_fd;
551 }
552 
553 static int create_map_spin_lock(void)
554 {
555 	struct bpf_create_map_attr attr = {
556 		.name = "test_map",
557 		.map_type = BPF_MAP_TYPE_ARRAY,
558 		.key_size = 4,
559 		.value_size = 8,
560 		.max_entries = 1,
561 		.btf_key_type_id = 1,
562 		.btf_value_type_id = 3,
563 	};
564 	int fd, btf_fd;
565 
566 	btf_fd = load_btf();
567 	if (btf_fd < 0)
568 		return -1;
569 	attr.btf_fd = btf_fd;
570 	fd = bpf_create_map_xattr(&attr);
571 	if (fd < 0)
572 		printf("Failed to create map with spin_lock\n");
573 	return fd;
574 }
575 
576 static int create_sk_storage_map(void)
577 {
578 	struct bpf_create_map_attr attr = {
579 		.name = "test_map",
580 		.map_type = BPF_MAP_TYPE_SK_STORAGE,
581 		.key_size = 4,
582 		.value_size = 8,
583 		.max_entries = 0,
584 		.map_flags = BPF_F_NO_PREALLOC,
585 		.btf_key_type_id = 1,
586 		.btf_value_type_id = 3,
587 	};
588 	int fd, btf_fd;
589 
590 	btf_fd = load_btf();
591 	if (btf_fd < 0)
592 		return -1;
593 	attr.btf_fd = btf_fd;
594 	fd = bpf_create_map_xattr(&attr);
595 	close(attr.btf_fd);
596 	if (fd < 0)
597 		printf("Failed to create sk_storage_map\n");
598 	return fd;
599 }
600 
601 static char bpf_vlog[UINT_MAX >> 8];
602 
603 static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type,
604 			  struct bpf_insn *prog, int *map_fds)
605 {
606 	int *fixup_map_hash_8b = test->fixup_map_hash_8b;
607 	int *fixup_map_hash_48b = test->fixup_map_hash_48b;
608 	int *fixup_map_hash_16b = test->fixup_map_hash_16b;
609 	int *fixup_map_array_48b = test->fixup_map_array_48b;
610 	int *fixup_map_sockmap = test->fixup_map_sockmap;
611 	int *fixup_map_sockhash = test->fixup_map_sockhash;
612 	int *fixup_map_xskmap = test->fixup_map_xskmap;
613 	int *fixup_map_stacktrace = test->fixup_map_stacktrace;
614 	int *fixup_prog1 = test->fixup_prog1;
615 	int *fixup_prog2 = test->fixup_prog2;
616 	int *fixup_map_in_map = test->fixup_map_in_map;
617 	int *fixup_cgroup_storage = test->fixup_cgroup_storage;
618 	int *fixup_percpu_cgroup_storage = test->fixup_percpu_cgroup_storage;
619 	int *fixup_map_spin_lock = test->fixup_map_spin_lock;
620 	int *fixup_map_array_ro = test->fixup_map_array_ro;
621 	int *fixup_map_array_wo = test->fixup_map_array_wo;
622 	int *fixup_map_array_small = test->fixup_map_array_small;
623 	int *fixup_sk_storage_map = test->fixup_sk_storage_map;
624 
625 	if (test->fill_helper) {
626 		test->fill_insns = calloc(MAX_TEST_INSNS, sizeof(struct bpf_insn));
627 		test->fill_helper(test);
628 	}
629 
630 	/* Allocating HTs with 1 elem is fine here, since we only test
631 	 * for verifier and not do a runtime lookup, so the only thing
632 	 * that really matters is value size in this case.
633 	 */
634 	if (*fixup_map_hash_8b) {
635 		map_fds[0] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long),
636 					sizeof(long long), 1);
637 		do {
638 			prog[*fixup_map_hash_8b].imm = map_fds[0];
639 			fixup_map_hash_8b++;
640 		} while (*fixup_map_hash_8b);
641 	}
642 
643 	if (*fixup_map_hash_48b) {
644 		map_fds[1] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long),
645 					sizeof(struct test_val), 1);
646 		do {
647 			prog[*fixup_map_hash_48b].imm = map_fds[1];
648 			fixup_map_hash_48b++;
649 		} while (*fixup_map_hash_48b);
650 	}
651 
652 	if (*fixup_map_hash_16b) {
653 		map_fds[2] = create_map(BPF_MAP_TYPE_HASH, sizeof(long long),
654 					sizeof(struct other_val), 1);
655 		do {
656 			prog[*fixup_map_hash_16b].imm = map_fds[2];
657 			fixup_map_hash_16b++;
658 		} while (*fixup_map_hash_16b);
659 	}
660 
661 	if (*fixup_map_array_48b) {
662 		map_fds[3] = create_map(BPF_MAP_TYPE_ARRAY, sizeof(int),
663 					sizeof(struct test_val), 1);
664 		update_map(map_fds[3], 0);
665 		do {
666 			prog[*fixup_map_array_48b].imm = map_fds[3];
667 			fixup_map_array_48b++;
668 		} while (*fixup_map_array_48b);
669 	}
670 
671 	if (*fixup_prog1) {
672 		map_fds[4] = create_prog_array(prog_type, 4, 0);
673 		do {
674 			prog[*fixup_prog1].imm = map_fds[4];
675 			fixup_prog1++;
676 		} while (*fixup_prog1);
677 	}
678 
679 	if (*fixup_prog2) {
680 		map_fds[5] = create_prog_array(prog_type, 8, 7);
681 		do {
682 			prog[*fixup_prog2].imm = map_fds[5];
683 			fixup_prog2++;
684 		} while (*fixup_prog2);
685 	}
686 
687 	if (*fixup_map_in_map) {
688 		map_fds[6] = create_map_in_map();
689 		do {
690 			prog[*fixup_map_in_map].imm = map_fds[6];
691 			fixup_map_in_map++;
692 		} while (*fixup_map_in_map);
693 	}
694 
695 	if (*fixup_cgroup_storage) {
696 		map_fds[7] = create_cgroup_storage(false);
697 		do {
698 			prog[*fixup_cgroup_storage].imm = map_fds[7];
699 			fixup_cgroup_storage++;
700 		} while (*fixup_cgroup_storage);
701 	}
702 
703 	if (*fixup_percpu_cgroup_storage) {
704 		map_fds[8] = create_cgroup_storage(true);
705 		do {
706 			prog[*fixup_percpu_cgroup_storage].imm = map_fds[8];
707 			fixup_percpu_cgroup_storage++;
708 		} while (*fixup_percpu_cgroup_storage);
709 	}
710 	if (*fixup_map_sockmap) {
711 		map_fds[9] = create_map(BPF_MAP_TYPE_SOCKMAP, sizeof(int),
712 					sizeof(int), 1);
713 		do {
714 			prog[*fixup_map_sockmap].imm = map_fds[9];
715 			fixup_map_sockmap++;
716 		} while (*fixup_map_sockmap);
717 	}
718 	if (*fixup_map_sockhash) {
719 		map_fds[10] = create_map(BPF_MAP_TYPE_SOCKHASH, sizeof(int),
720 					sizeof(int), 1);
721 		do {
722 			prog[*fixup_map_sockhash].imm = map_fds[10];
723 			fixup_map_sockhash++;
724 		} while (*fixup_map_sockhash);
725 	}
726 	if (*fixup_map_xskmap) {
727 		map_fds[11] = create_map(BPF_MAP_TYPE_XSKMAP, sizeof(int),
728 					sizeof(int), 1);
729 		do {
730 			prog[*fixup_map_xskmap].imm = map_fds[11];
731 			fixup_map_xskmap++;
732 		} while (*fixup_map_xskmap);
733 	}
734 	if (*fixup_map_stacktrace) {
735 		map_fds[12] = create_map(BPF_MAP_TYPE_STACK_TRACE, sizeof(u32),
736 					 sizeof(u64), 1);
737 		do {
738 			prog[*fixup_map_stacktrace].imm = map_fds[12];
739 			fixup_map_stacktrace++;
740 		} while (*fixup_map_stacktrace);
741 	}
742 	if (*fixup_map_spin_lock) {
743 		map_fds[13] = create_map_spin_lock();
744 		do {
745 			prog[*fixup_map_spin_lock].imm = map_fds[13];
746 			fixup_map_spin_lock++;
747 		} while (*fixup_map_spin_lock);
748 	}
749 	if (*fixup_map_array_ro) {
750 		map_fds[14] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int),
751 					   sizeof(struct test_val), 1,
752 					   BPF_F_RDONLY_PROG);
753 		update_map(map_fds[14], 0);
754 		do {
755 			prog[*fixup_map_array_ro].imm = map_fds[14];
756 			fixup_map_array_ro++;
757 		} while (*fixup_map_array_ro);
758 	}
759 	if (*fixup_map_array_wo) {
760 		map_fds[15] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int),
761 					   sizeof(struct test_val), 1,
762 					   BPF_F_WRONLY_PROG);
763 		update_map(map_fds[15], 0);
764 		do {
765 			prog[*fixup_map_array_wo].imm = map_fds[15];
766 			fixup_map_array_wo++;
767 		} while (*fixup_map_array_wo);
768 	}
769 	if (*fixup_map_array_small) {
770 		map_fds[16] = __create_map(BPF_MAP_TYPE_ARRAY, sizeof(int),
771 					   1, 1, 0);
772 		update_map(map_fds[16], 0);
773 		do {
774 			prog[*fixup_map_array_small].imm = map_fds[16];
775 			fixup_map_array_small++;
776 		} while (*fixup_map_array_small);
777 	}
778 	if (*fixup_sk_storage_map) {
779 		map_fds[17] = create_sk_storage_map();
780 		do {
781 			prog[*fixup_sk_storage_map].imm = map_fds[17];
782 			fixup_sk_storage_map++;
783 		} while (*fixup_sk_storage_map);
784 	}
785 }
786 
787 static int set_admin(bool admin)
788 {
789 	cap_t caps;
790 	const cap_value_t cap_val = CAP_SYS_ADMIN;
791 	int ret = -1;
792 
793 	caps = cap_get_proc();
794 	if (!caps) {
795 		perror("cap_get_proc");
796 		return -1;
797 	}
798 	if (cap_set_flag(caps, CAP_EFFECTIVE, 1, &cap_val,
799 				admin ? CAP_SET : CAP_CLEAR)) {
800 		perror("cap_set_flag");
801 		goto out;
802 	}
803 	if (cap_set_proc(caps)) {
804 		perror("cap_set_proc");
805 		goto out;
806 	}
807 	ret = 0;
808 out:
809 	if (cap_free(caps))
810 		perror("cap_free");
811 	return ret;
812 }
813 
814 static int do_prog_test_run(int fd_prog, bool unpriv, uint32_t expected_val,
815 			    void *data, size_t size_data)
816 {
817 	__u8 tmp[TEST_DATA_LEN << 2];
818 	__u32 size_tmp = sizeof(tmp);
819 	uint32_t retval;
820 	int err;
821 
822 	if (unpriv)
823 		set_admin(true);
824 	err = bpf_prog_test_run(fd_prog, 1, data, size_data,
825 				tmp, &size_tmp, &retval, NULL);
826 	if (unpriv)
827 		set_admin(false);
828 	if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) {
829 		printf("Unexpected bpf_prog_test_run error ");
830 		return err;
831 	}
832 	if (!err && retval != expected_val &&
833 	    expected_val != POINTER_VALUE) {
834 		printf("FAIL retval %d != %d ", retval, expected_val);
835 		return 1;
836 	}
837 
838 	return 0;
839 }
840 
841 static void do_test_single(struct bpf_test *test, bool unpriv,
842 			   int *passes, int *errors)
843 {
844 	int fd_prog, expected_ret, alignment_prevented_execution;
845 	int prog_len, prog_type = test->prog_type;
846 	struct bpf_insn *prog = test->insns;
847 	int run_errs, run_successes;
848 	int map_fds[MAX_NR_MAPS];
849 	const char *expected_err;
850 	int fixup_skips;
851 	__u32 pflags;
852 	int i, err;
853 
854 	for (i = 0; i < MAX_NR_MAPS; i++)
855 		map_fds[i] = -1;
856 
857 	if (!prog_type)
858 		prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
859 	fixup_skips = skips;
860 	do_test_fixup(test, prog_type, prog, map_fds);
861 	if (test->fill_insns) {
862 		prog = test->fill_insns;
863 		prog_len = test->prog_len;
864 	} else {
865 		prog_len = probe_filter_length(prog);
866 	}
867 	/* If there were some map skips during fixup due to missing bpf
868 	 * features, skip this test.
869 	 */
870 	if (fixup_skips != skips)
871 		return;
872 
873 	pflags = 0;
874 	if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT)
875 		pflags |= BPF_F_STRICT_ALIGNMENT;
876 	if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)
877 		pflags |= BPF_F_ANY_ALIGNMENT;
878 	fd_prog = bpf_verify_program(prog_type, prog, prog_len, pflags,
879 				     "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 4);
880 	if (fd_prog < 0 && !bpf_probe_prog_type(prog_type, 0)) {
881 		printf("SKIP (unsupported program type %d)\n", prog_type);
882 		skips++;
883 		goto close_fds;
884 	}
885 
886 	expected_ret = unpriv && test->result_unpriv != UNDEF ?
887 		       test->result_unpriv : test->result;
888 	expected_err = unpriv && test->errstr_unpriv ?
889 		       test->errstr_unpriv : test->errstr;
890 
891 	alignment_prevented_execution = 0;
892 
893 	if (expected_ret == ACCEPT) {
894 		if (fd_prog < 0) {
895 			printf("FAIL\nFailed to load prog '%s'!\n",
896 			       strerror(errno));
897 			goto fail_log;
898 		}
899 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
900 		if (fd_prog >= 0 &&
901 		    (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS))
902 			alignment_prevented_execution = 1;
903 #endif
904 	} else {
905 		if (fd_prog >= 0) {
906 			printf("FAIL\nUnexpected success to load!\n");
907 			goto fail_log;
908 		}
909 		if (!strstr(bpf_vlog, expected_err)) {
910 			printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",
911 			      expected_err, bpf_vlog);
912 			goto fail_log;
913 		}
914 	}
915 
916 	if (test->insn_processed) {
917 		uint32_t insn_processed;
918 		char *proc;
919 
920 		proc = strstr(bpf_vlog, "processed ");
921 		insn_processed = atoi(proc + 10);
922 		if (test->insn_processed != insn_processed) {
923 			printf("FAIL\nUnexpected insn_processed %u vs %u\n",
924 			       insn_processed, test->insn_processed);
925 			goto fail_log;
926 		}
927 	}
928 
929 	run_errs = 0;
930 	run_successes = 0;
931 	if (!alignment_prevented_execution && fd_prog >= 0) {
932 		uint32_t expected_val;
933 		int i;
934 
935 		if (!test->runs) {
936 			expected_val = unpriv && test->retval_unpriv ?
937 				test->retval_unpriv : test->retval;
938 
939 			err = do_prog_test_run(fd_prog, unpriv, expected_val,
940 					       test->data, sizeof(test->data));
941 			if (err)
942 				run_errs++;
943 			else
944 				run_successes++;
945 		}
946 
947 		for (i = 0; i < test->runs; i++) {
948 			if (unpriv && test->retvals[i].retval_unpriv)
949 				expected_val = test->retvals[i].retval_unpriv;
950 			else
951 				expected_val = test->retvals[i].retval;
952 
953 			err = do_prog_test_run(fd_prog, unpriv, expected_val,
954 					       test->retvals[i].data,
955 					       sizeof(test->retvals[i].data));
956 			if (err) {
957 				printf("(run %d/%d) ", i + 1, test->runs);
958 				run_errs++;
959 			} else {
960 				run_successes++;
961 			}
962 		}
963 	}
964 
965 	if (!run_errs) {
966 		(*passes)++;
967 		if (run_successes > 1)
968 			printf("%d cases ", run_successes);
969 		printf("OK");
970 		if (alignment_prevented_execution)
971 			printf(" (NOTE: not executed due to unknown alignment)");
972 		printf("\n");
973 	} else {
974 		printf("\n");
975 		goto fail_log;
976 	}
977 close_fds:
978 	if (test->fill_insns)
979 		free(test->fill_insns);
980 	close(fd_prog);
981 	for (i = 0; i < MAX_NR_MAPS; i++)
982 		close(map_fds[i]);
983 	sched_yield();
984 	return;
985 fail_log:
986 	(*errors)++;
987 	printf("%s", bpf_vlog);
988 	goto close_fds;
989 }
990 
991 static bool is_admin(void)
992 {
993 	cap_t caps;
994 	cap_flag_value_t sysadmin = CAP_CLEAR;
995 	const cap_value_t cap_val = CAP_SYS_ADMIN;
996 
997 #ifdef CAP_IS_SUPPORTED
998 	if (!CAP_IS_SUPPORTED(CAP_SETFCAP)) {
999 		perror("cap_get_flag");
1000 		return false;
1001 	}
1002 #endif
1003 	caps = cap_get_proc();
1004 	if (!caps) {
1005 		perror("cap_get_proc");
1006 		return false;
1007 	}
1008 	if (cap_get_flag(caps, cap_val, CAP_EFFECTIVE, &sysadmin))
1009 		perror("cap_get_flag");
1010 	if (cap_free(caps))
1011 		perror("cap_free");
1012 	return (sysadmin == CAP_SET);
1013 }
1014 
1015 static void get_unpriv_disabled()
1016 {
1017 	char buf[2];
1018 	FILE *fd;
1019 
1020 	fd = fopen("/proc/sys/"UNPRIV_SYSCTL, "r");
1021 	if (!fd) {
1022 		perror("fopen /proc/sys/"UNPRIV_SYSCTL);
1023 		unpriv_disabled = true;
1024 		return;
1025 	}
1026 	if (fgets(buf, 2, fd) == buf && atoi(buf))
1027 		unpriv_disabled = true;
1028 	fclose(fd);
1029 }
1030 
1031 static bool test_as_unpriv(struct bpf_test *test)
1032 {
1033 	return !test->prog_type ||
1034 	       test->prog_type == BPF_PROG_TYPE_SOCKET_FILTER ||
1035 	       test->prog_type == BPF_PROG_TYPE_CGROUP_SKB;
1036 }
1037 
1038 static int do_test(bool unpriv, unsigned int from, unsigned int to)
1039 {
1040 	int i, passes = 0, errors = 0;
1041 
1042 	for (i = from; i < to; i++) {
1043 		struct bpf_test *test = &tests[i];
1044 
1045 		/* Program types that are not supported by non-root we
1046 		 * skip right away.
1047 		 */
1048 		if (test_as_unpriv(test) && unpriv_disabled) {
1049 			printf("#%d/u %s SKIP\n", i, test->descr);
1050 			skips++;
1051 		} else if (test_as_unpriv(test)) {
1052 			if (!unpriv)
1053 				set_admin(false);
1054 			printf("#%d/u %s ", i, test->descr);
1055 			do_test_single(test, true, &passes, &errors);
1056 			if (!unpriv)
1057 				set_admin(true);
1058 		}
1059 
1060 		if (unpriv) {
1061 			printf("#%d/p %s SKIP\n", i, test->descr);
1062 			skips++;
1063 		} else {
1064 			printf("#%d/p %s ", i, test->descr);
1065 			do_test_single(test, false, &passes, &errors);
1066 		}
1067 	}
1068 
1069 	printf("Summary: %d PASSED, %d SKIPPED, %d FAILED\n", passes,
1070 	       skips, errors);
1071 	return errors ? EXIT_FAILURE : EXIT_SUCCESS;
1072 }
1073 
1074 int main(int argc, char **argv)
1075 {
1076 	unsigned int from = 0, to = ARRAY_SIZE(tests);
1077 	bool unpriv = !is_admin();
1078 
1079 	if (argc == 3) {
1080 		unsigned int l = atoi(argv[argc - 2]);
1081 		unsigned int u = atoi(argv[argc - 1]);
1082 
1083 		if (l < to && u < to) {
1084 			from = l;
1085 			to   = u + 1;
1086 		}
1087 	} else if (argc == 2) {
1088 		unsigned int t = atoi(argv[argc - 1]);
1089 
1090 		if (t < to) {
1091 			from = t;
1092 			to   = t + 1;
1093 		}
1094 	}
1095 
1096 	get_unpriv_disabled();
1097 	if (unpriv && unpriv_disabled) {
1098 		printf("Cannot run as unprivileged user with sysctl %s.\n",
1099 		       UNPRIV_SYSCTL);
1100 		return EXIT_FAILURE;
1101 	}
1102 
1103 	bpf_semi_rand_init();
1104 	return do_test(unpriv, from, to);
1105 }
1106