xref: /openbmc/linux/kernel/bpf/verifier.c (revision 22c03398)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5  */
6 #include <uapi/linux/btf.h>
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/slab.h>
10 #include <linux/bpf.h>
11 #include <linux/btf.h>
12 #include <linux/bpf_verifier.h>
13 #include <linux/filter.h>
14 #include <net/netlink.h>
15 #include <linux/file.h>
16 #include <linux/vmalloc.h>
17 #include <linux/stringify.h>
18 #include <linux/bsearch.h>
19 #include <linux/sort.h>
20 #include <linux/perf_event.h>
21 #include <linux/ctype.h>
22 #include <linux/error-injection.h>
23 #include <linux/bpf_lsm.h>
24 #include <linux/btf_ids.h>
25 
26 #include "disasm.h"
27 
28 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
29 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
30 	[_id] = & _name ## _verifier_ops,
31 #define BPF_MAP_TYPE(_id, _ops)
32 #define BPF_LINK_TYPE(_id, _name)
33 #include <linux/bpf_types.h>
34 #undef BPF_PROG_TYPE
35 #undef BPF_MAP_TYPE
36 #undef BPF_LINK_TYPE
37 };
38 
39 /* bpf_check() is a static code analyzer that walks eBPF program
40  * instruction by instruction and updates register/stack state.
41  * All paths of conditional branches are analyzed until 'bpf_exit' insn.
42  *
43  * The first pass is depth-first-search to check that the program is a DAG.
44  * It rejects the following programs:
45  * - larger than BPF_MAXINSNS insns
46  * - if loop is present (detected via back-edge)
47  * - unreachable insns exist (shouldn't be a forest. program = one function)
48  * - out of bounds or malformed jumps
49  * The second pass is all possible path descent from the 1st insn.
50  * Since it's analyzing all paths through the program, the length of the
51  * analysis is limited to 64k insn, which may be hit even if total number of
52  * insn is less then 4K, but there are too many branches that change stack/regs.
53  * Number of 'branches to be analyzed' is limited to 1k
54  *
55  * On entry to each instruction, each register has a type, and the instruction
56  * changes the types of the registers depending on instruction semantics.
57  * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
58  * copied to R1.
59  *
60  * All registers are 64-bit.
61  * R0 - return register
62  * R1-R5 argument passing registers
63  * R6-R9 callee saved registers
64  * R10 - frame pointer read-only
65  *
66  * At the start of BPF program the register R1 contains a pointer to bpf_context
67  * and has type PTR_TO_CTX.
68  *
69  * Verifier tracks arithmetic operations on pointers in case:
70  *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
71  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
72  * 1st insn copies R10 (which has FRAME_PTR) type into R1
73  * and 2nd arithmetic instruction is pattern matched to recognize
74  * that it wants to construct a pointer to some element within stack.
75  * So after 2nd insn, the register R1 has type PTR_TO_STACK
76  * (and -20 constant is saved for further stack bounds checking).
77  * Meaning that this reg is a pointer to stack plus known immediate constant.
78  *
79  * Most of the time the registers have SCALAR_VALUE type, which
80  * means the register has some value, but it's not a valid pointer.
81  * (like pointer plus pointer becomes SCALAR_VALUE type)
82  *
83  * When verifier sees load or store instructions the type of base register
84  * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
85  * four pointer types recognized by check_mem_access() function.
86  *
87  * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
88  * and the range of [ptr, ptr + map's value_size) is accessible.
89  *
90  * registers used to pass values to function calls are checked against
91  * function argument constraints.
92  *
93  * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
94  * It means that the register type passed to this function must be
95  * PTR_TO_STACK and it will be used inside the function as
96  * 'pointer to map element key'
97  *
98  * For example the argument constraints for bpf_map_lookup_elem():
99  *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
100  *   .arg1_type = ARG_CONST_MAP_PTR,
101  *   .arg2_type = ARG_PTR_TO_MAP_KEY,
102  *
103  * ret_type says that this function returns 'pointer to map elem value or null'
104  * function expects 1st argument to be a const pointer to 'struct bpf_map' and
105  * 2nd argument should be a pointer to stack, which will be used inside
106  * the helper function as a pointer to map element key.
107  *
108  * On the kernel side the helper function looks like:
109  * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
110  * {
111  *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
112  *    void *key = (void *) (unsigned long) r2;
113  *    void *value;
114  *
115  *    here kernel can access 'key' and 'map' pointers safely, knowing that
116  *    [key, key + map->key_size) bytes are valid and were initialized on
117  *    the stack of eBPF program.
118  * }
119  *
120  * Corresponding eBPF program may look like:
121  *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
122  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
123  *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
124  *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
125  * here verifier looks at prototype of map_lookup_elem() and sees:
126  * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
127  * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
128  *
129  * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
130  * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
131  * and were initialized prior to this call.
132  * If it's ok, then verifier allows this BPF_CALL insn and looks at
133  * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
134  * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
135  * returns either pointer to map value or NULL.
136  *
137  * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
138  * insn, the register holding that pointer in the true branch changes state to
139  * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
140  * branch. See check_cond_jmp_op().
141  *
142  * After the call R0 is set to return type of the function and registers R1-R5
143  * are set to NOT_INIT to indicate that they are no longer readable.
144  *
145  * The following reference types represent a potential reference to a kernel
146  * resource which, after first being allocated, must be checked and freed by
147  * the BPF program:
148  * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
149  *
150  * When the verifier sees a helper call return a reference type, it allocates a
151  * pointer id for the reference and stores it in the current function state.
152  * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
153  * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
154  * passes through a NULL-check conditional. For the branch wherein the state is
155  * changed to CONST_IMM, the verifier releases the reference.
156  *
157  * For each helper function that allocates a reference, such as
158  * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
159  * bpf_sk_release(). When a reference type passes into the release function,
160  * the verifier also releases the reference. If any unchecked or unreleased
161  * reference remains at the end of the program, the verifier rejects it.
162  */
163 
164 /* verifier_state + insn_idx are pushed to stack when branch is encountered */
165 struct bpf_verifier_stack_elem {
166 	/* verifer state is 'st'
167 	 * before processing instruction 'insn_idx'
168 	 * and after processing instruction 'prev_insn_idx'
169 	 */
170 	struct bpf_verifier_state st;
171 	int insn_idx;
172 	int prev_insn_idx;
173 	struct bpf_verifier_stack_elem *next;
174 	/* length of verifier log at the time this state was pushed on stack */
175 	u32 log_pos;
176 };
177 
178 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ	8192
179 #define BPF_COMPLEXITY_LIMIT_STATES	64
180 
181 #define BPF_MAP_KEY_POISON	(1ULL << 63)
182 #define BPF_MAP_KEY_SEEN	(1ULL << 62)
183 
184 #define BPF_MAP_PTR_UNPRIV	1UL
185 #define BPF_MAP_PTR_POISON	((void *)((0xeB9FUL << 1) +	\
186 					  POISON_POINTER_DELTA))
187 #define BPF_MAP_PTR(X)		((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
188 
189 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
190 {
191 	return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON;
192 }
193 
194 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
195 {
196 	return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV;
197 }
198 
199 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
200 			      const struct bpf_map *map, bool unpriv)
201 {
202 	BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
203 	unpriv |= bpf_map_ptr_unpriv(aux);
204 	aux->map_ptr_state = (unsigned long)map |
205 			     (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
206 }
207 
208 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
209 {
210 	return aux->map_key_state & BPF_MAP_KEY_POISON;
211 }
212 
213 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
214 {
215 	return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
216 }
217 
218 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
219 {
220 	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
221 }
222 
223 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
224 {
225 	bool poisoned = bpf_map_key_poisoned(aux);
226 
227 	aux->map_key_state = state | BPF_MAP_KEY_SEEN |
228 			     (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229 }
230 
231 static bool bpf_pseudo_call(const struct bpf_insn *insn)
232 {
233 	return insn->code == (BPF_JMP | BPF_CALL) &&
234 	       insn->src_reg == BPF_PSEUDO_CALL;
235 }
236 
237 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
238 {
239 	return insn->code == (BPF_JMP | BPF_CALL) &&
240 	       insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
241 }
242 
243 struct bpf_call_arg_meta {
244 	struct bpf_map *map_ptr;
245 	bool raw_mode;
246 	bool pkt_access;
247 	int regno;
248 	int access_size;
249 	int mem_size;
250 	u64 msize_max_value;
251 	int ref_obj_id;
252 	int map_uid;
253 	int func_id;
254 	struct btf *btf;
255 	u32 btf_id;
256 	struct btf *ret_btf;
257 	u32 ret_btf_id;
258 	u32 subprogno;
259 };
260 
261 struct btf *btf_vmlinux;
262 
263 static DEFINE_MUTEX(bpf_verifier_lock);
264 
265 static const struct bpf_line_info *
266 find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
267 {
268 	const struct bpf_line_info *linfo;
269 	const struct bpf_prog *prog;
270 	u32 i, nr_linfo;
271 
272 	prog = env->prog;
273 	nr_linfo = prog->aux->nr_linfo;
274 
275 	if (!nr_linfo || insn_off >= prog->len)
276 		return NULL;
277 
278 	linfo = prog->aux->linfo;
279 	for (i = 1; i < nr_linfo; i++)
280 		if (insn_off < linfo[i].insn_off)
281 			break;
282 
283 	return &linfo[i - 1];
284 }
285 
286 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
287 		       va_list args)
288 {
289 	unsigned int n;
290 
291 	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
292 
293 	WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
294 		  "verifier log line truncated - local buffer too short\n");
295 
296 	n = min(log->len_total - log->len_used - 1, n);
297 	log->kbuf[n] = '\0';
298 
299 	if (log->level == BPF_LOG_KERNEL) {
300 		pr_err("BPF:%s\n", log->kbuf);
301 		return;
302 	}
303 	if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
304 		log->len_used += n;
305 	else
306 		log->ubuf = NULL;
307 }
308 
309 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos)
310 {
311 	char zero = 0;
312 
313 	if (!bpf_verifier_log_needed(log))
314 		return;
315 
316 	log->len_used = new_pos;
317 	if (put_user(zero, log->ubuf + new_pos))
318 		log->ubuf = NULL;
319 }
320 
321 /* log_level controls verbosity level of eBPF verifier.
322  * bpf_verifier_log_write() is used to dump the verification trace to the log,
323  * so the user can figure out what's wrong with the program
324  */
325 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
326 					   const char *fmt, ...)
327 {
328 	va_list args;
329 
330 	if (!bpf_verifier_log_needed(&env->log))
331 		return;
332 
333 	va_start(args, fmt);
334 	bpf_verifier_vlog(&env->log, fmt, args);
335 	va_end(args);
336 }
337 EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
338 
339 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
340 {
341 	struct bpf_verifier_env *env = private_data;
342 	va_list args;
343 
344 	if (!bpf_verifier_log_needed(&env->log))
345 		return;
346 
347 	va_start(args, fmt);
348 	bpf_verifier_vlog(&env->log, fmt, args);
349 	va_end(args);
350 }
351 
352 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
353 			    const char *fmt, ...)
354 {
355 	va_list args;
356 
357 	if (!bpf_verifier_log_needed(log))
358 		return;
359 
360 	va_start(args, fmt);
361 	bpf_verifier_vlog(log, fmt, args);
362 	va_end(args);
363 }
364 
365 static const char *ltrim(const char *s)
366 {
367 	while (isspace(*s))
368 		s++;
369 
370 	return s;
371 }
372 
373 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
374 					 u32 insn_off,
375 					 const char *prefix_fmt, ...)
376 {
377 	const struct bpf_line_info *linfo;
378 
379 	if (!bpf_verifier_log_needed(&env->log))
380 		return;
381 
382 	linfo = find_linfo(env, insn_off);
383 	if (!linfo || linfo == env->prev_linfo)
384 		return;
385 
386 	if (prefix_fmt) {
387 		va_list args;
388 
389 		va_start(args, prefix_fmt);
390 		bpf_verifier_vlog(&env->log, prefix_fmt, args);
391 		va_end(args);
392 	}
393 
394 	verbose(env, "%s\n",
395 		ltrim(btf_name_by_offset(env->prog->aux->btf,
396 					 linfo->line_off)));
397 
398 	env->prev_linfo = linfo;
399 }
400 
401 static void verbose_invalid_scalar(struct bpf_verifier_env *env,
402 				   struct bpf_reg_state *reg,
403 				   struct tnum *range, const char *ctx,
404 				   const char *reg_name)
405 {
406 	char tn_buf[48];
407 
408 	verbose(env, "At %s the register %s ", ctx, reg_name);
409 	if (!tnum_is_unknown(reg->var_off)) {
410 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
411 		verbose(env, "has value %s", tn_buf);
412 	} else {
413 		verbose(env, "has unknown scalar value");
414 	}
415 	tnum_strn(tn_buf, sizeof(tn_buf), *range);
416 	verbose(env, " should have been in %s\n", tn_buf);
417 }
418 
419 static bool type_is_pkt_pointer(enum bpf_reg_type type)
420 {
421 	return type == PTR_TO_PACKET ||
422 	       type == PTR_TO_PACKET_META;
423 }
424 
425 static bool type_is_sk_pointer(enum bpf_reg_type type)
426 {
427 	return type == PTR_TO_SOCKET ||
428 		type == PTR_TO_SOCK_COMMON ||
429 		type == PTR_TO_TCP_SOCK ||
430 		type == PTR_TO_XDP_SOCK;
431 }
432 
433 static bool reg_type_not_null(enum bpf_reg_type type)
434 {
435 	return type == PTR_TO_SOCKET ||
436 		type == PTR_TO_TCP_SOCK ||
437 		type == PTR_TO_MAP_VALUE ||
438 		type == PTR_TO_MAP_KEY ||
439 		type == PTR_TO_SOCK_COMMON;
440 }
441 
442 static bool reg_type_may_be_null(enum bpf_reg_type type)
443 {
444 	return type == PTR_TO_MAP_VALUE_OR_NULL ||
445 	       type == PTR_TO_SOCKET_OR_NULL ||
446 	       type == PTR_TO_SOCK_COMMON_OR_NULL ||
447 	       type == PTR_TO_TCP_SOCK_OR_NULL ||
448 	       type == PTR_TO_BTF_ID_OR_NULL ||
449 	       type == PTR_TO_MEM_OR_NULL ||
450 	       type == PTR_TO_RDONLY_BUF_OR_NULL ||
451 	       type == PTR_TO_RDWR_BUF_OR_NULL;
452 }
453 
454 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
455 {
456 	return reg->type == PTR_TO_MAP_VALUE &&
457 		map_value_has_spin_lock(reg->map_ptr);
458 }
459 
460 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
461 {
462 	return type == PTR_TO_SOCKET ||
463 		type == PTR_TO_SOCKET_OR_NULL ||
464 		type == PTR_TO_TCP_SOCK ||
465 		type == PTR_TO_TCP_SOCK_OR_NULL ||
466 		type == PTR_TO_MEM ||
467 		type == PTR_TO_MEM_OR_NULL;
468 }
469 
470 static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
471 {
472 	return type == ARG_PTR_TO_SOCK_COMMON;
473 }
474 
475 static bool arg_type_may_be_null(enum bpf_arg_type type)
476 {
477 	return type == ARG_PTR_TO_MAP_VALUE_OR_NULL ||
478 	       type == ARG_PTR_TO_MEM_OR_NULL ||
479 	       type == ARG_PTR_TO_CTX_OR_NULL ||
480 	       type == ARG_PTR_TO_SOCKET_OR_NULL ||
481 	       type == ARG_PTR_TO_ALLOC_MEM_OR_NULL ||
482 	       type == ARG_PTR_TO_STACK_OR_NULL;
483 }
484 
485 /* Determine whether the function releases some resources allocated by another
486  * function call. The first reference type argument will be assumed to be
487  * released by release_reference().
488  */
489 static bool is_release_function(enum bpf_func_id func_id)
490 {
491 	return func_id == BPF_FUNC_sk_release ||
492 	       func_id == BPF_FUNC_ringbuf_submit ||
493 	       func_id == BPF_FUNC_ringbuf_discard;
494 }
495 
496 static bool may_be_acquire_function(enum bpf_func_id func_id)
497 {
498 	return func_id == BPF_FUNC_sk_lookup_tcp ||
499 		func_id == BPF_FUNC_sk_lookup_udp ||
500 		func_id == BPF_FUNC_skc_lookup_tcp ||
501 		func_id == BPF_FUNC_map_lookup_elem ||
502 	        func_id == BPF_FUNC_ringbuf_reserve;
503 }
504 
505 static bool is_acquire_function(enum bpf_func_id func_id,
506 				const struct bpf_map *map)
507 {
508 	enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
509 
510 	if (func_id == BPF_FUNC_sk_lookup_tcp ||
511 	    func_id == BPF_FUNC_sk_lookup_udp ||
512 	    func_id == BPF_FUNC_skc_lookup_tcp ||
513 	    func_id == BPF_FUNC_ringbuf_reserve)
514 		return true;
515 
516 	if (func_id == BPF_FUNC_map_lookup_elem &&
517 	    (map_type == BPF_MAP_TYPE_SOCKMAP ||
518 	     map_type == BPF_MAP_TYPE_SOCKHASH))
519 		return true;
520 
521 	return false;
522 }
523 
524 static bool is_ptr_cast_function(enum bpf_func_id func_id)
525 {
526 	return func_id == BPF_FUNC_tcp_sock ||
527 		func_id == BPF_FUNC_sk_fullsock ||
528 		func_id == BPF_FUNC_skc_to_tcp_sock ||
529 		func_id == BPF_FUNC_skc_to_tcp6_sock ||
530 		func_id == BPF_FUNC_skc_to_udp6_sock ||
531 		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
532 		func_id == BPF_FUNC_skc_to_tcp_request_sock;
533 }
534 
535 static bool is_cmpxchg_insn(const struct bpf_insn *insn)
536 {
537 	return BPF_CLASS(insn->code) == BPF_STX &&
538 	       BPF_MODE(insn->code) == BPF_ATOMIC &&
539 	       insn->imm == BPF_CMPXCHG;
540 }
541 
542 /* string representation of 'enum bpf_reg_type' */
543 static const char * const reg_type_str[] = {
544 	[NOT_INIT]		= "?",
545 	[SCALAR_VALUE]		= "inv",
546 	[PTR_TO_CTX]		= "ctx",
547 	[CONST_PTR_TO_MAP]	= "map_ptr",
548 	[PTR_TO_MAP_VALUE]	= "map_value",
549 	[PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
550 	[PTR_TO_STACK]		= "fp",
551 	[PTR_TO_PACKET]		= "pkt",
552 	[PTR_TO_PACKET_META]	= "pkt_meta",
553 	[PTR_TO_PACKET_END]	= "pkt_end",
554 	[PTR_TO_FLOW_KEYS]	= "flow_keys",
555 	[PTR_TO_SOCKET]		= "sock",
556 	[PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
557 	[PTR_TO_SOCK_COMMON]	= "sock_common",
558 	[PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
559 	[PTR_TO_TCP_SOCK]	= "tcp_sock",
560 	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
561 	[PTR_TO_TP_BUFFER]	= "tp_buffer",
562 	[PTR_TO_XDP_SOCK]	= "xdp_sock",
563 	[PTR_TO_BTF_ID]		= "ptr_",
564 	[PTR_TO_BTF_ID_OR_NULL]	= "ptr_or_null_",
565 	[PTR_TO_PERCPU_BTF_ID]	= "percpu_ptr_",
566 	[PTR_TO_MEM]		= "mem",
567 	[PTR_TO_MEM_OR_NULL]	= "mem_or_null",
568 	[PTR_TO_RDONLY_BUF]	= "rdonly_buf",
569 	[PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",
570 	[PTR_TO_RDWR_BUF]	= "rdwr_buf",
571 	[PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",
572 	[PTR_TO_FUNC]		= "func",
573 	[PTR_TO_MAP_KEY]	= "map_key",
574 };
575 
576 static char slot_type_char[] = {
577 	[STACK_INVALID]	= '?',
578 	[STACK_SPILL]	= 'r',
579 	[STACK_MISC]	= 'm',
580 	[STACK_ZERO]	= '0',
581 };
582 
583 static void print_liveness(struct bpf_verifier_env *env,
584 			   enum bpf_reg_liveness live)
585 {
586 	if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
587 	    verbose(env, "_");
588 	if (live & REG_LIVE_READ)
589 		verbose(env, "r");
590 	if (live & REG_LIVE_WRITTEN)
591 		verbose(env, "w");
592 	if (live & REG_LIVE_DONE)
593 		verbose(env, "D");
594 }
595 
596 static struct bpf_func_state *func(struct bpf_verifier_env *env,
597 				   const struct bpf_reg_state *reg)
598 {
599 	struct bpf_verifier_state *cur = env->cur_state;
600 
601 	return cur->frame[reg->frameno];
602 }
603 
604 static const char *kernel_type_name(const struct btf* btf, u32 id)
605 {
606 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
607 }
608 
609 /* The reg state of a pointer or a bounded scalar was saved when
610  * it was spilled to the stack.
611  */
612 static bool is_spilled_reg(const struct bpf_stack_state *stack)
613 {
614 	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
615 }
616 
617 static void scrub_spilled_slot(u8 *stype)
618 {
619 	if (*stype != STACK_INVALID)
620 		*stype = STACK_MISC;
621 }
622 
623 static void print_verifier_state(struct bpf_verifier_env *env,
624 				 const struct bpf_func_state *state)
625 {
626 	const struct bpf_reg_state *reg;
627 	enum bpf_reg_type t;
628 	int i;
629 
630 	if (state->frameno)
631 		verbose(env, " frame%d:", state->frameno);
632 	for (i = 0; i < MAX_BPF_REG; i++) {
633 		reg = &state->regs[i];
634 		t = reg->type;
635 		if (t == NOT_INIT)
636 			continue;
637 		verbose(env, " R%d", i);
638 		print_liveness(env, reg->live);
639 		verbose(env, "=%s", reg_type_str[t]);
640 		if (t == SCALAR_VALUE && reg->precise)
641 			verbose(env, "P");
642 		if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
643 		    tnum_is_const(reg->var_off)) {
644 			/* reg->off should be 0 for SCALAR_VALUE */
645 			verbose(env, "%lld", reg->var_off.value + reg->off);
646 		} else {
647 			if (t == PTR_TO_BTF_ID ||
648 			    t == PTR_TO_BTF_ID_OR_NULL ||
649 			    t == PTR_TO_PERCPU_BTF_ID)
650 				verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
651 			verbose(env, "(id=%d", reg->id);
652 			if (reg_type_may_be_refcounted_or_null(t))
653 				verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
654 			if (t != SCALAR_VALUE)
655 				verbose(env, ",off=%d", reg->off);
656 			if (type_is_pkt_pointer(t))
657 				verbose(env, ",r=%d", reg->range);
658 			else if (t == CONST_PTR_TO_MAP ||
659 				 t == PTR_TO_MAP_KEY ||
660 				 t == PTR_TO_MAP_VALUE ||
661 				 t == PTR_TO_MAP_VALUE_OR_NULL)
662 				verbose(env, ",ks=%d,vs=%d",
663 					reg->map_ptr->key_size,
664 					reg->map_ptr->value_size);
665 			if (tnum_is_const(reg->var_off)) {
666 				/* Typically an immediate SCALAR_VALUE, but
667 				 * could be a pointer whose offset is too big
668 				 * for reg->off
669 				 */
670 				verbose(env, ",imm=%llx", reg->var_off.value);
671 			} else {
672 				if (reg->smin_value != reg->umin_value &&
673 				    reg->smin_value != S64_MIN)
674 					verbose(env, ",smin_value=%lld",
675 						(long long)reg->smin_value);
676 				if (reg->smax_value != reg->umax_value &&
677 				    reg->smax_value != S64_MAX)
678 					verbose(env, ",smax_value=%lld",
679 						(long long)reg->smax_value);
680 				if (reg->umin_value != 0)
681 					verbose(env, ",umin_value=%llu",
682 						(unsigned long long)reg->umin_value);
683 				if (reg->umax_value != U64_MAX)
684 					verbose(env, ",umax_value=%llu",
685 						(unsigned long long)reg->umax_value);
686 				if (!tnum_is_unknown(reg->var_off)) {
687 					char tn_buf[48];
688 
689 					tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
690 					verbose(env, ",var_off=%s", tn_buf);
691 				}
692 				if (reg->s32_min_value != reg->smin_value &&
693 				    reg->s32_min_value != S32_MIN)
694 					verbose(env, ",s32_min_value=%d",
695 						(int)(reg->s32_min_value));
696 				if (reg->s32_max_value != reg->smax_value &&
697 				    reg->s32_max_value != S32_MAX)
698 					verbose(env, ",s32_max_value=%d",
699 						(int)(reg->s32_max_value));
700 				if (reg->u32_min_value != reg->umin_value &&
701 				    reg->u32_min_value != U32_MIN)
702 					verbose(env, ",u32_min_value=%d",
703 						(int)(reg->u32_min_value));
704 				if (reg->u32_max_value != reg->umax_value &&
705 				    reg->u32_max_value != U32_MAX)
706 					verbose(env, ",u32_max_value=%d",
707 						(int)(reg->u32_max_value));
708 			}
709 			verbose(env, ")");
710 		}
711 	}
712 	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
713 		char types_buf[BPF_REG_SIZE + 1];
714 		bool valid = false;
715 		int j;
716 
717 		for (j = 0; j < BPF_REG_SIZE; j++) {
718 			if (state->stack[i].slot_type[j] != STACK_INVALID)
719 				valid = true;
720 			types_buf[j] = slot_type_char[
721 					state->stack[i].slot_type[j]];
722 		}
723 		types_buf[BPF_REG_SIZE] = 0;
724 		if (!valid)
725 			continue;
726 		verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
727 		print_liveness(env, state->stack[i].spilled_ptr.live);
728 		if (is_spilled_reg(&state->stack[i])) {
729 			reg = &state->stack[i].spilled_ptr;
730 			t = reg->type;
731 			verbose(env, "=%s", reg_type_str[t]);
732 			if (t == SCALAR_VALUE && reg->precise)
733 				verbose(env, "P");
734 			if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
735 				verbose(env, "%lld", reg->var_off.value + reg->off);
736 		} else {
737 			verbose(env, "=%s", types_buf);
738 		}
739 	}
740 	if (state->acquired_refs && state->refs[0].id) {
741 		verbose(env, " refs=%d", state->refs[0].id);
742 		for (i = 1; i < state->acquired_refs; i++)
743 			if (state->refs[i].id)
744 				verbose(env, ",%d", state->refs[i].id);
745 	}
746 	if (state->in_callback_fn)
747 		verbose(env, " cb");
748 	if (state->in_async_callback_fn)
749 		verbose(env, " async_cb");
750 	verbose(env, "\n");
751 }
752 
753 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
754  * small to hold src. This is different from krealloc since we don't want to preserve
755  * the contents of dst.
756  *
757  * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
758  * not be allocated.
759  */
760 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
761 {
762 	size_t bytes;
763 
764 	if (ZERO_OR_NULL_PTR(src))
765 		goto out;
766 
767 	if (unlikely(check_mul_overflow(n, size, &bytes)))
768 		return NULL;
769 
770 	if (ksize(dst) < bytes) {
771 		kfree(dst);
772 		dst = kmalloc_track_caller(bytes, flags);
773 		if (!dst)
774 			return NULL;
775 	}
776 
777 	memcpy(dst, src, bytes);
778 out:
779 	return dst ? dst : ZERO_SIZE_PTR;
780 }
781 
782 /* resize an array from old_n items to new_n items. the array is reallocated if it's too
783  * small to hold new_n items. new items are zeroed out if the array grows.
784  *
785  * Contrary to krealloc_array, does not free arr if new_n is zero.
786  */
787 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
788 {
789 	if (!new_n || old_n == new_n)
790 		goto out;
791 
792 	arr = krealloc_array(arr, new_n, size, GFP_KERNEL);
793 	if (!arr)
794 		return NULL;
795 
796 	if (new_n > old_n)
797 		memset(arr + old_n * size, 0, (new_n - old_n) * size);
798 
799 out:
800 	return arr ? arr : ZERO_SIZE_PTR;
801 }
802 
803 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
804 {
805 	dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
806 			       sizeof(struct bpf_reference_state), GFP_KERNEL);
807 	if (!dst->refs)
808 		return -ENOMEM;
809 
810 	dst->acquired_refs = src->acquired_refs;
811 	return 0;
812 }
813 
814 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
815 {
816 	size_t n = src->allocated_stack / BPF_REG_SIZE;
817 
818 	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
819 				GFP_KERNEL);
820 	if (!dst->stack)
821 		return -ENOMEM;
822 
823 	dst->allocated_stack = src->allocated_stack;
824 	return 0;
825 }
826 
827 static int resize_reference_state(struct bpf_func_state *state, size_t n)
828 {
829 	state->refs = realloc_array(state->refs, state->acquired_refs, n,
830 				    sizeof(struct bpf_reference_state));
831 	if (!state->refs)
832 		return -ENOMEM;
833 
834 	state->acquired_refs = n;
835 	return 0;
836 }
837 
838 static int grow_stack_state(struct bpf_func_state *state, int size)
839 {
840 	size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE;
841 
842 	if (old_n >= n)
843 		return 0;
844 
845 	state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
846 	if (!state->stack)
847 		return -ENOMEM;
848 
849 	state->allocated_stack = size;
850 	return 0;
851 }
852 
853 /* Acquire a pointer id from the env and update the state->refs to include
854  * this new pointer reference.
855  * On success, returns a valid pointer id to associate with the register
856  * On failure, returns a negative errno.
857  */
858 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
859 {
860 	struct bpf_func_state *state = cur_func(env);
861 	int new_ofs = state->acquired_refs;
862 	int id, err;
863 
864 	err = resize_reference_state(state, state->acquired_refs + 1);
865 	if (err)
866 		return err;
867 	id = ++env->id_gen;
868 	state->refs[new_ofs].id = id;
869 	state->refs[new_ofs].insn_idx = insn_idx;
870 
871 	return id;
872 }
873 
874 /* release function corresponding to acquire_reference_state(). Idempotent. */
875 static int release_reference_state(struct bpf_func_state *state, int ptr_id)
876 {
877 	int i, last_idx;
878 
879 	last_idx = state->acquired_refs - 1;
880 	for (i = 0; i < state->acquired_refs; i++) {
881 		if (state->refs[i].id == ptr_id) {
882 			if (last_idx && i != last_idx)
883 				memcpy(&state->refs[i], &state->refs[last_idx],
884 				       sizeof(*state->refs));
885 			memset(&state->refs[last_idx], 0, sizeof(*state->refs));
886 			state->acquired_refs--;
887 			return 0;
888 		}
889 	}
890 	return -EINVAL;
891 }
892 
893 static void free_func_state(struct bpf_func_state *state)
894 {
895 	if (!state)
896 		return;
897 	kfree(state->refs);
898 	kfree(state->stack);
899 	kfree(state);
900 }
901 
902 static void clear_jmp_history(struct bpf_verifier_state *state)
903 {
904 	kfree(state->jmp_history);
905 	state->jmp_history = NULL;
906 	state->jmp_history_cnt = 0;
907 }
908 
909 static void free_verifier_state(struct bpf_verifier_state *state,
910 				bool free_self)
911 {
912 	int i;
913 
914 	for (i = 0; i <= state->curframe; i++) {
915 		free_func_state(state->frame[i]);
916 		state->frame[i] = NULL;
917 	}
918 	clear_jmp_history(state);
919 	if (free_self)
920 		kfree(state);
921 }
922 
923 /* copy verifier state from src to dst growing dst stack space
924  * when necessary to accommodate larger src stack
925  */
926 static int copy_func_state(struct bpf_func_state *dst,
927 			   const struct bpf_func_state *src)
928 {
929 	int err;
930 
931 	memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
932 	err = copy_reference_state(dst, src);
933 	if (err)
934 		return err;
935 	return copy_stack_state(dst, src);
936 }
937 
938 static int copy_verifier_state(struct bpf_verifier_state *dst_state,
939 			       const struct bpf_verifier_state *src)
940 {
941 	struct bpf_func_state *dst;
942 	int i, err;
943 
944 	dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
945 					    src->jmp_history_cnt, sizeof(struct bpf_idx_pair),
946 					    GFP_USER);
947 	if (!dst_state->jmp_history)
948 		return -ENOMEM;
949 	dst_state->jmp_history_cnt = src->jmp_history_cnt;
950 
951 	/* if dst has more stack frames then src frame, free them */
952 	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
953 		free_func_state(dst_state->frame[i]);
954 		dst_state->frame[i] = NULL;
955 	}
956 	dst_state->speculative = src->speculative;
957 	dst_state->curframe = src->curframe;
958 	dst_state->active_spin_lock = src->active_spin_lock;
959 	dst_state->branches = src->branches;
960 	dst_state->parent = src->parent;
961 	dst_state->first_insn_idx = src->first_insn_idx;
962 	dst_state->last_insn_idx = src->last_insn_idx;
963 	for (i = 0; i <= src->curframe; i++) {
964 		dst = dst_state->frame[i];
965 		if (!dst) {
966 			dst = kzalloc(sizeof(*dst), GFP_KERNEL);
967 			if (!dst)
968 				return -ENOMEM;
969 			dst_state->frame[i] = dst;
970 		}
971 		err = copy_func_state(dst, src->frame[i]);
972 		if (err)
973 			return err;
974 	}
975 	return 0;
976 }
977 
978 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
979 {
980 	while (st) {
981 		u32 br = --st->branches;
982 
983 		/* WARN_ON(br > 1) technically makes sense here,
984 		 * but see comment in push_stack(), hence:
985 		 */
986 		WARN_ONCE((int)br < 0,
987 			  "BUG update_branch_counts:branches_to_explore=%d\n",
988 			  br);
989 		if (br)
990 			break;
991 		st = st->parent;
992 	}
993 }
994 
995 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
996 		     int *insn_idx, bool pop_log)
997 {
998 	struct bpf_verifier_state *cur = env->cur_state;
999 	struct bpf_verifier_stack_elem *elem, *head = env->head;
1000 	int err;
1001 
1002 	if (env->head == NULL)
1003 		return -ENOENT;
1004 
1005 	if (cur) {
1006 		err = copy_verifier_state(cur, &head->st);
1007 		if (err)
1008 			return err;
1009 	}
1010 	if (pop_log)
1011 		bpf_vlog_reset(&env->log, head->log_pos);
1012 	if (insn_idx)
1013 		*insn_idx = head->insn_idx;
1014 	if (prev_insn_idx)
1015 		*prev_insn_idx = head->prev_insn_idx;
1016 	elem = head->next;
1017 	free_verifier_state(&head->st, false);
1018 	kfree(head);
1019 	env->head = elem;
1020 	env->stack_size--;
1021 	return 0;
1022 }
1023 
1024 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
1025 					     int insn_idx, int prev_insn_idx,
1026 					     bool speculative)
1027 {
1028 	struct bpf_verifier_state *cur = env->cur_state;
1029 	struct bpf_verifier_stack_elem *elem;
1030 	int err;
1031 
1032 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
1033 	if (!elem)
1034 		goto err;
1035 
1036 	elem->insn_idx = insn_idx;
1037 	elem->prev_insn_idx = prev_insn_idx;
1038 	elem->next = env->head;
1039 	elem->log_pos = env->log.len_used;
1040 	env->head = elem;
1041 	env->stack_size++;
1042 	err = copy_verifier_state(&elem->st, cur);
1043 	if (err)
1044 		goto err;
1045 	elem->st.speculative |= speculative;
1046 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
1047 		verbose(env, "The sequence of %d jumps is too complex.\n",
1048 			env->stack_size);
1049 		goto err;
1050 	}
1051 	if (elem->st.parent) {
1052 		++elem->st.parent->branches;
1053 		/* WARN_ON(branches > 2) technically makes sense here,
1054 		 * but
1055 		 * 1. speculative states will bump 'branches' for non-branch
1056 		 * instructions
1057 		 * 2. is_state_visited() heuristics may decide not to create
1058 		 * a new state for a sequence of branches and all such current
1059 		 * and cloned states will be pointing to a single parent state
1060 		 * which might have large 'branches' count.
1061 		 */
1062 	}
1063 	return &elem->st;
1064 err:
1065 	free_verifier_state(env->cur_state, true);
1066 	env->cur_state = NULL;
1067 	/* pop all elements and return */
1068 	while (!pop_stack(env, NULL, NULL, false));
1069 	return NULL;
1070 }
1071 
1072 #define CALLER_SAVED_REGS 6
1073 static const int caller_saved[CALLER_SAVED_REGS] = {
1074 	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
1075 };
1076 
1077 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1078 				struct bpf_reg_state *reg);
1079 
1080 /* This helper doesn't clear reg->id */
1081 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1082 {
1083 	reg->var_off = tnum_const(imm);
1084 	reg->smin_value = (s64)imm;
1085 	reg->smax_value = (s64)imm;
1086 	reg->umin_value = imm;
1087 	reg->umax_value = imm;
1088 
1089 	reg->s32_min_value = (s32)imm;
1090 	reg->s32_max_value = (s32)imm;
1091 	reg->u32_min_value = (u32)imm;
1092 	reg->u32_max_value = (u32)imm;
1093 }
1094 
1095 /* Mark the unknown part of a register (variable offset or scalar value) as
1096  * known to have the value @imm.
1097  */
1098 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1099 {
1100 	/* Clear id, off, and union(map_ptr, range) */
1101 	memset(((u8 *)reg) + sizeof(reg->type), 0,
1102 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
1103 	___mark_reg_known(reg, imm);
1104 }
1105 
1106 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
1107 {
1108 	reg->var_off = tnum_const_subreg(reg->var_off, imm);
1109 	reg->s32_min_value = (s32)imm;
1110 	reg->s32_max_value = (s32)imm;
1111 	reg->u32_min_value = (u32)imm;
1112 	reg->u32_max_value = (u32)imm;
1113 }
1114 
1115 /* Mark the 'variable offset' part of a register as zero.  This should be
1116  * used only on registers holding a pointer type.
1117  */
1118 static void __mark_reg_known_zero(struct bpf_reg_state *reg)
1119 {
1120 	__mark_reg_known(reg, 0);
1121 }
1122 
1123 static void __mark_reg_const_zero(struct bpf_reg_state *reg)
1124 {
1125 	__mark_reg_known(reg, 0);
1126 	reg->type = SCALAR_VALUE;
1127 }
1128 
1129 static void mark_reg_known_zero(struct bpf_verifier_env *env,
1130 				struct bpf_reg_state *regs, u32 regno)
1131 {
1132 	if (WARN_ON(regno >= MAX_BPF_REG)) {
1133 		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
1134 		/* Something bad happened, let's kill all regs */
1135 		for (regno = 0; regno < MAX_BPF_REG; regno++)
1136 			__mark_reg_not_init(env, regs + regno);
1137 		return;
1138 	}
1139 	__mark_reg_known_zero(regs + regno);
1140 }
1141 
1142 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1143 {
1144 	switch (reg->type) {
1145 	case PTR_TO_MAP_VALUE_OR_NULL: {
1146 		const struct bpf_map *map = reg->map_ptr;
1147 
1148 		if (map->inner_map_meta) {
1149 			reg->type = CONST_PTR_TO_MAP;
1150 			reg->map_ptr = map->inner_map_meta;
1151 			/* transfer reg's id which is unique for every map_lookup_elem
1152 			 * as UID of the inner map.
1153 			 */
1154 			if (map_value_has_timer(map->inner_map_meta))
1155 				reg->map_uid = reg->id;
1156 		} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1157 			reg->type = PTR_TO_XDP_SOCK;
1158 		} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1159 			   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1160 			reg->type = PTR_TO_SOCKET;
1161 		} else {
1162 			reg->type = PTR_TO_MAP_VALUE;
1163 		}
1164 		break;
1165 	}
1166 	case PTR_TO_SOCKET_OR_NULL:
1167 		reg->type = PTR_TO_SOCKET;
1168 		break;
1169 	case PTR_TO_SOCK_COMMON_OR_NULL:
1170 		reg->type = PTR_TO_SOCK_COMMON;
1171 		break;
1172 	case PTR_TO_TCP_SOCK_OR_NULL:
1173 		reg->type = PTR_TO_TCP_SOCK;
1174 		break;
1175 	case PTR_TO_BTF_ID_OR_NULL:
1176 		reg->type = PTR_TO_BTF_ID;
1177 		break;
1178 	case PTR_TO_MEM_OR_NULL:
1179 		reg->type = PTR_TO_MEM;
1180 		break;
1181 	case PTR_TO_RDONLY_BUF_OR_NULL:
1182 		reg->type = PTR_TO_RDONLY_BUF;
1183 		break;
1184 	case PTR_TO_RDWR_BUF_OR_NULL:
1185 		reg->type = PTR_TO_RDWR_BUF;
1186 		break;
1187 	default:
1188 		WARN_ONCE(1, "unknown nullable register type");
1189 	}
1190 }
1191 
1192 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
1193 {
1194 	return type_is_pkt_pointer(reg->type);
1195 }
1196 
1197 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
1198 {
1199 	return reg_is_pkt_pointer(reg) ||
1200 	       reg->type == PTR_TO_PACKET_END;
1201 }
1202 
1203 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
1204 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
1205 				    enum bpf_reg_type which)
1206 {
1207 	/* The register can already have a range from prior markings.
1208 	 * This is fine as long as it hasn't been advanced from its
1209 	 * origin.
1210 	 */
1211 	return reg->type == which &&
1212 	       reg->id == 0 &&
1213 	       reg->off == 0 &&
1214 	       tnum_equals_const(reg->var_off, 0);
1215 }
1216 
1217 /* Reset the min/max bounds of a register */
1218 static void __mark_reg_unbounded(struct bpf_reg_state *reg)
1219 {
1220 	reg->smin_value = S64_MIN;
1221 	reg->smax_value = S64_MAX;
1222 	reg->umin_value = 0;
1223 	reg->umax_value = U64_MAX;
1224 
1225 	reg->s32_min_value = S32_MIN;
1226 	reg->s32_max_value = S32_MAX;
1227 	reg->u32_min_value = 0;
1228 	reg->u32_max_value = U32_MAX;
1229 }
1230 
1231 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
1232 {
1233 	reg->smin_value = S64_MIN;
1234 	reg->smax_value = S64_MAX;
1235 	reg->umin_value = 0;
1236 	reg->umax_value = U64_MAX;
1237 }
1238 
1239 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
1240 {
1241 	reg->s32_min_value = S32_MIN;
1242 	reg->s32_max_value = S32_MAX;
1243 	reg->u32_min_value = 0;
1244 	reg->u32_max_value = U32_MAX;
1245 }
1246 
1247 static void __update_reg32_bounds(struct bpf_reg_state *reg)
1248 {
1249 	struct tnum var32_off = tnum_subreg(reg->var_off);
1250 
1251 	/* min signed is max(sign bit) | min(other bits) */
1252 	reg->s32_min_value = max_t(s32, reg->s32_min_value,
1253 			var32_off.value | (var32_off.mask & S32_MIN));
1254 	/* max signed is min(sign bit) | max(other bits) */
1255 	reg->s32_max_value = min_t(s32, reg->s32_max_value,
1256 			var32_off.value | (var32_off.mask & S32_MAX));
1257 	reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
1258 	reg->u32_max_value = min(reg->u32_max_value,
1259 				 (u32)(var32_off.value | var32_off.mask));
1260 }
1261 
1262 static void __update_reg64_bounds(struct bpf_reg_state *reg)
1263 {
1264 	/* min signed is max(sign bit) | min(other bits) */
1265 	reg->smin_value = max_t(s64, reg->smin_value,
1266 				reg->var_off.value | (reg->var_off.mask & S64_MIN));
1267 	/* max signed is min(sign bit) | max(other bits) */
1268 	reg->smax_value = min_t(s64, reg->smax_value,
1269 				reg->var_off.value | (reg->var_off.mask & S64_MAX));
1270 	reg->umin_value = max(reg->umin_value, reg->var_off.value);
1271 	reg->umax_value = min(reg->umax_value,
1272 			      reg->var_off.value | reg->var_off.mask);
1273 }
1274 
1275 static void __update_reg_bounds(struct bpf_reg_state *reg)
1276 {
1277 	__update_reg32_bounds(reg);
1278 	__update_reg64_bounds(reg);
1279 }
1280 
1281 /* Uses signed min/max values to inform unsigned, and vice-versa */
1282 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
1283 {
1284 	/* Learn sign from signed bounds.
1285 	 * If we cannot cross the sign boundary, then signed and unsigned bounds
1286 	 * are the same, so combine.  This works even in the negative case, e.g.
1287 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
1288 	 */
1289 	if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) {
1290 		reg->s32_min_value = reg->u32_min_value =
1291 			max_t(u32, reg->s32_min_value, reg->u32_min_value);
1292 		reg->s32_max_value = reg->u32_max_value =
1293 			min_t(u32, reg->s32_max_value, reg->u32_max_value);
1294 		return;
1295 	}
1296 	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
1297 	 * boundary, so we must be careful.
1298 	 */
1299 	if ((s32)reg->u32_max_value >= 0) {
1300 		/* Positive.  We can't learn anything from the smin, but smax
1301 		 * is positive, hence safe.
1302 		 */
1303 		reg->s32_min_value = reg->u32_min_value;
1304 		reg->s32_max_value = reg->u32_max_value =
1305 			min_t(u32, reg->s32_max_value, reg->u32_max_value);
1306 	} else if ((s32)reg->u32_min_value < 0) {
1307 		/* Negative.  We can't learn anything from the smax, but smin
1308 		 * is negative, hence safe.
1309 		 */
1310 		reg->s32_min_value = reg->u32_min_value =
1311 			max_t(u32, reg->s32_min_value, reg->u32_min_value);
1312 		reg->s32_max_value = reg->u32_max_value;
1313 	}
1314 }
1315 
1316 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
1317 {
1318 	/* Learn sign from signed bounds.
1319 	 * If we cannot cross the sign boundary, then signed and unsigned bounds
1320 	 * are the same, so combine.  This works even in the negative case, e.g.
1321 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
1322 	 */
1323 	if (reg->smin_value >= 0 || reg->smax_value < 0) {
1324 		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
1325 							  reg->umin_value);
1326 		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
1327 							  reg->umax_value);
1328 		return;
1329 	}
1330 	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
1331 	 * boundary, so we must be careful.
1332 	 */
1333 	if ((s64)reg->umax_value >= 0) {
1334 		/* Positive.  We can't learn anything from the smin, but smax
1335 		 * is positive, hence safe.
1336 		 */
1337 		reg->smin_value = reg->umin_value;
1338 		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
1339 							  reg->umax_value);
1340 	} else if ((s64)reg->umin_value < 0) {
1341 		/* Negative.  We can't learn anything from the smax, but smin
1342 		 * is negative, hence safe.
1343 		 */
1344 		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
1345 							  reg->umin_value);
1346 		reg->smax_value = reg->umax_value;
1347 	}
1348 }
1349 
1350 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
1351 {
1352 	__reg32_deduce_bounds(reg);
1353 	__reg64_deduce_bounds(reg);
1354 }
1355 
1356 /* Attempts to improve var_off based on unsigned min/max information */
1357 static void __reg_bound_offset(struct bpf_reg_state *reg)
1358 {
1359 	struct tnum var64_off = tnum_intersect(reg->var_off,
1360 					       tnum_range(reg->umin_value,
1361 							  reg->umax_value));
1362 	struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off),
1363 						tnum_range(reg->u32_min_value,
1364 							   reg->u32_max_value));
1365 
1366 	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
1367 }
1368 
1369 static bool __reg32_bound_s64(s32 a)
1370 {
1371 	return a >= 0 && a <= S32_MAX;
1372 }
1373 
1374 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
1375 {
1376 	reg->umin_value = reg->u32_min_value;
1377 	reg->umax_value = reg->u32_max_value;
1378 
1379 	/* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
1380 	 * be positive otherwise set to worse case bounds and refine later
1381 	 * from tnum.
1382 	 */
1383 	if (__reg32_bound_s64(reg->s32_min_value) &&
1384 	    __reg32_bound_s64(reg->s32_max_value)) {
1385 		reg->smin_value = reg->s32_min_value;
1386 		reg->smax_value = reg->s32_max_value;
1387 	} else {
1388 		reg->smin_value = 0;
1389 		reg->smax_value = U32_MAX;
1390 	}
1391 }
1392 
1393 static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
1394 {
1395 	/* special case when 64-bit register has upper 32-bit register
1396 	 * zeroed. Typically happens after zext or <<32, >>32 sequence
1397 	 * allowing us to use 32-bit bounds directly,
1398 	 */
1399 	if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) {
1400 		__reg_assign_32_into_64(reg);
1401 	} else {
1402 		/* Otherwise the best we can do is push lower 32bit known and
1403 		 * unknown bits into register (var_off set from jmp logic)
1404 		 * then learn as much as possible from the 64-bit tnum
1405 		 * known and unknown bits. The previous smin/smax bounds are
1406 		 * invalid here because of jmp32 compare so mark them unknown
1407 		 * so they do not impact tnum bounds calculation.
1408 		 */
1409 		__mark_reg64_unbounded(reg);
1410 		__update_reg_bounds(reg);
1411 	}
1412 
1413 	/* Intersecting with the old var_off might have improved our bounds
1414 	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
1415 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
1416 	 */
1417 	__reg_deduce_bounds(reg);
1418 	__reg_bound_offset(reg);
1419 	__update_reg_bounds(reg);
1420 }
1421 
1422 static bool __reg64_bound_s32(s64 a)
1423 {
1424 	return a >= S32_MIN && a <= S32_MAX;
1425 }
1426 
1427 static bool __reg64_bound_u32(u64 a)
1428 {
1429 	return a >= U32_MIN && a <= U32_MAX;
1430 }
1431 
1432 static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
1433 {
1434 	__mark_reg32_unbounded(reg);
1435 
1436 	if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) {
1437 		reg->s32_min_value = (s32)reg->smin_value;
1438 		reg->s32_max_value = (s32)reg->smax_value;
1439 	}
1440 	if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) {
1441 		reg->u32_min_value = (u32)reg->umin_value;
1442 		reg->u32_max_value = (u32)reg->umax_value;
1443 	}
1444 
1445 	/* Intersecting with the old var_off might have improved our bounds
1446 	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
1447 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
1448 	 */
1449 	__reg_deduce_bounds(reg);
1450 	__reg_bound_offset(reg);
1451 	__update_reg_bounds(reg);
1452 }
1453 
1454 /* Mark a register as having a completely unknown (scalar) value. */
1455 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
1456 			       struct bpf_reg_state *reg)
1457 {
1458 	/*
1459 	 * Clear type, id, off, and union(map_ptr, range) and
1460 	 * padding between 'type' and union
1461 	 */
1462 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
1463 	reg->type = SCALAR_VALUE;
1464 	reg->var_off = tnum_unknown;
1465 	reg->frameno = 0;
1466 	reg->precise = env->subprog_cnt > 1 || !env->bpf_capable;
1467 	__mark_reg_unbounded(reg);
1468 }
1469 
1470 static void mark_reg_unknown(struct bpf_verifier_env *env,
1471 			     struct bpf_reg_state *regs, u32 regno)
1472 {
1473 	if (WARN_ON(regno >= MAX_BPF_REG)) {
1474 		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
1475 		/* Something bad happened, let's kill all regs except FP */
1476 		for (regno = 0; regno < BPF_REG_FP; regno++)
1477 			__mark_reg_not_init(env, regs + regno);
1478 		return;
1479 	}
1480 	__mark_reg_unknown(env, regs + regno);
1481 }
1482 
1483 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1484 				struct bpf_reg_state *reg)
1485 {
1486 	__mark_reg_unknown(env, reg);
1487 	reg->type = NOT_INIT;
1488 }
1489 
1490 static void mark_reg_not_init(struct bpf_verifier_env *env,
1491 			      struct bpf_reg_state *regs, u32 regno)
1492 {
1493 	if (WARN_ON(regno >= MAX_BPF_REG)) {
1494 		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
1495 		/* Something bad happened, let's kill all regs except FP */
1496 		for (regno = 0; regno < BPF_REG_FP; regno++)
1497 			__mark_reg_not_init(env, regs + regno);
1498 		return;
1499 	}
1500 	__mark_reg_not_init(env, regs + regno);
1501 }
1502 
1503 static void mark_btf_ld_reg(struct bpf_verifier_env *env,
1504 			    struct bpf_reg_state *regs, u32 regno,
1505 			    enum bpf_reg_type reg_type,
1506 			    struct btf *btf, u32 btf_id)
1507 {
1508 	if (reg_type == SCALAR_VALUE) {
1509 		mark_reg_unknown(env, regs, regno);
1510 		return;
1511 	}
1512 	mark_reg_known_zero(env, regs, regno);
1513 	regs[regno].type = PTR_TO_BTF_ID;
1514 	regs[regno].btf = btf;
1515 	regs[regno].btf_id = btf_id;
1516 }
1517 
1518 #define DEF_NOT_SUBREG	(0)
1519 static void init_reg_state(struct bpf_verifier_env *env,
1520 			   struct bpf_func_state *state)
1521 {
1522 	struct bpf_reg_state *regs = state->regs;
1523 	int i;
1524 
1525 	for (i = 0; i < MAX_BPF_REG; i++) {
1526 		mark_reg_not_init(env, regs, i);
1527 		regs[i].live = REG_LIVE_NONE;
1528 		regs[i].parent = NULL;
1529 		regs[i].subreg_def = DEF_NOT_SUBREG;
1530 	}
1531 
1532 	/* frame pointer */
1533 	regs[BPF_REG_FP].type = PTR_TO_STACK;
1534 	mark_reg_known_zero(env, regs, BPF_REG_FP);
1535 	regs[BPF_REG_FP].frameno = state->frameno;
1536 }
1537 
1538 #define BPF_MAIN_FUNC (-1)
1539 static void init_func_state(struct bpf_verifier_env *env,
1540 			    struct bpf_func_state *state,
1541 			    int callsite, int frameno, int subprogno)
1542 {
1543 	state->callsite = callsite;
1544 	state->frameno = frameno;
1545 	state->subprogno = subprogno;
1546 	init_reg_state(env, state);
1547 }
1548 
1549 /* Similar to push_stack(), but for async callbacks */
1550 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
1551 						int insn_idx, int prev_insn_idx,
1552 						int subprog)
1553 {
1554 	struct bpf_verifier_stack_elem *elem;
1555 	struct bpf_func_state *frame;
1556 
1557 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
1558 	if (!elem)
1559 		goto err;
1560 
1561 	elem->insn_idx = insn_idx;
1562 	elem->prev_insn_idx = prev_insn_idx;
1563 	elem->next = env->head;
1564 	elem->log_pos = env->log.len_used;
1565 	env->head = elem;
1566 	env->stack_size++;
1567 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
1568 		verbose(env,
1569 			"The sequence of %d jumps is too complex for async cb.\n",
1570 			env->stack_size);
1571 		goto err;
1572 	}
1573 	/* Unlike push_stack() do not copy_verifier_state().
1574 	 * The caller state doesn't matter.
1575 	 * This is async callback. It starts in a fresh stack.
1576 	 * Initialize it similar to do_check_common().
1577 	 */
1578 	elem->st.branches = 1;
1579 	frame = kzalloc(sizeof(*frame), GFP_KERNEL);
1580 	if (!frame)
1581 		goto err;
1582 	init_func_state(env, frame,
1583 			BPF_MAIN_FUNC /* callsite */,
1584 			0 /* frameno within this callchain */,
1585 			subprog /* subprog number within this prog */);
1586 	elem->st.frame[0] = frame;
1587 	return &elem->st;
1588 err:
1589 	free_verifier_state(env->cur_state, true);
1590 	env->cur_state = NULL;
1591 	/* pop all elements and return */
1592 	while (!pop_stack(env, NULL, NULL, false));
1593 	return NULL;
1594 }
1595 
1596 
1597 enum reg_arg_type {
1598 	SRC_OP,		/* register is used as source operand */
1599 	DST_OP,		/* register is used as destination operand */
1600 	DST_OP_NO_MARK	/* same as above, check only, don't mark */
1601 };
1602 
1603 static int cmp_subprogs(const void *a, const void *b)
1604 {
1605 	return ((struct bpf_subprog_info *)a)->start -
1606 	       ((struct bpf_subprog_info *)b)->start;
1607 }
1608 
1609 static int find_subprog(struct bpf_verifier_env *env, int off)
1610 {
1611 	struct bpf_subprog_info *p;
1612 
1613 	p = bsearch(&off, env->subprog_info, env->subprog_cnt,
1614 		    sizeof(env->subprog_info[0]), cmp_subprogs);
1615 	if (!p)
1616 		return -ENOENT;
1617 	return p - env->subprog_info;
1618 
1619 }
1620 
1621 static int add_subprog(struct bpf_verifier_env *env, int off)
1622 {
1623 	int insn_cnt = env->prog->len;
1624 	int ret;
1625 
1626 	if (off >= insn_cnt || off < 0) {
1627 		verbose(env, "call to invalid destination\n");
1628 		return -EINVAL;
1629 	}
1630 	ret = find_subprog(env, off);
1631 	if (ret >= 0)
1632 		return ret;
1633 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
1634 		verbose(env, "too many subprograms\n");
1635 		return -E2BIG;
1636 	}
1637 	/* determine subprog starts. The end is one before the next starts */
1638 	env->subprog_info[env->subprog_cnt++].start = off;
1639 	sort(env->subprog_info, env->subprog_cnt,
1640 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1641 	return env->subprog_cnt - 1;
1642 }
1643 
1644 #define MAX_KFUNC_DESCS 256
1645 #define MAX_KFUNC_BTFS	256
1646 
1647 struct bpf_kfunc_desc {
1648 	struct btf_func_model func_model;
1649 	u32 func_id;
1650 	s32 imm;
1651 	u16 offset;
1652 };
1653 
1654 struct bpf_kfunc_btf {
1655 	struct btf *btf;
1656 	struct module *module;
1657 	u16 offset;
1658 };
1659 
1660 struct bpf_kfunc_desc_tab {
1661 	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
1662 	u32 nr_descs;
1663 };
1664 
1665 struct bpf_kfunc_btf_tab {
1666 	struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
1667 	u32 nr_descs;
1668 };
1669 
1670 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
1671 {
1672 	const struct bpf_kfunc_desc *d0 = a;
1673 	const struct bpf_kfunc_desc *d1 = b;
1674 
1675 	/* func_id is not greater than BTF_MAX_TYPE */
1676 	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
1677 }
1678 
1679 static int kfunc_btf_cmp_by_off(const void *a, const void *b)
1680 {
1681 	const struct bpf_kfunc_btf *d0 = a;
1682 	const struct bpf_kfunc_btf *d1 = b;
1683 
1684 	return d0->offset - d1->offset;
1685 }
1686 
1687 static const struct bpf_kfunc_desc *
1688 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
1689 {
1690 	struct bpf_kfunc_desc desc = {
1691 		.func_id = func_id,
1692 		.offset = offset,
1693 	};
1694 	struct bpf_kfunc_desc_tab *tab;
1695 
1696 	tab = prog->aux->kfunc_tab;
1697 	return bsearch(&desc, tab->descs, tab->nr_descs,
1698 		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
1699 }
1700 
1701 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
1702 					 s16 offset, struct module **btf_modp)
1703 {
1704 	struct bpf_kfunc_btf kf_btf = { .offset = offset };
1705 	struct bpf_kfunc_btf_tab *tab;
1706 	struct bpf_kfunc_btf *b;
1707 	struct module *mod;
1708 	struct btf *btf;
1709 	int btf_fd;
1710 
1711 	tab = env->prog->aux->kfunc_btf_tab;
1712 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
1713 		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
1714 	if (!b) {
1715 		if (tab->nr_descs == MAX_KFUNC_BTFS) {
1716 			verbose(env, "too many different module BTFs\n");
1717 			return ERR_PTR(-E2BIG);
1718 		}
1719 
1720 		if (bpfptr_is_null(env->fd_array)) {
1721 			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
1722 			return ERR_PTR(-EPROTO);
1723 		}
1724 
1725 		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
1726 					    offset * sizeof(btf_fd),
1727 					    sizeof(btf_fd)))
1728 			return ERR_PTR(-EFAULT);
1729 
1730 		btf = btf_get_by_fd(btf_fd);
1731 		if (IS_ERR(btf)) {
1732 			verbose(env, "invalid module BTF fd specified\n");
1733 			return btf;
1734 		}
1735 
1736 		if (!btf_is_module(btf)) {
1737 			verbose(env, "BTF fd for kfunc is not a module BTF\n");
1738 			btf_put(btf);
1739 			return ERR_PTR(-EINVAL);
1740 		}
1741 
1742 		mod = btf_try_get_module(btf);
1743 		if (!mod) {
1744 			btf_put(btf);
1745 			return ERR_PTR(-ENXIO);
1746 		}
1747 
1748 		b = &tab->descs[tab->nr_descs++];
1749 		b->btf = btf;
1750 		b->module = mod;
1751 		b->offset = offset;
1752 
1753 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1754 		     kfunc_btf_cmp_by_off, NULL);
1755 	}
1756 	if (btf_modp)
1757 		*btf_modp = b->module;
1758 	return b->btf;
1759 }
1760 
1761 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
1762 {
1763 	if (!tab)
1764 		return;
1765 
1766 	while (tab->nr_descs--) {
1767 		module_put(tab->descs[tab->nr_descs].module);
1768 		btf_put(tab->descs[tab->nr_descs].btf);
1769 	}
1770 	kfree(tab);
1771 }
1772 
1773 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env,
1774 				       u32 func_id, s16 offset,
1775 				       struct module **btf_modp)
1776 {
1777 	if (offset) {
1778 		if (offset < 0) {
1779 			/* In the future, this can be allowed to increase limit
1780 			 * of fd index into fd_array, interpreted as u16.
1781 			 */
1782 			verbose(env, "negative offset disallowed for kernel module function call\n");
1783 			return ERR_PTR(-EINVAL);
1784 		}
1785 
1786 		return __find_kfunc_desc_btf(env, offset, btf_modp);
1787 	}
1788 	return btf_vmlinux ?: ERR_PTR(-ENOENT);
1789 }
1790 
1791 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
1792 {
1793 	const struct btf_type *func, *func_proto;
1794 	struct bpf_kfunc_btf_tab *btf_tab;
1795 	struct bpf_kfunc_desc_tab *tab;
1796 	struct bpf_prog_aux *prog_aux;
1797 	struct bpf_kfunc_desc *desc;
1798 	const char *func_name;
1799 	struct btf *desc_btf;
1800 	unsigned long addr;
1801 	int err;
1802 
1803 	prog_aux = env->prog->aux;
1804 	tab = prog_aux->kfunc_tab;
1805 	btf_tab = prog_aux->kfunc_btf_tab;
1806 	if (!tab) {
1807 		if (!btf_vmlinux) {
1808 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
1809 			return -ENOTSUPP;
1810 		}
1811 
1812 		if (!env->prog->jit_requested) {
1813 			verbose(env, "JIT is required for calling kernel function\n");
1814 			return -ENOTSUPP;
1815 		}
1816 
1817 		if (!bpf_jit_supports_kfunc_call()) {
1818 			verbose(env, "JIT does not support calling kernel function\n");
1819 			return -ENOTSUPP;
1820 		}
1821 
1822 		if (!env->prog->gpl_compatible) {
1823 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");
1824 			return -EINVAL;
1825 		}
1826 
1827 		tab = kzalloc(sizeof(*tab), GFP_KERNEL);
1828 		if (!tab)
1829 			return -ENOMEM;
1830 		prog_aux->kfunc_tab = tab;
1831 	}
1832 
1833 	/* func_id == 0 is always invalid, but instead of returning an error, be
1834 	 * conservative and wait until the code elimination pass before returning
1835 	 * error, so that invalid calls that get pruned out can be in BPF programs
1836 	 * loaded from userspace.  It is also required that offset be untouched
1837 	 * for such calls.
1838 	 */
1839 	if (!func_id && !offset)
1840 		return 0;
1841 
1842 	if (!btf_tab && offset) {
1843 		btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
1844 		if (!btf_tab)
1845 			return -ENOMEM;
1846 		prog_aux->kfunc_btf_tab = btf_tab;
1847 	}
1848 
1849 	desc_btf = find_kfunc_desc_btf(env, func_id, offset, NULL);
1850 	if (IS_ERR(desc_btf)) {
1851 		verbose(env, "failed to find BTF for kernel function\n");
1852 		return PTR_ERR(desc_btf);
1853 	}
1854 
1855 	if (find_kfunc_desc(env->prog, func_id, offset))
1856 		return 0;
1857 
1858 	if (tab->nr_descs == MAX_KFUNC_DESCS) {
1859 		verbose(env, "too many different kernel function calls\n");
1860 		return -E2BIG;
1861 	}
1862 
1863 	func = btf_type_by_id(desc_btf, func_id);
1864 	if (!func || !btf_type_is_func(func)) {
1865 		verbose(env, "kernel btf_id %u is not a function\n",
1866 			func_id);
1867 		return -EINVAL;
1868 	}
1869 	func_proto = btf_type_by_id(desc_btf, func->type);
1870 	if (!func_proto || !btf_type_is_func_proto(func_proto)) {
1871 		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
1872 			func_id);
1873 		return -EINVAL;
1874 	}
1875 
1876 	func_name = btf_name_by_offset(desc_btf, func->name_off);
1877 	addr = kallsyms_lookup_name(func_name);
1878 	if (!addr) {
1879 		verbose(env, "cannot find address for kernel function %s\n",
1880 			func_name);
1881 		return -EINVAL;
1882 	}
1883 
1884 	desc = &tab->descs[tab->nr_descs++];
1885 	desc->func_id = func_id;
1886 	desc->imm = BPF_CALL_IMM(addr);
1887 	desc->offset = offset;
1888 	err = btf_distill_func_proto(&env->log, desc_btf,
1889 				     func_proto, func_name,
1890 				     &desc->func_model);
1891 	if (!err)
1892 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1893 		     kfunc_desc_cmp_by_id_off, NULL);
1894 	return err;
1895 }
1896 
1897 static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
1898 {
1899 	const struct bpf_kfunc_desc *d0 = a;
1900 	const struct bpf_kfunc_desc *d1 = b;
1901 
1902 	if (d0->imm > d1->imm)
1903 		return 1;
1904 	else if (d0->imm < d1->imm)
1905 		return -1;
1906 	return 0;
1907 }
1908 
1909 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
1910 {
1911 	struct bpf_kfunc_desc_tab *tab;
1912 
1913 	tab = prog->aux->kfunc_tab;
1914 	if (!tab)
1915 		return;
1916 
1917 	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1918 	     kfunc_desc_cmp_by_imm, NULL);
1919 }
1920 
1921 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1922 {
1923 	return !!prog->aux->kfunc_tab;
1924 }
1925 
1926 const struct btf_func_model *
1927 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1928 			 const struct bpf_insn *insn)
1929 {
1930 	const struct bpf_kfunc_desc desc = {
1931 		.imm = insn->imm,
1932 	};
1933 	const struct bpf_kfunc_desc *res;
1934 	struct bpf_kfunc_desc_tab *tab;
1935 
1936 	tab = prog->aux->kfunc_tab;
1937 	res = bsearch(&desc, tab->descs, tab->nr_descs,
1938 		      sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
1939 
1940 	return res ? &res->func_model : NULL;
1941 }
1942 
1943 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1944 {
1945 	struct bpf_subprog_info *subprog = env->subprog_info;
1946 	struct bpf_insn *insn = env->prog->insnsi;
1947 	int i, ret, insn_cnt = env->prog->len;
1948 
1949 	/* Add entry function. */
1950 	ret = add_subprog(env, 0);
1951 	if (ret)
1952 		return ret;
1953 
1954 	for (i = 0; i < insn_cnt; i++, insn++) {
1955 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1956 		    !bpf_pseudo_kfunc_call(insn))
1957 			continue;
1958 
1959 		if (!env->bpf_capable) {
1960 			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1961 			return -EPERM;
1962 		}
1963 
1964 		if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
1965 			ret = add_subprog(env, i + insn->imm + 1);
1966 		else
1967 			ret = add_kfunc_call(env, insn->imm, insn->off);
1968 
1969 		if (ret < 0)
1970 			return ret;
1971 	}
1972 
1973 	/* Add a fake 'exit' subprog which could simplify subprog iteration
1974 	 * logic. 'subprog_cnt' should not be increased.
1975 	 */
1976 	subprog[env->subprog_cnt].start = insn_cnt;
1977 
1978 	if (env->log.level & BPF_LOG_LEVEL2)
1979 		for (i = 0; i < env->subprog_cnt; i++)
1980 			verbose(env, "func#%d @%d\n", i, subprog[i].start);
1981 
1982 	return 0;
1983 }
1984 
1985 static int check_subprogs(struct bpf_verifier_env *env)
1986 {
1987 	int i, subprog_start, subprog_end, off, cur_subprog = 0;
1988 	struct bpf_subprog_info *subprog = env->subprog_info;
1989 	struct bpf_insn *insn = env->prog->insnsi;
1990 	int insn_cnt = env->prog->len;
1991 
1992 	/* now check that all jumps are within the same subprog */
1993 	subprog_start = subprog[cur_subprog].start;
1994 	subprog_end = subprog[cur_subprog + 1].start;
1995 	for (i = 0; i < insn_cnt; i++) {
1996 		u8 code = insn[i].code;
1997 
1998 		if (code == (BPF_JMP | BPF_CALL) &&
1999 		    insn[i].imm == BPF_FUNC_tail_call &&
2000 		    insn[i].src_reg != BPF_PSEUDO_CALL)
2001 			subprog[cur_subprog].has_tail_call = true;
2002 		if (BPF_CLASS(code) == BPF_LD &&
2003 		    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
2004 			subprog[cur_subprog].has_ld_abs = true;
2005 		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
2006 			goto next;
2007 		if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
2008 			goto next;
2009 		off = i + insn[i].off + 1;
2010 		if (off < subprog_start || off >= subprog_end) {
2011 			verbose(env, "jump out of range from insn %d to %d\n", i, off);
2012 			return -EINVAL;
2013 		}
2014 next:
2015 		if (i == subprog_end - 1) {
2016 			/* to avoid fall-through from one subprog into another
2017 			 * the last insn of the subprog should be either exit
2018 			 * or unconditional jump back
2019 			 */
2020 			if (code != (BPF_JMP | BPF_EXIT) &&
2021 			    code != (BPF_JMP | BPF_JA)) {
2022 				verbose(env, "last insn is not an exit or jmp\n");
2023 				return -EINVAL;
2024 			}
2025 			subprog_start = subprog_end;
2026 			cur_subprog++;
2027 			if (cur_subprog < env->subprog_cnt)
2028 				subprog_end = subprog[cur_subprog + 1].start;
2029 		}
2030 	}
2031 	return 0;
2032 }
2033 
2034 /* Parentage chain of this register (or stack slot) should take care of all
2035  * issues like callee-saved registers, stack slot allocation time, etc.
2036  */
2037 static int mark_reg_read(struct bpf_verifier_env *env,
2038 			 const struct bpf_reg_state *state,
2039 			 struct bpf_reg_state *parent, u8 flag)
2040 {
2041 	bool writes = parent == state->parent; /* Observe write marks */
2042 	int cnt = 0;
2043 
2044 	while (parent) {
2045 		/* if read wasn't screened by an earlier write ... */
2046 		if (writes && state->live & REG_LIVE_WRITTEN)
2047 			break;
2048 		if (parent->live & REG_LIVE_DONE) {
2049 			verbose(env, "verifier BUG type %s var_off %lld off %d\n",
2050 				reg_type_str[parent->type],
2051 				parent->var_off.value, parent->off);
2052 			return -EFAULT;
2053 		}
2054 		/* The first condition is more likely to be true than the
2055 		 * second, checked it first.
2056 		 */
2057 		if ((parent->live & REG_LIVE_READ) == flag ||
2058 		    parent->live & REG_LIVE_READ64)
2059 			/* The parentage chain never changes and
2060 			 * this parent was already marked as LIVE_READ.
2061 			 * There is no need to keep walking the chain again and
2062 			 * keep re-marking all parents as LIVE_READ.
2063 			 * This case happens when the same register is read
2064 			 * multiple times without writes into it in-between.
2065 			 * Also, if parent has the stronger REG_LIVE_READ64 set,
2066 			 * then no need to set the weak REG_LIVE_READ32.
2067 			 */
2068 			break;
2069 		/* ... then we depend on parent's value */
2070 		parent->live |= flag;
2071 		/* REG_LIVE_READ64 overrides REG_LIVE_READ32. */
2072 		if (flag == REG_LIVE_READ64)
2073 			parent->live &= ~REG_LIVE_READ32;
2074 		state = parent;
2075 		parent = state->parent;
2076 		writes = true;
2077 		cnt++;
2078 	}
2079 
2080 	if (env->longest_mark_read_walk < cnt)
2081 		env->longest_mark_read_walk = cnt;
2082 	return 0;
2083 }
2084 
2085 /* This function is supposed to be used by the following 32-bit optimization
2086  * code only. It returns TRUE if the source or destination register operates
2087  * on 64-bit, otherwise return FALSE.
2088  */
2089 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
2090 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
2091 {
2092 	u8 code, class, op;
2093 
2094 	code = insn->code;
2095 	class = BPF_CLASS(code);
2096 	op = BPF_OP(code);
2097 	if (class == BPF_JMP) {
2098 		/* BPF_EXIT for "main" will reach here. Return TRUE
2099 		 * conservatively.
2100 		 */
2101 		if (op == BPF_EXIT)
2102 			return true;
2103 		if (op == BPF_CALL) {
2104 			/* BPF to BPF call will reach here because of marking
2105 			 * caller saved clobber with DST_OP_NO_MARK for which we
2106 			 * don't care the register def because they are anyway
2107 			 * marked as NOT_INIT already.
2108 			 */
2109 			if (insn->src_reg == BPF_PSEUDO_CALL)
2110 				return false;
2111 			/* Helper call will reach here because of arg type
2112 			 * check, conservatively return TRUE.
2113 			 */
2114 			if (t == SRC_OP)
2115 				return true;
2116 
2117 			return false;
2118 		}
2119 	}
2120 
2121 	if (class == BPF_ALU64 || class == BPF_JMP ||
2122 	    /* BPF_END always use BPF_ALU class. */
2123 	    (class == BPF_ALU && op == BPF_END && insn->imm == 64))
2124 		return true;
2125 
2126 	if (class == BPF_ALU || class == BPF_JMP32)
2127 		return false;
2128 
2129 	if (class == BPF_LDX) {
2130 		if (t != SRC_OP)
2131 			return BPF_SIZE(code) == BPF_DW;
2132 		/* LDX source must be ptr. */
2133 		return true;
2134 	}
2135 
2136 	if (class == BPF_STX) {
2137 		/* BPF_STX (including atomic variants) has multiple source
2138 		 * operands, one of which is a ptr. Check whether the caller is
2139 		 * asking about it.
2140 		 */
2141 		if (t == SRC_OP && reg->type != SCALAR_VALUE)
2142 			return true;
2143 		return BPF_SIZE(code) == BPF_DW;
2144 	}
2145 
2146 	if (class == BPF_LD) {
2147 		u8 mode = BPF_MODE(code);
2148 
2149 		/* LD_IMM64 */
2150 		if (mode == BPF_IMM)
2151 			return true;
2152 
2153 		/* Both LD_IND and LD_ABS return 32-bit data. */
2154 		if (t != SRC_OP)
2155 			return  false;
2156 
2157 		/* Implicit ctx ptr. */
2158 		if (regno == BPF_REG_6)
2159 			return true;
2160 
2161 		/* Explicit source could be any width. */
2162 		return true;
2163 	}
2164 
2165 	if (class == BPF_ST)
2166 		/* The only source register for BPF_ST is a ptr. */
2167 		return true;
2168 
2169 	/* Conservatively return true at default. */
2170 	return true;
2171 }
2172 
2173 /* Return the regno defined by the insn, or -1. */
2174 static int insn_def_regno(const struct bpf_insn *insn)
2175 {
2176 	switch (BPF_CLASS(insn->code)) {
2177 	case BPF_JMP:
2178 	case BPF_JMP32:
2179 	case BPF_ST:
2180 		return -1;
2181 	case BPF_STX:
2182 		if (BPF_MODE(insn->code) == BPF_ATOMIC &&
2183 		    (insn->imm & BPF_FETCH)) {
2184 			if (insn->imm == BPF_CMPXCHG)
2185 				return BPF_REG_0;
2186 			else
2187 				return insn->src_reg;
2188 		} else {
2189 			return -1;
2190 		}
2191 	default:
2192 		return insn->dst_reg;
2193 	}
2194 }
2195 
2196 /* Return TRUE if INSN has defined any 32-bit value explicitly. */
2197 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
2198 {
2199 	int dst_reg = insn_def_regno(insn);
2200 
2201 	if (dst_reg == -1)
2202 		return false;
2203 
2204 	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
2205 }
2206 
2207 static void mark_insn_zext(struct bpf_verifier_env *env,
2208 			   struct bpf_reg_state *reg)
2209 {
2210 	s32 def_idx = reg->subreg_def;
2211 
2212 	if (def_idx == DEF_NOT_SUBREG)
2213 		return;
2214 
2215 	env->insn_aux_data[def_idx - 1].zext_dst = true;
2216 	/* The dst will be zero extended, so won't be sub-register anymore. */
2217 	reg->subreg_def = DEF_NOT_SUBREG;
2218 }
2219 
2220 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
2221 			 enum reg_arg_type t)
2222 {
2223 	struct bpf_verifier_state *vstate = env->cur_state;
2224 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
2225 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
2226 	struct bpf_reg_state *reg, *regs = state->regs;
2227 	bool rw64;
2228 
2229 	if (regno >= MAX_BPF_REG) {
2230 		verbose(env, "R%d is invalid\n", regno);
2231 		return -EINVAL;
2232 	}
2233 
2234 	reg = &regs[regno];
2235 	rw64 = is_reg64(env, insn, regno, reg, t);
2236 	if (t == SRC_OP) {
2237 		/* check whether register used as source operand can be read */
2238 		if (reg->type == NOT_INIT) {
2239 			verbose(env, "R%d !read_ok\n", regno);
2240 			return -EACCES;
2241 		}
2242 		/* We don't need to worry about FP liveness because it's read-only */
2243 		if (regno == BPF_REG_FP)
2244 			return 0;
2245 
2246 		if (rw64)
2247 			mark_insn_zext(env, reg);
2248 
2249 		return mark_reg_read(env, reg, reg->parent,
2250 				     rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
2251 	} else {
2252 		/* check whether register used as dest operand can be written to */
2253 		if (regno == BPF_REG_FP) {
2254 			verbose(env, "frame pointer is read only\n");
2255 			return -EACCES;
2256 		}
2257 		reg->live |= REG_LIVE_WRITTEN;
2258 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
2259 		if (t == DST_OP)
2260 			mark_reg_unknown(env, regs, regno);
2261 	}
2262 	return 0;
2263 }
2264 
2265 /* for any branch, call, exit record the history of jmps in the given state */
2266 static int push_jmp_history(struct bpf_verifier_env *env,
2267 			    struct bpf_verifier_state *cur)
2268 {
2269 	u32 cnt = cur->jmp_history_cnt;
2270 	struct bpf_idx_pair *p;
2271 
2272 	cnt++;
2273 	p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER);
2274 	if (!p)
2275 		return -ENOMEM;
2276 	p[cnt - 1].idx = env->insn_idx;
2277 	p[cnt - 1].prev_idx = env->prev_insn_idx;
2278 	cur->jmp_history = p;
2279 	cur->jmp_history_cnt = cnt;
2280 	return 0;
2281 }
2282 
2283 /* Backtrack one insn at a time. If idx is not at the top of recorded
2284  * history then previous instruction came from straight line execution.
2285  */
2286 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
2287 			     u32 *history)
2288 {
2289 	u32 cnt = *history;
2290 
2291 	if (cnt && st->jmp_history[cnt - 1].idx == i) {
2292 		i = st->jmp_history[cnt - 1].prev_idx;
2293 		(*history)--;
2294 	} else {
2295 		i--;
2296 	}
2297 	return i;
2298 }
2299 
2300 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
2301 {
2302 	const struct btf_type *func;
2303 	struct btf *desc_btf;
2304 
2305 	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
2306 		return NULL;
2307 
2308 	desc_btf = find_kfunc_desc_btf(data, insn->imm, insn->off, NULL);
2309 	if (IS_ERR(desc_btf))
2310 		return "<error>";
2311 
2312 	func = btf_type_by_id(desc_btf, insn->imm);
2313 	return btf_name_by_offset(desc_btf, func->name_off);
2314 }
2315 
2316 /* For given verifier state backtrack_insn() is called from the last insn to
2317  * the first insn. Its purpose is to compute a bitmask of registers and
2318  * stack slots that needs precision in the parent verifier state.
2319  */
2320 static int backtrack_insn(struct bpf_verifier_env *env, int idx,
2321 			  u32 *reg_mask, u64 *stack_mask)
2322 {
2323 	const struct bpf_insn_cbs cbs = {
2324 		.cb_call	= disasm_kfunc_name,
2325 		.cb_print	= verbose,
2326 		.private_data	= env,
2327 	};
2328 	struct bpf_insn *insn = env->prog->insnsi + idx;
2329 	u8 class = BPF_CLASS(insn->code);
2330 	u8 opcode = BPF_OP(insn->code);
2331 	u8 mode = BPF_MODE(insn->code);
2332 	u32 dreg = 1u << insn->dst_reg;
2333 	u32 sreg = 1u << insn->src_reg;
2334 	u32 spi;
2335 
2336 	if (insn->code == 0)
2337 		return 0;
2338 	if (env->log.level & BPF_LOG_LEVEL) {
2339 		verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
2340 		verbose(env, "%d: ", idx);
2341 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
2342 	}
2343 
2344 	if (class == BPF_ALU || class == BPF_ALU64) {
2345 		if (!(*reg_mask & dreg))
2346 			return 0;
2347 		if (opcode == BPF_MOV) {
2348 			if (BPF_SRC(insn->code) == BPF_X) {
2349 				/* dreg = sreg
2350 				 * dreg needs precision after this insn
2351 				 * sreg needs precision before this insn
2352 				 */
2353 				*reg_mask &= ~dreg;
2354 				*reg_mask |= sreg;
2355 			} else {
2356 				/* dreg = K
2357 				 * dreg needs precision after this insn.
2358 				 * Corresponding register is already marked
2359 				 * as precise=true in this verifier state.
2360 				 * No further markings in parent are necessary
2361 				 */
2362 				*reg_mask &= ~dreg;
2363 			}
2364 		} else {
2365 			if (BPF_SRC(insn->code) == BPF_X) {
2366 				/* dreg += sreg
2367 				 * both dreg and sreg need precision
2368 				 * before this insn
2369 				 */
2370 				*reg_mask |= sreg;
2371 			} /* else dreg += K
2372 			   * dreg still needs precision before this insn
2373 			   */
2374 		}
2375 	} else if (class == BPF_LDX) {
2376 		if (!(*reg_mask & dreg))
2377 			return 0;
2378 		*reg_mask &= ~dreg;
2379 
2380 		/* scalars can only be spilled into stack w/o losing precision.
2381 		 * Load from any other memory can be zero extended.
2382 		 * The desire to keep that precision is already indicated
2383 		 * by 'precise' mark in corresponding register of this state.
2384 		 * No further tracking necessary.
2385 		 */
2386 		if (insn->src_reg != BPF_REG_FP)
2387 			return 0;
2388 
2389 		/* dreg = *(u64 *)[fp - off] was a fill from the stack.
2390 		 * that [fp - off] slot contains scalar that needs to be
2391 		 * tracked with precision
2392 		 */
2393 		spi = (-insn->off - 1) / BPF_REG_SIZE;
2394 		if (spi >= 64) {
2395 			verbose(env, "BUG spi %d\n", spi);
2396 			WARN_ONCE(1, "verifier backtracking bug");
2397 			return -EFAULT;
2398 		}
2399 		*stack_mask |= 1ull << spi;
2400 	} else if (class == BPF_STX || class == BPF_ST) {
2401 		if (*reg_mask & dreg)
2402 			/* stx & st shouldn't be using _scalar_ dst_reg
2403 			 * to access memory. It means backtracking
2404 			 * encountered a case of pointer subtraction.
2405 			 */
2406 			return -ENOTSUPP;
2407 		/* scalars can only be spilled into stack */
2408 		if (insn->dst_reg != BPF_REG_FP)
2409 			return 0;
2410 		spi = (-insn->off - 1) / BPF_REG_SIZE;
2411 		if (spi >= 64) {
2412 			verbose(env, "BUG spi %d\n", spi);
2413 			WARN_ONCE(1, "verifier backtracking bug");
2414 			return -EFAULT;
2415 		}
2416 		if (!(*stack_mask & (1ull << spi)))
2417 			return 0;
2418 		*stack_mask &= ~(1ull << spi);
2419 		if (class == BPF_STX)
2420 			*reg_mask |= sreg;
2421 	} else if (class == BPF_JMP || class == BPF_JMP32) {
2422 		if (opcode == BPF_CALL) {
2423 			if (insn->src_reg == BPF_PSEUDO_CALL)
2424 				return -ENOTSUPP;
2425 			/* regular helper call sets R0 */
2426 			*reg_mask &= ~1;
2427 			if (*reg_mask & 0x3f) {
2428 				/* if backtracing was looking for registers R1-R5
2429 				 * they should have been found already.
2430 				 */
2431 				verbose(env, "BUG regs %x\n", *reg_mask);
2432 				WARN_ONCE(1, "verifier backtracking bug");
2433 				return -EFAULT;
2434 			}
2435 		} else if (opcode == BPF_EXIT) {
2436 			return -ENOTSUPP;
2437 		}
2438 	} else if (class == BPF_LD) {
2439 		if (!(*reg_mask & dreg))
2440 			return 0;
2441 		*reg_mask &= ~dreg;
2442 		/* It's ld_imm64 or ld_abs or ld_ind.
2443 		 * For ld_imm64 no further tracking of precision
2444 		 * into parent is necessary
2445 		 */
2446 		if (mode == BPF_IND || mode == BPF_ABS)
2447 			/* to be analyzed */
2448 			return -ENOTSUPP;
2449 	}
2450 	return 0;
2451 }
2452 
2453 /* the scalar precision tracking algorithm:
2454  * . at the start all registers have precise=false.
2455  * . scalar ranges are tracked as normal through alu and jmp insns.
2456  * . once precise value of the scalar register is used in:
2457  *   .  ptr + scalar alu
2458  *   . if (scalar cond K|scalar)
2459  *   .  helper_call(.., scalar, ...) where ARG_CONST is expected
2460  *   backtrack through the verifier states and mark all registers and
2461  *   stack slots with spilled constants that these scalar regisers
2462  *   should be precise.
2463  * . during state pruning two registers (or spilled stack slots)
2464  *   are equivalent if both are not precise.
2465  *
2466  * Note the verifier cannot simply walk register parentage chain,
2467  * since many different registers and stack slots could have been
2468  * used to compute single precise scalar.
2469  *
2470  * The approach of starting with precise=true for all registers and then
2471  * backtrack to mark a register as not precise when the verifier detects
2472  * that program doesn't care about specific value (e.g., when helper
2473  * takes register as ARG_ANYTHING parameter) is not safe.
2474  *
2475  * It's ok to walk single parentage chain of the verifier states.
2476  * It's possible that this backtracking will go all the way till 1st insn.
2477  * All other branches will be explored for needing precision later.
2478  *
2479  * The backtracking needs to deal with cases like:
2480  *   R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
2481  * r9 -= r8
2482  * r5 = r9
2483  * if r5 > 0x79f goto pc+7
2484  *    R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
2485  * r5 += 1
2486  * ...
2487  * call bpf_perf_event_output#25
2488  *   where .arg5_type = ARG_CONST_SIZE_OR_ZERO
2489  *
2490  * and this case:
2491  * r6 = 1
2492  * call foo // uses callee's r6 inside to compute r0
2493  * r0 += r6
2494  * if r0 == 0 goto
2495  *
2496  * to track above reg_mask/stack_mask needs to be independent for each frame.
2497  *
2498  * Also if parent's curframe > frame where backtracking started,
2499  * the verifier need to mark registers in both frames, otherwise callees
2500  * may incorrectly prune callers. This is similar to
2501  * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
2502  *
2503  * For now backtracking falls back into conservative marking.
2504  */
2505 static void mark_all_scalars_precise(struct bpf_verifier_env *env,
2506 				     struct bpf_verifier_state *st)
2507 {
2508 	struct bpf_func_state *func;
2509 	struct bpf_reg_state *reg;
2510 	int i, j;
2511 
2512 	/* big hammer: mark all scalars precise in this path.
2513 	 * pop_stack may still get !precise scalars.
2514 	 */
2515 	for (; st; st = st->parent)
2516 		for (i = 0; i <= st->curframe; i++) {
2517 			func = st->frame[i];
2518 			for (j = 0; j < BPF_REG_FP; j++) {
2519 				reg = &func->regs[j];
2520 				if (reg->type != SCALAR_VALUE)
2521 					continue;
2522 				reg->precise = true;
2523 			}
2524 			for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
2525 				if (!is_spilled_reg(&func->stack[j]))
2526 					continue;
2527 				reg = &func->stack[j].spilled_ptr;
2528 				if (reg->type != SCALAR_VALUE)
2529 					continue;
2530 				reg->precise = true;
2531 			}
2532 		}
2533 }
2534 
2535 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
2536 				  int spi)
2537 {
2538 	struct bpf_verifier_state *st = env->cur_state;
2539 	int first_idx = st->first_insn_idx;
2540 	int last_idx = env->insn_idx;
2541 	struct bpf_func_state *func;
2542 	struct bpf_reg_state *reg;
2543 	u32 reg_mask = regno >= 0 ? 1u << regno : 0;
2544 	u64 stack_mask = spi >= 0 ? 1ull << spi : 0;
2545 	bool skip_first = true;
2546 	bool new_marks = false;
2547 	int i, err;
2548 
2549 	if (!env->bpf_capable)
2550 		return 0;
2551 
2552 	func = st->frame[st->curframe];
2553 	if (regno >= 0) {
2554 		reg = &func->regs[regno];
2555 		if (reg->type != SCALAR_VALUE) {
2556 			WARN_ONCE(1, "backtracing misuse");
2557 			return -EFAULT;
2558 		}
2559 		if (!reg->precise)
2560 			new_marks = true;
2561 		else
2562 			reg_mask = 0;
2563 		reg->precise = true;
2564 	}
2565 
2566 	while (spi >= 0) {
2567 		if (!is_spilled_reg(&func->stack[spi])) {
2568 			stack_mask = 0;
2569 			break;
2570 		}
2571 		reg = &func->stack[spi].spilled_ptr;
2572 		if (reg->type != SCALAR_VALUE) {
2573 			stack_mask = 0;
2574 			break;
2575 		}
2576 		if (!reg->precise)
2577 			new_marks = true;
2578 		else
2579 			stack_mask = 0;
2580 		reg->precise = true;
2581 		break;
2582 	}
2583 
2584 	if (!new_marks)
2585 		return 0;
2586 	if (!reg_mask && !stack_mask)
2587 		return 0;
2588 	for (;;) {
2589 		DECLARE_BITMAP(mask, 64);
2590 		u32 history = st->jmp_history_cnt;
2591 
2592 		if (env->log.level & BPF_LOG_LEVEL)
2593 			verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2594 		for (i = last_idx;;) {
2595 			if (skip_first) {
2596 				err = 0;
2597 				skip_first = false;
2598 			} else {
2599 				err = backtrack_insn(env, i, &reg_mask, &stack_mask);
2600 			}
2601 			if (err == -ENOTSUPP) {
2602 				mark_all_scalars_precise(env, st);
2603 				return 0;
2604 			} else if (err) {
2605 				return err;
2606 			}
2607 			if (!reg_mask && !stack_mask)
2608 				/* Found assignment(s) into tracked register in this state.
2609 				 * Since this state is already marked, just return.
2610 				 * Nothing to be tracked further in the parent state.
2611 				 */
2612 				return 0;
2613 			if (i == first_idx)
2614 				break;
2615 			i = get_prev_insn_idx(st, i, &history);
2616 			if (i >= env->prog->len) {
2617 				/* This can happen if backtracking reached insn 0
2618 				 * and there are still reg_mask or stack_mask
2619 				 * to backtrack.
2620 				 * It means the backtracking missed the spot where
2621 				 * particular register was initialized with a constant.
2622 				 */
2623 				verbose(env, "BUG backtracking idx %d\n", i);
2624 				WARN_ONCE(1, "verifier backtracking bug");
2625 				return -EFAULT;
2626 			}
2627 		}
2628 		st = st->parent;
2629 		if (!st)
2630 			break;
2631 
2632 		new_marks = false;
2633 		func = st->frame[st->curframe];
2634 		bitmap_from_u64(mask, reg_mask);
2635 		for_each_set_bit(i, mask, 32) {
2636 			reg = &func->regs[i];
2637 			if (reg->type != SCALAR_VALUE) {
2638 				reg_mask &= ~(1u << i);
2639 				continue;
2640 			}
2641 			if (!reg->precise)
2642 				new_marks = true;
2643 			reg->precise = true;
2644 		}
2645 
2646 		bitmap_from_u64(mask, stack_mask);
2647 		for_each_set_bit(i, mask, 64) {
2648 			if (i >= func->allocated_stack / BPF_REG_SIZE) {
2649 				/* the sequence of instructions:
2650 				 * 2: (bf) r3 = r10
2651 				 * 3: (7b) *(u64 *)(r3 -8) = r0
2652 				 * 4: (79) r4 = *(u64 *)(r10 -8)
2653 				 * doesn't contain jmps. It's backtracked
2654 				 * as a single block.
2655 				 * During backtracking insn 3 is not recognized as
2656 				 * stack access, so at the end of backtracking
2657 				 * stack slot fp-8 is still marked in stack_mask.
2658 				 * However the parent state may not have accessed
2659 				 * fp-8 and it's "unallocated" stack space.
2660 				 * In such case fallback to conservative.
2661 				 */
2662 				mark_all_scalars_precise(env, st);
2663 				return 0;
2664 			}
2665 
2666 			if (!is_spilled_reg(&func->stack[i])) {
2667 				stack_mask &= ~(1ull << i);
2668 				continue;
2669 			}
2670 			reg = &func->stack[i].spilled_ptr;
2671 			if (reg->type != SCALAR_VALUE) {
2672 				stack_mask &= ~(1ull << i);
2673 				continue;
2674 			}
2675 			if (!reg->precise)
2676 				new_marks = true;
2677 			reg->precise = true;
2678 		}
2679 		if (env->log.level & BPF_LOG_LEVEL) {
2680 			print_verifier_state(env, func);
2681 			verbose(env, "parent %s regs=%x stack=%llx marks\n",
2682 				new_marks ? "didn't have" : "already had",
2683 				reg_mask, stack_mask);
2684 		}
2685 
2686 		if (!reg_mask && !stack_mask)
2687 			break;
2688 		if (!new_marks)
2689 			break;
2690 
2691 		last_idx = st->last_insn_idx;
2692 		first_idx = st->first_insn_idx;
2693 	}
2694 	return 0;
2695 }
2696 
2697 static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
2698 {
2699 	return __mark_chain_precision(env, regno, -1);
2700 }
2701 
2702 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
2703 {
2704 	return __mark_chain_precision(env, -1, spi);
2705 }
2706 
2707 static bool is_spillable_regtype(enum bpf_reg_type type)
2708 {
2709 	switch (type) {
2710 	case PTR_TO_MAP_VALUE:
2711 	case PTR_TO_MAP_VALUE_OR_NULL:
2712 	case PTR_TO_STACK:
2713 	case PTR_TO_CTX:
2714 	case PTR_TO_PACKET:
2715 	case PTR_TO_PACKET_META:
2716 	case PTR_TO_PACKET_END:
2717 	case PTR_TO_FLOW_KEYS:
2718 	case CONST_PTR_TO_MAP:
2719 	case PTR_TO_SOCKET:
2720 	case PTR_TO_SOCKET_OR_NULL:
2721 	case PTR_TO_SOCK_COMMON:
2722 	case PTR_TO_SOCK_COMMON_OR_NULL:
2723 	case PTR_TO_TCP_SOCK:
2724 	case PTR_TO_TCP_SOCK_OR_NULL:
2725 	case PTR_TO_XDP_SOCK:
2726 	case PTR_TO_BTF_ID:
2727 	case PTR_TO_BTF_ID_OR_NULL:
2728 	case PTR_TO_RDONLY_BUF:
2729 	case PTR_TO_RDONLY_BUF_OR_NULL:
2730 	case PTR_TO_RDWR_BUF:
2731 	case PTR_TO_RDWR_BUF_OR_NULL:
2732 	case PTR_TO_PERCPU_BTF_ID:
2733 	case PTR_TO_MEM:
2734 	case PTR_TO_MEM_OR_NULL:
2735 	case PTR_TO_FUNC:
2736 	case PTR_TO_MAP_KEY:
2737 		return true;
2738 	default:
2739 		return false;
2740 	}
2741 }
2742 
2743 /* Does this register contain a constant zero? */
2744 static bool register_is_null(struct bpf_reg_state *reg)
2745 {
2746 	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
2747 }
2748 
2749 static bool register_is_const(struct bpf_reg_state *reg)
2750 {
2751 	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2752 }
2753 
2754 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
2755 {
2756 	return tnum_is_unknown(reg->var_off) &&
2757 	       reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
2758 	       reg->umin_value == 0 && reg->umax_value == U64_MAX &&
2759 	       reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
2760 	       reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
2761 }
2762 
2763 static bool register_is_bounded(struct bpf_reg_state *reg)
2764 {
2765 	return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
2766 }
2767 
2768 static bool __is_pointer_value(bool allow_ptr_leaks,
2769 			       const struct bpf_reg_state *reg)
2770 {
2771 	if (allow_ptr_leaks)
2772 		return false;
2773 
2774 	return reg->type != SCALAR_VALUE;
2775 }
2776 
2777 static void save_register_state(struct bpf_func_state *state,
2778 				int spi, struct bpf_reg_state *reg,
2779 				int size)
2780 {
2781 	int i;
2782 
2783 	state->stack[spi].spilled_ptr = *reg;
2784 	if (size == BPF_REG_SIZE)
2785 		state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
2786 
2787 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
2788 		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
2789 
2790 	/* size < 8 bytes spill */
2791 	for (; i; i--)
2792 		scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]);
2793 }
2794 
2795 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
2796  * stack boundary and alignment are checked in check_mem_access()
2797  */
2798 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2799 				       /* stack frame we're writing to */
2800 				       struct bpf_func_state *state,
2801 				       int off, int size, int value_regno,
2802 				       int insn_idx)
2803 {
2804 	struct bpf_func_state *cur; /* state of the current function */
2805 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
2806 	u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
2807 	struct bpf_reg_state *reg = NULL;
2808 
2809 	err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE));
2810 	if (err)
2811 		return err;
2812 	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
2813 	 * so it's aligned access and [off, off + size) are within stack limits
2814 	 */
2815 	if (!env->allow_ptr_leaks &&
2816 	    state->stack[spi].slot_type[0] == STACK_SPILL &&
2817 	    size != BPF_REG_SIZE) {
2818 		verbose(env, "attempt to corrupt spilled pointer on stack\n");
2819 		return -EACCES;
2820 	}
2821 
2822 	cur = env->cur_state->frame[env->cur_state->curframe];
2823 	if (value_regno >= 0)
2824 		reg = &cur->regs[value_regno];
2825 	if (!env->bypass_spec_v4) {
2826 		bool sanitize = reg && is_spillable_regtype(reg->type);
2827 
2828 		for (i = 0; i < size; i++) {
2829 			if (state->stack[spi].slot_type[i] == STACK_INVALID) {
2830 				sanitize = true;
2831 				break;
2832 			}
2833 		}
2834 
2835 		if (sanitize)
2836 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2837 	}
2838 
2839 	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
2840 	    !register_is_null(reg) && env->bpf_capable) {
2841 		if (dst_reg != BPF_REG_FP) {
2842 			/* The backtracking logic can only recognize explicit
2843 			 * stack slot address like [fp - 8]. Other spill of
2844 			 * scalar via different register has to be conservative.
2845 			 * Backtrack from here and mark all registers as precise
2846 			 * that contributed into 'reg' being a constant.
2847 			 */
2848 			err = mark_chain_precision(env, value_regno);
2849 			if (err)
2850 				return err;
2851 		}
2852 		save_register_state(state, spi, reg, size);
2853 	} else if (reg && is_spillable_regtype(reg->type)) {
2854 		/* register containing pointer is being spilled into stack */
2855 		if (size != BPF_REG_SIZE) {
2856 			verbose_linfo(env, insn_idx, "; ");
2857 			verbose(env, "invalid size of register spill\n");
2858 			return -EACCES;
2859 		}
2860 		if (state != cur && reg->type == PTR_TO_STACK) {
2861 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
2862 			return -EINVAL;
2863 		}
2864 		save_register_state(state, spi, reg, size);
2865 	} else {
2866 		u8 type = STACK_MISC;
2867 
2868 		/* regular write of data into stack destroys any spilled ptr */
2869 		state->stack[spi].spilled_ptr.type = NOT_INIT;
2870 		/* Mark slots as STACK_MISC if they belonged to spilled ptr. */
2871 		if (is_spilled_reg(&state->stack[spi]))
2872 			for (i = 0; i < BPF_REG_SIZE; i++)
2873 				scrub_spilled_slot(&state->stack[spi].slot_type[i]);
2874 
2875 		/* only mark the slot as written if all 8 bytes were written
2876 		 * otherwise read propagation may incorrectly stop too soon
2877 		 * when stack slots are partially written.
2878 		 * This heuristic means that read propagation will be
2879 		 * conservative, since it will add reg_live_read marks
2880 		 * to stack slots all the way to first state when programs
2881 		 * writes+reads less than 8 bytes
2882 		 */
2883 		if (size == BPF_REG_SIZE)
2884 			state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
2885 
2886 		/* when we zero initialize stack slots mark them as such */
2887 		if (reg && register_is_null(reg)) {
2888 			/* backtracking doesn't work for STACK_ZERO yet. */
2889 			err = mark_chain_precision(env, value_regno);
2890 			if (err)
2891 				return err;
2892 			type = STACK_ZERO;
2893 		}
2894 
2895 		/* Mark slots affected by this stack write. */
2896 		for (i = 0; i < size; i++)
2897 			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
2898 				type;
2899 	}
2900 	return 0;
2901 }
2902 
2903 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
2904  * known to contain a variable offset.
2905  * This function checks whether the write is permitted and conservatively
2906  * tracks the effects of the write, considering that each stack slot in the
2907  * dynamic range is potentially written to.
2908  *
2909  * 'off' includes 'regno->off'.
2910  * 'value_regno' can be -1, meaning that an unknown value is being written to
2911  * the stack.
2912  *
2913  * Spilled pointers in range are not marked as written because we don't know
2914  * what's going to be actually written. This means that read propagation for
2915  * future reads cannot be terminated by this write.
2916  *
2917  * For privileged programs, uninitialized stack slots are considered
2918  * initialized by this write (even though we don't know exactly what offsets
2919  * are going to be written to). The idea is that we don't want the verifier to
2920  * reject future reads that access slots written to through variable offsets.
2921  */
2922 static int check_stack_write_var_off(struct bpf_verifier_env *env,
2923 				     /* func where register points to */
2924 				     struct bpf_func_state *state,
2925 				     int ptr_regno, int off, int size,
2926 				     int value_regno, int insn_idx)
2927 {
2928 	struct bpf_func_state *cur; /* state of the current function */
2929 	int min_off, max_off;
2930 	int i, err;
2931 	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
2932 	bool writing_zero = false;
2933 	/* set if the fact that we're writing a zero is used to let any
2934 	 * stack slots remain STACK_ZERO
2935 	 */
2936 	bool zero_used = false;
2937 
2938 	cur = env->cur_state->frame[env->cur_state->curframe];
2939 	ptr_reg = &cur->regs[ptr_regno];
2940 	min_off = ptr_reg->smin_value + off;
2941 	max_off = ptr_reg->smax_value + off + size;
2942 	if (value_regno >= 0)
2943 		value_reg = &cur->regs[value_regno];
2944 	if (value_reg && register_is_null(value_reg))
2945 		writing_zero = true;
2946 
2947 	err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE));
2948 	if (err)
2949 		return err;
2950 
2951 
2952 	/* Variable offset writes destroy any spilled pointers in range. */
2953 	for (i = min_off; i < max_off; i++) {
2954 		u8 new_type, *stype;
2955 		int slot, spi;
2956 
2957 		slot = -i - 1;
2958 		spi = slot / BPF_REG_SIZE;
2959 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2960 
2961 		if (!env->allow_ptr_leaks
2962 				&& *stype != NOT_INIT
2963 				&& *stype != SCALAR_VALUE) {
2964 			/* Reject the write if there's are spilled pointers in
2965 			 * range. If we didn't reject here, the ptr status
2966 			 * would be erased below (even though not all slots are
2967 			 * actually overwritten), possibly opening the door to
2968 			 * leaks.
2969 			 */
2970 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
2971 				insn_idx, i);
2972 			return -EINVAL;
2973 		}
2974 
2975 		/* Erase all spilled pointers. */
2976 		state->stack[spi].spilled_ptr.type = NOT_INIT;
2977 
2978 		/* Update the slot type. */
2979 		new_type = STACK_MISC;
2980 		if (writing_zero && *stype == STACK_ZERO) {
2981 			new_type = STACK_ZERO;
2982 			zero_used = true;
2983 		}
2984 		/* If the slot is STACK_INVALID, we check whether it's OK to
2985 		 * pretend that it will be initialized by this write. The slot
2986 		 * might not actually be written to, and so if we mark it as
2987 		 * initialized future reads might leak uninitialized memory.
2988 		 * For privileged programs, we will accept such reads to slots
2989 		 * that may or may not be written because, if we're reject
2990 		 * them, the error would be too confusing.
2991 		 */
2992 		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2993 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
2994 					insn_idx, i);
2995 			return -EINVAL;
2996 		}
2997 		*stype = new_type;
2998 	}
2999 	if (zero_used) {
3000 		/* backtracking doesn't work for STACK_ZERO yet. */
3001 		err = mark_chain_precision(env, value_regno);
3002 		if (err)
3003 			return err;
3004 	}
3005 	return 0;
3006 }
3007 
3008 /* When register 'dst_regno' is assigned some values from stack[min_off,
3009  * max_off), we set the register's type according to the types of the
3010  * respective stack slots. If all the stack values are known to be zeros, then
3011  * so is the destination reg. Otherwise, the register is considered to be
3012  * SCALAR. This function does not deal with register filling; the caller must
3013  * ensure that all spilled registers in the stack range have been marked as
3014  * read.
3015  */
3016 static void mark_reg_stack_read(struct bpf_verifier_env *env,
3017 				/* func where src register points to */
3018 				struct bpf_func_state *ptr_state,
3019 				int min_off, int max_off, int dst_regno)
3020 {
3021 	struct bpf_verifier_state *vstate = env->cur_state;
3022 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3023 	int i, slot, spi;
3024 	u8 *stype;
3025 	int zeros = 0;
3026 
3027 	for (i = min_off; i < max_off; i++) {
3028 		slot = -i - 1;
3029 		spi = slot / BPF_REG_SIZE;
3030 		stype = ptr_state->stack[spi].slot_type;
3031 		if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
3032 			break;
3033 		zeros++;
3034 	}
3035 	if (zeros == max_off - min_off) {
3036 		/* any access_size read into register is zero extended,
3037 		 * so the whole register == const_zero
3038 		 */
3039 		__mark_reg_const_zero(&state->regs[dst_regno]);
3040 		/* backtracking doesn't support STACK_ZERO yet,
3041 		 * so mark it precise here, so that later
3042 		 * backtracking can stop here.
3043 		 * Backtracking may not need this if this register
3044 		 * doesn't participate in pointer adjustment.
3045 		 * Forward propagation of precise flag is not
3046 		 * necessary either. This mark is only to stop
3047 		 * backtracking. Any register that contributed
3048 		 * to const 0 was marked precise before spill.
3049 		 */
3050 		state->regs[dst_regno].precise = true;
3051 	} else {
3052 		/* have read misc data from the stack */
3053 		mark_reg_unknown(env, state->regs, dst_regno);
3054 	}
3055 	state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
3056 }
3057 
3058 /* Read the stack at 'off' and put the results into the register indicated by
3059  * 'dst_regno'. It handles reg filling if the addressed stack slot is a
3060  * spilled reg.
3061  *
3062  * 'dst_regno' can be -1, meaning that the read value is not going to a
3063  * register.
3064  *
3065  * The access is assumed to be within the current stack bounds.
3066  */
3067 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
3068 				      /* func where src register points to */
3069 				      struct bpf_func_state *reg_state,
3070 				      int off, int size, int dst_regno)
3071 {
3072 	struct bpf_verifier_state *vstate = env->cur_state;
3073 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3074 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
3075 	struct bpf_reg_state *reg;
3076 	u8 *stype, type;
3077 
3078 	stype = reg_state->stack[spi].slot_type;
3079 	reg = &reg_state->stack[spi].spilled_ptr;
3080 
3081 	if (is_spilled_reg(&reg_state->stack[spi])) {
3082 		u8 spill_size = 1;
3083 
3084 		for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
3085 			spill_size++;
3086 
3087 		if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
3088 			if (reg->type != SCALAR_VALUE) {
3089 				verbose_linfo(env, env->insn_idx, "; ");
3090 				verbose(env, "invalid size of register fill\n");
3091 				return -EACCES;
3092 			}
3093 
3094 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
3095 			if (dst_regno < 0)
3096 				return 0;
3097 
3098 			if (!(off % BPF_REG_SIZE) && size == spill_size) {
3099 				/* The earlier check_reg_arg() has decided the
3100 				 * subreg_def for this insn.  Save it first.
3101 				 */
3102 				s32 subreg_def = state->regs[dst_regno].subreg_def;
3103 
3104 				state->regs[dst_regno] = *reg;
3105 				state->regs[dst_regno].subreg_def = subreg_def;
3106 			} else {
3107 				for (i = 0; i < size; i++) {
3108 					type = stype[(slot - i) % BPF_REG_SIZE];
3109 					if (type == STACK_SPILL)
3110 						continue;
3111 					if (type == STACK_MISC)
3112 						continue;
3113 					verbose(env, "invalid read from stack off %d+%d size %d\n",
3114 						off, i, size);
3115 					return -EACCES;
3116 				}
3117 				mark_reg_unknown(env, state->regs, dst_regno);
3118 			}
3119 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
3120 			return 0;
3121 		}
3122 
3123 		if (dst_regno >= 0) {
3124 			/* restore register state from stack */
3125 			state->regs[dst_regno] = *reg;
3126 			/* mark reg as written since spilled pointer state likely
3127 			 * has its liveness marks cleared by is_state_visited()
3128 			 * which resets stack/reg liveness for state transitions
3129 			 */
3130 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
3131 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
3132 			/* If dst_regno==-1, the caller is asking us whether
3133 			 * it is acceptable to use this value as a SCALAR_VALUE
3134 			 * (e.g. for XADD).
3135 			 * We must not allow unprivileged callers to do that
3136 			 * with spilled pointers.
3137 			 */
3138 			verbose(env, "leaking pointer from stack off %d\n",
3139 				off);
3140 			return -EACCES;
3141 		}
3142 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
3143 	} else {
3144 		for (i = 0; i < size; i++) {
3145 			type = stype[(slot - i) % BPF_REG_SIZE];
3146 			if (type == STACK_MISC)
3147 				continue;
3148 			if (type == STACK_ZERO)
3149 				continue;
3150 			verbose(env, "invalid read from stack off %d+%d size %d\n",
3151 				off, i, size);
3152 			return -EACCES;
3153 		}
3154 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
3155 		if (dst_regno >= 0)
3156 			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
3157 	}
3158 	return 0;
3159 }
3160 
3161 enum stack_access_src {
3162 	ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
3163 	ACCESS_HELPER = 2,  /* the access is performed by a helper */
3164 };
3165 
3166 static int check_stack_range_initialized(struct bpf_verifier_env *env,
3167 					 int regno, int off, int access_size,
3168 					 bool zero_size_allowed,
3169 					 enum stack_access_src type,
3170 					 struct bpf_call_arg_meta *meta);
3171 
3172 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
3173 {
3174 	return cur_regs(env) + regno;
3175 }
3176 
3177 /* Read the stack at 'ptr_regno + off' and put the result into the register
3178  * 'dst_regno'.
3179  * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
3180  * but not its variable offset.
3181  * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
3182  *
3183  * As opposed to check_stack_read_fixed_off, this function doesn't deal with
3184  * filling registers (i.e. reads of spilled register cannot be detected when
3185  * the offset is not fixed). We conservatively mark 'dst_regno' as containing
3186  * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
3187  * offset; for a fixed offset check_stack_read_fixed_off should be used
3188  * instead.
3189  */
3190 static int check_stack_read_var_off(struct bpf_verifier_env *env,
3191 				    int ptr_regno, int off, int size, int dst_regno)
3192 {
3193 	/* The state of the source register. */
3194 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
3195 	struct bpf_func_state *ptr_state = func(env, reg);
3196 	int err;
3197 	int min_off, max_off;
3198 
3199 	/* Note that we pass a NULL meta, so raw access will not be permitted.
3200 	 */
3201 	err = check_stack_range_initialized(env, ptr_regno, off, size,
3202 					    false, ACCESS_DIRECT, NULL);
3203 	if (err)
3204 		return err;
3205 
3206 	min_off = reg->smin_value + off;
3207 	max_off = reg->smax_value + off;
3208 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
3209 	return 0;
3210 }
3211 
3212 /* check_stack_read dispatches to check_stack_read_fixed_off or
3213  * check_stack_read_var_off.
3214  *
3215  * The caller must ensure that the offset falls within the allocated stack
3216  * bounds.
3217  *
3218  * 'dst_regno' is a register which will receive the value from the stack. It
3219  * can be -1, meaning that the read value is not going to a register.
3220  */
3221 static int check_stack_read(struct bpf_verifier_env *env,
3222 			    int ptr_regno, int off, int size,
3223 			    int dst_regno)
3224 {
3225 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
3226 	struct bpf_func_state *state = func(env, reg);
3227 	int err;
3228 	/* Some accesses are only permitted with a static offset. */
3229 	bool var_off = !tnum_is_const(reg->var_off);
3230 
3231 	/* The offset is required to be static when reads don't go to a
3232 	 * register, in order to not leak pointers (see
3233 	 * check_stack_read_fixed_off).
3234 	 */
3235 	if (dst_regno < 0 && var_off) {
3236 		char tn_buf[48];
3237 
3238 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3239 		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
3240 			tn_buf, off, size);
3241 		return -EACCES;
3242 	}
3243 	/* Variable offset is prohibited for unprivileged mode for simplicity
3244 	 * since it requires corresponding support in Spectre masking for stack
3245 	 * ALU. See also retrieve_ptr_limit().
3246 	 */
3247 	if (!env->bypass_spec_v1 && var_off) {
3248 		char tn_buf[48];
3249 
3250 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3251 		verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
3252 				ptr_regno, tn_buf);
3253 		return -EACCES;
3254 	}
3255 
3256 	if (!var_off) {
3257 		off += reg->var_off.value;
3258 		err = check_stack_read_fixed_off(env, state, off, size,
3259 						 dst_regno);
3260 	} else {
3261 		/* Variable offset stack reads need more conservative handling
3262 		 * than fixed offset ones. Note that dst_regno >= 0 on this
3263 		 * branch.
3264 		 */
3265 		err = check_stack_read_var_off(env, ptr_regno, off, size,
3266 					       dst_regno);
3267 	}
3268 	return err;
3269 }
3270 
3271 
3272 /* check_stack_write dispatches to check_stack_write_fixed_off or
3273  * check_stack_write_var_off.
3274  *
3275  * 'ptr_regno' is the register used as a pointer into the stack.
3276  * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
3277  * 'value_regno' is the register whose value we're writing to the stack. It can
3278  * be -1, meaning that we're not writing from a register.
3279  *
3280  * The caller must ensure that the offset falls within the maximum stack size.
3281  */
3282 static int check_stack_write(struct bpf_verifier_env *env,
3283 			     int ptr_regno, int off, int size,
3284 			     int value_regno, int insn_idx)
3285 {
3286 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
3287 	struct bpf_func_state *state = func(env, reg);
3288 	int err;
3289 
3290 	if (tnum_is_const(reg->var_off)) {
3291 		off += reg->var_off.value;
3292 		err = check_stack_write_fixed_off(env, state, off, size,
3293 						  value_regno, insn_idx);
3294 	} else {
3295 		/* Variable offset stack reads need more conservative handling
3296 		 * than fixed offset ones.
3297 		 */
3298 		err = check_stack_write_var_off(env, state,
3299 						ptr_regno, off, size,
3300 						value_regno, insn_idx);
3301 	}
3302 	return err;
3303 }
3304 
3305 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
3306 				 int off, int size, enum bpf_access_type type)
3307 {
3308 	struct bpf_reg_state *regs = cur_regs(env);
3309 	struct bpf_map *map = regs[regno].map_ptr;
3310 	u32 cap = bpf_map_flags_to_cap(map);
3311 
3312 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
3313 		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
3314 			map->value_size, off, size);
3315 		return -EACCES;
3316 	}
3317 
3318 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
3319 		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
3320 			map->value_size, off, size);
3321 		return -EACCES;
3322 	}
3323 
3324 	return 0;
3325 }
3326 
3327 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
3328 static int __check_mem_access(struct bpf_verifier_env *env, int regno,
3329 			      int off, int size, u32 mem_size,
3330 			      bool zero_size_allowed)
3331 {
3332 	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
3333 	struct bpf_reg_state *reg;
3334 
3335 	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
3336 		return 0;
3337 
3338 	reg = &cur_regs(env)[regno];
3339 	switch (reg->type) {
3340 	case PTR_TO_MAP_KEY:
3341 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
3342 			mem_size, off, size);
3343 		break;
3344 	case PTR_TO_MAP_VALUE:
3345 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
3346 			mem_size, off, size);
3347 		break;
3348 	case PTR_TO_PACKET:
3349 	case PTR_TO_PACKET_META:
3350 	case PTR_TO_PACKET_END:
3351 		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
3352 			off, size, regno, reg->id, off, mem_size);
3353 		break;
3354 	case PTR_TO_MEM:
3355 	default:
3356 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
3357 			mem_size, off, size);
3358 	}
3359 
3360 	return -EACCES;
3361 }
3362 
3363 /* check read/write into a memory region with possible variable offset */
3364 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
3365 				   int off, int size, u32 mem_size,
3366 				   bool zero_size_allowed)
3367 {
3368 	struct bpf_verifier_state *vstate = env->cur_state;
3369 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3370 	struct bpf_reg_state *reg = &state->regs[regno];
3371 	int err;
3372 
3373 	/* We may have adjusted the register pointing to memory region, so we
3374 	 * need to try adding each of min_value and max_value to off
3375 	 * to make sure our theoretical access will be safe.
3376 	 */
3377 	if (env->log.level & BPF_LOG_LEVEL)
3378 		print_verifier_state(env, state);
3379 
3380 	/* The minimum value is only important with signed
3381 	 * comparisons where we can't assume the floor of a
3382 	 * value is 0.  If we are using signed variables for our
3383 	 * index'es we need to make sure that whatever we use
3384 	 * will have a set floor within our range.
3385 	 */
3386 	if (reg->smin_value < 0 &&
3387 	    (reg->smin_value == S64_MIN ||
3388 	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
3389 	      reg->smin_value + off < 0)) {
3390 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3391 			regno);
3392 		return -EACCES;
3393 	}
3394 	err = __check_mem_access(env, regno, reg->smin_value + off, size,
3395 				 mem_size, zero_size_allowed);
3396 	if (err) {
3397 		verbose(env, "R%d min value is outside of the allowed memory range\n",
3398 			regno);
3399 		return err;
3400 	}
3401 
3402 	/* If we haven't set a max value then we need to bail since we can't be
3403 	 * sure we won't do bad things.
3404 	 * If reg->umax_value + off could overflow, treat that as unbounded too.
3405 	 */
3406 	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
3407 		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
3408 			regno);
3409 		return -EACCES;
3410 	}
3411 	err = __check_mem_access(env, regno, reg->umax_value + off, size,
3412 				 mem_size, zero_size_allowed);
3413 	if (err) {
3414 		verbose(env, "R%d max value is outside of the allowed memory range\n",
3415 			regno);
3416 		return err;
3417 	}
3418 
3419 	return 0;
3420 }
3421 
3422 /* check read/write into a map element with possible variable offset */
3423 static int check_map_access(struct bpf_verifier_env *env, u32 regno,
3424 			    int off, int size, bool zero_size_allowed)
3425 {
3426 	struct bpf_verifier_state *vstate = env->cur_state;
3427 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3428 	struct bpf_reg_state *reg = &state->regs[regno];
3429 	struct bpf_map *map = reg->map_ptr;
3430 	int err;
3431 
3432 	err = check_mem_region_access(env, regno, off, size, map->value_size,
3433 				      zero_size_allowed);
3434 	if (err)
3435 		return err;
3436 
3437 	if (map_value_has_spin_lock(map)) {
3438 		u32 lock = map->spin_lock_off;
3439 
3440 		/* if any part of struct bpf_spin_lock can be touched by
3441 		 * load/store reject this program.
3442 		 * To check that [x1, x2) overlaps with [y1, y2)
3443 		 * it is sufficient to check x1 < y2 && y1 < x2.
3444 		 */
3445 		if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
3446 		     lock < reg->umax_value + off + size) {
3447 			verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
3448 			return -EACCES;
3449 		}
3450 	}
3451 	if (map_value_has_timer(map)) {
3452 		u32 t = map->timer_off;
3453 
3454 		if (reg->smin_value + off < t + sizeof(struct bpf_timer) &&
3455 		     t < reg->umax_value + off + size) {
3456 			verbose(env, "bpf_timer cannot be accessed directly by load/store\n");
3457 			return -EACCES;
3458 		}
3459 	}
3460 	return err;
3461 }
3462 
3463 #define MAX_PACKET_OFF 0xffff
3464 
3465 static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
3466 {
3467 	return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type;
3468 }
3469 
3470 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
3471 				       const struct bpf_call_arg_meta *meta,
3472 				       enum bpf_access_type t)
3473 {
3474 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
3475 
3476 	switch (prog_type) {
3477 	/* Program types only with direct read access go here! */
3478 	case BPF_PROG_TYPE_LWT_IN:
3479 	case BPF_PROG_TYPE_LWT_OUT:
3480 	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
3481 	case BPF_PROG_TYPE_SK_REUSEPORT:
3482 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
3483 	case BPF_PROG_TYPE_CGROUP_SKB:
3484 		if (t == BPF_WRITE)
3485 			return false;
3486 		fallthrough;
3487 
3488 	/* Program types with direct read + write access go here! */
3489 	case BPF_PROG_TYPE_SCHED_CLS:
3490 	case BPF_PROG_TYPE_SCHED_ACT:
3491 	case BPF_PROG_TYPE_XDP:
3492 	case BPF_PROG_TYPE_LWT_XMIT:
3493 	case BPF_PROG_TYPE_SK_SKB:
3494 	case BPF_PROG_TYPE_SK_MSG:
3495 		if (meta)
3496 			return meta->pkt_access;
3497 
3498 		env->seen_direct_write = true;
3499 		return true;
3500 
3501 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3502 		if (t == BPF_WRITE)
3503 			env->seen_direct_write = true;
3504 
3505 		return true;
3506 
3507 	default:
3508 		return false;
3509 	}
3510 }
3511 
3512 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
3513 			       int size, bool zero_size_allowed)
3514 {
3515 	struct bpf_reg_state *regs = cur_regs(env);
3516 	struct bpf_reg_state *reg = &regs[regno];
3517 	int err;
3518 
3519 	/* We may have added a variable offset to the packet pointer; but any
3520 	 * reg->range we have comes after that.  We are only checking the fixed
3521 	 * offset.
3522 	 */
3523 
3524 	/* We don't allow negative numbers, because we aren't tracking enough
3525 	 * detail to prove they're safe.
3526 	 */
3527 	if (reg->smin_value < 0) {
3528 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3529 			regno);
3530 		return -EACCES;
3531 	}
3532 
3533 	err = reg->range < 0 ? -EINVAL :
3534 	      __check_mem_access(env, regno, off, size, reg->range,
3535 				 zero_size_allowed);
3536 	if (err) {
3537 		verbose(env, "R%d offset is outside of the packet\n", regno);
3538 		return err;
3539 	}
3540 
3541 	/* __check_mem_access has made sure "off + size - 1" is within u16.
3542 	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
3543 	 * otherwise find_good_pkt_pointers would have refused to set range info
3544 	 * that __check_mem_access would have rejected this pkt access.
3545 	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
3546 	 */
3547 	env->prog->aux->max_pkt_offset =
3548 		max_t(u32, env->prog->aux->max_pkt_offset,
3549 		      off + reg->umax_value + size - 1);
3550 
3551 	return err;
3552 }
3553 
3554 /* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
3555 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
3556 			    enum bpf_access_type t, enum bpf_reg_type *reg_type,
3557 			    struct btf **btf, u32 *btf_id)
3558 {
3559 	struct bpf_insn_access_aux info = {
3560 		.reg_type = *reg_type,
3561 		.log = &env->log,
3562 	};
3563 
3564 	if (env->ops->is_valid_access &&
3565 	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {
3566 		/* A non zero info.ctx_field_size indicates that this field is a
3567 		 * candidate for later verifier transformation to load the whole
3568 		 * field and then apply a mask when accessed with a narrower
3569 		 * access than actual ctx access size. A zero info.ctx_field_size
3570 		 * will only allow for whole field access and rejects any other
3571 		 * type of narrower access.
3572 		 */
3573 		*reg_type = info.reg_type;
3574 
3575 		if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) {
3576 			*btf = info.btf;
3577 			*btf_id = info.btf_id;
3578 		} else {
3579 			env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
3580 		}
3581 		/* remember the offset of last byte accessed in ctx */
3582 		if (env->prog->aux->max_ctx_offset < off + size)
3583 			env->prog->aux->max_ctx_offset = off + size;
3584 		return 0;
3585 	}
3586 
3587 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
3588 	return -EACCES;
3589 }
3590 
3591 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
3592 				  int size)
3593 {
3594 	if (size < 0 || off < 0 ||
3595 	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
3596 		verbose(env, "invalid access to flow keys off=%d size=%d\n",
3597 			off, size);
3598 		return -EACCES;
3599 	}
3600 	return 0;
3601 }
3602 
3603 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
3604 			     u32 regno, int off, int size,
3605 			     enum bpf_access_type t)
3606 {
3607 	struct bpf_reg_state *regs = cur_regs(env);
3608 	struct bpf_reg_state *reg = &regs[regno];
3609 	struct bpf_insn_access_aux info = {};
3610 	bool valid;
3611 
3612 	if (reg->smin_value < 0) {
3613 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3614 			regno);
3615 		return -EACCES;
3616 	}
3617 
3618 	switch (reg->type) {
3619 	case PTR_TO_SOCK_COMMON:
3620 		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
3621 		break;
3622 	case PTR_TO_SOCKET:
3623 		valid = bpf_sock_is_valid_access(off, size, t, &info);
3624 		break;
3625 	case PTR_TO_TCP_SOCK:
3626 		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
3627 		break;
3628 	case PTR_TO_XDP_SOCK:
3629 		valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
3630 		break;
3631 	default:
3632 		valid = false;
3633 	}
3634 
3635 
3636 	if (valid) {
3637 		env->insn_aux_data[insn_idx].ctx_field_size =
3638 			info.ctx_field_size;
3639 		return 0;
3640 	}
3641 
3642 	verbose(env, "R%d invalid %s access off=%d size=%d\n",
3643 		regno, reg_type_str[reg->type], off, size);
3644 
3645 	return -EACCES;
3646 }
3647 
3648 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
3649 {
3650 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
3651 }
3652 
3653 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
3654 {
3655 	const struct bpf_reg_state *reg = reg_state(env, regno);
3656 
3657 	return reg->type == PTR_TO_CTX;
3658 }
3659 
3660 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
3661 {
3662 	const struct bpf_reg_state *reg = reg_state(env, regno);
3663 
3664 	return type_is_sk_pointer(reg->type);
3665 }
3666 
3667 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
3668 {
3669 	const struct bpf_reg_state *reg = reg_state(env, regno);
3670 
3671 	return type_is_pkt_pointer(reg->type);
3672 }
3673 
3674 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
3675 {
3676 	const struct bpf_reg_state *reg = reg_state(env, regno);
3677 
3678 	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
3679 	return reg->type == PTR_TO_FLOW_KEYS;
3680 }
3681 
3682 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
3683 				   const struct bpf_reg_state *reg,
3684 				   int off, int size, bool strict)
3685 {
3686 	struct tnum reg_off;
3687 	int ip_align;
3688 
3689 	/* Byte size accesses are always allowed. */
3690 	if (!strict || size == 1)
3691 		return 0;
3692 
3693 	/* For platforms that do not have a Kconfig enabling
3694 	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
3695 	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
3696 	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
3697 	 * to this code only in strict mode where we want to emulate
3698 	 * the NET_IP_ALIGN==2 checking.  Therefore use an
3699 	 * unconditional IP align value of '2'.
3700 	 */
3701 	ip_align = 2;
3702 
3703 	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
3704 	if (!tnum_is_aligned(reg_off, size)) {
3705 		char tn_buf[48];
3706 
3707 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3708 		verbose(env,
3709 			"misaligned packet access off %d+%s+%d+%d size %d\n",
3710 			ip_align, tn_buf, reg->off, off, size);
3711 		return -EACCES;
3712 	}
3713 
3714 	return 0;
3715 }
3716 
3717 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
3718 				       const struct bpf_reg_state *reg,
3719 				       const char *pointer_desc,
3720 				       int off, int size, bool strict)
3721 {
3722 	struct tnum reg_off;
3723 
3724 	/* Byte size accesses are always allowed. */
3725 	if (!strict || size == 1)
3726 		return 0;
3727 
3728 	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
3729 	if (!tnum_is_aligned(reg_off, size)) {
3730 		char tn_buf[48];
3731 
3732 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3733 		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
3734 			pointer_desc, tn_buf, reg->off, off, size);
3735 		return -EACCES;
3736 	}
3737 
3738 	return 0;
3739 }
3740 
3741 static int check_ptr_alignment(struct bpf_verifier_env *env,
3742 			       const struct bpf_reg_state *reg, int off,
3743 			       int size, bool strict_alignment_once)
3744 {
3745 	bool strict = env->strict_alignment || strict_alignment_once;
3746 	const char *pointer_desc = "";
3747 
3748 	switch (reg->type) {
3749 	case PTR_TO_PACKET:
3750 	case PTR_TO_PACKET_META:
3751 		/* Special case, because of NET_IP_ALIGN. Given metadata sits
3752 		 * right in front, treat it the very same way.
3753 		 */
3754 		return check_pkt_ptr_alignment(env, reg, off, size, strict);
3755 	case PTR_TO_FLOW_KEYS:
3756 		pointer_desc = "flow keys ";
3757 		break;
3758 	case PTR_TO_MAP_KEY:
3759 		pointer_desc = "key ";
3760 		break;
3761 	case PTR_TO_MAP_VALUE:
3762 		pointer_desc = "value ";
3763 		break;
3764 	case PTR_TO_CTX:
3765 		pointer_desc = "context ";
3766 		break;
3767 	case PTR_TO_STACK:
3768 		pointer_desc = "stack ";
3769 		/* The stack spill tracking logic in check_stack_write_fixed_off()
3770 		 * and check_stack_read_fixed_off() relies on stack accesses being
3771 		 * aligned.
3772 		 */
3773 		strict = true;
3774 		break;
3775 	case PTR_TO_SOCKET:
3776 		pointer_desc = "sock ";
3777 		break;
3778 	case PTR_TO_SOCK_COMMON:
3779 		pointer_desc = "sock_common ";
3780 		break;
3781 	case PTR_TO_TCP_SOCK:
3782 		pointer_desc = "tcp_sock ";
3783 		break;
3784 	case PTR_TO_XDP_SOCK:
3785 		pointer_desc = "xdp_sock ";
3786 		break;
3787 	default:
3788 		break;
3789 	}
3790 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
3791 					   strict);
3792 }
3793 
3794 static int update_stack_depth(struct bpf_verifier_env *env,
3795 			      const struct bpf_func_state *func,
3796 			      int off)
3797 {
3798 	u16 stack = env->subprog_info[func->subprogno].stack_depth;
3799 
3800 	if (stack >= -off)
3801 		return 0;
3802 
3803 	/* update known max for given subprogram */
3804 	env->subprog_info[func->subprogno].stack_depth = -off;
3805 	return 0;
3806 }
3807 
3808 /* starting from main bpf function walk all instructions of the function
3809  * and recursively walk all callees that given function can call.
3810  * Ignore jump and exit insns.
3811  * Since recursion is prevented by check_cfg() this algorithm
3812  * only needs a local stack of MAX_CALL_FRAMES to remember callsites
3813  */
3814 static int check_max_stack_depth(struct bpf_verifier_env *env)
3815 {
3816 	int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
3817 	struct bpf_subprog_info *subprog = env->subprog_info;
3818 	struct bpf_insn *insn = env->prog->insnsi;
3819 	bool tail_call_reachable = false;
3820 	int ret_insn[MAX_CALL_FRAMES];
3821 	int ret_prog[MAX_CALL_FRAMES];
3822 	int j;
3823 
3824 process_func:
3825 	/* protect against potential stack overflow that might happen when
3826 	 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
3827 	 * depth for such case down to 256 so that the worst case scenario
3828 	 * would result in 8k stack size (32 which is tailcall limit * 256 =
3829 	 * 8k).
3830 	 *
3831 	 * To get the idea what might happen, see an example:
3832 	 * func1 -> sub rsp, 128
3833 	 *  subfunc1 -> sub rsp, 256
3834 	 *  tailcall1 -> add rsp, 256
3835 	 *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
3836 	 *   subfunc2 -> sub rsp, 64
3837 	 *   subfunc22 -> sub rsp, 128
3838 	 *   tailcall2 -> add rsp, 128
3839 	 *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
3840 	 *
3841 	 * tailcall will unwind the current stack frame but it will not get rid
3842 	 * of caller's stack as shown on the example above.
3843 	 */
3844 	if (idx && subprog[idx].has_tail_call && depth >= 256) {
3845 		verbose(env,
3846 			"tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
3847 			depth);
3848 		return -EACCES;
3849 	}
3850 	/* round up to 32-bytes, since this is granularity
3851 	 * of interpreter stack size
3852 	 */
3853 	depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
3854 	if (depth > MAX_BPF_STACK) {
3855 		verbose(env, "combined stack size of %d calls is %d. Too large\n",
3856 			frame + 1, depth);
3857 		return -EACCES;
3858 	}
3859 continue_func:
3860 	subprog_end = subprog[idx + 1].start;
3861 	for (; i < subprog_end; i++) {
3862 		int next_insn;
3863 
3864 		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
3865 			continue;
3866 		/* remember insn and function to return to */
3867 		ret_insn[frame] = i + 1;
3868 		ret_prog[frame] = idx;
3869 
3870 		/* find the callee */
3871 		next_insn = i + insn[i].imm + 1;
3872 		idx = find_subprog(env, next_insn);
3873 		if (idx < 0) {
3874 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
3875 				  next_insn);
3876 			return -EFAULT;
3877 		}
3878 		if (subprog[idx].is_async_cb) {
3879 			if (subprog[idx].has_tail_call) {
3880 				verbose(env, "verifier bug. subprog has tail_call and async cb\n");
3881 				return -EFAULT;
3882 			}
3883 			 /* async callbacks don't increase bpf prog stack size */
3884 			continue;
3885 		}
3886 		i = next_insn;
3887 
3888 		if (subprog[idx].has_tail_call)
3889 			tail_call_reachable = true;
3890 
3891 		frame++;
3892 		if (frame >= MAX_CALL_FRAMES) {
3893 			verbose(env, "the call stack of %d frames is too deep !\n",
3894 				frame);
3895 			return -E2BIG;
3896 		}
3897 		goto process_func;
3898 	}
3899 	/* if tail call got detected across bpf2bpf calls then mark each of the
3900 	 * currently present subprog frames as tail call reachable subprogs;
3901 	 * this info will be utilized by JIT so that we will be preserving the
3902 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
3903 	 */
3904 	if (tail_call_reachable)
3905 		for (j = 0; j < frame; j++)
3906 			subprog[ret_prog[j]].tail_call_reachable = true;
3907 	if (subprog[0].tail_call_reachable)
3908 		env->prog->aux->tail_call_reachable = true;
3909 
3910 	/* end of for() loop means the last insn of the 'subprog'
3911 	 * was reached. Doesn't matter whether it was JA or EXIT
3912 	 */
3913 	if (frame == 0)
3914 		return 0;
3915 	depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
3916 	frame--;
3917 	i = ret_insn[frame];
3918 	idx = ret_prog[frame];
3919 	goto continue_func;
3920 }
3921 
3922 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
3923 static int get_callee_stack_depth(struct bpf_verifier_env *env,
3924 				  const struct bpf_insn *insn, int idx)
3925 {
3926 	int start = idx + insn->imm + 1, subprog;
3927 
3928 	subprog = find_subprog(env, start);
3929 	if (subprog < 0) {
3930 		WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
3931 			  start);
3932 		return -EFAULT;
3933 	}
3934 	return env->subprog_info[subprog].stack_depth;
3935 }
3936 #endif
3937 
3938 int check_ctx_reg(struct bpf_verifier_env *env,
3939 		  const struct bpf_reg_state *reg, int regno)
3940 {
3941 	/* Access to ctx or passing it to a helper is only allowed in
3942 	 * its original, unmodified form.
3943 	 */
3944 
3945 	if (reg->off) {
3946 		verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
3947 			regno, reg->off);
3948 		return -EACCES;
3949 	}
3950 
3951 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
3952 		char tn_buf[48];
3953 
3954 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3955 		verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
3956 		return -EACCES;
3957 	}
3958 
3959 	return 0;
3960 }
3961 
3962 static int __check_buffer_access(struct bpf_verifier_env *env,
3963 				 const char *buf_info,
3964 				 const struct bpf_reg_state *reg,
3965 				 int regno, int off, int size)
3966 {
3967 	if (off < 0) {
3968 		verbose(env,
3969 			"R%d invalid %s buffer access: off=%d, size=%d\n",
3970 			regno, buf_info, off, size);
3971 		return -EACCES;
3972 	}
3973 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
3974 		char tn_buf[48];
3975 
3976 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3977 		verbose(env,
3978 			"R%d invalid variable buffer offset: off=%d, var_off=%s\n",
3979 			regno, off, tn_buf);
3980 		return -EACCES;
3981 	}
3982 
3983 	return 0;
3984 }
3985 
3986 static int check_tp_buffer_access(struct bpf_verifier_env *env,
3987 				  const struct bpf_reg_state *reg,
3988 				  int regno, int off, int size)
3989 {
3990 	int err;
3991 
3992 	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
3993 	if (err)
3994 		return err;
3995 
3996 	if (off + size > env->prog->aux->max_tp_access)
3997 		env->prog->aux->max_tp_access = off + size;
3998 
3999 	return 0;
4000 }
4001 
4002 static int check_buffer_access(struct bpf_verifier_env *env,
4003 			       const struct bpf_reg_state *reg,
4004 			       int regno, int off, int size,
4005 			       bool zero_size_allowed,
4006 			       const char *buf_info,
4007 			       u32 *max_access)
4008 {
4009 	int err;
4010 
4011 	err = __check_buffer_access(env, buf_info, reg, regno, off, size);
4012 	if (err)
4013 		return err;
4014 
4015 	if (off + size > *max_access)
4016 		*max_access = off + size;
4017 
4018 	return 0;
4019 }
4020 
4021 /* BPF architecture zero extends alu32 ops into 64-bit registesr */
4022 static void zext_32_to_64(struct bpf_reg_state *reg)
4023 {
4024 	reg->var_off = tnum_subreg(reg->var_off);
4025 	__reg_assign_32_into_64(reg);
4026 }
4027 
4028 /* truncate register to smaller size (in bytes)
4029  * must be called with size < BPF_REG_SIZE
4030  */
4031 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
4032 {
4033 	u64 mask;
4034 
4035 	/* clear high bits in bit representation */
4036 	reg->var_off = tnum_cast(reg->var_off, size);
4037 
4038 	/* fix arithmetic bounds */
4039 	mask = ((u64)1 << (size * 8)) - 1;
4040 	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
4041 		reg->umin_value &= mask;
4042 		reg->umax_value &= mask;
4043 	} else {
4044 		reg->umin_value = 0;
4045 		reg->umax_value = mask;
4046 	}
4047 	reg->smin_value = reg->umin_value;
4048 	reg->smax_value = reg->umax_value;
4049 
4050 	/* If size is smaller than 32bit register the 32bit register
4051 	 * values are also truncated so we push 64-bit bounds into
4052 	 * 32-bit bounds. Above were truncated < 32-bits already.
4053 	 */
4054 	if (size >= 4)
4055 		return;
4056 	__reg_combine_64_into_32(reg);
4057 }
4058 
4059 static bool bpf_map_is_rdonly(const struct bpf_map *map)
4060 {
4061 	/* A map is considered read-only if the following condition are true:
4062 	 *
4063 	 * 1) BPF program side cannot change any of the map content. The
4064 	 *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
4065 	 *    and was set at map creation time.
4066 	 * 2) The map value(s) have been initialized from user space by a
4067 	 *    loader and then "frozen", such that no new map update/delete
4068 	 *    operations from syscall side are possible for the rest of
4069 	 *    the map's lifetime from that point onwards.
4070 	 * 3) Any parallel/pending map update/delete operations from syscall
4071 	 *    side have been completed. Only after that point, it's safe to
4072 	 *    assume that map value(s) are immutable.
4073 	 */
4074 	return (map->map_flags & BPF_F_RDONLY_PROG) &&
4075 	       READ_ONCE(map->frozen) &&
4076 	       !bpf_map_write_active(map);
4077 }
4078 
4079 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val)
4080 {
4081 	void *ptr;
4082 	u64 addr;
4083 	int err;
4084 
4085 	err = map->ops->map_direct_value_addr(map, &addr, off);
4086 	if (err)
4087 		return err;
4088 	ptr = (void *)(long)addr + off;
4089 
4090 	switch (size) {
4091 	case sizeof(u8):
4092 		*val = (u64)*(u8 *)ptr;
4093 		break;
4094 	case sizeof(u16):
4095 		*val = (u64)*(u16 *)ptr;
4096 		break;
4097 	case sizeof(u32):
4098 		*val = (u64)*(u32 *)ptr;
4099 		break;
4100 	case sizeof(u64):
4101 		*val = *(u64 *)ptr;
4102 		break;
4103 	default:
4104 		return -EINVAL;
4105 	}
4106 	return 0;
4107 }
4108 
4109 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
4110 				   struct bpf_reg_state *regs,
4111 				   int regno, int off, int size,
4112 				   enum bpf_access_type atype,
4113 				   int value_regno)
4114 {
4115 	struct bpf_reg_state *reg = regs + regno;
4116 	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
4117 	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
4118 	u32 btf_id;
4119 	int ret;
4120 
4121 	if (off < 0) {
4122 		verbose(env,
4123 			"R%d is ptr_%s invalid negative access: off=%d\n",
4124 			regno, tname, off);
4125 		return -EACCES;
4126 	}
4127 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
4128 		char tn_buf[48];
4129 
4130 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4131 		verbose(env,
4132 			"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
4133 			regno, tname, off, tn_buf);
4134 		return -EACCES;
4135 	}
4136 
4137 	if (env->ops->btf_struct_access) {
4138 		ret = env->ops->btf_struct_access(&env->log, reg->btf, t,
4139 						  off, size, atype, &btf_id);
4140 	} else {
4141 		if (atype != BPF_READ) {
4142 			verbose(env, "only read is supported\n");
4143 			return -EACCES;
4144 		}
4145 
4146 		ret = btf_struct_access(&env->log, reg->btf, t, off, size,
4147 					atype, &btf_id);
4148 	}
4149 
4150 	if (ret < 0)
4151 		return ret;
4152 
4153 	if (atype == BPF_READ && value_regno >= 0)
4154 		mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id);
4155 
4156 	return 0;
4157 }
4158 
4159 static int check_ptr_to_map_access(struct bpf_verifier_env *env,
4160 				   struct bpf_reg_state *regs,
4161 				   int regno, int off, int size,
4162 				   enum bpf_access_type atype,
4163 				   int value_regno)
4164 {
4165 	struct bpf_reg_state *reg = regs + regno;
4166 	struct bpf_map *map = reg->map_ptr;
4167 	const struct btf_type *t;
4168 	const char *tname;
4169 	u32 btf_id;
4170 	int ret;
4171 
4172 	if (!btf_vmlinux) {
4173 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
4174 		return -ENOTSUPP;
4175 	}
4176 
4177 	if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
4178 		verbose(env, "map_ptr access not supported for map type %d\n",
4179 			map->map_type);
4180 		return -ENOTSUPP;
4181 	}
4182 
4183 	t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
4184 	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
4185 
4186 	if (!env->allow_ptr_to_map_access) {
4187 		verbose(env,
4188 			"%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
4189 			tname);
4190 		return -EPERM;
4191 	}
4192 
4193 	if (off < 0) {
4194 		verbose(env, "R%d is %s invalid negative access: off=%d\n",
4195 			regno, tname, off);
4196 		return -EACCES;
4197 	}
4198 
4199 	if (atype != BPF_READ) {
4200 		verbose(env, "only read from %s is supported\n", tname);
4201 		return -EACCES;
4202 	}
4203 
4204 	ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id);
4205 	if (ret < 0)
4206 		return ret;
4207 
4208 	if (value_regno >= 0)
4209 		mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id);
4210 
4211 	return 0;
4212 }
4213 
4214 /* Check that the stack access at the given offset is within bounds. The
4215  * maximum valid offset is -1.
4216  *
4217  * The minimum valid offset is -MAX_BPF_STACK for writes, and
4218  * -state->allocated_stack for reads.
4219  */
4220 static int check_stack_slot_within_bounds(int off,
4221 					  struct bpf_func_state *state,
4222 					  enum bpf_access_type t)
4223 {
4224 	int min_valid_off;
4225 
4226 	if (t == BPF_WRITE)
4227 		min_valid_off = -MAX_BPF_STACK;
4228 	else
4229 		min_valid_off = -state->allocated_stack;
4230 
4231 	if (off < min_valid_off || off > -1)
4232 		return -EACCES;
4233 	return 0;
4234 }
4235 
4236 /* Check that the stack access at 'regno + off' falls within the maximum stack
4237  * bounds.
4238  *
4239  * 'off' includes `regno->offset`, but not its dynamic part (if any).
4240  */
4241 static int check_stack_access_within_bounds(
4242 		struct bpf_verifier_env *env,
4243 		int regno, int off, int access_size,
4244 		enum stack_access_src src, enum bpf_access_type type)
4245 {
4246 	struct bpf_reg_state *regs = cur_regs(env);
4247 	struct bpf_reg_state *reg = regs + regno;
4248 	struct bpf_func_state *state = func(env, reg);
4249 	int min_off, max_off;
4250 	int err;
4251 	char *err_extra;
4252 
4253 	if (src == ACCESS_HELPER)
4254 		/* We don't know if helpers are reading or writing (or both). */
4255 		err_extra = " indirect access to";
4256 	else if (type == BPF_READ)
4257 		err_extra = " read from";
4258 	else
4259 		err_extra = " write to";
4260 
4261 	if (tnum_is_const(reg->var_off)) {
4262 		min_off = reg->var_off.value + off;
4263 		if (access_size > 0)
4264 			max_off = min_off + access_size - 1;
4265 		else
4266 			max_off = min_off;
4267 	} else {
4268 		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
4269 		    reg->smin_value <= -BPF_MAX_VAR_OFF) {
4270 			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
4271 				err_extra, regno);
4272 			return -EACCES;
4273 		}
4274 		min_off = reg->smin_value + off;
4275 		if (access_size > 0)
4276 			max_off = reg->smax_value + off + access_size - 1;
4277 		else
4278 			max_off = min_off;
4279 	}
4280 
4281 	err = check_stack_slot_within_bounds(min_off, state, type);
4282 	if (!err)
4283 		err = check_stack_slot_within_bounds(max_off, state, type);
4284 
4285 	if (err) {
4286 		if (tnum_is_const(reg->var_off)) {
4287 			verbose(env, "invalid%s stack R%d off=%d size=%d\n",
4288 				err_extra, regno, off, access_size);
4289 		} else {
4290 			char tn_buf[48];
4291 
4292 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4293 			verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
4294 				err_extra, regno, tn_buf, access_size);
4295 		}
4296 	}
4297 	return err;
4298 }
4299 
4300 /* check whether memory at (regno + off) is accessible for t = (read | write)
4301  * if t==write, value_regno is a register which value is stored into memory
4302  * if t==read, value_regno is a register which will receive the value from memory
4303  * if t==write && value_regno==-1, some unknown value is stored into memory
4304  * if t==read && value_regno==-1, don't care what we read from memory
4305  */
4306 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
4307 			    int off, int bpf_size, enum bpf_access_type t,
4308 			    int value_regno, bool strict_alignment_once)
4309 {
4310 	struct bpf_reg_state *regs = cur_regs(env);
4311 	struct bpf_reg_state *reg = regs + regno;
4312 	struct bpf_func_state *state;
4313 	int size, err = 0;
4314 
4315 	size = bpf_size_to_bytes(bpf_size);
4316 	if (size < 0)
4317 		return size;
4318 
4319 	/* alignment checks will add in reg->off themselves */
4320 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
4321 	if (err)
4322 		return err;
4323 
4324 	/* for access checks, reg->off is just part of off */
4325 	off += reg->off;
4326 
4327 	if (reg->type == PTR_TO_MAP_KEY) {
4328 		if (t == BPF_WRITE) {
4329 			verbose(env, "write to change key R%d not allowed\n", regno);
4330 			return -EACCES;
4331 		}
4332 
4333 		err = check_mem_region_access(env, regno, off, size,
4334 					      reg->map_ptr->key_size, false);
4335 		if (err)
4336 			return err;
4337 		if (value_regno >= 0)
4338 			mark_reg_unknown(env, regs, value_regno);
4339 	} else if (reg->type == PTR_TO_MAP_VALUE) {
4340 		if (t == BPF_WRITE && value_regno >= 0 &&
4341 		    is_pointer_value(env, value_regno)) {
4342 			verbose(env, "R%d leaks addr into map\n", value_regno);
4343 			return -EACCES;
4344 		}
4345 		err = check_map_access_type(env, regno, off, size, t);
4346 		if (err)
4347 			return err;
4348 		err = check_map_access(env, regno, off, size, false);
4349 		if (!err && t == BPF_READ && value_regno >= 0) {
4350 			struct bpf_map *map = reg->map_ptr;
4351 
4352 			/* if map is read-only, track its contents as scalars */
4353 			if (tnum_is_const(reg->var_off) &&
4354 			    bpf_map_is_rdonly(map) &&
4355 			    map->ops->map_direct_value_addr) {
4356 				int map_off = off + reg->var_off.value;
4357 				u64 val = 0;
4358 
4359 				err = bpf_map_direct_read(map, map_off, size,
4360 							  &val);
4361 				if (err)
4362 					return err;
4363 
4364 				regs[value_regno].type = SCALAR_VALUE;
4365 				__mark_reg_known(&regs[value_regno], val);
4366 			} else {
4367 				mark_reg_unknown(env, regs, value_regno);
4368 			}
4369 		}
4370 	} else if (reg->type == PTR_TO_MEM) {
4371 		if (t == BPF_WRITE && value_regno >= 0 &&
4372 		    is_pointer_value(env, value_regno)) {
4373 			verbose(env, "R%d leaks addr into mem\n", value_regno);
4374 			return -EACCES;
4375 		}
4376 		err = check_mem_region_access(env, regno, off, size,
4377 					      reg->mem_size, false);
4378 		if (!err && t == BPF_READ && value_regno >= 0)
4379 			mark_reg_unknown(env, regs, value_regno);
4380 	} else if (reg->type == PTR_TO_CTX) {
4381 		enum bpf_reg_type reg_type = SCALAR_VALUE;
4382 		struct btf *btf = NULL;
4383 		u32 btf_id = 0;
4384 
4385 		if (t == BPF_WRITE && value_regno >= 0 &&
4386 		    is_pointer_value(env, value_regno)) {
4387 			verbose(env, "R%d leaks addr into ctx\n", value_regno);
4388 			return -EACCES;
4389 		}
4390 
4391 		err = check_ctx_reg(env, reg, regno);
4392 		if (err < 0)
4393 			return err;
4394 
4395 		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf, &btf_id);
4396 		if (err)
4397 			verbose_linfo(env, insn_idx, "; ");
4398 		if (!err && t == BPF_READ && value_regno >= 0) {
4399 			/* ctx access returns either a scalar, or a
4400 			 * PTR_TO_PACKET[_META,_END]. In the latter
4401 			 * case, we know the offset is zero.
4402 			 */
4403 			if (reg_type == SCALAR_VALUE) {
4404 				mark_reg_unknown(env, regs, value_regno);
4405 			} else {
4406 				mark_reg_known_zero(env, regs,
4407 						    value_regno);
4408 				if (reg_type_may_be_null(reg_type))
4409 					regs[value_regno].id = ++env->id_gen;
4410 				/* A load of ctx field could have different
4411 				 * actual load size with the one encoded in the
4412 				 * insn. When the dst is PTR, it is for sure not
4413 				 * a sub-register.
4414 				 */
4415 				regs[value_regno].subreg_def = DEF_NOT_SUBREG;
4416 				if (reg_type == PTR_TO_BTF_ID ||
4417 				    reg_type == PTR_TO_BTF_ID_OR_NULL) {
4418 					regs[value_regno].btf = btf;
4419 					regs[value_regno].btf_id = btf_id;
4420 				}
4421 			}
4422 			regs[value_regno].type = reg_type;
4423 		}
4424 
4425 	} else if (reg->type == PTR_TO_STACK) {
4426 		/* Basic bounds checks. */
4427 		err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
4428 		if (err)
4429 			return err;
4430 
4431 		state = func(env, reg);
4432 		err = update_stack_depth(env, state, off);
4433 		if (err)
4434 			return err;
4435 
4436 		if (t == BPF_READ)
4437 			err = check_stack_read(env, regno, off, size,
4438 					       value_regno);
4439 		else
4440 			err = check_stack_write(env, regno, off, size,
4441 						value_regno, insn_idx);
4442 	} else if (reg_is_pkt_pointer(reg)) {
4443 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
4444 			verbose(env, "cannot write into packet\n");
4445 			return -EACCES;
4446 		}
4447 		if (t == BPF_WRITE && value_regno >= 0 &&
4448 		    is_pointer_value(env, value_regno)) {
4449 			verbose(env, "R%d leaks addr into packet\n",
4450 				value_regno);
4451 			return -EACCES;
4452 		}
4453 		err = check_packet_access(env, regno, off, size, false);
4454 		if (!err && t == BPF_READ && value_regno >= 0)
4455 			mark_reg_unknown(env, regs, value_regno);
4456 	} else if (reg->type == PTR_TO_FLOW_KEYS) {
4457 		if (t == BPF_WRITE && value_regno >= 0 &&
4458 		    is_pointer_value(env, value_regno)) {
4459 			verbose(env, "R%d leaks addr into flow keys\n",
4460 				value_regno);
4461 			return -EACCES;
4462 		}
4463 
4464 		err = check_flow_keys_access(env, off, size);
4465 		if (!err && t == BPF_READ && value_regno >= 0)
4466 			mark_reg_unknown(env, regs, value_regno);
4467 	} else if (type_is_sk_pointer(reg->type)) {
4468 		if (t == BPF_WRITE) {
4469 			verbose(env, "R%d cannot write into %s\n",
4470 				regno, reg_type_str[reg->type]);
4471 			return -EACCES;
4472 		}
4473 		err = check_sock_access(env, insn_idx, regno, off, size, t);
4474 		if (!err && value_regno >= 0)
4475 			mark_reg_unknown(env, regs, value_regno);
4476 	} else if (reg->type == PTR_TO_TP_BUFFER) {
4477 		err = check_tp_buffer_access(env, reg, regno, off, size);
4478 		if (!err && t == BPF_READ && value_regno >= 0)
4479 			mark_reg_unknown(env, regs, value_regno);
4480 	} else if (reg->type == PTR_TO_BTF_ID) {
4481 		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
4482 					      value_regno);
4483 	} else if (reg->type == CONST_PTR_TO_MAP) {
4484 		err = check_ptr_to_map_access(env, regs, regno, off, size, t,
4485 					      value_regno);
4486 	} else if (reg->type == PTR_TO_RDONLY_BUF) {
4487 		if (t == BPF_WRITE) {
4488 			verbose(env, "R%d cannot write into %s\n",
4489 				regno, reg_type_str[reg->type]);
4490 			return -EACCES;
4491 		}
4492 		err = check_buffer_access(env, reg, regno, off, size, false,
4493 					  "rdonly",
4494 					  &env->prog->aux->max_rdonly_access);
4495 		if (!err && value_regno >= 0)
4496 			mark_reg_unknown(env, regs, value_regno);
4497 	} else if (reg->type == PTR_TO_RDWR_BUF) {
4498 		err = check_buffer_access(env, reg, regno, off, size, false,
4499 					  "rdwr",
4500 					  &env->prog->aux->max_rdwr_access);
4501 		if (!err && t == BPF_READ && value_regno >= 0)
4502 			mark_reg_unknown(env, regs, value_regno);
4503 	} else {
4504 		verbose(env, "R%d invalid mem access '%s'\n", regno,
4505 			reg_type_str[reg->type]);
4506 		return -EACCES;
4507 	}
4508 
4509 	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
4510 	    regs[value_regno].type == SCALAR_VALUE) {
4511 		/* b/h/w load zero-extends, mark upper bits as known 0 */
4512 		coerce_reg_to_size(&regs[value_regno], size);
4513 	}
4514 	return err;
4515 }
4516 
4517 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
4518 {
4519 	int load_reg;
4520 	int err;
4521 
4522 	switch (insn->imm) {
4523 	case BPF_ADD:
4524 	case BPF_ADD | BPF_FETCH:
4525 	case BPF_AND:
4526 	case BPF_AND | BPF_FETCH:
4527 	case BPF_OR:
4528 	case BPF_OR | BPF_FETCH:
4529 	case BPF_XOR:
4530 	case BPF_XOR | BPF_FETCH:
4531 	case BPF_XCHG:
4532 	case BPF_CMPXCHG:
4533 		break;
4534 	default:
4535 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
4536 		return -EINVAL;
4537 	}
4538 
4539 	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
4540 		verbose(env, "invalid atomic operand size\n");
4541 		return -EINVAL;
4542 	}
4543 
4544 	/* check src1 operand */
4545 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
4546 	if (err)
4547 		return err;
4548 
4549 	/* check src2 operand */
4550 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4551 	if (err)
4552 		return err;
4553 
4554 	if (insn->imm == BPF_CMPXCHG) {
4555 		/* Check comparison of R0 with memory location */
4556 		const u32 aux_reg = BPF_REG_0;
4557 
4558 		err = check_reg_arg(env, aux_reg, SRC_OP);
4559 		if (err)
4560 			return err;
4561 
4562 		if (is_pointer_value(env, aux_reg)) {
4563 			verbose(env, "R%d leaks addr into mem\n", aux_reg);
4564 			return -EACCES;
4565 		}
4566 	}
4567 
4568 	if (is_pointer_value(env, insn->src_reg)) {
4569 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
4570 		return -EACCES;
4571 	}
4572 
4573 	if (is_ctx_reg(env, insn->dst_reg) ||
4574 	    is_pkt_reg(env, insn->dst_reg) ||
4575 	    is_flow_key_reg(env, insn->dst_reg) ||
4576 	    is_sk_reg(env, insn->dst_reg)) {
4577 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
4578 			insn->dst_reg,
4579 			reg_type_str[reg_state(env, insn->dst_reg)->type]);
4580 		return -EACCES;
4581 	}
4582 
4583 	if (insn->imm & BPF_FETCH) {
4584 		if (insn->imm == BPF_CMPXCHG)
4585 			load_reg = BPF_REG_0;
4586 		else
4587 			load_reg = insn->src_reg;
4588 
4589 		/* check and record load of old value */
4590 		err = check_reg_arg(env, load_reg, DST_OP);
4591 		if (err)
4592 			return err;
4593 	} else {
4594 		/* This instruction accesses a memory location but doesn't
4595 		 * actually load it into a register.
4596 		 */
4597 		load_reg = -1;
4598 	}
4599 
4600 	/* Check whether we can read the memory, with second call for fetch
4601 	 * case to simulate the register fill.
4602 	 */
4603 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
4604 			       BPF_SIZE(insn->code), BPF_READ, -1, true);
4605 	if (!err && load_reg >= 0)
4606 		err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
4607 				       BPF_SIZE(insn->code), BPF_READ, load_reg,
4608 				       true);
4609 	if (err)
4610 		return err;
4611 
4612 	/* Check whether we can write into the same memory. */
4613 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
4614 			       BPF_SIZE(insn->code), BPF_WRITE, -1, true);
4615 	if (err)
4616 		return err;
4617 
4618 	return 0;
4619 }
4620 
4621 /* When register 'regno' is used to read the stack (either directly or through
4622  * a helper function) make sure that it's within stack boundary and, depending
4623  * on the access type, that all elements of the stack are initialized.
4624  *
4625  * 'off' includes 'regno->off', but not its dynamic part (if any).
4626  *
4627  * All registers that have been spilled on the stack in the slots within the
4628  * read offsets are marked as read.
4629  */
4630 static int check_stack_range_initialized(
4631 		struct bpf_verifier_env *env, int regno, int off,
4632 		int access_size, bool zero_size_allowed,
4633 		enum stack_access_src type, struct bpf_call_arg_meta *meta)
4634 {
4635 	struct bpf_reg_state *reg = reg_state(env, regno);
4636 	struct bpf_func_state *state = func(env, reg);
4637 	int err, min_off, max_off, i, j, slot, spi;
4638 	char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
4639 	enum bpf_access_type bounds_check_type;
4640 	/* Some accesses can write anything into the stack, others are
4641 	 * read-only.
4642 	 */
4643 	bool clobber = false;
4644 
4645 	if (access_size == 0 && !zero_size_allowed) {
4646 		verbose(env, "invalid zero-sized read\n");
4647 		return -EACCES;
4648 	}
4649 
4650 	if (type == ACCESS_HELPER) {
4651 		/* The bounds checks for writes are more permissive than for
4652 		 * reads. However, if raw_mode is not set, we'll do extra
4653 		 * checks below.
4654 		 */
4655 		bounds_check_type = BPF_WRITE;
4656 		clobber = true;
4657 	} else {
4658 		bounds_check_type = BPF_READ;
4659 	}
4660 	err = check_stack_access_within_bounds(env, regno, off, access_size,
4661 					       type, bounds_check_type);
4662 	if (err)
4663 		return err;
4664 
4665 
4666 	if (tnum_is_const(reg->var_off)) {
4667 		min_off = max_off = reg->var_off.value + off;
4668 	} else {
4669 		/* Variable offset is prohibited for unprivileged mode for
4670 		 * simplicity since it requires corresponding support in
4671 		 * Spectre masking for stack ALU.
4672 		 * See also retrieve_ptr_limit().
4673 		 */
4674 		if (!env->bypass_spec_v1) {
4675 			char tn_buf[48];
4676 
4677 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4678 			verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
4679 				regno, err_extra, tn_buf);
4680 			return -EACCES;
4681 		}
4682 		/* Only initialized buffer on stack is allowed to be accessed
4683 		 * with variable offset. With uninitialized buffer it's hard to
4684 		 * guarantee that whole memory is marked as initialized on
4685 		 * helper return since specific bounds are unknown what may
4686 		 * cause uninitialized stack leaking.
4687 		 */
4688 		if (meta && meta->raw_mode)
4689 			meta = NULL;
4690 
4691 		min_off = reg->smin_value + off;
4692 		max_off = reg->smax_value + off;
4693 	}
4694 
4695 	if (meta && meta->raw_mode) {
4696 		meta->access_size = access_size;
4697 		meta->regno = regno;
4698 		return 0;
4699 	}
4700 
4701 	for (i = min_off; i < max_off + access_size; i++) {
4702 		u8 *stype;
4703 
4704 		slot = -i - 1;
4705 		spi = slot / BPF_REG_SIZE;
4706 		if (state->allocated_stack <= slot)
4707 			goto err;
4708 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
4709 		if (*stype == STACK_MISC)
4710 			goto mark;
4711 		if (*stype == STACK_ZERO) {
4712 			if (clobber) {
4713 				/* helper can write anything into the stack */
4714 				*stype = STACK_MISC;
4715 			}
4716 			goto mark;
4717 		}
4718 
4719 		if (is_spilled_reg(&state->stack[spi]) &&
4720 		    state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID)
4721 			goto mark;
4722 
4723 		if (is_spilled_reg(&state->stack[spi]) &&
4724 		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
4725 		     env->allow_ptr_leaks)) {
4726 			if (clobber) {
4727 				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
4728 				for (j = 0; j < BPF_REG_SIZE; j++)
4729 					scrub_spilled_slot(&state->stack[spi].slot_type[j]);
4730 			}
4731 			goto mark;
4732 		}
4733 
4734 err:
4735 		if (tnum_is_const(reg->var_off)) {
4736 			verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
4737 				err_extra, regno, min_off, i - min_off, access_size);
4738 		} else {
4739 			char tn_buf[48];
4740 
4741 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4742 			verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
4743 				err_extra, regno, tn_buf, i - min_off, access_size);
4744 		}
4745 		return -EACCES;
4746 mark:
4747 		/* reading any byte out of 8-byte 'spill_slot' will cause
4748 		 * the whole slot to be marked as 'read'
4749 		 */
4750 		mark_reg_read(env, &state->stack[spi].spilled_ptr,
4751 			      state->stack[spi].spilled_ptr.parent,
4752 			      REG_LIVE_READ64);
4753 	}
4754 	return update_stack_depth(env, state, min_off);
4755 }
4756 
4757 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
4758 				   int access_size, bool zero_size_allowed,
4759 				   struct bpf_call_arg_meta *meta)
4760 {
4761 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
4762 
4763 	switch (reg->type) {
4764 	case PTR_TO_PACKET:
4765 	case PTR_TO_PACKET_META:
4766 		return check_packet_access(env, regno, reg->off, access_size,
4767 					   zero_size_allowed);
4768 	case PTR_TO_MAP_KEY:
4769 		return check_mem_region_access(env, regno, reg->off, access_size,
4770 					       reg->map_ptr->key_size, false);
4771 	case PTR_TO_MAP_VALUE:
4772 		if (check_map_access_type(env, regno, reg->off, access_size,
4773 					  meta && meta->raw_mode ? BPF_WRITE :
4774 					  BPF_READ))
4775 			return -EACCES;
4776 		return check_map_access(env, regno, reg->off, access_size,
4777 					zero_size_allowed);
4778 	case PTR_TO_MEM:
4779 		return check_mem_region_access(env, regno, reg->off,
4780 					       access_size, reg->mem_size,
4781 					       zero_size_allowed);
4782 	case PTR_TO_RDONLY_BUF:
4783 		if (meta && meta->raw_mode)
4784 			return -EACCES;
4785 		return check_buffer_access(env, reg, regno, reg->off,
4786 					   access_size, zero_size_allowed,
4787 					   "rdonly",
4788 					   &env->prog->aux->max_rdonly_access);
4789 	case PTR_TO_RDWR_BUF:
4790 		return check_buffer_access(env, reg, regno, reg->off,
4791 					   access_size, zero_size_allowed,
4792 					   "rdwr",
4793 					   &env->prog->aux->max_rdwr_access);
4794 	case PTR_TO_STACK:
4795 		return check_stack_range_initialized(
4796 				env,
4797 				regno, reg->off, access_size,
4798 				zero_size_allowed, ACCESS_HELPER, meta);
4799 	default: /* scalar_value or invalid ptr */
4800 		/* Allow zero-byte read from NULL, regardless of pointer type */
4801 		if (zero_size_allowed && access_size == 0 &&
4802 		    register_is_null(reg))
4803 			return 0;
4804 
4805 		verbose(env, "R%d type=%s expected=%s\n", regno,
4806 			reg_type_str[reg->type],
4807 			reg_type_str[PTR_TO_STACK]);
4808 		return -EACCES;
4809 	}
4810 }
4811 
4812 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4813 		   u32 regno, u32 mem_size)
4814 {
4815 	if (register_is_null(reg))
4816 		return 0;
4817 
4818 	if (reg_type_may_be_null(reg->type)) {
4819 		/* Assuming that the register contains a value check if the memory
4820 		 * access is safe. Temporarily save and restore the register's state as
4821 		 * the conversion shouldn't be visible to a caller.
4822 		 */
4823 		const struct bpf_reg_state saved_reg = *reg;
4824 		int rv;
4825 
4826 		mark_ptr_not_null_reg(reg);
4827 		rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
4828 		*reg = saved_reg;
4829 		return rv;
4830 	}
4831 
4832 	return check_helper_mem_access(env, regno, mem_size, true, NULL);
4833 }
4834 
4835 /* Implementation details:
4836  * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
4837  * Two bpf_map_lookups (even with the same key) will have different reg->id.
4838  * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after
4839  * value_or_null->value transition, since the verifier only cares about
4840  * the range of access to valid map value pointer and doesn't care about actual
4841  * address of the map element.
4842  * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
4843  * reg->id > 0 after value_or_null->value transition. By doing so
4844  * two bpf_map_lookups will be considered two different pointers that
4845  * point to different bpf_spin_locks.
4846  * The verifier allows taking only one bpf_spin_lock at a time to avoid
4847  * dead-locks.
4848  * Since only one bpf_spin_lock is allowed the checks are simpler than
4849  * reg_is_refcounted() logic. The verifier needs to remember only
4850  * one spin_lock instead of array of acquired_refs.
4851  * cur_state->active_spin_lock remembers which map value element got locked
4852  * and clears it after bpf_spin_unlock.
4853  */
4854 static int process_spin_lock(struct bpf_verifier_env *env, int regno,
4855 			     bool is_lock)
4856 {
4857 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
4858 	struct bpf_verifier_state *cur = env->cur_state;
4859 	bool is_const = tnum_is_const(reg->var_off);
4860 	struct bpf_map *map = reg->map_ptr;
4861 	u64 val = reg->var_off.value;
4862 
4863 	if (!is_const) {
4864 		verbose(env,
4865 			"R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
4866 			regno);
4867 		return -EINVAL;
4868 	}
4869 	if (!map->btf) {
4870 		verbose(env,
4871 			"map '%s' has to have BTF in order to use bpf_spin_lock\n",
4872 			map->name);
4873 		return -EINVAL;
4874 	}
4875 	if (!map_value_has_spin_lock(map)) {
4876 		if (map->spin_lock_off == -E2BIG)
4877 			verbose(env,
4878 				"map '%s' has more than one 'struct bpf_spin_lock'\n",
4879 				map->name);
4880 		else if (map->spin_lock_off == -ENOENT)
4881 			verbose(env,
4882 				"map '%s' doesn't have 'struct bpf_spin_lock'\n",
4883 				map->name);
4884 		else
4885 			verbose(env,
4886 				"map '%s' is not a struct type or bpf_spin_lock is mangled\n",
4887 				map->name);
4888 		return -EINVAL;
4889 	}
4890 	if (map->spin_lock_off != val + reg->off) {
4891 		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
4892 			val + reg->off);
4893 		return -EINVAL;
4894 	}
4895 	if (is_lock) {
4896 		if (cur->active_spin_lock) {
4897 			verbose(env,
4898 				"Locking two bpf_spin_locks are not allowed\n");
4899 			return -EINVAL;
4900 		}
4901 		cur->active_spin_lock = reg->id;
4902 	} else {
4903 		if (!cur->active_spin_lock) {
4904 			verbose(env, "bpf_spin_unlock without taking a lock\n");
4905 			return -EINVAL;
4906 		}
4907 		if (cur->active_spin_lock != reg->id) {
4908 			verbose(env, "bpf_spin_unlock of different lock\n");
4909 			return -EINVAL;
4910 		}
4911 		cur->active_spin_lock = 0;
4912 	}
4913 	return 0;
4914 }
4915 
4916 static int process_timer_func(struct bpf_verifier_env *env, int regno,
4917 			      struct bpf_call_arg_meta *meta)
4918 {
4919 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
4920 	bool is_const = tnum_is_const(reg->var_off);
4921 	struct bpf_map *map = reg->map_ptr;
4922 	u64 val = reg->var_off.value;
4923 
4924 	if (!is_const) {
4925 		verbose(env,
4926 			"R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n",
4927 			regno);
4928 		return -EINVAL;
4929 	}
4930 	if (!map->btf) {
4931 		verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",
4932 			map->name);
4933 		return -EINVAL;
4934 	}
4935 	if (!map_value_has_timer(map)) {
4936 		if (map->timer_off == -E2BIG)
4937 			verbose(env,
4938 				"map '%s' has more than one 'struct bpf_timer'\n",
4939 				map->name);
4940 		else if (map->timer_off == -ENOENT)
4941 			verbose(env,
4942 				"map '%s' doesn't have 'struct bpf_timer'\n",
4943 				map->name);
4944 		else
4945 			verbose(env,
4946 				"map '%s' is not a struct type or bpf_timer is mangled\n",
4947 				map->name);
4948 		return -EINVAL;
4949 	}
4950 	if (map->timer_off != val + reg->off) {
4951 		verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",
4952 			val + reg->off, map->timer_off);
4953 		return -EINVAL;
4954 	}
4955 	if (meta->map_ptr) {
4956 		verbose(env, "verifier bug. Two map pointers in a timer helper\n");
4957 		return -EFAULT;
4958 	}
4959 	meta->map_uid = reg->map_uid;
4960 	meta->map_ptr = map;
4961 	return 0;
4962 }
4963 
4964 static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
4965 {
4966 	return type == ARG_PTR_TO_MEM ||
4967 	       type == ARG_PTR_TO_MEM_OR_NULL ||
4968 	       type == ARG_PTR_TO_UNINIT_MEM;
4969 }
4970 
4971 static bool arg_type_is_mem_size(enum bpf_arg_type type)
4972 {
4973 	return type == ARG_CONST_SIZE ||
4974 	       type == ARG_CONST_SIZE_OR_ZERO;
4975 }
4976 
4977 static bool arg_type_is_alloc_size(enum bpf_arg_type type)
4978 {
4979 	return type == ARG_CONST_ALLOC_SIZE_OR_ZERO;
4980 }
4981 
4982 static bool arg_type_is_int_ptr(enum bpf_arg_type type)
4983 {
4984 	return type == ARG_PTR_TO_INT ||
4985 	       type == ARG_PTR_TO_LONG;
4986 }
4987 
4988 static int int_ptr_type_to_size(enum bpf_arg_type type)
4989 {
4990 	if (type == ARG_PTR_TO_INT)
4991 		return sizeof(u32);
4992 	else if (type == ARG_PTR_TO_LONG)
4993 		return sizeof(u64);
4994 
4995 	return -EINVAL;
4996 }
4997 
4998 static int resolve_map_arg_type(struct bpf_verifier_env *env,
4999 				 const struct bpf_call_arg_meta *meta,
5000 				 enum bpf_arg_type *arg_type)
5001 {
5002 	if (!meta->map_ptr) {
5003 		/* kernel subsystem misconfigured verifier */
5004 		verbose(env, "invalid map_ptr to access map->type\n");
5005 		return -EACCES;
5006 	}
5007 
5008 	switch (meta->map_ptr->map_type) {
5009 	case BPF_MAP_TYPE_SOCKMAP:
5010 	case BPF_MAP_TYPE_SOCKHASH:
5011 		if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
5012 			*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
5013 		} else {
5014 			verbose(env, "invalid arg_type for sockmap/sockhash\n");
5015 			return -EINVAL;
5016 		}
5017 		break;
5018 	case BPF_MAP_TYPE_BLOOM_FILTER:
5019 		if (meta->func_id == BPF_FUNC_map_peek_elem)
5020 			*arg_type = ARG_PTR_TO_MAP_VALUE;
5021 		break;
5022 	default:
5023 		break;
5024 	}
5025 	return 0;
5026 }
5027 
5028 struct bpf_reg_types {
5029 	const enum bpf_reg_type types[10];
5030 	u32 *btf_id;
5031 };
5032 
5033 static const struct bpf_reg_types map_key_value_types = {
5034 	.types = {
5035 		PTR_TO_STACK,
5036 		PTR_TO_PACKET,
5037 		PTR_TO_PACKET_META,
5038 		PTR_TO_MAP_KEY,
5039 		PTR_TO_MAP_VALUE,
5040 	},
5041 };
5042 
5043 static const struct bpf_reg_types sock_types = {
5044 	.types = {
5045 		PTR_TO_SOCK_COMMON,
5046 		PTR_TO_SOCKET,
5047 		PTR_TO_TCP_SOCK,
5048 		PTR_TO_XDP_SOCK,
5049 	},
5050 };
5051 
5052 #ifdef CONFIG_NET
5053 static const struct bpf_reg_types btf_id_sock_common_types = {
5054 	.types = {
5055 		PTR_TO_SOCK_COMMON,
5056 		PTR_TO_SOCKET,
5057 		PTR_TO_TCP_SOCK,
5058 		PTR_TO_XDP_SOCK,
5059 		PTR_TO_BTF_ID,
5060 	},
5061 	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5062 };
5063 #endif
5064 
5065 static const struct bpf_reg_types mem_types = {
5066 	.types = {
5067 		PTR_TO_STACK,
5068 		PTR_TO_PACKET,
5069 		PTR_TO_PACKET_META,
5070 		PTR_TO_MAP_KEY,
5071 		PTR_TO_MAP_VALUE,
5072 		PTR_TO_MEM,
5073 		PTR_TO_RDONLY_BUF,
5074 		PTR_TO_RDWR_BUF,
5075 	},
5076 };
5077 
5078 static const struct bpf_reg_types int_ptr_types = {
5079 	.types = {
5080 		PTR_TO_STACK,
5081 		PTR_TO_PACKET,
5082 		PTR_TO_PACKET_META,
5083 		PTR_TO_MAP_KEY,
5084 		PTR_TO_MAP_VALUE,
5085 	},
5086 };
5087 
5088 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
5089 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
5090 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
5091 static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } };
5092 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
5093 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
5094 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
5095 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
5096 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
5097 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
5098 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
5099 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
5100 
5101 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
5102 	[ARG_PTR_TO_MAP_KEY]		= &map_key_value_types,
5103 	[ARG_PTR_TO_MAP_VALUE]		= &map_key_value_types,
5104 	[ARG_PTR_TO_UNINIT_MAP_VALUE]	= &map_key_value_types,
5105 	[ARG_PTR_TO_MAP_VALUE_OR_NULL]	= &map_key_value_types,
5106 	[ARG_CONST_SIZE]		= &scalar_types,
5107 	[ARG_CONST_SIZE_OR_ZERO]	= &scalar_types,
5108 	[ARG_CONST_ALLOC_SIZE_OR_ZERO]	= &scalar_types,
5109 	[ARG_CONST_MAP_PTR]		= &const_map_ptr_types,
5110 	[ARG_PTR_TO_CTX]		= &context_types,
5111 	[ARG_PTR_TO_CTX_OR_NULL]	= &context_types,
5112 	[ARG_PTR_TO_SOCK_COMMON]	= &sock_types,
5113 #ifdef CONFIG_NET
5114 	[ARG_PTR_TO_BTF_ID_SOCK_COMMON]	= &btf_id_sock_common_types,
5115 #endif
5116 	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
5117 	[ARG_PTR_TO_SOCKET_OR_NULL]	= &fullsock_types,
5118 	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
5119 	[ARG_PTR_TO_SPIN_LOCK]		= &spin_lock_types,
5120 	[ARG_PTR_TO_MEM]		= &mem_types,
5121 	[ARG_PTR_TO_MEM_OR_NULL]	= &mem_types,
5122 	[ARG_PTR_TO_UNINIT_MEM]		= &mem_types,
5123 	[ARG_PTR_TO_ALLOC_MEM]		= &alloc_mem_types,
5124 	[ARG_PTR_TO_ALLOC_MEM_OR_NULL]	= &alloc_mem_types,
5125 	[ARG_PTR_TO_INT]		= &int_ptr_types,
5126 	[ARG_PTR_TO_LONG]		= &int_ptr_types,
5127 	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
5128 	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
5129 	[ARG_PTR_TO_STACK_OR_NULL]	= &stack_ptr_types,
5130 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
5131 	[ARG_PTR_TO_TIMER]		= &timer_types,
5132 };
5133 
5134 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
5135 			  enum bpf_arg_type arg_type,
5136 			  const u32 *arg_btf_id)
5137 {
5138 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
5139 	enum bpf_reg_type expected, type = reg->type;
5140 	const struct bpf_reg_types *compatible;
5141 	int i, j;
5142 
5143 	compatible = compatible_reg_types[arg_type];
5144 	if (!compatible) {
5145 		verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
5146 		return -EFAULT;
5147 	}
5148 
5149 	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
5150 		expected = compatible->types[i];
5151 		if (expected == NOT_INIT)
5152 			break;
5153 
5154 		if (type == expected)
5155 			goto found;
5156 	}
5157 
5158 	verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
5159 	for (j = 0; j + 1 < i; j++)
5160 		verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
5161 	verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
5162 	return -EACCES;
5163 
5164 found:
5165 	if (type == PTR_TO_BTF_ID) {
5166 		if (!arg_btf_id) {
5167 			if (!compatible->btf_id) {
5168 				verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
5169 				return -EFAULT;
5170 			}
5171 			arg_btf_id = compatible->btf_id;
5172 		}
5173 
5174 		if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
5175 					  btf_vmlinux, *arg_btf_id)) {
5176 			verbose(env, "R%d is of type %s but %s is expected\n",
5177 				regno, kernel_type_name(reg->btf, reg->btf_id),
5178 				kernel_type_name(btf_vmlinux, *arg_btf_id));
5179 			return -EACCES;
5180 		}
5181 
5182 		if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
5183 			verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
5184 				regno);
5185 			return -EACCES;
5186 		}
5187 	}
5188 
5189 	return 0;
5190 }
5191 
5192 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
5193 			  struct bpf_call_arg_meta *meta,
5194 			  const struct bpf_func_proto *fn)
5195 {
5196 	u32 regno = BPF_REG_1 + arg;
5197 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
5198 	enum bpf_arg_type arg_type = fn->arg_type[arg];
5199 	enum bpf_reg_type type = reg->type;
5200 	int err = 0;
5201 
5202 	if (arg_type == ARG_DONTCARE)
5203 		return 0;
5204 
5205 	err = check_reg_arg(env, regno, SRC_OP);
5206 	if (err)
5207 		return err;
5208 
5209 	if (arg_type == ARG_ANYTHING) {
5210 		if (is_pointer_value(env, regno)) {
5211 			verbose(env, "R%d leaks addr into helper function\n",
5212 				regno);
5213 			return -EACCES;
5214 		}
5215 		return 0;
5216 	}
5217 
5218 	if (type_is_pkt_pointer(type) &&
5219 	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {
5220 		verbose(env, "helper access to the packet is not allowed\n");
5221 		return -EACCES;
5222 	}
5223 
5224 	if (arg_type == ARG_PTR_TO_MAP_VALUE ||
5225 	    arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
5226 	    arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
5227 		err = resolve_map_arg_type(env, meta, &arg_type);
5228 		if (err)
5229 			return err;
5230 	}
5231 
5232 	if (register_is_null(reg) && arg_type_may_be_null(arg_type))
5233 		/* A NULL register has a SCALAR_VALUE type, so skip
5234 		 * type checking.
5235 		 */
5236 		goto skip_type_check;
5237 
5238 	err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]);
5239 	if (err)
5240 		return err;
5241 
5242 	if (type == PTR_TO_CTX) {
5243 		err = check_ctx_reg(env, reg, regno);
5244 		if (err < 0)
5245 			return err;
5246 	}
5247 
5248 skip_type_check:
5249 	if (reg->ref_obj_id) {
5250 		if (meta->ref_obj_id) {
5251 			verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
5252 				regno, reg->ref_obj_id,
5253 				meta->ref_obj_id);
5254 			return -EFAULT;
5255 		}
5256 		meta->ref_obj_id = reg->ref_obj_id;
5257 	}
5258 
5259 	if (arg_type == ARG_CONST_MAP_PTR) {
5260 		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
5261 		if (meta->map_ptr) {
5262 			/* Use map_uid (which is unique id of inner map) to reject:
5263 			 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
5264 			 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
5265 			 * if (inner_map1 && inner_map2) {
5266 			 *     timer = bpf_map_lookup_elem(inner_map1);
5267 			 *     if (timer)
5268 			 *         // mismatch would have been allowed
5269 			 *         bpf_timer_init(timer, inner_map2);
5270 			 * }
5271 			 *
5272 			 * Comparing map_ptr is enough to distinguish normal and outer maps.
5273 			 */
5274 			if (meta->map_ptr != reg->map_ptr ||
5275 			    meta->map_uid != reg->map_uid) {
5276 				verbose(env,
5277 					"timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
5278 					meta->map_uid, reg->map_uid);
5279 				return -EINVAL;
5280 			}
5281 		}
5282 		meta->map_ptr = reg->map_ptr;
5283 		meta->map_uid = reg->map_uid;
5284 	} else if (arg_type == ARG_PTR_TO_MAP_KEY) {
5285 		/* bpf_map_xxx(..., map_ptr, ..., key) call:
5286 		 * check that [key, key + map->key_size) are within
5287 		 * stack limits and initialized
5288 		 */
5289 		if (!meta->map_ptr) {
5290 			/* in function declaration map_ptr must come before
5291 			 * map_key, so that it's verified and known before
5292 			 * we have to check map_key here. Otherwise it means
5293 			 * that kernel subsystem misconfigured verifier
5294 			 */
5295 			verbose(env, "invalid map_ptr to access map->key\n");
5296 			return -EACCES;
5297 		}
5298 		err = check_helper_mem_access(env, regno,
5299 					      meta->map_ptr->key_size, false,
5300 					      NULL);
5301 	} else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
5302 		   (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
5303 		    !register_is_null(reg)) ||
5304 		   arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
5305 		/* bpf_map_xxx(..., map_ptr, ..., value) call:
5306 		 * check [value, value + map->value_size) validity
5307 		 */
5308 		if (!meta->map_ptr) {
5309 			/* kernel subsystem misconfigured verifier */
5310 			verbose(env, "invalid map_ptr to access map->value\n");
5311 			return -EACCES;
5312 		}
5313 		meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
5314 		err = check_helper_mem_access(env, regno,
5315 					      meta->map_ptr->value_size, false,
5316 					      meta);
5317 	} else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
5318 		if (!reg->btf_id) {
5319 			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
5320 			return -EACCES;
5321 		}
5322 		meta->ret_btf = reg->btf;
5323 		meta->ret_btf_id = reg->btf_id;
5324 	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
5325 		if (meta->func_id == BPF_FUNC_spin_lock) {
5326 			if (process_spin_lock(env, regno, true))
5327 				return -EACCES;
5328 		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
5329 			if (process_spin_lock(env, regno, false))
5330 				return -EACCES;
5331 		} else {
5332 			verbose(env, "verifier internal error\n");
5333 			return -EFAULT;
5334 		}
5335 	} else if (arg_type == ARG_PTR_TO_TIMER) {
5336 		if (process_timer_func(env, regno, meta))
5337 			return -EACCES;
5338 	} else if (arg_type == ARG_PTR_TO_FUNC) {
5339 		meta->subprogno = reg->subprogno;
5340 	} else if (arg_type_is_mem_ptr(arg_type)) {
5341 		/* The access to this pointer is only checked when we hit the
5342 		 * next is_mem_size argument below.
5343 		 */
5344 		meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM);
5345 	} else if (arg_type_is_mem_size(arg_type)) {
5346 		bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
5347 
5348 		/* This is used to refine r0 return value bounds for helpers
5349 		 * that enforce this value as an upper bound on return values.
5350 		 * See do_refine_retval_range() for helpers that can refine
5351 		 * the return value. C type of helper is u32 so we pull register
5352 		 * bound from umax_value however, if negative verifier errors
5353 		 * out. Only upper bounds can be learned because retval is an
5354 		 * int type and negative retvals are allowed.
5355 		 */
5356 		meta->msize_max_value = reg->umax_value;
5357 
5358 		/* The register is SCALAR_VALUE; the access check
5359 		 * happens using its boundaries.
5360 		 */
5361 		if (!tnum_is_const(reg->var_off))
5362 			/* For unprivileged variable accesses, disable raw
5363 			 * mode so that the program is required to
5364 			 * initialize all the memory that the helper could
5365 			 * just partially fill up.
5366 			 */
5367 			meta = NULL;
5368 
5369 		if (reg->smin_value < 0) {
5370 			verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
5371 				regno);
5372 			return -EACCES;
5373 		}
5374 
5375 		if (reg->umin_value == 0) {
5376 			err = check_helper_mem_access(env, regno - 1, 0,
5377 						      zero_size_allowed,
5378 						      meta);
5379 			if (err)
5380 				return err;
5381 		}
5382 
5383 		if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
5384 			verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
5385 				regno);
5386 			return -EACCES;
5387 		}
5388 		err = check_helper_mem_access(env, regno - 1,
5389 					      reg->umax_value,
5390 					      zero_size_allowed, meta);
5391 		if (!err)
5392 			err = mark_chain_precision(env, regno);
5393 	} else if (arg_type_is_alloc_size(arg_type)) {
5394 		if (!tnum_is_const(reg->var_off)) {
5395 			verbose(env, "R%d is not a known constant'\n",
5396 				regno);
5397 			return -EACCES;
5398 		}
5399 		meta->mem_size = reg->var_off.value;
5400 	} else if (arg_type_is_int_ptr(arg_type)) {
5401 		int size = int_ptr_type_to_size(arg_type);
5402 
5403 		err = check_helper_mem_access(env, regno, size, false, meta);
5404 		if (err)
5405 			return err;
5406 		err = check_ptr_alignment(env, reg, 0, size, true);
5407 	} else if (arg_type == ARG_PTR_TO_CONST_STR) {
5408 		struct bpf_map *map = reg->map_ptr;
5409 		int map_off;
5410 		u64 map_addr;
5411 		char *str_ptr;
5412 
5413 		if (!bpf_map_is_rdonly(map)) {
5414 			verbose(env, "R%d does not point to a readonly map'\n", regno);
5415 			return -EACCES;
5416 		}
5417 
5418 		if (!tnum_is_const(reg->var_off)) {
5419 			verbose(env, "R%d is not a constant address'\n", regno);
5420 			return -EACCES;
5421 		}
5422 
5423 		if (!map->ops->map_direct_value_addr) {
5424 			verbose(env, "no direct value access support for this map type\n");
5425 			return -EACCES;
5426 		}
5427 
5428 		err = check_map_access(env, regno, reg->off,
5429 				       map->value_size - reg->off, false);
5430 		if (err)
5431 			return err;
5432 
5433 		map_off = reg->off + reg->var_off.value;
5434 		err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
5435 		if (err) {
5436 			verbose(env, "direct value access on string failed\n");
5437 			return err;
5438 		}
5439 
5440 		str_ptr = (char *)(long)(map_addr);
5441 		if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
5442 			verbose(env, "string is not zero-terminated\n");
5443 			return -EINVAL;
5444 		}
5445 	}
5446 
5447 	return err;
5448 }
5449 
5450 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
5451 {
5452 	enum bpf_attach_type eatype = env->prog->expected_attach_type;
5453 	enum bpf_prog_type type = resolve_prog_type(env->prog);
5454 
5455 	if (func_id != BPF_FUNC_map_update_elem)
5456 		return false;
5457 
5458 	/* It's not possible to get access to a locked struct sock in these
5459 	 * contexts, so updating is safe.
5460 	 */
5461 	switch (type) {
5462 	case BPF_PROG_TYPE_TRACING:
5463 		if (eatype == BPF_TRACE_ITER)
5464 			return true;
5465 		break;
5466 	case BPF_PROG_TYPE_SOCKET_FILTER:
5467 	case BPF_PROG_TYPE_SCHED_CLS:
5468 	case BPF_PROG_TYPE_SCHED_ACT:
5469 	case BPF_PROG_TYPE_XDP:
5470 	case BPF_PROG_TYPE_SK_REUSEPORT:
5471 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
5472 	case BPF_PROG_TYPE_SK_LOOKUP:
5473 		return true;
5474 	default:
5475 		break;
5476 	}
5477 
5478 	verbose(env, "cannot update sockmap in this context\n");
5479 	return false;
5480 }
5481 
5482 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
5483 {
5484 	return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64);
5485 }
5486 
5487 static int check_map_func_compatibility(struct bpf_verifier_env *env,
5488 					struct bpf_map *map, int func_id)
5489 {
5490 	if (!map)
5491 		return 0;
5492 
5493 	/* We need a two way check, first is from map perspective ... */
5494 	switch (map->map_type) {
5495 	case BPF_MAP_TYPE_PROG_ARRAY:
5496 		if (func_id != BPF_FUNC_tail_call)
5497 			goto error;
5498 		break;
5499 	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5500 		if (func_id != BPF_FUNC_perf_event_read &&
5501 		    func_id != BPF_FUNC_perf_event_output &&
5502 		    func_id != BPF_FUNC_skb_output &&
5503 		    func_id != BPF_FUNC_perf_event_read_value &&
5504 		    func_id != BPF_FUNC_xdp_output)
5505 			goto error;
5506 		break;
5507 	case BPF_MAP_TYPE_RINGBUF:
5508 		if (func_id != BPF_FUNC_ringbuf_output &&
5509 		    func_id != BPF_FUNC_ringbuf_reserve &&
5510 		    func_id != BPF_FUNC_ringbuf_query)
5511 			goto error;
5512 		break;
5513 	case BPF_MAP_TYPE_STACK_TRACE:
5514 		if (func_id != BPF_FUNC_get_stackid)
5515 			goto error;
5516 		break;
5517 	case BPF_MAP_TYPE_CGROUP_ARRAY:
5518 		if (func_id != BPF_FUNC_skb_under_cgroup &&
5519 		    func_id != BPF_FUNC_current_task_under_cgroup)
5520 			goto error;
5521 		break;
5522 	case BPF_MAP_TYPE_CGROUP_STORAGE:
5523 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
5524 		if (func_id != BPF_FUNC_get_local_storage)
5525 			goto error;
5526 		break;
5527 	case BPF_MAP_TYPE_DEVMAP:
5528 	case BPF_MAP_TYPE_DEVMAP_HASH:
5529 		if (func_id != BPF_FUNC_redirect_map &&
5530 		    func_id != BPF_FUNC_map_lookup_elem)
5531 			goto error;
5532 		break;
5533 	/* Restrict bpf side of cpumap and xskmap, open when use-cases
5534 	 * appear.
5535 	 */
5536 	case BPF_MAP_TYPE_CPUMAP:
5537 		if (func_id != BPF_FUNC_redirect_map)
5538 			goto error;
5539 		break;
5540 	case BPF_MAP_TYPE_XSKMAP:
5541 		if (func_id != BPF_FUNC_redirect_map &&
5542 		    func_id != BPF_FUNC_map_lookup_elem)
5543 			goto error;
5544 		break;
5545 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5546 	case BPF_MAP_TYPE_HASH_OF_MAPS:
5547 		if (func_id != BPF_FUNC_map_lookup_elem)
5548 			goto error;
5549 		break;
5550 	case BPF_MAP_TYPE_SOCKMAP:
5551 		if (func_id != BPF_FUNC_sk_redirect_map &&
5552 		    func_id != BPF_FUNC_sock_map_update &&
5553 		    func_id != BPF_FUNC_map_delete_elem &&
5554 		    func_id != BPF_FUNC_msg_redirect_map &&
5555 		    func_id != BPF_FUNC_sk_select_reuseport &&
5556 		    func_id != BPF_FUNC_map_lookup_elem &&
5557 		    !may_update_sockmap(env, func_id))
5558 			goto error;
5559 		break;
5560 	case BPF_MAP_TYPE_SOCKHASH:
5561 		if (func_id != BPF_FUNC_sk_redirect_hash &&
5562 		    func_id != BPF_FUNC_sock_hash_update &&
5563 		    func_id != BPF_FUNC_map_delete_elem &&
5564 		    func_id != BPF_FUNC_msg_redirect_hash &&
5565 		    func_id != BPF_FUNC_sk_select_reuseport &&
5566 		    func_id != BPF_FUNC_map_lookup_elem &&
5567 		    !may_update_sockmap(env, func_id))
5568 			goto error;
5569 		break;
5570 	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
5571 		if (func_id != BPF_FUNC_sk_select_reuseport)
5572 			goto error;
5573 		break;
5574 	case BPF_MAP_TYPE_QUEUE:
5575 	case BPF_MAP_TYPE_STACK:
5576 		if (func_id != BPF_FUNC_map_peek_elem &&
5577 		    func_id != BPF_FUNC_map_pop_elem &&
5578 		    func_id != BPF_FUNC_map_push_elem)
5579 			goto error;
5580 		break;
5581 	case BPF_MAP_TYPE_SK_STORAGE:
5582 		if (func_id != BPF_FUNC_sk_storage_get &&
5583 		    func_id != BPF_FUNC_sk_storage_delete)
5584 			goto error;
5585 		break;
5586 	case BPF_MAP_TYPE_INODE_STORAGE:
5587 		if (func_id != BPF_FUNC_inode_storage_get &&
5588 		    func_id != BPF_FUNC_inode_storage_delete)
5589 			goto error;
5590 		break;
5591 	case BPF_MAP_TYPE_TASK_STORAGE:
5592 		if (func_id != BPF_FUNC_task_storage_get &&
5593 		    func_id != BPF_FUNC_task_storage_delete)
5594 			goto error;
5595 		break;
5596 	case BPF_MAP_TYPE_BLOOM_FILTER:
5597 		if (func_id != BPF_FUNC_map_peek_elem &&
5598 		    func_id != BPF_FUNC_map_push_elem)
5599 			goto error;
5600 		break;
5601 	default:
5602 		break;
5603 	}
5604 
5605 	/* ... and second from the function itself. */
5606 	switch (func_id) {
5607 	case BPF_FUNC_tail_call:
5608 		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
5609 			goto error;
5610 		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
5611 			verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
5612 			return -EINVAL;
5613 		}
5614 		break;
5615 	case BPF_FUNC_perf_event_read:
5616 	case BPF_FUNC_perf_event_output:
5617 	case BPF_FUNC_perf_event_read_value:
5618 	case BPF_FUNC_skb_output:
5619 	case BPF_FUNC_xdp_output:
5620 		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
5621 			goto error;
5622 		break;
5623 	case BPF_FUNC_ringbuf_output:
5624 	case BPF_FUNC_ringbuf_reserve:
5625 	case BPF_FUNC_ringbuf_query:
5626 		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
5627 			goto error;
5628 		break;
5629 	case BPF_FUNC_get_stackid:
5630 		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
5631 			goto error;
5632 		break;
5633 	case BPF_FUNC_current_task_under_cgroup:
5634 	case BPF_FUNC_skb_under_cgroup:
5635 		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
5636 			goto error;
5637 		break;
5638 	case BPF_FUNC_redirect_map:
5639 		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
5640 		    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
5641 		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
5642 		    map->map_type != BPF_MAP_TYPE_XSKMAP)
5643 			goto error;
5644 		break;
5645 	case BPF_FUNC_sk_redirect_map:
5646 	case BPF_FUNC_msg_redirect_map:
5647 	case BPF_FUNC_sock_map_update:
5648 		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
5649 			goto error;
5650 		break;
5651 	case BPF_FUNC_sk_redirect_hash:
5652 	case BPF_FUNC_msg_redirect_hash:
5653 	case BPF_FUNC_sock_hash_update:
5654 		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
5655 			goto error;
5656 		break;
5657 	case BPF_FUNC_get_local_storage:
5658 		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
5659 		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
5660 			goto error;
5661 		break;
5662 	case BPF_FUNC_sk_select_reuseport:
5663 		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
5664 		    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
5665 		    map->map_type != BPF_MAP_TYPE_SOCKHASH)
5666 			goto error;
5667 		break;
5668 	case BPF_FUNC_map_pop_elem:
5669 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
5670 		    map->map_type != BPF_MAP_TYPE_STACK)
5671 			goto error;
5672 		break;
5673 	case BPF_FUNC_map_peek_elem:
5674 	case BPF_FUNC_map_push_elem:
5675 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
5676 		    map->map_type != BPF_MAP_TYPE_STACK &&
5677 		    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
5678 			goto error;
5679 		break;
5680 	case BPF_FUNC_sk_storage_get:
5681 	case BPF_FUNC_sk_storage_delete:
5682 		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
5683 			goto error;
5684 		break;
5685 	case BPF_FUNC_inode_storage_get:
5686 	case BPF_FUNC_inode_storage_delete:
5687 		if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
5688 			goto error;
5689 		break;
5690 	case BPF_FUNC_task_storage_get:
5691 	case BPF_FUNC_task_storage_delete:
5692 		if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
5693 			goto error;
5694 		break;
5695 	default:
5696 		break;
5697 	}
5698 
5699 	return 0;
5700 error:
5701 	verbose(env, "cannot pass map_type %d into func %s#%d\n",
5702 		map->map_type, func_id_name(func_id), func_id);
5703 	return -EINVAL;
5704 }
5705 
5706 static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
5707 {
5708 	int count = 0;
5709 
5710 	if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
5711 		count++;
5712 	if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
5713 		count++;
5714 	if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
5715 		count++;
5716 	if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
5717 		count++;
5718 	if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
5719 		count++;
5720 
5721 	/* We only support one arg being in raw mode at the moment,
5722 	 * which is sufficient for the helper functions we have
5723 	 * right now.
5724 	 */
5725 	return count <= 1;
5726 }
5727 
5728 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
5729 				    enum bpf_arg_type arg_next)
5730 {
5731 	return (arg_type_is_mem_ptr(arg_curr) &&
5732 	        !arg_type_is_mem_size(arg_next)) ||
5733 	       (!arg_type_is_mem_ptr(arg_curr) &&
5734 		arg_type_is_mem_size(arg_next));
5735 }
5736 
5737 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
5738 {
5739 	/* bpf_xxx(..., buf, len) call will access 'len'
5740 	 * bytes from memory 'buf'. Both arg types need
5741 	 * to be paired, so make sure there's no buggy
5742 	 * helper function specification.
5743 	 */
5744 	if (arg_type_is_mem_size(fn->arg1_type) ||
5745 	    arg_type_is_mem_ptr(fn->arg5_type)  ||
5746 	    check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
5747 	    check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
5748 	    check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
5749 	    check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
5750 		return false;
5751 
5752 	return true;
5753 }
5754 
5755 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
5756 {
5757 	int count = 0;
5758 
5759 	if (arg_type_may_be_refcounted(fn->arg1_type))
5760 		count++;
5761 	if (arg_type_may_be_refcounted(fn->arg2_type))
5762 		count++;
5763 	if (arg_type_may_be_refcounted(fn->arg3_type))
5764 		count++;
5765 	if (arg_type_may_be_refcounted(fn->arg4_type))
5766 		count++;
5767 	if (arg_type_may_be_refcounted(fn->arg5_type))
5768 		count++;
5769 
5770 	/* A reference acquiring function cannot acquire
5771 	 * another refcounted ptr.
5772 	 */
5773 	if (may_be_acquire_function(func_id) && count)
5774 		return false;
5775 
5776 	/* We only support one arg being unreferenced at the moment,
5777 	 * which is sufficient for the helper functions we have right now.
5778 	 */
5779 	return count <= 1;
5780 }
5781 
5782 static bool check_btf_id_ok(const struct bpf_func_proto *fn)
5783 {
5784 	int i;
5785 
5786 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
5787 		if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
5788 			return false;
5789 
5790 		if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])
5791 			return false;
5792 	}
5793 
5794 	return true;
5795 }
5796 
5797 static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
5798 {
5799 	return check_raw_mode_ok(fn) &&
5800 	       check_arg_pair_ok(fn) &&
5801 	       check_btf_id_ok(fn) &&
5802 	       check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
5803 }
5804 
5805 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
5806  * are now invalid, so turn them into unknown SCALAR_VALUE.
5807  */
5808 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
5809 				     struct bpf_func_state *state)
5810 {
5811 	struct bpf_reg_state *regs = state->regs, *reg;
5812 	int i;
5813 
5814 	for (i = 0; i < MAX_BPF_REG; i++)
5815 		if (reg_is_pkt_pointer_any(&regs[i]))
5816 			mark_reg_unknown(env, regs, i);
5817 
5818 	bpf_for_each_spilled_reg(i, state, reg) {
5819 		if (!reg)
5820 			continue;
5821 		if (reg_is_pkt_pointer_any(reg))
5822 			__mark_reg_unknown(env, reg);
5823 	}
5824 }
5825 
5826 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
5827 {
5828 	struct bpf_verifier_state *vstate = env->cur_state;
5829 	int i;
5830 
5831 	for (i = 0; i <= vstate->curframe; i++)
5832 		__clear_all_pkt_pointers(env, vstate->frame[i]);
5833 }
5834 
5835 enum {
5836 	AT_PKT_END = -1,
5837 	BEYOND_PKT_END = -2,
5838 };
5839 
5840 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
5841 {
5842 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
5843 	struct bpf_reg_state *reg = &state->regs[regn];
5844 
5845 	if (reg->type != PTR_TO_PACKET)
5846 		/* PTR_TO_PACKET_META is not supported yet */
5847 		return;
5848 
5849 	/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
5850 	 * How far beyond pkt_end it goes is unknown.
5851 	 * if (!range_open) it's the case of pkt >= pkt_end
5852 	 * if (range_open) it's the case of pkt > pkt_end
5853 	 * hence this pointer is at least 1 byte bigger than pkt_end
5854 	 */
5855 	if (range_open)
5856 		reg->range = BEYOND_PKT_END;
5857 	else
5858 		reg->range = AT_PKT_END;
5859 }
5860 
5861 static void release_reg_references(struct bpf_verifier_env *env,
5862 				   struct bpf_func_state *state,
5863 				   int ref_obj_id)
5864 {
5865 	struct bpf_reg_state *regs = state->regs, *reg;
5866 	int i;
5867 
5868 	for (i = 0; i < MAX_BPF_REG; i++)
5869 		if (regs[i].ref_obj_id == ref_obj_id)
5870 			mark_reg_unknown(env, regs, i);
5871 
5872 	bpf_for_each_spilled_reg(i, state, reg) {
5873 		if (!reg)
5874 			continue;
5875 		if (reg->ref_obj_id == ref_obj_id)
5876 			__mark_reg_unknown(env, reg);
5877 	}
5878 }
5879 
5880 /* The pointer with the specified id has released its reference to kernel
5881  * resources. Identify all copies of the same pointer and clear the reference.
5882  */
5883 static int release_reference(struct bpf_verifier_env *env,
5884 			     int ref_obj_id)
5885 {
5886 	struct bpf_verifier_state *vstate = env->cur_state;
5887 	int err;
5888 	int i;
5889 
5890 	err = release_reference_state(cur_func(env), ref_obj_id);
5891 	if (err)
5892 		return err;
5893 
5894 	for (i = 0; i <= vstate->curframe; i++)
5895 		release_reg_references(env, vstate->frame[i], ref_obj_id);
5896 
5897 	return 0;
5898 }
5899 
5900 static void clear_caller_saved_regs(struct bpf_verifier_env *env,
5901 				    struct bpf_reg_state *regs)
5902 {
5903 	int i;
5904 
5905 	/* after the call registers r0 - r5 were scratched */
5906 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
5907 		mark_reg_not_init(env, regs, caller_saved[i]);
5908 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5909 	}
5910 }
5911 
5912 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
5913 				   struct bpf_func_state *caller,
5914 				   struct bpf_func_state *callee,
5915 				   int insn_idx);
5916 
5917 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
5918 			     int *insn_idx, int subprog,
5919 			     set_callee_state_fn set_callee_state_cb)
5920 {
5921 	struct bpf_verifier_state *state = env->cur_state;
5922 	struct bpf_func_info_aux *func_info_aux;
5923 	struct bpf_func_state *caller, *callee;
5924 	int err;
5925 	bool is_global = false;
5926 
5927 	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
5928 		verbose(env, "the call stack of %d frames is too deep\n",
5929 			state->curframe + 2);
5930 		return -E2BIG;
5931 	}
5932 
5933 	caller = state->frame[state->curframe];
5934 	if (state->frame[state->curframe + 1]) {
5935 		verbose(env, "verifier bug. Frame %d already allocated\n",
5936 			state->curframe + 1);
5937 		return -EFAULT;
5938 	}
5939 
5940 	func_info_aux = env->prog->aux->func_info_aux;
5941 	if (func_info_aux)
5942 		is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5943 	err = btf_check_subprog_arg_match(env, subprog, caller->regs);
5944 	if (err == -EFAULT)
5945 		return err;
5946 	if (is_global) {
5947 		if (err) {
5948 			verbose(env, "Caller passes invalid args into func#%d\n",
5949 				subprog);
5950 			return err;
5951 		} else {
5952 			if (env->log.level & BPF_LOG_LEVEL)
5953 				verbose(env,
5954 					"Func#%d is global and valid. Skipping.\n",
5955 					subprog);
5956 			clear_caller_saved_regs(env, caller->regs);
5957 
5958 			/* All global functions return a 64-bit SCALAR_VALUE */
5959 			mark_reg_unknown(env, caller->regs, BPF_REG_0);
5960 			caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
5961 
5962 			/* continue with next insn after call */
5963 			return 0;
5964 		}
5965 	}
5966 
5967 	if (insn->code == (BPF_JMP | BPF_CALL) &&
5968 	    insn->imm == BPF_FUNC_timer_set_callback) {
5969 		struct bpf_verifier_state *async_cb;
5970 
5971 		/* there is no real recursion here. timer callbacks are async */
5972 		env->subprog_info[subprog].is_async_cb = true;
5973 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
5974 					 *insn_idx, subprog);
5975 		if (!async_cb)
5976 			return -EFAULT;
5977 		callee = async_cb->frame[0];
5978 		callee->async_entry_cnt = caller->async_entry_cnt + 1;
5979 
5980 		/* Convert bpf_timer_set_callback() args into timer callback args */
5981 		err = set_callee_state_cb(env, caller, callee, *insn_idx);
5982 		if (err)
5983 			return err;
5984 
5985 		clear_caller_saved_regs(env, caller->regs);
5986 		mark_reg_unknown(env, caller->regs, BPF_REG_0);
5987 		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
5988 		/* continue with next insn after call */
5989 		return 0;
5990 	}
5991 
5992 	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
5993 	if (!callee)
5994 		return -ENOMEM;
5995 	state->frame[state->curframe + 1] = callee;
5996 
5997 	/* callee cannot access r0, r6 - r9 for reading and has to write
5998 	 * into its own stack before reading from it.
5999 	 * callee can read/write into caller's stack
6000 	 */
6001 	init_func_state(env, callee,
6002 			/* remember the callsite, it will be used by bpf_exit */
6003 			*insn_idx /* callsite */,
6004 			state->curframe + 1 /* frameno within this callchain */,
6005 			subprog /* subprog number within this prog */);
6006 
6007 	/* Transfer references to the callee */
6008 	err = copy_reference_state(callee, caller);
6009 	if (err)
6010 		return err;
6011 
6012 	err = set_callee_state_cb(env, caller, callee, *insn_idx);
6013 	if (err)
6014 		return err;
6015 
6016 	clear_caller_saved_regs(env, caller->regs);
6017 
6018 	/* only increment it after check_reg_arg() finished */
6019 	state->curframe++;
6020 
6021 	/* and go analyze first insn of the callee */
6022 	*insn_idx = env->subprog_info[subprog].start - 1;
6023 
6024 	if (env->log.level & BPF_LOG_LEVEL) {
6025 		verbose(env, "caller:\n");
6026 		print_verifier_state(env, caller);
6027 		verbose(env, "callee:\n");
6028 		print_verifier_state(env, callee);
6029 	}
6030 	return 0;
6031 }
6032 
6033 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
6034 				   struct bpf_func_state *caller,
6035 				   struct bpf_func_state *callee)
6036 {
6037 	/* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
6038 	 *      void *callback_ctx, u64 flags);
6039 	 * callback_fn(struct bpf_map *map, void *key, void *value,
6040 	 *      void *callback_ctx);
6041 	 */
6042 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
6043 
6044 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
6045 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
6046 	callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
6047 
6048 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
6049 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
6050 	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
6051 
6052 	/* pointer to stack or null */
6053 	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
6054 
6055 	/* unused */
6056 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
6057 	return 0;
6058 }
6059 
6060 static int set_callee_state(struct bpf_verifier_env *env,
6061 			    struct bpf_func_state *caller,
6062 			    struct bpf_func_state *callee, int insn_idx)
6063 {
6064 	int i;
6065 
6066 	/* copy r1 - r5 args that callee can access.  The copy includes parent
6067 	 * pointers, which connects us up to the liveness chain
6068 	 */
6069 	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
6070 		callee->regs[i] = caller->regs[i];
6071 	return 0;
6072 }
6073 
6074 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
6075 			   int *insn_idx)
6076 {
6077 	int subprog, target_insn;
6078 
6079 	target_insn = *insn_idx + insn->imm + 1;
6080 	subprog = find_subprog(env, target_insn);
6081 	if (subprog < 0) {
6082 		verbose(env, "verifier bug. No program starts at insn %d\n",
6083 			target_insn);
6084 		return -EFAULT;
6085 	}
6086 
6087 	return __check_func_call(env, insn, insn_idx, subprog, set_callee_state);
6088 }
6089 
6090 static int set_map_elem_callback_state(struct bpf_verifier_env *env,
6091 				       struct bpf_func_state *caller,
6092 				       struct bpf_func_state *callee,
6093 				       int insn_idx)
6094 {
6095 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
6096 	struct bpf_map *map;
6097 	int err;
6098 
6099 	if (bpf_map_ptr_poisoned(insn_aux)) {
6100 		verbose(env, "tail_call abusing map_ptr\n");
6101 		return -EINVAL;
6102 	}
6103 
6104 	map = BPF_MAP_PTR(insn_aux->map_ptr_state);
6105 	if (!map->ops->map_set_for_each_callback_args ||
6106 	    !map->ops->map_for_each_callback) {
6107 		verbose(env, "callback function not allowed for map\n");
6108 		return -ENOTSUPP;
6109 	}
6110 
6111 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);
6112 	if (err)
6113 		return err;
6114 
6115 	callee->in_callback_fn = true;
6116 	return 0;
6117 }
6118 
6119 static int set_timer_callback_state(struct bpf_verifier_env *env,
6120 				    struct bpf_func_state *caller,
6121 				    struct bpf_func_state *callee,
6122 				    int insn_idx)
6123 {
6124 	struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
6125 
6126 	/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
6127 	 * callback_fn(struct bpf_map *map, void *key, void *value);
6128 	 */
6129 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
6130 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
6131 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
6132 
6133 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
6134 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
6135 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
6136 
6137 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
6138 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
6139 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
6140 
6141 	/* unused */
6142 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
6143 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
6144 	callee->in_async_callback_fn = true;
6145 	return 0;
6146 }
6147 
6148 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
6149 {
6150 	struct bpf_verifier_state *state = env->cur_state;
6151 	struct bpf_func_state *caller, *callee;
6152 	struct bpf_reg_state *r0;
6153 	int err;
6154 
6155 	callee = state->frame[state->curframe];
6156 	r0 = &callee->regs[BPF_REG_0];
6157 	if (r0->type == PTR_TO_STACK) {
6158 		/* technically it's ok to return caller's stack pointer
6159 		 * (or caller's caller's pointer) back to the caller,
6160 		 * since these pointers are valid. Only current stack
6161 		 * pointer will be invalid as soon as function exits,
6162 		 * but let's be conservative
6163 		 */
6164 		verbose(env, "cannot return stack pointer to the caller\n");
6165 		return -EINVAL;
6166 	}
6167 
6168 	state->curframe--;
6169 	caller = state->frame[state->curframe];
6170 	if (callee->in_callback_fn) {
6171 		/* enforce R0 return value range [0, 1]. */
6172 		struct tnum range = tnum_range(0, 1);
6173 
6174 		if (r0->type != SCALAR_VALUE) {
6175 			verbose(env, "R0 not a scalar value\n");
6176 			return -EACCES;
6177 		}
6178 		if (!tnum_in(range, r0->var_off)) {
6179 			verbose_invalid_scalar(env, r0, &range, "callback return", "R0");
6180 			return -EINVAL;
6181 		}
6182 	} else {
6183 		/* return to the caller whatever r0 had in the callee */
6184 		caller->regs[BPF_REG_0] = *r0;
6185 	}
6186 
6187 	/* Transfer references to the caller */
6188 	err = copy_reference_state(caller, callee);
6189 	if (err)
6190 		return err;
6191 
6192 	*insn_idx = callee->callsite + 1;
6193 	if (env->log.level & BPF_LOG_LEVEL) {
6194 		verbose(env, "returning from callee:\n");
6195 		print_verifier_state(env, callee);
6196 		verbose(env, "to caller at %d:\n", *insn_idx);
6197 		print_verifier_state(env, caller);
6198 	}
6199 	/* clear everything in the callee */
6200 	free_func_state(callee);
6201 	state->frame[state->curframe + 1] = NULL;
6202 	return 0;
6203 }
6204 
6205 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
6206 				   int func_id,
6207 				   struct bpf_call_arg_meta *meta)
6208 {
6209 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
6210 
6211 	if (ret_type != RET_INTEGER ||
6212 	    (func_id != BPF_FUNC_get_stack &&
6213 	     func_id != BPF_FUNC_get_task_stack &&
6214 	     func_id != BPF_FUNC_probe_read_str &&
6215 	     func_id != BPF_FUNC_probe_read_kernel_str &&
6216 	     func_id != BPF_FUNC_probe_read_user_str))
6217 		return;
6218 
6219 	ret_reg->smax_value = meta->msize_max_value;
6220 	ret_reg->s32_max_value = meta->msize_max_value;
6221 	ret_reg->smin_value = -MAX_ERRNO;
6222 	ret_reg->s32_min_value = -MAX_ERRNO;
6223 	__reg_deduce_bounds(ret_reg);
6224 	__reg_bound_offset(ret_reg);
6225 	__update_reg_bounds(ret_reg);
6226 }
6227 
6228 static int
6229 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
6230 		int func_id, int insn_idx)
6231 {
6232 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
6233 	struct bpf_map *map = meta->map_ptr;
6234 
6235 	if (func_id != BPF_FUNC_tail_call &&
6236 	    func_id != BPF_FUNC_map_lookup_elem &&
6237 	    func_id != BPF_FUNC_map_update_elem &&
6238 	    func_id != BPF_FUNC_map_delete_elem &&
6239 	    func_id != BPF_FUNC_map_push_elem &&
6240 	    func_id != BPF_FUNC_map_pop_elem &&
6241 	    func_id != BPF_FUNC_map_peek_elem &&
6242 	    func_id != BPF_FUNC_for_each_map_elem &&
6243 	    func_id != BPF_FUNC_redirect_map)
6244 		return 0;
6245 
6246 	if (map == NULL) {
6247 		verbose(env, "kernel subsystem misconfigured verifier\n");
6248 		return -EINVAL;
6249 	}
6250 
6251 	/* In case of read-only, some additional restrictions
6252 	 * need to be applied in order to prevent altering the
6253 	 * state of the map from program side.
6254 	 */
6255 	if ((map->map_flags & BPF_F_RDONLY_PROG) &&
6256 	    (func_id == BPF_FUNC_map_delete_elem ||
6257 	     func_id == BPF_FUNC_map_update_elem ||
6258 	     func_id == BPF_FUNC_map_push_elem ||
6259 	     func_id == BPF_FUNC_map_pop_elem)) {
6260 		verbose(env, "write into map forbidden\n");
6261 		return -EACCES;
6262 	}
6263 
6264 	if (!BPF_MAP_PTR(aux->map_ptr_state))
6265 		bpf_map_ptr_store(aux, meta->map_ptr,
6266 				  !meta->map_ptr->bypass_spec_v1);
6267 	else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr)
6268 		bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
6269 				  !meta->map_ptr->bypass_spec_v1);
6270 	return 0;
6271 }
6272 
6273 static int
6274 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
6275 		int func_id, int insn_idx)
6276 {
6277 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
6278 	struct bpf_reg_state *regs = cur_regs(env), *reg;
6279 	struct bpf_map *map = meta->map_ptr;
6280 	struct tnum range;
6281 	u64 val;
6282 	int err;
6283 
6284 	if (func_id != BPF_FUNC_tail_call)
6285 		return 0;
6286 	if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
6287 		verbose(env, "kernel subsystem misconfigured verifier\n");
6288 		return -EINVAL;
6289 	}
6290 
6291 	range = tnum_range(0, map->max_entries - 1);
6292 	reg = &regs[BPF_REG_3];
6293 
6294 	if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) {
6295 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
6296 		return 0;
6297 	}
6298 
6299 	err = mark_chain_precision(env, BPF_REG_3);
6300 	if (err)
6301 		return err;
6302 
6303 	val = reg->var_off.value;
6304 	if (bpf_map_key_unseen(aux))
6305 		bpf_map_key_store(aux, val);
6306 	else if (!bpf_map_key_poisoned(aux) &&
6307 		  bpf_map_key_immediate(aux) != val)
6308 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
6309 	return 0;
6310 }
6311 
6312 static int check_reference_leak(struct bpf_verifier_env *env)
6313 {
6314 	struct bpf_func_state *state = cur_func(env);
6315 	int i;
6316 
6317 	for (i = 0; i < state->acquired_refs; i++) {
6318 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
6319 			state->refs[i].id, state->refs[i].insn_idx);
6320 	}
6321 	return state->acquired_refs ? -EINVAL : 0;
6322 }
6323 
6324 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
6325 				   struct bpf_reg_state *regs)
6326 {
6327 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
6328 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
6329 	struct bpf_map *fmt_map = fmt_reg->map_ptr;
6330 	int err, fmt_map_off, num_args;
6331 	u64 fmt_addr;
6332 	char *fmt;
6333 
6334 	/* data must be an array of u64 */
6335 	if (data_len_reg->var_off.value % 8)
6336 		return -EINVAL;
6337 	num_args = data_len_reg->var_off.value / 8;
6338 
6339 	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
6340 	 * and map_direct_value_addr is set.
6341 	 */
6342 	fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
6343 	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
6344 						  fmt_map_off);
6345 	if (err) {
6346 		verbose(env, "verifier bug\n");
6347 		return -EFAULT;
6348 	}
6349 	fmt = (char *)(long)fmt_addr + fmt_map_off;
6350 
6351 	/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
6352 	 * can focus on validating the format specifiers.
6353 	 */
6354 	err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args);
6355 	if (err < 0)
6356 		verbose(env, "Invalid format string\n");
6357 
6358 	return err;
6359 }
6360 
6361 static int check_get_func_ip(struct bpf_verifier_env *env)
6362 {
6363 	enum bpf_attach_type eatype = env->prog->expected_attach_type;
6364 	enum bpf_prog_type type = resolve_prog_type(env->prog);
6365 	int func_id = BPF_FUNC_get_func_ip;
6366 
6367 	if (type == BPF_PROG_TYPE_TRACING) {
6368 		if (eatype != BPF_TRACE_FENTRY && eatype != BPF_TRACE_FEXIT &&
6369 		    eatype != BPF_MODIFY_RETURN) {
6370 			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
6371 				func_id_name(func_id), func_id);
6372 			return -ENOTSUPP;
6373 		}
6374 		return 0;
6375 	} else if (type == BPF_PROG_TYPE_KPROBE) {
6376 		return 0;
6377 	}
6378 
6379 	verbose(env, "func %s#%d not supported for program type %d\n",
6380 		func_id_name(func_id), func_id, type);
6381 	return -ENOTSUPP;
6382 }
6383 
6384 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
6385 			     int *insn_idx_p)
6386 {
6387 	const struct bpf_func_proto *fn = NULL;
6388 	struct bpf_reg_state *regs;
6389 	struct bpf_call_arg_meta meta;
6390 	int insn_idx = *insn_idx_p;
6391 	bool changes_data;
6392 	int i, err, func_id;
6393 
6394 	/* find function prototype */
6395 	func_id = insn->imm;
6396 	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
6397 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
6398 			func_id);
6399 		return -EINVAL;
6400 	}
6401 
6402 	if (env->ops->get_func_proto)
6403 		fn = env->ops->get_func_proto(func_id, env->prog);
6404 	if (!fn) {
6405 		verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
6406 			func_id);
6407 		return -EINVAL;
6408 	}
6409 
6410 	/* eBPF programs must be GPL compatible to use GPL-ed functions */
6411 	if (!env->prog->gpl_compatible && fn->gpl_only) {
6412 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
6413 		return -EINVAL;
6414 	}
6415 
6416 	if (fn->allowed && !fn->allowed(env->prog)) {
6417 		verbose(env, "helper call is not allowed in probe\n");
6418 		return -EINVAL;
6419 	}
6420 
6421 	/* With LD_ABS/IND some JITs save/restore skb from r1. */
6422 	changes_data = bpf_helper_changes_pkt_data(fn->func);
6423 	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
6424 		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
6425 			func_id_name(func_id), func_id);
6426 		return -EINVAL;
6427 	}
6428 
6429 	memset(&meta, 0, sizeof(meta));
6430 	meta.pkt_access = fn->pkt_access;
6431 
6432 	err = check_func_proto(fn, func_id);
6433 	if (err) {
6434 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",
6435 			func_id_name(func_id), func_id);
6436 		return err;
6437 	}
6438 
6439 	meta.func_id = func_id;
6440 	/* check args */
6441 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
6442 		err = check_func_arg(env, i, &meta, fn);
6443 		if (err)
6444 			return err;
6445 	}
6446 
6447 	err = record_func_map(env, &meta, func_id, insn_idx);
6448 	if (err)
6449 		return err;
6450 
6451 	err = record_func_key(env, &meta, func_id, insn_idx);
6452 	if (err)
6453 		return err;
6454 
6455 	/* Mark slots with STACK_MISC in case of raw mode, stack offset
6456 	 * is inferred from register state.
6457 	 */
6458 	for (i = 0; i < meta.access_size; i++) {
6459 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
6460 				       BPF_WRITE, -1, false);
6461 		if (err)
6462 			return err;
6463 	}
6464 
6465 	if (func_id == BPF_FUNC_tail_call) {
6466 		err = check_reference_leak(env);
6467 		if (err) {
6468 			verbose(env, "tail_call would lead to reference leak\n");
6469 			return err;
6470 		}
6471 	} else if (is_release_function(func_id)) {
6472 		err = release_reference(env, meta.ref_obj_id);
6473 		if (err) {
6474 			verbose(env, "func %s#%d reference has not been acquired before\n",
6475 				func_id_name(func_id), func_id);
6476 			return err;
6477 		}
6478 	}
6479 
6480 	regs = cur_regs(env);
6481 
6482 	/* check that flags argument in get_local_storage(map, flags) is 0,
6483 	 * this is required because get_local_storage() can't return an error.
6484 	 */
6485 	if (func_id == BPF_FUNC_get_local_storage &&
6486 	    !register_is_null(&regs[BPF_REG_2])) {
6487 		verbose(env, "get_local_storage() doesn't support non-zero flags\n");
6488 		return -EINVAL;
6489 	}
6490 
6491 	if (func_id == BPF_FUNC_for_each_map_elem) {
6492 		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
6493 					set_map_elem_callback_state);
6494 		if (err < 0)
6495 			return -EINVAL;
6496 	}
6497 
6498 	if (func_id == BPF_FUNC_timer_set_callback) {
6499 		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
6500 					set_timer_callback_state);
6501 		if (err < 0)
6502 			return -EINVAL;
6503 	}
6504 
6505 	if (func_id == BPF_FUNC_snprintf) {
6506 		err = check_bpf_snprintf_call(env, regs);
6507 		if (err < 0)
6508 			return err;
6509 	}
6510 
6511 	/* reset caller saved regs */
6512 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
6513 		mark_reg_not_init(env, regs, caller_saved[i]);
6514 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
6515 	}
6516 
6517 	/* helper call returns 64-bit value. */
6518 	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
6519 
6520 	/* update return register (already marked as written above) */
6521 	if (fn->ret_type == RET_INTEGER) {
6522 		/* sets type to SCALAR_VALUE */
6523 		mark_reg_unknown(env, regs, BPF_REG_0);
6524 	} else if (fn->ret_type == RET_VOID) {
6525 		regs[BPF_REG_0].type = NOT_INIT;
6526 	} else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
6527 		   fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6528 		/* There is no offset yet applied, variable or fixed */
6529 		mark_reg_known_zero(env, regs, BPF_REG_0);
6530 		/* remember map_ptr, so that check_map_access()
6531 		 * can check 'value_size' boundary of memory access
6532 		 * to map element returned from bpf_map_lookup_elem()
6533 		 */
6534 		if (meta.map_ptr == NULL) {
6535 			verbose(env,
6536 				"kernel subsystem misconfigured verifier\n");
6537 			return -EINVAL;
6538 		}
6539 		regs[BPF_REG_0].map_ptr = meta.map_ptr;
6540 		regs[BPF_REG_0].map_uid = meta.map_uid;
6541 		if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6542 			regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
6543 			if (map_value_has_spin_lock(meta.map_ptr))
6544 				regs[BPF_REG_0].id = ++env->id_gen;
6545 		} else {
6546 			regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
6547 		}
6548 	} else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
6549 		mark_reg_known_zero(env, regs, BPF_REG_0);
6550 		regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
6551 	} else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
6552 		mark_reg_known_zero(env, regs, BPF_REG_0);
6553 		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
6554 	} else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
6555 		mark_reg_known_zero(env, regs, BPF_REG_0);
6556 		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
6557 	} else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {
6558 		mark_reg_known_zero(env, regs, BPF_REG_0);
6559 		regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
6560 		regs[BPF_REG_0].mem_size = meta.mem_size;
6561 	} else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL ||
6562 		   fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
6563 		const struct btf_type *t;
6564 
6565 		mark_reg_known_zero(env, regs, BPF_REG_0);
6566 		t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
6567 		if (!btf_type_is_struct(t)) {
6568 			u32 tsize;
6569 			const struct btf_type *ret;
6570 			const char *tname;
6571 
6572 			/* resolve the type size of ksym. */
6573 			ret = btf_resolve_size(meta.ret_btf, t, &tsize);
6574 			if (IS_ERR(ret)) {
6575 				tname = btf_name_by_offset(meta.ret_btf, t->name_off);
6576 				verbose(env, "unable to resolve the size of type '%s': %ld\n",
6577 					tname, PTR_ERR(ret));
6578 				return -EINVAL;
6579 			}
6580 			regs[BPF_REG_0].type =
6581 				fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6582 				PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
6583 			regs[BPF_REG_0].mem_size = tsize;
6584 		} else {
6585 			regs[BPF_REG_0].type =
6586 				fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6587 				PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
6588 			regs[BPF_REG_0].btf = meta.ret_btf;
6589 			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
6590 		}
6591 	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL ||
6592 		   fn->ret_type == RET_PTR_TO_BTF_ID) {
6593 		int ret_btf_id;
6594 
6595 		mark_reg_known_zero(env, regs, BPF_REG_0);
6596 		regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
6597 						     PTR_TO_BTF_ID :
6598 						     PTR_TO_BTF_ID_OR_NULL;
6599 		ret_btf_id = *fn->ret_btf_id;
6600 		if (ret_btf_id == 0) {
6601 			verbose(env, "invalid return type %d of func %s#%d\n",
6602 				fn->ret_type, func_id_name(func_id), func_id);
6603 			return -EINVAL;
6604 		}
6605 		/* current BPF helper definitions are only coming from
6606 		 * built-in code with type IDs from  vmlinux BTF
6607 		 */
6608 		regs[BPF_REG_0].btf = btf_vmlinux;
6609 		regs[BPF_REG_0].btf_id = ret_btf_id;
6610 	} else {
6611 		verbose(env, "unknown return type %d of func %s#%d\n",
6612 			fn->ret_type, func_id_name(func_id), func_id);
6613 		return -EINVAL;
6614 	}
6615 
6616 	if (reg_type_may_be_null(regs[BPF_REG_0].type))
6617 		regs[BPF_REG_0].id = ++env->id_gen;
6618 
6619 	if (is_ptr_cast_function(func_id)) {
6620 		/* For release_reference() */
6621 		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
6622 	} else if (is_acquire_function(func_id, meta.map_ptr)) {
6623 		int id = acquire_reference_state(env, insn_idx);
6624 
6625 		if (id < 0)
6626 			return id;
6627 		/* For mark_ptr_or_null_reg() */
6628 		regs[BPF_REG_0].id = id;
6629 		/* For release_reference() */
6630 		regs[BPF_REG_0].ref_obj_id = id;
6631 	}
6632 
6633 	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
6634 
6635 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
6636 	if (err)
6637 		return err;
6638 
6639 	if ((func_id == BPF_FUNC_get_stack ||
6640 	     func_id == BPF_FUNC_get_task_stack) &&
6641 	    !env->prog->has_callchain_buf) {
6642 		const char *err_str;
6643 
6644 #ifdef CONFIG_PERF_EVENTS
6645 		err = get_callchain_buffers(sysctl_perf_event_max_stack);
6646 		err_str = "cannot get callchain buffer for func %s#%d\n";
6647 #else
6648 		err = -ENOTSUPP;
6649 		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
6650 #endif
6651 		if (err) {
6652 			verbose(env, err_str, func_id_name(func_id), func_id);
6653 			return err;
6654 		}
6655 
6656 		env->prog->has_callchain_buf = true;
6657 	}
6658 
6659 	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
6660 		env->prog->call_get_stack = true;
6661 
6662 	if (func_id == BPF_FUNC_get_func_ip) {
6663 		if (check_get_func_ip(env))
6664 			return -ENOTSUPP;
6665 		env->prog->call_get_func_ip = true;
6666 	}
6667 
6668 	if (changes_data)
6669 		clear_all_pkt_pointers(env);
6670 	return 0;
6671 }
6672 
6673 /* mark_btf_func_reg_size() is used when the reg size is determined by
6674  * the BTF func_proto's return value size and argument.
6675  */
6676 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
6677 				   size_t reg_size)
6678 {
6679 	struct bpf_reg_state *reg = &cur_regs(env)[regno];
6680 
6681 	if (regno == BPF_REG_0) {
6682 		/* Function return value */
6683 		reg->live |= REG_LIVE_WRITTEN;
6684 		reg->subreg_def = reg_size == sizeof(u64) ?
6685 			DEF_NOT_SUBREG : env->insn_idx + 1;
6686 	} else {
6687 		/* Function argument */
6688 		if (reg_size == sizeof(u64)) {
6689 			mark_insn_zext(env, reg);
6690 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
6691 		} else {
6692 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
6693 		}
6694 	}
6695 }
6696 
6697 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
6698 {
6699 	const struct btf_type *t, *func, *func_proto, *ptr_type;
6700 	struct bpf_reg_state *regs = cur_regs(env);
6701 	const char *func_name, *ptr_type_name;
6702 	u32 i, nargs, func_id, ptr_type_id;
6703 	struct module *btf_mod = NULL;
6704 	const struct btf_param *args;
6705 	struct btf *desc_btf;
6706 	int err;
6707 
6708 	/* skip for now, but return error when we find this in fixup_kfunc_call */
6709 	if (!insn->imm)
6710 		return 0;
6711 
6712 	desc_btf = find_kfunc_desc_btf(env, insn->imm, insn->off, &btf_mod);
6713 	if (IS_ERR(desc_btf))
6714 		return PTR_ERR(desc_btf);
6715 
6716 	func_id = insn->imm;
6717 	func = btf_type_by_id(desc_btf, func_id);
6718 	func_name = btf_name_by_offset(desc_btf, func->name_off);
6719 	func_proto = btf_type_by_id(desc_btf, func->type);
6720 
6721 	if (!env->ops->check_kfunc_call ||
6722 	    !env->ops->check_kfunc_call(func_id, btf_mod)) {
6723 		verbose(env, "calling kernel function %s is not allowed\n",
6724 			func_name);
6725 		return -EACCES;
6726 	}
6727 
6728 	/* Check the arguments */
6729 	err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs);
6730 	if (err)
6731 		return err;
6732 
6733 	for (i = 0; i < CALLER_SAVED_REGS; i++)
6734 		mark_reg_not_init(env, regs, caller_saved[i]);
6735 
6736 	/* Check return type */
6737 	t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL);
6738 	if (btf_type_is_scalar(t)) {
6739 		mark_reg_unknown(env, regs, BPF_REG_0);
6740 		mark_btf_func_reg_size(env, BPF_REG_0, t->size);
6741 	} else if (btf_type_is_ptr(t)) {
6742 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type,
6743 						   &ptr_type_id);
6744 		if (!btf_type_is_struct(ptr_type)) {
6745 			ptr_type_name = btf_name_by_offset(desc_btf,
6746 							   ptr_type->name_off);
6747 			verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
6748 				func_name, btf_type_str(ptr_type),
6749 				ptr_type_name);
6750 			return -EINVAL;
6751 		}
6752 		mark_reg_known_zero(env, regs, BPF_REG_0);
6753 		regs[BPF_REG_0].btf = desc_btf;
6754 		regs[BPF_REG_0].type = PTR_TO_BTF_ID;
6755 		regs[BPF_REG_0].btf_id = ptr_type_id;
6756 		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
6757 	} /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
6758 
6759 	nargs = btf_type_vlen(func_proto);
6760 	args = (const struct btf_param *)(func_proto + 1);
6761 	for (i = 0; i < nargs; i++) {
6762 		u32 regno = i + 1;
6763 
6764 		t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
6765 		if (btf_type_is_ptr(t))
6766 			mark_btf_func_reg_size(env, regno, sizeof(void *));
6767 		else
6768 			/* scalar. ensured by btf_check_kfunc_arg_match() */
6769 			mark_btf_func_reg_size(env, regno, t->size);
6770 	}
6771 
6772 	return 0;
6773 }
6774 
6775 static bool signed_add_overflows(s64 a, s64 b)
6776 {
6777 	/* Do the add in u64, where overflow is well-defined */
6778 	s64 res = (s64)((u64)a + (u64)b);
6779 
6780 	if (b < 0)
6781 		return res > a;
6782 	return res < a;
6783 }
6784 
6785 static bool signed_add32_overflows(s32 a, s32 b)
6786 {
6787 	/* Do the add in u32, where overflow is well-defined */
6788 	s32 res = (s32)((u32)a + (u32)b);
6789 
6790 	if (b < 0)
6791 		return res > a;
6792 	return res < a;
6793 }
6794 
6795 static bool signed_sub_overflows(s64 a, s64 b)
6796 {
6797 	/* Do the sub in u64, where overflow is well-defined */
6798 	s64 res = (s64)((u64)a - (u64)b);
6799 
6800 	if (b < 0)
6801 		return res < a;
6802 	return res > a;
6803 }
6804 
6805 static bool signed_sub32_overflows(s32 a, s32 b)
6806 {
6807 	/* Do the sub in u32, where overflow is well-defined */
6808 	s32 res = (s32)((u32)a - (u32)b);
6809 
6810 	if (b < 0)
6811 		return res < a;
6812 	return res > a;
6813 }
6814 
6815 static bool check_reg_sane_offset(struct bpf_verifier_env *env,
6816 				  const struct bpf_reg_state *reg,
6817 				  enum bpf_reg_type type)
6818 {
6819 	bool known = tnum_is_const(reg->var_off);
6820 	s64 val = reg->var_off.value;
6821 	s64 smin = reg->smin_value;
6822 
6823 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
6824 		verbose(env, "math between %s pointer and %lld is not allowed\n",
6825 			reg_type_str[type], val);
6826 		return false;
6827 	}
6828 
6829 	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
6830 		verbose(env, "%s pointer offset %d is not allowed\n",
6831 			reg_type_str[type], reg->off);
6832 		return false;
6833 	}
6834 
6835 	if (smin == S64_MIN) {
6836 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
6837 			reg_type_str[type]);
6838 		return false;
6839 	}
6840 
6841 	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
6842 		verbose(env, "value %lld makes %s pointer be out of bounds\n",
6843 			smin, reg_type_str[type]);
6844 		return false;
6845 	}
6846 
6847 	return true;
6848 }
6849 
6850 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
6851 {
6852 	return &env->insn_aux_data[env->insn_idx];
6853 }
6854 
6855 enum {
6856 	REASON_BOUNDS	= -1,
6857 	REASON_TYPE	= -2,
6858 	REASON_PATHS	= -3,
6859 	REASON_LIMIT	= -4,
6860 	REASON_STACK	= -5,
6861 };
6862 
6863 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
6864 			      u32 *alu_limit, bool mask_to_left)
6865 {
6866 	u32 max = 0, ptr_limit = 0;
6867 
6868 	switch (ptr_reg->type) {
6869 	case PTR_TO_STACK:
6870 		/* Offset 0 is out-of-bounds, but acceptable start for the
6871 		 * left direction, see BPF_REG_FP. Also, unknown scalar
6872 		 * offset where we would need to deal with min/max bounds is
6873 		 * currently prohibited for unprivileged.
6874 		 */
6875 		max = MAX_BPF_STACK + mask_to_left;
6876 		ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
6877 		break;
6878 	case PTR_TO_MAP_VALUE:
6879 		max = ptr_reg->map_ptr->value_size;
6880 		ptr_limit = (mask_to_left ?
6881 			     ptr_reg->smin_value :
6882 			     ptr_reg->umax_value) + ptr_reg->off;
6883 		break;
6884 	default:
6885 		return REASON_TYPE;
6886 	}
6887 
6888 	if (ptr_limit >= max)
6889 		return REASON_LIMIT;
6890 	*alu_limit = ptr_limit;
6891 	return 0;
6892 }
6893 
6894 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
6895 				    const struct bpf_insn *insn)
6896 {
6897 	return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
6898 }
6899 
6900 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
6901 				       u32 alu_state, u32 alu_limit)
6902 {
6903 	/* If we arrived here from different branches with different
6904 	 * state or limits to sanitize, then this won't work.
6905 	 */
6906 	if (aux->alu_state &&
6907 	    (aux->alu_state != alu_state ||
6908 	     aux->alu_limit != alu_limit))
6909 		return REASON_PATHS;
6910 
6911 	/* Corresponding fixup done in do_misc_fixups(). */
6912 	aux->alu_state = alu_state;
6913 	aux->alu_limit = alu_limit;
6914 	return 0;
6915 }
6916 
6917 static int sanitize_val_alu(struct bpf_verifier_env *env,
6918 			    struct bpf_insn *insn)
6919 {
6920 	struct bpf_insn_aux_data *aux = cur_aux(env);
6921 
6922 	if (can_skip_alu_sanitation(env, insn))
6923 		return 0;
6924 
6925 	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
6926 }
6927 
6928 static bool sanitize_needed(u8 opcode)
6929 {
6930 	return opcode == BPF_ADD || opcode == BPF_SUB;
6931 }
6932 
6933 struct bpf_sanitize_info {
6934 	struct bpf_insn_aux_data aux;
6935 	bool mask_to_left;
6936 };
6937 
6938 static struct bpf_verifier_state *
6939 sanitize_speculative_path(struct bpf_verifier_env *env,
6940 			  const struct bpf_insn *insn,
6941 			  u32 next_idx, u32 curr_idx)
6942 {
6943 	struct bpf_verifier_state *branch;
6944 	struct bpf_reg_state *regs;
6945 
6946 	branch = push_stack(env, next_idx, curr_idx, true);
6947 	if (branch && insn) {
6948 		regs = branch->frame[branch->curframe]->regs;
6949 		if (BPF_SRC(insn->code) == BPF_K) {
6950 			mark_reg_unknown(env, regs, insn->dst_reg);
6951 		} else if (BPF_SRC(insn->code) == BPF_X) {
6952 			mark_reg_unknown(env, regs, insn->dst_reg);
6953 			mark_reg_unknown(env, regs, insn->src_reg);
6954 		}
6955 	}
6956 	return branch;
6957 }
6958 
6959 static int sanitize_ptr_alu(struct bpf_verifier_env *env,
6960 			    struct bpf_insn *insn,
6961 			    const struct bpf_reg_state *ptr_reg,
6962 			    const struct bpf_reg_state *off_reg,
6963 			    struct bpf_reg_state *dst_reg,
6964 			    struct bpf_sanitize_info *info,
6965 			    const bool commit_window)
6966 {
6967 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
6968 	struct bpf_verifier_state *vstate = env->cur_state;
6969 	bool off_is_imm = tnum_is_const(off_reg->var_off);
6970 	bool off_is_neg = off_reg->smin_value < 0;
6971 	bool ptr_is_dst_reg = ptr_reg == dst_reg;
6972 	u8 opcode = BPF_OP(insn->code);
6973 	u32 alu_state, alu_limit;
6974 	struct bpf_reg_state tmp;
6975 	bool ret;
6976 	int err;
6977 
6978 	if (can_skip_alu_sanitation(env, insn))
6979 		return 0;
6980 
6981 	/* We already marked aux for masking from non-speculative
6982 	 * paths, thus we got here in the first place. We only care
6983 	 * to explore bad access from here.
6984 	 */
6985 	if (vstate->speculative)
6986 		goto do_sim;
6987 
6988 	if (!commit_window) {
6989 		if (!tnum_is_const(off_reg->var_off) &&
6990 		    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
6991 			return REASON_BOUNDS;
6992 
6993 		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
6994 				     (opcode == BPF_SUB && !off_is_neg);
6995 	}
6996 
6997 	err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
6998 	if (err < 0)
6999 		return err;
7000 
7001 	if (commit_window) {
7002 		/* In commit phase we narrow the masking window based on
7003 		 * the observed pointer move after the simulated operation.
7004 		 */
7005 		alu_state = info->aux.alu_state;
7006 		alu_limit = abs(info->aux.alu_limit - alu_limit);
7007 	} else {
7008 		alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
7009 		alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
7010 		alu_state |= ptr_is_dst_reg ?
7011 			     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
7012 
7013 		/* Limit pruning on unknown scalars to enable deep search for
7014 		 * potential masking differences from other program paths.
7015 		 */
7016 		if (!off_is_imm)
7017 			env->explore_alu_limits = true;
7018 	}
7019 
7020 	err = update_alu_sanitation_state(aux, alu_state, alu_limit);
7021 	if (err < 0)
7022 		return err;
7023 do_sim:
7024 	/* If we're in commit phase, we're done here given we already
7025 	 * pushed the truncated dst_reg into the speculative verification
7026 	 * stack.
7027 	 *
7028 	 * Also, when register is a known constant, we rewrite register-based
7029 	 * operation to immediate-based, and thus do not need masking (and as
7030 	 * a consequence, do not need to simulate the zero-truncation either).
7031 	 */
7032 	if (commit_window || off_is_imm)
7033 		return 0;
7034 
7035 	/* Simulate and find potential out-of-bounds access under
7036 	 * speculative execution from truncation as a result of
7037 	 * masking when off was not within expected range. If off
7038 	 * sits in dst, then we temporarily need to move ptr there
7039 	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
7040 	 * for cases where we use K-based arithmetic in one direction
7041 	 * and truncated reg-based in the other in order to explore
7042 	 * bad access.
7043 	 */
7044 	if (!ptr_is_dst_reg) {
7045 		tmp = *dst_reg;
7046 		*dst_reg = *ptr_reg;
7047 	}
7048 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
7049 					env->insn_idx);
7050 	if (!ptr_is_dst_reg && ret)
7051 		*dst_reg = tmp;
7052 	return !ret ? REASON_STACK : 0;
7053 }
7054 
7055 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
7056 {
7057 	struct bpf_verifier_state *vstate = env->cur_state;
7058 
7059 	/* If we simulate paths under speculation, we don't update the
7060 	 * insn as 'seen' such that when we verify unreachable paths in
7061 	 * the non-speculative domain, sanitize_dead_code() can still
7062 	 * rewrite/sanitize them.
7063 	 */
7064 	if (!vstate->speculative)
7065 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
7066 }
7067 
7068 static int sanitize_err(struct bpf_verifier_env *env,
7069 			const struct bpf_insn *insn, int reason,
7070 			const struct bpf_reg_state *off_reg,
7071 			const struct bpf_reg_state *dst_reg)
7072 {
7073 	static const char *err = "pointer arithmetic with it prohibited for !root";
7074 	const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
7075 	u32 dst = insn->dst_reg, src = insn->src_reg;
7076 
7077 	switch (reason) {
7078 	case REASON_BOUNDS:
7079 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
7080 			off_reg == dst_reg ? dst : src, err);
7081 		break;
7082 	case REASON_TYPE:
7083 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
7084 			off_reg == dst_reg ? src : dst, err);
7085 		break;
7086 	case REASON_PATHS:
7087 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
7088 			dst, op, err);
7089 		break;
7090 	case REASON_LIMIT:
7091 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
7092 			dst, op, err);
7093 		break;
7094 	case REASON_STACK:
7095 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",
7096 			dst, err);
7097 		break;
7098 	default:
7099 		verbose(env, "verifier internal error: unknown reason (%d)\n",
7100 			reason);
7101 		break;
7102 	}
7103 
7104 	return -EACCES;
7105 }
7106 
7107 /* check that stack access falls within stack limits and that 'reg' doesn't
7108  * have a variable offset.
7109  *
7110  * Variable offset is prohibited for unprivileged mode for simplicity since it
7111  * requires corresponding support in Spectre masking for stack ALU.  See also
7112  * retrieve_ptr_limit().
7113  *
7114  *
7115  * 'off' includes 'reg->off'.
7116  */
7117 static int check_stack_access_for_ptr_arithmetic(
7118 				struct bpf_verifier_env *env,
7119 				int regno,
7120 				const struct bpf_reg_state *reg,
7121 				int off)
7122 {
7123 	if (!tnum_is_const(reg->var_off)) {
7124 		char tn_buf[48];
7125 
7126 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7127 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
7128 			regno, tn_buf, off);
7129 		return -EACCES;
7130 	}
7131 
7132 	if (off >= 0 || off < -MAX_BPF_STACK) {
7133 		verbose(env, "R%d stack pointer arithmetic goes out of range, "
7134 			"prohibited for !root; off=%d\n", regno, off);
7135 		return -EACCES;
7136 	}
7137 
7138 	return 0;
7139 }
7140 
7141 static int sanitize_check_bounds(struct bpf_verifier_env *env,
7142 				 const struct bpf_insn *insn,
7143 				 const struct bpf_reg_state *dst_reg)
7144 {
7145 	u32 dst = insn->dst_reg;
7146 
7147 	/* For unprivileged we require that resulting offset must be in bounds
7148 	 * in order to be able to sanitize access later on.
7149 	 */
7150 	if (env->bypass_spec_v1)
7151 		return 0;
7152 
7153 	switch (dst_reg->type) {
7154 	case PTR_TO_STACK:
7155 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
7156 					dst_reg->off + dst_reg->var_off.value))
7157 			return -EACCES;
7158 		break;
7159 	case PTR_TO_MAP_VALUE:
7160 		if (check_map_access(env, dst, dst_reg->off, 1, false)) {
7161 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
7162 				"prohibited for !root\n", dst);
7163 			return -EACCES;
7164 		}
7165 		break;
7166 	default:
7167 		break;
7168 	}
7169 
7170 	return 0;
7171 }
7172 
7173 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
7174  * Caller should also handle BPF_MOV case separately.
7175  * If we return -EACCES, caller may want to try again treating pointer as a
7176  * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
7177  */
7178 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
7179 				   struct bpf_insn *insn,
7180 				   const struct bpf_reg_state *ptr_reg,
7181 				   const struct bpf_reg_state *off_reg)
7182 {
7183 	struct bpf_verifier_state *vstate = env->cur_state;
7184 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
7185 	struct bpf_reg_state *regs = state->regs, *dst_reg;
7186 	bool known = tnum_is_const(off_reg->var_off);
7187 	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
7188 	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
7189 	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
7190 	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
7191 	struct bpf_sanitize_info info = {};
7192 	u8 opcode = BPF_OP(insn->code);
7193 	u32 dst = insn->dst_reg;
7194 	int ret;
7195 
7196 	dst_reg = &regs[dst];
7197 
7198 	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
7199 	    smin_val > smax_val || umin_val > umax_val) {
7200 		/* Taint dst register if offset had invalid bounds derived from
7201 		 * e.g. dead branches.
7202 		 */
7203 		__mark_reg_unknown(env, dst_reg);
7204 		return 0;
7205 	}
7206 
7207 	if (BPF_CLASS(insn->code) != BPF_ALU64) {
7208 		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
7209 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {
7210 			__mark_reg_unknown(env, dst_reg);
7211 			return 0;
7212 		}
7213 
7214 		verbose(env,
7215 			"R%d 32-bit pointer arithmetic prohibited\n",
7216 			dst);
7217 		return -EACCES;
7218 	}
7219 
7220 	switch (ptr_reg->type) {
7221 	case PTR_TO_MAP_VALUE_OR_NULL:
7222 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
7223 			dst, reg_type_str[ptr_reg->type]);
7224 		return -EACCES;
7225 	case CONST_PTR_TO_MAP:
7226 		/* smin_val represents the known value */
7227 		if (known && smin_val == 0 && opcode == BPF_ADD)
7228 			break;
7229 		fallthrough;
7230 	case PTR_TO_PACKET_END:
7231 	case PTR_TO_SOCKET:
7232 	case PTR_TO_SOCKET_OR_NULL:
7233 	case PTR_TO_SOCK_COMMON:
7234 	case PTR_TO_SOCK_COMMON_OR_NULL:
7235 	case PTR_TO_TCP_SOCK:
7236 	case PTR_TO_TCP_SOCK_OR_NULL:
7237 	case PTR_TO_XDP_SOCK:
7238 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
7239 			dst, reg_type_str[ptr_reg->type]);
7240 		return -EACCES;
7241 	default:
7242 		break;
7243 	}
7244 
7245 	/* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
7246 	 * The id may be overwritten later if we create a new variable offset.
7247 	 */
7248 	dst_reg->type = ptr_reg->type;
7249 	dst_reg->id = ptr_reg->id;
7250 
7251 	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
7252 	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
7253 		return -EINVAL;
7254 
7255 	/* pointer types do not carry 32-bit bounds at the moment. */
7256 	__mark_reg32_unbounded(dst_reg);
7257 
7258 	if (sanitize_needed(opcode)) {
7259 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
7260 				       &info, false);
7261 		if (ret < 0)
7262 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
7263 	}
7264 
7265 	switch (opcode) {
7266 	case BPF_ADD:
7267 		/* We can take a fixed offset as long as it doesn't overflow
7268 		 * the s32 'off' field
7269 		 */
7270 		if (known && (ptr_reg->off + smin_val ==
7271 			      (s64)(s32)(ptr_reg->off + smin_val))) {
7272 			/* pointer += K.  Accumulate it into fixed offset */
7273 			dst_reg->smin_value = smin_ptr;
7274 			dst_reg->smax_value = smax_ptr;
7275 			dst_reg->umin_value = umin_ptr;
7276 			dst_reg->umax_value = umax_ptr;
7277 			dst_reg->var_off = ptr_reg->var_off;
7278 			dst_reg->off = ptr_reg->off + smin_val;
7279 			dst_reg->raw = ptr_reg->raw;
7280 			break;
7281 		}
7282 		/* A new variable offset is created.  Note that off_reg->off
7283 		 * == 0, since it's a scalar.
7284 		 * dst_reg gets the pointer type and since some positive
7285 		 * integer value was added to the pointer, give it a new 'id'
7286 		 * if it's a PTR_TO_PACKET.
7287 		 * this creates a new 'base' pointer, off_reg (variable) gets
7288 		 * added into the variable offset, and we copy the fixed offset
7289 		 * from ptr_reg.
7290 		 */
7291 		if (signed_add_overflows(smin_ptr, smin_val) ||
7292 		    signed_add_overflows(smax_ptr, smax_val)) {
7293 			dst_reg->smin_value = S64_MIN;
7294 			dst_reg->smax_value = S64_MAX;
7295 		} else {
7296 			dst_reg->smin_value = smin_ptr + smin_val;
7297 			dst_reg->smax_value = smax_ptr + smax_val;
7298 		}
7299 		if (umin_ptr + umin_val < umin_ptr ||
7300 		    umax_ptr + umax_val < umax_ptr) {
7301 			dst_reg->umin_value = 0;
7302 			dst_reg->umax_value = U64_MAX;
7303 		} else {
7304 			dst_reg->umin_value = umin_ptr + umin_val;
7305 			dst_reg->umax_value = umax_ptr + umax_val;
7306 		}
7307 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
7308 		dst_reg->off = ptr_reg->off;
7309 		dst_reg->raw = ptr_reg->raw;
7310 		if (reg_is_pkt_pointer(ptr_reg)) {
7311 			dst_reg->id = ++env->id_gen;
7312 			/* something was added to pkt_ptr, set range to zero */
7313 			memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
7314 		}
7315 		break;
7316 	case BPF_SUB:
7317 		if (dst_reg == off_reg) {
7318 			/* scalar -= pointer.  Creates an unknown scalar */
7319 			verbose(env, "R%d tried to subtract pointer from scalar\n",
7320 				dst);
7321 			return -EACCES;
7322 		}
7323 		/* We don't allow subtraction from FP, because (according to
7324 		 * test_verifier.c test "invalid fp arithmetic", JITs might not
7325 		 * be able to deal with it.
7326 		 */
7327 		if (ptr_reg->type == PTR_TO_STACK) {
7328 			verbose(env, "R%d subtraction from stack pointer prohibited\n",
7329 				dst);
7330 			return -EACCES;
7331 		}
7332 		if (known && (ptr_reg->off - smin_val ==
7333 			      (s64)(s32)(ptr_reg->off - smin_val))) {
7334 			/* pointer -= K.  Subtract it from fixed offset */
7335 			dst_reg->smin_value = smin_ptr;
7336 			dst_reg->smax_value = smax_ptr;
7337 			dst_reg->umin_value = umin_ptr;
7338 			dst_reg->umax_value = umax_ptr;
7339 			dst_reg->var_off = ptr_reg->var_off;
7340 			dst_reg->id = ptr_reg->id;
7341 			dst_reg->off = ptr_reg->off - smin_val;
7342 			dst_reg->raw = ptr_reg->raw;
7343 			break;
7344 		}
7345 		/* A new variable offset is created.  If the subtrahend is known
7346 		 * nonnegative, then any reg->range we had before is still good.
7347 		 */
7348 		if (signed_sub_overflows(smin_ptr, smax_val) ||
7349 		    signed_sub_overflows(smax_ptr, smin_val)) {
7350 			/* Overflow possible, we know nothing */
7351 			dst_reg->smin_value = S64_MIN;
7352 			dst_reg->smax_value = S64_MAX;
7353 		} else {
7354 			dst_reg->smin_value = smin_ptr - smax_val;
7355 			dst_reg->smax_value = smax_ptr - smin_val;
7356 		}
7357 		if (umin_ptr < umax_val) {
7358 			/* Overflow possible, we know nothing */
7359 			dst_reg->umin_value = 0;
7360 			dst_reg->umax_value = U64_MAX;
7361 		} else {
7362 			/* Cannot overflow (as long as bounds are consistent) */
7363 			dst_reg->umin_value = umin_ptr - umax_val;
7364 			dst_reg->umax_value = umax_ptr - umin_val;
7365 		}
7366 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
7367 		dst_reg->off = ptr_reg->off;
7368 		dst_reg->raw = ptr_reg->raw;
7369 		if (reg_is_pkt_pointer(ptr_reg)) {
7370 			dst_reg->id = ++env->id_gen;
7371 			/* something was added to pkt_ptr, set range to zero */
7372 			if (smin_val < 0)
7373 				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
7374 		}
7375 		break;
7376 	case BPF_AND:
7377 	case BPF_OR:
7378 	case BPF_XOR:
7379 		/* bitwise ops on pointers are troublesome, prohibit. */
7380 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
7381 			dst, bpf_alu_string[opcode >> 4]);
7382 		return -EACCES;
7383 	default:
7384 		/* other operators (e.g. MUL,LSH) produce non-pointer results */
7385 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
7386 			dst, bpf_alu_string[opcode >> 4]);
7387 		return -EACCES;
7388 	}
7389 
7390 	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
7391 		return -EINVAL;
7392 
7393 	__update_reg_bounds(dst_reg);
7394 	__reg_deduce_bounds(dst_reg);
7395 	__reg_bound_offset(dst_reg);
7396 
7397 	if (sanitize_check_bounds(env, insn, dst_reg) < 0)
7398 		return -EACCES;
7399 	if (sanitize_needed(opcode)) {
7400 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
7401 				       &info, true);
7402 		if (ret < 0)
7403 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
7404 	}
7405 
7406 	return 0;
7407 }
7408 
7409 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
7410 				 struct bpf_reg_state *src_reg)
7411 {
7412 	s32 smin_val = src_reg->s32_min_value;
7413 	s32 smax_val = src_reg->s32_max_value;
7414 	u32 umin_val = src_reg->u32_min_value;
7415 	u32 umax_val = src_reg->u32_max_value;
7416 
7417 	if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) ||
7418 	    signed_add32_overflows(dst_reg->s32_max_value, smax_val)) {
7419 		dst_reg->s32_min_value = S32_MIN;
7420 		dst_reg->s32_max_value = S32_MAX;
7421 	} else {
7422 		dst_reg->s32_min_value += smin_val;
7423 		dst_reg->s32_max_value += smax_val;
7424 	}
7425 	if (dst_reg->u32_min_value + umin_val < umin_val ||
7426 	    dst_reg->u32_max_value + umax_val < umax_val) {
7427 		dst_reg->u32_min_value = 0;
7428 		dst_reg->u32_max_value = U32_MAX;
7429 	} else {
7430 		dst_reg->u32_min_value += umin_val;
7431 		dst_reg->u32_max_value += umax_val;
7432 	}
7433 }
7434 
7435 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
7436 			       struct bpf_reg_state *src_reg)
7437 {
7438 	s64 smin_val = src_reg->smin_value;
7439 	s64 smax_val = src_reg->smax_value;
7440 	u64 umin_val = src_reg->umin_value;
7441 	u64 umax_val = src_reg->umax_value;
7442 
7443 	if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
7444 	    signed_add_overflows(dst_reg->smax_value, smax_val)) {
7445 		dst_reg->smin_value = S64_MIN;
7446 		dst_reg->smax_value = S64_MAX;
7447 	} else {
7448 		dst_reg->smin_value += smin_val;
7449 		dst_reg->smax_value += smax_val;
7450 	}
7451 	if (dst_reg->umin_value + umin_val < umin_val ||
7452 	    dst_reg->umax_value + umax_val < umax_val) {
7453 		dst_reg->umin_value = 0;
7454 		dst_reg->umax_value = U64_MAX;
7455 	} else {
7456 		dst_reg->umin_value += umin_val;
7457 		dst_reg->umax_value += umax_val;
7458 	}
7459 }
7460 
7461 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
7462 				 struct bpf_reg_state *src_reg)
7463 {
7464 	s32 smin_val = src_reg->s32_min_value;
7465 	s32 smax_val = src_reg->s32_max_value;
7466 	u32 umin_val = src_reg->u32_min_value;
7467 	u32 umax_val = src_reg->u32_max_value;
7468 
7469 	if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) ||
7470 	    signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) {
7471 		/* Overflow possible, we know nothing */
7472 		dst_reg->s32_min_value = S32_MIN;
7473 		dst_reg->s32_max_value = S32_MAX;
7474 	} else {
7475 		dst_reg->s32_min_value -= smax_val;
7476 		dst_reg->s32_max_value -= smin_val;
7477 	}
7478 	if (dst_reg->u32_min_value < umax_val) {
7479 		/* Overflow possible, we know nothing */
7480 		dst_reg->u32_min_value = 0;
7481 		dst_reg->u32_max_value = U32_MAX;
7482 	} else {
7483 		/* Cannot overflow (as long as bounds are consistent) */
7484 		dst_reg->u32_min_value -= umax_val;
7485 		dst_reg->u32_max_value -= umin_val;
7486 	}
7487 }
7488 
7489 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
7490 			       struct bpf_reg_state *src_reg)
7491 {
7492 	s64 smin_val = src_reg->smin_value;
7493 	s64 smax_val = src_reg->smax_value;
7494 	u64 umin_val = src_reg->umin_value;
7495 	u64 umax_val = src_reg->umax_value;
7496 
7497 	if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
7498 	    signed_sub_overflows(dst_reg->smax_value, smin_val)) {
7499 		/* Overflow possible, we know nothing */
7500 		dst_reg->smin_value = S64_MIN;
7501 		dst_reg->smax_value = S64_MAX;
7502 	} else {
7503 		dst_reg->smin_value -= smax_val;
7504 		dst_reg->smax_value -= smin_val;
7505 	}
7506 	if (dst_reg->umin_value < umax_val) {
7507 		/* Overflow possible, we know nothing */
7508 		dst_reg->umin_value = 0;
7509 		dst_reg->umax_value = U64_MAX;
7510 	} else {
7511 		/* Cannot overflow (as long as bounds are consistent) */
7512 		dst_reg->umin_value -= umax_val;
7513 		dst_reg->umax_value -= umin_val;
7514 	}
7515 }
7516 
7517 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
7518 				 struct bpf_reg_state *src_reg)
7519 {
7520 	s32 smin_val = src_reg->s32_min_value;
7521 	u32 umin_val = src_reg->u32_min_value;
7522 	u32 umax_val = src_reg->u32_max_value;
7523 
7524 	if (smin_val < 0 || dst_reg->s32_min_value < 0) {
7525 		/* Ain't nobody got time to multiply that sign */
7526 		__mark_reg32_unbounded(dst_reg);
7527 		return;
7528 	}
7529 	/* Both values are positive, so we can work with unsigned and
7530 	 * copy the result to signed (unless it exceeds S32_MAX).
7531 	 */
7532 	if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) {
7533 		/* Potential overflow, we know nothing */
7534 		__mark_reg32_unbounded(dst_reg);
7535 		return;
7536 	}
7537 	dst_reg->u32_min_value *= umin_val;
7538 	dst_reg->u32_max_value *= umax_val;
7539 	if (dst_reg->u32_max_value > S32_MAX) {
7540 		/* Overflow possible, we know nothing */
7541 		dst_reg->s32_min_value = S32_MIN;
7542 		dst_reg->s32_max_value = S32_MAX;
7543 	} else {
7544 		dst_reg->s32_min_value = dst_reg->u32_min_value;
7545 		dst_reg->s32_max_value = dst_reg->u32_max_value;
7546 	}
7547 }
7548 
7549 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
7550 			       struct bpf_reg_state *src_reg)
7551 {
7552 	s64 smin_val = src_reg->smin_value;
7553 	u64 umin_val = src_reg->umin_value;
7554 	u64 umax_val = src_reg->umax_value;
7555 
7556 	if (smin_val < 0 || dst_reg->smin_value < 0) {
7557 		/* Ain't nobody got time to multiply that sign */
7558 		__mark_reg64_unbounded(dst_reg);
7559 		return;
7560 	}
7561 	/* Both values are positive, so we can work with unsigned and
7562 	 * copy the result to signed (unless it exceeds S64_MAX).
7563 	 */
7564 	if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
7565 		/* Potential overflow, we know nothing */
7566 		__mark_reg64_unbounded(dst_reg);
7567 		return;
7568 	}
7569 	dst_reg->umin_value *= umin_val;
7570 	dst_reg->umax_value *= umax_val;
7571 	if (dst_reg->umax_value > S64_MAX) {
7572 		/* Overflow possible, we know nothing */
7573 		dst_reg->smin_value = S64_MIN;
7574 		dst_reg->smax_value = S64_MAX;
7575 	} else {
7576 		dst_reg->smin_value = dst_reg->umin_value;
7577 		dst_reg->smax_value = dst_reg->umax_value;
7578 	}
7579 }
7580 
7581 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
7582 				 struct bpf_reg_state *src_reg)
7583 {
7584 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
7585 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7586 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7587 	s32 smin_val = src_reg->s32_min_value;
7588 	u32 umax_val = src_reg->u32_max_value;
7589 
7590 	if (src_known && dst_known) {
7591 		__mark_reg32_known(dst_reg, var32_off.value);
7592 		return;
7593 	}
7594 
7595 	/* We get our minimum from the var_off, since that's inherently
7596 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
7597 	 */
7598 	dst_reg->u32_min_value = var32_off.value;
7599 	dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
7600 	if (dst_reg->s32_min_value < 0 || smin_val < 0) {
7601 		/* Lose signed bounds when ANDing negative numbers,
7602 		 * ain't nobody got time for that.
7603 		 */
7604 		dst_reg->s32_min_value = S32_MIN;
7605 		dst_reg->s32_max_value = S32_MAX;
7606 	} else {
7607 		/* ANDing two positives gives a positive, so safe to
7608 		 * cast result into s64.
7609 		 */
7610 		dst_reg->s32_min_value = dst_reg->u32_min_value;
7611 		dst_reg->s32_max_value = dst_reg->u32_max_value;
7612 	}
7613 }
7614 
7615 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
7616 			       struct bpf_reg_state *src_reg)
7617 {
7618 	bool src_known = tnum_is_const(src_reg->var_off);
7619 	bool dst_known = tnum_is_const(dst_reg->var_off);
7620 	s64 smin_val = src_reg->smin_value;
7621 	u64 umax_val = src_reg->umax_value;
7622 
7623 	if (src_known && dst_known) {
7624 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
7625 		return;
7626 	}
7627 
7628 	/* We get our minimum from the var_off, since that's inherently
7629 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
7630 	 */
7631 	dst_reg->umin_value = dst_reg->var_off.value;
7632 	dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
7633 	if (dst_reg->smin_value < 0 || smin_val < 0) {
7634 		/* Lose signed bounds when ANDing negative numbers,
7635 		 * ain't nobody got time for that.
7636 		 */
7637 		dst_reg->smin_value = S64_MIN;
7638 		dst_reg->smax_value = S64_MAX;
7639 	} else {
7640 		/* ANDing two positives gives a positive, so safe to
7641 		 * cast result into s64.
7642 		 */
7643 		dst_reg->smin_value = dst_reg->umin_value;
7644 		dst_reg->smax_value = dst_reg->umax_value;
7645 	}
7646 	/* We may learn something more from the var_off */
7647 	__update_reg_bounds(dst_reg);
7648 }
7649 
7650 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
7651 				struct bpf_reg_state *src_reg)
7652 {
7653 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
7654 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7655 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7656 	s32 smin_val = src_reg->s32_min_value;
7657 	u32 umin_val = src_reg->u32_min_value;
7658 
7659 	if (src_known && dst_known) {
7660 		__mark_reg32_known(dst_reg, var32_off.value);
7661 		return;
7662 	}
7663 
7664 	/* We get our maximum from the var_off, and our minimum is the
7665 	 * maximum of the operands' minima
7666 	 */
7667 	dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
7668 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
7669 	if (dst_reg->s32_min_value < 0 || smin_val < 0) {
7670 		/* Lose signed bounds when ORing negative numbers,
7671 		 * ain't nobody got time for that.
7672 		 */
7673 		dst_reg->s32_min_value = S32_MIN;
7674 		dst_reg->s32_max_value = S32_MAX;
7675 	} else {
7676 		/* ORing two positives gives a positive, so safe to
7677 		 * cast result into s64.
7678 		 */
7679 		dst_reg->s32_min_value = dst_reg->u32_min_value;
7680 		dst_reg->s32_max_value = dst_reg->u32_max_value;
7681 	}
7682 }
7683 
7684 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
7685 			      struct bpf_reg_state *src_reg)
7686 {
7687 	bool src_known = tnum_is_const(src_reg->var_off);
7688 	bool dst_known = tnum_is_const(dst_reg->var_off);
7689 	s64 smin_val = src_reg->smin_value;
7690 	u64 umin_val = src_reg->umin_value;
7691 
7692 	if (src_known && dst_known) {
7693 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
7694 		return;
7695 	}
7696 
7697 	/* We get our maximum from the var_off, and our minimum is the
7698 	 * maximum of the operands' minima
7699 	 */
7700 	dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
7701 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
7702 	if (dst_reg->smin_value < 0 || smin_val < 0) {
7703 		/* Lose signed bounds when ORing negative numbers,
7704 		 * ain't nobody got time for that.
7705 		 */
7706 		dst_reg->smin_value = S64_MIN;
7707 		dst_reg->smax_value = S64_MAX;
7708 	} else {
7709 		/* ORing two positives gives a positive, so safe to
7710 		 * cast result into s64.
7711 		 */
7712 		dst_reg->smin_value = dst_reg->umin_value;
7713 		dst_reg->smax_value = dst_reg->umax_value;
7714 	}
7715 	/* We may learn something more from the var_off */
7716 	__update_reg_bounds(dst_reg);
7717 }
7718 
7719 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
7720 				 struct bpf_reg_state *src_reg)
7721 {
7722 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
7723 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7724 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7725 	s32 smin_val = src_reg->s32_min_value;
7726 
7727 	if (src_known && dst_known) {
7728 		__mark_reg32_known(dst_reg, var32_off.value);
7729 		return;
7730 	}
7731 
7732 	/* We get both minimum and maximum from the var32_off. */
7733 	dst_reg->u32_min_value = var32_off.value;
7734 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
7735 
7736 	if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
7737 		/* XORing two positive sign numbers gives a positive,
7738 		 * so safe to cast u32 result into s32.
7739 		 */
7740 		dst_reg->s32_min_value = dst_reg->u32_min_value;
7741 		dst_reg->s32_max_value = dst_reg->u32_max_value;
7742 	} else {
7743 		dst_reg->s32_min_value = S32_MIN;
7744 		dst_reg->s32_max_value = S32_MAX;
7745 	}
7746 }
7747 
7748 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
7749 			       struct bpf_reg_state *src_reg)
7750 {
7751 	bool src_known = tnum_is_const(src_reg->var_off);
7752 	bool dst_known = tnum_is_const(dst_reg->var_off);
7753 	s64 smin_val = src_reg->smin_value;
7754 
7755 	if (src_known && dst_known) {
7756 		/* dst_reg->var_off.value has been updated earlier */
7757 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
7758 		return;
7759 	}
7760 
7761 	/* We get both minimum and maximum from the var_off. */
7762 	dst_reg->umin_value = dst_reg->var_off.value;
7763 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
7764 
7765 	if (dst_reg->smin_value >= 0 && smin_val >= 0) {
7766 		/* XORing two positive sign numbers gives a positive,
7767 		 * so safe to cast u64 result into s64.
7768 		 */
7769 		dst_reg->smin_value = dst_reg->umin_value;
7770 		dst_reg->smax_value = dst_reg->umax_value;
7771 	} else {
7772 		dst_reg->smin_value = S64_MIN;
7773 		dst_reg->smax_value = S64_MAX;
7774 	}
7775 
7776 	__update_reg_bounds(dst_reg);
7777 }
7778 
7779 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
7780 				   u64 umin_val, u64 umax_val)
7781 {
7782 	/* We lose all sign bit information (except what we can pick
7783 	 * up from var_off)
7784 	 */
7785 	dst_reg->s32_min_value = S32_MIN;
7786 	dst_reg->s32_max_value = S32_MAX;
7787 	/* If we might shift our top bit out, then we know nothing */
7788 	if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
7789 		dst_reg->u32_min_value = 0;
7790 		dst_reg->u32_max_value = U32_MAX;
7791 	} else {
7792 		dst_reg->u32_min_value <<= umin_val;
7793 		dst_reg->u32_max_value <<= umax_val;
7794 	}
7795 }
7796 
7797 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
7798 				 struct bpf_reg_state *src_reg)
7799 {
7800 	u32 umax_val = src_reg->u32_max_value;
7801 	u32 umin_val = src_reg->u32_min_value;
7802 	/* u32 alu operation will zext upper bits */
7803 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
7804 
7805 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
7806 	dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
7807 	/* Not required but being careful mark reg64 bounds as unknown so
7808 	 * that we are forced to pick them up from tnum and zext later and
7809 	 * if some path skips this step we are still safe.
7810 	 */
7811 	__mark_reg64_unbounded(dst_reg);
7812 	__update_reg32_bounds(dst_reg);
7813 }
7814 
7815 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
7816 				   u64 umin_val, u64 umax_val)
7817 {
7818 	/* Special case <<32 because it is a common compiler pattern to sign
7819 	 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
7820 	 * positive we know this shift will also be positive so we can track
7821 	 * bounds correctly. Otherwise we lose all sign bit information except
7822 	 * what we can pick up from var_off. Perhaps we can generalize this
7823 	 * later to shifts of any length.
7824 	 */
7825 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
7826 		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
7827 	else
7828 		dst_reg->smax_value = S64_MAX;
7829 
7830 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
7831 		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
7832 	else
7833 		dst_reg->smin_value = S64_MIN;
7834 
7835 	/* If we might shift our top bit out, then we know nothing */
7836 	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
7837 		dst_reg->umin_value = 0;
7838 		dst_reg->umax_value = U64_MAX;
7839 	} else {
7840 		dst_reg->umin_value <<= umin_val;
7841 		dst_reg->umax_value <<= umax_val;
7842 	}
7843 }
7844 
7845 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
7846 			       struct bpf_reg_state *src_reg)
7847 {
7848 	u64 umax_val = src_reg->umax_value;
7849 	u64 umin_val = src_reg->umin_value;
7850 
7851 	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
7852 	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
7853 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
7854 
7855 	dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
7856 	/* We may learn something more from the var_off */
7857 	__update_reg_bounds(dst_reg);
7858 }
7859 
7860 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
7861 				 struct bpf_reg_state *src_reg)
7862 {
7863 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
7864 	u32 umax_val = src_reg->u32_max_value;
7865 	u32 umin_val = src_reg->u32_min_value;
7866 
7867 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
7868 	 * be negative, then either:
7869 	 * 1) src_reg might be zero, so the sign bit of the result is
7870 	 *    unknown, so we lose our signed bounds
7871 	 * 2) it's known negative, thus the unsigned bounds capture the
7872 	 *    signed bounds
7873 	 * 3) the signed bounds cross zero, so they tell us nothing
7874 	 *    about the result
7875 	 * If the value in dst_reg is known nonnegative, then again the
7876 	 * unsigned bounds capture the signed bounds.
7877 	 * Thus, in all cases it suffices to blow away our signed bounds
7878 	 * and rely on inferring new ones from the unsigned bounds and
7879 	 * var_off of the result.
7880 	 */
7881 	dst_reg->s32_min_value = S32_MIN;
7882 	dst_reg->s32_max_value = S32_MAX;
7883 
7884 	dst_reg->var_off = tnum_rshift(subreg, umin_val);
7885 	dst_reg->u32_min_value >>= umax_val;
7886 	dst_reg->u32_max_value >>= umin_val;
7887 
7888 	__mark_reg64_unbounded(dst_reg);
7889 	__update_reg32_bounds(dst_reg);
7890 }
7891 
7892 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
7893 			       struct bpf_reg_state *src_reg)
7894 {
7895 	u64 umax_val = src_reg->umax_value;
7896 	u64 umin_val = src_reg->umin_value;
7897 
7898 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
7899 	 * be negative, then either:
7900 	 * 1) src_reg might be zero, so the sign bit of the result is
7901 	 *    unknown, so we lose our signed bounds
7902 	 * 2) it's known negative, thus the unsigned bounds capture the
7903 	 *    signed bounds
7904 	 * 3) the signed bounds cross zero, so they tell us nothing
7905 	 *    about the result
7906 	 * If the value in dst_reg is known nonnegative, then again the
7907 	 * unsigned bounds capture the signed bounds.
7908 	 * Thus, in all cases it suffices to blow away our signed bounds
7909 	 * and rely on inferring new ones from the unsigned bounds and
7910 	 * var_off of the result.
7911 	 */
7912 	dst_reg->smin_value = S64_MIN;
7913 	dst_reg->smax_value = S64_MAX;
7914 	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
7915 	dst_reg->umin_value >>= umax_val;
7916 	dst_reg->umax_value >>= umin_val;
7917 
7918 	/* Its not easy to operate on alu32 bounds here because it depends
7919 	 * on bits being shifted in. Take easy way out and mark unbounded
7920 	 * so we can recalculate later from tnum.
7921 	 */
7922 	__mark_reg32_unbounded(dst_reg);
7923 	__update_reg_bounds(dst_reg);
7924 }
7925 
7926 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
7927 				  struct bpf_reg_state *src_reg)
7928 {
7929 	u64 umin_val = src_reg->u32_min_value;
7930 
7931 	/* Upon reaching here, src_known is true and
7932 	 * umax_val is equal to umin_val.
7933 	 */
7934 	dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
7935 	dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
7936 
7937 	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
7938 
7939 	/* blow away the dst_reg umin_value/umax_value and rely on
7940 	 * dst_reg var_off to refine the result.
7941 	 */
7942 	dst_reg->u32_min_value = 0;
7943 	dst_reg->u32_max_value = U32_MAX;
7944 
7945 	__mark_reg64_unbounded(dst_reg);
7946 	__update_reg32_bounds(dst_reg);
7947 }
7948 
7949 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
7950 				struct bpf_reg_state *src_reg)
7951 {
7952 	u64 umin_val = src_reg->umin_value;
7953 
7954 	/* Upon reaching here, src_known is true and umax_val is equal
7955 	 * to umin_val.
7956 	 */
7957 	dst_reg->smin_value >>= umin_val;
7958 	dst_reg->smax_value >>= umin_val;
7959 
7960 	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
7961 
7962 	/* blow away the dst_reg umin_value/umax_value and rely on
7963 	 * dst_reg var_off to refine the result.
7964 	 */
7965 	dst_reg->umin_value = 0;
7966 	dst_reg->umax_value = U64_MAX;
7967 
7968 	/* Its not easy to operate on alu32 bounds here because it depends
7969 	 * on bits being shifted in from upper 32-bits. Take easy way out
7970 	 * and mark unbounded so we can recalculate later from tnum.
7971 	 */
7972 	__mark_reg32_unbounded(dst_reg);
7973 	__update_reg_bounds(dst_reg);
7974 }
7975 
7976 /* WARNING: This function does calculations on 64-bit values, but the actual
7977  * execution may occur on 32-bit values. Therefore, things like bitshifts
7978  * need extra checks in the 32-bit case.
7979  */
7980 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
7981 				      struct bpf_insn *insn,
7982 				      struct bpf_reg_state *dst_reg,
7983 				      struct bpf_reg_state src_reg)
7984 {
7985 	struct bpf_reg_state *regs = cur_regs(env);
7986 	u8 opcode = BPF_OP(insn->code);
7987 	bool src_known;
7988 	s64 smin_val, smax_val;
7989 	u64 umin_val, umax_val;
7990 	s32 s32_min_val, s32_max_val;
7991 	u32 u32_min_val, u32_max_val;
7992 	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
7993 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
7994 	int ret;
7995 
7996 	smin_val = src_reg.smin_value;
7997 	smax_val = src_reg.smax_value;
7998 	umin_val = src_reg.umin_value;
7999 	umax_val = src_reg.umax_value;
8000 
8001 	s32_min_val = src_reg.s32_min_value;
8002 	s32_max_val = src_reg.s32_max_value;
8003 	u32_min_val = src_reg.u32_min_value;
8004 	u32_max_val = src_reg.u32_max_value;
8005 
8006 	if (alu32) {
8007 		src_known = tnum_subreg_is_const(src_reg.var_off);
8008 		if ((src_known &&
8009 		     (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) ||
8010 		    s32_min_val > s32_max_val || u32_min_val > u32_max_val) {
8011 			/* Taint dst register if offset had invalid bounds
8012 			 * derived from e.g. dead branches.
8013 			 */
8014 			__mark_reg_unknown(env, dst_reg);
8015 			return 0;
8016 		}
8017 	} else {
8018 		src_known = tnum_is_const(src_reg.var_off);
8019 		if ((src_known &&
8020 		     (smin_val != smax_val || umin_val != umax_val)) ||
8021 		    smin_val > smax_val || umin_val > umax_val) {
8022 			/* Taint dst register if offset had invalid bounds
8023 			 * derived from e.g. dead branches.
8024 			 */
8025 			__mark_reg_unknown(env, dst_reg);
8026 			return 0;
8027 		}
8028 	}
8029 
8030 	if (!src_known &&
8031 	    opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
8032 		__mark_reg_unknown(env, dst_reg);
8033 		return 0;
8034 	}
8035 
8036 	if (sanitize_needed(opcode)) {
8037 		ret = sanitize_val_alu(env, insn);
8038 		if (ret < 0)
8039 			return sanitize_err(env, insn, ret, NULL, NULL);
8040 	}
8041 
8042 	/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
8043 	 * There are two classes of instructions: The first class we track both
8044 	 * alu32 and alu64 sign/unsigned bounds independently this provides the
8045 	 * greatest amount of precision when alu operations are mixed with jmp32
8046 	 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
8047 	 * and BPF_OR. This is possible because these ops have fairly easy to
8048 	 * understand and calculate behavior in both 32-bit and 64-bit alu ops.
8049 	 * See alu32 verifier tests for examples. The second class of
8050 	 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
8051 	 * with regards to tracking sign/unsigned bounds because the bits may
8052 	 * cross subreg boundaries in the alu64 case. When this happens we mark
8053 	 * the reg unbounded in the subreg bound space and use the resulting
8054 	 * tnum to calculate an approximation of the sign/unsigned bounds.
8055 	 */
8056 	switch (opcode) {
8057 	case BPF_ADD:
8058 		scalar32_min_max_add(dst_reg, &src_reg);
8059 		scalar_min_max_add(dst_reg, &src_reg);
8060 		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
8061 		break;
8062 	case BPF_SUB:
8063 		scalar32_min_max_sub(dst_reg, &src_reg);
8064 		scalar_min_max_sub(dst_reg, &src_reg);
8065 		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
8066 		break;
8067 	case BPF_MUL:
8068 		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
8069 		scalar32_min_max_mul(dst_reg, &src_reg);
8070 		scalar_min_max_mul(dst_reg, &src_reg);
8071 		break;
8072 	case BPF_AND:
8073 		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
8074 		scalar32_min_max_and(dst_reg, &src_reg);
8075 		scalar_min_max_and(dst_reg, &src_reg);
8076 		break;
8077 	case BPF_OR:
8078 		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
8079 		scalar32_min_max_or(dst_reg, &src_reg);
8080 		scalar_min_max_or(dst_reg, &src_reg);
8081 		break;
8082 	case BPF_XOR:
8083 		dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
8084 		scalar32_min_max_xor(dst_reg, &src_reg);
8085 		scalar_min_max_xor(dst_reg, &src_reg);
8086 		break;
8087 	case BPF_LSH:
8088 		if (umax_val >= insn_bitness) {
8089 			/* Shifts greater than 31 or 63 are undefined.
8090 			 * This includes shifts by a negative number.
8091 			 */
8092 			mark_reg_unknown(env, regs, insn->dst_reg);
8093 			break;
8094 		}
8095 		if (alu32)
8096 			scalar32_min_max_lsh(dst_reg, &src_reg);
8097 		else
8098 			scalar_min_max_lsh(dst_reg, &src_reg);
8099 		break;
8100 	case BPF_RSH:
8101 		if (umax_val >= insn_bitness) {
8102 			/* Shifts greater than 31 or 63 are undefined.
8103 			 * This includes shifts by a negative number.
8104 			 */
8105 			mark_reg_unknown(env, regs, insn->dst_reg);
8106 			break;
8107 		}
8108 		if (alu32)
8109 			scalar32_min_max_rsh(dst_reg, &src_reg);
8110 		else
8111 			scalar_min_max_rsh(dst_reg, &src_reg);
8112 		break;
8113 	case BPF_ARSH:
8114 		if (umax_val >= insn_bitness) {
8115 			/* Shifts greater than 31 or 63 are undefined.
8116 			 * This includes shifts by a negative number.
8117 			 */
8118 			mark_reg_unknown(env, regs, insn->dst_reg);
8119 			break;
8120 		}
8121 		if (alu32)
8122 			scalar32_min_max_arsh(dst_reg, &src_reg);
8123 		else
8124 			scalar_min_max_arsh(dst_reg, &src_reg);
8125 		break;
8126 	default:
8127 		mark_reg_unknown(env, regs, insn->dst_reg);
8128 		break;
8129 	}
8130 
8131 	/* ALU32 ops are zero extended into 64bit register */
8132 	if (alu32)
8133 		zext_32_to_64(dst_reg);
8134 
8135 	__update_reg_bounds(dst_reg);
8136 	__reg_deduce_bounds(dst_reg);
8137 	__reg_bound_offset(dst_reg);
8138 	return 0;
8139 }
8140 
8141 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
8142  * and var_off.
8143  */
8144 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
8145 				   struct bpf_insn *insn)
8146 {
8147 	struct bpf_verifier_state *vstate = env->cur_state;
8148 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
8149 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
8150 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
8151 	u8 opcode = BPF_OP(insn->code);
8152 	int err;
8153 
8154 	dst_reg = &regs[insn->dst_reg];
8155 	src_reg = NULL;
8156 	if (dst_reg->type != SCALAR_VALUE)
8157 		ptr_reg = dst_reg;
8158 	else
8159 		/* Make sure ID is cleared otherwise dst_reg min/max could be
8160 		 * incorrectly propagated into other registers by find_equal_scalars()
8161 		 */
8162 		dst_reg->id = 0;
8163 	if (BPF_SRC(insn->code) == BPF_X) {
8164 		src_reg = &regs[insn->src_reg];
8165 		if (src_reg->type != SCALAR_VALUE) {
8166 			if (dst_reg->type != SCALAR_VALUE) {
8167 				/* Combining two pointers by any ALU op yields
8168 				 * an arbitrary scalar. Disallow all math except
8169 				 * pointer subtraction
8170 				 */
8171 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
8172 					mark_reg_unknown(env, regs, insn->dst_reg);
8173 					return 0;
8174 				}
8175 				verbose(env, "R%d pointer %s pointer prohibited\n",
8176 					insn->dst_reg,
8177 					bpf_alu_string[opcode >> 4]);
8178 				return -EACCES;
8179 			} else {
8180 				/* scalar += pointer
8181 				 * This is legal, but we have to reverse our
8182 				 * src/dest handling in computing the range
8183 				 */
8184 				err = mark_chain_precision(env, insn->dst_reg);
8185 				if (err)
8186 					return err;
8187 				return adjust_ptr_min_max_vals(env, insn,
8188 							       src_reg, dst_reg);
8189 			}
8190 		} else if (ptr_reg) {
8191 			/* pointer += scalar */
8192 			err = mark_chain_precision(env, insn->src_reg);
8193 			if (err)
8194 				return err;
8195 			return adjust_ptr_min_max_vals(env, insn,
8196 						       dst_reg, src_reg);
8197 		}
8198 	} else {
8199 		/* Pretend the src is a reg with a known value, since we only
8200 		 * need to be able to read from this state.
8201 		 */
8202 		off_reg.type = SCALAR_VALUE;
8203 		__mark_reg_known(&off_reg, insn->imm);
8204 		src_reg = &off_reg;
8205 		if (ptr_reg) /* pointer += K */
8206 			return adjust_ptr_min_max_vals(env, insn,
8207 						       ptr_reg, src_reg);
8208 	}
8209 
8210 	/* Got here implies adding two SCALAR_VALUEs */
8211 	if (WARN_ON_ONCE(ptr_reg)) {
8212 		print_verifier_state(env, state);
8213 		verbose(env, "verifier internal error: unexpected ptr_reg\n");
8214 		return -EINVAL;
8215 	}
8216 	if (WARN_ON(!src_reg)) {
8217 		print_verifier_state(env, state);
8218 		verbose(env, "verifier internal error: no src_reg\n");
8219 		return -EINVAL;
8220 	}
8221 	return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
8222 }
8223 
8224 /* check validity of 32-bit and 64-bit arithmetic operations */
8225 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
8226 {
8227 	struct bpf_reg_state *regs = cur_regs(env);
8228 	u8 opcode = BPF_OP(insn->code);
8229 	int err;
8230 
8231 	if (opcode == BPF_END || opcode == BPF_NEG) {
8232 		if (opcode == BPF_NEG) {
8233 			if (BPF_SRC(insn->code) != 0 ||
8234 			    insn->src_reg != BPF_REG_0 ||
8235 			    insn->off != 0 || insn->imm != 0) {
8236 				verbose(env, "BPF_NEG uses reserved fields\n");
8237 				return -EINVAL;
8238 			}
8239 		} else {
8240 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
8241 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
8242 			    BPF_CLASS(insn->code) == BPF_ALU64) {
8243 				verbose(env, "BPF_END uses reserved fields\n");
8244 				return -EINVAL;
8245 			}
8246 		}
8247 
8248 		/* check src operand */
8249 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
8250 		if (err)
8251 			return err;
8252 
8253 		if (is_pointer_value(env, insn->dst_reg)) {
8254 			verbose(env, "R%d pointer arithmetic prohibited\n",
8255 				insn->dst_reg);
8256 			return -EACCES;
8257 		}
8258 
8259 		/* check dest operand */
8260 		err = check_reg_arg(env, insn->dst_reg, DST_OP);
8261 		if (err)
8262 			return err;
8263 
8264 	} else if (opcode == BPF_MOV) {
8265 
8266 		if (BPF_SRC(insn->code) == BPF_X) {
8267 			if (insn->imm != 0 || insn->off != 0) {
8268 				verbose(env, "BPF_MOV uses reserved fields\n");
8269 				return -EINVAL;
8270 			}
8271 
8272 			/* check src operand */
8273 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
8274 			if (err)
8275 				return err;
8276 		} else {
8277 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
8278 				verbose(env, "BPF_MOV uses reserved fields\n");
8279 				return -EINVAL;
8280 			}
8281 		}
8282 
8283 		/* check dest operand, mark as required later */
8284 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
8285 		if (err)
8286 			return err;
8287 
8288 		if (BPF_SRC(insn->code) == BPF_X) {
8289 			struct bpf_reg_state *src_reg = regs + insn->src_reg;
8290 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
8291 
8292 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
8293 				/* case: R1 = R2
8294 				 * copy register state to dest reg
8295 				 */
8296 				if (src_reg->type == SCALAR_VALUE && !src_reg->id)
8297 					/* Assign src and dst registers the same ID
8298 					 * that will be used by find_equal_scalars()
8299 					 * to propagate min/max range.
8300 					 */
8301 					src_reg->id = ++env->id_gen;
8302 				*dst_reg = *src_reg;
8303 				dst_reg->live |= REG_LIVE_WRITTEN;
8304 				dst_reg->subreg_def = DEF_NOT_SUBREG;
8305 			} else {
8306 				/* R1 = (u32) R2 */
8307 				if (is_pointer_value(env, insn->src_reg)) {
8308 					verbose(env,
8309 						"R%d partial copy of pointer\n",
8310 						insn->src_reg);
8311 					return -EACCES;
8312 				} else if (src_reg->type == SCALAR_VALUE) {
8313 					*dst_reg = *src_reg;
8314 					/* Make sure ID is cleared otherwise
8315 					 * dst_reg min/max could be incorrectly
8316 					 * propagated into src_reg by find_equal_scalars()
8317 					 */
8318 					dst_reg->id = 0;
8319 					dst_reg->live |= REG_LIVE_WRITTEN;
8320 					dst_reg->subreg_def = env->insn_idx + 1;
8321 				} else {
8322 					mark_reg_unknown(env, regs,
8323 							 insn->dst_reg);
8324 				}
8325 				zext_32_to_64(dst_reg);
8326 
8327 				__update_reg_bounds(dst_reg);
8328 				__reg_deduce_bounds(dst_reg);
8329 				__reg_bound_offset(dst_reg);
8330 			}
8331 		} else {
8332 			/* case: R = imm
8333 			 * remember the value we stored into this reg
8334 			 */
8335 			/* clear any state __mark_reg_known doesn't set */
8336 			mark_reg_unknown(env, regs, insn->dst_reg);
8337 			regs[insn->dst_reg].type = SCALAR_VALUE;
8338 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
8339 				__mark_reg_known(regs + insn->dst_reg,
8340 						 insn->imm);
8341 			} else {
8342 				__mark_reg_known(regs + insn->dst_reg,
8343 						 (u32)insn->imm);
8344 			}
8345 		}
8346 
8347 	} else if (opcode > BPF_END) {
8348 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
8349 		return -EINVAL;
8350 
8351 	} else {	/* all other ALU ops: and, sub, xor, add, ... */
8352 
8353 		if (BPF_SRC(insn->code) == BPF_X) {
8354 			if (insn->imm != 0 || insn->off != 0) {
8355 				verbose(env, "BPF_ALU uses reserved fields\n");
8356 				return -EINVAL;
8357 			}
8358 			/* check src1 operand */
8359 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
8360 			if (err)
8361 				return err;
8362 		} else {
8363 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
8364 				verbose(env, "BPF_ALU uses reserved fields\n");
8365 				return -EINVAL;
8366 			}
8367 		}
8368 
8369 		/* check src2 operand */
8370 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
8371 		if (err)
8372 			return err;
8373 
8374 		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
8375 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
8376 			verbose(env, "div by zero\n");
8377 			return -EINVAL;
8378 		}
8379 
8380 		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
8381 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
8382 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
8383 
8384 			if (insn->imm < 0 || insn->imm >= size) {
8385 				verbose(env, "invalid shift %d\n", insn->imm);
8386 				return -EINVAL;
8387 			}
8388 		}
8389 
8390 		/* check dest operand */
8391 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
8392 		if (err)
8393 			return err;
8394 
8395 		return adjust_reg_min_max_vals(env, insn);
8396 	}
8397 
8398 	return 0;
8399 }
8400 
8401 static void __find_good_pkt_pointers(struct bpf_func_state *state,
8402 				     struct bpf_reg_state *dst_reg,
8403 				     enum bpf_reg_type type, int new_range)
8404 {
8405 	struct bpf_reg_state *reg;
8406 	int i;
8407 
8408 	for (i = 0; i < MAX_BPF_REG; i++) {
8409 		reg = &state->regs[i];
8410 		if (reg->type == type && reg->id == dst_reg->id)
8411 			/* keep the maximum range already checked */
8412 			reg->range = max(reg->range, new_range);
8413 	}
8414 
8415 	bpf_for_each_spilled_reg(i, state, reg) {
8416 		if (!reg)
8417 			continue;
8418 		if (reg->type == type && reg->id == dst_reg->id)
8419 			reg->range = max(reg->range, new_range);
8420 	}
8421 }
8422 
8423 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
8424 				   struct bpf_reg_state *dst_reg,
8425 				   enum bpf_reg_type type,
8426 				   bool range_right_open)
8427 {
8428 	int new_range, i;
8429 
8430 	if (dst_reg->off < 0 ||
8431 	    (dst_reg->off == 0 && range_right_open))
8432 		/* This doesn't give us any range */
8433 		return;
8434 
8435 	if (dst_reg->umax_value > MAX_PACKET_OFF ||
8436 	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
8437 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
8438 		 * than pkt_end, but that's because it's also less than pkt.
8439 		 */
8440 		return;
8441 
8442 	new_range = dst_reg->off;
8443 	if (range_right_open)
8444 		new_range++;
8445 
8446 	/* Examples for register markings:
8447 	 *
8448 	 * pkt_data in dst register:
8449 	 *
8450 	 *   r2 = r3;
8451 	 *   r2 += 8;
8452 	 *   if (r2 > pkt_end) goto <handle exception>
8453 	 *   <access okay>
8454 	 *
8455 	 *   r2 = r3;
8456 	 *   r2 += 8;
8457 	 *   if (r2 < pkt_end) goto <access okay>
8458 	 *   <handle exception>
8459 	 *
8460 	 *   Where:
8461 	 *     r2 == dst_reg, pkt_end == src_reg
8462 	 *     r2=pkt(id=n,off=8,r=0)
8463 	 *     r3=pkt(id=n,off=0,r=0)
8464 	 *
8465 	 * pkt_data in src register:
8466 	 *
8467 	 *   r2 = r3;
8468 	 *   r2 += 8;
8469 	 *   if (pkt_end >= r2) goto <access okay>
8470 	 *   <handle exception>
8471 	 *
8472 	 *   r2 = r3;
8473 	 *   r2 += 8;
8474 	 *   if (pkt_end <= r2) goto <handle exception>
8475 	 *   <access okay>
8476 	 *
8477 	 *   Where:
8478 	 *     pkt_end == dst_reg, r2 == src_reg
8479 	 *     r2=pkt(id=n,off=8,r=0)
8480 	 *     r3=pkt(id=n,off=0,r=0)
8481 	 *
8482 	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
8483 	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
8484 	 * and [r3, r3 + 8-1) respectively is safe to access depending on
8485 	 * the check.
8486 	 */
8487 
8488 	/* If our ids match, then we must have the same max_value.  And we
8489 	 * don't care about the other reg's fixed offset, since if it's too big
8490 	 * the range won't allow anything.
8491 	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
8492 	 */
8493 	for (i = 0; i <= vstate->curframe; i++)
8494 		__find_good_pkt_pointers(vstate->frame[i], dst_reg, type,
8495 					 new_range);
8496 }
8497 
8498 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
8499 {
8500 	struct tnum subreg = tnum_subreg(reg->var_off);
8501 	s32 sval = (s32)val;
8502 
8503 	switch (opcode) {
8504 	case BPF_JEQ:
8505 		if (tnum_is_const(subreg))
8506 			return !!tnum_equals_const(subreg, val);
8507 		break;
8508 	case BPF_JNE:
8509 		if (tnum_is_const(subreg))
8510 			return !tnum_equals_const(subreg, val);
8511 		break;
8512 	case BPF_JSET:
8513 		if ((~subreg.mask & subreg.value) & val)
8514 			return 1;
8515 		if (!((subreg.mask | subreg.value) & val))
8516 			return 0;
8517 		break;
8518 	case BPF_JGT:
8519 		if (reg->u32_min_value > val)
8520 			return 1;
8521 		else if (reg->u32_max_value <= val)
8522 			return 0;
8523 		break;
8524 	case BPF_JSGT:
8525 		if (reg->s32_min_value > sval)
8526 			return 1;
8527 		else if (reg->s32_max_value <= sval)
8528 			return 0;
8529 		break;
8530 	case BPF_JLT:
8531 		if (reg->u32_max_value < val)
8532 			return 1;
8533 		else if (reg->u32_min_value >= val)
8534 			return 0;
8535 		break;
8536 	case BPF_JSLT:
8537 		if (reg->s32_max_value < sval)
8538 			return 1;
8539 		else if (reg->s32_min_value >= sval)
8540 			return 0;
8541 		break;
8542 	case BPF_JGE:
8543 		if (reg->u32_min_value >= val)
8544 			return 1;
8545 		else if (reg->u32_max_value < val)
8546 			return 0;
8547 		break;
8548 	case BPF_JSGE:
8549 		if (reg->s32_min_value >= sval)
8550 			return 1;
8551 		else if (reg->s32_max_value < sval)
8552 			return 0;
8553 		break;
8554 	case BPF_JLE:
8555 		if (reg->u32_max_value <= val)
8556 			return 1;
8557 		else if (reg->u32_min_value > val)
8558 			return 0;
8559 		break;
8560 	case BPF_JSLE:
8561 		if (reg->s32_max_value <= sval)
8562 			return 1;
8563 		else if (reg->s32_min_value > sval)
8564 			return 0;
8565 		break;
8566 	}
8567 
8568 	return -1;
8569 }
8570 
8571 
8572 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
8573 {
8574 	s64 sval = (s64)val;
8575 
8576 	switch (opcode) {
8577 	case BPF_JEQ:
8578 		if (tnum_is_const(reg->var_off))
8579 			return !!tnum_equals_const(reg->var_off, val);
8580 		break;
8581 	case BPF_JNE:
8582 		if (tnum_is_const(reg->var_off))
8583 			return !tnum_equals_const(reg->var_off, val);
8584 		break;
8585 	case BPF_JSET:
8586 		if ((~reg->var_off.mask & reg->var_off.value) & val)
8587 			return 1;
8588 		if (!((reg->var_off.mask | reg->var_off.value) & val))
8589 			return 0;
8590 		break;
8591 	case BPF_JGT:
8592 		if (reg->umin_value > val)
8593 			return 1;
8594 		else if (reg->umax_value <= val)
8595 			return 0;
8596 		break;
8597 	case BPF_JSGT:
8598 		if (reg->smin_value > sval)
8599 			return 1;
8600 		else if (reg->smax_value <= sval)
8601 			return 0;
8602 		break;
8603 	case BPF_JLT:
8604 		if (reg->umax_value < val)
8605 			return 1;
8606 		else if (reg->umin_value >= val)
8607 			return 0;
8608 		break;
8609 	case BPF_JSLT:
8610 		if (reg->smax_value < sval)
8611 			return 1;
8612 		else if (reg->smin_value >= sval)
8613 			return 0;
8614 		break;
8615 	case BPF_JGE:
8616 		if (reg->umin_value >= val)
8617 			return 1;
8618 		else if (reg->umax_value < val)
8619 			return 0;
8620 		break;
8621 	case BPF_JSGE:
8622 		if (reg->smin_value >= sval)
8623 			return 1;
8624 		else if (reg->smax_value < sval)
8625 			return 0;
8626 		break;
8627 	case BPF_JLE:
8628 		if (reg->umax_value <= val)
8629 			return 1;
8630 		else if (reg->umin_value > val)
8631 			return 0;
8632 		break;
8633 	case BPF_JSLE:
8634 		if (reg->smax_value <= sval)
8635 			return 1;
8636 		else if (reg->smin_value > sval)
8637 			return 0;
8638 		break;
8639 	}
8640 
8641 	return -1;
8642 }
8643 
8644 /* compute branch direction of the expression "if (reg opcode val) goto target;"
8645  * and return:
8646  *  1 - branch will be taken and "goto target" will be executed
8647  *  0 - branch will not be taken and fall-through to next insn
8648  * -1 - unknown. Example: "if (reg < 5)" is unknown when register value
8649  *      range [0,10]
8650  */
8651 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
8652 			   bool is_jmp32)
8653 {
8654 	if (__is_pointer_value(false, reg)) {
8655 		if (!reg_type_not_null(reg->type))
8656 			return -1;
8657 
8658 		/* If pointer is valid tests against zero will fail so we can
8659 		 * use this to direct branch taken.
8660 		 */
8661 		if (val != 0)
8662 			return -1;
8663 
8664 		switch (opcode) {
8665 		case BPF_JEQ:
8666 			return 0;
8667 		case BPF_JNE:
8668 			return 1;
8669 		default:
8670 			return -1;
8671 		}
8672 	}
8673 
8674 	if (is_jmp32)
8675 		return is_branch32_taken(reg, val, opcode);
8676 	return is_branch64_taken(reg, val, opcode);
8677 }
8678 
8679 static int flip_opcode(u32 opcode)
8680 {
8681 	/* How can we transform "a <op> b" into "b <op> a"? */
8682 	static const u8 opcode_flip[16] = {
8683 		/* these stay the same */
8684 		[BPF_JEQ  >> 4] = BPF_JEQ,
8685 		[BPF_JNE  >> 4] = BPF_JNE,
8686 		[BPF_JSET >> 4] = BPF_JSET,
8687 		/* these swap "lesser" and "greater" (L and G in the opcodes) */
8688 		[BPF_JGE  >> 4] = BPF_JLE,
8689 		[BPF_JGT  >> 4] = BPF_JLT,
8690 		[BPF_JLE  >> 4] = BPF_JGE,
8691 		[BPF_JLT  >> 4] = BPF_JGT,
8692 		[BPF_JSGE >> 4] = BPF_JSLE,
8693 		[BPF_JSGT >> 4] = BPF_JSLT,
8694 		[BPF_JSLE >> 4] = BPF_JSGE,
8695 		[BPF_JSLT >> 4] = BPF_JSGT
8696 	};
8697 	return opcode_flip[opcode >> 4];
8698 }
8699 
8700 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
8701 				   struct bpf_reg_state *src_reg,
8702 				   u8 opcode)
8703 {
8704 	struct bpf_reg_state *pkt;
8705 
8706 	if (src_reg->type == PTR_TO_PACKET_END) {
8707 		pkt = dst_reg;
8708 	} else if (dst_reg->type == PTR_TO_PACKET_END) {
8709 		pkt = src_reg;
8710 		opcode = flip_opcode(opcode);
8711 	} else {
8712 		return -1;
8713 	}
8714 
8715 	if (pkt->range >= 0)
8716 		return -1;
8717 
8718 	switch (opcode) {
8719 	case BPF_JLE:
8720 		/* pkt <= pkt_end */
8721 		fallthrough;
8722 	case BPF_JGT:
8723 		/* pkt > pkt_end */
8724 		if (pkt->range == BEYOND_PKT_END)
8725 			/* pkt has at last one extra byte beyond pkt_end */
8726 			return opcode == BPF_JGT;
8727 		break;
8728 	case BPF_JLT:
8729 		/* pkt < pkt_end */
8730 		fallthrough;
8731 	case BPF_JGE:
8732 		/* pkt >= pkt_end */
8733 		if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
8734 			return opcode == BPF_JGE;
8735 		break;
8736 	}
8737 	return -1;
8738 }
8739 
8740 /* Adjusts the register min/max values in the case that the dst_reg is the
8741  * variable register that we are working on, and src_reg is a constant or we're
8742  * simply doing a BPF_K check.
8743  * In JEQ/JNE cases we also adjust the var_off values.
8744  */
8745 static void reg_set_min_max(struct bpf_reg_state *true_reg,
8746 			    struct bpf_reg_state *false_reg,
8747 			    u64 val, u32 val32,
8748 			    u8 opcode, bool is_jmp32)
8749 {
8750 	struct tnum false_32off = tnum_subreg(false_reg->var_off);
8751 	struct tnum false_64off = false_reg->var_off;
8752 	struct tnum true_32off = tnum_subreg(true_reg->var_off);
8753 	struct tnum true_64off = true_reg->var_off;
8754 	s64 sval = (s64)val;
8755 	s32 sval32 = (s32)val32;
8756 
8757 	/* If the dst_reg is a pointer, we can't learn anything about its
8758 	 * variable offset from the compare (unless src_reg were a pointer into
8759 	 * the same object, but we don't bother with that.
8760 	 * Since false_reg and true_reg have the same type by construction, we
8761 	 * only need to check one of them for pointerness.
8762 	 */
8763 	if (__is_pointer_value(false, false_reg))
8764 		return;
8765 
8766 	switch (opcode) {
8767 	case BPF_JEQ:
8768 	case BPF_JNE:
8769 	{
8770 		struct bpf_reg_state *reg =
8771 			opcode == BPF_JEQ ? true_reg : false_reg;
8772 
8773 		/* JEQ/JNE comparison doesn't change the register equivalence.
8774 		 * r1 = r2;
8775 		 * if (r1 == 42) goto label;
8776 		 * ...
8777 		 * label: // here both r1 and r2 are known to be 42.
8778 		 *
8779 		 * Hence when marking register as known preserve it's ID.
8780 		 */
8781 		if (is_jmp32)
8782 			__mark_reg32_known(reg, val32);
8783 		else
8784 			___mark_reg_known(reg, val);
8785 		break;
8786 	}
8787 	case BPF_JSET:
8788 		if (is_jmp32) {
8789 			false_32off = tnum_and(false_32off, tnum_const(~val32));
8790 			if (is_power_of_2(val32))
8791 				true_32off = tnum_or(true_32off,
8792 						     tnum_const(val32));
8793 		} else {
8794 			false_64off = tnum_and(false_64off, tnum_const(~val));
8795 			if (is_power_of_2(val))
8796 				true_64off = tnum_or(true_64off,
8797 						     tnum_const(val));
8798 		}
8799 		break;
8800 	case BPF_JGE:
8801 	case BPF_JGT:
8802 	{
8803 		if (is_jmp32) {
8804 			u32 false_umax = opcode == BPF_JGT ? val32  : val32 - 1;
8805 			u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32;
8806 
8807 			false_reg->u32_max_value = min(false_reg->u32_max_value,
8808 						       false_umax);
8809 			true_reg->u32_min_value = max(true_reg->u32_min_value,
8810 						      true_umin);
8811 		} else {
8812 			u64 false_umax = opcode == BPF_JGT ? val    : val - 1;
8813 			u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
8814 
8815 			false_reg->umax_value = min(false_reg->umax_value, false_umax);
8816 			true_reg->umin_value = max(true_reg->umin_value, true_umin);
8817 		}
8818 		break;
8819 	}
8820 	case BPF_JSGE:
8821 	case BPF_JSGT:
8822 	{
8823 		if (is_jmp32) {
8824 			s32 false_smax = opcode == BPF_JSGT ? sval32    : sval32 - 1;
8825 			s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32;
8826 
8827 			false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax);
8828 			true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin);
8829 		} else {
8830 			s64 false_smax = opcode == BPF_JSGT ? sval    : sval - 1;
8831 			s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
8832 
8833 			false_reg->smax_value = min(false_reg->smax_value, false_smax);
8834 			true_reg->smin_value = max(true_reg->smin_value, true_smin);
8835 		}
8836 		break;
8837 	}
8838 	case BPF_JLE:
8839 	case BPF_JLT:
8840 	{
8841 		if (is_jmp32) {
8842 			u32 false_umin = opcode == BPF_JLT ? val32  : val32 + 1;
8843 			u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32;
8844 
8845 			false_reg->u32_min_value = max(false_reg->u32_min_value,
8846 						       false_umin);
8847 			true_reg->u32_max_value = min(true_reg->u32_max_value,
8848 						      true_umax);
8849 		} else {
8850 			u64 false_umin = opcode == BPF_JLT ? val    : val + 1;
8851 			u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
8852 
8853 			false_reg->umin_value = max(false_reg->umin_value, false_umin);
8854 			true_reg->umax_value = min(true_reg->umax_value, true_umax);
8855 		}
8856 		break;
8857 	}
8858 	case BPF_JSLE:
8859 	case BPF_JSLT:
8860 	{
8861 		if (is_jmp32) {
8862 			s32 false_smin = opcode == BPF_JSLT ? sval32    : sval32 + 1;
8863 			s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32;
8864 
8865 			false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin);
8866 			true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax);
8867 		} else {
8868 			s64 false_smin = opcode == BPF_JSLT ? sval    : sval + 1;
8869 			s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
8870 
8871 			false_reg->smin_value = max(false_reg->smin_value, false_smin);
8872 			true_reg->smax_value = min(true_reg->smax_value, true_smax);
8873 		}
8874 		break;
8875 	}
8876 	default:
8877 		return;
8878 	}
8879 
8880 	if (is_jmp32) {
8881 		false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off),
8882 					     tnum_subreg(false_32off));
8883 		true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off),
8884 					    tnum_subreg(true_32off));
8885 		__reg_combine_32_into_64(false_reg);
8886 		__reg_combine_32_into_64(true_reg);
8887 	} else {
8888 		false_reg->var_off = false_64off;
8889 		true_reg->var_off = true_64off;
8890 		__reg_combine_64_into_32(false_reg);
8891 		__reg_combine_64_into_32(true_reg);
8892 	}
8893 }
8894 
8895 /* Same as above, but for the case that dst_reg holds a constant and src_reg is
8896  * the variable reg.
8897  */
8898 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
8899 				struct bpf_reg_state *false_reg,
8900 				u64 val, u32 val32,
8901 				u8 opcode, bool is_jmp32)
8902 {
8903 	opcode = flip_opcode(opcode);
8904 	/* This uses zero as "not present in table"; luckily the zero opcode,
8905 	 * BPF_JA, can't get here.
8906 	 */
8907 	if (opcode)
8908 		reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32);
8909 }
8910 
8911 /* Regs are known to be equal, so intersect their min/max/var_off */
8912 static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
8913 				  struct bpf_reg_state *dst_reg)
8914 {
8915 	src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
8916 							dst_reg->umin_value);
8917 	src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
8918 							dst_reg->umax_value);
8919 	src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
8920 							dst_reg->smin_value);
8921 	src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
8922 							dst_reg->smax_value);
8923 	src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
8924 							     dst_reg->var_off);
8925 	/* We might have learned new bounds from the var_off. */
8926 	__update_reg_bounds(src_reg);
8927 	__update_reg_bounds(dst_reg);
8928 	/* We might have learned something about the sign bit. */
8929 	__reg_deduce_bounds(src_reg);
8930 	__reg_deduce_bounds(dst_reg);
8931 	/* We might have learned some bits from the bounds. */
8932 	__reg_bound_offset(src_reg);
8933 	__reg_bound_offset(dst_reg);
8934 	/* Intersecting with the old var_off might have improved our bounds
8935 	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
8936 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
8937 	 */
8938 	__update_reg_bounds(src_reg);
8939 	__update_reg_bounds(dst_reg);
8940 }
8941 
8942 static void reg_combine_min_max(struct bpf_reg_state *true_src,
8943 				struct bpf_reg_state *true_dst,
8944 				struct bpf_reg_state *false_src,
8945 				struct bpf_reg_state *false_dst,
8946 				u8 opcode)
8947 {
8948 	switch (opcode) {
8949 	case BPF_JEQ:
8950 		__reg_combine_min_max(true_src, true_dst);
8951 		break;
8952 	case BPF_JNE:
8953 		__reg_combine_min_max(false_src, false_dst);
8954 		break;
8955 	}
8956 }
8957 
8958 static void mark_ptr_or_null_reg(struct bpf_func_state *state,
8959 				 struct bpf_reg_state *reg, u32 id,
8960 				 bool is_null)
8961 {
8962 	if (reg_type_may_be_null(reg->type) && reg->id == id &&
8963 	    !WARN_ON_ONCE(!reg->id)) {
8964 		/* Old offset (both fixed and variable parts) should
8965 		 * have been known-zero, because we don't allow pointer
8966 		 * arithmetic on pointers that might be NULL.
8967 		 */
8968 		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
8969 				 !tnum_equals_const(reg->var_off, 0) ||
8970 				 reg->off)) {
8971 			__mark_reg_known_zero(reg);
8972 			reg->off = 0;
8973 		}
8974 		if (is_null) {
8975 			reg->type = SCALAR_VALUE;
8976 			/* We don't need id and ref_obj_id from this point
8977 			 * onwards anymore, thus we should better reset it,
8978 			 * so that state pruning has chances to take effect.
8979 			 */
8980 			reg->id = 0;
8981 			reg->ref_obj_id = 0;
8982 
8983 			return;
8984 		}
8985 
8986 		mark_ptr_not_null_reg(reg);
8987 
8988 		if (!reg_may_point_to_spin_lock(reg)) {
8989 			/* For not-NULL ptr, reg->ref_obj_id will be reset
8990 			 * in release_reg_references().
8991 			 *
8992 			 * reg->id is still used by spin_lock ptr. Other
8993 			 * than spin_lock ptr type, reg->id can be reset.
8994 			 */
8995 			reg->id = 0;
8996 		}
8997 	}
8998 }
8999 
9000 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id,
9001 				    bool is_null)
9002 {
9003 	struct bpf_reg_state *reg;
9004 	int i;
9005 
9006 	for (i = 0; i < MAX_BPF_REG; i++)
9007 		mark_ptr_or_null_reg(state, &state->regs[i], id, is_null);
9008 
9009 	bpf_for_each_spilled_reg(i, state, reg) {
9010 		if (!reg)
9011 			continue;
9012 		mark_ptr_or_null_reg(state, reg, id, is_null);
9013 	}
9014 }
9015 
9016 /* The logic is similar to find_good_pkt_pointers(), both could eventually
9017  * be folded together at some point.
9018  */
9019 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
9020 				  bool is_null)
9021 {
9022 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
9023 	struct bpf_reg_state *regs = state->regs;
9024 	u32 ref_obj_id = regs[regno].ref_obj_id;
9025 	u32 id = regs[regno].id;
9026 	int i;
9027 
9028 	if (ref_obj_id && ref_obj_id == id && is_null)
9029 		/* regs[regno] is in the " == NULL" branch.
9030 		 * No one could have freed the reference state before
9031 		 * doing the NULL check.
9032 		 */
9033 		WARN_ON_ONCE(release_reference_state(state, id));
9034 
9035 	for (i = 0; i <= vstate->curframe; i++)
9036 		__mark_ptr_or_null_regs(vstate->frame[i], id, is_null);
9037 }
9038 
9039 static bool try_match_pkt_pointers(const struct bpf_insn *insn,
9040 				   struct bpf_reg_state *dst_reg,
9041 				   struct bpf_reg_state *src_reg,
9042 				   struct bpf_verifier_state *this_branch,
9043 				   struct bpf_verifier_state *other_branch)
9044 {
9045 	if (BPF_SRC(insn->code) != BPF_X)
9046 		return false;
9047 
9048 	/* Pointers are always 64-bit. */
9049 	if (BPF_CLASS(insn->code) == BPF_JMP32)
9050 		return false;
9051 
9052 	switch (BPF_OP(insn->code)) {
9053 	case BPF_JGT:
9054 		if ((dst_reg->type == PTR_TO_PACKET &&
9055 		     src_reg->type == PTR_TO_PACKET_END) ||
9056 		    (dst_reg->type == PTR_TO_PACKET_META &&
9057 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
9058 			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
9059 			find_good_pkt_pointers(this_branch, dst_reg,
9060 					       dst_reg->type, false);
9061 			mark_pkt_end(other_branch, insn->dst_reg, true);
9062 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
9063 			    src_reg->type == PTR_TO_PACKET) ||
9064 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
9065 			    src_reg->type == PTR_TO_PACKET_META)) {
9066 			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
9067 			find_good_pkt_pointers(other_branch, src_reg,
9068 					       src_reg->type, true);
9069 			mark_pkt_end(this_branch, insn->src_reg, false);
9070 		} else {
9071 			return false;
9072 		}
9073 		break;
9074 	case BPF_JLT:
9075 		if ((dst_reg->type == PTR_TO_PACKET &&
9076 		     src_reg->type == PTR_TO_PACKET_END) ||
9077 		    (dst_reg->type == PTR_TO_PACKET_META &&
9078 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
9079 			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
9080 			find_good_pkt_pointers(other_branch, dst_reg,
9081 					       dst_reg->type, true);
9082 			mark_pkt_end(this_branch, insn->dst_reg, false);
9083 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
9084 			    src_reg->type == PTR_TO_PACKET) ||
9085 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
9086 			    src_reg->type == PTR_TO_PACKET_META)) {
9087 			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
9088 			find_good_pkt_pointers(this_branch, src_reg,
9089 					       src_reg->type, false);
9090 			mark_pkt_end(other_branch, insn->src_reg, true);
9091 		} else {
9092 			return false;
9093 		}
9094 		break;
9095 	case BPF_JGE:
9096 		if ((dst_reg->type == PTR_TO_PACKET &&
9097 		     src_reg->type == PTR_TO_PACKET_END) ||
9098 		    (dst_reg->type == PTR_TO_PACKET_META &&
9099 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
9100 			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
9101 			find_good_pkt_pointers(this_branch, dst_reg,
9102 					       dst_reg->type, true);
9103 			mark_pkt_end(other_branch, insn->dst_reg, false);
9104 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
9105 			    src_reg->type == PTR_TO_PACKET) ||
9106 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
9107 			    src_reg->type == PTR_TO_PACKET_META)) {
9108 			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
9109 			find_good_pkt_pointers(other_branch, src_reg,
9110 					       src_reg->type, false);
9111 			mark_pkt_end(this_branch, insn->src_reg, true);
9112 		} else {
9113 			return false;
9114 		}
9115 		break;
9116 	case BPF_JLE:
9117 		if ((dst_reg->type == PTR_TO_PACKET &&
9118 		     src_reg->type == PTR_TO_PACKET_END) ||
9119 		    (dst_reg->type == PTR_TO_PACKET_META &&
9120 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
9121 			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
9122 			find_good_pkt_pointers(other_branch, dst_reg,
9123 					       dst_reg->type, false);
9124 			mark_pkt_end(this_branch, insn->dst_reg, true);
9125 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
9126 			    src_reg->type == PTR_TO_PACKET) ||
9127 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
9128 			    src_reg->type == PTR_TO_PACKET_META)) {
9129 			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
9130 			find_good_pkt_pointers(this_branch, src_reg,
9131 					       src_reg->type, true);
9132 			mark_pkt_end(other_branch, insn->src_reg, false);
9133 		} else {
9134 			return false;
9135 		}
9136 		break;
9137 	default:
9138 		return false;
9139 	}
9140 
9141 	return true;
9142 }
9143 
9144 static void find_equal_scalars(struct bpf_verifier_state *vstate,
9145 			       struct bpf_reg_state *known_reg)
9146 {
9147 	struct bpf_func_state *state;
9148 	struct bpf_reg_state *reg;
9149 	int i, j;
9150 
9151 	for (i = 0; i <= vstate->curframe; i++) {
9152 		state = vstate->frame[i];
9153 		for (j = 0; j < MAX_BPF_REG; j++) {
9154 			reg = &state->regs[j];
9155 			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
9156 				*reg = *known_reg;
9157 		}
9158 
9159 		bpf_for_each_spilled_reg(j, state, reg) {
9160 			if (!reg)
9161 				continue;
9162 			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
9163 				*reg = *known_reg;
9164 		}
9165 	}
9166 }
9167 
9168 static int check_cond_jmp_op(struct bpf_verifier_env *env,
9169 			     struct bpf_insn *insn, int *insn_idx)
9170 {
9171 	struct bpf_verifier_state *this_branch = env->cur_state;
9172 	struct bpf_verifier_state *other_branch;
9173 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
9174 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
9175 	u8 opcode = BPF_OP(insn->code);
9176 	bool is_jmp32;
9177 	int pred = -1;
9178 	int err;
9179 
9180 	/* Only conditional jumps are expected to reach here. */
9181 	if (opcode == BPF_JA || opcode > BPF_JSLE) {
9182 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
9183 		return -EINVAL;
9184 	}
9185 
9186 	if (BPF_SRC(insn->code) == BPF_X) {
9187 		if (insn->imm != 0) {
9188 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
9189 			return -EINVAL;
9190 		}
9191 
9192 		/* check src1 operand */
9193 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
9194 		if (err)
9195 			return err;
9196 
9197 		if (is_pointer_value(env, insn->src_reg)) {
9198 			verbose(env, "R%d pointer comparison prohibited\n",
9199 				insn->src_reg);
9200 			return -EACCES;
9201 		}
9202 		src_reg = &regs[insn->src_reg];
9203 	} else {
9204 		if (insn->src_reg != BPF_REG_0) {
9205 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
9206 			return -EINVAL;
9207 		}
9208 	}
9209 
9210 	/* check src2 operand */
9211 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
9212 	if (err)
9213 		return err;
9214 
9215 	dst_reg = &regs[insn->dst_reg];
9216 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
9217 
9218 	if (BPF_SRC(insn->code) == BPF_K) {
9219 		pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32);
9220 	} else if (src_reg->type == SCALAR_VALUE &&
9221 		   is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) {
9222 		pred = is_branch_taken(dst_reg,
9223 				       tnum_subreg(src_reg->var_off).value,
9224 				       opcode,
9225 				       is_jmp32);
9226 	} else if (src_reg->type == SCALAR_VALUE &&
9227 		   !is_jmp32 && tnum_is_const(src_reg->var_off)) {
9228 		pred = is_branch_taken(dst_reg,
9229 				       src_reg->var_off.value,
9230 				       opcode,
9231 				       is_jmp32);
9232 	} else if (reg_is_pkt_pointer_any(dst_reg) &&
9233 		   reg_is_pkt_pointer_any(src_reg) &&
9234 		   !is_jmp32) {
9235 		pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
9236 	}
9237 
9238 	if (pred >= 0) {
9239 		/* If we get here with a dst_reg pointer type it is because
9240 		 * above is_branch_taken() special cased the 0 comparison.
9241 		 */
9242 		if (!__is_pointer_value(false, dst_reg))
9243 			err = mark_chain_precision(env, insn->dst_reg);
9244 		if (BPF_SRC(insn->code) == BPF_X && !err &&
9245 		    !__is_pointer_value(false, src_reg))
9246 			err = mark_chain_precision(env, insn->src_reg);
9247 		if (err)
9248 			return err;
9249 	}
9250 
9251 	if (pred == 1) {
9252 		/* Only follow the goto, ignore fall-through. If needed, push
9253 		 * the fall-through branch for simulation under speculative
9254 		 * execution.
9255 		 */
9256 		if (!env->bypass_spec_v1 &&
9257 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,
9258 					       *insn_idx))
9259 			return -EFAULT;
9260 		*insn_idx += insn->off;
9261 		return 0;
9262 	} else if (pred == 0) {
9263 		/* Only follow the fall-through branch, since that's where the
9264 		 * program will go. If needed, push the goto branch for
9265 		 * simulation under speculative execution.
9266 		 */
9267 		if (!env->bypass_spec_v1 &&
9268 		    !sanitize_speculative_path(env, insn,
9269 					       *insn_idx + insn->off + 1,
9270 					       *insn_idx))
9271 			return -EFAULT;
9272 		return 0;
9273 	}
9274 
9275 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
9276 				  false);
9277 	if (!other_branch)
9278 		return -EFAULT;
9279 	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
9280 
9281 	/* detect if we are comparing against a constant value so we can adjust
9282 	 * our min/max values for our dst register.
9283 	 * this is only legit if both are scalars (or pointers to the same
9284 	 * object, I suppose, but we don't support that right now), because
9285 	 * otherwise the different base pointers mean the offsets aren't
9286 	 * comparable.
9287 	 */
9288 	if (BPF_SRC(insn->code) == BPF_X) {
9289 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];
9290 
9291 		if (dst_reg->type == SCALAR_VALUE &&
9292 		    src_reg->type == SCALAR_VALUE) {
9293 			if (tnum_is_const(src_reg->var_off) ||
9294 			    (is_jmp32 &&
9295 			     tnum_is_const(tnum_subreg(src_reg->var_off))))
9296 				reg_set_min_max(&other_branch_regs[insn->dst_reg],
9297 						dst_reg,
9298 						src_reg->var_off.value,
9299 						tnum_subreg(src_reg->var_off).value,
9300 						opcode, is_jmp32);
9301 			else if (tnum_is_const(dst_reg->var_off) ||
9302 				 (is_jmp32 &&
9303 				  tnum_is_const(tnum_subreg(dst_reg->var_off))))
9304 				reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
9305 						    src_reg,
9306 						    dst_reg->var_off.value,
9307 						    tnum_subreg(dst_reg->var_off).value,
9308 						    opcode, is_jmp32);
9309 			else if (!is_jmp32 &&
9310 				 (opcode == BPF_JEQ || opcode == BPF_JNE))
9311 				/* Comparing for equality, we can combine knowledge */
9312 				reg_combine_min_max(&other_branch_regs[insn->src_reg],
9313 						    &other_branch_regs[insn->dst_reg],
9314 						    src_reg, dst_reg, opcode);
9315 			if (src_reg->id &&
9316 			    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
9317 				find_equal_scalars(this_branch, src_reg);
9318 				find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
9319 			}
9320 
9321 		}
9322 	} else if (dst_reg->type == SCALAR_VALUE) {
9323 		reg_set_min_max(&other_branch_regs[insn->dst_reg],
9324 					dst_reg, insn->imm, (u32)insn->imm,
9325 					opcode, is_jmp32);
9326 	}
9327 
9328 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
9329 	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
9330 		find_equal_scalars(this_branch, dst_reg);
9331 		find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
9332 	}
9333 
9334 	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
9335 	 * NOTE: these optimizations below are related with pointer comparison
9336 	 *       which will never be JMP32.
9337 	 */
9338 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
9339 	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
9340 	    reg_type_may_be_null(dst_reg->type)) {
9341 		/* Mark all identical registers in each branch as either
9342 		 * safe or unknown depending R == 0 or R != 0 conditional.
9343 		 */
9344 		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
9345 				      opcode == BPF_JNE);
9346 		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
9347 				      opcode == BPF_JEQ);
9348 	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
9349 					   this_branch, other_branch) &&
9350 		   is_pointer_value(env, insn->dst_reg)) {
9351 		verbose(env, "R%d pointer comparison prohibited\n",
9352 			insn->dst_reg);
9353 		return -EACCES;
9354 	}
9355 	if (env->log.level & BPF_LOG_LEVEL)
9356 		print_verifier_state(env, this_branch->frame[this_branch->curframe]);
9357 	return 0;
9358 }
9359 
9360 /* verify BPF_LD_IMM64 instruction */
9361 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
9362 {
9363 	struct bpf_insn_aux_data *aux = cur_aux(env);
9364 	struct bpf_reg_state *regs = cur_regs(env);
9365 	struct bpf_reg_state *dst_reg;
9366 	struct bpf_map *map;
9367 	int err;
9368 
9369 	if (BPF_SIZE(insn->code) != BPF_DW) {
9370 		verbose(env, "invalid BPF_LD_IMM insn\n");
9371 		return -EINVAL;
9372 	}
9373 	if (insn->off != 0) {
9374 		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
9375 		return -EINVAL;
9376 	}
9377 
9378 	err = check_reg_arg(env, insn->dst_reg, DST_OP);
9379 	if (err)
9380 		return err;
9381 
9382 	dst_reg = &regs[insn->dst_reg];
9383 	if (insn->src_reg == 0) {
9384 		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
9385 
9386 		dst_reg->type = SCALAR_VALUE;
9387 		__mark_reg_known(&regs[insn->dst_reg], imm);
9388 		return 0;
9389 	}
9390 
9391 	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
9392 		mark_reg_known_zero(env, regs, insn->dst_reg);
9393 
9394 		dst_reg->type = aux->btf_var.reg_type;
9395 		switch (dst_reg->type) {
9396 		case PTR_TO_MEM:
9397 			dst_reg->mem_size = aux->btf_var.mem_size;
9398 			break;
9399 		case PTR_TO_BTF_ID:
9400 		case PTR_TO_PERCPU_BTF_ID:
9401 			dst_reg->btf = aux->btf_var.btf;
9402 			dst_reg->btf_id = aux->btf_var.btf_id;
9403 			break;
9404 		default:
9405 			verbose(env, "bpf verifier is misconfigured\n");
9406 			return -EFAULT;
9407 		}
9408 		return 0;
9409 	}
9410 
9411 	if (insn->src_reg == BPF_PSEUDO_FUNC) {
9412 		struct bpf_prog_aux *aux = env->prog->aux;
9413 		u32 subprogno = find_subprog(env,
9414 					     env->insn_idx + insn->imm + 1);
9415 
9416 		if (!aux->func_info) {
9417 			verbose(env, "missing btf func_info\n");
9418 			return -EINVAL;
9419 		}
9420 		if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
9421 			verbose(env, "callback function not static\n");
9422 			return -EINVAL;
9423 		}
9424 
9425 		dst_reg->type = PTR_TO_FUNC;
9426 		dst_reg->subprogno = subprogno;
9427 		return 0;
9428 	}
9429 
9430 	map = env->used_maps[aux->map_index];
9431 	mark_reg_known_zero(env, regs, insn->dst_reg);
9432 	dst_reg->map_ptr = map;
9433 
9434 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
9435 	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
9436 		dst_reg->type = PTR_TO_MAP_VALUE;
9437 		dst_reg->off = aux->map_off;
9438 		if (map_value_has_spin_lock(map))
9439 			dst_reg->id = ++env->id_gen;
9440 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
9441 		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
9442 		dst_reg->type = CONST_PTR_TO_MAP;
9443 	} else {
9444 		verbose(env, "bpf verifier is misconfigured\n");
9445 		return -EINVAL;
9446 	}
9447 
9448 	return 0;
9449 }
9450 
9451 static bool may_access_skb(enum bpf_prog_type type)
9452 {
9453 	switch (type) {
9454 	case BPF_PROG_TYPE_SOCKET_FILTER:
9455 	case BPF_PROG_TYPE_SCHED_CLS:
9456 	case BPF_PROG_TYPE_SCHED_ACT:
9457 		return true;
9458 	default:
9459 		return false;
9460 	}
9461 }
9462 
9463 /* verify safety of LD_ABS|LD_IND instructions:
9464  * - they can only appear in the programs where ctx == skb
9465  * - since they are wrappers of function calls, they scratch R1-R5 registers,
9466  *   preserve R6-R9, and store return value into R0
9467  *
9468  * Implicit input:
9469  *   ctx == skb == R6 == CTX
9470  *
9471  * Explicit input:
9472  *   SRC == any register
9473  *   IMM == 32-bit immediate
9474  *
9475  * Output:
9476  *   R0 - 8/16/32-bit skb data converted to cpu endianness
9477  */
9478 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
9479 {
9480 	struct bpf_reg_state *regs = cur_regs(env);
9481 	static const int ctx_reg = BPF_REG_6;
9482 	u8 mode = BPF_MODE(insn->code);
9483 	int i, err;
9484 
9485 	if (!may_access_skb(resolve_prog_type(env->prog))) {
9486 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
9487 		return -EINVAL;
9488 	}
9489 
9490 	if (!env->ops->gen_ld_abs) {
9491 		verbose(env, "bpf verifier is misconfigured\n");
9492 		return -EINVAL;
9493 	}
9494 
9495 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
9496 	    BPF_SIZE(insn->code) == BPF_DW ||
9497 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
9498 		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
9499 		return -EINVAL;
9500 	}
9501 
9502 	/* check whether implicit source operand (register R6) is readable */
9503 	err = check_reg_arg(env, ctx_reg, SRC_OP);
9504 	if (err)
9505 		return err;
9506 
9507 	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
9508 	 * gen_ld_abs() may terminate the program at runtime, leading to
9509 	 * reference leak.
9510 	 */
9511 	err = check_reference_leak(env);
9512 	if (err) {
9513 		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
9514 		return err;
9515 	}
9516 
9517 	if (env->cur_state->active_spin_lock) {
9518 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
9519 		return -EINVAL;
9520 	}
9521 
9522 	if (regs[ctx_reg].type != PTR_TO_CTX) {
9523 		verbose(env,
9524 			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
9525 		return -EINVAL;
9526 	}
9527 
9528 	if (mode == BPF_IND) {
9529 		/* check explicit source operand */
9530 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
9531 		if (err)
9532 			return err;
9533 	}
9534 
9535 	err = check_ctx_reg(env, &regs[ctx_reg], ctx_reg);
9536 	if (err < 0)
9537 		return err;
9538 
9539 	/* reset caller saved regs to unreadable */
9540 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
9541 		mark_reg_not_init(env, regs, caller_saved[i]);
9542 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
9543 	}
9544 
9545 	/* mark destination R0 register as readable, since it contains
9546 	 * the value fetched from the packet.
9547 	 * Already marked as written above.
9548 	 */
9549 	mark_reg_unknown(env, regs, BPF_REG_0);
9550 	/* ld_abs load up to 32-bit skb data. */
9551 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
9552 	return 0;
9553 }
9554 
9555 static int check_return_code(struct bpf_verifier_env *env)
9556 {
9557 	struct tnum enforce_attach_type_range = tnum_unknown;
9558 	const struct bpf_prog *prog = env->prog;
9559 	struct bpf_reg_state *reg;
9560 	struct tnum range = tnum_range(0, 1);
9561 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
9562 	int err;
9563 	struct bpf_func_state *frame = env->cur_state->frame[0];
9564 	const bool is_subprog = frame->subprogno;
9565 
9566 	/* LSM and struct_ops func-ptr's return type could be "void" */
9567 	if (!is_subprog &&
9568 	    (prog_type == BPF_PROG_TYPE_STRUCT_OPS ||
9569 	     prog_type == BPF_PROG_TYPE_LSM) &&
9570 	    !prog->aux->attach_func_proto->type)
9571 		return 0;
9572 
9573 	/* eBPF calling convention is such that R0 is used
9574 	 * to return the value from eBPF program.
9575 	 * Make sure that it's readable at this time
9576 	 * of bpf_exit, which means that program wrote
9577 	 * something into it earlier
9578 	 */
9579 	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
9580 	if (err)
9581 		return err;
9582 
9583 	if (is_pointer_value(env, BPF_REG_0)) {
9584 		verbose(env, "R0 leaks addr as return value\n");
9585 		return -EACCES;
9586 	}
9587 
9588 	reg = cur_regs(env) + BPF_REG_0;
9589 
9590 	if (frame->in_async_callback_fn) {
9591 		/* enforce return zero from async callbacks like timer */
9592 		if (reg->type != SCALAR_VALUE) {
9593 			verbose(env, "In async callback the register R0 is not a known value (%s)\n",
9594 				reg_type_str[reg->type]);
9595 			return -EINVAL;
9596 		}
9597 
9598 		if (!tnum_in(tnum_const(0), reg->var_off)) {
9599 			verbose_invalid_scalar(env, reg, &range, "async callback", "R0");
9600 			return -EINVAL;
9601 		}
9602 		return 0;
9603 	}
9604 
9605 	if (is_subprog) {
9606 		if (reg->type != SCALAR_VALUE) {
9607 			verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
9608 				reg_type_str[reg->type]);
9609 			return -EINVAL;
9610 		}
9611 		return 0;
9612 	}
9613 
9614 	switch (prog_type) {
9615 	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
9616 		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
9617 		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
9618 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
9619 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
9620 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
9621 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
9622 			range = tnum_range(1, 1);
9623 		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
9624 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
9625 			range = tnum_range(0, 3);
9626 		break;
9627 	case BPF_PROG_TYPE_CGROUP_SKB:
9628 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
9629 			range = tnum_range(0, 3);
9630 			enforce_attach_type_range = tnum_range(2, 3);
9631 		}
9632 		break;
9633 	case BPF_PROG_TYPE_CGROUP_SOCK:
9634 	case BPF_PROG_TYPE_SOCK_OPS:
9635 	case BPF_PROG_TYPE_CGROUP_DEVICE:
9636 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
9637 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
9638 		break;
9639 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
9640 		if (!env->prog->aux->attach_btf_id)
9641 			return 0;
9642 		range = tnum_const(0);
9643 		break;
9644 	case BPF_PROG_TYPE_TRACING:
9645 		switch (env->prog->expected_attach_type) {
9646 		case BPF_TRACE_FENTRY:
9647 		case BPF_TRACE_FEXIT:
9648 			range = tnum_const(0);
9649 			break;
9650 		case BPF_TRACE_RAW_TP:
9651 		case BPF_MODIFY_RETURN:
9652 			return 0;
9653 		case BPF_TRACE_ITER:
9654 			break;
9655 		default:
9656 			return -ENOTSUPP;
9657 		}
9658 		break;
9659 	case BPF_PROG_TYPE_SK_LOOKUP:
9660 		range = tnum_range(SK_DROP, SK_PASS);
9661 		break;
9662 	case BPF_PROG_TYPE_EXT:
9663 		/* freplace program can return anything as its return value
9664 		 * depends on the to-be-replaced kernel func or bpf program.
9665 		 */
9666 	default:
9667 		return 0;
9668 	}
9669 
9670 	if (reg->type != SCALAR_VALUE) {
9671 		verbose(env, "At program exit the register R0 is not a known value (%s)\n",
9672 			reg_type_str[reg->type]);
9673 		return -EINVAL;
9674 	}
9675 
9676 	if (!tnum_in(range, reg->var_off)) {
9677 		verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
9678 		return -EINVAL;
9679 	}
9680 
9681 	if (!tnum_is_unknown(enforce_attach_type_range) &&
9682 	    tnum_in(enforce_attach_type_range, reg->var_off))
9683 		env->prog->enforce_expected_attach_type = 1;
9684 	return 0;
9685 }
9686 
9687 /* non-recursive DFS pseudo code
9688  * 1  procedure DFS-iterative(G,v):
9689  * 2      label v as discovered
9690  * 3      let S be a stack
9691  * 4      S.push(v)
9692  * 5      while S is not empty
9693  * 6            t <- S.pop()
9694  * 7            if t is what we're looking for:
9695  * 8                return t
9696  * 9            for all edges e in G.adjacentEdges(t) do
9697  * 10               if edge e is already labelled
9698  * 11                   continue with the next edge
9699  * 12               w <- G.adjacentVertex(t,e)
9700  * 13               if vertex w is not discovered and not explored
9701  * 14                   label e as tree-edge
9702  * 15                   label w as discovered
9703  * 16                   S.push(w)
9704  * 17                   continue at 5
9705  * 18               else if vertex w is discovered
9706  * 19                   label e as back-edge
9707  * 20               else
9708  * 21                   // vertex w is explored
9709  * 22                   label e as forward- or cross-edge
9710  * 23           label t as explored
9711  * 24           S.pop()
9712  *
9713  * convention:
9714  * 0x10 - discovered
9715  * 0x11 - discovered and fall-through edge labelled
9716  * 0x12 - discovered and fall-through and branch edges labelled
9717  * 0x20 - explored
9718  */
9719 
9720 enum {
9721 	DISCOVERED = 0x10,
9722 	EXPLORED = 0x20,
9723 	FALLTHROUGH = 1,
9724 	BRANCH = 2,
9725 };
9726 
9727 static u32 state_htab_size(struct bpf_verifier_env *env)
9728 {
9729 	return env->prog->len;
9730 }
9731 
9732 static struct bpf_verifier_state_list **explored_state(
9733 					struct bpf_verifier_env *env,
9734 					int idx)
9735 {
9736 	struct bpf_verifier_state *cur = env->cur_state;
9737 	struct bpf_func_state *state = cur->frame[cur->curframe];
9738 
9739 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
9740 }
9741 
9742 static void init_explored_state(struct bpf_verifier_env *env, int idx)
9743 {
9744 	env->insn_aux_data[idx].prune_point = true;
9745 }
9746 
9747 enum {
9748 	DONE_EXPLORING = 0,
9749 	KEEP_EXPLORING = 1,
9750 };
9751 
9752 /* t, w, e - match pseudo-code above:
9753  * t - index of current instruction
9754  * w - next instruction
9755  * e - edge
9756  */
9757 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,
9758 		     bool loop_ok)
9759 {
9760 	int *insn_stack = env->cfg.insn_stack;
9761 	int *insn_state = env->cfg.insn_state;
9762 
9763 	if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
9764 		return DONE_EXPLORING;
9765 
9766 	if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
9767 		return DONE_EXPLORING;
9768 
9769 	if (w < 0 || w >= env->prog->len) {
9770 		verbose_linfo(env, t, "%d: ", t);
9771 		verbose(env, "jump out of range from insn %d to %d\n", t, w);
9772 		return -EINVAL;
9773 	}
9774 
9775 	if (e == BRANCH)
9776 		/* mark branch target for state pruning */
9777 		init_explored_state(env, w);
9778 
9779 	if (insn_state[w] == 0) {
9780 		/* tree-edge */
9781 		insn_state[t] = DISCOVERED | e;
9782 		insn_state[w] = DISCOVERED;
9783 		if (env->cfg.cur_stack >= env->prog->len)
9784 			return -E2BIG;
9785 		insn_stack[env->cfg.cur_stack++] = w;
9786 		return KEEP_EXPLORING;
9787 	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
9788 		if (loop_ok && env->bpf_capable)
9789 			return DONE_EXPLORING;
9790 		verbose_linfo(env, t, "%d: ", t);
9791 		verbose_linfo(env, w, "%d: ", w);
9792 		verbose(env, "back-edge from insn %d to %d\n", t, w);
9793 		return -EINVAL;
9794 	} else if (insn_state[w] == EXPLORED) {
9795 		/* forward- or cross-edge */
9796 		insn_state[t] = DISCOVERED | e;
9797 	} else {
9798 		verbose(env, "insn state internal bug\n");
9799 		return -EFAULT;
9800 	}
9801 	return DONE_EXPLORING;
9802 }
9803 
9804 static int visit_func_call_insn(int t, int insn_cnt,
9805 				struct bpf_insn *insns,
9806 				struct bpf_verifier_env *env,
9807 				bool visit_callee)
9808 {
9809 	int ret;
9810 
9811 	ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
9812 	if (ret)
9813 		return ret;
9814 
9815 	if (t + 1 < insn_cnt)
9816 		init_explored_state(env, t + 1);
9817 	if (visit_callee) {
9818 		init_explored_state(env, t);
9819 		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env,
9820 				/* It's ok to allow recursion from CFG point of
9821 				 * view. __check_func_call() will do the actual
9822 				 * check.
9823 				 */
9824 				bpf_pseudo_func(insns + t));
9825 	}
9826 	return ret;
9827 }
9828 
9829 /* Visits the instruction at index t and returns one of the following:
9830  *  < 0 - an error occurred
9831  *  DONE_EXPLORING - the instruction was fully explored
9832  *  KEEP_EXPLORING - there is still work to be done before it is fully explored
9833  */
9834 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env)
9835 {
9836 	struct bpf_insn *insns = env->prog->insnsi;
9837 	int ret;
9838 
9839 	if (bpf_pseudo_func(insns + t))
9840 		return visit_func_call_insn(t, insn_cnt, insns, env, true);
9841 
9842 	/* All non-branch instructions have a single fall-through edge. */
9843 	if (BPF_CLASS(insns[t].code) != BPF_JMP &&
9844 	    BPF_CLASS(insns[t].code) != BPF_JMP32)
9845 		return push_insn(t, t + 1, FALLTHROUGH, env, false);
9846 
9847 	switch (BPF_OP(insns[t].code)) {
9848 	case BPF_EXIT:
9849 		return DONE_EXPLORING;
9850 
9851 	case BPF_CALL:
9852 		if (insns[t].imm == BPF_FUNC_timer_set_callback)
9853 			/* Mark this call insn to trigger is_state_visited() check
9854 			 * before call itself is processed by __check_func_call().
9855 			 * Otherwise new async state will be pushed for further
9856 			 * exploration.
9857 			 */
9858 			init_explored_state(env, t);
9859 		return visit_func_call_insn(t, insn_cnt, insns, env,
9860 					    insns[t].src_reg == BPF_PSEUDO_CALL);
9861 
9862 	case BPF_JA:
9863 		if (BPF_SRC(insns[t].code) != BPF_K)
9864 			return -EINVAL;
9865 
9866 		/* unconditional jump with single edge */
9867 		ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env,
9868 				true);
9869 		if (ret)
9870 			return ret;
9871 
9872 		/* unconditional jmp is not a good pruning point,
9873 		 * but it's marked, since backtracking needs
9874 		 * to record jmp history in is_state_visited().
9875 		 */
9876 		init_explored_state(env, t + insns[t].off + 1);
9877 		/* tell verifier to check for equivalent states
9878 		 * after every call and jump
9879 		 */
9880 		if (t + 1 < insn_cnt)
9881 			init_explored_state(env, t + 1);
9882 
9883 		return ret;
9884 
9885 	default:
9886 		/* conditional jump with two edges */
9887 		init_explored_state(env, t);
9888 		ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
9889 		if (ret)
9890 			return ret;
9891 
9892 		return push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
9893 	}
9894 }
9895 
9896 /* non-recursive depth-first-search to detect loops in BPF program
9897  * loop == back-edge in directed graph
9898  */
9899 static int check_cfg(struct bpf_verifier_env *env)
9900 {
9901 	int insn_cnt = env->prog->len;
9902 	int *insn_stack, *insn_state;
9903 	int ret = 0;
9904 	int i;
9905 
9906 	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
9907 	if (!insn_state)
9908 		return -ENOMEM;
9909 
9910 	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
9911 	if (!insn_stack) {
9912 		kvfree(insn_state);
9913 		return -ENOMEM;
9914 	}
9915 
9916 	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
9917 	insn_stack[0] = 0; /* 0 is the first instruction */
9918 	env->cfg.cur_stack = 1;
9919 
9920 	while (env->cfg.cur_stack > 0) {
9921 		int t = insn_stack[env->cfg.cur_stack - 1];
9922 
9923 		ret = visit_insn(t, insn_cnt, env);
9924 		switch (ret) {
9925 		case DONE_EXPLORING:
9926 			insn_state[t] = EXPLORED;
9927 			env->cfg.cur_stack--;
9928 			break;
9929 		case KEEP_EXPLORING:
9930 			break;
9931 		default:
9932 			if (ret > 0) {
9933 				verbose(env, "visit_insn internal bug\n");
9934 				ret = -EFAULT;
9935 			}
9936 			goto err_free;
9937 		}
9938 	}
9939 
9940 	if (env->cfg.cur_stack < 0) {
9941 		verbose(env, "pop stack internal bug\n");
9942 		ret = -EFAULT;
9943 		goto err_free;
9944 	}
9945 
9946 	for (i = 0; i < insn_cnt; i++) {
9947 		if (insn_state[i] != EXPLORED) {
9948 			verbose(env, "unreachable insn %d\n", i);
9949 			ret = -EINVAL;
9950 			goto err_free;
9951 		}
9952 	}
9953 	ret = 0; /* cfg looks good */
9954 
9955 err_free:
9956 	kvfree(insn_state);
9957 	kvfree(insn_stack);
9958 	env->cfg.insn_state = env->cfg.insn_stack = NULL;
9959 	return ret;
9960 }
9961 
9962 static int check_abnormal_return(struct bpf_verifier_env *env)
9963 {
9964 	int i;
9965 
9966 	for (i = 1; i < env->subprog_cnt; i++) {
9967 		if (env->subprog_info[i].has_ld_abs) {
9968 			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
9969 			return -EINVAL;
9970 		}
9971 		if (env->subprog_info[i].has_tail_call) {
9972 			verbose(env, "tail_call is not allowed in subprogs without BTF\n");
9973 			return -EINVAL;
9974 		}
9975 	}
9976 	return 0;
9977 }
9978 
9979 /* The minimum supported BTF func info size */
9980 #define MIN_BPF_FUNCINFO_SIZE	8
9981 #define MAX_FUNCINFO_REC_SIZE	252
9982 
9983 static int check_btf_func(struct bpf_verifier_env *env,
9984 			  const union bpf_attr *attr,
9985 			  bpfptr_t uattr)
9986 {
9987 	const struct btf_type *type, *func_proto, *ret_type;
9988 	u32 i, nfuncs, urec_size, min_size;
9989 	u32 krec_size = sizeof(struct bpf_func_info);
9990 	struct bpf_func_info *krecord;
9991 	struct bpf_func_info_aux *info_aux = NULL;
9992 	struct bpf_prog *prog;
9993 	const struct btf *btf;
9994 	bpfptr_t urecord;
9995 	u32 prev_offset = 0;
9996 	bool scalar_return;
9997 	int ret = -ENOMEM;
9998 
9999 	nfuncs = attr->func_info_cnt;
10000 	if (!nfuncs) {
10001 		if (check_abnormal_return(env))
10002 			return -EINVAL;
10003 		return 0;
10004 	}
10005 
10006 	if (nfuncs != env->subprog_cnt) {
10007 		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
10008 		return -EINVAL;
10009 	}
10010 
10011 	urec_size = attr->func_info_rec_size;
10012 	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
10013 	    urec_size > MAX_FUNCINFO_REC_SIZE ||
10014 	    urec_size % sizeof(u32)) {
10015 		verbose(env, "invalid func info rec size %u\n", urec_size);
10016 		return -EINVAL;
10017 	}
10018 
10019 	prog = env->prog;
10020 	btf = prog->aux->btf;
10021 
10022 	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
10023 	min_size = min_t(u32, krec_size, urec_size);
10024 
10025 	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
10026 	if (!krecord)
10027 		return -ENOMEM;
10028 	info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
10029 	if (!info_aux)
10030 		goto err_free;
10031 
10032 	for (i = 0; i < nfuncs; i++) {
10033 		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
10034 		if (ret) {
10035 			if (ret == -E2BIG) {
10036 				verbose(env, "nonzero tailing record in func info");
10037 				/* set the size kernel expects so loader can zero
10038 				 * out the rest of the record.
10039 				 */
10040 				if (copy_to_bpfptr_offset(uattr,
10041 							  offsetof(union bpf_attr, func_info_rec_size),
10042 							  &min_size, sizeof(min_size)))
10043 					ret = -EFAULT;
10044 			}
10045 			goto err_free;
10046 		}
10047 
10048 		if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
10049 			ret = -EFAULT;
10050 			goto err_free;
10051 		}
10052 
10053 		/* check insn_off */
10054 		ret = -EINVAL;
10055 		if (i == 0) {
10056 			if (krecord[i].insn_off) {
10057 				verbose(env,
10058 					"nonzero insn_off %u for the first func info record",
10059 					krecord[i].insn_off);
10060 				goto err_free;
10061 			}
10062 		} else if (krecord[i].insn_off <= prev_offset) {
10063 			verbose(env,
10064 				"same or smaller insn offset (%u) than previous func info record (%u)",
10065 				krecord[i].insn_off, prev_offset);
10066 			goto err_free;
10067 		}
10068 
10069 		if (env->subprog_info[i].start != krecord[i].insn_off) {
10070 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
10071 			goto err_free;
10072 		}
10073 
10074 		/* check type_id */
10075 		type = btf_type_by_id(btf, krecord[i].type_id);
10076 		if (!type || !btf_type_is_func(type)) {
10077 			verbose(env, "invalid type id %d in func info",
10078 				krecord[i].type_id);
10079 			goto err_free;
10080 		}
10081 		info_aux[i].linkage = BTF_INFO_VLEN(type->info);
10082 
10083 		func_proto = btf_type_by_id(btf, type->type);
10084 		if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
10085 			/* btf_func_check() already verified it during BTF load */
10086 			goto err_free;
10087 		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
10088 		scalar_return =
10089 			btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type);
10090 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
10091 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
10092 			goto err_free;
10093 		}
10094 		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
10095 			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
10096 			goto err_free;
10097 		}
10098 
10099 		prev_offset = krecord[i].insn_off;
10100 		bpfptr_add(&urecord, urec_size);
10101 	}
10102 
10103 	prog->aux->func_info = krecord;
10104 	prog->aux->func_info_cnt = nfuncs;
10105 	prog->aux->func_info_aux = info_aux;
10106 	return 0;
10107 
10108 err_free:
10109 	kvfree(krecord);
10110 	kfree(info_aux);
10111 	return ret;
10112 }
10113 
10114 static void adjust_btf_func(struct bpf_verifier_env *env)
10115 {
10116 	struct bpf_prog_aux *aux = env->prog->aux;
10117 	int i;
10118 
10119 	if (!aux->func_info)
10120 		return;
10121 
10122 	for (i = 0; i < env->subprog_cnt; i++)
10123 		aux->func_info[i].insn_off = env->subprog_info[i].start;
10124 }
10125 
10126 #define MIN_BPF_LINEINFO_SIZE	(offsetof(struct bpf_line_info, line_col) + \
10127 		sizeof(((struct bpf_line_info *)(0))->line_col))
10128 #define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE
10129 
10130 static int check_btf_line(struct bpf_verifier_env *env,
10131 			  const union bpf_attr *attr,
10132 			  bpfptr_t uattr)
10133 {
10134 	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
10135 	struct bpf_subprog_info *sub;
10136 	struct bpf_line_info *linfo;
10137 	struct bpf_prog *prog;
10138 	const struct btf *btf;
10139 	bpfptr_t ulinfo;
10140 	int err;
10141 
10142 	nr_linfo = attr->line_info_cnt;
10143 	if (!nr_linfo)
10144 		return 0;
10145 	if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
10146 		return -EINVAL;
10147 
10148 	rec_size = attr->line_info_rec_size;
10149 	if (rec_size < MIN_BPF_LINEINFO_SIZE ||
10150 	    rec_size > MAX_LINEINFO_REC_SIZE ||
10151 	    rec_size & (sizeof(u32) - 1))
10152 		return -EINVAL;
10153 
10154 	/* Need to zero it in case the userspace may
10155 	 * pass in a smaller bpf_line_info object.
10156 	 */
10157 	linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
10158 			 GFP_KERNEL | __GFP_NOWARN);
10159 	if (!linfo)
10160 		return -ENOMEM;
10161 
10162 	prog = env->prog;
10163 	btf = prog->aux->btf;
10164 
10165 	s = 0;
10166 	sub = env->subprog_info;
10167 	ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
10168 	expected_size = sizeof(struct bpf_line_info);
10169 	ncopy = min_t(u32, expected_size, rec_size);
10170 	for (i = 0; i < nr_linfo; i++) {
10171 		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
10172 		if (err) {
10173 			if (err == -E2BIG) {
10174 				verbose(env, "nonzero tailing record in line_info");
10175 				if (copy_to_bpfptr_offset(uattr,
10176 							  offsetof(union bpf_attr, line_info_rec_size),
10177 							  &expected_size, sizeof(expected_size)))
10178 					err = -EFAULT;
10179 			}
10180 			goto err_free;
10181 		}
10182 
10183 		if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
10184 			err = -EFAULT;
10185 			goto err_free;
10186 		}
10187 
10188 		/*
10189 		 * Check insn_off to ensure
10190 		 * 1) strictly increasing AND
10191 		 * 2) bounded by prog->len
10192 		 *
10193 		 * The linfo[0].insn_off == 0 check logically falls into
10194 		 * the later "missing bpf_line_info for func..." case
10195 		 * because the first linfo[0].insn_off must be the
10196 		 * first sub also and the first sub must have
10197 		 * subprog_info[0].start == 0.
10198 		 */
10199 		if ((i && linfo[i].insn_off <= prev_offset) ||
10200 		    linfo[i].insn_off >= prog->len) {
10201 			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
10202 				i, linfo[i].insn_off, prev_offset,
10203 				prog->len);
10204 			err = -EINVAL;
10205 			goto err_free;
10206 		}
10207 
10208 		if (!prog->insnsi[linfo[i].insn_off].code) {
10209 			verbose(env,
10210 				"Invalid insn code at line_info[%u].insn_off\n",
10211 				i);
10212 			err = -EINVAL;
10213 			goto err_free;
10214 		}
10215 
10216 		if (!btf_name_by_offset(btf, linfo[i].line_off) ||
10217 		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
10218 			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
10219 			err = -EINVAL;
10220 			goto err_free;
10221 		}
10222 
10223 		if (s != env->subprog_cnt) {
10224 			if (linfo[i].insn_off == sub[s].start) {
10225 				sub[s].linfo_idx = i;
10226 				s++;
10227 			} else if (sub[s].start < linfo[i].insn_off) {
10228 				verbose(env, "missing bpf_line_info for func#%u\n", s);
10229 				err = -EINVAL;
10230 				goto err_free;
10231 			}
10232 		}
10233 
10234 		prev_offset = linfo[i].insn_off;
10235 		bpfptr_add(&ulinfo, rec_size);
10236 	}
10237 
10238 	if (s != env->subprog_cnt) {
10239 		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
10240 			env->subprog_cnt - s, s);
10241 		err = -EINVAL;
10242 		goto err_free;
10243 	}
10244 
10245 	prog->aux->linfo = linfo;
10246 	prog->aux->nr_linfo = nr_linfo;
10247 
10248 	return 0;
10249 
10250 err_free:
10251 	kvfree(linfo);
10252 	return err;
10253 }
10254 
10255 static int check_btf_info(struct bpf_verifier_env *env,
10256 			  const union bpf_attr *attr,
10257 			  bpfptr_t uattr)
10258 {
10259 	struct btf *btf;
10260 	int err;
10261 
10262 	if (!attr->func_info_cnt && !attr->line_info_cnt) {
10263 		if (check_abnormal_return(env))
10264 			return -EINVAL;
10265 		return 0;
10266 	}
10267 
10268 	btf = btf_get_by_fd(attr->prog_btf_fd);
10269 	if (IS_ERR(btf))
10270 		return PTR_ERR(btf);
10271 	if (btf_is_kernel(btf)) {
10272 		btf_put(btf);
10273 		return -EACCES;
10274 	}
10275 	env->prog->aux->btf = btf;
10276 
10277 	err = check_btf_func(env, attr, uattr);
10278 	if (err)
10279 		return err;
10280 
10281 	err = check_btf_line(env, attr, uattr);
10282 	if (err)
10283 		return err;
10284 
10285 	return 0;
10286 }
10287 
10288 /* check %cur's range satisfies %old's */
10289 static bool range_within(struct bpf_reg_state *old,
10290 			 struct bpf_reg_state *cur)
10291 {
10292 	return old->umin_value <= cur->umin_value &&
10293 	       old->umax_value >= cur->umax_value &&
10294 	       old->smin_value <= cur->smin_value &&
10295 	       old->smax_value >= cur->smax_value &&
10296 	       old->u32_min_value <= cur->u32_min_value &&
10297 	       old->u32_max_value >= cur->u32_max_value &&
10298 	       old->s32_min_value <= cur->s32_min_value &&
10299 	       old->s32_max_value >= cur->s32_max_value;
10300 }
10301 
10302 /* If in the old state two registers had the same id, then they need to have
10303  * the same id in the new state as well.  But that id could be different from
10304  * the old state, so we need to track the mapping from old to new ids.
10305  * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
10306  * regs with old id 5 must also have new id 9 for the new state to be safe.  But
10307  * regs with a different old id could still have new id 9, we don't care about
10308  * that.
10309  * So we look through our idmap to see if this old id has been seen before.  If
10310  * so, we require the new id to match; otherwise, we add the id pair to the map.
10311  */
10312 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap)
10313 {
10314 	unsigned int i;
10315 
10316 	for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
10317 		if (!idmap[i].old) {
10318 			/* Reached an empty slot; haven't seen this id before */
10319 			idmap[i].old = old_id;
10320 			idmap[i].cur = cur_id;
10321 			return true;
10322 		}
10323 		if (idmap[i].old == old_id)
10324 			return idmap[i].cur == cur_id;
10325 	}
10326 	/* We ran out of idmap slots, which should be impossible */
10327 	WARN_ON_ONCE(1);
10328 	return false;
10329 }
10330 
10331 static void clean_func_state(struct bpf_verifier_env *env,
10332 			     struct bpf_func_state *st)
10333 {
10334 	enum bpf_reg_liveness live;
10335 	int i, j;
10336 
10337 	for (i = 0; i < BPF_REG_FP; i++) {
10338 		live = st->regs[i].live;
10339 		/* liveness must not touch this register anymore */
10340 		st->regs[i].live |= REG_LIVE_DONE;
10341 		if (!(live & REG_LIVE_READ))
10342 			/* since the register is unused, clear its state
10343 			 * to make further comparison simpler
10344 			 */
10345 			__mark_reg_not_init(env, &st->regs[i]);
10346 	}
10347 
10348 	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
10349 		live = st->stack[i].spilled_ptr.live;
10350 		/* liveness must not touch this stack slot anymore */
10351 		st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
10352 		if (!(live & REG_LIVE_READ)) {
10353 			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);
10354 			for (j = 0; j < BPF_REG_SIZE; j++)
10355 				st->stack[i].slot_type[j] = STACK_INVALID;
10356 		}
10357 	}
10358 }
10359 
10360 static void clean_verifier_state(struct bpf_verifier_env *env,
10361 				 struct bpf_verifier_state *st)
10362 {
10363 	int i;
10364 
10365 	if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
10366 		/* all regs in this state in all frames were already marked */
10367 		return;
10368 
10369 	for (i = 0; i <= st->curframe; i++)
10370 		clean_func_state(env, st->frame[i]);
10371 }
10372 
10373 /* the parentage chains form a tree.
10374  * the verifier states are added to state lists at given insn and
10375  * pushed into state stack for future exploration.
10376  * when the verifier reaches bpf_exit insn some of the verifer states
10377  * stored in the state lists have their final liveness state already,
10378  * but a lot of states will get revised from liveness point of view when
10379  * the verifier explores other branches.
10380  * Example:
10381  * 1: r0 = 1
10382  * 2: if r1 == 100 goto pc+1
10383  * 3: r0 = 2
10384  * 4: exit
10385  * when the verifier reaches exit insn the register r0 in the state list of
10386  * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
10387  * of insn 2 and goes exploring further. At the insn 4 it will walk the
10388  * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
10389  *
10390  * Since the verifier pushes the branch states as it sees them while exploring
10391  * the program the condition of walking the branch instruction for the second
10392  * time means that all states below this branch were already explored and
10393  * their final liveness marks are already propagated.
10394  * Hence when the verifier completes the search of state list in is_state_visited()
10395  * we can call this clean_live_states() function to mark all liveness states
10396  * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
10397  * will not be used.
10398  * This function also clears the registers and stack for states that !READ
10399  * to simplify state merging.
10400  *
10401  * Important note here that walking the same branch instruction in the callee
10402  * doesn't meant that the states are DONE. The verifier has to compare
10403  * the callsites
10404  */
10405 static void clean_live_states(struct bpf_verifier_env *env, int insn,
10406 			      struct bpf_verifier_state *cur)
10407 {
10408 	struct bpf_verifier_state_list *sl;
10409 	int i;
10410 
10411 	sl = *explored_state(env, insn);
10412 	while (sl) {
10413 		if (sl->state.branches)
10414 			goto next;
10415 		if (sl->state.insn_idx != insn ||
10416 		    sl->state.curframe != cur->curframe)
10417 			goto next;
10418 		for (i = 0; i <= cur->curframe; i++)
10419 			if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
10420 				goto next;
10421 		clean_verifier_state(env, &sl->state);
10422 next:
10423 		sl = sl->next;
10424 	}
10425 }
10426 
10427 /* Returns true if (rold safe implies rcur safe) */
10428 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
10429 		    struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)
10430 {
10431 	bool equal;
10432 
10433 	if (!(rold->live & REG_LIVE_READ))
10434 		/* explored state didn't use this */
10435 		return true;
10436 
10437 	equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
10438 
10439 	if (rold->type == PTR_TO_STACK)
10440 		/* two stack pointers are equal only if they're pointing to
10441 		 * the same stack frame, since fp-8 in foo != fp-8 in bar
10442 		 */
10443 		return equal && rold->frameno == rcur->frameno;
10444 
10445 	if (equal)
10446 		return true;
10447 
10448 	if (rold->type == NOT_INIT)
10449 		/* explored state can't have used this */
10450 		return true;
10451 	if (rcur->type == NOT_INIT)
10452 		return false;
10453 	switch (rold->type) {
10454 	case SCALAR_VALUE:
10455 		if (env->explore_alu_limits)
10456 			return false;
10457 		if (rcur->type == SCALAR_VALUE) {
10458 			if (!rold->precise && !rcur->precise)
10459 				return true;
10460 			/* new val must satisfy old val knowledge */
10461 			return range_within(rold, rcur) &&
10462 			       tnum_in(rold->var_off, rcur->var_off);
10463 		} else {
10464 			/* We're trying to use a pointer in place of a scalar.
10465 			 * Even if the scalar was unbounded, this could lead to
10466 			 * pointer leaks because scalars are allowed to leak
10467 			 * while pointers are not. We could make this safe in
10468 			 * special cases if root is calling us, but it's
10469 			 * probably not worth the hassle.
10470 			 */
10471 			return false;
10472 		}
10473 	case PTR_TO_MAP_KEY:
10474 	case PTR_TO_MAP_VALUE:
10475 		/* If the new min/max/var_off satisfy the old ones and
10476 		 * everything else matches, we are OK.
10477 		 * 'id' is not compared, since it's only used for maps with
10478 		 * bpf_spin_lock inside map element and in such cases if
10479 		 * the rest of the prog is valid for one map element then
10480 		 * it's valid for all map elements regardless of the key
10481 		 * used in bpf_map_lookup()
10482 		 */
10483 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
10484 		       range_within(rold, rcur) &&
10485 		       tnum_in(rold->var_off, rcur->var_off);
10486 	case PTR_TO_MAP_VALUE_OR_NULL:
10487 		/* a PTR_TO_MAP_VALUE could be safe to use as a
10488 		 * PTR_TO_MAP_VALUE_OR_NULL into the same map.
10489 		 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
10490 		 * checked, doing so could have affected others with the same
10491 		 * id, and we can't check for that because we lost the id when
10492 		 * we converted to a PTR_TO_MAP_VALUE.
10493 		 */
10494 		if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
10495 			return false;
10496 		if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
10497 			return false;
10498 		/* Check our ids match any regs they're supposed to */
10499 		return check_ids(rold->id, rcur->id, idmap);
10500 	case PTR_TO_PACKET_META:
10501 	case PTR_TO_PACKET:
10502 		if (rcur->type != rold->type)
10503 			return false;
10504 		/* We must have at least as much range as the old ptr
10505 		 * did, so that any accesses which were safe before are
10506 		 * still safe.  This is true even if old range < old off,
10507 		 * since someone could have accessed through (ptr - k), or
10508 		 * even done ptr -= k in a register, to get a safe access.
10509 		 */
10510 		if (rold->range > rcur->range)
10511 			return false;
10512 		/* If the offsets don't match, we can't trust our alignment;
10513 		 * nor can we be sure that we won't fall out of range.
10514 		 */
10515 		if (rold->off != rcur->off)
10516 			return false;
10517 		/* id relations must be preserved */
10518 		if (rold->id && !check_ids(rold->id, rcur->id, idmap))
10519 			return false;
10520 		/* new val must satisfy old val knowledge */
10521 		return range_within(rold, rcur) &&
10522 		       tnum_in(rold->var_off, rcur->var_off);
10523 	case PTR_TO_CTX:
10524 	case CONST_PTR_TO_MAP:
10525 	case PTR_TO_PACKET_END:
10526 	case PTR_TO_FLOW_KEYS:
10527 	case PTR_TO_SOCKET:
10528 	case PTR_TO_SOCKET_OR_NULL:
10529 	case PTR_TO_SOCK_COMMON:
10530 	case PTR_TO_SOCK_COMMON_OR_NULL:
10531 	case PTR_TO_TCP_SOCK:
10532 	case PTR_TO_TCP_SOCK_OR_NULL:
10533 	case PTR_TO_XDP_SOCK:
10534 		/* Only valid matches are exact, which memcmp() above
10535 		 * would have accepted
10536 		 */
10537 	default:
10538 		/* Don't know what's going on, just say it's not safe */
10539 		return false;
10540 	}
10541 
10542 	/* Shouldn't get here; if we do, say it's not safe */
10543 	WARN_ON_ONCE(1);
10544 	return false;
10545 }
10546 
10547 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
10548 		      struct bpf_func_state *cur, struct bpf_id_pair *idmap)
10549 {
10550 	int i, spi;
10551 
10552 	/* walk slots of the explored stack and ignore any additional
10553 	 * slots in the current stack, since explored(safe) state
10554 	 * didn't use them
10555 	 */
10556 	for (i = 0; i < old->allocated_stack; i++) {
10557 		spi = i / BPF_REG_SIZE;
10558 
10559 		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
10560 			i += BPF_REG_SIZE - 1;
10561 			/* explored state didn't use this */
10562 			continue;
10563 		}
10564 
10565 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
10566 			continue;
10567 
10568 		/* explored stack has more populated slots than current stack
10569 		 * and these slots were used
10570 		 */
10571 		if (i >= cur->allocated_stack)
10572 			return false;
10573 
10574 		/* if old state was safe with misc data in the stack
10575 		 * it will be safe with zero-initialized stack.
10576 		 * The opposite is not true
10577 		 */
10578 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
10579 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
10580 			continue;
10581 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
10582 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
10583 			/* Ex: old explored (safe) state has STACK_SPILL in
10584 			 * this stack slot, but current has STACK_MISC ->
10585 			 * this verifier states are not equivalent,
10586 			 * return false to continue verification of this path
10587 			 */
10588 			return false;
10589 		if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
10590 			continue;
10591 		if (!is_spilled_reg(&old->stack[spi]))
10592 			continue;
10593 		if (!regsafe(env, &old->stack[spi].spilled_ptr,
10594 			     &cur->stack[spi].spilled_ptr, idmap))
10595 			/* when explored and current stack slot are both storing
10596 			 * spilled registers, check that stored pointers types
10597 			 * are the same as well.
10598 			 * Ex: explored safe path could have stored
10599 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
10600 			 * but current path has stored:
10601 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
10602 			 * such verifier states are not equivalent.
10603 			 * return false to continue verification of this path
10604 			 */
10605 			return false;
10606 	}
10607 	return true;
10608 }
10609 
10610 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
10611 {
10612 	if (old->acquired_refs != cur->acquired_refs)
10613 		return false;
10614 	return !memcmp(old->refs, cur->refs,
10615 		       sizeof(*old->refs) * old->acquired_refs);
10616 }
10617 
10618 /* compare two verifier states
10619  *
10620  * all states stored in state_list are known to be valid, since
10621  * verifier reached 'bpf_exit' instruction through them
10622  *
10623  * this function is called when verifier exploring different branches of
10624  * execution popped from the state stack. If it sees an old state that has
10625  * more strict register state and more strict stack state then this execution
10626  * branch doesn't need to be explored further, since verifier already
10627  * concluded that more strict state leads to valid finish.
10628  *
10629  * Therefore two states are equivalent if register state is more conservative
10630  * and explored stack state is more conservative than the current one.
10631  * Example:
10632  *       explored                   current
10633  * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
10634  * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
10635  *
10636  * In other words if current stack state (one being explored) has more
10637  * valid slots than old one that already passed validation, it means
10638  * the verifier can stop exploring and conclude that current state is valid too
10639  *
10640  * Similarly with registers. If explored state has register type as invalid
10641  * whereas register type in current state is meaningful, it means that
10642  * the current state will reach 'bpf_exit' instruction safely
10643  */
10644 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
10645 			      struct bpf_func_state *cur)
10646 {
10647 	int i;
10648 
10649 	memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
10650 	for (i = 0; i < MAX_BPF_REG; i++)
10651 		if (!regsafe(env, &old->regs[i], &cur->regs[i],
10652 			     env->idmap_scratch))
10653 			return false;
10654 
10655 	if (!stacksafe(env, old, cur, env->idmap_scratch))
10656 		return false;
10657 
10658 	if (!refsafe(old, cur))
10659 		return false;
10660 
10661 	return true;
10662 }
10663 
10664 static bool states_equal(struct bpf_verifier_env *env,
10665 			 struct bpf_verifier_state *old,
10666 			 struct bpf_verifier_state *cur)
10667 {
10668 	int i;
10669 
10670 	if (old->curframe != cur->curframe)
10671 		return false;
10672 
10673 	/* Verification state from speculative execution simulation
10674 	 * must never prune a non-speculative execution one.
10675 	 */
10676 	if (old->speculative && !cur->speculative)
10677 		return false;
10678 
10679 	if (old->active_spin_lock != cur->active_spin_lock)
10680 		return false;
10681 
10682 	/* for states to be equal callsites have to be the same
10683 	 * and all frame states need to be equivalent
10684 	 */
10685 	for (i = 0; i <= old->curframe; i++) {
10686 		if (old->frame[i]->callsite != cur->frame[i]->callsite)
10687 			return false;
10688 		if (!func_states_equal(env, old->frame[i], cur->frame[i]))
10689 			return false;
10690 	}
10691 	return true;
10692 }
10693 
10694 /* Return 0 if no propagation happened. Return negative error code if error
10695  * happened. Otherwise, return the propagated bit.
10696  */
10697 static int propagate_liveness_reg(struct bpf_verifier_env *env,
10698 				  struct bpf_reg_state *reg,
10699 				  struct bpf_reg_state *parent_reg)
10700 {
10701 	u8 parent_flag = parent_reg->live & REG_LIVE_READ;
10702 	u8 flag = reg->live & REG_LIVE_READ;
10703 	int err;
10704 
10705 	/* When comes here, read flags of PARENT_REG or REG could be any of
10706 	 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need
10707 	 * of propagation if PARENT_REG has strongest REG_LIVE_READ64.
10708 	 */
10709 	if (parent_flag == REG_LIVE_READ64 ||
10710 	    /* Or if there is no read flag from REG. */
10711 	    !flag ||
10712 	    /* Or if the read flag from REG is the same as PARENT_REG. */
10713 	    parent_flag == flag)
10714 		return 0;
10715 
10716 	err = mark_reg_read(env, reg, parent_reg, flag);
10717 	if (err)
10718 		return err;
10719 
10720 	return flag;
10721 }
10722 
10723 /* A write screens off any subsequent reads; but write marks come from the
10724  * straight-line code between a state and its parent.  When we arrive at an
10725  * equivalent state (jump target or such) we didn't arrive by the straight-line
10726  * code, so read marks in the state must propagate to the parent regardless
10727  * of the state's write marks. That's what 'parent == state->parent' comparison
10728  * in mark_reg_read() is for.
10729  */
10730 static int propagate_liveness(struct bpf_verifier_env *env,
10731 			      const struct bpf_verifier_state *vstate,
10732 			      struct bpf_verifier_state *vparent)
10733 {
10734 	struct bpf_reg_state *state_reg, *parent_reg;
10735 	struct bpf_func_state *state, *parent;
10736 	int i, frame, err = 0;
10737 
10738 	if (vparent->curframe != vstate->curframe) {
10739 		WARN(1, "propagate_live: parent frame %d current frame %d\n",
10740 		     vparent->curframe, vstate->curframe);
10741 		return -EFAULT;
10742 	}
10743 	/* Propagate read liveness of registers... */
10744 	BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
10745 	for (frame = 0; frame <= vstate->curframe; frame++) {
10746 		parent = vparent->frame[frame];
10747 		state = vstate->frame[frame];
10748 		parent_reg = parent->regs;
10749 		state_reg = state->regs;
10750 		/* We don't need to worry about FP liveness, it's read-only */
10751 		for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
10752 			err = propagate_liveness_reg(env, &state_reg[i],
10753 						     &parent_reg[i]);
10754 			if (err < 0)
10755 				return err;
10756 			if (err == REG_LIVE_READ64)
10757 				mark_insn_zext(env, &parent_reg[i]);
10758 		}
10759 
10760 		/* Propagate stack slots. */
10761 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
10762 			    i < parent->allocated_stack / BPF_REG_SIZE; i++) {
10763 			parent_reg = &parent->stack[i].spilled_ptr;
10764 			state_reg = &state->stack[i].spilled_ptr;
10765 			err = propagate_liveness_reg(env, state_reg,
10766 						     parent_reg);
10767 			if (err < 0)
10768 				return err;
10769 		}
10770 	}
10771 	return 0;
10772 }
10773 
10774 /* find precise scalars in the previous equivalent state and
10775  * propagate them into the current state
10776  */
10777 static int propagate_precision(struct bpf_verifier_env *env,
10778 			       const struct bpf_verifier_state *old)
10779 {
10780 	struct bpf_reg_state *state_reg;
10781 	struct bpf_func_state *state;
10782 	int i, err = 0;
10783 
10784 	state = old->frame[old->curframe];
10785 	state_reg = state->regs;
10786 	for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
10787 		if (state_reg->type != SCALAR_VALUE ||
10788 		    !state_reg->precise)
10789 			continue;
10790 		if (env->log.level & BPF_LOG_LEVEL2)
10791 			verbose(env, "propagating r%d\n", i);
10792 		err = mark_chain_precision(env, i);
10793 		if (err < 0)
10794 			return err;
10795 	}
10796 
10797 	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
10798 		if (!is_spilled_reg(&state->stack[i]))
10799 			continue;
10800 		state_reg = &state->stack[i].spilled_ptr;
10801 		if (state_reg->type != SCALAR_VALUE ||
10802 		    !state_reg->precise)
10803 			continue;
10804 		if (env->log.level & BPF_LOG_LEVEL2)
10805 			verbose(env, "propagating fp%d\n",
10806 				(-i - 1) * BPF_REG_SIZE);
10807 		err = mark_chain_precision_stack(env, i);
10808 		if (err < 0)
10809 			return err;
10810 	}
10811 	return 0;
10812 }
10813 
10814 static bool states_maybe_looping(struct bpf_verifier_state *old,
10815 				 struct bpf_verifier_state *cur)
10816 {
10817 	struct bpf_func_state *fold, *fcur;
10818 	int i, fr = cur->curframe;
10819 
10820 	if (old->curframe != fr)
10821 		return false;
10822 
10823 	fold = old->frame[fr];
10824 	fcur = cur->frame[fr];
10825 	for (i = 0; i < MAX_BPF_REG; i++)
10826 		if (memcmp(&fold->regs[i], &fcur->regs[i],
10827 			   offsetof(struct bpf_reg_state, parent)))
10828 			return false;
10829 	return true;
10830 }
10831 
10832 
10833 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
10834 {
10835 	struct bpf_verifier_state_list *new_sl;
10836 	struct bpf_verifier_state_list *sl, **pprev;
10837 	struct bpf_verifier_state *cur = env->cur_state, *new;
10838 	int i, j, err, states_cnt = 0;
10839 	bool add_new_state = env->test_state_freq ? true : false;
10840 
10841 	cur->last_insn_idx = env->prev_insn_idx;
10842 	if (!env->insn_aux_data[insn_idx].prune_point)
10843 		/* this 'insn_idx' instruction wasn't marked, so we will not
10844 		 * be doing state search here
10845 		 */
10846 		return 0;
10847 
10848 	/* bpf progs typically have pruning point every 4 instructions
10849 	 * http://vger.kernel.org/bpfconf2019.html#session-1
10850 	 * Do not add new state for future pruning if the verifier hasn't seen
10851 	 * at least 2 jumps and at least 8 instructions.
10852 	 * This heuristics helps decrease 'total_states' and 'peak_states' metric.
10853 	 * In tests that amounts to up to 50% reduction into total verifier
10854 	 * memory consumption and 20% verifier time speedup.
10855 	 */
10856 	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
10857 	    env->insn_processed - env->prev_insn_processed >= 8)
10858 		add_new_state = true;
10859 
10860 	pprev = explored_state(env, insn_idx);
10861 	sl = *pprev;
10862 
10863 	clean_live_states(env, insn_idx, cur);
10864 
10865 	while (sl) {
10866 		states_cnt++;
10867 		if (sl->state.insn_idx != insn_idx)
10868 			goto next;
10869 
10870 		if (sl->state.branches) {
10871 			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
10872 
10873 			if (frame->in_async_callback_fn &&
10874 			    frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
10875 				/* Different async_entry_cnt means that the verifier is
10876 				 * processing another entry into async callback.
10877 				 * Seeing the same state is not an indication of infinite
10878 				 * loop or infinite recursion.
10879 				 * But finding the same state doesn't mean that it's safe
10880 				 * to stop processing the current state. The previous state
10881 				 * hasn't yet reached bpf_exit, since state.branches > 0.
10882 				 * Checking in_async_callback_fn alone is not enough either.
10883 				 * Since the verifier still needs to catch infinite loops
10884 				 * inside async callbacks.
10885 				 */
10886 			} else if (states_maybe_looping(&sl->state, cur) &&
10887 				   states_equal(env, &sl->state, cur)) {
10888 				verbose_linfo(env, insn_idx, "; ");
10889 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);
10890 				return -EINVAL;
10891 			}
10892 			/* if the verifier is processing a loop, avoid adding new state
10893 			 * too often, since different loop iterations have distinct
10894 			 * states and may not help future pruning.
10895 			 * This threshold shouldn't be too low to make sure that
10896 			 * a loop with large bound will be rejected quickly.
10897 			 * The most abusive loop will be:
10898 			 * r1 += 1
10899 			 * if r1 < 1000000 goto pc-2
10900 			 * 1M insn_procssed limit / 100 == 10k peak states.
10901 			 * This threshold shouldn't be too high either, since states
10902 			 * at the end of the loop are likely to be useful in pruning.
10903 			 */
10904 			if (env->jmps_processed - env->prev_jmps_processed < 20 &&
10905 			    env->insn_processed - env->prev_insn_processed < 100)
10906 				add_new_state = false;
10907 			goto miss;
10908 		}
10909 		if (states_equal(env, &sl->state, cur)) {
10910 			sl->hit_cnt++;
10911 			/* reached equivalent register/stack state,
10912 			 * prune the search.
10913 			 * Registers read by the continuation are read by us.
10914 			 * If we have any write marks in env->cur_state, they
10915 			 * will prevent corresponding reads in the continuation
10916 			 * from reaching our parent (an explored_state).  Our
10917 			 * own state will get the read marks recorded, but
10918 			 * they'll be immediately forgotten as we're pruning
10919 			 * this state and will pop a new one.
10920 			 */
10921 			err = propagate_liveness(env, &sl->state, cur);
10922 
10923 			/* if previous state reached the exit with precision and
10924 			 * current state is equivalent to it (except precsion marks)
10925 			 * the precision needs to be propagated back in
10926 			 * the current state.
10927 			 */
10928 			err = err ? : push_jmp_history(env, cur);
10929 			err = err ? : propagate_precision(env, &sl->state);
10930 			if (err)
10931 				return err;
10932 			return 1;
10933 		}
10934 miss:
10935 		/* when new state is not going to be added do not increase miss count.
10936 		 * Otherwise several loop iterations will remove the state
10937 		 * recorded earlier. The goal of these heuristics is to have
10938 		 * states from some iterations of the loop (some in the beginning
10939 		 * and some at the end) to help pruning.
10940 		 */
10941 		if (add_new_state)
10942 			sl->miss_cnt++;
10943 		/* heuristic to determine whether this state is beneficial
10944 		 * to keep checking from state equivalence point of view.
10945 		 * Higher numbers increase max_states_per_insn and verification time,
10946 		 * but do not meaningfully decrease insn_processed.
10947 		 */
10948 		if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
10949 			/* the state is unlikely to be useful. Remove it to
10950 			 * speed up verification
10951 			 */
10952 			*pprev = sl->next;
10953 			if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
10954 				u32 br = sl->state.branches;
10955 
10956 				WARN_ONCE(br,
10957 					  "BUG live_done but branches_to_explore %d\n",
10958 					  br);
10959 				free_verifier_state(&sl->state, false);
10960 				kfree(sl);
10961 				env->peak_states--;
10962 			} else {
10963 				/* cannot free this state, since parentage chain may
10964 				 * walk it later. Add it for free_list instead to
10965 				 * be freed at the end of verification
10966 				 */
10967 				sl->next = env->free_list;
10968 				env->free_list = sl;
10969 			}
10970 			sl = *pprev;
10971 			continue;
10972 		}
10973 next:
10974 		pprev = &sl->next;
10975 		sl = *pprev;
10976 	}
10977 
10978 	if (env->max_states_per_insn < states_cnt)
10979 		env->max_states_per_insn = states_cnt;
10980 
10981 	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
10982 		return push_jmp_history(env, cur);
10983 
10984 	if (!add_new_state)
10985 		return push_jmp_history(env, cur);
10986 
10987 	/* There were no equivalent states, remember the current one.
10988 	 * Technically the current state is not proven to be safe yet,
10989 	 * but it will either reach outer most bpf_exit (which means it's safe)
10990 	 * or it will be rejected. When there are no loops the verifier won't be
10991 	 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
10992 	 * again on the way to bpf_exit.
10993 	 * When looping the sl->state.branches will be > 0 and this state
10994 	 * will not be considered for equivalence until branches == 0.
10995 	 */
10996 	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
10997 	if (!new_sl)
10998 		return -ENOMEM;
10999 	env->total_states++;
11000 	env->peak_states++;
11001 	env->prev_jmps_processed = env->jmps_processed;
11002 	env->prev_insn_processed = env->insn_processed;
11003 
11004 	/* add new state to the head of linked list */
11005 	new = &new_sl->state;
11006 	err = copy_verifier_state(new, cur);
11007 	if (err) {
11008 		free_verifier_state(new, false);
11009 		kfree(new_sl);
11010 		return err;
11011 	}
11012 	new->insn_idx = insn_idx;
11013 	WARN_ONCE(new->branches != 1,
11014 		  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
11015 
11016 	cur->parent = new;
11017 	cur->first_insn_idx = insn_idx;
11018 	clear_jmp_history(cur);
11019 	new_sl->next = *explored_state(env, insn_idx);
11020 	*explored_state(env, insn_idx) = new_sl;
11021 	/* connect new state to parentage chain. Current frame needs all
11022 	 * registers connected. Only r6 - r9 of the callers are alive (pushed
11023 	 * to the stack implicitly by JITs) so in callers' frames connect just
11024 	 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
11025 	 * the state of the call instruction (with WRITTEN set), and r0 comes
11026 	 * from callee with its full parentage chain, anyway.
11027 	 */
11028 	/* clear write marks in current state: the writes we did are not writes
11029 	 * our child did, so they don't screen off its reads from us.
11030 	 * (There are no read marks in current state, because reads always mark
11031 	 * their parent and current state never has children yet.  Only
11032 	 * explored_states can get read marks.)
11033 	 */
11034 	for (j = 0; j <= cur->curframe; j++) {
11035 		for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
11036 			cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
11037 		for (i = 0; i < BPF_REG_FP; i++)
11038 			cur->frame[j]->regs[i].live = REG_LIVE_NONE;
11039 	}
11040 
11041 	/* all stack frames are accessible from callee, clear them all */
11042 	for (j = 0; j <= cur->curframe; j++) {
11043 		struct bpf_func_state *frame = cur->frame[j];
11044 		struct bpf_func_state *newframe = new->frame[j];
11045 
11046 		for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
11047 			frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
11048 			frame->stack[i].spilled_ptr.parent =
11049 						&newframe->stack[i].spilled_ptr;
11050 		}
11051 	}
11052 	return 0;
11053 }
11054 
11055 /* Return true if it's OK to have the same insn return a different type. */
11056 static bool reg_type_mismatch_ok(enum bpf_reg_type type)
11057 {
11058 	switch (type) {
11059 	case PTR_TO_CTX:
11060 	case PTR_TO_SOCKET:
11061 	case PTR_TO_SOCKET_OR_NULL:
11062 	case PTR_TO_SOCK_COMMON:
11063 	case PTR_TO_SOCK_COMMON_OR_NULL:
11064 	case PTR_TO_TCP_SOCK:
11065 	case PTR_TO_TCP_SOCK_OR_NULL:
11066 	case PTR_TO_XDP_SOCK:
11067 	case PTR_TO_BTF_ID:
11068 	case PTR_TO_BTF_ID_OR_NULL:
11069 		return false;
11070 	default:
11071 		return true;
11072 	}
11073 }
11074 
11075 /* If an instruction was previously used with particular pointer types, then we
11076  * need to be careful to avoid cases such as the below, where it may be ok
11077  * for one branch accessing the pointer, but not ok for the other branch:
11078  *
11079  * R1 = sock_ptr
11080  * goto X;
11081  * ...
11082  * R1 = some_other_valid_ptr;
11083  * goto X;
11084  * ...
11085  * R2 = *(u32 *)(R1 + 0);
11086  */
11087 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
11088 {
11089 	return src != prev && (!reg_type_mismatch_ok(src) ||
11090 			       !reg_type_mismatch_ok(prev));
11091 }
11092 
11093 static int do_check(struct bpf_verifier_env *env)
11094 {
11095 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
11096 	struct bpf_verifier_state *state = env->cur_state;
11097 	struct bpf_insn *insns = env->prog->insnsi;
11098 	struct bpf_reg_state *regs;
11099 	int insn_cnt = env->prog->len;
11100 	bool do_print_state = false;
11101 	int prev_insn_idx = -1;
11102 
11103 	for (;;) {
11104 		struct bpf_insn *insn;
11105 		u8 class;
11106 		int err;
11107 
11108 		env->prev_insn_idx = prev_insn_idx;
11109 		if (env->insn_idx >= insn_cnt) {
11110 			verbose(env, "invalid insn idx %d insn_cnt %d\n",
11111 				env->insn_idx, insn_cnt);
11112 			return -EFAULT;
11113 		}
11114 
11115 		insn = &insns[env->insn_idx];
11116 		class = BPF_CLASS(insn->code);
11117 
11118 		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
11119 			verbose(env,
11120 				"BPF program is too large. Processed %d insn\n",
11121 				env->insn_processed);
11122 			return -E2BIG;
11123 		}
11124 
11125 		err = is_state_visited(env, env->insn_idx);
11126 		if (err < 0)
11127 			return err;
11128 		if (err == 1) {
11129 			/* found equivalent state, can prune the search */
11130 			if (env->log.level & BPF_LOG_LEVEL) {
11131 				if (do_print_state)
11132 					verbose(env, "\nfrom %d to %d%s: safe\n",
11133 						env->prev_insn_idx, env->insn_idx,
11134 						env->cur_state->speculative ?
11135 						" (speculative execution)" : "");
11136 				else
11137 					verbose(env, "%d: safe\n", env->insn_idx);
11138 			}
11139 			goto process_bpf_exit;
11140 		}
11141 
11142 		if (signal_pending(current))
11143 			return -EAGAIN;
11144 
11145 		if (need_resched())
11146 			cond_resched();
11147 
11148 		if (env->log.level & BPF_LOG_LEVEL2 ||
11149 		    (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
11150 			if (env->log.level & BPF_LOG_LEVEL2)
11151 				verbose(env, "%d:", env->insn_idx);
11152 			else
11153 				verbose(env, "\nfrom %d to %d%s:",
11154 					env->prev_insn_idx, env->insn_idx,
11155 					env->cur_state->speculative ?
11156 					" (speculative execution)" : "");
11157 			print_verifier_state(env, state->frame[state->curframe]);
11158 			do_print_state = false;
11159 		}
11160 
11161 		if (env->log.level & BPF_LOG_LEVEL) {
11162 			const struct bpf_insn_cbs cbs = {
11163 				.cb_call	= disasm_kfunc_name,
11164 				.cb_print	= verbose,
11165 				.private_data	= env,
11166 			};
11167 
11168 			verbose_linfo(env, env->insn_idx, "; ");
11169 			verbose(env, "%d: ", env->insn_idx);
11170 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
11171 		}
11172 
11173 		if (bpf_prog_is_dev_bound(env->prog->aux)) {
11174 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
11175 							   env->prev_insn_idx);
11176 			if (err)
11177 				return err;
11178 		}
11179 
11180 		regs = cur_regs(env);
11181 		sanitize_mark_insn_seen(env);
11182 		prev_insn_idx = env->insn_idx;
11183 
11184 		if (class == BPF_ALU || class == BPF_ALU64) {
11185 			err = check_alu_op(env, insn);
11186 			if (err)
11187 				return err;
11188 
11189 		} else if (class == BPF_LDX) {
11190 			enum bpf_reg_type *prev_src_type, src_reg_type;
11191 
11192 			/* check for reserved fields is already done */
11193 
11194 			/* check src operand */
11195 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
11196 			if (err)
11197 				return err;
11198 
11199 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
11200 			if (err)
11201 				return err;
11202 
11203 			src_reg_type = regs[insn->src_reg].type;
11204 
11205 			/* check that memory (src_reg + off) is readable,
11206 			 * the state of dst_reg will be updated by this func
11207 			 */
11208 			err = check_mem_access(env, env->insn_idx, insn->src_reg,
11209 					       insn->off, BPF_SIZE(insn->code),
11210 					       BPF_READ, insn->dst_reg, false);
11211 			if (err)
11212 				return err;
11213 
11214 			prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
11215 
11216 			if (*prev_src_type == NOT_INIT) {
11217 				/* saw a valid insn
11218 				 * dst_reg = *(u32 *)(src_reg + off)
11219 				 * save type to validate intersecting paths
11220 				 */
11221 				*prev_src_type = src_reg_type;
11222 
11223 			} else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
11224 				/* ABuser program is trying to use the same insn
11225 				 * dst_reg = *(u32*) (src_reg + off)
11226 				 * with different pointer types:
11227 				 * src_reg == ctx in one branch and
11228 				 * src_reg == stack|map in some other branch.
11229 				 * Reject it.
11230 				 */
11231 				verbose(env, "same insn cannot be used with different pointers\n");
11232 				return -EINVAL;
11233 			}
11234 
11235 		} else if (class == BPF_STX) {
11236 			enum bpf_reg_type *prev_dst_type, dst_reg_type;
11237 
11238 			if (BPF_MODE(insn->code) == BPF_ATOMIC) {
11239 				err = check_atomic(env, env->insn_idx, insn);
11240 				if (err)
11241 					return err;
11242 				env->insn_idx++;
11243 				continue;
11244 			}
11245 
11246 			if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
11247 				verbose(env, "BPF_STX uses reserved fields\n");
11248 				return -EINVAL;
11249 			}
11250 
11251 			/* check src1 operand */
11252 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
11253 			if (err)
11254 				return err;
11255 			/* check src2 operand */
11256 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
11257 			if (err)
11258 				return err;
11259 
11260 			dst_reg_type = regs[insn->dst_reg].type;
11261 
11262 			/* check that memory (dst_reg + off) is writeable */
11263 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
11264 					       insn->off, BPF_SIZE(insn->code),
11265 					       BPF_WRITE, insn->src_reg, false);
11266 			if (err)
11267 				return err;
11268 
11269 			prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
11270 
11271 			if (*prev_dst_type == NOT_INIT) {
11272 				*prev_dst_type = dst_reg_type;
11273 			} else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
11274 				verbose(env, "same insn cannot be used with different pointers\n");
11275 				return -EINVAL;
11276 			}
11277 
11278 		} else if (class == BPF_ST) {
11279 			if (BPF_MODE(insn->code) != BPF_MEM ||
11280 			    insn->src_reg != BPF_REG_0) {
11281 				verbose(env, "BPF_ST uses reserved fields\n");
11282 				return -EINVAL;
11283 			}
11284 			/* check src operand */
11285 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
11286 			if (err)
11287 				return err;
11288 
11289 			if (is_ctx_reg(env, insn->dst_reg)) {
11290 				verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
11291 					insn->dst_reg,
11292 					reg_type_str[reg_state(env, insn->dst_reg)->type]);
11293 				return -EACCES;
11294 			}
11295 
11296 			/* check that memory (dst_reg + off) is writeable */
11297 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
11298 					       insn->off, BPF_SIZE(insn->code),
11299 					       BPF_WRITE, -1, false);
11300 			if (err)
11301 				return err;
11302 
11303 		} else if (class == BPF_JMP || class == BPF_JMP32) {
11304 			u8 opcode = BPF_OP(insn->code);
11305 
11306 			env->jmps_processed++;
11307 			if (opcode == BPF_CALL) {
11308 				if (BPF_SRC(insn->code) != BPF_K ||
11309 				    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL
11310 				     && insn->off != 0) ||
11311 				    (insn->src_reg != BPF_REG_0 &&
11312 				     insn->src_reg != BPF_PSEUDO_CALL &&
11313 				     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
11314 				    insn->dst_reg != BPF_REG_0 ||
11315 				    class == BPF_JMP32) {
11316 					verbose(env, "BPF_CALL uses reserved fields\n");
11317 					return -EINVAL;
11318 				}
11319 
11320 				if (env->cur_state->active_spin_lock &&
11321 				    (insn->src_reg == BPF_PSEUDO_CALL ||
11322 				     insn->imm != BPF_FUNC_spin_unlock)) {
11323 					verbose(env, "function calls are not allowed while holding a lock\n");
11324 					return -EINVAL;
11325 				}
11326 				if (insn->src_reg == BPF_PSEUDO_CALL)
11327 					err = check_func_call(env, insn, &env->insn_idx);
11328 				else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
11329 					err = check_kfunc_call(env, insn);
11330 				else
11331 					err = check_helper_call(env, insn, &env->insn_idx);
11332 				if (err)
11333 					return err;
11334 			} else if (opcode == BPF_JA) {
11335 				if (BPF_SRC(insn->code) != BPF_K ||
11336 				    insn->imm != 0 ||
11337 				    insn->src_reg != BPF_REG_0 ||
11338 				    insn->dst_reg != BPF_REG_0 ||
11339 				    class == BPF_JMP32) {
11340 					verbose(env, "BPF_JA uses reserved fields\n");
11341 					return -EINVAL;
11342 				}
11343 
11344 				env->insn_idx += insn->off + 1;
11345 				continue;
11346 
11347 			} else if (opcode == BPF_EXIT) {
11348 				if (BPF_SRC(insn->code) != BPF_K ||
11349 				    insn->imm != 0 ||
11350 				    insn->src_reg != BPF_REG_0 ||
11351 				    insn->dst_reg != BPF_REG_0 ||
11352 				    class == BPF_JMP32) {
11353 					verbose(env, "BPF_EXIT uses reserved fields\n");
11354 					return -EINVAL;
11355 				}
11356 
11357 				if (env->cur_state->active_spin_lock) {
11358 					verbose(env, "bpf_spin_unlock is missing\n");
11359 					return -EINVAL;
11360 				}
11361 
11362 				if (state->curframe) {
11363 					/* exit from nested function */
11364 					err = prepare_func_exit(env, &env->insn_idx);
11365 					if (err)
11366 						return err;
11367 					do_print_state = true;
11368 					continue;
11369 				}
11370 
11371 				err = check_reference_leak(env);
11372 				if (err)
11373 					return err;
11374 
11375 				err = check_return_code(env);
11376 				if (err)
11377 					return err;
11378 process_bpf_exit:
11379 				update_branch_counts(env, env->cur_state);
11380 				err = pop_stack(env, &prev_insn_idx,
11381 						&env->insn_idx, pop_log);
11382 				if (err < 0) {
11383 					if (err != -ENOENT)
11384 						return err;
11385 					break;
11386 				} else {
11387 					do_print_state = true;
11388 					continue;
11389 				}
11390 			} else {
11391 				err = check_cond_jmp_op(env, insn, &env->insn_idx);
11392 				if (err)
11393 					return err;
11394 			}
11395 		} else if (class == BPF_LD) {
11396 			u8 mode = BPF_MODE(insn->code);
11397 
11398 			if (mode == BPF_ABS || mode == BPF_IND) {
11399 				err = check_ld_abs(env, insn);
11400 				if (err)
11401 					return err;
11402 
11403 			} else if (mode == BPF_IMM) {
11404 				err = check_ld_imm(env, insn);
11405 				if (err)
11406 					return err;
11407 
11408 				env->insn_idx++;
11409 				sanitize_mark_insn_seen(env);
11410 			} else {
11411 				verbose(env, "invalid BPF_LD mode\n");
11412 				return -EINVAL;
11413 			}
11414 		} else {
11415 			verbose(env, "unknown insn class %d\n", class);
11416 			return -EINVAL;
11417 		}
11418 
11419 		env->insn_idx++;
11420 	}
11421 
11422 	return 0;
11423 }
11424 
11425 static int find_btf_percpu_datasec(struct btf *btf)
11426 {
11427 	const struct btf_type *t;
11428 	const char *tname;
11429 	int i, n;
11430 
11431 	/*
11432 	 * Both vmlinux and module each have their own ".data..percpu"
11433 	 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
11434 	 * types to look at only module's own BTF types.
11435 	 */
11436 	n = btf_nr_types(btf);
11437 	if (btf_is_module(btf))
11438 		i = btf_nr_types(btf_vmlinux);
11439 	else
11440 		i = 1;
11441 
11442 	for(; i < n; i++) {
11443 		t = btf_type_by_id(btf, i);
11444 		if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
11445 			continue;
11446 
11447 		tname = btf_name_by_offset(btf, t->name_off);
11448 		if (!strcmp(tname, ".data..percpu"))
11449 			return i;
11450 	}
11451 
11452 	return -ENOENT;
11453 }
11454 
11455 /* replace pseudo btf_id with kernel symbol address */
11456 static int check_pseudo_btf_id(struct bpf_verifier_env *env,
11457 			       struct bpf_insn *insn,
11458 			       struct bpf_insn_aux_data *aux)
11459 {
11460 	const struct btf_var_secinfo *vsi;
11461 	const struct btf_type *datasec;
11462 	struct btf_mod_pair *btf_mod;
11463 	const struct btf_type *t;
11464 	const char *sym_name;
11465 	bool percpu = false;
11466 	u32 type, id = insn->imm;
11467 	struct btf *btf;
11468 	s32 datasec_id;
11469 	u64 addr;
11470 	int i, btf_fd, err;
11471 
11472 	btf_fd = insn[1].imm;
11473 	if (btf_fd) {
11474 		btf = btf_get_by_fd(btf_fd);
11475 		if (IS_ERR(btf)) {
11476 			verbose(env, "invalid module BTF object FD specified.\n");
11477 			return -EINVAL;
11478 		}
11479 	} else {
11480 		if (!btf_vmlinux) {
11481 			verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
11482 			return -EINVAL;
11483 		}
11484 		btf = btf_vmlinux;
11485 		btf_get(btf);
11486 	}
11487 
11488 	t = btf_type_by_id(btf, id);
11489 	if (!t) {
11490 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
11491 		err = -ENOENT;
11492 		goto err_put;
11493 	}
11494 
11495 	if (!btf_type_is_var(t)) {
11496 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id);
11497 		err = -EINVAL;
11498 		goto err_put;
11499 	}
11500 
11501 	sym_name = btf_name_by_offset(btf, t->name_off);
11502 	addr = kallsyms_lookup_name(sym_name);
11503 	if (!addr) {
11504 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
11505 			sym_name);
11506 		err = -ENOENT;
11507 		goto err_put;
11508 	}
11509 
11510 	datasec_id = find_btf_percpu_datasec(btf);
11511 	if (datasec_id > 0) {
11512 		datasec = btf_type_by_id(btf, datasec_id);
11513 		for_each_vsi(i, datasec, vsi) {
11514 			if (vsi->type == id) {
11515 				percpu = true;
11516 				break;
11517 			}
11518 		}
11519 	}
11520 
11521 	insn[0].imm = (u32)addr;
11522 	insn[1].imm = addr >> 32;
11523 
11524 	type = t->type;
11525 	t = btf_type_skip_modifiers(btf, type, NULL);
11526 	if (percpu) {
11527 		aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
11528 		aux->btf_var.btf = btf;
11529 		aux->btf_var.btf_id = type;
11530 	} else if (!btf_type_is_struct(t)) {
11531 		const struct btf_type *ret;
11532 		const char *tname;
11533 		u32 tsize;
11534 
11535 		/* resolve the type size of ksym. */
11536 		ret = btf_resolve_size(btf, t, &tsize);
11537 		if (IS_ERR(ret)) {
11538 			tname = btf_name_by_offset(btf, t->name_off);
11539 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
11540 				tname, PTR_ERR(ret));
11541 			err = -EINVAL;
11542 			goto err_put;
11543 		}
11544 		aux->btf_var.reg_type = PTR_TO_MEM;
11545 		aux->btf_var.mem_size = tsize;
11546 	} else {
11547 		aux->btf_var.reg_type = PTR_TO_BTF_ID;
11548 		aux->btf_var.btf = btf;
11549 		aux->btf_var.btf_id = type;
11550 	}
11551 
11552 	/* check whether we recorded this BTF (and maybe module) already */
11553 	for (i = 0; i < env->used_btf_cnt; i++) {
11554 		if (env->used_btfs[i].btf == btf) {
11555 			btf_put(btf);
11556 			return 0;
11557 		}
11558 	}
11559 
11560 	if (env->used_btf_cnt >= MAX_USED_BTFS) {
11561 		err = -E2BIG;
11562 		goto err_put;
11563 	}
11564 
11565 	btf_mod = &env->used_btfs[env->used_btf_cnt];
11566 	btf_mod->btf = btf;
11567 	btf_mod->module = NULL;
11568 
11569 	/* if we reference variables from kernel module, bump its refcount */
11570 	if (btf_is_module(btf)) {
11571 		btf_mod->module = btf_try_get_module(btf);
11572 		if (!btf_mod->module) {
11573 			err = -ENXIO;
11574 			goto err_put;
11575 		}
11576 	}
11577 
11578 	env->used_btf_cnt++;
11579 
11580 	return 0;
11581 err_put:
11582 	btf_put(btf);
11583 	return err;
11584 }
11585 
11586 static int check_map_prealloc(struct bpf_map *map)
11587 {
11588 	return (map->map_type != BPF_MAP_TYPE_HASH &&
11589 		map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
11590 		map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
11591 		!(map->map_flags & BPF_F_NO_PREALLOC);
11592 }
11593 
11594 static bool is_tracing_prog_type(enum bpf_prog_type type)
11595 {
11596 	switch (type) {
11597 	case BPF_PROG_TYPE_KPROBE:
11598 	case BPF_PROG_TYPE_TRACEPOINT:
11599 	case BPF_PROG_TYPE_PERF_EVENT:
11600 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
11601 		return true;
11602 	default:
11603 		return false;
11604 	}
11605 }
11606 
11607 static bool is_preallocated_map(struct bpf_map *map)
11608 {
11609 	if (!check_map_prealloc(map))
11610 		return false;
11611 	if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta))
11612 		return false;
11613 	return true;
11614 }
11615 
11616 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
11617 					struct bpf_map *map,
11618 					struct bpf_prog *prog)
11619 
11620 {
11621 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
11622 	/*
11623 	 * Validate that trace type programs use preallocated hash maps.
11624 	 *
11625 	 * For programs attached to PERF events this is mandatory as the
11626 	 * perf NMI can hit any arbitrary code sequence.
11627 	 *
11628 	 * All other trace types using preallocated hash maps are unsafe as
11629 	 * well because tracepoint or kprobes can be inside locked regions
11630 	 * of the memory allocator or at a place where a recursion into the
11631 	 * memory allocator would see inconsistent state.
11632 	 *
11633 	 * On RT enabled kernels run-time allocation of all trace type
11634 	 * programs is strictly prohibited due to lock type constraints. On
11635 	 * !RT kernels it is allowed for backwards compatibility reasons for
11636 	 * now, but warnings are emitted so developers are made aware of
11637 	 * the unsafety and can fix their programs before this is enforced.
11638 	 */
11639 	if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
11640 		if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
11641 			verbose(env, "perf_event programs can only use preallocated hash map\n");
11642 			return -EINVAL;
11643 		}
11644 		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
11645 			verbose(env, "trace type programs can only use preallocated hash map\n");
11646 			return -EINVAL;
11647 		}
11648 		WARN_ONCE(1, "trace type BPF program uses run-time allocation\n");
11649 		verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
11650 	}
11651 
11652 	if (map_value_has_spin_lock(map)) {
11653 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
11654 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
11655 			return -EINVAL;
11656 		}
11657 
11658 		if (is_tracing_prog_type(prog_type)) {
11659 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
11660 			return -EINVAL;
11661 		}
11662 
11663 		if (prog->aux->sleepable) {
11664 			verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n");
11665 			return -EINVAL;
11666 		}
11667 	}
11668 
11669 	if (map_value_has_timer(map)) {
11670 		if (is_tracing_prog_type(prog_type)) {
11671 			verbose(env, "tracing progs cannot use bpf_timer yet\n");
11672 			return -EINVAL;
11673 		}
11674 	}
11675 
11676 	if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
11677 	    !bpf_offload_prog_map_match(prog, map)) {
11678 		verbose(env, "offload device mismatch between prog and map\n");
11679 		return -EINVAL;
11680 	}
11681 
11682 	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
11683 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");
11684 		return -EINVAL;
11685 	}
11686 
11687 	if (prog->aux->sleepable)
11688 		switch (map->map_type) {
11689 		case BPF_MAP_TYPE_HASH:
11690 		case BPF_MAP_TYPE_LRU_HASH:
11691 		case BPF_MAP_TYPE_ARRAY:
11692 		case BPF_MAP_TYPE_PERCPU_HASH:
11693 		case BPF_MAP_TYPE_PERCPU_ARRAY:
11694 		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
11695 		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
11696 		case BPF_MAP_TYPE_HASH_OF_MAPS:
11697 			if (!is_preallocated_map(map)) {
11698 				verbose(env,
11699 					"Sleepable programs can only use preallocated maps\n");
11700 				return -EINVAL;
11701 			}
11702 			break;
11703 		case BPF_MAP_TYPE_RINGBUF:
11704 			break;
11705 		default:
11706 			verbose(env,
11707 				"Sleepable programs can only use array, hash, and ringbuf maps\n");
11708 			return -EINVAL;
11709 		}
11710 
11711 	return 0;
11712 }
11713 
11714 static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
11715 {
11716 	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
11717 		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
11718 }
11719 
11720 /* find and rewrite pseudo imm in ld_imm64 instructions:
11721  *
11722  * 1. if it accesses map FD, replace it with actual map pointer.
11723  * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
11724  *
11725  * NOTE: btf_vmlinux is required for converting pseudo btf_id.
11726  */
11727 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
11728 {
11729 	struct bpf_insn *insn = env->prog->insnsi;
11730 	int insn_cnt = env->prog->len;
11731 	int i, j, err;
11732 
11733 	err = bpf_prog_calc_tag(env->prog);
11734 	if (err)
11735 		return err;
11736 
11737 	for (i = 0; i < insn_cnt; i++, insn++) {
11738 		if (BPF_CLASS(insn->code) == BPF_LDX &&
11739 		    (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
11740 			verbose(env, "BPF_LDX uses reserved fields\n");
11741 			return -EINVAL;
11742 		}
11743 
11744 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
11745 			struct bpf_insn_aux_data *aux;
11746 			struct bpf_map *map;
11747 			struct fd f;
11748 			u64 addr;
11749 			u32 fd;
11750 
11751 			if (i == insn_cnt - 1 || insn[1].code != 0 ||
11752 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
11753 			    insn[1].off != 0) {
11754 				verbose(env, "invalid bpf_ld_imm64 insn\n");
11755 				return -EINVAL;
11756 			}
11757 
11758 			if (insn[0].src_reg == 0)
11759 				/* valid generic load 64-bit imm */
11760 				goto next_insn;
11761 
11762 			if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
11763 				aux = &env->insn_aux_data[i];
11764 				err = check_pseudo_btf_id(env, insn, aux);
11765 				if (err)
11766 					return err;
11767 				goto next_insn;
11768 			}
11769 
11770 			if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
11771 				aux = &env->insn_aux_data[i];
11772 				aux->ptr_type = PTR_TO_FUNC;
11773 				goto next_insn;
11774 			}
11775 
11776 			/* In final convert_pseudo_ld_imm64() step, this is
11777 			 * converted into regular 64-bit imm load insn.
11778 			 */
11779 			switch (insn[0].src_reg) {
11780 			case BPF_PSEUDO_MAP_VALUE:
11781 			case BPF_PSEUDO_MAP_IDX_VALUE:
11782 				break;
11783 			case BPF_PSEUDO_MAP_FD:
11784 			case BPF_PSEUDO_MAP_IDX:
11785 				if (insn[1].imm == 0)
11786 					break;
11787 				fallthrough;
11788 			default:
11789 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
11790 				return -EINVAL;
11791 			}
11792 
11793 			switch (insn[0].src_reg) {
11794 			case BPF_PSEUDO_MAP_IDX_VALUE:
11795 			case BPF_PSEUDO_MAP_IDX:
11796 				if (bpfptr_is_null(env->fd_array)) {
11797 					verbose(env, "fd_idx without fd_array is invalid\n");
11798 					return -EPROTO;
11799 				}
11800 				if (copy_from_bpfptr_offset(&fd, env->fd_array,
11801 							    insn[0].imm * sizeof(fd),
11802 							    sizeof(fd)))
11803 					return -EFAULT;
11804 				break;
11805 			default:
11806 				fd = insn[0].imm;
11807 				break;
11808 			}
11809 
11810 			f = fdget(fd);
11811 			map = __bpf_map_get(f);
11812 			if (IS_ERR(map)) {
11813 				verbose(env, "fd %d is not pointing to valid bpf_map\n",
11814 					insn[0].imm);
11815 				return PTR_ERR(map);
11816 			}
11817 
11818 			err = check_map_prog_compatibility(env, map, env->prog);
11819 			if (err) {
11820 				fdput(f);
11821 				return err;
11822 			}
11823 
11824 			aux = &env->insn_aux_data[i];
11825 			if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
11826 			    insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
11827 				addr = (unsigned long)map;
11828 			} else {
11829 				u32 off = insn[1].imm;
11830 
11831 				if (off >= BPF_MAX_VAR_OFF) {
11832 					verbose(env, "direct value offset of %u is not allowed\n", off);
11833 					fdput(f);
11834 					return -EINVAL;
11835 				}
11836 
11837 				if (!map->ops->map_direct_value_addr) {
11838 					verbose(env, "no direct value access support for this map type\n");
11839 					fdput(f);
11840 					return -EINVAL;
11841 				}
11842 
11843 				err = map->ops->map_direct_value_addr(map, &addr, off);
11844 				if (err) {
11845 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
11846 						map->value_size, off);
11847 					fdput(f);
11848 					return err;
11849 				}
11850 
11851 				aux->map_off = off;
11852 				addr += off;
11853 			}
11854 
11855 			insn[0].imm = (u32)addr;
11856 			insn[1].imm = addr >> 32;
11857 
11858 			/* check whether we recorded this map already */
11859 			for (j = 0; j < env->used_map_cnt; j++) {
11860 				if (env->used_maps[j] == map) {
11861 					aux->map_index = j;
11862 					fdput(f);
11863 					goto next_insn;
11864 				}
11865 			}
11866 
11867 			if (env->used_map_cnt >= MAX_USED_MAPS) {
11868 				fdput(f);
11869 				return -E2BIG;
11870 			}
11871 
11872 			/* hold the map. If the program is rejected by verifier,
11873 			 * the map will be released by release_maps() or it
11874 			 * will be used by the valid program until it's unloaded
11875 			 * and all maps are released in free_used_maps()
11876 			 */
11877 			bpf_map_inc(map);
11878 
11879 			aux->map_index = env->used_map_cnt;
11880 			env->used_maps[env->used_map_cnt++] = map;
11881 
11882 			if (bpf_map_is_cgroup_storage(map) &&
11883 			    bpf_cgroup_storage_assign(env->prog->aux, map)) {
11884 				verbose(env, "only one cgroup storage of each type is allowed\n");
11885 				fdput(f);
11886 				return -EBUSY;
11887 			}
11888 
11889 			fdput(f);
11890 next_insn:
11891 			insn++;
11892 			i++;
11893 			continue;
11894 		}
11895 
11896 		/* Basic sanity check before we invest more work here. */
11897 		if (!bpf_opcode_in_insntable(insn->code)) {
11898 			verbose(env, "unknown opcode %02x\n", insn->code);
11899 			return -EINVAL;
11900 		}
11901 	}
11902 
11903 	/* now all pseudo BPF_LD_IMM64 instructions load valid
11904 	 * 'struct bpf_map *' into a register instead of user map_fd.
11905 	 * These pointers will be used later by verifier to validate map access.
11906 	 */
11907 	return 0;
11908 }
11909 
11910 /* drop refcnt of maps used by the rejected program */
11911 static void release_maps(struct bpf_verifier_env *env)
11912 {
11913 	__bpf_free_used_maps(env->prog->aux, env->used_maps,
11914 			     env->used_map_cnt);
11915 }
11916 
11917 /* drop refcnt of maps used by the rejected program */
11918 static void release_btfs(struct bpf_verifier_env *env)
11919 {
11920 	__bpf_free_used_btfs(env->prog->aux, env->used_btfs,
11921 			     env->used_btf_cnt);
11922 }
11923 
11924 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
11925 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
11926 {
11927 	struct bpf_insn *insn = env->prog->insnsi;
11928 	int insn_cnt = env->prog->len;
11929 	int i;
11930 
11931 	for (i = 0; i < insn_cnt; i++, insn++) {
11932 		if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
11933 			continue;
11934 		if (insn->src_reg == BPF_PSEUDO_FUNC)
11935 			continue;
11936 		insn->src_reg = 0;
11937 	}
11938 }
11939 
11940 /* single env->prog->insni[off] instruction was replaced with the range
11941  * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
11942  * [0, off) and [off, end) to new locations, so the patched range stays zero
11943  */
11944 static void adjust_insn_aux_data(struct bpf_verifier_env *env,
11945 				 struct bpf_insn_aux_data *new_data,
11946 				 struct bpf_prog *new_prog, u32 off, u32 cnt)
11947 {
11948 	struct bpf_insn_aux_data *old_data = env->insn_aux_data;
11949 	struct bpf_insn *insn = new_prog->insnsi;
11950 	u32 old_seen = old_data[off].seen;
11951 	u32 prog_len;
11952 	int i;
11953 
11954 	/* aux info at OFF always needs adjustment, no matter fast path
11955 	 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
11956 	 * original insn at old prog.
11957 	 */
11958 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
11959 
11960 	if (cnt == 1)
11961 		return;
11962 	prog_len = new_prog->len;
11963 
11964 	memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
11965 	memcpy(new_data + off + cnt - 1, old_data + off,
11966 	       sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
11967 	for (i = off; i < off + cnt - 1; i++) {
11968 		/* Expand insni[off]'s seen count to the patched range. */
11969 		new_data[i].seen = old_seen;
11970 		new_data[i].zext_dst = insn_has_def32(env, insn + i);
11971 	}
11972 	env->insn_aux_data = new_data;
11973 	vfree(old_data);
11974 }
11975 
11976 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
11977 {
11978 	int i;
11979 
11980 	if (len == 1)
11981 		return;
11982 	/* NOTE: fake 'exit' subprog should be updated as well. */
11983 	for (i = 0; i <= env->subprog_cnt; i++) {
11984 		if (env->subprog_info[i].start <= off)
11985 			continue;
11986 		env->subprog_info[i].start += len - 1;
11987 	}
11988 }
11989 
11990 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
11991 {
11992 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
11993 	int i, sz = prog->aux->size_poke_tab;
11994 	struct bpf_jit_poke_descriptor *desc;
11995 
11996 	for (i = 0; i < sz; i++) {
11997 		desc = &tab[i];
11998 		if (desc->insn_idx <= off)
11999 			continue;
12000 		desc->insn_idx += len - 1;
12001 	}
12002 }
12003 
12004 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
12005 					    const struct bpf_insn *patch, u32 len)
12006 {
12007 	struct bpf_prog *new_prog;
12008 	struct bpf_insn_aux_data *new_data = NULL;
12009 
12010 	if (len > 1) {
12011 		new_data = vzalloc(array_size(env->prog->len + len - 1,
12012 					      sizeof(struct bpf_insn_aux_data)));
12013 		if (!new_data)
12014 			return NULL;
12015 	}
12016 
12017 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
12018 	if (IS_ERR(new_prog)) {
12019 		if (PTR_ERR(new_prog) == -ERANGE)
12020 			verbose(env,
12021 				"insn %d cannot be patched due to 16-bit range\n",
12022 				env->insn_aux_data[off].orig_idx);
12023 		vfree(new_data);
12024 		return NULL;
12025 	}
12026 	adjust_insn_aux_data(env, new_data, new_prog, off, len);
12027 	adjust_subprog_starts(env, off, len);
12028 	adjust_poke_descs(new_prog, off, len);
12029 	return new_prog;
12030 }
12031 
12032 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
12033 					      u32 off, u32 cnt)
12034 {
12035 	int i, j;
12036 
12037 	/* find first prog starting at or after off (first to remove) */
12038 	for (i = 0; i < env->subprog_cnt; i++)
12039 		if (env->subprog_info[i].start >= off)
12040 			break;
12041 	/* find first prog starting at or after off + cnt (first to stay) */
12042 	for (j = i; j < env->subprog_cnt; j++)
12043 		if (env->subprog_info[j].start >= off + cnt)
12044 			break;
12045 	/* if j doesn't start exactly at off + cnt, we are just removing
12046 	 * the front of previous prog
12047 	 */
12048 	if (env->subprog_info[j].start != off + cnt)
12049 		j--;
12050 
12051 	if (j > i) {
12052 		struct bpf_prog_aux *aux = env->prog->aux;
12053 		int move;
12054 
12055 		/* move fake 'exit' subprog as well */
12056 		move = env->subprog_cnt + 1 - j;
12057 
12058 		memmove(env->subprog_info + i,
12059 			env->subprog_info + j,
12060 			sizeof(*env->subprog_info) * move);
12061 		env->subprog_cnt -= j - i;
12062 
12063 		/* remove func_info */
12064 		if (aux->func_info) {
12065 			move = aux->func_info_cnt - j;
12066 
12067 			memmove(aux->func_info + i,
12068 				aux->func_info + j,
12069 				sizeof(*aux->func_info) * move);
12070 			aux->func_info_cnt -= j - i;
12071 			/* func_info->insn_off is set after all code rewrites,
12072 			 * in adjust_btf_func() - no need to adjust
12073 			 */
12074 		}
12075 	} else {
12076 		/* convert i from "first prog to remove" to "first to adjust" */
12077 		if (env->subprog_info[i].start == off)
12078 			i++;
12079 	}
12080 
12081 	/* update fake 'exit' subprog as well */
12082 	for (; i <= env->subprog_cnt; i++)
12083 		env->subprog_info[i].start -= cnt;
12084 
12085 	return 0;
12086 }
12087 
12088 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
12089 				      u32 cnt)
12090 {
12091 	struct bpf_prog *prog = env->prog;
12092 	u32 i, l_off, l_cnt, nr_linfo;
12093 	struct bpf_line_info *linfo;
12094 
12095 	nr_linfo = prog->aux->nr_linfo;
12096 	if (!nr_linfo)
12097 		return 0;
12098 
12099 	linfo = prog->aux->linfo;
12100 
12101 	/* find first line info to remove, count lines to be removed */
12102 	for (i = 0; i < nr_linfo; i++)
12103 		if (linfo[i].insn_off >= off)
12104 			break;
12105 
12106 	l_off = i;
12107 	l_cnt = 0;
12108 	for (; i < nr_linfo; i++)
12109 		if (linfo[i].insn_off < off + cnt)
12110 			l_cnt++;
12111 		else
12112 			break;
12113 
12114 	/* First live insn doesn't match first live linfo, it needs to "inherit"
12115 	 * last removed linfo.  prog is already modified, so prog->len == off
12116 	 * means no live instructions after (tail of the program was removed).
12117 	 */
12118 	if (prog->len != off && l_cnt &&
12119 	    (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
12120 		l_cnt--;
12121 		linfo[--i].insn_off = off + cnt;
12122 	}
12123 
12124 	/* remove the line info which refer to the removed instructions */
12125 	if (l_cnt) {
12126 		memmove(linfo + l_off, linfo + i,
12127 			sizeof(*linfo) * (nr_linfo - i));
12128 
12129 		prog->aux->nr_linfo -= l_cnt;
12130 		nr_linfo = prog->aux->nr_linfo;
12131 	}
12132 
12133 	/* pull all linfo[i].insn_off >= off + cnt in by cnt */
12134 	for (i = l_off; i < nr_linfo; i++)
12135 		linfo[i].insn_off -= cnt;
12136 
12137 	/* fix up all subprogs (incl. 'exit') which start >= off */
12138 	for (i = 0; i <= env->subprog_cnt; i++)
12139 		if (env->subprog_info[i].linfo_idx > l_off) {
12140 			/* program may have started in the removed region but
12141 			 * may not be fully removed
12142 			 */
12143 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
12144 				env->subprog_info[i].linfo_idx -= l_cnt;
12145 			else
12146 				env->subprog_info[i].linfo_idx = l_off;
12147 		}
12148 
12149 	return 0;
12150 }
12151 
12152 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
12153 {
12154 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
12155 	unsigned int orig_prog_len = env->prog->len;
12156 	int err;
12157 
12158 	if (bpf_prog_is_dev_bound(env->prog->aux))
12159 		bpf_prog_offload_remove_insns(env, off, cnt);
12160 
12161 	err = bpf_remove_insns(env->prog, off, cnt);
12162 	if (err)
12163 		return err;
12164 
12165 	err = adjust_subprog_starts_after_remove(env, off, cnt);
12166 	if (err)
12167 		return err;
12168 
12169 	err = bpf_adj_linfo_after_remove(env, off, cnt);
12170 	if (err)
12171 		return err;
12172 
12173 	memmove(aux_data + off,	aux_data + off + cnt,
12174 		sizeof(*aux_data) * (orig_prog_len - off - cnt));
12175 
12176 	return 0;
12177 }
12178 
12179 /* The verifier does more data flow analysis than llvm and will not
12180  * explore branches that are dead at run time. Malicious programs can
12181  * have dead code too. Therefore replace all dead at-run-time code
12182  * with 'ja -1'.
12183  *
12184  * Just nops are not optimal, e.g. if they would sit at the end of the
12185  * program and through another bug we would manage to jump there, then
12186  * we'd execute beyond program memory otherwise. Returning exception
12187  * code also wouldn't work since we can have subprogs where the dead
12188  * code could be located.
12189  */
12190 static void sanitize_dead_code(struct bpf_verifier_env *env)
12191 {
12192 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
12193 	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
12194 	struct bpf_insn *insn = env->prog->insnsi;
12195 	const int insn_cnt = env->prog->len;
12196 	int i;
12197 
12198 	for (i = 0; i < insn_cnt; i++) {
12199 		if (aux_data[i].seen)
12200 			continue;
12201 		memcpy(insn + i, &trap, sizeof(trap));
12202 		aux_data[i].zext_dst = false;
12203 	}
12204 }
12205 
12206 static bool insn_is_cond_jump(u8 code)
12207 {
12208 	u8 op;
12209 
12210 	if (BPF_CLASS(code) == BPF_JMP32)
12211 		return true;
12212 
12213 	if (BPF_CLASS(code) != BPF_JMP)
12214 		return false;
12215 
12216 	op = BPF_OP(code);
12217 	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
12218 }
12219 
12220 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
12221 {
12222 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
12223 	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
12224 	struct bpf_insn *insn = env->prog->insnsi;
12225 	const int insn_cnt = env->prog->len;
12226 	int i;
12227 
12228 	for (i = 0; i < insn_cnt; i++, insn++) {
12229 		if (!insn_is_cond_jump(insn->code))
12230 			continue;
12231 
12232 		if (!aux_data[i + 1].seen)
12233 			ja.off = insn->off;
12234 		else if (!aux_data[i + 1 + insn->off].seen)
12235 			ja.off = 0;
12236 		else
12237 			continue;
12238 
12239 		if (bpf_prog_is_dev_bound(env->prog->aux))
12240 			bpf_prog_offload_replace_insn(env, i, &ja);
12241 
12242 		memcpy(insn, &ja, sizeof(ja));
12243 	}
12244 }
12245 
12246 static int opt_remove_dead_code(struct bpf_verifier_env *env)
12247 {
12248 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
12249 	int insn_cnt = env->prog->len;
12250 	int i, err;
12251 
12252 	for (i = 0; i < insn_cnt; i++) {
12253 		int j;
12254 
12255 		j = 0;
12256 		while (i + j < insn_cnt && !aux_data[i + j].seen)
12257 			j++;
12258 		if (!j)
12259 			continue;
12260 
12261 		err = verifier_remove_insns(env, i, j);
12262 		if (err)
12263 			return err;
12264 		insn_cnt = env->prog->len;
12265 	}
12266 
12267 	return 0;
12268 }
12269 
12270 static int opt_remove_nops(struct bpf_verifier_env *env)
12271 {
12272 	const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
12273 	struct bpf_insn *insn = env->prog->insnsi;
12274 	int insn_cnt = env->prog->len;
12275 	int i, err;
12276 
12277 	for (i = 0; i < insn_cnt; i++) {
12278 		if (memcmp(&insn[i], &ja, sizeof(ja)))
12279 			continue;
12280 
12281 		err = verifier_remove_insns(env, i, 1);
12282 		if (err)
12283 			return err;
12284 		insn_cnt--;
12285 		i--;
12286 	}
12287 
12288 	return 0;
12289 }
12290 
12291 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
12292 					 const union bpf_attr *attr)
12293 {
12294 	struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
12295 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
12296 	int i, patch_len, delta = 0, len = env->prog->len;
12297 	struct bpf_insn *insns = env->prog->insnsi;
12298 	struct bpf_prog *new_prog;
12299 	bool rnd_hi32;
12300 
12301 	rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
12302 	zext_patch[1] = BPF_ZEXT_REG(0);
12303 	rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
12304 	rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
12305 	rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
12306 	for (i = 0; i < len; i++) {
12307 		int adj_idx = i + delta;
12308 		struct bpf_insn insn;
12309 		int load_reg;
12310 
12311 		insn = insns[adj_idx];
12312 		load_reg = insn_def_regno(&insn);
12313 		if (!aux[adj_idx].zext_dst) {
12314 			u8 code, class;
12315 			u32 imm_rnd;
12316 
12317 			if (!rnd_hi32)
12318 				continue;
12319 
12320 			code = insn.code;
12321 			class = BPF_CLASS(code);
12322 			if (load_reg == -1)
12323 				continue;
12324 
12325 			/* NOTE: arg "reg" (the fourth one) is only used for
12326 			 *       BPF_STX + SRC_OP, so it is safe to pass NULL
12327 			 *       here.
12328 			 */
12329 			if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
12330 				if (class == BPF_LD &&
12331 				    BPF_MODE(code) == BPF_IMM)
12332 					i++;
12333 				continue;
12334 			}
12335 
12336 			/* ctx load could be transformed into wider load. */
12337 			if (class == BPF_LDX &&
12338 			    aux[adj_idx].ptr_type == PTR_TO_CTX)
12339 				continue;
12340 
12341 			imm_rnd = get_random_int();
12342 			rnd_hi32_patch[0] = insn;
12343 			rnd_hi32_patch[1].imm = imm_rnd;
12344 			rnd_hi32_patch[3].dst_reg = load_reg;
12345 			patch = rnd_hi32_patch;
12346 			patch_len = 4;
12347 			goto apply_patch_buffer;
12348 		}
12349 
12350 		/* Add in an zero-extend instruction if a) the JIT has requested
12351 		 * it or b) it's a CMPXCHG.
12352 		 *
12353 		 * The latter is because: BPF_CMPXCHG always loads a value into
12354 		 * R0, therefore always zero-extends. However some archs'
12355 		 * equivalent instruction only does this load when the
12356 		 * comparison is successful. This detail of CMPXCHG is
12357 		 * orthogonal to the general zero-extension behaviour of the
12358 		 * CPU, so it's treated independently of bpf_jit_needs_zext.
12359 		 */
12360 		if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
12361 			continue;
12362 
12363 		if (WARN_ON(load_reg == -1)) {
12364 			verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
12365 			return -EFAULT;
12366 		}
12367 
12368 		zext_patch[0] = insn;
12369 		zext_patch[1].dst_reg = load_reg;
12370 		zext_patch[1].src_reg = load_reg;
12371 		patch = zext_patch;
12372 		patch_len = 2;
12373 apply_patch_buffer:
12374 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
12375 		if (!new_prog)
12376 			return -ENOMEM;
12377 		env->prog = new_prog;
12378 		insns = new_prog->insnsi;
12379 		aux = env->insn_aux_data;
12380 		delta += patch_len - 1;
12381 	}
12382 
12383 	return 0;
12384 }
12385 
12386 /* convert load instructions that access fields of a context type into a
12387  * sequence of instructions that access fields of the underlying structure:
12388  *     struct __sk_buff    -> struct sk_buff
12389  *     struct bpf_sock_ops -> struct sock
12390  */
12391 static int convert_ctx_accesses(struct bpf_verifier_env *env)
12392 {
12393 	const struct bpf_verifier_ops *ops = env->ops;
12394 	int i, cnt, size, ctx_field_size, delta = 0;
12395 	const int insn_cnt = env->prog->len;
12396 	struct bpf_insn insn_buf[16], *insn;
12397 	u32 target_size, size_default, off;
12398 	struct bpf_prog *new_prog;
12399 	enum bpf_access_type type;
12400 	bool is_narrower_load;
12401 
12402 	if (ops->gen_prologue || env->seen_direct_write) {
12403 		if (!ops->gen_prologue) {
12404 			verbose(env, "bpf verifier is misconfigured\n");
12405 			return -EINVAL;
12406 		}
12407 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
12408 					env->prog);
12409 		if (cnt >= ARRAY_SIZE(insn_buf)) {
12410 			verbose(env, "bpf verifier is misconfigured\n");
12411 			return -EINVAL;
12412 		} else if (cnt) {
12413 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
12414 			if (!new_prog)
12415 				return -ENOMEM;
12416 
12417 			env->prog = new_prog;
12418 			delta += cnt - 1;
12419 		}
12420 	}
12421 
12422 	if (bpf_prog_is_dev_bound(env->prog->aux))
12423 		return 0;
12424 
12425 	insn = env->prog->insnsi + delta;
12426 
12427 	for (i = 0; i < insn_cnt; i++, insn++) {
12428 		bpf_convert_ctx_access_t convert_ctx_access;
12429 		bool ctx_access;
12430 
12431 		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
12432 		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
12433 		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
12434 		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) {
12435 			type = BPF_READ;
12436 			ctx_access = true;
12437 		} else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
12438 			   insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
12439 			   insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
12440 			   insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
12441 			   insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
12442 			   insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
12443 			   insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
12444 			   insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
12445 			type = BPF_WRITE;
12446 			ctx_access = BPF_CLASS(insn->code) == BPF_STX;
12447 		} else {
12448 			continue;
12449 		}
12450 
12451 		if (type == BPF_WRITE &&
12452 		    env->insn_aux_data[i + delta].sanitize_stack_spill) {
12453 			struct bpf_insn patch[] = {
12454 				*insn,
12455 				BPF_ST_NOSPEC(),
12456 			};
12457 
12458 			cnt = ARRAY_SIZE(patch);
12459 			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
12460 			if (!new_prog)
12461 				return -ENOMEM;
12462 
12463 			delta    += cnt - 1;
12464 			env->prog = new_prog;
12465 			insn      = new_prog->insnsi + i + delta;
12466 			continue;
12467 		}
12468 
12469 		if (!ctx_access)
12470 			continue;
12471 
12472 		switch (env->insn_aux_data[i + delta].ptr_type) {
12473 		case PTR_TO_CTX:
12474 			if (!ops->convert_ctx_access)
12475 				continue;
12476 			convert_ctx_access = ops->convert_ctx_access;
12477 			break;
12478 		case PTR_TO_SOCKET:
12479 		case PTR_TO_SOCK_COMMON:
12480 			convert_ctx_access = bpf_sock_convert_ctx_access;
12481 			break;
12482 		case PTR_TO_TCP_SOCK:
12483 			convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
12484 			break;
12485 		case PTR_TO_XDP_SOCK:
12486 			convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
12487 			break;
12488 		case PTR_TO_BTF_ID:
12489 			if (type == BPF_READ) {
12490 				insn->code = BPF_LDX | BPF_PROBE_MEM |
12491 					BPF_SIZE((insn)->code);
12492 				env->prog->aux->num_exentries++;
12493 			} else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
12494 				verbose(env, "Writes through BTF pointers are not allowed\n");
12495 				return -EINVAL;
12496 			}
12497 			continue;
12498 		default:
12499 			continue;
12500 		}
12501 
12502 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
12503 		size = BPF_LDST_BYTES(insn);
12504 
12505 		/* If the read access is a narrower load of the field,
12506 		 * convert to a 4/8-byte load, to minimum program type specific
12507 		 * convert_ctx_access changes. If conversion is successful,
12508 		 * we will apply proper mask to the result.
12509 		 */
12510 		is_narrower_load = size < ctx_field_size;
12511 		size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
12512 		off = insn->off;
12513 		if (is_narrower_load) {
12514 			u8 size_code;
12515 
12516 			if (type == BPF_WRITE) {
12517 				verbose(env, "bpf verifier narrow ctx access misconfigured\n");
12518 				return -EINVAL;
12519 			}
12520 
12521 			size_code = BPF_H;
12522 			if (ctx_field_size == 4)
12523 				size_code = BPF_W;
12524 			else if (ctx_field_size == 8)
12525 				size_code = BPF_DW;
12526 
12527 			insn->off = off & ~(size_default - 1);
12528 			insn->code = BPF_LDX | BPF_MEM | size_code;
12529 		}
12530 
12531 		target_size = 0;
12532 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
12533 					 &target_size);
12534 		if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
12535 		    (ctx_field_size && !target_size)) {
12536 			verbose(env, "bpf verifier is misconfigured\n");
12537 			return -EINVAL;
12538 		}
12539 
12540 		if (is_narrower_load && size < target_size) {
12541 			u8 shift = bpf_ctx_narrow_access_offset(
12542 				off, size, size_default) * 8;
12543 			if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) {
12544 				verbose(env, "bpf verifier narrow ctx load misconfigured\n");
12545 				return -EINVAL;
12546 			}
12547 			if (ctx_field_size <= 4) {
12548 				if (shift)
12549 					insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
12550 									insn->dst_reg,
12551 									shift);
12552 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
12553 								(1 << size * 8) - 1);
12554 			} else {
12555 				if (shift)
12556 					insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
12557 									insn->dst_reg,
12558 									shift);
12559 				insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
12560 								(1ULL << size * 8) - 1);
12561 			}
12562 		}
12563 
12564 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12565 		if (!new_prog)
12566 			return -ENOMEM;
12567 
12568 		delta += cnt - 1;
12569 
12570 		/* keep walking new program and skip insns we just inserted */
12571 		env->prog = new_prog;
12572 		insn      = new_prog->insnsi + i + delta;
12573 	}
12574 
12575 	return 0;
12576 }
12577 
12578 static int jit_subprogs(struct bpf_verifier_env *env)
12579 {
12580 	struct bpf_prog *prog = env->prog, **func, *tmp;
12581 	int i, j, subprog_start, subprog_end = 0, len, subprog;
12582 	struct bpf_map *map_ptr;
12583 	struct bpf_insn *insn;
12584 	void *old_bpf_func;
12585 	int err, num_exentries;
12586 
12587 	if (env->subprog_cnt <= 1)
12588 		return 0;
12589 
12590 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12591 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
12592 			continue;
12593 
12594 		/* Upon error here we cannot fall back to interpreter but
12595 		 * need a hard reject of the program. Thus -EFAULT is
12596 		 * propagated in any case.
12597 		 */
12598 		subprog = find_subprog(env, i + insn->imm + 1);
12599 		if (subprog < 0) {
12600 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
12601 				  i + insn->imm + 1);
12602 			return -EFAULT;
12603 		}
12604 		/* temporarily remember subprog id inside insn instead of
12605 		 * aux_data, since next loop will split up all insns into funcs
12606 		 */
12607 		insn->off = subprog;
12608 		/* remember original imm in case JIT fails and fallback
12609 		 * to interpreter will be needed
12610 		 */
12611 		env->insn_aux_data[i].call_imm = insn->imm;
12612 		/* point imm to __bpf_call_base+1 from JITs point of view */
12613 		insn->imm = 1;
12614 		if (bpf_pseudo_func(insn))
12615 			/* jit (e.g. x86_64) may emit fewer instructions
12616 			 * if it learns a u32 imm is the same as a u64 imm.
12617 			 * Force a non zero here.
12618 			 */
12619 			insn[1].imm = 1;
12620 	}
12621 
12622 	err = bpf_prog_alloc_jited_linfo(prog);
12623 	if (err)
12624 		goto out_undo_insn;
12625 
12626 	err = -ENOMEM;
12627 	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
12628 	if (!func)
12629 		goto out_undo_insn;
12630 
12631 	for (i = 0; i < env->subprog_cnt; i++) {
12632 		subprog_start = subprog_end;
12633 		subprog_end = env->subprog_info[i + 1].start;
12634 
12635 		len = subprog_end - subprog_start;
12636 		/* bpf_prog_run() doesn't call subprogs directly,
12637 		 * hence main prog stats include the runtime of subprogs.
12638 		 * subprogs don't have IDs and not reachable via prog_get_next_id
12639 		 * func[i]->stats will never be accessed and stays NULL
12640 		 */
12641 		func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
12642 		if (!func[i])
12643 			goto out_free;
12644 		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
12645 		       len * sizeof(struct bpf_insn));
12646 		func[i]->type = prog->type;
12647 		func[i]->len = len;
12648 		if (bpf_prog_calc_tag(func[i]))
12649 			goto out_free;
12650 		func[i]->is_func = 1;
12651 		func[i]->aux->func_idx = i;
12652 		/* Below members will be freed only at prog->aux */
12653 		func[i]->aux->btf = prog->aux->btf;
12654 		func[i]->aux->func_info = prog->aux->func_info;
12655 		func[i]->aux->poke_tab = prog->aux->poke_tab;
12656 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
12657 
12658 		for (j = 0; j < prog->aux->size_poke_tab; j++) {
12659 			struct bpf_jit_poke_descriptor *poke;
12660 
12661 			poke = &prog->aux->poke_tab[j];
12662 			if (poke->insn_idx < subprog_end &&
12663 			    poke->insn_idx >= subprog_start)
12664 				poke->aux = func[i]->aux;
12665 		}
12666 
12667 		/* Use bpf_prog_F_tag to indicate functions in stack traces.
12668 		 * Long term would need debug info to populate names
12669 		 */
12670 		func[i]->aux->name[0] = 'F';
12671 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
12672 		func[i]->jit_requested = 1;
12673 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
12674 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
12675 		func[i]->aux->linfo = prog->aux->linfo;
12676 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
12677 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
12678 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
12679 		num_exentries = 0;
12680 		insn = func[i]->insnsi;
12681 		for (j = 0; j < func[i]->len; j++, insn++) {
12682 			if (BPF_CLASS(insn->code) == BPF_LDX &&
12683 			    BPF_MODE(insn->code) == BPF_PROBE_MEM)
12684 				num_exentries++;
12685 		}
12686 		func[i]->aux->num_exentries = num_exentries;
12687 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
12688 		func[i] = bpf_int_jit_compile(func[i]);
12689 		if (!func[i]->jited) {
12690 			err = -ENOTSUPP;
12691 			goto out_free;
12692 		}
12693 		cond_resched();
12694 	}
12695 
12696 	/* at this point all bpf functions were successfully JITed
12697 	 * now populate all bpf_calls with correct addresses and
12698 	 * run last pass of JIT
12699 	 */
12700 	for (i = 0; i < env->subprog_cnt; i++) {
12701 		insn = func[i]->insnsi;
12702 		for (j = 0; j < func[i]->len; j++, insn++) {
12703 			if (bpf_pseudo_func(insn)) {
12704 				subprog = insn->off;
12705 				insn[0].imm = (u32)(long)func[subprog]->bpf_func;
12706 				insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
12707 				continue;
12708 			}
12709 			if (!bpf_pseudo_call(insn))
12710 				continue;
12711 			subprog = insn->off;
12712 			insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
12713 		}
12714 
12715 		/* we use the aux data to keep a list of the start addresses
12716 		 * of the JITed images for each function in the program
12717 		 *
12718 		 * for some architectures, such as powerpc64, the imm field
12719 		 * might not be large enough to hold the offset of the start
12720 		 * address of the callee's JITed image from __bpf_call_base
12721 		 *
12722 		 * in such cases, we can lookup the start address of a callee
12723 		 * by using its subprog id, available from the off field of
12724 		 * the call instruction, as an index for this list
12725 		 */
12726 		func[i]->aux->func = func;
12727 		func[i]->aux->func_cnt = env->subprog_cnt;
12728 	}
12729 	for (i = 0; i < env->subprog_cnt; i++) {
12730 		old_bpf_func = func[i]->bpf_func;
12731 		tmp = bpf_int_jit_compile(func[i]);
12732 		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
12733 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
12734 			err = -ENOTSUPP;
12735 			goto out_free;
12736 		}
12737 		cond_resched();
12738 	}
12739 
12740 	/* finally lock prog and jit images for all functions and
12741 	 * populate kallsysm
12742 	 */
12743 	for (i = 0; i < env->subprog_cnt; i++) {
12744 		bpf_prog_lock_ro(func[i]);
12745 		bpf_prog_kallsyms_add(func[i]);
12746 	}
12747 
12748 	/* Last step: make now unused interpreter insns from main
12749 	 * prog consistent for later dump requests, so they can
12750 	 * later look the same as if they were interpreted only.
12751 	 */
12752 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12753 		if (bpf_pseudo_func(insn)) {
12754 			insn[0].imm = env->insn_aux_data[i].call_imm;
12755 			insn[1].imm = insn->off;
12756 			insn->off = 0;
12757 			continue;
12758 		}
12759 		if (!bpf_pseudo_call(insn))
12760 			continue;
12761 		insn->off = env->insn_aux_data[i].call_imm;
12762 		subprog = find_subprog(env, i + insn->off + 1);
12763 		insn->imm = subprog;
12764 	}
12765 
12766 	prog->jited = 1;
12767 	prog->bpf_func = func[0]->bpf_func;
12768 	prog->aux->func = func;
12769 	prog->aux->func_cnt = env->subprog_cnt;
12770 	bpf_prog_jit_attempt_done(prog);
12771 	return 0;
12772 out_free:
12773 	/* We failed JIT'ing, so at this point we need to unregister poke
12774 	 * descriptors from subprogs, so that kernel is not attempting to
12775 	 * patch it anymore as we're freeing the subprog JIT memory.
12776 	 */
12777 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
12778 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
12779 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
12780 	}
12781 	/* At this point we're guaranteed that poke descriptors are not
12782 	 * live anymore. We can just unlink its descriptor table as it's
12783 	 * released with the main prog.
12784 	 */
12785 	for (i = 0; i < env->subprog_cnt; i++) {
12786 		if (!func[i])
12787 			continue;
12788 		func[i]->aux->poke_tab = NULL;
12789 		bpf_jit_free(func[i]);
12790 	}
12791 	kfree(func);
12792 out_undo_insn:
12793 	/* cleanup main prog to be interpreted */
12794 	prog->jit_requested = 0;
12795 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12796 		if (!bpf_pseudo_call(insn))
12797 			continue;
12798 		insn->off = 0;
12799 		insn->imm = env->insn_aux_data[i].call_imm;
12800 	}
12801 	bpf_prog_jit_attempt_done(prog);
12802 	return err;
12803 }
12804 
12805 static int fixup_call_args(struct bpf_verifier_env *env)
12806 {
12807 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
12808 	struct bpf_prog *prog = env->prog;
12809 	struct bpf_insn *insn = prog->insnsi;
12810 	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
12811 	int i, depth;
12812 #endif
12813 	int err = 0;
12814 
12815 	if (env->prog->jit_requested &&
12816 	    !bpf_prog_is_dev_bound(env->prog->aux)) {
12817 		err = jit_subprogs(env);
12818 		if (err == 0)
12819 			return 0;
12820 		if (err == -EFAULT)
12821 			return err;
12822 	}
12823 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
12824 	if (has_kfunc_call) {
12825 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
12826 		return -EINVAL;
12827 	}
12828 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
12829 		/* When JIT fails the progs with bpf2bpf calls and tail_calls
12830 		 * have to be rejected, since interpreter doesn't support them yet.
12831 		 */
12832 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
12833 		return -EINVAL;
12834 	}
12835 	for (i = 0; i < prog->len; i++, insn++) {
12836 		if (bpf_pseudo_func(insn)) {
12837 			/* When JIT fails the progs with callback calls
12838 			 * have to be rejected, since interpreter doesn't support them yet.
12839 			 */
12840 			verbose(env, "callbacks are not allowed in non-JITed programs\n");
12841 			return -EINVAL;
12842 		}
12843 
12844 		if (!bpf_pseudo_call(insn))
12845 			continue;
12846 		depth = get_callee_stack_depth(env, insn, i);
12847 		if (depth < 0)
12848 			return depth;
12849 		bpf_patch_call_args(insn, depth);
12850 	}
12851 	err = 0;
12852 #endif
12853 	return err;
12854 }
12855 
12856 static int fixup_kfunc_call(struct bpf_verifier_env *env,
12857 			    struct bpf_insn *insn)
12858 {
12859 	const struct bpf_kfunc_desc *desc;
12860 
12861 	if (!insn->imm) {
12862 		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
12863 		return -EINVAL;
12864 	}
12865 
12866 	/* insn->imm has the btf func_id. Replace it with
12867 	 * an address (relative to __bpf_base_call).
12868 	 */
12869 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
12870 	if (!desc) {
12871 		verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
12872 			insn->imm);
12873 		return -EFAULT;
12874 	}
12875 
12876 	insn->imm = desc->imm;
12877 
12878 	return 0;
12879 }
12880 
12881 /* Do various post-verification rewrites in a single program pass.
12882  * These rewrites simplify JIT and interpreter implementations.
12883  */
12884 static int do_misc_fixups(struct bpf_verifier_env *env)
12885 {
12886 	struct bpf_prog *prog = env->prog;
12887 	bool expect_blinding = bpf_jit_blinding_enabled(prog);
12888 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
12889 	struct bpf_insn *insn = prog->insnsi;
12890 	const struct bpf_func_proto *fn;
12891 	const int insn_cnt = prog->len;
12892 	const struct bpf_map_ops *ops;
12893 	struct bpf_insn_aux_data *aux;
12894 	struct bpf_insn insn_buf[16];
12895 	struct bpf_prog *new_prog;
12896 	struct bpf_map *map_ptr;
12897 	int i, ret, cnt, delta = 0;
12898 
12899 	for (i = 0; i < insn_cnt; i++, insn++) {
12900 		/* Make divide-by-zero exceptions impossible. */
12901 		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
12902 		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
12903 		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
12904 		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
12905 			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
12906 			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
12907 			struct bpf_insn *patchlet;
12908 			struct bpf_insn chk_and_div[] = {
12909 				/* [R,W]x div 0 -> 0 */
12910 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
12911 					     BPF_JNE | BPF_K, insn->src_reg,
12912 					     0, 2, 0),
12913 				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
12914 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
12915 				*insn,
12916 			};
12917 			struct bpf_insn chk_and_mod[] = {
12918 				/* [R,W]x mod 0 -> [R,W]x */
12919 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
12920 					     BPF_JEQ | BPF_K, insn->src_reg,
12921 					     0, 1 + (is64 ? 0 : 1), 0),
12922 				*insn,
12923 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
12924 				BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
12925 			};
12926 
12927 			patchlet = isdiv ? chk_and_div : chk_and_mod;
12928 			cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
12929 				      ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
12930 
12931 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
12932 			if (!new_prog)
12933 				return -ENOMEM;
12934 
12935 			delta    += cnt - 1;
12936 			env->prog = prog = new_prog;
12937 			insn      = new_prog->insnsi + i + delta;
12938 			continue;
12939 		}
12940 
12941 		/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
12942 		if (BPF_CLASS(insn->code) == BPF_LD &&
12943 		    (BPF_MODE(insn->code) == BPF_ABS ||
12944 		     BPF_MODE(insn->code) == BPF_IND)) {
12945 			cnt = env->ops->gen_ld_abs(insn, insn_buf);
12946 			if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
12947 				verbose(env, "bpf verifier is misconfigured\n");
12948 				return -EINVAL;
12949 			}
12950 
12951 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12952 			if (!new_prog)
12953 				return -ENOMEM;
12954 
12955 			delta    += cnt - 1;
12956 			env->prog = prog = new_prog;
12957 			insn      = new_prog->insnsi + i + delta;
12958 			continue;
12959 		}
12960 
12961 		/* Rewrite pointer arithmetic to mitigate speculation attacks. */
12962 		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
12963 		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
12964 			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
12965 			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
12966 			struct bpf_insn *patch = &insn_buf[0];
12967 			bool issrc, isneg, isimm;
12968 			u32 off_reg;
12969 
12970 			aux = &env->insn_aux_data[i + delta];
12971 			if (!aux->alu_state ||
12972 			    aux->alu_state == BPF_ALU_NON_POINTER)
12973 				continue;
12974 
12975 			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
12976 			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
12977 				BPF_ALU_SANITIZE_SRC;
12978 			isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
12979 
12980 			off_reg = issrc ? insn->src_reg : insn->dst_reg;
12981 			if (isimm) {
12982 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
12983 			} else {
12984 				if (isneg)
12985 					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
12986 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
12987 				*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
12988 				*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
12989 				*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
12990 				*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
12991 				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
12992 			}
12993 			if (!issrc)
12994 				*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
12995 			insn->src_reg = BPF_REG_AX;
12996 			if (isneg)
12997 				insn->code = insn->code == code_add ?
12998 					     code_sub : code_add;
12999 			*patch++ = *insn;
13000 			if (issrc && isneg && !isimm)
13001 				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
13002 			cnt = patch - insn_buf;
13003 
13004 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
13005 			if (!new_prog)
13006 				return -ENOMEM;
13007 
13008 			delta    += cnt - 1;
13009 			env->prog = prog = new_prog;
13010 			insn      = new_prog->insnsi + i + delta;
13011 			continue;
13012 		}
13013 
13014 		if (insn->code != (BPF_JMP | BPF_CALL))
13015 			continue;
13016 		if (insn->src_reg == BPF_PSEUDO_CALL)
13017 			continue;
13018 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
13019 			ret = fixup_kfunc_call(env, insn);
13020 			if (ret)
13021 				return ret;
13022 			continue;
13023 		}
13024 
13025 		if (insn->imm == BPF_FUNC_get_route_realm)
13026 			prog->dst_needed = 1;
13027 		if (insn->imm == BPF_FUNC_get_prandom_u32)
13028 			bpf_user_rnd_init_once();
13029 		if (insn->imm == BPF_FUNC_override_return)
13030 			prog->kprobe_override = 1;
13031 		if (insn->imm == BPF_FUNC_tail_call) {
13032 			/* If we tail call into other programs, we
13033 			 * cannot make any assumptions since they can
13034 			 * be replaced dynamically during runtime in
13035 			 * the program array.
13036 			 */
13037 			prog->cb_access = 1;
13038 			if (!allow_tail_call_in_subprogs(env))
13039 				prog->aux->stack_depth = MAX_BPF_STACK;
13040 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;
13041 
13042 			/* mark bpf_tail_call as different opcode to avoid
13043 			 * conditional branch in the interpreter for every normal
13044 			 * call and to prevent accidental JITing by JIT compiler
13045 			 * that doesn't support bpf_tail_call yet
13046 			 */
13047 			insn->imm = 0;
13048 			insn->code = BPF_JMP | BPF_TAIL_CALL;
13049 
13050 			aux = &env->insn_aux_data[i + delta];
13051 			if (env->bpf_capable && !expect_blinding &&
13052 			    prog->jit_requested &&
13053 			    !bpf_map_key_poisoned(aux) &&
13054 			    !bpf_map_ptr_poisoned(aux) &&
13055 			    !bpf_map_ptr_unpriv(aux)) {
13056 				struct bpf_jit_poke_descriptor desc = {
13057 					.reason = BPF_POKE_REASON_TAIL_CALL,
13058 					.tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
13059 					.tail_call.key = bpf_map_key_immediate(aux),
13060 					.insn_idx = i + delta,
13061 				};
13062 
13063 				ret = bpf_jit_add_poke_descriptor(prog, &desc);
13064 				if (ret < 0) {
13065 					verbose(env, "adding tail call poke descriptor failed\n");
13066 					return ret;
13067 				}
13068 
13069 				insn->imm = ret + 1;
13070 				continue;
13071 			}
13072 
13073 			if (!bpf_map_ptr_unpriv(aux))
13074 				continue;
13075 
13076 			/* instead of changing every JIT dealing with tail_call
13077 			 * emit two extra insns:
13078 			 * if (index >= max_entries) goto out;
13079 			 * index &= array->index_mask;
13080 			 * to avoid out-of-bounds cpu speculation
13081 			 */
13082 			if (bpf_map_ptr_poisoned(aux)) {
13083 				verbose(env, "tail_call abusing map_ptr\n");
13084 				return -EINVAL;
13085 			}
13086 
13087 			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
13088 			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
13089 						  map_ptr->max_entries, 2);
13090 			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
13091 						    container_of(map_ptr,
13092 								 struct bpf_array,
13093 								 map)->index_mask);
13094 			insn_buf[2] = *insn;
13095 			cnt = 3;
13096 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
13097 			if (!new_prog)
13098 				return -ENOMEM;
13099 
13100 			delta    += cnt - 1;
13101 			env->prog = prog = new_prog;
13102 			insn      = new_prog->insnsi + i + delta;
13103 			continue;
13104 		}
13105 
13106 		if (insn->imm == BPF_FUNC_timer_set_callback) {
13107 			/* The verifier will process callback_fn as many times as necessary
13108 			 * with different maps and the register states prepared by
13109 			 * set_timer_callback_state will be accurate.
13110 			 *
13111 			 * The following use case is valid:
13112 			 *   map1 is shared by prog1, prog2, prog3.
13113 			 *   prog1 calls bpf_timer_init for some map1 elements
13114 			 *   prog2 calls bpf_timer_set_callback for some map1 elements.
13115 			 *     Those that were not bpf_timer_init-ed will return -EINVAL.
13116 			 *   prog3 calls bpf_timer_start for some map1 elements.
13117 			 *     Those that were not both bpf_timer_init-ed and
13118 			 *     bpf_timer_set_callback-ed will return -EINVAL.
13119 			 */
13120 			struct bpf_insn ld_addrs[2] = {
13121 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
13122 			};
13123 
13124 			insn_buf[0] = ld_addrs[0];
13125 			insn_buf[1] = ld_addrs[1];
13126 			insn_buf[2] = *insn;
13127 			cnt = 3;
13128 
13129 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
13130 			if (!new_prog)
13131 				return -ENOMEM;
13132 
13133 			delta    += cnt - 1;
13134 			env->prog = prog = new_prog;
13135 			insn      = new_prog->insnsi + i + delta;
13136 			goto patch_call_imm;
13137 		}
13138 
13139 		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
13140 		 * and other inlining handlers are currently limited to 64 bit
13141 		 * only.
13142 		 */
13143 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
13144 		    (insn->imm == BPF_FUNC_map_lookup_elem ||
13145 		     insn->imm == BPF_FUNC_map_update_elem ||
13146 		     insn->imm == BPF_FUNC_map_delete_elem ||
13147 		     insn->imm == BPF_FUNC_map_push_elem   ||
13148 		     insn->imm == BPF_FUNC_map_pop_elem    ||
13149 		     insn->imm == BPF_FUNC_map_peek_elem   ||
13150 		     insn->imm == BPF_FUNC_redirect_map    ||
13151 		     insn->imm == BPF_FUNC_for_each_map_elem)) {
13152 			aux = &env->insn_aux_data[i + delta];
13153 			if (bpf_map_ptr_poisoned(aux))
13154 				goto patch_call_imm;
13155 
13156 			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
13157 			ops = map_ptr->ops;
13158 			if (insn->imm == BPF_FUNC_map_lookup_elem &&
13159 			    ops->map_gen_lookup) {
13160 				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
13161 				if (cnt == -EOPNOTSUPP)
13162 					goto patch_map_ops_generic;
13163 				if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
13164 					verbose(env, "bpf verifier is misconfigured\n");
13165 					return -EINVAL;
13166 				}
13167 
13168 				new_prog = bpf_patch_insn_data(env, i + delta,
13169 							       insn_buf, cnt);
13170 				if (!new_prog)
13171 					return -ENOMEM;
13172 
13173 				delta    += cnt - 1;
13174 				env->prog = prog = new_prog;
13175 				insn      = new_prog->insnsi + i + delta;
13176 				continue;
13177 			}
13178 
13179 			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
13180 				     (void *(*)(struct bpf_map *map, void *key))NULL));
13181 			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
13182 				     (int (*)(struct bpf_map *map, void *key))NULL));
13183 			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
13184 				     (int (*)(struct bpf_map *map, void *key, void *value,
13185 					      u64 flags))NULL));
13186 			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
13187 				     (int (*)(struct bpf_map *map, void *value,
13188 					      u64 flags))NULL));
13189 			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
13190 				     (int (*)(struct bpf_map *map, void *value))NULL));
13191 			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
13192 				     (int (*)(struct bpf_map *map, void *value))NULL));
13193 			BUILD_BUG_ON(!__same_type(ops->map_redirect,
13194 				     (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL));
13195 			BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
13196 				     (int (*)(struct bpf_map *map,
13197 					      bpf_callback_t callback_fn,
13198 					      void *callback_ctx,
13199 					      u64 flags))NULL));
13200 
13201 patch_map_ops_generic:
13202 			switch (insn->imm) {
13203 			case BPF_FUNC_map_lookup_elem:
13204 				insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
13205 				continue;
13206 			case BPF_FUNC_map_update_elem:
13207 				insn->imm = BPF_CALL_IMM(ops->map_update_elem);
13208 				continue;
13209 			case BPF_FUNC_map_delete_elem:
13210 				insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
13211 				continue;
13212 			case BPF_FUNC_map_push_elem:
13213 				insn->imm = BPF_CALL_IMM(ops->map_push_elem);
13214 				continue;
13215 			case BPF_FUNC_map_pop_elem:
13216 				insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
13217 				continue;
13218 			case BPF_FUNC_map_peek_elem:
13219 				insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
13220 				continue;
13221 			case BPF_FUNC_redirect_map:
13222 				insn->imm = BPF_CALL_IMM(ops->map_redirect);
13223 				continue;
13224 			case BPF_FUNC_for_each_map_elem:
13225 				insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
13226 				continue;
13227 			}
13228 
13229 			goto patch_call_imm;
13230 		}
13231 
13232 		/* Implement bpf_jiffies64 inline. */
13233 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
13234 		    insn->imm == BPF_FUNC_jiffies64) {
13235 			struct bpf_insn ld_jiffies_addr[2] = {
13236 				BPF_LD_IMM64(BPF_REG_0,
13237 					     (unsigned long)&jiffies),
13238 			};
13239 
13240 			insn_buf[0] = ld_jiffies_addr[0];
13241 			insn_buf[1] = ld_jiffies_addr[1];
13242 			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
13243 						  BPF_REG_0, 0);
13244 			cnt = 3;
13245 
13246 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
13247 						       cnt);
13248 			if (!new_prog)
13249 				return -ENOMEM;
13250 
13251 			delta    += cnt - 1;
13252 			env->prog = prog = new_prog;
13253 			insn      = new_prog->insnsi + i + delta;
13254 			continue;
13255 		}
13256 
13257 		/* Implement bpf_get_func_ip inline. */
13258 		if (prog_type == BPF_PROG_TYPE_TRACING &&
13259 		    insn->imm == BPF_FUNC_get_func_ip) {
13260 			/* Load IP address from ctx - 8 */
13261 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
13262 
13263 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
13264 			if (!new_prog)
13265 				return -ENOMEM;
13266 
13267 			env->prog = prog = new_prog;
13268 			insn      = new_prog->insnsi + i + delta;
13269 			continue;
13270 		}
13271 
13272 patch_call_imm:
13273 		fn = env->ops->get_func_proto(insn->imm, env->prog);
13274 		/* all functions that have prototype and verifier allowed
13275 		 * programs to call them, must be real in-kernel functions
13276 		 */
13277 		if (!fn->func) {
13278 			verbose(env,
13279 				"kernel subsystem misconfigured func %s#%d\n",
13280 				func_id_name(insn->imm), insn->imm);
13281 			return -EFAULT;
13282 		}
13283 		insn->imm = fn->func - __bpf_call_base;
13284 	}
13285 
13286 	/* Since poke tab is now finalized, publish aux to tracker. */
13287 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
13288 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
13289 		if (!map_ptr->ops->map_poke_track ||
13290 		    !map_ptr->ops->map_poke_untrack ||
13291 		    !map_ptr->ops->map_poke_run) {
13292 			verbose(env, "bpf verifier is misconfigured\n");
13293 			return -EINVAL;
13294 		}
13295 
13296 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
13297 		if (ret < 0) {
13298 			verbose(env, "tracking tail call prog failed\n");
13299 			return ret;
13300 		}
13301 	}
13302 
13303 	sort_kfunc_descs_by_imm(env->prog);
13304 
13305 	return 0;
13306 }
13307 
13308 static void free_states(struct bpf_verifier_env *env)
13309 {
13310 	struct bpf_verifier_state_list *sl, *sln;
13311 	int i;
13312 
13313 	sl = env->free_list;
13314 	while (sl) {
13315 		sln = sl->next;
13316 		free_verifier_state(&sl->state, false);
13317 		kfree(sl);
13318 		sl = sln;
13319 	}
13320 	env->free_list = NULL;
13321 
13322 	if (!env->explored_states)
13323 		return;
13324 
13325 	for (i = 0; i < state_htab_size(env); i++) {
13326 		sl = env->explored_states[i];
13327 
13328 		while (sl) {
13329 			sln = sl->next;
13330 			free_verifier_state(&sl->state, false);
13331 			kfree(sl);
13332 			sl = sln;
13333 		}
13334 		env->explored_states[i] = NULL;
13335 	}
13336 }
13337 
13338 static int do_check_common(struct bpf_verifier_env *env, int subprog)
13339 {
13340 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
13341 	struct bpf_verifier_state *state;
13342 	struct bpf_reg_state *regs;
13343 	int ret, i;
13344 
13345 	env->prev_linfo = NULL;
13346 	env->pass_cnt++;
13347 
13348 	state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
13349 	if (!state)
13350 		return -ENOMEM;
13351 	state->curframe = 0;
13352 	state->speculative = false;
13353 	state->branches = 1;
13354 	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
13355 	if (!state->frame[0]) {
13356 		kfree(state);
13357 		return -ENOMEM;
13358 	}
13359 	env->cur_state = state;
13360 	init_func_state(env, state->frame[0],
13361 			BPF_MAIN_FUNC /* callsite */,
13362 			0 /* frameno */,
13363 			subprog);
13364 
13365 	regs = state->frame[state->curframe]->regs;
13366 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
13367 		ret = btf_prepare_func_args(env, subprog, regs);
13368 		if (ret)
13369 			goto out;
13370 		for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
13371 			if (regs[i].type == PTR_TO_CTX)
13372 				mark_reg_known_zero(env, regs, i);
13373 			else if (regs[i].type == SCALAR_VALUE)
13374 				mark_reg_unknown(env, regs, i);
13375 			else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
13376 				const u32 mem_size = regs[i].mem_size;
13377 
13378 				mark_reg_known_zero(env, regs, i);
13379 				regs[i].mem_size = mem_size;
13380 				regs[i].id = ++env->id_gen;
13381 			}
13382 		}
13383 	} else {
13384 		/* 1st arg to a function */
13385 		regs[BPF_REG_1].type = PTR_TO_CTX;
13386 		mark_reg_known_zero(env, regs, BPF_REG_1);
13387 		ret = btf_check_subprog_arg_match(env, subprog, regs);
13388 		if (ret == -EFAULT)
13389 			/* unlikely verifier bug. abort.
13390 			 * ret == 0 and ret < 0 are sadly acceptable for
13391 			 * main() function due to backward compatibility.
13392 			 * Like socket filter program may be written as:
13393 			 * int bpf_prog(struct pt_regs *ctx)
13394 			 * and never dereference that ctx in the program.
13395 			 * 'struct pt_regs' is a type mismatch for socket
13396 			 * filter that should be using 'struct __sk_buff'.
13397 			 */
13398 			goto out;
13399 	}
13400 
13401 	ret = do_check(env);
13402 out:
13403 	/* check for NULL is necessary, since cur_state can be freed inside
13404 	 * do_check() under memory pressure.
13405 	 */
13406 	if (env->cur_state) {
13407 		free_verifier_state(env->cur_state, true);
13408 		env->cur_state = NULL;
13409 	}
13410 	while (!pop_stack(env, NULL, NULL, false));
13411 	if (!ret && pop_log)
13412 		bpf_vlog_reset(&env->log, 0);
13413 	free_states(env);
13414 	return ret;
13415 }
13416 
13417 /* Verify all global functions in a BPF program one by one based on their BTF.
13418  * All global functions must pass verification. Otherwise the whole program is rejected.
13419  * Consider:
13420  * int bar(int);
13421  * int foo(int f)
13422  * {
13423  *    return bar(f);
13424  * }
13425  * int bar(int b)
13426  * {
13427  *    ...
13428  * }
13429  * foo() will be verified first for R1=any_scalar_value. During verification it
13430  * will be assumed that bar() already verified successfully and call to bar()
13431  * from foo() will be checked for type match only. Later bar() will be verified
13432  * independently to check that it's safe for R1=any_scalar_value.
13433  */
13434 static int do_check_subprogs(struct bpf_verifier_env *env)
13435 {
13436 	struct bpf_prog_aux *aux = env->prog->aux;
13437 	int i, ret;
13438 
13439 	if (!aux->func_info)
13440 		return 0;
13441 
13442 	for (i = 1; i < env->subprog_cnt; i++) {
13443 		if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL)
13444 			continue;
13445 		env->insn_idx = env->subprog_info[i].start;
13446 		WARN_ON_ONCE(env->insn_idx == 0);
13447 		ret = do_check_common(env, i);
13448 		if (ret) {
13449 			return ret;
13450 		} else if (env->log.level & BPF_LOG_LEVEL) {
13451 			verbose(env,
13452 				"Func#%d is safe for any args that match its prototype\n",
13453 				i);
13454 		}
13455 	}
13456 	return 0;
13457 }
13458 
13459 static int do_check_main(struct bpf_verifier_env *env)
13460 {
13461 	int ret;
13462 
13463 	env->insn_idx = 0;
13464 	ret = do_check_common(env, 0);
13465 	if (!ret)
13466 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
13467 	return ret;
13468 }
13469 
13470 
13471 static void print_verification_stats(struct bpf_verifier_env *env)
13472 {
13473 	int i;
13474 
13475 	if (env->log.level & BPF_LOG_STATS) {
13476 		verbose(env, "verification time %lld usec\n",
13477 			div_u64(env->verification_time, 1000));
13478 		verbose(env, "stack depth ");
13479 		for (i = 0; i < env->subprog_cnt; i++) {
13480 			u32 depth = env->subprog_info[i].stack_depth;
13481 
13482 			verbose(env, "%d", depth);
13483 			if (i + 1 < env->subprog_cnt)
13484 				verbose(env, "+");
13485 		}
13486 		verbose(env, "\n");
13487 	}
13488 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
13489 		"total_states %d peak_states %d mark_read %d\n",
13490 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
13491 		env->max_states_per_insn, env->total_states,
13492 		env->peak_states, env->longest_mark_read_walk);
13493 }
13494 
13495 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
13496 {
13497 	const struct btf_type *t, *func_proto;
13498 	const struct bpf_struct_ops *st_ops;
13499 	const struct btf_member *member;
13500 	struct bpf_prog *prog = env->prog;
13501 	u32 btf_id, member_idx;
13502 	const char *mname;
13503 
13504 	if (!prog->gpl_compatible) {
13505 		verbose(env, "struct ops programs must have a GPL compatible license\n");
13506 		return -EINVAL;
13507 	}
13508 
13509 	btf_id = prog->aux->attach_btf_id;
13510 	st_ops = bpf_struct_ops_find(btf_id);
13511 	if (!st_ops) {
13512 		verbose(env, "attach_btf_id %u is not a supported struct\n",
13513 			btf_id);
13514 		return -ENOTSUPP;
13515 	}
13516 
13517 	t = st_ops->type;
13518 	member_idx = prog->expected_attach_type;
13519 	if (member_idx >= btf_type_vlen(t)) {
13520 		verbose(env, "attach to invalid member idx %u of struct %s\n",
13521 			member_idx, st_ops->name);
13522 		return -EINVAL;
13523 	}
13524 
13525 	member = &btf_type_member(t)[member_idx];
13526 	mname = btf_name_by_offset(btf_vmlinux, member->name_off);
13527 	func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
13528 					       NULL);
13529 	if (!func_proto) {
13530 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
13531 			mname, member_idx, st_ops->name);
13532 		return -EINVAL;
13533 	}
13534 
13535 	if (st_ops->check_member) {
13536 		int err = st_ops->check_member(t, member);
13537 
13538 		if (err) {
13539 			verbose(env, "attach to unsupported member %s of struct %s\n",
13540 				mname, st_ops->name);
13541 			return err;
13542 		}
13543 	}
13544 
13545 	prog->aux->attach_func_proto = func_proto;
13546 	prog->aux->attach_func_name = mname;
13547 	env->ops = st_ops->verifier_ops;
13548 
13549 	return 0;
13550 }
13551 #define SECURITY_PREFIX "security_"
13552 
13553 static int check_attach_modify_return(unsigned long addr, const char *func_name)
13554 {
13555 	if (within_error_injection_list(addr) ||
13556 	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
13557 		return 0;
13558 
13559 	return -EINVAL;
13560 }
13561 
13562 /* list of non-sleepable functions that are otherwise on
13563  * ALLOW_ERROR_INJECTION list
13564  */
13565 BTF_SET_START(btf_non_sleepable_error_inject)
13566 /* Three functions below can be called from sleepable and non-sleepable context.
13567  * Assume non-sleepable from bpf safety point of view.
13568  */
13569 BTF_ID(func, __filemap_add_folio)
13570 BTF_ID(func, should_fail_alloc_page)
13571 BTF_ID(func, should_failslab)
13572 BTF_SET_END(btf_non_sleepable_error_inject)
13573 
13574 static int check_non_sleepable_error_inject(u32 btf_id)
13575 {
13576 	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
13577 }
13578 
13579 int bpf_check_attach_target(struct bpf_verifier_log *log,
13580 			    const struct bpf_prog *prog,
13581 			    const struct bpf_prog *tgt_prog,
13582 			    u32 btf_id,
13583 			    struct bpf_attach_target_info *tgt_info)
13584 {
13585 	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
13586 	const char prefix[] = "btf_trace_";
13587 	int ret = 0, subprog = -1, i;
13588 	const struct btf_type *t;
13589 	bool conservative = true;
13590 	const char *tname;
13591 	struct btf *btf;
13592 	long addr = 0;
13593 
13594 	if (!btf_id) {
13595 		bpf_log(log, "Tracing programs must provide btf_id\n");
13596 		return -EINVAL;
13597 	}
13598 	btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
13599 	if (!btf) {
13600 		bpf_log(log,
13601 			"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
13602 		return -EINVAL;
13603 	}
13604 	t = btf_type_by_id(btf, btf_id);
13605 	if (!t) {
13606 		bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
13607 		return -EINVAL;
13608 	}
13609 	tname = btf_name_by_offset(btf, t->name_off);
13610 	if (!tname) {
13611 		bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
13612 		return -EINVAL;
13613 	}
13614 	if (tgt_prog) {
13615 		struct bpf_prog_aux *aux = tgt_prog->aux;
13616 
13617 		for (i = 0; i < aux->func_info_cnt; i++)
13618 			if (aux->func_info[i].type_id == btf_id) {
13619 				subprog = i;
13620 				break;
13621 			}
13622 		if (subprog == -1) {
13623 			bpf_log(log, "Subprog %s doesn't exist\n", tname);
13624 			return -EINVAL;
13625 		}
13626 		conservative = aux->func_info_aux[subprog].unreliable;
13627 		if (prog_extension) {
13628 			if (conservative) {
13629 				bpf_log(log,
13630 					"Cannot replace static functions\n");
13631 				return -EINVAL;
13632 			}
13633 			if (!prog->jit_requested) {
13634 				bpf_log(log,
13635 					"Extension programs should be JITed\n");
13636 				return -EINVAL;
13637 			}
13638 		}
13639 		if (!tgt_prog->jited) {
13640 			bpf_log(log, "Can attach to only JITed progs\n");
13641 			return -EINVAL;
13642 		}
13643 		if (tgt_prog->type == prog->type) {
13644 			/* Cannot fentry/fexit another fentry/fexit program.
13645 			 * Cannot attach program extension to another extension.
13646 			 * It's ok to attach fentry/fexit to extension program.
13647 			 */
13648 			bpf_log(log, "Cannot recursively attach\n");
13649 			return -EINVAL;
13650 		}
13651 		if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
13652 		    prog_extension &&
13653 		    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
13654 		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
13655 			/* Program extensions can extend all program types
13656 			 * except fentry/fexit. The reason is the following.
13657 			 * The fentry/fexit programs are used for performance
13658 			 * analysis, stats and can be attached to any program
13659 			 * type except themselves. When extension program is
13660 			 * replacing XDP function it is necessary to allow
13661 			 * performance analysis of all functions. Both original
13662 			 * XDP program and its program extension. Hence
13663 			 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is
13664 			 * allowed. If extending of fentry/fexit was allowed it
13665 			 * would be possible to create long call chain
13666 			 * fentry->extension->fentry->extension beyond
13667 			 * reasonable stack size. Hence extending fentry is not
13668 			 * allowed.
13669 			 */
13670 			bpf_log(log, "Cannot extend fentry/fexit\n");
13671 			return -EINVAL;
13672 		}
13673 	} else {
13674 		if (prog_extension) {
13675 			bpf_log(log, "Cannot replace kernel functions\n");
13676 			return -EINVAL;
13677 		}
13678 	}
13679 
13680 	switch (prog->expected_attach_type) {
13681 	case BPF_TRACE_RAW_TP:
13682 		if (tgt_prog) {
13683 			bpf_log(log,
13684 				"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
13685 			return -EINVAL;
13686 		}
13687 		if (!btf_type_is_typedef(t)) {
13688 			bpf_log(log, "attach_btf_id %u is not a typedef\n",
13689 				btf_id);
13690 			return -EINVAL;
13691 		}
13692 		if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
13693 			bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
13694 				btf_id, tname);
13695 			return -EINVAL;
13696 		}
13697 		tname += sizeof(prefix) - 1;
13698 		t = btf_type_by_id(btf, t->type);
13699 		if (!btf_type_is_ptr(t))
13700 			/* should never happen in valid vmlinux build */
13701 			return -EINVAL;
13702 		t = btf_type_by_id(btf, t->type);
13703 		if (!btf_type_is_func_proto(t))
13704 			/* should never happen in valid vmlinux build */
13705 			return -EINVAL;
13706 
13707 		break;
13708 	case BPF_TRACE_ITER:
13709 		if (!btf_type_is_func(t)) {
13710 			bpf_log(log, "attach_btf_id %u is not a function\n",
13711 				btf_id);
13712 			return -EINVAL;
13713 		}
13714 		t = btf_type_by_id(btf, t->type);
13715 		if (!btf_type_is_func_proto(t))
13716 			return -EINVAL;
13717 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
13718 		if (ret)
13719 			return ret;
13720 		break;
13721 	default:
13722 		if (!prog_extension)
13723 			return -EINVAL;
13724 		fallthrough;
13725 	case BPF_MODIFY_RETURN:
13726 	case BPF_LSM_MAC:
13727 	case BPF_TRACE_FENTRY:
13728 	case BPF_TRACE_FEXIT:
13729 		if (!btf_type_is_func(t)) {
13730 			bpf_log(log, "attach_btf_id %u is not a function\n",
13731 				btf_id);
13732 			return -EINVAL;
13733 		}
13734 		if (prog_extension &&
13735 		    btf_check_type_match(log, prog, btf, t))
13736 			return -EINVAL;
13737 		t = btf_type_by_id(btf, t->type);
13738 		if (!btf_type_is_func_proto(t))
13739 			return -EINVAL;
13740 
13741 		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
13742 		    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
13743 		     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
13744 			return -EINVAL;
13745 
13746 		if (tgt_prog && conservative)
13747 			t = NULL;
13748 
13749 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
13750 		if (ret < 0)
13751 			return ret;
13752 
13753 		if (tgt_prog) {
13754 			if (subprog == 0)
13755 				addr = (long) tgt_prog->bpf_func;
13756 			else
13757 				addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
13758 		} else {
13759 			addr = kallsyms_lookup_name(tname);
13760 			if (!addr) {
13761 				bpf_log(log,
13762 					"The address of function %s cannot be found\n",
13763 					tname);
13764 				return -ENOENT;
13765 			}
13766 		}
13767 
13768 		if (prog->aux->sleepable) {
13769 			ret = -EINVAL;
13770 			switch (prog->type) {
13771 			case BPF_PROG_TYPE_TRACING:
13772 				/* fentry/fexit/fmod_ret progs can be sleepable only if they are
13773 				 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
13774 				 */
13775 				if (!check_non_sleepable_error_inject(btf_id) &&
13776 				    within_error_injection_list(addr))
13777 					ret = 0;
13778 				break;
13779 			case BPF_PROG_TYPE_LSM:
13780 				/* LSM progs check that they are attached to bpf_lsm_*() funcs.
13781 				 * Only some of them are sleepable.
13782 				 */
13783 				if (bpf_lsm_is_sleepable_hook(btf_id))
13784 					ret = 0;
13785 				break;
13786 			default:
13787 				break;
13788 			}
13789 			if (ret) {
13790 				bpf_log(log, "%s is not sleepable\n", tname);
13791 				return ret;
13792 			}
13793 		} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
13794 			if (tgt_prog) {
13795 				bpf_log(log, "can't modify return codes of BPF programs\n");
13796 				return -EINVAL;
13797 			}
13798 			ret = check_attach_modify_return(addr, tname);
13799 			if (ret) {
13800 				bpf_log(log, "%s() is not modifiable\n", tname);
13801 				return ret;
13802 			}
13803 		}
13804 
13805 		break;
13806 	}
13807 	tgt_info->tgt_addr = addr;
13808 	tgt_info->tgt_name = tname;
13809 	tgt_info->tgt_type = t;
13810 	return 0;
13811 }
13812 
13813 BTF_SET_START(btf_id_deny)
13814 BTF_ID_UNUSED
13815 #ifdef CONFIG_SMP
13816 BTF_ID(func, migrate_disable)
13817 BTF_ID(func, migrate_enable)
13818 #endif
13819 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
13820 BTF_ID(func, rcu_read_unlock_strict)
13821 #endif
13822 BTF_SET_END(btf_id_deny)
13823 
13824 static int check_attach_btf_id(struct bpf_verifier_env *env)
13825 {
13826 	struct bpf_prog *prog = env->prog;
13827 	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
13828 	struct bpf_attach_target_info tgt_info = {};
13829 	u32 btf_id = prog->aux->attach_btf_id;
13830 	struct bpf_trampoline *tr;
13831 	int ret;
13832 	u64 key;
13833 
13834 	if (prog->type == BPF_PROG_TYPE_SYSCALL) {
13835 		if (prog->aux->sleepable)
13836 			/* attach_btf_id checked to be zero already */
13837 			return 0;
13838 		verbose(env, "Syscall programs can only be sleepable\n");
13839 		return -EINVAL;
13840 	}
13841 
13842 	if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
13843 	    prog->type != BPF_PROG_TYPE_LSM) {
13844 		verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
13845 		return -EINVAL;
13846 	}
13847 
13848 	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
13849 		return check_struct_ops_btf_id(env);
13850 
13851 	if (prog->type != BPF_PROG_TYPE_TRACING &&
13852 	    prog->type != BPF_PROG_TYPE_LSM &&
13853 	    prog->type != BPF_PROG_TYPE_EXT)
13854 		return 0;
13855 
13856 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
13857 	if (ret)
13858 		return ret;
13859 
13860 	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
13861 		/* to make freplace equivalent to their targets, they need to
13862 		 * inherit env->ops and expected_attach_type for the rest of the
13863 		 * verification
13864 		 */
13865 		env->ops = bpf_verifier_ops[tgt_prog->type];
13866 		prog->expected_attach_type = tgt_prog->expected_attach_type;
13867 	}
13868 
13869 	/* store info about the attachment target that will be used later */
13870 	prog->aux->attach_func_proto = tgt_info.tgt_type;
13871 	prog->aux->attach_func_name = tgt_info.tgt_name;
13872 
13873 	if (tgt_prog) {
13874 		prog->aux->saved_dst_prog_type = tgt_prog->type;
13875 		prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
13876 	}
13877 
13878 	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
13879 		prog->aux->attach_btf_trace = true;
13880 		return 0;
13881 	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
13882 		if (!bpf_iter_prog_supported(prog))
13883 			return -EINVAL;
13884 		return 0;
13885 	}
13886 
13887 	if (prog->type == BPF_PROG_TYPE_LSM) {
13888 		ret = bpf_lsm_verify_prog(&env->log, prog);
13889 		if (ret < 0)
13890 			return ret;
13891 	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
13892 		   btf_id_set_contains(&btf_id_deny, btf_id)) {
13893 		return -EINVAL;
13894 	}
13895 
13896 	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
13897 	tr = bpf_trampoline_get(key, &tgt_info);
13898 	if (!tr)
13899 		return -ENOMEM;
13900 
13901 	prog->aux->dst_trampoline = tr;
13902 	return 0;
13903 }
13904 
13905 struct btf *bpf_get_btf_vmlinux(void)
13906 {
13907 	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
13908 		mutex_lock(&bpf_verifier_lock);
13909 		if (!btf_vmlinux)
13910 			btf_vmlinux = btf_parse_vmlinux();
13911 		mutex_unlock(&bpf_verifier_lock);
13912 	}
13913 	return btf_vmlinux;
13914 }
13915 
13916 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr)
13917 {
13918 	u64 start_time = ktime_get_ns();
13919 	struct bpf_verifier_env *env;
13920 	struct bpf_verifier_log *log;
13921 	int i, len, ret = -EINVAL;
13922 	bool is_priv;
13923 
13924 	/* no program is valid */
13925 	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
13926 		return -EINVAL;
13927 
13928 	/* 'struct bpf_verifier_env' can be global, but since it's not small,
13929 	 * allocate/free it every time bpf_check() is called
13930 	 */
13931 	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
13932 	if (!env)
13933 		return -ENOMEM;
13934 	log = &env->log;
13935 
13936 	len = (*prog)->len;
13937 	env->insn_aux_data =
13938 		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
13939 	ret = -ENOMEM;
13940 	if (!env->insn_aux_data)
13941 		goto err_free_env;
13942 	for (i = 0; i < len; i++)
13943 		env->insn_aux_data[i].orig_idx = i;
13944 	env->prog = *prog;
13945 	env->ops = bpf_verifier_ops[env->prog->type];
13946 	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
13947 	is_priv = bpf_capable();
13948 
13949 	bpf_get_btf_vmlinux();
13950 
13951 	/* grab the mutex to protect few globals used by verifier */
13952 	if (!is_priv)
13953 		mutex_lock(&bpf_verifier_lock);
13954 
13955 	if (attr->log_level || attr->log_buf || attr->log_size) {
13956 		/* user requested verbose verifier output
13957 		 * and supplied buffer to store the verification trace
13958 		 */
13959 		log->level = attr->log_level;
13960 		log->ubuf = (char __user *) (unsigned long) attr->log_buf;
13961 		log->len_total = attr->log_size;
13962 
13963 		ret = -EINVAL;
13964 		/* log attributes have to be sane */
13965 		if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
13966 		    !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
13967 			goto err_unlock;
13968 	}
13969 
13970 	if (IS_ERR(btf_vmlinux)) {
13971 		/* Either gcc or pahole or kernel are broken. */
13972 		verbose(env, "in-kernel BTF is malformed\n");
13973 		ret = PTR_ERR(btf_vmlinux);
13974 		goto skip_full_check;
13975 	}
13976 
13977 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
13978 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
13979 		env->strict_alignment = true;
13980 	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
13981 		env->strict_alignment = false;
13982 
13983 	env->allow_ptr_leaks = bpf_allow_ptr_leaks();
13984 	env->allow_uninit_stack = bpf_allow_uninit_stack();
13985 	env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
13986 	env->bypass_spec_v1 = bpf_bypass_spec_v1();
13987 	env->bypass_spec_v4 = bpf_bypass_spec_v4();
13988 	env->bpf_capable = bpf_capable();
13989 
13990 	if (is_priv)
13991 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
13992 
13993 	env->explored_states = kvcalloc(state_htab_size(env),
13994 				       sizeof(struct bpf_verifier_state_list *),
13995 				       GFP_USER);
13996 	ret = -ENOMEM;
13997 	if (!env->explored_states)
13998 		goto skip_full_check;
13999 
14000 	ret = add_subprog_and_kfunc(env);
14001 	if (ret < 0)
14002 		goto skip_full_check;
14003 
14004 	ret = check_subprogs(env);
14005 	if (ret < 0)
14006 		goto skip_full_check;
14007 
14008 	ret = check_btf_info(env, attr, uattr);
14009 	if (ret < 0)
14010 		goto skip_full_check;
14011 
14012 	ret = check_attach_btf_id(env);
14013 	if (ret)
14014 		goto skip_full_check;
14015 
14016 	ret = resolve_pseudo_ldimm64(env);
14017 	if (ret < 0)
14018 		goto skip_full_check;
14019 
14020 	if (bpf_prog_is_dev_bound(env->prog->aux)) {
14021 		ret = bpf_prog_offload_verifier_prep(env->prog);
14022 		if (ret)
14023 			goto skip_full_check;
14024 	}
14025 
14026 	ret = check_cfg(env);
14027 	if (ret < 0)
14028 		goto skip_full_check;
14029 
14030 	ret = do_check_subprogs(env);
14031 	ret = ret ?: do_check_main(env);
14032 
14033 	if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
14034 		ret = bpf_prog_offload_finalize(env);
14035 
14036 skip_full_check:
14037 	kvfree(env->explored_states);
14038 
14039 	if (ret == 0)
14040 		ret = check_max_stack_depth(env);
14041 
14042 	/* instruction rewrites happen after this point */
14043 	if (is_priv) {
14044 		if (ret == 0)
14045 			opt_hard_wire_dead_code_branches(env);
14046 		if (ret == 0)
14047 			ret = opt_remove_dead_code(env);
14048 		if (ret == 0)
14049 			ret = opt_remove_nops(env);
14050 	} else {
14051 		if (ret == 0)
14052 			sanitize_dead_code(env);
14053 	}
14054 
14055 	if (ret == 0)
14056 		/* program is valid, convert *(u32*)(ctx + off) accesses */
14057 		ret = convert_ctx_accesses(env);
14058 
14059 	if (ret == 0)
14060 		ret = do_misc_fixups(env);
14061 
14062 	/* do 32-bit optimization after insn patching has done so those patched
14063 	 * insns could be handled correctly.
14064 	 */
14065 	if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
14066 		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
14067 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
14068 								     : false;
14069 	}
14070 
14071 	if (ret == 0)
14072 		ret = fixup_call_args(env);
14073 
14074 	env->verification_time = ktime_get_ns() - start_time;
14075 	print_verification_stats(env);
14076 	env->prog->aux->verified_insns = env->insn_processed;
14077 
14078 	if (log->level && bpf_verifier_log_full(log))
14079 		ret = -ENOSPC;
14080 	if (log->level && !log->ubuf) {
14081 		ret = -EFAULT;
14082 		goto err_release_maps;
14083 	}
14084 
14085 	if (ret)
14086 		goto err_release_maps;
14087 
14088 	if (env->used_map_cnt) {
14089 		/* if program passed verifier, update used_maps in bpf_prog_info */
14090 		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
14091 							  sizeof(env->used_maps[0]),
14092 							  GFP_KERNEL);
14093 
14094 		if (!env->prog->aux->used_maps) {
14095 			ret = -ENOMEM;
14096 			goto err_release_maps;
14097 		}
14098 
14099 		memcpy(env->prog->aux->used_maps, env->used_maps,
14100 		       sizeof(env->used_maps[0]) * env->used_map_cnt);
14101 		env->prog->aux->used_map_cnt = env->used_map_cnt;
14102 	}
14103 	if (env->used_btf_cnt) {
14104 		/* if program passed verifier, update used_btfs in bpf_prog_aux */
14105 		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
14106 							  sizeof(env->used_btfs[0]),
14107 							  GFP_KERNEL);
14108 		if (!env->prog->aux->used_btfs) {
14109 			ret = -ENOMEM;
14110 			goto err_release_maps;
14111 		}
14112 
14113 		memcpy(env->prog->aux->used_btfs, env->used_btfs,
14114 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
14115 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;
14116 	}
14117 	if (env->used_map_cnt || env->used_btf_cnt) {
14118 		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
14119 		 * bpf_ld_imm64 instructions
14120 		 */
14121 		convert_pseudo_ld_imm64(env);
14122 	}
14123 
14124 	adjust_btf_func(env);
14125 
14126 err_release_maps:
14127 	if (!env->prog->aux->used_maps)
14128 		/* if we didn't copy map pointers into bpf_prog_info, release
14129 		 * them now. Otherwise free_used_maps() will release them.
14130 		 */
14131 		release_maps(env);
14132 	if (!env->prog->aux->used_btfs)
14133 		release_btfs(env);
14134 
14135 	/* extension progs temporarily inherit the attach_type of their targets
14136 	   for verification purposes, so set it back to zero before returning
14137 	 */
14138 	if (env->prog->type == BPF_PROG_TYPE_EXT)
14139 		env->prog->expected_attach_type = 0;
14140 
14141 	*prog = env->prog;
14142 err_unlock:
14143 	if (!is_priv)
14144 		mutex_unlock(&bpf_verifier_lock);
14145 	vfree(env->insn_aux_data);
14146 err_free_env:
14147 	kfree(env);
14148 	return ret;
14149 }
14150