xref: /openbmc/linux/kernel/bpf/verifier.c (revision 2ad2f2ed)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  * Copyright (c) 2016 Facebook
4  * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
5  */
6 #include <uapi/linux/btf.h>
7 #include <linux/bpf-cgroup.h>
8 #include <linux/kernel.h>
9 #include <linux/types.h>
10 #include <linux/slab.h>
11 #include <linux/bpf.h>
12 #include <linux/btf.h>
13 #include <linux/bpf_verifier.h>
14 #include <linux/filter.h>
15 #include <net/netlink.h>
16 #include <linux/file.h>
17 #include <linux/vmalloc.h>
18 #include <linux/stringify.h>
19 #include <linux/bsearch.h>
20 #include <linux/sort.h>
21 #include <linux/perf_event.h>
22 #include <linux/ctype.h>
23 #include <linux/error-injection.h>
24 #include <linux/bpf_lsm.h>
25 #include <linux/btf_ids.h>
26 #include <linux/poison.h>
27 #include <linux/module.h>
28 #include <linux/cpumask.h>
29 #include <net/xdp.h>
30 
31 #include "disasm.h"
32 
33 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
34 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
35 	[_id] = & _name ## _verifier_ops,
36 #define BPF_MAP_TYPE(_id, _ops)
37 #define BPF_LINK_TYPE(_id, _name)
38 #include <linux/bpf_types.h>
39 #undef BPF_PROG_TYPE
40 #undef BPF_MAP_TYPE
41 #undef BPF_LINK_TYPE
42 };
43 
44 /* bpf_check() is a static code analyzer that walks eBPF program
45  * instruction by instruction and updates register/stack state.
46  * All paths of conditional branches are analyzed until 'bpf_exit' insn.
47  *
48  * The first pass is depth-first-search to check that the program is a DAG.
49  * It rejects the following programs:
50  * - larger than BPF_MAXINSNS insns
51  * - if loop is present (detected via back-edge)
52  * - unreachable insns exist (shouldn't be a forest. program = one function)
53  * - out of bounds or malformed jumps
54  * The second pass is all possible path descent from the 1st insn.
55  * Since it's analyzing all paths through the program, the length of the
56  * analysis is limited to 64k insn, which may be hit even if total number of
57  * insn is less then 4K, but there are too many branches that change stack/regs.
58  * Number of 'branches to be analyzed' is limited to 1k
59  *
60  * On entry to each instruction, each register has a type, and the instruction
61  * changes the types of the registers depending on instruction semantics.
62  * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
63  * copied to R1.
64  *
65  * All registers are 64-bit.
66  * R0 - return register
67  * R1-R5 argument passing registers
68  * R6-R9 callee saved registers
69  * R10 - frame pointer read-only
70  *
71  * At the start of BPF program the register R1 contains a pointer to bpf_context
72  * and has type PTR_TO_CTX.
73  *
74  * Verifier tracks arithmetic operations on pointers in case:
75  *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
76  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
77  * 1st insn copies R10 (which has FRAME_PTR) type into R1
78  * and 2nd arithmetic instruction is pattern matched to recognize
79  * that it wants to construct a pointer to some element within stack.
80  * So after 2nd insn, the register R1 has type PTR_TO_STACK
81  * (and -20 constant is saved for further stack bounds checking).
82  * Meaning that this reg is a pointer to stack plus known immediate constant.
83  *
84  * Most of the time the registers have SCALAR_VALUE type, which
85  * means the register has some value, but it's not a valid pointer.
86  * (like pointer plus pointer becomes SCALAR_VALUE type)
87  *
88  * When verifier sees load or store instructions the type of base register
89  * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
90  * four pointer types recognized by check_mem_access() function.
91  *
92  * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
93  * and the range of [ptr, ptr + map's value_size) is accessible.
94  *
95  * registers used to pass values to function calls are checked against
96  * function argument constraints.
97  *
98  * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
99  * It means that the register type passed to this function must be
100  * PTR_TO_STACK and it will be used inside the function as
101  * 'pointer to map element key'
102  *
103  * For example the argument constraints for bpf_map_lookup_elem():
104  *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
105  *   .arg1_type = ARG_CONST_MAP_PTR,
106  *   .arg2_type = ARG_PTR_TO_MAP_KEY,
107  *
108  * ret_type says that this function returns 'pointer to map elem value or null'
109  * function expects 1st argument to be a const pointer to 'struct bpf_map' and
110  * 2nd argument should be a pointer to stack, which will be used inside
111  * the helper function as a pointer to map element key.
112  *
113  * On the kernel side the helper function looks like:
114  * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
115  * {
116  *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
117  *    void *key = (void *) (unsigned long) r2;
118  *    void *value;
119  *
120  *    here kernel can access 'key' and 'map' pointers safely, knowing that
121  *    [key, key + map->key_size) bytes are valid and were initialized on
122  *    the stack of eBPF program.
123  * }
124  *
125  * Corresponding eBPF program may look like:
126  *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
127  *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
128  *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
129  *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
130  * here verifier looks at prototype of map_lookup_elem() and sees:
131  * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
132  * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
133  *
134  * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
135  * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
136  * and were initialized prior to this call.
137  * If it's ok, then verifier allows this BPF_CALL insn and looks at
138  * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
139  * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
140  * returns either pointer to map value or NULL.
141  *
142  * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
143  * insn, the register holding that pointer in the true branch changes state to
144  * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
145  * branch. See check_cond_jmp_op().
146  *
147  * After the call R0 is set to return type of the function and registers R1-R5
148  * are set to NOT_INIT to indicate that they are no longer readable.
149  *
150  * The following reference types represent a potential reference to a kernel
151  * resource which, after first being allocated, must be checked and freed by
152  * the BPF program:
153  * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
154  *
155  * When the verifier sees a helper call return a reference type, it allocates a
156  * pointer id for the reference and stores it in the current function state.
157  * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
158  * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
159  * passes through a NULL-check conditional. For the branch wherein the state is
160  * changed to CONST_IMM, the verifier releases the reference.
161  *
162  * For each helper function that allocates a reference, such as
163  * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
164  * bpf_sk_release(). When a reference type passes into the release function,
165  * the verifier also releases the reference. If any unchecked or unreleased
166  * reference remains at the end of the program, the verifier rejects it.
167  */
168 
169 /* verifier_state + insn_idx are pushed to stack when branch is encountered */
170 struct bpf_verifier_stack_elem {
171 	/* verifer state is 'st'
172 	 * before processing instruction 'insn_idx'
173 	 * and after processing instruction 'prev_insn_idx'
174 	 */
175 	struct bpf_verifier_state st;
176 	int insn_idx;
177 	int prev_insn_idx;
178 	struct bpf_verifier_stack_elem *next;
179 	/* length of verifier log at the time this state was pushed on stack */
180 	u32 log_pos;
181 };
182 
183 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ	8192
184 #define BPF_COMPLEXITY_LIMIT_STATES	64
185 
186 #define BPF_MAP_KEY_POISON	(1ULL << 63)
187 #define BPF_MAP_KEY_SEEN	(1ULL << 62)
188 
189 #define BPF_MAP_PTR_UNPRIV	1UL
190 #define BPF_MAP_PTR_POISON	((void *)((0xeB9FUL << 1) +	\
191 					  POISON_POINTER_DELTA))
192 #define BPF_MAP_PTR(X)		((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
193 
194 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
195 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
196 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
197 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
198 static int ref_set_non_owning(struct bpf_verifier_env *env,
199 			      struct bpf_reg_state *reg);
200 static void specialize_kfunc(struct bpf_verifier_env *env,
201 			     u32 func_id, u16 offset, unsigned long *addr);
202 static bool is_trusted_reg(const struct bpf_reg_state *reg);
203 
204 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
205 {
206 	return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON;
207 }
208 
209 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
210 {
211 	return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV;
212 }
213 
214 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
215 			      const struct bpf_map *map, bool unpriv)
216 {
217 	BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
218 	unpriv |= bpf_map_ptr_unpriv(aux);
219 	aux->map_ptr_state = (unsigned long)map |
220 			     (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
221 }
222 
223 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
224 {
225 	return aux->map_key_state & BPF_MAP_KEY_POISON;
226 }
227 
228 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
229 {
230 	return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
231 }
232 
233 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
234 {
235 	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
236 }
237 
238 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
239 {
240 	bool poisoned = bpf_map_key_poisoned(aux);
241 
242 	aux->map_key_state = state | BPF_MAP_KEY_SEEN |
243 			     (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
244 }
245 
246 static bool bpf_helper_call(const struct bpf_insn *insn)
247 {
248 	return insn->code == (BPF_JMP | BPF_CALL) &&
249 	       insn->src_reg == 0;
250 }
251 
252 static bool bpf_pseudo_call(const struct bpf_insn *insn)
253 {
254 	return insn->code == (BPF_JMP | BPF_CALL) &&
255 	       insn->src_reg == BPF_PSEUDO_CALL;
256 }
257 
258 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
259 {
260 	return insn->code == (BPF_JMP | BPF_CALL) &&
261 	       insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
262 }
263 
264 struct bpf_call_arg_meta {
265 	struct bpf_map *map_ptr;
266 	bool raw_mode;
267 	bool pkt_access;
268 	u8 release_regno;
269 	int regno;
270 	int access_size;
271 	int mem_size;
272 	u64 msize_max_value;
273 	int ref_obj_id;
274 	int dynptr_id;
275 	int map_uid;
276 	int func_id;
277 	struct btf *btf;
278 	u32 btf_id;
279 	struct btf *ret_btf;
280 	u32 ret_btf_id;
281 	u32 subprogno;
282 	struct btf_field *kptr_field;
283 };
284 
285 struct bpf_kfunc_call_arg_meta {
286 	/* In parameters */
287 	struct btf *btf;
288 	u32 func_id;
289 	u32 kfunc_flags;
290 	const struct btf_type *func_proto;
291 	const char *func_name;
292 	/* Out parameters */
293 	u32 ref_obj_id;
294 	u8 release_regno;
295 	bool r0_rdonly;
296 	u32 ret_btf_id;
297 	u64 r0_size;
298 	u32 subprogno;
299 	struct {
300 		u64 value;
301 		bool found;
302 	} arg_constant;
303 
304 	/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
305 	 * generally to pass info about user-defined local kptr types to later
306 	 * verification logic
307 	 *   bpf_obj_drop
308 	 *     Record the local kptr type to be drop'd
309 	 *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
310 	 *     Record the local kptr type to be refcount_incr'd and use
311 	 *     arg_owning_ref to determine whether refcount_acquire should be
312 	 *     fallible
313 	 */
314 	struct btf *arg_btf;
315 	u32 arg_btf_id;
316 	bool arg_owning_ref;
317 
318 	struct {
319 		struct btf_field *field;
320 	} arg_list_head;
321 	struct {
322 		struct btf_field *field;
323 	} arg_rbtree_root;
324 	struct {
325 		enum bpf_dynptr_type type;
326 		u32 id;
327 		u32 ref_obj_id;
328 	} initialized_dynptr;
329 	struct {
330 		u8 spi;
331 		u8 frameno;
332 	} iter;
333 	u64 mem_size;
334 };
335 
336 struct btf *btf_vmlinux;
337 
338 static DEFINE_MUTEX(bpf_verifier_lock);
339 
340 static const struct bpf_line_info *
341 find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
342 {
343 	const struct bpf_line_info *linfo;
344 	const struct bpf_prog *prog;
345 	u32 i, nr_linfo;
346 
347 	prog = env->prog;
348 	nr_linfo = prog->aux->nr_linfo;
349 
350 	if (!nr_linfo || insn_off >= prog->len)
351 		return NULL;
352 
353 	linfo = prog->aux->linfo;
354 	for (i = 1; i < nr_linfo; i++)
355 		if (insn_off < linfo[i].insn_off)
356 			break;
357 
358 	return &linfo[i - 1];
359 }
360 
361 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
362 {
363 	struct bpf_verifier_env *env = private_data;
364 	va_list args;
365 
366 	if (!bpf_verifier_log_needed(&env->log))
367 		return;
368 
369 	va_start(args, fmt);
370 	bpf_verifier_vlog(&env->log, fmt, args);
371 	va_end(args);
372 }
373 
374 static const char *ltrim(const char *s)
375 {
376 	while (isspace(*s))
377 		s++;
378 
379 	return s;
380 }
381 
382 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
383 					 u32 insn_off,
384 					 const char *prefix_fmt, ...)
385 {
386 	const struct bpf_line_info *linfo;
387 
388 	if (!bpf_verifier_log_needed(&env->log))
389 		return;
390 
391 	linfo = find_linfo(env, insn_off);
392 	if (!linfo || linfo == env->prev_linfo)
393 		return;
394 
395 	if (prefix_fmt) {
396 		va_list args;
397 
398 		va_start(args, prefix_fmt);
399 		bpf_verifier_vlog(&env->log, prefix_fmt, args);
400 		va_end(args);
401 	}
402 
403 	verbose(env, "%s\n",
404 		ltrim(btf_name_by_offset(env->prog->aux->btf,
405 					 linfo->line_off)));
406 
407 	env->prev_linfo = linfo;
408 }
409 
410 static void verbose_invalid_scalar(struct bpf_verifier_env *env,
411 				   struct bpf_reg_state *reg,
412 				   struct tnum *range, const char *ctx,
413 				   const char *reg_name)
414 {
415 	char tn_buf[48];
416 
417 	verbose(env, "At %s the register %s ", ctx, reg_name);
418 	if (!tnum_is_unknown(reg->var_off)) {
419 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
420 		verbose(env, "has value %s", tn_buf);
421 	} else {
422 		verbose(env, "has unknown scalar value");
423 	}
424 	tnum_strn(tn_buf, sizeof(tn_buf), *range);
425 	verbose(env, " should have been in %s\n", tn_buf);
426 }
427 
428 static bool type_is_pkt_pointer(enum bpf_reg_type type)
429 {
430 	type = base_type(type);
431 	return type == PTR_TO_PACKET ||
432 	       type == PTR_TO_PACKET_META;
433 }
434 
435 static bool type_is_sk_pointer(enum bpf_reg_type type)
436 {
437 	return type == PTR_TO_SOCKET ||
438 		type == PTR_TO_SOCK_COMMON ||
439 		type == PTR_TO_TCP_SOCK ||
440 		type == PTR_TO_XDP_SOCK;
441 }
442 
443 static bool type_may_be_null(u32 type)
444 {
445 	return type & PTR_MAYBE_NULL;
446 }
447 
448 static bool reg_not_null(const struct bpf_reg_state *reg)
449 {
450 	enum bpf_reg_type type;
451 
452 	type = reg->type;
453 	if (type_may_be_null(type))
454 		return false;
455 
456 	type = base_type(type);
457 	return type == PTR_TO_SOCKET ||
458 		type == PTR_TO_TCP_SOCK ||
459 		type == PTR_TO_MAP_VALUE ||
460 		type == PTR_TO_MAP_KEY ||
461 		type == PTR_TO_SOCK_COMMON ||
462 		(type == PTR_TO_BTF_ID && is_trusted_reg(reg)) ||
463 		type == PTR_TO_MEM;
464 }
465 
466 static bool type_is_ptr_alloc_obj(u32 type)
467 {
468 	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
469 }
470 
471 static bool type_is_non_owning_ref(u32 type)
472 {
473 	return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
474 }
475 
476 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
477 {
478 	struct btf_record *rec = NULL;
479 	struct btf_struct_meta *meta;
480 
481 	if (reg->type == PTR_TO_MAP_VALUE) {
482 		rec = reg->map_ptr->record;
483 	} else if (type_is_ptr_alloc_obj(reg->type)) {
484 		meta = btf_find_struct_meta(reg->btf, reg->btf_id);
485 		if (meta)
486 			rec = meta->record;
487 	}
488 	return rec;
489 }
490 
491 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
492 {
493 	struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
494 
495 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
496 }
497 
498 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
499 {
500 	return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK);
501 }
502 
503 static bool type_is_rdonly_mem(u32 type)
504 {
505 	return type & MEM_RDONLY;
506 }
507 
508 static bool is_acquire_function(enum bpf_func_id func_id,
509 				const struct bpf_map *map)
510 {
511 	enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
512 
513 	if (func_id == BPF_FUNC_sk_lookup_tcp ||
514 	    func_id == BPF_FUNC_sk_lookup_udp ||
515 	    func_id == BPF_FUNC_skc_lookup_tcp ||
516 	    func_id == BPF_FUNC_ringbuf_reserve ||
517 	    func_id == BPF_FUNC_kptr_xchg)
518 		return true;
519 
520 	if (func_id == BPF_FUNC_map_lookup_elem &&
521 	    (map_type == BPF_MAP_TYPE_SOCKMAP ||
522 	     map_type == BPF_MAP_TYPE_SOCKHASH))
523 		return true;
524 
525 	return false;
526 }
527 
528 static bool is_ptr_cast_function(enum bpf_func_id func_id)
529 {
530 	return func_id == BPF_FUNC_tcp_sock ||
531 		func_id == BPF_FUNC_sk_fullsock ||
532 		func_id == BPF_FUNC_skc_to_tcp_sock ||
533 		func_id == BPF_FUNC_skc_to_tcp6_sock ||
534 		func_id == BPF_FUNC_skc_to_udp6_sock ||
535 		func_id == BPF_FUNC_skc_to_mptcp_sock ||
536 		func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
537 		func_id == BPF_FUNC_skc_to_tcp_request_sock;
538 }
539 
540 static bool is_dynptr_ref_function(enum bpf_func_id func_id)
541 {
542 	return func_id == BPF_FUNC_dynptr_data;
543 }
544 
545 static bool is_sync_callback_calling_kfunc(u32 btf_id);
546 
547 static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
548 {
549 	return func_id == BPF_FUNC_for_each_map_elem ||
550 	       func_id == BPF_FUNC_find_vma ||
551 	       func_id == BPF_FUNC_loop ||
552 	       func_id == BPF_FUNC_user_ringbuf_drain;
553 }
554 
555 static bool is_async_callback_calling_function(enum bpf_func_id func_id)
556 {
557 	return func_id == BPF_FUNC_timer_set_callback;
558 }
559 
560 static bool is_callback_calling_function(enum bpf_func_id func_id)
561 {
562 	return is_sync_callback_calling_function(func_id) ||
563 	       is_async_callback_calling_function(func_id);
564 }
565 
566 static bool is_sync_callback_calling_insn(struct bpf_insn *insn)
567 {
568 	return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
569 	       (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
570 }
571 
572 static bool is_storage_get_function(enum bpf_func_id func_id)
573 {
574 	return func_id == BPF_FUNC_sk_storage_get ||
575 	       func_id == BPF_FUNC_inode_storage_get ||
576 	       func_id == BPF_FUNC_task_storage_get ||
577 	       func_id == BPF_FUNC_cgrp_storage_get;
578 }
579 
580 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
581 					const struct bpf_map *map)
582 {
583 	int ref_obj_uses = 0;
584 
585 	if (is_ptr_cast_function(func_id))
586 		ref_obj_uses++;
587 	if (is_acquire_function(func_id, map))
588 		ref_obj_uses++;
589 	if (is_dynptr_ref_function(func_id))
590 		ref_obj_uses++;
591 
592 	return ref_obj_uses > 1;
593 }
594 
595 static bool is_cmpxchg_insn(const struct bpf_insn *insn)
596 {
597 	return BPF_CLASS(insn->code) == BPF_STX &&
598 	       BPF_MODE(insn->code) == BPF_ATOMIC &&
599 	       insn->imm == BPF_CMPXCHG;
600 }
601 
602 /* string representation of 'enum bpf_reg_type'
603  *
604  * Note that reg_type_str() can not appear more than once in a single verbose()
605  * statement.
606  */
607 static const char *reg_type_str(struct bpf_verifier_env *env,
608 				enum bpf_reg_type type)
609 {
610 	char postfix[16] = {0}, prefix[64] = {0};
611 	static const char * const str[] = {
612 		[NOT_INIT]		= "?",
613 		[SCALAR_VALUE]		= "scalar",
614 		[PTR_TO_CTX]		= "ctx",
615 		[CONST_PTR_TO_MAP]	= "map_ptr",
616 		[PTR_TO_MAP_VALUE]	= "map_value",
617 		[PTR_TO_STACK]		= "fp",
618 		[PTR_TO_PACKET]		= "pkt",
619 		[PTR_TO_PACKET_META]	= "pkt_meta",
620 		[PTR_TO_PACKET_END]	= "pkt_end",
621 		[PTR_TO_FLOW_KEYS]	= "flow_keys",
622 		[PTR_TO_SOCKET]		= "sock",
623 		[PTR_TO_SOCK_COMMON]	= "sock_common",
624 		[PTR_TO_TCP_SOCK]	= "tcp_sock",
625 		[PTR_TO_TP_BUFFER]	= "tp_buffer",
626 		[PTR_TO_XDP_SOCK]	= "xdp_sock",
627 		[PTR_TO_BTF_ID]		= "ptr_",
628 		[PTR_TO_MEM]		= "mem",
629 		[PTR_TO_BUF]		= "buf",
630 		[PTR_TO_FUNC]		= "func",
631 		[PTR_TO_MAP_KEY]	= "map_key",
632 		[CONST_PTR_TO_DYNPTR]	= "dynptr_ptr",
633 	};
634 
635 	if (type & PTR_MAYBE_NULL) {
636 		if (base_type(type) == PTR_TO_BTF_ID)
637 			strncpy(postfix, "or_null_", 16);
638 		else
639 			strncpy(postfix, "_or_null", 16);
640 	}
641 
642 	snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s",
643 		 type & MEM_RDONLY ? "rdonly_" : "",
644 		 type & MEM_RINGBUF ? "ringbuf_" : "",
645 		 type & MEM_USER ? "user_" : "",
646 		 type & MEM_PERCPU ? "percpu_" : "",
647 		 type & MEM_RCU ? "rcu_" : "",
648 		 type & PTR_UNTRUSTED ? "untrusted_" : "",
649 		 type & PTR_TRUSTED ? "trusted_" : ""
650 	);
651 
652 	snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",
653 		 prefix, str[base_type(type)], postfix);
654 	return env->tmp_str_buf;
655 }
656 
657 static char slot_type_char[] = {
658 	[STACK_INVALID]	= '?',
659 	[STACK_SPILL]	= 'r',
660 	[STACK_MISC]	= 'm',
661 	[STACK_ZERO]	= '0',
662 	[STACK_DYNPTR]	= 'd',
663 	[STACK_ITER]	= 'i',
664 };
665 
666 static void print_liveness(struct bpf_verifier_env *env,
667 			   enum bpf_reg_liveness live)
668 {
669 	if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
670 	    verbose(env, "_");
671 	if (live & REG_LIVE_READ)
672 		verbose(env, "r");
673 	if (live & REG_LIVE_WRITTEN)
674 		verbose(env, "w");
675 	if (live & REG_LIVE_DONE)
676 		verbose(env, "D");
677 }
678 
679 static int __get_spi(s32 off)
680 {
681 	return (-off - 1) / BPF_REG_SIZE;
682 }
683 
684 static struct bpf_func_state *func(struct bpf_verifier_env *env,
685 				   const struct bpf_reg_state *reg)
686 {
687 	struct bpf_verifier_state *cur = env->cur_state;
688 
689 	return cur->frame[reg->frameno];
690 }
691 
692 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
693 {
694        int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
695 
696        /* We need to check that slots between [spi - nr_slots + 1, spi] are
697 	* within [0, allocated_stack).
698 	*
699 	* Please note that the spi grows downwards. For example, a dynptr
700 	* takes the size of two stack slots; the first slot will be at
701 	* spi and the second slot will be at spi - 1.
702 	*/
703        return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
704 }
705 
706 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
707 			          const char *obj_kind, int nr_slots)
708 {
709 	int off, spi;
710 
711 	if (!tnum_is_const(reg->var_off)) {
712 		verbose(env, "%s has to be at a constant offset\n", obj_kind);
713 		return -EINVAL;
714 	}
715 
716 	off = reg->off + reg->var_off.value;
717 	if (off % BPF_REG_SIZE) {
718 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
719 		return -EINVAL;
720 	}
721 
722 	spi = __get_spi(off);
723 	if (spi + 1 < nr_slots) {
724 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
725 		return -EINVAL;
726 	}
727 
728 	if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots))
729 		return -ERANGE;
730 	return spi;
731 }
732 
733 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
734 {
735 	return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS);
736 }
737 
738 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots)
739 {
740 	return stack_slot_obj_get_spi(env, reg, "iter", nr_slots);
741 }
742 
743 static const char *btf_type_name(const struct btf *btf, u32 id)
744 {
745 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
746 }
747 
748 static const char *dynptr_type_str(enum bpf_dynptr_type type)
749 {
750 	switch (type) {
751 	case BPF_DYNPTR_TYPE_LOCAL:
752 		return "local";
753 	case BPF_DYNPTR_TYPE_RINGBUF:
754 		return "ringbuf";
755 	case BPF_DYNPTR_TYPE_SKB:
756 		return "skb";
757 	case BPF_DYNPTR_TYPE_XDP:
758 		return "xdp";
759 	case BPF_DYNPTR_TYPE_INVALID:
760 		return "<invalid>";
761 	default:
762 		WARN_ONCE(1, "unknown dynptr type %d\n", type);
763 		return "<unknown>";
764 	}
765 }
766 
767 static const char *iter_type_str(const struct btf *btf, u32 btf_id)
768 {
769 	if (!btf || btf_id == 0)
770 		return "<invalid>";
771 
772 	/* we already validated that type is valid and has conforming name */
773 	return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;
774 }
775 
776 static const char *iter_state_str(enum bpf_iter_state state)
777 {
778 	switch (state) {
779 	case BPF_ITER_STATE_ACTIVE:
780 		return "active";
781 	case BPF_ITER_STATE_DRAINED:
782 		return "drained";
783 	case BPF_ITER_STATE_INVALID:
784 		return "<invalid>";
785 	default:
786 		WARN_ONCE(1, "unknown iter state %d\n", state);
787 		return "<unknown>";
788 	}
789 }
790 
791 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
792 {
793 	env->scratched_regs |= 1U << regno;
794 }
795 
796 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
797 {
798 	env->scratched_stack_slots |= 1ULL << spi;
799 }
800 
801 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
802 {
803 	return (env->scratched_regs >> regno) & 1;
804 }
805 
806 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
807 {
808 	return (env->scratched_stack_slots >> regno) & 1;
809 }
810 
811 static bool verifier_state_scratched(const struct bpf_verifier_env *env)
812 {
813 	return env->scratched_regs || env->scratched_stack_slots;
814 }
815 
816 static void mark_verifier_state_clean(struct bpf_verifier_env *env)
817 {
818 	env->scratched_regs = 0U;
819 	env->scratched_stack_slots = 0ULL;
820 }
821 
822 /* Used for printing the entire verifier state. */
823 static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
824 {
825 	env->scratched_regs = ~0U;
826 	env->scratched_stack_slots = ~0ULL;
827 }
828 
829 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
830 {
831 	switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
832 	case DYNPTR_TYPE_LOCAL:
833 		return BPF_DYNPTR_TYPE_LOCAL;
834 	case DYNPTR_TYPE_RINGBUF:
835 		return BPF_DYNPTR_TYPE_RINGBUF;
836 	case DYNPTR_TYPE_SKB:
837 		return BPF_DYNPTR_TYPE_SKB;
838 	case DYNPTR_TYPE_XDP:
839 		return BPF_DYNPTR_TYPE_XDP;
840 	default:
841 		return BPF_DYNPTR_TYPE_INVALID;
842 	}
843 }
844 
845 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type)
846 {
847 	switch (type) {
848 	case BPF_DYNPTR_TYPE_LOCAL:
849 		return DYNPTR_TYPE_LOCAL;
850 	case BPF_DYNPTR_TYPE_RINGBUF:
851 		return DYNPTR_TYPE_RINGBUF;
852 	case BPF_DYNPTR_TYPE_SKB:
853 		return DYNPTR_TYPE_SKB;
854 	case BPF_DYNPTR_TYPE_XDP:
855 		return DYNPTR_TYPE_XDP;
856 	default:
857 		return 0;
858 	}
859 }
860 
861 static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
862 {
863 	return type == BPF_DYNPTR_TYPE_RINGBUF;
864 }
865 
866 static void __mark_dynptr_reg(struct bpf_reg_state *reg,
867 			      enum bpf_dynptr_type type,
868 			      bool first_slot, int dynptr_id);
869 
870 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
871 				struct bpf_reg_state *reg);
872 
873 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
874 				   struct bpf_reg_state *sreg1,
875 				   struct bpf_reg_state *sreg2,
876 				   enum bpf_dynptr_type type)
877 {
878 	int id = ++env->id_gen;
879 
880 	__mark_dynptr_reg(sreg1, type, true, id);
881 	__mark_dynptr_reg(sreg2, type, false, id);
882 }
883 
884 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
885 			       struct bpf_reg_state *reg,
886 			       enum bpf_dynptr_type type)
887 {
888 	__mark_dynptr_reg(reg, type, true, ++env->id_gen);
889 }
890 
891 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
892 				        struct bpf_func_state *state, int spi);
893 
894 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
895 				   enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)
896 {
897 	struct bpf_func_state *state = func(env, reg);
898 	enum bpf_dynptr_type type;
899 	int spi, i, err;
900 
901 	spi = dynptr_get_spi(env, reg);
902 	if (spi < 0)
903 		return spi;
904 
905 	/* We cannot assume both spi and spi - 1 belong to the same dynptr,
906 	 * hence we need to call destroy_if_dynptr_stack_slot twice for both,
907 	 * to ensure that for the following example:
908 	 *	[d1][d1][d2][d2]
909 	 * spi    3   2   1   0
910 	 * So marking spi = 2 should lead to destruction of both d1 and d2. In
911 	 * case they do belong to same dynptr, second call won't see slot_type
912 	 * as STACK_DYNPTR and will simply skip destruction.
913 	 */
914 	err = destroy_if_dynptr_stack_slot(env, state, spi);
915 	if (err)
916 		return err;
917 	err = destroy_if_dynptr_stack_slot(env, state, spi - 1);
918 	if (err)
919 		return err;
920 
921 	for (i = 0; i < BPF_REG_SIZE; i++) {
922 		state->stack[spi].slot_type[i] = STACK_DYNPTR;
923 		state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
924 	}
925 
926 	type = arg_to_dynptr_type(arg_type);
927 	if (type == BPF_DYNPTR_TYPE_INVALID)
928 		return -EINVAL;
929 
930 	mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
931 			       &state->stack[spi - 1].spilled_ptr, type);
932 
933 	if (dynptr_type_refcounted(type)) {
934 		/* The id is used to track proper releasing */
935 		int id;
936 
937 		if (clone_ref_obj_id)
938 			id = clone_ref_obj_id;
939 		else
940 			id = acquire_reference_state(env, insn_idx);
941 
942 		if (id < 0)
943 			return id;
944 
945 		state->stack[spi].spilled_ptr.ref_obj_id = id;
946 		state->stack[spi - 1].spilled_ptr.ref_obj_id = id;
947 	}
948 
949 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
950 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
951 
952 	return 0;
953 }
954 
955 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)
956 {
957 	int i;
958 
959 	for (i = 0; i < BPF_REG_SIZE; i++) {
960 		state->stack[spi].slot_type[i] = STACK_INVALID;
961 		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
962 	}
963 
964 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
965 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
966 
967 	/* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot?
968 	 *
969 	 * While we don't allow reading STACK_INVALID, it is still possible to
970 	 * do <8 byte writes marking some but not all slots as STACK_MISC. Then,
971 	 * helpers or insns can do partial read of that part without failing,
972 	 * but check_stack_range_initialized, check_stack_read_var_off, and
973 	 * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of
974 	 * the slot conservatively. Hence we need to prevent those liveness
975 	 * marking walks.
976 	 *
977 	 * This was not a problem before because STACK_INVALID is only set by
978 	 * default (where the default reg state has its reg->parent as NULL), or
979 	 * in clean_live_states after REG_LIVE_DONE (at which point
980 	 * mark_reg_read won't walk reg->parent chain), but not randomly during
981 	 * verifier state exploration (like we did above). Hence, for our case
982 	 * parentage chain will still be live (i.e. reg->parent may be
983 	 * non-NULL), while earlier reg->parent was NULL, so we need
984 	 * REG_LIVE_WRITTEN to screen off read marker propagation when it is
985 	 * done later on reads or by mark_dynptr_read as well to unnecessary
986 	 * mark registers in verifier state.
987 	 */
988 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
989 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
990 }
991 
992 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
993 {
994 	struct bpf_func_state *state = func(env, reg);
995 	int spi, ref_obj_id, i;
996 
997 	spi = dynptr_get_spi(env, reg);
998 	if (spi < 0)
999 		return spi;
1000 
1001 	if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
1002 		invalidate_dynptr(env, state, spi);
1003 		return 0;
1004 	}
1005 
1006 	ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;
1007 
1008 	/* If the dynptr has a ref_obj_id, then we need to invalidate
1009 	 * two things:
1010 	 *
1011 	 * 1) Any dynptrs with a matching ref_obj_id (clones)
1012 	 * 2) Any slices derived from this dynptr.
1013 	 */
1014 
1015 	/* Invalidate any slices associated with this dynptr */
1016 	WARN_ON_ONCE(release_reference(env, ref_obj_id));
1017 
1018 	/* Invalidate any dynptr clones */
1019 	for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) {
1020 		if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id)
1021 			continue;
1022 
1023 		/* it should always be the case that if the ref obj id
1024 		 * matches then the stack slot also belongs to a
1025 		 * dynptr
1026 		 */
1027 		if (state->stack[i].slot_type[0] != STACK_DYNPTR) {
1028 			verbose(env, "verifier internal error: misconfigured ref_obj_id\n");
1029 			return -EFAULT;
1030 		}
1031 		if (state->stack[i].spilled_ptr.dynptr.first_slot)
1032 			invalidate_dynptr(env, state, i);
1033 	}
1034 
1035 	return 0;
1036 }
1037 
1038 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
1039 			       struct bpf_reg_state *reg);
1040 
1041 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1042 {
1043 	if (!env->allow_ptr_leaks)
1044 		__mark_reg_not_init(env, reg);
1045 	else
1046 		__mark_reg_unknown(env, reg);
1047 }
1048 
1049 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
1050 				        struct bpf_func_state *state, int spi)
1051 {
1052 	struct bpf_func_state *fstate;
1053 	struct bpf_reg_state *dreg;
1054 	int i, dynptr_id;
1055 
1056 	/* We always ensure that STACK_DYNPTR is never set partially,
1057 	 * hence just checking for slot_type[0] is enough. This is
1058 	 * different for STACK_SPILL, where it may be only set for
1059 	 * 1 byte, so code has to use is_spilled_reg.
1060 	 */
1061 	if (state->stack[spi].slot_type[0] != STACK_DYNPTR)
1062 		return 0;
1063 
1064 	/* Reposition spi to first slot */
1065 	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
1066 		spi = spi + 1;
1067 
1068 	if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
1069 		verbose(env, "cannot overwrite referenced dynptr\n");
1070 		return -EINVAL;
1071 	}
1072 
1073 	mark_stack_slot_scratched(env, spi);
1074 	mark_stack_slot_scratched(env, spi - 1);
1075 
1076 	/* Writing partially to one dynptr stack slot destroys both. */
1077 	for (i = 0; i < BPF_REG_SIZE; i++) {
1078 		state->stack[spi].slot_type[i] = STACK_INVALID;
1079 		state->stack[spi - 1].slot_type[i] = STACK_INVALID;
1080 	}
1081 
1082 	dynptr_id = state->stack[spi].spilled_ptr.id;
1083 	/* Invalidate any slices associated with this dynptr */
1084 	bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({
1085 		/* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */
1086 		if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM)
1087 			continue;
1088 		if (dreg->dynptr_id == dynptr_id)
1089 			mark_reg_invalid(env, dreg);
1090 	}));
1091 
1092 	/* Do not release reference state, we are destroying dynptr on stack,
1093 	 * not using some helper to release it. Just reset register.
1094 	 */
1095 	__mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
1096 	__mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
1097 
1098 	/* Same reason as unmark_stack_slots_dynptr above */
1099 	state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1100 	state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN;
1101 
1102 	return 0;
1103 }
1104 
1105 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1106 {
1107 	int spi;
1108 
1109 	if (reg->type == CONST_PTR_TO_DYNPTR)
1110 		return false;
1111 
1112 	spi = dynptr_get_spi(env, reg);
1113 
1114 	/* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an
1115 	 * error because this just means the stack state hasn't been updated yet.
1116 	 * We will do check_mem_access to check and update stack bounds later.
1117 	 */
1118 	if (spi < 0 && spi != -ERANGE)
1119 		return false;
1120 
1121 	/* We don't need to check if the stack slots are marked by previous
1122 	 * dynptr initializations because we allow overwriting existing unreferenced
1123 	 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls
1124 	 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are
1125 	 * touching are completely destructed before we reinitialize them for a new
1126 	 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early
1127 	 * instead of delaying it until the end where the user will get "Unreleased
1128 	 * reference" error.
1129 	 */
1130 	return true;
1131 }
1132 
1133 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1134 {
1135 	struct bpf_func_state *state = func(env, reg);
1136 	int i, spi;
1137 
1138 	/* This already represents first slot of initialized bpf_dynptr.
1139 	 *
1140 	 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to
1141 	 * check_func_arg_reg_off's logic, so we don't need to check its
1142 	 * offset and alignment.
1143 	 */
1144 	if (reg->type == CONST_PTR_TO_DYNPTR)
1145 		return true;
1146 
1147 	spi = dynptr_get_spi(env, reg);
1148 	if (spi < 0)
1149 		return false;
1150 	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
1151 		return false;
1152 
1153 	for (i = 0; i < BPF_REG_SIZE; i++) {
1154 		if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
1155 		    state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
1156 			return false;
1157 	}
1158 
1159 	return true;
1160 }
1161 
1162 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1163 				    enum bpf_arg_type arg_type)
1164 {
1165 	struct bpf_func_state *state = func(env, reg);
1166 	enum bpf_dynptr_type dynptr_type;
1167 	int spi;
1168 
1169 	/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
1170 	if (arg_type == ARG_PTR_TO_DYNPTR)
1171 		return true;
1172 
1173 	dynptr_type = arg_to_dynptr_type(arg_type);
1174 	if (reg->type == CONST_PTR_TO_DYNPTR) {
1175 		return reg->dynptr.type == dynptr_type;
1176 	} else {
1177 		spi = dynptr_get_spi(env, reg);
1178 		if (spi < 0)
1179 			return false;
1180 		return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
1181 	}
1182 }
1183 
1184 static void __mark_reg_known_zero(struct bpf_reg_state *reg);
1185 
1186 static int mark_stack_slots_iter(struct bpf_verifier_env *env,
1187 				 struct bpf_reg_state *reg, int insn_idx,
1188 				 struct btf *btf, u32 btf_id, int nr_slots)
1189 {
1190 	struct bpf_func_state *state = func(env, reg);
1191 	int spi, i, j, id;
1192 
1193 	spi = iter_get_spi(env, reg, nr_slots);
1194 	if (spi < 0)
1195 		return spi;
1196 
1197 	id = acquire_reference_state(env, insn_idx);
1198 	if (id < 0)
1199 		return id;
1200 
1201 	for (i = 0; i < nr_slots; i++) {
1202 		struct bpf_stack_state *slot = &state->stack[spi - i];
1203 		struct bpf_reg_state *st = &slot->spilled_ptr;
1204 
1205 		__mark_reg_known_zero(st);
1206 		st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
1207 		st->live |= REG_LIVE_WRITTEN;
1208 		st->ref_obj_id = i == 0 ? id : 0;
1209 		st->iter.btf = btf;
1210 		st->iter.btf_id = btf_id;
1211 		st->iter.state = BPF_ITER_STATE_ACTIVE;
1212 		st->iter.depth = 0;
1213 
1214 		for (j = 0; j < BPF_REG_SIZE; j++)
1215 			slot->slot_type[j] = STACK_ITER;
1216 
1217 		mark_stack_slot_scratched(env, spi - i);
1218 	}
1219 
1220 	return 0;
1221 }
1222 
1223 static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
1224 				   struct bpf_reg_state *reg, int nr_slots)
1225 {
1226 	struct bpf_func_state *state = func(env, reg);
1227 	int spi, i, j;
1228 
1229 	spi = iter_get_spi(env, reg, nr_slots);
1230 	if (spi < 0)
1231 		return spi;
1232 
1233 	for (i = 0; i < nr_slots; i++) {
1234 		struct bpf_stack_state *slot = &state->stack[spi - i];
1235 		struct bpf_reg_state *st = &slot->spilled_ptr;
1236 
1237 		if (i == 0)
1238 			WARN_ON_ONCE(release_reference(env, st->ref_obj_id));
1239 
1240 		__mark_reg_not_init(env, st);
1241 
1242 		/* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */
1243 		st->live |= REG_LIVE_WRITTEN;
1244 
1245 		for (j = 0; j < BPF_REG_SIZE; j++)
1246 			slot->slot_type[j] = STACK_INVALID;
1247 
1248 		mark_stack_slot_scratched(env, spi - i);
1249 	}
1250 
1251 	return 0;
1252 }
1253 
1254 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
1255 				     struct bpf_reg_state *reg, int nr_slots)
1256 {
1257 	struct bpf_func_state *state = func(env, reg);
1258 	int spi, i, j;
1259 
1260 	/* For -ERANGE (i.e. spi not falling into allocated stack slots), we
1261 	 * will do check_mem_access to check and update stack bounds later, so
1262 	 * return true for that case.
1263 	 */
1264 	spi = iter_get_spi(env, reg, nr_slots);
1265 	if (spi == -ERANGE)
1266 		return true;
1267 	if (spi < 0)
1268 		return false;
1269 
1270 	for (i = 0; i < nr_slots; i++) {
1271 		struct bpf_stack_state *slot = &state->stack[spi - i];
1272 
1273 		for (j = 0; j < BPF_REG_SIZE; j++)
1274 			if (slot->slot_type[j] == STACK_ITER)
1275 				return false;
1276 	}
1277 
1278 	return true;
1279 }
1280 
1281 static bool is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
1282 				   struct btf *btf, u32 btf_id, int nr_slots)
1283 {
1284 	struct bpf_func_state *state = func(env, reg);
1285 	int spi, i, j;
1286 
1287 	spi = iter_get_spi(env, reg, nr_slots);
1288 	if (spi < 0)
1289 		return false;
1290 
1291 	for (i = 0; i < nr_slots; i++) {
1292 		struct bpf_stack_state *slot = &state->stack[spi - i];
1293 		struct bpf_reg_state *st = &slot->spilled_ptr;
1294 
1295 		/* only main (first) slot has ref_obj_id set */
1296 		if (i == 0 && !st->ref_obj_id)
1297 			return false;
1298 		if (i != 0 && st->ref_obj_id)
1299 			return false;
1300 		if (st->iter.btf != btf || st->iter.btf_id != btf_id)
1301 			return false;
1302 
1303 		for (j = 0; j < BPF_REG_SIZE; j++)
1304 			if (slot->slot_type[j] != STACK_ITER)
1305 				return false;
1306 	}
1307 
1308 	return true;
1309 }
1310 
1311 /* Check if given stack slot is "special":
1312  *   - spilled register state (STACK_SPILL);
1313  *   - dynptr state (STACK_DYNPTR);
1314  *   - iter state (STACK_ITER).
1315  */
1316 static bool is_stack_slot_special(const struct bpf_stack_state *stack)
1317 {
1318 	enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1];
1319 
1320 	switch (type) {
1321 	case STACK_SPILL:
1322 	case STACK_DYNPTR:
1323 	case STACK_ITER:
1324 		return true;
1325 	case STACK_INVALID:
1326 	case STACK_MISC:
1327 	case STACK_ZERO:
1328 		return false;
1329 	default:
1330 		WARN_ONCE(1, "unknown stack slot type %d\n", type);
1331 		return true;
1332 	}
1333 }
1334 
1335 /* The reg state of a pointer or a bounded scalar was saved when
1336  * it was spilled to the stack.
1337  */
1338 static bool is_spilled_reg(const struct bpf_stack_state *stack)
1339 {
1340 	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
1341 }
1342 
1343 static bool is_spilled_scalar_reg(const struct bpf_stack_state *stack)
1344 {
1345 	return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL &&
1346 	       stack->spilled_ptr.type == SCALAR_VALUE;
1347 }
1348 
1349 static void scrub_spilled_slot(u8 *stype)
1350 {
1351 	if (*stype != STACK_INVALID)
1352 		*stype = STACK_MISC;
1353 }
1354 
1355 static void print_verifier_state(struct bpf_verifier_env *env,
1356 				 const struct bpf_func_state *state,
1357 				 bool print_all)
1358 {
1359 	const struct bpf_reg_state *reg;
1360 	enum bpf_reg_type t;
1361 	int i;
1362 
1363 	if (state->frameno)
1364 		verbose(env, " frame%d:", state->frameno);
1365 	for (i = 0; i < MAX_BPF_REG; i++) {
1366 		reg = &state->regs[i];
1367 		t = reg->type;
1368 		if (t == NOT_INIT)
1369 			continue;
1370 		if (!print_all && !reg_scratched(env, i))
1371 			continue;
1372 		verbose(env, " R%d", i);
1373 		print_liveness(env, reg->live);
1374 		verbose(env, "=");
1375 		if (t == SCALAR_VALUE && reg->precise)
1376 			verbose(env, "P");
1377 		if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
1378 		    tnum_is_const(reg->var_off)) {
1379 			/* reg->off should be 0 for SCALAR_VALUE */
1380 			verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
1381 			verbose(env, "%lld", reg->var_off.value + reg->off);
1382 		} else {
1383 			const char *sep = "";
1384 
1385 			verbose(env, "%s", reg_type_str(env, t));
1386 			if (base_type(t) == PTR_TO_BTF_ID)
1387 				verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));
1388 			verbose(env, "(");
1389 /*
1390  * _a stands for append, was shortened to avoid multiline statements below.
1391  * This macro is used to output a comma separated list of attributes.
1392  */
1393 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; })
1394 
1395 			if (reg->id)
1396 				verbose_a("id=%d", reg->id);
1397 			if (reg->ref_obj_id)
1398 				verbose_a("ref_obj_id=%d", reg->ref_obj_id);
1399 			if (type_is_non_owning_ref(reg->type))
1400 				verbose_a("%s", "non_own_ref");
1401 			if (t != SCALAR_VALUE)
1402 				verbose_a("off=%d", reg->off);
1403 			if (type_is_pkt_pointer(t))
1404 				verbose_a("r=%d", reg->range);
1405 			else if (base_type(t) == CONST_PTR_TO_MAP ||
1406 				 base_type(t) == PTR_TO_MAP_KEY ||
1407 				 base_type(t) == PTR_TO_MAP_VALUE)
1408 				verbose_a("ks=%d,vs=%d",
1409 					  reg->map_ptr->key_size,
1410 					  reg->map_ptr->value_size);
1411 			if (tnum_is_const(reg->var_off)) {
1412 				/* Typically an immediate SCALAR_VALUE, but
1413 				 * could be a pointer whose offset is too big
1414 				 * for reg->off
1415 				 */
1416 				verbose_a("imm=%llx", reg->var_off.value);
1417 			} else {
1418 				if (reg->smin_value != reg->umin_value &&
1419 				    reg->smin_value != S64_MIN)
1420 					verbose_a("smin=%lld", (long long)reg->smin_value);
1421 				if (reg->smax_value != reg->umax_value &&
1422 				    reg->smax_value != S64_MAX)
1423 					verbose_a("smax=%lld", (long long)reg->smax_value);
1424 				if (reg->umin_value != 0)
1425 					verbose_a("umin=%llu", (unsigned long long)reg->umin_value);
1426 				if (reg->umax_value != U64_MAX)
1427 					verbose_a("umax=%llu", (unsigned long long)reg->umax_value);
1428 				if (!tnum_is_unknown(reg->var_off)) {
1429 					char tn_buf[48];
1430 
1431 					tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1432 					verbose_a("var_off=%s", tn_buf);
1433 				}
1434 				if (reg->s32_min_value != reg->smin_value &&
1435 				    reg->s32_min_value != S32_MIN)
1436 					verbose_a("s32_min=%d", (int)(reg->s32_min_value));
1437 				if (reg->s32_max_value != reg->smax_value &&
1438 				    reg->s32_max_value != S32_MAX)
1439 					verbose_a("s32_max=%d", (int)(reg->s32_max_value));
1440 				if (reg->u32_min_value != reg->umin_value &&
1441 				    reg->u32_min_value != U32_MIN)
1442 					verbose_a("u32_min=%d", (int)(reg->u32_min_value));
1443 				if (reg->u32_max_value != reg->umax_value &&
1444 				    reg->u32_max_value != U32_MAX)
1445 					verbose_a("u32_max=%d", (int)(reg->u32_max_value));
1446 			}
1447 #undef verbose_a
1448 
1449 			verbose(env, ")");
1450 		}
1451 	}
1452 	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
1453 		char types_buf[BPF_REG_SIZE + 1];
1454 		bool valid = false;
1455 		int j;
1456 
1457 		for (j = 0; j < BPF_REG_SIZE; j++) {
1458 			if (state->stack[i].slot_type[j] != STACK_INVALID)
1459 				valid = true;
1460 			types_buf[j] = slot_type_char[state->stack[i].slot_type[j]];
1461 		}
1462 		types_buf[BPF_REG_SIZE] = 0;
1463 		if (!valid)
1464 			continue;
1465 		if (!print_all && !stack_slot_scratched(env, i))
1466 			continue;
1467 		switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
1468 		case STACK_SPILL:
1469 			reg = &state->stack[i].spilled_ptr;
1470 			t = reg->type;
1471 
1472 			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
1473 			print_liveness(env, reg->live);
1474 			verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
1475 			if (t == SCALAR_VALUE && reg->precise)
1476 				verbose(env, "P");
1477 			if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
1478 				verbose(env, "%lld", reg->var_off.value + reg->off);
1479 			break;
1480 		case STACK_DYNPTR:
1481 			i += BPF_DYNPTR_NR_SLOTS - 1;
1482 			reg = &state->stack[i].spilled_ptr;
1483 
1484 			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
1485 			print_liveness(env, reg->live);
1486 			verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type));
1487 			if (reg->ref_obj_id)
1488 				verbose(env, "(ref_id=%d)", reg->ref_obj_id);
1489 			break;
1490 		case STACK_ITER:
1491 			/* only main slot has ref_obj_id set; skip others */
1492 			reg = &state->stack[i].spilled_ptr;
1493 			if (!reg->ref_obj_id)
1494 				continue;
1495 
1496 			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
1497 			print_liveness(env, reg->live);
1498 			verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)",
1499 				iter_type_str(reg->iter.btf, reg->iter.btf_id),
1500 				reg->ref_obj_id, iter_state_str(reg->iter.state),
1501 				reg->iter.depth);
1502 			break;
1503 		case STACK_MISC:
1504 		case STACK_ZERO:
1505 		default:
1506 			reg = &state->stack[i].spilled_ptr;
1507 
1508 			for (j = 0; j < BPF_REG_SIZE; j++)
1509 				types_buf[j] = slot_type_char[state->stack[i].slot_type[j]];
1510 			types_buf[BPF_REG_SIZE] = 0;
1511 
1512 			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
1513 			print_liveness(env, reg->live);
1514 			verbose(env, "=%s", types_buf);
1515 			break;
1516 		}
1517 	}
1518 	if (state->acquired_refs && state->refs[0].id) {
1519 		verbose(env, " refs=%d", state->refs[0].id);
1520 		for (i = 1; i < state->acquired_refs; i++)
1521 			if (state->refs[i].id)
1522 				verbose(env, ",%d", state->refs[i].id);
1523 	}
1524 	if (state->in_callback_fn)
1525 		verbose(env, " cb");
1526 	if (state->in_async_callback_fn)
1527 		verbose(env, " async_cb");
1528 	verbose(env, "\n");
1529 	if (!print_all)
1530 		mark_verifier_state_clean(env);
1531 }
1532 
1533 static inline u32 vlog_alignment(u32 pos)
1534 {
1535 	return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
1536 			BPF_LOG_MIN_ALIGNMENT) - pos - 1;
1537 }
1538 
1539 static void print_insn_state(struct bpf_verifier_env *env,
1540 			     const struct bpf_func_state *state)
1541 {
1542 	if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
1543 		/* remove new line character */
1544 		bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
1545 		verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' ');
1546 	} else {
1547 		verbose(env, "%d:", env->insn_idx);
1548 	}
1549 	print_verifier_state(env, state, false);
1550 }
1551 
1552 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
1553  * small to hold src. This is different from krealloc since we don't want to preserve
1554  * the contents of dst.
1555  *
1556  * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
1557  * not be allocated.
1558  */
1559 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
1560 {
1561 	size_t alloc_bytes;
1562 	void *orig = dst;
1563 	size_t bytes;
1564 
1565 	if (ZERO_OR_NULL_PTR(src))
1566 		goto out;
1567 
1568 	if (unlikely(check_mul_overflow(n, size, &bytes)))
1569 		return NULL;
1570 
1571 	alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
1572 	dst = krealloc(orig, alloc_bytes, flags);
1573 	if (!dst) {
1574 		kfree(orig);
1575 		return NULL;
1576 	}
1577 
1578 	memcpy(dst, src, bytes);
1579 out:
1580 	return dst ? dst : ZERO_SIZE_PTR;
1581 }
1582 
1583 /* resize an array from old_n items to new_n items. the array is reallocated if it's too
1584  * small to hold new_n items. new items are zeroed out if the array grows.
1585  *
1586  * Contrary to krealloc_array, does not free arr if new_n is zero.
1587  */
1588 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
1589 {
1590 	size_t alloc_size;
1591 	void *new_arr;
1592 
1593 	if (!new_n || old_n == new_n)
1594 		goto out;
1595 
1596 	alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
1597 	new_arr = krealloc(arr, alloc_size, GFP_KERNEL);
1598 	if (!new_arr) {
1599 		kfree(arr);
1600 		return NULL;
1601 	}
1602 	arr = new_arr;
1603 
1604 	if (new_n > old_n)
1605 		memset(arr + old_n * size, 0, (new_n - old_n) * size);
1606 
1607 out:
1608 	return arr ? arr : ZERO_SIZE_PTR;
1609 }
1610 
1611 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
1612 {
1613 	dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
1614 			       sizeof(struct bpf_reference_state), GFP_KERNEL);
1615 	if (!dst->refs)
1616 		return -ENOMEM;
1617 
1618 	dst->acquired_refs = src->acquired_refs;
1619 	return 0;
1620 }
1621 
1622 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
1623 {
1624 	size_t n = src->allocated_stack / BPF_REG_SIZE;
1625 
1626 	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
1627 				GFP_KERNEL);
1628 	if (!dst->stack)
1629 		return -ENOMEM;
1630 
1631 	dst->allocated_stack = src->allocated_stack;
1632 	return 0;
1633 }
1634 
1635 static int resize_reference_state(struct bpf_func_state *state, size_t n)
1636 {
1637 	state->refs = realloc_array(state->refs, state->acquired_refs, n,
1638 				    sizeof(struct bpf_reference_state));
1639 	if (!state->refs)
1640 		return -ENOMEM;
1641 
1642 	state->acquired_refs = n;
1643 	return 0;
1644 }
1645 
1646 /* Possibly update state->allocated_stack to be at least size bytes. Also
1647  * possibly update the function's high-water mark in its bpf_subprog_info.
1648  */
1649 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
1650 {
1651 	size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE;
1652 
1653 	if (old_n >= n)
1654 		return 0;
1655 
1656 	state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
1657 	if (!state->stack)
1658 		return -ENOMEM;
1659 
1660 	state->allocated_stack = size;
1661 
1662 	/* update known max for given subprogram */
1663 	if (env->subprog_info[state->subprogno].stack_depth < size)
1664 		env->subprog_info[state->subprogno].stack_depth = size;
1665 
1666 	return 0;
1667 }
1668 
1669 /* Acquire a pointer id from the env and update the state->refs to include
1670  * this new pointer reference.
1671  * On success, returns a valid pointer id to associate with the register
1672  * On failure, returns a negative errno.
1673  */
1674 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
1675 {
1676 	struct bpf_func_state *state = cur_func(env);
1677 	int new_ofs = state->acquired_refs;
1678 	int id, err;
1679 
1680 	err = resize_reference_state(state, state->acquired_refs + 1);
1681 	if (err)
1682 		return err;
1683 	id = ++env->id_gen;
1684 	state->refs[new_ofs].id = id;
1685 	state->refs[new_ofs].insn_idx = insn_idx;
1686 	state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
1687 
1688 	return id;
1689 }
1690 
1691 /* release function corresponding to acquire_reference_state(). Idempotent. */
1692 static int release_reference_state(struct bpf_func_state *state, int ptr_id)
1693 {
1694 	int i, last_idx;
1695 
1696 	last_idx = state->acquired_refs - 1;
1697 	for (i = 0; i < state->acquired_refs; i++) {
1698 		if (state->refs[i].id == ptr_id) {
1699 			/* Cannot release caller references in callbacks */
1700 			if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
1701 				return -EINVAL;
1702 			if (last_idx && i != last_idx)
1703 				memcpy(&state->refs[i], &state->refs[last_idx],
1704 				       sizeof(*state->refs));
1705 			memset(&state->refs[last_idx], 0, sizeof(*state->refs));
1706 			state->acquired_refs--;
1707 			return 0;
1708 		}
1709 	}
1710 	return -EINVAL;
1711 }
1712 
1713 static void free_func_state(struct bpf_func_state *state)
1714 {
1715 	if (!state)
1716 		return;
1717 	kfree(state->refs);
1718 	kfree(state->stack);
1719 	kfree(state);
1720 }
1721 
1722 static void clear_jmp_history(struct bpf_verifier_state *state)
1723 {
1724 	kfree(state->jmp_history);
1725 	state->jmp_history = NULL;
1726 	state->jmp_history_cnt = 0;
1727 }
1728 
1729 static void free_verifier_state(struct bpf_verifier_state *state,
1730 				bool free_self)
1731 {
1732 	int i;
1733 
1734 	for (i = 0; i <= state->curframe; i++) {
1735 		free_func_state(state->frame[i]);
1736 		state->frame[i] = NULL;
1737 	}
1738 	clear_jmp_history(state);
1739 	if (free_self)
1740 		kfree(state);
1741 }
1742 
1743 /* copy verifier state from src to dst growing dst stack space
1744  * when necessary to accommodate larger src stack
1745  */
1746 static int copy_func_state(struct bpf_func_state *dst,
1747 			   const struct bpf_func_state *src)
1748 {
1749 	int err;
1750 
1751 	memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
1752 	err = copy_reference_state(dst, src);
1753 	if (err)
1754 		return err;
1755 	return copy_stack_state(dst, src);
1756 }
1757 
1758 static int copy_verifier_state(struct bpf_verifier_state *dst_state,
1759 			       const struct bpf_verifier_state *src)
1760 {
1761 	struct bpf_func_state *dst;
1762 	int i, err;
1763 
1764 	dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
1765 					    src->jmp_history_cnt, sizeof(struct bpf_idx_pair),
1766 					    GFP_USER);
1767 	if (!dst_state->jmp_history)
1768 		return -ENOMEM;
1769 	dst_state->jmp_history_cnt = src->jmp_history_cnt;
1770 
1771 	/* if dst has more stack frames then src frame, free them */
1772 	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
1773 		free_func_state(dst_state->frame[i]);
1774 		dst_state->frame[i] = NULL;
1775 	}
1776 	dst_state->speculative = src->speculative;
1777 	dst_state->active_rcu_lock = src->active_rcu_lock;
1778 	dst_state->curframe = src->curframe;
1779 	dst_state->active_lock.ptr = src->active_lock.ptr;
1780 	dst_state->active_lock.id = src->active_lock.id;
1781 	dst_state->branches = src->branches;
1782 	dst_state->parent = src->parent;
1783 	dst_state->first_insn_idx = src->first_insn_idx;
1784 	dst_state->last_insn_idx = src->last_insn_idx;
1785 	dst_state->dfs_depth = src->dfs_depth;
1786 	dst_state->callback_unroll_depth = src->callback_unroll_depth;
1787 	dst_state->used_as_loop_entry = src->used_as_loop_entry;
1788 	for (i = 0; i <= src->curframe; i++) {
1789 		dst = dst_state->frame[i];
1790 		if (!dst) {
1791 			dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1792 			if (!dst)
1793 				return -ENOMEM;
1794 			dst_state->frame[i] = dst;
1795 		}
1796 		err = copy_func_state(dst, src->frame[i]);
1797 		if (err)
1798 			return err;
1799 	}
1800 	return 0;
1801 }
1802 
1803 static u32 state_htab_size(struct bpf_verifier_env *env)
1804 {
1805 	return env->prog->len;
1806 }
1807 
1808 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx)
1809 {
1810 	struct bpf_verifier_state *cur = env->cur_state;
1811 	struct bpf_func_state *state = cur->frame[cur->curframe];
1812 
1813 	return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
1814 }
1815 
1816 static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_state *b)
1817 {
1818 	int fr;
1819 
1820 	if (a->curframe != b->curframe)
1821 		return false;
1822 
1823 	for (fr = a->curframe; fr >= 0; fr--)
1824 		if (a->frame[fr]->callsite != b->frame[fr]->callsite)
1825 			return false;
1826 
1827 	return true;
1828 }
1829 
1830 /* Open coded iterators allow back-edges in the state graph in order to
1831  * check unbounded loops that iterators.
1832  *
1833  * In is_state_visited() it is necessary to know if explored states are
1834  * part of some loops in order to decide whether non-exact states
1835  * comparison could be used:
1836  * - non-exact states comparison establishes sub-state relation and uses
1837  *   read and precision marks to do so, these marks are propagated from
1838  *   children states and thus are not guaranteed to be final in a loop;
1839  * - exact states comparison just checks if current and explored states
1840  *   are identical (and thus form a back-edge).
1841  *
1842  * Paper "A New Algorithm for Identifying Loops in Decompilation"
1843  * by Tao Wei, Jian Mao, Wei Zou and Yu Chen [1] presents a convenient
1844  * algorithm for loop structure detection and gives an overview of
1845  * relevant terminology. It also has helpful illustrations.
1846  *
1847  * [1] https://api.semanticscholar.org/CorpusID:15784067
1848  *
1849  * We use a similar algorithm but because loop nested structure is
1850  * irrelevant for verifier ours is significantly simpler and resembles
1851  * strongly connected components algorithm from Sedgewick's textbook.
1852  *
1853  * Define topmost loop entry as a first node of the loop traversed in a
1854  * depth first search starting from initial state. The goal of the loop
1855  * tracking algorithm is to associate topmost loop entries with states
1856  * derived from these entries.
1857  *
1858  * For each step in the DFS states traversal algorithm needs to identify
1859  * the following situations:
1860  *
1861  *          initial                     initial                   initial
1862  *            |                           |                         |
1863  *            V                           V                         V
1864  *           ...                         ...           .---------> hdr
1865  *            |                           |            |            |
1866  *            V                           V            |            V
1867  *           cur                     .-> succ          |    .------...
1868  *            |                      |    |            |    |       |
1869  *            V                      |    V            |    V       V
1870  *           succ                    '-- cur           |   ...     ...
1871  *                                                     |    |       |
1872  *                                                     |    V       V
1873  *                                                     |   succ <- cur
1874  *                                                     |    |
1875  *                                                     |    V
1876  *                                                     |   ...
1877  *                                                     |    |
1878  *                                                     '----'
1879  *
1880  *  (A) successor state of cur   (B) successor state of cur or it's entry
1881  *      not yet traversed            are in current DFS path, thus cur and succ
1882  *                                   are members of the same outermost loop
1883  *
1884  *                      initial                  initial
1885  *                        |                        |
1886  *                        V                        V
1887  *                       ...                      ...
1888  *                        |                        |
1889  *                        V                        V
1890  *                .------...               .------...
1891  *                |       |                |       |
1892  *                V       V                V       V
1893  *           .-> hdr     ...              ...     ...
1894  *           |    |       |                |       |
1895  *           |    V       V                V       V
1896  *           |   succ <- cur              succ <- cur
1897  *           |    |                        |
1898  *           |    V                        V
1899  *           |   ...                      ...
1900  *           |    |                        |
1901  *           '----'                       exit
1902  *
1903  * (C) successor state of cur is a part of some loop but this loop
1904  *     does not include cur or successor state is not in a loop at all.
1905  *
1906  * Algorithm could be described as the following python code:
1907  *
1908  *     traversed = set()   # Set of traversed nodes
1909  *     entries = {}        # Mapping from node to loop entry
1910  *     depths = {}         # Depth level assigned to graph node
1911  *     path = set()        # Current DFS path
1912  *
1913  *     # Find outermost loop entry known for n
1914  *     def get_loop_entry(n):
1915  *         h = entries.get(n, None)
1916  *         while h in entries and entries[h] != h:
1917  *             h = entries[h]
1918  *         return h
1919  *
1920  *     # Update n's loop entry if h's outermost entry comes
1921  *     # before n's outermost entry in current DFS path.
1922  *     def update_loop_entry(n, h):
1923  *         n1 = get_loop_entry(n) or n
1924  *         h1 = get_loop_entry(h) or h
1925  *         if h1 in path and depths[h1] <= depths[n1]:
1926  *             entries[n] = h1
1927  *
1928  *     def dfs(n, depth):
1929  *         traversed.add(n)
1930  *         path.add(n)
1931  *         depths[n] = depth
1932  *         for succ in G.successors(n):
1933  *             if succ not in traversed:
1934  *                 # Case A: explore succ and update cur's loop entry
1935  *                 #         only if succ's entry is in current DFS path.
1936  *                 dfs(succ, depth + 1)
1937  *                 h = get_loop_entry(succ)
1938  *                 update_loop_entry(n, h)
1939  *             else:
1940  *                 # Case B or C depending on `h1 in path` check in update_loop_entry().
1941  *                 update_loop_entry(n, succ)
1942  *         path.remove(n)
1943  *
1944  * To adapt this algorithm for use with verifier:
1945  * - use st->branch == 0 as a signal that DFS of succ had been finished
1946  *   and cur's loop entry has to be updated (case A), handle this in
1947  *   update_branch_counts();
1948  * - use st->branch > 0 as a signal that st is in the current DFS path;
1949  * - handle cases B and C in is_state_visited();
1950  * - update topmost loop entry for intermediate states in get_loop_entry().
1951  */
1952 static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_state *st)
1953 {
1954 	struct bpf_verifier_state *topmost = st->loop_entry, *old;
1955 
1956 	while (topmost && topmost->loop_entry && topmost != topmost->loop_entry)
1957 		topmost = topmost->loop_entry;
1958 	/* Update loop entries for intermediate states to avoid this
1959 	 * traversal in future get_loop_entry() calls.
1960 	 */
1961 	while (st && st->loop_entry != topmost) {
1962 		old = st->loop_entry;
1963 		st->loop_entry = topmost;
1964 		st = old;
1965 	}
1966 	return topmost;
1967 }
1968 
1969 static void update_loop_entry(struct bpf_verifier_state *cur, struct bpf_verifier_state *hdr)
1970 {
1971 	struct bpf_verifier_state *cur1, *hdr1;
1972 
1973 	cur1 = get_loop_entry(cur) ?: cur;
1974 	hdr1 = get_loop_entry(hdr) ?: hdr;
1975 	/* The head1->branches check decides between cases B and C in
1976 	 * comment for get_loop_entry(). If hdr1->branches == 0 then
1977 	 * head's topmost loop entry is not in current DFS path,
1978 	 * hence 'cur' and 'hdr' are not in the same loop and there is
1979 	 * no need to update cur->loop_entry.
1980 	 */
1981 	if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) {
1982 		cur->loop_entry = hdr;
1983 		hdr->used_as_loop_entry = true;
1984 	}
1985 }
1986 
1987 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
1988 {
1989 	while (st) {
1990 		u32 br = --st->branches;
1991 
1992 		/* br == 0 signals that DFS exploration for 'st' is finished,
1993 		 * thus it is necessary to update parent's loop entry if it
1994 		 * turned out that st is a part of some loop.
1995 		 * This is a part of 'case A' in get_loop_entry() comment.
1996 		 */
1997 		if (br == 0 && st->parent && st->loop_entry)
1998 			update_loop_entry(st->parent, st->loop_entry);
1999 
2000 		/* WARN_ON(br > 1) technically makes sense here,
2001 		 * but see comment in push_stack(), hence:
2002 		 */
2003 		WARN_ONCE((int)br < 0,
2004 			  "BUG update_branch_counts:branches_to_explore=%d\n",
2005 			  br);
2006 		if (br)
2007 			break;
2008 		st = st->parent;
2009 	}
2010 }
2011 
2012 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
2013 		     int *insn_idx, bool pop_log)
2014 {
2015 	struct bpf_verifier_state *cur = env->cur_state;
2016 	struct bpf_verifier_stack_elem *elem, *head = env->head;
2017 	int err;
2018 
2019 	if (env->head == NULL)
2020 		return -ENOENT;
2021 
2022 	if (cur) {
2023 		err = copy_verifier_state(cur, &head->st);
2024 		if (err)
2025 			return err;
2026 	}
2027 	if (pop_log)
2028 		bpf_vlog_reset(&env->log, head->log_pos);
2029 	if (insn_idx)
2030 		*insn_idx = head->insn_idx;
2031 	if (prev_insn_idx)
2032 		*prev_insn_idx = head->prev_insn_idx;
2033 	elem = head->next;
2034 	free_verifier_state(&head->st, false);
2035 	kfree(head);
2036 	env->head = elem;
2037 	env->stack_size--;
2038 	return 0;
2039 }
2040 
2041 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
2042 					     int insn_idx, int prev_insn_idx,
2043 					     bool speculative)
2044 {
2045 	struct bpf_verifier_state *cur = env->cur_state;
2046 	struct bpf_verifier_stack_elem *elem;
2047 	int err;
2048 
2049 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
2050 	if (!elem)
2051 		goto err;
2052 
2053 	elem->insn_idx = insn_idx;
2054 	elem->prev_insn_idx = prev_insn_idx;
2055 	elem->next = env->head;
2056 	elem->log_pos = env->log.end_pos;
2057 	env->head = elem;
2058 	env->stack_size++;
2059 	err = copy_verifier_state(&elem->st, cur);
2060 	if (err)
2061 		goto err;
2062 	elem->st.speculative |= speculative;
2063 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
2064 		verbose(env, "The sequence of %d jumps is too complex.\n",
2065 			env->stack_size);
2066 		goto err;
2067 	}
2068 	if (elem->st.parent) {
2069 		++elem->st.parent->branches;
2070 		/* WARN_ON(branches > 2) technically makes sense here,
2071 		 * but
2072 		 * 1. speculative states will bump 'branches' for non-branch
2073 		 * instructions
2074 		 * 2. is_state_visited() heuristics may decide not to create
2075 		 * a new state for a sequence of branches and all such current
2076 		 * and cloned states will be pointing to a single parent state
2077 		 * which might have large 'branches' count.
2078 		 */
2079 	}
2080 	return &elem->st;
2081 err:
2082 	free_verifier_state(env->cur_state, true);
2083 	env->cur_state = NULL;
2084 	/* pop all elements and return */
2085 	while (!pop_stack(env, NULL, NULL, false));
2086 	return NULL;
2087 }
2088 
2089 #define CALLER_SAVED_REGS 6
2090 static const int caller_saved[CALLER_SAVED_REGS] = {
2091 	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
2092 };
2093 
2094 /* This helper doesn't clear reg->id */
2095 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
2096 {
2097 	reg->var_off = tnum_const(imm);
2098 	reg->smin_value = (s64)imm;
2099 	reg->smax_value = (s64)imm;
2100 	reg->umin_value = imm;
2101 	reg->umax_value = imm;
2102 
2103 	reg->s32_min_value = (s32)imm;
2104 	reg->s32_max_value = (s32)imm;
2105 	reg->u32_min_value = (u32)imm;
2106 	reg->u32_max_value = (u32)imm;
2107 }
2108 
2109 /* Mark the unknown part of a register (variable offset or scalar value) as
2110  * known to have the value @imm.
2111  */
2112 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
2113 {
2114 	/* Clear off and union(map_ptr, range) */
2115 	memset(((u8 *)reg) + sizeof(reg->type), 0,
2116 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
2117 	reg->id = 0;
2118 	reg->ref_obj_id = 0;
2119 	___mark_reg_known(reg, imm);
2120 }
2121 
2122 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
2123 {
2124 	reg->var_off = tnum_const_subreg(reg->var_off, imm);
2125 	reg->s32_min_value = (s32)imm;
2126 	reg->s32_max_value = (s32)imm;
2127 	reg->u32_min_value = (u32)imm;
2128 	reg->u32_max_value = (u32)imm;
2129 }
2130 
2131 /* Mark the 'variable offset' part of a register as zero.  This should be
2132  * used only on registers holding a pointer type.
2133  */
2134 static void __mark_reg_known_zero(struct bpf_reg_state *reg)
2135 {
2136 	__mark_reg_known(reg, 0);
2137 }
2138 
2139 static void __mark_reg_const_zero(struct bpf_reg_state *reg)
2140 {
2141 	__mark_reg_known(reg, 0);
2142 	reg->type = SCALAR_VALUE;
2143 }
2144 
2145 static void mark_reg_known_zero(struct bpf_verifier_env *env,
2146 				struct bpf_reg_state *regs, u32 regno)
2147 {
2148 	if (WARN_ON(regno >= MAX_BPF_REG)) {
2149 		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
2150 		/* Something bad happened, let's kill all regs */
2151 		for (regno = 0; regno < MAX_BPF_REG; regno++)
2152 			__mark_reg_not_init(env, regs + regno);
2153 		return;
2154 	}
2155 	__mark_reg_known_zero(regs + regno);
2156 }
2157 
2158 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
2159 			      bool first_slot, int dynptr_id)
2160 {
2161 	/* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
2162 	 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
2163 	 * set it unconditionally as it is ignored for STACK_DYNPTR anyway.
2164 	 */
2165 	__mark_reg_known_zero(reg);
2166 	reg->type = CONST_PTR_TO_DYNPTR;
2167 	/* Give each dynptr a unique id to uniquely associate slices to it. */
2168 	reg->id = dynptr_id;
2169 	reg->dynptr.type = type;
2170 	reg->dynptr.first_slot = first_slot;
2171 }
2172 
2173 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
2174 {
2175 	if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
2176 		const struct bpf_map *map = reg->map_ptr;
2177 
2178 		if (map->inner_map_meta) {
2179 			reg->type = CONST_PTR_TO_MAP;
2180 			reg->map_ptr = map->inner_map_meta;
2181 			/* transfer reg's id which is unique for every map_lookup_elem
2182 			 * as UID of the inner map.
2183 			 */
2184 			if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER))
2185 				reg->map_uid = reg->id;
2186 		} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
2187 			reg->type = PTR_TO_XDP_SOCK;
2188 		} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
2189 			   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
2190 			reg->type = PTR_TO_SOCKET;
2191 		} else {
2192 			reg->type = PTR_TO_MAP_VALUE;
2193 		}
2194 		return;
2195 	}
2196 
2197 	reg->type &= ~PTR_MAYBE_NULL;
2198 }
2199 
2200 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno,
2201 				struct btf_field_graph_root *ds_head)
2202 {
2203 	__mark_reg_known_zero(&regs[regno]);
2204 	regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC;
2205 	regs[regno].btf = ds_head->btf;
2206 	regs[regno].btf_id = ds_head->value_btf_id;
2207 	regs[regno].off = ds_head->node_offset;
2208 }
2209 
2210 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
2211 {
2212 	return type_is_pkt_pointer(reg->type);
2213 }
2214 
2215 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
2216 {
2217 	return reg_is_pkt_pointer(reg) ||
2218 	       reg->type == PTR_TO_PACKET_END;
2219 }
2220 
2221 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg)
2222 {
2223 	return base_type(reg->type) == PTR_TO_MEM &&
2224 		(reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP);
2225 }
2226 
2227 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
2228 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
2229 				    enum bpf_reg_type which)
2230 {
2231 	/* The register can already have a range from prior markings.
2232 	 * This is fine as long as it hasn't been advanced from its
2233 	 * origin.
2234 	 */
2235 	return reg->type == which &&
2236 	       reg->id == 0 &&
2237 	       reg->off == 0 &&
2238 	       tnum_equals_const(reg->var_off, 0);
2239 }
2240 
2241 /* Reset the min/max bounds of a register */
2242 static void __mark_reg_unbounded(struct bpf_reg_state *reg)
2243 {
2244 	reg->smin_value = S64_MIN;
2245 	reg->smax_value = S64_MAX;
2246 	reg->umin_value = 0;
2247 	reg->umax_value = U64_MAX;
2248 
2249 	reg->s32_min_value = S32_MIN;
2250 	reg->s32_max_value = S32_MAX;
2251 	reg->u32_min_value = 0;
2252 	reg->u32_max_value = U32_MAX;
2253 }
2254 
2255 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
2256 {
2257 	reg->smin_value = S64_MIN;
2258 	reg->smax_value = S64_MAX;
2259 	reg->umin_value = 0;
2260 	reg->umax_value = U64_MAX;
2261 }
2262 
2263 static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
2264 {
2265 	reg->s32_min_value = S32_MIN;
2266 	reg->s32_max_value = S32_MAX;
2267 	reg->u32_min_value = 0;
2268 	reg->u32_max_value = U32_MAX;
2269 }
2270 
2271 static void __update_reg32_bounds(struct bpf_reg_state *reg)
2272 {
2273 	struct tnum var32_off = tnum_subreg(reg->var_off);
2274 
2275 	/* min signed is max(sign bit) | min(other bits) */
2276 	reg->s32_min_value = max_t(s32, reg->s32_min_value,
2277 			var32_off.value | (var32_off.mask & S32_MIN));
2278 	/* max signed is min(sign bit) | max(other bits) */
2279 	reg->s32_max_value = min_t(s32, reg->s32_max_value,
2280 			var32_off.value | (var32_off.mask & S32_MAX));
2281 	reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
2282 	reg->u32_max_value = min(reg->u32_max_value,
2283 				 (u32)(var32_off.value | var32_off.mask));
2284 }
2285 
2286 static void __update_reg64_bounds(struct bpf_reg_state *reg)
2287 {
2288 	/* min signed is max(sign bit) | min(other bits) */
2289 	reg->smin_value = max_t(s64, reg->smin_value,
2290 				reg->var_off.value | (reg->var_off.mask & S64_MIN));
2291 	/* max signed is min(sign bit) | max(other bits) */
2292 	reg->smax_value = min_t(s64, reg->smax_value,
2293 				reg->var_off.value | (reg->var_off.mask & S64_MAX));
2294 	reg->umin_value = max(reg->umin_value, reg->var_off.value);
2295 	reg->umax_value = min(reg->umax_value,
2296 			      reg->var_off.value | reg->var_off.mask);
2297 }
2298 
2299 static void __update_reg_bounds(struct bpf_reg_state *reg)
2300 {
2301 	__update_reg32_bounds(reg);
2302 	__update_reg64_bounds(reg);
2303 }
2304 
2305 /* Uses signed min/max values to inform unsigned, and vice-versa */
2306 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
2307 {
2308 	/* Learn sign from signed bounds.
2309 	 * If we cannot cross the sign boundary, then signed and unsigned bounds
2310 	 * are the same, so combine.  This works even in the negative case, e.g.
2311 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
2312 	 */
2313 	if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) {
2314 		reg->s32_min_value = reg->u32_min_value =
2315 			max_t(u32, reg->s32_min_value, reg->u32_min_value);
2316 		reg->s32_max_value = reg->u32_max_value =
2317 			min_t(u32, reg->s32_max_value, reg->u32_max_value);
2318 		return;
2319 	}
2320 	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
2321 	 * boundary, so we must be careful.
2322 	 */
2323 	if ((s32)reg->u32_max_value >= 0) {
2324 		/* Positive.  We can't learn anything from the smin, but smax
2325 		 * is positive, hence safe.
2326 		 */
2327 		reg->s32_min_value = reg->u32_min_value;
2328 		reg->s32_max_value = reg->u32_max_value =
2329 			min_t(u32, reg->s32_max_value, reg->u32_max_value);
2330 	} else if ((s32)reg->u32_min_value < 0) {
2331 		/* Negative.  We can't learn anything from the smax, but smin
2332 		 * is negative, hence safe.
2333 		 */
2334 		reg->s32_min_value = reg->u32_min_value =
2335 			max_t(u32, reg->s32_min_value, reg->u32_min_value);
2336 		reg->s32_max_value = reg->u32_max_value;
2337 	}
2338 }
2339 
2340 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
2341 {
2342 	/* Learn sign from signed bounds.
2343 	 * If we cannot cross the sign boundary, then signed and unsigned bounds
2344 	 * are the same, so combine.  This works even in the negative case, e.g.
2345 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
2346 	 */
2347 	if (reg->smin_value >= 0 || reg->smax_value < 0) {
2348 		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
2349 							  reg->umin_value);
2350 		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
2351 							  reg->umax_value);
2352 		return;
2353 	}
2354 	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
2355 	 * boundary, so we must be careful.
2356 	 */
2357 	if ((s64)reg->umax_value >= 0) {
2358 		/* Positive.  We can't learn anything from the smin, but smax
2359 		 * is positive, hence safe.
2360 		 */
2361 		reg->smin_value = reg->umin_value;
2362 		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
2363 							  reg->umax_value);
2364 	} else if ((s64)reg->umin_value < 0) {
2365 		/* Negative.  We can't learn anything from the smax, but smin
2366 		 * is negative, hence safe.
2367 		 */
2368 		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
2369 							  reg->umin_value);
2370 		reg->smax_value = reg->umax_value;
2371 	}
2372 }
2373 
2374 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
2375 {
2376 	__reg32_deduce_bounds(reg);
2377 	__reg64_deduce_bounds(reg);
2378 }
2379 
2380 /* Attempts to improve var_off based on unsigned min/max information */
2381 static void __reg_bound_offset(struct bpf_reg_state *reg)
2382 {
2383 	struct tnum var64_off = tnum_intersect(reg->var_off,
2384 					       tnum_range(reg->umin_value,
2385 							  reg->umax_value));
2386 	struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
2387 					       tnum_range(reg->u32_min_value,
2388 							  reg->u32_max_value));
2389 
2390 	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
2391 }
2392 
2393 static void reg_bounds_sync(struct bpf_reg_state *reg)
2394 {
2395 	/* We might have learned new bounds from the var_off. */
2396 	__update_reg_bounds(reg);
2397 	/* We might have learned something about the sign bit. */
2398 	__reg_deduce_bounds(reg);
2399 	/* We might have learned some bits from the bounds. */
2400 	__reg_bound_offset(reg);
2401 	/* Intersecting with the old var_off might have improved our bounds
2402 	 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
2403 	 * then new var_off is (0; 0x7f...fc) which improves our umax.
2404 	 */
2405 	__update_reg_bounds(reg);
2406 }
2407 
2408 static bool __reg32_bound_s64(s32 a)
2409 {
2410 	return a >= 0 && a <= S32_MAX;
2411 }
2412 
2413 static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
2414 {
2415 	reg->umin_value = reg->u32_min_value;
2416 	reg->umax_value = reg->u32_max_value;
2417 
2418 	/* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
2419 	 * be positive otherwise set to worse case bounds and refine later
2420 	 * from tnum.
2421 	 */
2422 	if (__reg32_bound_s64(reg->s32_min_value) &&
2423 	    __reg32_bound_s64(reg->s32_max_value)) {
2424 		reg->smin_value = reg->s32_min_value;
2425 		reg->smax_value = reg->s32_max_value;
2426 	} else {
2427 		reg->smin_value = 0;
2428 		reg->smax_value = U32_MAX;
2429 	}
2430 }
2431 
2432 static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
2433 {
2434 	/* special case when 64-bit register has upper 32-bit register
2435 	 * zeroed. Typically happens after zext or <<32, >>32 sequence
2436 	 * allowing us to use 32-bit bounds directly,
2437 	 */
2438 	if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) {
2439 		__reg_assign_32_into_64(reg);
2440 	} else {
2441 		/* Otherwise the best we can do is push lower 32bit known and
2442 		 * unknown bits into register (var_off set from jmp logic)
2443 		 * then learn as much as possible from the 64-bit tnum
2444 		 * known and unknown bits. The previous smin/smax bounds are
2445 		 * invalid here because of jmp32 compare so mark them unknown
2446 		 * so they do not impact tnum bounds calculation.
2447 		 */
2448 		__mark_reg64_unbounded(reg);
2449 	}
2450 	reg_bounds_sync(reg);
2451 }
2452 
2453 static bool __reg64_bound_s32(s64 a)
2454 {
2455 	return a >= S32_MIN && a <= S32_MAX;
2456 }
2457 
2458 static bool __reg64_bound_u32(u64 a)
2459 {
2460 	return a >= U32_MIN && a <= U32_MAX;
2461 }
2462 
2463 static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
2464 {
2465 	__mark_reg32_unbounded(reg);
2466 	if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) {
2467 		reg->s32_min_value = (s32)reg->smin_value;
2468 		reg->s32_max_value = (s32)reg->smax_value;
2469 	}
2470 	if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) {
2471 		reg->u32_min_value = (u32)reg->umin_value;
2472 		reg->u32_max_value = (u32)reg->umax_value;
2473 	}
2474 	reg_bounds_sync(reg);
2475 }
2476 
2477 /* Mark a register as having a completely unknown (scalar) value. */
2478 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
2479 			       struct bpf_reg_state *reg)
2480 {
2481 	/*
2482 	 * Clear type, off, and union(map_ptr, range) and
2483 	 * padding between 'type' and union
2484 	 */
2485 	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
2486 	reg->type = SCALAR_VALUE;
2487 	reg->id = 0;
2488 	reg->ref_obj_id = 0;
2489 	reg->var_off = tnum_unknown;
2490 	reg->frameno = 0;
2491 	reg->precise = !env->bpf_capable;
2492 	__mark_reg_unbounded(reg);
2493 }
2494 
2495 static void mark_reg_unknown(struct bpf_verifier_env *env,
2496 			     struct bpf_reg_state *regs, u32 regno)
2497 {
2498 	if (WARN_ON(regno >= MAX_BPF_REG)) {
2499 		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
2500 		/* Something bad happened, let's kill all regs except FP */
2501 		for (regno = 0; regno < BPF_REG_FP; regno++)
2502 			__mark_reg_not_init(env, regs + regno);
2503 		return;
2504 	}
2505 	__mark_reg_unknown(env, regs + regno);
2506 }
2507 
2508 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
2509 				struct bpf_reg_state *reg)
2510 {
2511 	__mark_reg_unknown(env, reg);
2512 	reg->type = NOT_INIT;
2513 }
2514 
2515 static void mark_reg_not_init(struct bpf_verifier_env *env,
2516 			      struct bpf_reg_state *regs, u32 regno)
2517 {
2518 	if (WARN_ON(regno >= MAX_BPF_REG)) {
2519 		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
2520 		/* Something bad happened, let's kill all regs except FP */
2521 		for (regno = 0; regno < BPF_REG_FP; regno++)
2522 			__mark_reg_not_init(env, regs + regno);
2523 		return;
2524 	}
2525 	__mark_reg_not_init(env, regs + regno);
2526 }
2527 
2528 static void mark_btf_ld_reg(struct bpf_verifier_env *env,
2529 			    struct bpf_reg_state *regs, u32 regno,
2530 			    enum bpf_reg_type reg_type,
2531 			    struct btf *btf, u32 btf_id,
2532 			    enum bpf_type_flag flag)
2533 {
2534 	if (reg_type == SCALAR_VALUE) {
2535 		mark_reg_unknown(env, regs, regno);
2536 		return;
2537 	}
2538 	mark_reg_known_zero(env, regs, regno);
2539 	regs[regno].type = PTR_TO_BTF_ID | flag;
2540 	regs[regno].btf = btf;
2541 	regs[regno].btf_id = btf_id;
2542 	if (type_may_be_null(flag))
2543 		regs[regno].id = ++env->id_gen;
2544 }
2545 
2546 #define DEF_NOT_SUBREG	(0)
2547 static void init_reg_state(struct bpf_verifier_env *env,
2548 			   struct bpf_func_state *state)
2549 {
2550 	struct bpf_reg_state *regs = state->regs;
2551 	int i;
2552 
2553 	for (i = 0; i < MAX_BPF_REG; i++) {
2554 		mark_reg_not_init(env, regs, i);
2555 		regs[i].live = REG_LIVE_NONE;
2556 		regs[i].parent = NULL;
2557 		regs[i].subreg_def = DEF_NOT_SUBREG;
2558 	}
2559 
2560 	/* frame pointer */
2561 	regs[BPF_REG_FP].type = PTR_TO_STACK;
2562 	mark_reg_known_zero(env, regs, BPF_REG_FP);
2563 	regs[BPF_REG_FP].frameno = state->frameno;
2564 }
2565 
2566 #define BPF_MAIN_FUNC (-1)
2567 static void init_func_state(struct bpf_verifier_env *env,
2568 			    struct bpf_func_state *state,
2569 			    int callsite, int frameno, int subprogno)
2570 {
2571 	state->callsite = callsite;
2572 	state->frameno = frameno;
2573 	state->subprogno = subprogno;
2574 	state->callback_ret_range = tnum_range(0, 0);
2575 	init_reg_state(env, state);
2576 	mark_verifier_state_scratched(env);
2577 }
2578 
2579 /* Similar to push_stack(), but for async callbacks */
2580 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
2581 						int insn_idx, int prev_insn_idx,
2582 						int subprog)
2583 {
2584 	struct bpf_verifier_stack_elem *elem;
2585 	struct bpf_func_state *frame;
2586 
2587 	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
2588 	if (!elem)
2589 		goto err;
2590 
2591 	elem->insn_idx = insn_idx;
2592 	elem->prev_insn_idx = prev_insn_idx;
2593 	elem->next = env->head;
2594 	elem->log_pos = env->log.end_pos;
2595 	env->head = elem;
2596 	env->stack_size++;
2597 	if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
2598 		verbose(env,
2599 			"The sequence of %d jumps is too complex for async cb.\n",
2600 			env->stack_size);
2601 		goto err;
2602 	}
2603 	/* Unlike push_stack() do not copy_verifier_state().
2604 	 * The caller state doesn't matter.
2605 	 * This is async callback. It starts in a fresh stack.
2606 	 * Initialize it similar to do_check_common().
2607 	 */
2608 	elem->st.branches = 1;
2609 	frame = kzalloc(sizeof(*frame), GFP_KERNEL);
2610 	if (!frame)
2611 		goto err;
2612 	init_func_state(env, frame,
2613 			BPF_MAIN_FUNC /* callsite */,
2614 			0 /* frameno within this callchain */,
2615 			subprog /* subprog number within this prog */);
2616 	elem->st.frame[0] = frame;
2617 	return &elem->st;
2618 err:
2619 	free_verifier_state(env->cur_state, true);
2620 	env->cur_state = NULL;
2621 	/* pop all elements and return */
2622 	while (!pop_stack(env, NULL, NULL, false));
2623 	return NULL;
2624 }
2625 
2626 
2627 enum reg_arg_type {
2628 	SRC_OP,		/* register is used as source operand */
2629 	DST_OP,		/* register is used as destination operand */
2630 	DST_OP_NO_MARK	/* same as above, check only, don't mark */
2631 };
2632 
2633 static int cmp_subprogs(const void *a, const void *b)
2634 {
2635 	return ((struct bpf_subprog_info *)a)->start -
2636 	       ((struct bpf_subprog_info *)b)->start;
2637 }
2638 
2639 static int find_subprog(struct bpf_verifier_env *env, int off)
2640 {
2641 	struct bpf_subprog_info *p;
2642 
2643 	p = bsearch(&off, env->subprog_info, env->subprog_cnt,
2644 		    sizeof(env->subprog_info[0]), cmp_subprogs);
2645 	if (!p)
2646 		return -ENOENT;
2647 	return p - env->subprog_info;
2648 
2649 }
2650 
2651 static int add_subprog(struct bpf_verifier_env *env, int off)
2652 {
2653 	int insn_cnt = env->prog->len;
2654 	int ret;
2655 
2656 	if (off >= insn_cnt || off < 0) {
2657 		verbose(env, "call to invalid destination\n");
2658 		return -EINVAL;
2659 	}
2660 	ret = find_subprog(env, off);
2661 	if (ret >= 0)
2662 		return ret;
2663 	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
2664 		verbose(env, "too many subprograms\n");
2665 		return -E2BIG;
2666 	}
2667 	/* determine subprog starts. The end is one before the next starts */
2668 	env->subprog_info[env->subprog_cnt++].start = off;
2669 	sort(env->subprog_info, env->subprog_cnt,
2670 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
2671 	return env->subprog_cnt - 1;
2672 }
2673 
2674 #define MAX_KFUNC_DESCS 256
2675 #define MAX_KFUNC_BTFS	256
2676 
2677 struct bpf_kfunc_desc {
2678 	struct btf_func_model func_model;
2679 	u32 func_id;
2680 	s32 imm;
2681 	u16 offset;
2682 	unsigned long addr;
2683 };
2684 
2685 struct bpf_kfunc_btf {
2686 	struct btf *btf;
2687 	struct module *module;
2688 	u16 offset;
2689 };
2690 
2691 struct bpf_kfunc_desc_tab {
2692 	/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
2693 	 * verification. JITs do lookups by bpf_insn, where func_id may not be
2694 	 * available, therefore at the end of verification do_misc_fixups()
2695 	 * sorts this by imm and offset.
2696 	 */
2697 	struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
2698 	u32 nr_descs;
2699 };
2700 
2701 struct bpf_kfunc_btf_tab {
2702 	struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
2703 	u32 nr_descs;
2704 };
2705 
2706 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
2707 {
2708 	const struct bpf_kfunc_desc *d0 = a;
2709 	const struct bpf_kfunc_desc *d1 = b;
2710 
2711 	/* func_id is not greater than BTF_MAX_TYPE */
2712 	return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
2713 }
2714 
2715 static int kfunc_btf_cmp_by_off(const void *a, const void *b)
2716 {
2717 	const struct bpf_kfunc_btf *d0 = a;
2718 	const struct bpf_kfunc_btf *d1 = b;
2719 
2720 	return d0->offset - d1->offset;
2721 }
2722 
2723 static const struct bpf_kfunc_desc *
2724 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
2725 {
2726 	struct bpf_kfunc_desc desc = {
2727 		.func_id = func_id,
2728 		.offset = offset,
2729 	};
2730 	struct bpf_kfunc_desc_tab *tab;
2731 
2732 	tab = prog->aux->kfunc_tab;
2733 	return bsearch(&desc, tab->descs, tab->nr_descs,
2734 		       sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
2735 }
2736 
2737 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2738 		       u16 btf_fd_idx, u8 **func_addr)
2739 {
2740 	const struct bpf_kfunc_desc *desc;
2741 
2742 	desc = find_kfunc_desc(prog, func_id, btf_fd_idx);
2743 	if (!desc)
2744 		return -EFAULT;
2745 
2746 	*func_addr = (u8 *)desc->addr;
2747 	return 0;
2748 }
2749 
2750 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
2751 					 s16 offset)
2752 {
2753 	struct bpf_kfunc_btf kf_btf = { .offset = offset };
2754 	struct bpf_kfunc_btf_tab *tab;
2755 	struct bpf_kfunc_btf *b;
2756 	struct module *mod;
2757 	struct btf *btf;
2758 	int btf_fd;
2759 
2760 	tab = env->prog->aux->kfunc_btf_tab;
2761 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
2762 		    sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
2763 	if (!b) {
2764 		if (tab->nr_descs == MAX_KFUNC_BTFS) {
2765 			verbose(env, "too many different module BTFs\n");
2766 			return ERR_PTR(-E2BIG);
2767 		}
2768 
2769 		if (bpfptr_is_null(env->fd_array)) {
2770 			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
2771 			return ERR_PTR(-EPROTO);
2772 		}
2773 
2774 		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
2775 					    offset * sizeof(btf_fd),
2776 					    sizeof(btf_fd)))
2777 			return ERR_PTR(-EFAULT);
2778 
2779 		btf = btf_get_by_fd(btf_fd);
2780 		if (IS_ERR(btf)) {
2781 			verbose(env, "invalid module BTF fd specified\n");
2782 			return btf;
2783 		}
2784 
2785 		if (!btf_is_module(btf)) {
2786 			verbose(env, "BTF fd for kfunc is not a module BTF\n");
2787 			btf_put(btf);
2788 			return ERR_PTR(-EINVAL);
2789 		}
2790 
2791 		mod = btf_try_get_module(btf);
2792 		if (!mod) {
2793 			btf_put(btf);
2794 			return ERR_PTR(-ENXIO);
2795 		}
2796 
2797 		b = &tab->descs[tab->nr_descs++];
2798 		b->btf = btf;
2799 		b->module = mod;
2800 		b->offset = offset;
2801 
2802 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2803 		     kfunc_btf_cmp_by_off, NULL);
2804 	}
2805 	return b->btf;
2806 }
2807 
2808 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
2809 {
2810 	if (!tab)
2811 		return;
2812 
2813 	while (tab->nr_descs--) {
2814 		module_put(tab->descs[tab->nr_descs].module);
2815 		btf_put(tab->descs[tab->nr_descs].btf);
2816 	}
2817 	kfree(tab);
2818 }
2819 
2820 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
2821 {
2822 	if (offset) {
2823 		if (offset < 0) {
2824 			/* In the future, this can be allowed to increase limit
2825 			 * of fd index into fd_array, interpreted as u16.
2826 			 */
2827 			verbose(env, "negative offset disallowed for kernel module function call\n");
2828 			return ERR_PTR(-EINVAL);
2829 		}
2830 
2831 		return __find_kfunc_desc_btf(env, offset);
2832 	}
2833 	return btf_vmlinux ?: ERR_PTR(-ENOENT);
2834 }
2835 
2836 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
2837 {
2838 	const struct btf_type *func, *func_proto;
2839 	struct bpf_kfunc_btf_tab *btf_tab;
2840 	struct bpf_kfunc_desc_tab *tab;
2841 	struct bpf_prog_aux *prog_aux;
2842 	struct bpf_kfunc_desc *desc;
2843 	const char *func_name;
2844 	struct btf *desc_btf;
2845 	unsigned long call_imm;
2846 	unsigned long addr;
2847 	int err;
2848 
2849 	prog_aux = env->prog->aux;
2850 	tab = prog_aux->kfunc_tab;
2851 	btf_tab = prog_aux->kfunc_btf_tab;
2852 	if (!tab) {
2853 		if (!btf_vmlinux) {
2854 			verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
2855 			return -ENOTSUPP;
2856 		}
2857 
2858 		if (!env->prog->jit_requested) {
2859 			verbose(env, "JIT is required for calling kernel function\n");
2860 			return -ENOTSUPP;
2861 		}
2862 
2863 		if (!bpf_jit_supports_kfunc_call()) {
2864 			verbose(env, "JIT does not support calling kernel function\n");
2865 			return -ENOTSUPP;
2866 		}
2867 
2868 		if (!env->prog->gpl_compatible) {
2869 			verbose(env, "cannot call kernel function from non-GPL compatible program\n");
2870 			return -EINVAL;
2871 		}
2872 
2873 		tab = kzalloc(sizeof(*tab), GFP_KERNEL);
2874 		if (!tab)
2875 			return -ENOMEM;
2876 		prog_aux->kfunc_tab = tab;
2877 	}
2878 
2879 	/* func_id == 0 is always invalid, but instead of returning an error, be
2880 	 * conservative and wait until the code elimination pass before returning
2881 	 * error, so that invalid calls that get pruned out can be in BPF programs
2882 	 * loaded from userspace.  It is also required that offset be untouched
2883 	 * for such calls.
2884 	 */
2885 	if (!func_id && !offset)
2886 		return 0;
2887 
2888 	if (!btf_tab && offset) {
2889 		btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
2890 		if (!btf_tab)
2891 			return -ENOMEM;
2892 		prog_aux->kfunc_btf_tab = btf_tab;
2893 	}
2894 
2895 	desc_btf = find_kfunc_desc_btf(env, offset);
2896 	if (IS_ERR(desc_btf)) {
2897 		verbose(env, "failed to find BTF for kernel function\n");
2898 		return PTR_ERR(desc_btf);
2899 	}
2900 
2901 	if (find_kfunc_desc(env->prog, func_id, offset))
2902 		return 0;
2903 
2904 	if (tab->nr_descs == MAX_KFUNC_DESCS) {
2905 		verbose(env, "too many different kernel function calls\n");
2906 		return -E2BIG;
2907 	}
2908 
2909 	func = btf_type_by_id(desc_btf, func_id);
2910 	if (!func || !btf_type_is_func(func)) {
2911 		verbose(env, "kernel btf_id %u is not a function\n",
2912 			func_id);
2913 		return -EINVAL;
2914 	}
2915 	func_proto = btf_type_by_id(desc_btf, func->type);
2916 	if (!func_proto || !btf_type_is_func_proto(func_proto)) {
2917 		verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
2918 			func_id);
2919 		return -EINVAL;
2920 	}
2921 
2922 	func_name = btf_name_by_offset(desc_btf, func->name_off);
2923 	addr = kallsyms_lookup_name(func_name);
2924 	if (!addr) {
2925 		verbose(env, "cannot find address for kernel function %s\n",
2926 			func_name);
2927 		return -EINVAL;
2928 	}
2929 	specialize_kfunc(env, func_id, offset, &addr);
2930 
2931 	if (bpf_jit_supports_far_kfunc_call()) {
2932 		call_imm = func_id;
2933 	} else {
2934 		call_imm = BPF_CALL_IMM(addr);
2935 		/* Check whether the relative offset overflows desc->imm */
2936 		if ((unsigned long)(s32)call_imm != call_imm) {
2937 			verbose(env, "address of kernel function %s is out of range\n",
2938 				func_name);
2939 			return -EINVAL;
2940 		}
2941 	}
2942 
2943 	if (bpf_dev_bound_kfunc_id(func_id)) {
2944 		err = bpf_dev_bound_kfunc_check(&env->log, prog_aux);
2945 		if (err)
2946 			return err;
2947 	}
2948 
2949 	desc = &tab->descs[tab->nr_descs++];
2950 	desc->func_id = func_id;
2951 	desc->imm = call_imm;
2952 	desc->offset = offset;
2953 	desc->addr = addr;
2954 	err = btf_distill_func_proto(&env->log, desc_btf,
2955 				     func_proto, func_name,
2956 				     &desc->func_model);
2957 	if (!err)
2958 		sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2959 		     kfunc_desc_cmp_by_id_off, NULL);
2960 	return err;
2961 }
2962 
2963 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b)
2964 {
2965 	const struct bpf_kfunc_desc *d0 = a;
2966 	const struct bpf_kfunc_desc *d1 = b;
2967 
2968 	if (d0->imm != d1->imm)
2969 		return d0->imm < d1->imm ? -1 : 1;
2970 	if (d0->offset != d1->offset)
2971 		return d0->offset < d1->offset ? -1 : 1;
2972 	return 0;
2973 }
2974 
2975 static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog)
2976 {
2977 	struct bpf_kfunc_desc_tab *tab;
2978 
2979 	tab = prog->aux->kfunc_tab;
2980 	if (!tab)
2981 		return;
2982 
2983 	sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
2984 	     kfunc_desc_cmp_by_imm_off, NULL);
2985 }
2986 
2987 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2988 {
2989 	return !!prog->aux->kfunc_tab;
2990 }
2991 
2992 const struct btf_func_model *
2993 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2994 			 const struct bpf_insn *insn)
2995 {
2996 	const struct bpf_kfunc_desc desc = {
2997 		.imm = insn->imm,
2998 		.offset = insn->off,
2999 	};
3000 	const struct bpf_kfunc_desc *res;
3001 	struct bpf_kfunc_desc_tab *tab;
3002 
3003 	tab = prog->aux->kfunc_tab;
3004 	res = bsearch(&desc, tab->descs, tab->nr_descs,
3005 		      sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off);
3006 
3007 	return res ? &res->func_model : NULL;
3008 }
3009 
3010 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
3011 {
3012 	struct bpf_subprog_info *subprog = env->subprog_info;
3013 	struct bpf_insn *insn = env->prog->insnsi;
3014 	int i, ret, insn_cnt = env->prog->len;
3015 
3016 	/* Add entry function. */
3017 	ret = add_subprog(env, 0);
3018 	if (ret)
3019 		return ret;
3020 
3021 	for (i = 0; i < insn_cnt; i++, insn++) {
3022 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
3023 		    !bpf_pseudo_kfunc_call(insn))
3024 			continue;
3025 
3026 		if (!env->bpf_capable) {
3027 			verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
3028 			return -EPERM;
3029 		}
3030 
3031 		if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
3032 			ret = add_subprog(env, i + insn->imm + 1);
3033 		else
3034 			ret = add_kfunc_call(env, insn->imm, insn->off);
3035 
3036 		if (ret < 0)
3037 			return ret;
3038 	}
3039 
3040 	/* Add a fake 'exit' subprog which could simplify subprog iteration
3041 	 * logic. 'subprog_cnt' should not be increased.
3042 	 */
3043 	subprog[env->subprog_cnt].start = insn_cnt;
3044 
3045 	if (env->log.level & BPF_LOG_LEVEL2)
3046 		for (i = 0; i < env->subprog_cnt; i++)
3047 			verbose(env, "func#%d @%d\n", i, subprog[i].start);
3048 
3049 	return 0;
3050 }
3051 
3052 static int check_subprogs(struct bpf_verifier_env *env)
3053 {
3054 	int i, subprog_start, subprog_end, off, cur_subprog = 0;
3055 	struct bpf_subprog_info *subprog = env->subprog_info;
3056 	struct bpf_insn *insn = env->prog->insnsi;
3057 	int insn_cnt = env->prog->len;
3058 
3059 	/* now check that all jumps are within the same subprog */
3060 	subprog_start = subprog[cur_subprog].start;
3061 	subprog_end = subprog[cur_subprog + 1].start;
3062 	for (i = 0; i < insn_cnt; i++) {
3063 		u8 code = insn[i].code;
3064 
3065 		if (code == (BPF_JMP | BPF_CALL) &&
3066 		    insn[i].src_reg == 0 &&
3067 		    insn[i].imm == BPF_FUNC_tail_call)
3068 			subprog[cur_subprog].has_tail_call = true;
3069 		if (BPF_CLASS(code) == BPF_LD &&
3070 		    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
3071 			subprog[cur_subprog].has_ld_abs = true;
3072 		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
3073 			goto next;
3074 		if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
3075 			goto next;
3076 		if (code == (BPF_JMP32 | BPF_JA))
3077 			off = i + insn[i].imm + 1;
3078 		else
3079 			off = i + insn[i].off + 1;
3080 		if (off < subprog_start || off >= subprog_end) {
3081 			verbose(env, "jump out of range from insn %d to %d\n", i, off);
3082 			return -EINVAL;
3083 		}
3084 next:
3085 		if (i == subprog_end - 1) {
3086 			/* to avoid fall-through from one subprog into another
3087 			 * the last insn of the subprog should be either exit
3088 			 * or unconditional jump back
3089 			 */
3090 			if (code != (BPF_JMP | BPF_EXIT) &&
3091 			    code != (BPF_JMP32 | BPF_JA) &&
3092 			    code != (BPF_JMP | BPF_JA)) {
3093 				verbose(env, "last insn is not an exit or jmp\n");
3094 				return -EINVAL;
3095 			}
3096 			subprog_start = subprog_end;
3097 			cur_subprog++;
3098 			if (cur_subprog < env->subprog_cnt)
3099 				subprog_end = subprog[cur_subprog + 1].start;
3100 		}
3101 	}
3102 	return 0;
3103 }
3104 
3105 /* Parentage chain of this register (or stack slot) should take care of all
3106  * issues like callee-saved registers, stack slot allocation time, etc.
3107  */
3108 static int mark_reg_read(struct bpf_verifier_env *env,
3109 			 const struct bpf_reg_state *state,
3110 			 struct bpf_reg_state *parent, u8 flag)
3111 {
3112 	bool writes = parent == state->parent; /* Observe write marks */
3113 	int cnt = 0;
3114 
3115 	while (parent) {
3116 		/* if read wasn't screened by an earlier write ... */
3117 		if (writes && state->live & REG_LIVE_WRITTEN)
3118 			break;
3119 		if (parent->live & REG_LIVE_DONE) {
3120 			verbose(env, "verifier BUG type %s var_off %lld off %d\n",
3121 				reg_type_str(env, parent->type),
3122 				parent->var_off.value, parent->off);
3123 			return -EFAULT;
3124 		}
3125 		/* The first condition is more likely to be true than the
3126 		 * second, checked it first.
3127 		 */
3128 		if ((parent->live & REG_LIVE_READ) == flag ||
3129 		    parent->live & REG_LIVE_READ64)
3130 			/* The parentage chain never changes and
3131 			 * this parent was already marked as LIVE_READ.
3132 			 * There is no need to keep walking the chain again and
3133 			 * keep re-marking all parents as LIVE_READ.
3134 			 * This case happens when the same register is read
3135 			 * multiple times without writes into it in-between.
3136 			 * Also, if parent has the stronger REG_LIVE_READ64 set,
3137 			 * then no need to set the weak REG_LIVE_READ32.
3138 			 */
3139 			break;
3140 		/* ... then we depend on parent's value */
3141 		parent->live |= flag;
3142 		/* REG_LIVE_READ64 overrides REG_LIVE_READ32. */
3143 		if (flag == REG_LIVE_READ64)
3144 			parent->live &= ~REG_LIVE_READ32;
3145 		state = parent;
3146 		parent = state->parent;
3147 		writes = true;
3148 		cnt++;
3149 	}
3150 
3151 	if (env->longest_mark_read_walk < cnt)
3152 		env->longest_mark_read_walk = cnt;
3153 	return 0;
3154 }
3155 
3156 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
3157 {
3158 	struct bpf_func_state *state = func(env, reg);
3159 	int spi, ret;
3160 
3161 	/* For CONST_PTR_TO_DYNPTR, it must have already been done by
3162 	 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in
3163 	 * check_kfunc_call.
3164 	 */
3165 	if (reg->type == CONST_PTR_TO_DYNPTR)
3166 		return 0;
3167 	spi = dynptr_get_spi(env, reg);
3168 	if (spi < 0)
3169 		return spi;
3170 	/* Caller ensures dynptr is valid and initialized, which means spi is in
3171 	 * bounds and spi is the first dynptr slot. Simply mark stack slot as
3172 	 * read.
3173 	 */
3174 	ret = mark_reg_read(env, &state->stack[spi].spilled_ptr,
3175 			    state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64);
3176 	if (ret)
3177 		return ret;
3178 	return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr,
3179 			     state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64);
3180 }
3181 
3182 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
3183 			  int spi, int nr_slots)
3184 {
3185 	struct bpf_func_state *state = func(env, reg);
3186 	int err, i;
3187 
3188 	for (i = 0; i < nr_slots; i++) {
3189 		struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr;
3190 
3191 		err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64);
3192 		if (err)
3193 			return err;
3194 
3195 		mark_stack_slot_scratched(env, spi - i);
3196 	}
3197 
3198 	return 0;
3199 }
3200 
3201 /* This function is supposed to be used by the following 32-bit optimization
3202  * code only. It returns TRUE if the source or destination register operates
3203  * on 64-bit, otherwise return FALSE.
3204  */
3205 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
3206 		     u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
3207 {
3208 	u8 code, class, op;
3209 
3210 	code = insn->code;
3211 	class = BPF_CLASS(code);
3212 	op = BPF_OP(code);
3213 	if (class == BPF_JMP) {
3214 		/* BPF_EXIT for "main" will reach here. Return TRUE
3215 		 * conservatively.
3216 		 */
3217 		if (op == BPF_EXIT)
3218 			return true;
3219 		if (op == BPF_CALL) {
3220 			/* BPF to BPF call will reach here because of marking
3221 			 * caller saved clobber with DST_OP_NO_MARK for which we
3222 			 * don't care the register def because they are anyway
3223 			 * marked as NOT_INIT already.
3224 			 */
3225 			if (insn->src_reg == BPF_PSEUDO_CALL)
3226 				return false;
3227 			/* Helper call will reach here because of arg type
3228 			 * check, conservatively return TRUE.
3229 			 */
3230 			if (t == SRC_OP)
3231 				return true;
3232 
3233 			return false;
3234 		}
3235 	}
3236 
3237 	if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32))
3238 		return false;
3239 
3240 	if (class == BPF_ALU64 || class == BPF_JMP ||
3241 	    (class == BPF_ALU && op == BPF_END && insn->imm == 64))
3242 		return true;
3243 
3244 	if (class == BPF_ALU || class == BPF_JMP32)
3245 		return false;
3246 
3247 	if (class == BPF_LDX) {
3248 		if (t != SRC_OP)
3249 			return BPF_SIZE(code) == BPF_DW;
3250 		/* LDX source must be ptr. */
3251 		return true;
3252 	}
3253 
3254 	if (class == BPF_STX) {
3255 		/* BPF_STX (including atomic variants) has multiple source
3256 		 * operands, one of which is a ptr. Check whether the caller is
3257 		 * asking about it.
3258 		 */
3259 		if (t == SRC_OP && reg->type != SCALAR_VALUE)
3260 			return true;
3261 		return BPF_SIZE(code) == BPF_DW;
3262 	}
3263 
3264 	if (class == BPF_LD) {
3265 		u8 mode = BPF_MODE(code);
3266 
3267 		/* LD_IMM64 */
3268 		if (mode == BPF_IMM)
3269 			return true;
3270 
3271 		/* Both LD_IND and LD_ABS return 32-bit data. */
3272 		if (t != SRC_OP)
3273 			return  false;
3274 
3275 		/* Implicit ctx ptr. */
3276 		if (regno == BPF_REG_6)
3277 			return true;
3278 
3279 		/* Explicit source could be any width. */
3280 		return true;
3281 	}
3282 
3283 	if (class == BPF_ST)
3284 		/* The only source register for BPF_ST is a ptr. */
3285 		return true;
3286 
3287 	/* Conservatively return true at default. */
3288 	return true;
3289 }
3290 
3291 /* Return the regno defined by the insn, or -1. */
3292 static int insn_def_regno(const struct bpf_insn *insn)
3293 {
3294 	switch (BPF_CLASS(insn->code)) {
3295 	case BPF_JMP:
3296 	case BPF_JMP32:
3297 	case BPF_ST:
3298 		return -1;
3299 	case BPF_STX:
3300 		if (BPF_MODE(insn->code) == BPF_ATOMIC &&
3301 		    (insn->imm & BPF_FETCH)) {
3302 			if (insn->imm == BPF_CMPXCHG)
3303 				return BPF_REG_0;
3304 			else
3305 				return insn->src_reg;
3306 		} else {
3307 			return -1;
3308 		}
3309 	default:
3310 		return insn->dst_reg;
3311 	}
3312 }
3313 
3314 /* Return TRUE if INSN has defined any 32-bit value explicitly. */
3315 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
3316 {
3317 	int dst_reg = insn_def_regno(insn);
3318 
3319 	if (dst_reg == -1)
3320 		return false;
3321 
3322 	return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
3323 }
3324 
3325 static void mark_insn_zext(struct bpf_verifier_env *env,
3326 			   struct bpf_reg_state *reg)
3327 {
3328 	s32 def_idx = reg->subreg_def;
3329 
3330 	if (def_idx == DEF_NOT_SUBREG)
3331 		return;
3332 
3333 	env->insn_aux_data[def_idx - 1].zext_dst = true;
3334 	/* The dst will be zero extended, so won't be sub-register anymore. */
3335 	reg->subreg_def = DEF_NOT_SUBREG;
3336 }
3337 
3338 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
3339 			   enum reg_arg_type t)
3340 {
3341 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
3342 	struct bpf_reg_state *reg;
3343 	bool rw64;
3344 
3345 	if (regno >= MAX_BPF_REG) {
3346 		verbose(env, "R%d is invalid\n", regno);
3347 		return -EINVAL;
3348 	}
3349 
3350 	mark_reg_scratched(env, regno);
3351 
3352 	reg = &regs[regno];
3353 	rw64 = is_reg64(env, insn, regno, reg, t);
3354 	if (t == SRC_OP) {
3355 		/* check whether register used as source operand can be read */
3356 		if (reg->type == NOT_INIT) {
3357 			verbose(env, "R%d !read_ok\n", regno);
3358 			return -EACCES;
3359 		}
3360 		/* We don't need to worry about FP liveness because it's read-only */
3361 		if (regno == BPF_REG_FP)
3362 			return 0;
3363 
3364 		if (rw64)
3365 			mark_insn_zext(env, reg);
3366 
3367 		return mark_reg_read(env, reg, reg->parent,
3368 				     rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
3369 	} else {
3370 		/* check whether register used as dest operand can be written to */
3371 		if (regno == BPF_REG_FP) {
3372 			verbose(env, "frame pointer is read only\n");
3373 			return -EACCES;
3374 		}
3375 		reg->live |= REG_LIVE_WRITTEN;
3376 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
3377 		if (t == DST_OP)
3378 			mark_reg_unknown(env, regs, regno);
3379 	}
3380 	return 0;
3381 }
3382 
3383 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
3384 			 enum reg_arg_type t)
3385 {
3386 	struct bpf_verifier_state *vstate = env->cur_state;
3387 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
3388 
3389 	return __check_reg_arg(env, state->regs, regno, t);
3390 }
3391 
3392 static void mark_jmp_point(struct bpf_verifier_env *env, int idx)
3393 {
3394 	env->insn_aux_data[idx].jmp_point = true;
3395 }
3396 
3397 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
3398 {
3399 	return env->insn_aux_data[insn_idx].jmp_point;
3400 }
3401 
3402 /* for any branch, call, exit record the history of jmps in the given state */
3403 static int push_jmp_history(struct bpf_verifier_env *env,
3404 			    struct bpf_verifier_state *cur)
3405 {
3406 	u32 cnt = cur->jmp_history_cnt;
3407 	struct bpf_idx_pair *p;
3408 	size_t alloc_size;
3409 
3410 	if (!is_jmp_point(env, env->insn_idx))
3411 		return 0;
3412 
3413 	cnt++;
3414 	alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
3415 	p = krealloc(cur->jmp_history, alloc_size, GFP_USER);
3416 	if (!p)
3417 		return -ENOMEM;
3418 	p[cnt - 1].idx = env->insn_idx;
3419 	p[cnt - 1].prev_idx = env->prev_insn_idx;
3420 	cur->jmp_history = p;
3421 	cur->jmp_history_cnt = cnt;
3422 	return 0;
3423 }
3424 
3425 /* Backtrack one insn at a time. If idx is not at the top of recorded
3426  * history then previous instruction came from straight line execution.
3427  * Return -ENOENT if we exhausted all instructions within given state.
3428  *
3429  * It's legal to have a bit of a looping with the same starting and ending
3430  * insn index within the same state, e.g.: 3->4->5->3, so just because current
3431  * instruction index is the same as state's first_idx doesn't mean we are
3432  * done. If there is still some jump history left, we should keep going. We
3433  * need to take into account that we might have a jump history between given
3434  * state's parent and itself, due to checkpointing. In this case, we'll have
3435  * history entry recording a jump from last instruction of parent state and
3436  * first instruction of given state.
3437  */
3438 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
3439 			     u32 *history)
3440 {
3441 	u32 cnt = *history;
3442 
3443 	if (i == st->first_insn_idx) {
3444 		if (cnt == 0)
3445 			return -ENOENT;
3446 		if (cnt == 1 && st->jmp_history[0].idx == i)
3447 			return -ENOENT;
3448 	}
3449 
3450 	if (cnt && st->jmp_history[cnt - 1].idx == i) {
3451 		i = st->jmp_history[cnt - 1].prev_idx;
3452 		(*history)--;
3453 	} else {
3454 		i--;
3455 	}
3456 	return i;
3457 }
3458 
3459 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
3460 {
3461 	const struct btf_type *func;
3462 	struct btf *desc_btf;
3463 
3464 	if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
3465 		return NULL;
3466 
3467 	desc_btf = find_kfunc_desc_btf(data, insn->off);
3468 	if (IS_ERR(desc_btf))
3469 		return "<error>";
3470 
3471 	func = btf_type_by_id(desc_btf, insn->imm);
3472 	return btf_name_by_offset(desc_btf, func->name_off);
3473 }
3474 
3475 static inline void bt_init(struct backtrack_state *bt, u32 frame)
3476 {
3477 	bt->frame = frame;
3478 }
3479 
3480 static inline void bt_reset(struct backtrack_state *bt)
3481 {
3482 	struct bpf_verifier_env *env = bt->env;
3483 
3484 	memset(bt, 0, sizeof(*bt));
3485 	bt->env = env;
3486 }
3487 
3488 static inline u32 bt_empty(struct backtrack_state *bt)
3489 {
3490 	u64 mask = 0;
3491 	int i;
3492 
3493 	for (i = 0; i <= bt->frame; i++)
3494 		mask |= bt->reg_masks[i] | bt->stack_masks[i];
3495 
3496 	return mask == 0;
3497 }
3498 
3499 static inline int bt_subprog_enter(struct backtrack_state *bt)
3500 {
3501 	if (bt->frame == MAX_CALL_FRAMES - 1) {
3502 		verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame);
3503 		WARN_ONCE(1, "verifier backtracking bug");
3504 		return -EFAULT;
3505 	}
3506 	bt->frame++;
3507 	return 0;
3508 }
3509 
3510 static inline int bt_subprog_exit(struct backtrack_state *bt)
3511 {
3512 	if (bt->frame == 0) {
3513 		verbose(bt->env, "BUG subprog exit from frame 0\n");
3514 		WARN_ONCE(1, "verifier backtracking bug");
3515 		return -EFAULT;
3516 	}
3517 	bt->frame--;
3518 	return 0;
3519 }
3520 
3521 static inline void bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
3522 {
3523 	bt->reg_masks[frame] |= 1 << reg;
3524 }
3525 
3526 static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
3527 {
3528 	bt->reg_masks[frame] &= ~(1 << reg);
3529 }
3530 
3531 static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
3532 {
3533 	bt_set_frame_reg(bt, bt->frame, reg);
3534 }
3535 
3536 static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
3537 {
3538 	bt_clear_frame_reg(bt, bt->frame, reg);
3539 }
3540 
3541 static inline void bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
3542 {
3543 	bt->stack_masks[frame] |= 1ull << slot;
3544 }
3545 
3546 static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
3547 {
3548 	bt->stack_masks[frame] &= ~(1ull << slot);
3549 }
3550 
3551 static inline void bt_set_slot(struct backtrack_state *bt, u32 slot)
3552 {
3553 	bt_set_frame_slot(bt, bt->frame, slot);
3554 }
3555 
3556 static inline void bt_clear_slot(struct backtrack_state *bt, u32 slot)
3557 {
3558 	bt_clear_frame_slot(bt, bt->frame, slot);
3559 }
3560 
3561 static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
3562 {
3563 	return bt->reg_masks[frame];
3564 }
3565 
3566 static inline u32 bt_reg_mask(struct backtrack_state *bt)
3567 {
3568 	return bt->reg_masks[bt->frame];
3569 }
3570 
3571 static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
3572 {
3573 	return bt->stack_masks[frame];
3574 }
3575 
3576 static inline u64 bt_stack_mask(struct backtrack_state *bt)
3577 {
3578 	return bt->stack_masks[bt->frame];
3579 }
3580 
3581 static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
3582 {
3583 	return bt->reg_masks[bt->frame] & (1 << reg);
3584 }
3585 
3586 static inline bool bt_is_slot_set(struct backtrack_state *bt, u32 slot)
3587 {
3588 	return bt->stack_masks[bt->frame] & (1ull << slot);
3589 }
3590 
3591 /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
3592 static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
3593 {
3594 	DECLARE_BITMAP(mask, 64);
3595 	bool first = true;
3596 	int i, n;
3597 
3598 	buf[0] = '\0';
3599 
3600 	bitmap_from_u64(mask, reg_mask);
3601 	for_each_set_bit(i, mask, 32) {
3602 		n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
3603 		first = false;
3604 		buf += n;
3605 		buf_sz -= n;
3606 		if (buf_sz < 0)
3607 			break;
3608 	}
3609 }
3610 /* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
3611 static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
3612 {
3613 	DECLARE_BITMAP(mask, 64);
3614 	bool first = true;
3615 	int i, n;
3616 
3617 	buf[0] = '\0';
3618 
3619 	bitmap_from_u64(mask, stack_mask);
3620 	for_each_set_bit(i, mask, 64) {
3621 		n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
3622 		first = false;
3623 		buf += n;
3624 		buf_sz -= n;
3625 		if (buf_sz < 0)
3626 			break;
3627 	}
3628 }
3629 
3630 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
3631 
3632 /* For given verifier state backtrack_insn() is called from the last insn to
3633  * the first insn. Its purpose is to compute a bitmask of registers and
3634  * stack slots that needs precision in the parent verifier state.
3635  *
3636  * @idx is an index of the instruction we are currently processing;
3637  * @subseq_idx is an index of the subsequent instruction that:
3638  *   - *would be* executed next, if jump history is viewed in forward order;
3639  *   - *was* processed previously during backtracking.
3640  */
3641 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
3642 			  struct backtrack_state *bt)
3643 {
3644 	const struct bpf_insn_cbs cbs = {
3645 		.cb_call	= disasm_kfunc_name,
3646 		.cb_print	= verbose,
3647 		.private_data	= env,
3648 	};
3649 	struct bpf_insn *insn = env->prog->insnsi + idx;
3650 	u8 class = BPF_CLASS(insn->code);
3651 	u8 opcode = BPF_OP(insn->code);
3652 	u8 mode = BPF_MODE(insn->code);
3653 	u32 dreg = insn->dst_reg;
3654 	u32 sreg = insn->src_reg;
3655 	u32 spi, i;
3656 
3657 	if (insn->code == 0)
3658 		return 0;
3659 	if (env->log.level & BPF_LOG_LEVEL2) {
3660 		fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
3661 		verbose(env, "mark_precise: frame%d: regs=%s ",
3662 			bt->frame, env->tmp_str_buf);
3663 		fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
3664 		verbose(env, "stack=%s before ", env->tmp_str_buf);
3665 		verbose(env, "%d: ", idx);
3666 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
3667 	}
3668 
3669 	if (class == BPF_ALU || class == BPF_ALU64) {
3670 		if (!bt_is_reg_set(bt, dreg))
3671 			return 0;
3672 		if (opcode == BPF_END || opcode == BPF_NEG) {
3673 			/* sreg is reserved and unused
3674 			 * dreg still need precision before this insn
3675 			 */
3676 			return 0;
3677 		} else if (opcode == BPF_MOV) {
3678 			if (BPF_SRC(insn->code) == BPF_X) {
3679 				/* dreg = sreg or dreg = (s8, s16, s32)sreg
3680 				 * dreg needs precision after this insn
3681 				 * sreg needs precision before this insn
3682 				 */
3683 				bt_clear_reg(bt, dreg);
3684 				if (sreg != BPF_REG_FP)
3685 					bt_set_reg(bt, sreg);
3686 			} else {
3687 				/* dreg = K
3688 				 * dreg needs precision after this insn.
3689 				 * Corresponding register is already marked
3690 				 * as precise=true in this verifier state.
3691 				 * No further markings in parent are necessary
3692 				 */
3693 				bt_clear_reg(bt, dreg);
3694 			}
3695 		} else {
3696 			if (BPF_SRC(insn->code) == BPF_X) {
3697 				/* dreg += sreg
3698 				 * both dreg and sreg need precision
3699 				 * before this insn
3700 				 */
3701 				if (sreg != BPF_REG_FP)
3702 					bt_set_reg(bt, sreg);
3703 			} /* else dreg += K
3704 			   * dreg still needs precision before this insn
3705 			   */
3706 		}
3707 	} else if (class == BPF_LDX) {
3708 		if (!bt_is_reg_set(bt, dreg))
3709 			return 0;
3710 		bt_clear_reg(bt, dreg);
3711 
3712 		/* scalars can only be spilled into stack w/o losing precision.
3713 		 * Load from any other memory can be zero extended.
3714 		 * The desire to keep that precision is already indicated
3715 		 * by 'precise' mark in corresponding register of this state.
3716 		 * No further tracking necessary.
3717 		 */
3718 		if (insn->src_reg != BPF_REG_FP)
3719 			return 0;
3720 
3721 		/* dreg = *(u64 *)[fp - off] was a fill from the stack.
3722 		 * that [fp - off] slot contains scalar that needs to be
3723 		 * tracked with precision
3724 		 */
3725 		spi = (-insn->off - 1) / BPF_REG_SIZE;
3726 		if (spi >= 64) {
3727 			verbose(env, "BUG spi %d\n", spi);
3728 			WARN_ONCE(1, "verifier backtracking bug");
3729 			return -EFAULT;
3730 		}
3731 		bt_set_slot(bt, spi);
3732 	} else if (class == BPF_STX || class == BPF_ST) {
3733 		if (bt_is_reg_set(bt, dreg))
3734 			/* stx & st shouldn't be using _scalar_ dst_reg
3735 			 * to access memory. It means backtracking
3736 			 * encountered a case of pointer subtraction.
3737 			 */
3738 			return -ENOTSUPP;
3739 		/* scalars can only be spilled into stack */
3740 		if (insn->dst_reg != BPF_REG_FP)
3741 			return 0;
3742 		spi = (-insn->off - 1) / BPF_REG_SIZE;
3743 		if (spi >= 64) {
3744 			verbose(env, "BUG spi %d\n", spi);
3745 			WARN_ONCE(1, "verifier backtracking bug");
3746 			return -EFAULT;
3747 		}
3748 		if (!bt_is_slot_set(bt, spi))
3749 			return 0;
3750 		bt_clear_slot(bt, spi);
3751 		if (class == BPF_STX)
3752 			bt_set_reg(bt, sreg);
3753 	} else if (class == BPF_JMP || class == BPF_JMP32) {
3754 		if (bpf_pseudo_call(insn)) {
3755 			int subprog_insn_idx, subprog;
3756 
3757 			subprog_insn_idx = idx + insn->imm + 1;
3758 			subprog = find_subprog(env, subprog_insn_idx);
3759 			if (subprog < 0)
3760 				return -EFAULT;
3761 
3762 			if (subprog_is_global(env, subprog)) {
3763 				/* check that jump history doesn't have any
3764 				 * extra instructions from subprog; the next
3765 				 * instruction after call to global subprog
3766 				 * should be literally next instruction in
3767 				 * caller program
3768 				 */
3769 				WARN_ONCE(idx + 1 != subseq_idx, "verifier backtracking bug");
3770 				/* r1-r5 are invalidated after subprog call,
3771 				 * so for global func call it shouldn't be set
3772 				 * anymore
3773 				 */
3774 				if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3775 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3776 					WARN_ONCE(1, "verifier backtracking bug");
3777 					return -EFAULT;
3778 				}
3779 				/* global subprog always sets R0 */
3780 				bt_clear_reg(bt, BPF_REG_0);
3781 				return 0;
3782 			} else {
3783 				/* static subprog call instruction, which
3784 				 * means that we are exiting current subprog,
3785 				 * so only r1-r5 could be still requested as
3786 				 * precise, r0 and r6-r10 or any stack slot in
3787 				 * the current frame should be zero by now
3788 				 */
3789 				if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
3790 					verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3791 					WARN_ONCE(1, "verifier backtracking bug");
3792 					return -EFAULT;
3793 				}
3794 				/* we don't track register spills perfectly,
3795 				 * so fallback to force-precise instead of failing */
3796 				if (bt_stack_mask(bt) != 0)
3797 					return -ENOTSUPP;
3798 				/* propagate r1-r5 to the caller */
3799 				for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
3800 					if (bt_is_reg_set(bt, i)) {
3801 						bt_clear_reg(bt, i);
3802 						bt_set_frame_reg(bt, bt->frame - 1, i);
3803 					}
3804 				}
3805 				if (bt_subprog_exit(bt))
3806 					return -EFAULT;
3807 				return 0;
3808 			}
3809 		} else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
3810 			/* exit from callback subprog to callback-calling helper or
3811 			 * kfunc call. Use idx/subseq_idx check to discern it from
3812 			 * straight line code backtracking.
3813 			 * Unlike the subprog call handling above, we shouldn't
3814 			 * propagate precision of r1-r5 (if any requested), as they are
3815 			 * not actually arguments passed directly to callback subprogs
3816 			 */
3817 			if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
3818 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3819 				WARN_ONCE(1, "verifier backtracking bug");
3820 				return -EFAULT;
3821 			}
3822 			if (bt_stack_mask(bt) != 0)
3823 				return -ENOTSUPP;
3824 			/* clear r1-r5 in callback subprog's mask */
3825 			for (i = BPF_REG_1; i <= BPF_REG_5; i++)
3826 				bt_clear_reg(bt, i);
3827 			if (bt_subprog_exit(bt))
3828 				return -EFAULT;
3829 			return 0;
3830 		} else if (opcode == BPF_CALL) {
3831 			/* kfunc with imm==0 is invalid and fixup_kfunc_call will
3832 			 * catch this error later. Make backtracking conservative
3833 			 * with ENOTSUPP.
3834 			 */
3835 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
3836 				return -ENOTSUPP;
3837 			/* regular helper call sets R0 */
3838 			bt_clear_reg(bt, BPF_REG_0);
3839 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3840 				/* if backtracing was looking for registers R1-R5
3841 				 * they should have been found already.
3842 				 */
3843 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3844 				WARN_ONCE(1, "verifier backtracking bug");
3845 				return -EFAULT;
3846 			}
3847 		} else if (opcode == BPF_EXIT) {
3848 			bool r0_precise;
3849 
3850 			/* Backtracking to a nested function call, 'idx' is a part of
3851 			 * the inner frame 'subseq_idx' is a part of the outer frame.
3852 			 * In case of a regular function call, instructions giving
3853 			 * precision to registers R1-R5 should have been found already.
3854 			 * In case of a callback, it is ok to have R1-R5 marked for
3855 			 * backtracking, as these registers are set by the function
3856 			 * invoking callback.
3857 			 */
3858 			if (subseq_idx >= 0 && calls_callback(env, subseq_idx))
3859 				for (i = BPF_REG_1; i <= BPF_REG_5; i++)
3860 					bt_clear_reg(bt, i);
3861 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
3862 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
3863 				WARN_ONCE(1, "verifier backtracking bug");
3864 				return -EFAULT;
3865 			}
3866 
3867 			/* BPF_EXIT in subprog or callback always returns
3868 			 * right after the call instruction, so by checking
3869 			 * whether the instruction at subseq_idx-1 is subprog
3870 			 * call or not we can distinguish actual exit from
3871 			 * *subprog* from exit from *callback*. In the former
3872 			 * case, we need to propagate r0 precision, if
3873 			 * necessary. In the former we never do that.
3874 			 */
3875 			r0_precise = subseq_idx - 1 >= 0 &&
3876 				     bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
3877 				     bt_is_reg_set(bt, BPF_REG_0);
3878 
3879 			bt_clear_reg(bt, BPF_REG_0);
3880 			if (bt_subprog_enter(bt))
3881 				return -EFAULT;
3882 
3883 			if (r0_precise)
3884 				bt_set_reg(bt, BPF_REG_0);
3885 			/* r6-r9 and stack slots will stay set in caller frame
3886 			 * bitmasks until we return back from callee(s)
3887 			 */
3888 			return 0;
3889 		} else if (BPF_SRC(insn->code) == BPF_X) {
3890 			if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
3891 				return 0;
3892 			/* dreg <cond> sreg
3893 			 * Both dreg and sreg need precision before
3894 			 * this insn. If only sreg was marked precise
3895 			 * before it would be equally necessary to
3896 			 * propagate it to dreg.
3897 			 */
3898 			bt_set_reg(bt, dreg);
3899 			bt_set_reg(bt, sreg);
3900 			 /* else dreg <cond> K
3901 			  * Only dreg still needs precision before
3902 			  * this insn, so for the K-based conditional
3903 			  * there is nothing new to be marked.
3904 			  */
3905 		}
3906 	} else if (class == BPF_LD) {
3907 		if (!bt_is_reg_set(bt, dreg))
3908 			return 0;
3909 		bt_clear_reg(bt, dreg);
3910 		/* It's ld_imm64 or ld_abs or ld_ind.
3911 		 * For ld_imm64 no further tracking of precision
3912 		 * into parent is necessary
3913 		 */
3914 		if (mode == BPF_IND || mode == BPF_ABS)
3915 			/* to be analyzed */
3916 			return -ENOTSUPP;
3917 	}
3918 	return 0;
3919 }
3920 
3921 /* the scalar precision tracking algorithm:
3922  * . at the start all registers have precise=false.
3923  * . scalar ranges are tracked as normal through alu and jmp insns.
3924  * . once precise value of the scalar register is used in:
3925  *   .  ptr + scalar alu
3926  *   . if (scalar cond K|scalar)
3927  *   .  helper_call(.., scalar, ...) where ARG_CONST is expected
3928  *   backtrack through the verifier states and mark all registers and
3929  *   stack slots with spilled constants that these scalar regisers
3930  *   should be precise.
3931  * . during state pruning two registers (or spilled stack slots)
3932  *   are equivalent if both are not precise.
3933  *
3934  * Note the verifier cannot simply walk register parentage chain,
3935  * since many different registers and stack slots could have been
3936  * used to compute single precise scalar.
3937  *
3938  * The approach of starting with precise=true for all registers and then
3939  * backtrack to mark a register as not precise when the verifier detects
3940  * that program doesn't care about specific value (e.g., when helper
3941  * takes register as ARG_ANYTHING parameter) is not safe.
3942  *
3943  * It's ok to walk single parentage chain of the verifier states.
3944  * It's possible that this backtracking will go all the way till 1st insn.
3945  * All other branches will be explored for needing precision later.
3946  *
3947  * The backtracking needs to deal with cases like:
3948  *   R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
3949  * r9 -= r8
3950  * r5 = r9
3951  * if r5 > 0x79f goto pc+7
3952  *    R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
3953  * r5 += 1
3954  * ...
3955  * call bpf_perf_event_output#25
3956  *   where .arg5_type = ARG_CONST_SIZE_OR_ZERO
3957  *
3958  * and this case:
3959  * r6 = 1
3960  * call foo // uses callee's r6 inside to compute r0
3961  * r0 += r6
3962  * if r0 == 0 goto
3963  *
3964  * to track above reg_mask/stack_mask needs to be independent for each frame.
3965  *
3966  * Also if parent's curframe > frame where backtracking started,
3967  * the verifier need to mark registers in both frames, otherwise callees
3968  * may incorrectly prune callers. This is similar to
3969  * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
3970  *
3971  * For now backtracking falls back into conservative marking.
3972  */
3973 static void mark_all_scalars_precise(struct bpf_verifier_env *env,
3974 				     struct bpf_verifier_state *st)
3975 {
3976 	struct bpf_func_state *func;
3977 	struct bpf_reg_state *reg;
3978 	int i, j;
3979 
3980 	if (env->log.level & BPF_LOG_LEVEL2) {
3981 		verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
3982 			st->curframe);
3983 	}
3984 
3985 	/* big hammer: mark all scalars precise in this path.
3986 	 * pop_stack may still get !precise scalars.
3987 	 * We also skip current state and go straight to first parent state,
3988 	 * because precision markings in current non-checkpointed state are
3989 	 * not needed. See why in the comment in __mark_chain_precision below.
3990 	 */
3991 	for (st = st->parent; st; st = st->parent) {
3992 		for (i = 0; i <= st->curframe; i++) {
3993 			func = st->frame[i];
3994 			for (j = 0; j < BPF_REG_FP; j++) {
3995 				reg = &func->regs[j];
3996 				if (reg->type != SCALAR_VALUE || reg->precise)
3997 					continue;
3998 				reg->precise = true;
3999 				if (env->log.level & BPF_LOG_LEVEL2) {
4000 					verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
4001 						i, j);
4002 				}
4003 			}
4004 			for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
4005 				if (!is_spilled_reg(&func->stack[j]))
4006 					continue;
4007 				reg = &func->stack[j].spilled_ptr;
4008 				if (reg->type != SCALAR_VALUE || reg->precise)
4009 					continue;
4010 				reg->precise = true;
4011 				if (env->log.level & BPF_LOG_LEVEL2) {
4012 					verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
4013 						i, -(j + 1) * 8);
4014 				}
4015 			}
4016 		}
4017 	}
4018 }
4019 
4020 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
4021 {
4022 	struct bpf_func_state *func;
4023 	struct bpf_reg_state *reg;
4024 	int i, j;
4025 
4026 	for (i = 0; i <= st->curframe; i++) {
4027 		func = st->frame[i];
4028 		for (j = 0; j < BPF_REG_FP; j++) {
4029 			reg = &func->regs[j];
4030 			if (reg->type != SCALAR_VALUE)
4031 				continue;
4032 			reg->precise = false;
4033 		}
4034 		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
4035 			if (!is_spilled_reg(&func->stack[j]))
4036 				continue;
4037 			reg = &func->stack[j].spilled_ptr;
4038 			if (reg->type != SCALAR_VALUE)
4039 				continue;
4040 			reg->precise = false;
4041 		}
4042 	}
4043 }
4044 
4045 static bool idset_contains(struct bpf_idset *s, u32 id)
4046 {
4047 	u32 i;
4048 
4049 	for (i = 0; i < s->count; ++i)
4050 		if (s->ids[i] == id)
4051 			return true;
4052 
4053 	return false;
4054 }
4055 
4056 static int idset_push(struct bpf_idset *s, u32 id)
4057 {
4058 	if (WARN_ON_ONCE(s->count >= ARRAY_SIZE(s->ids)))
4059 		return -EFAULT;
4060 	s->ids[s->count++] = id;
4061 	return 0;
4062 }
4063 
4064 static void idset_reset(struct bpf_idset *s)
4065 {
4066 	s->count = 0;
4067 }
4068 
4069 /* Collect a set of IDs for all registers currently marked as precise in env->bt.
4070  * Mark all registers with these IDs as precise.
4071  */
4072 static int mark_precise_scalar_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
4073 {
4074 	struct bpf_idset *precise_ids = &env->idset_scratch;
4075 	struct backtrack_state *bt = &env->bt;
4076 	struct bpf_func_state *func;
4077 	struct bpf_reg_state *reg;
4078 	DECLARE_BITMAP(mask, 64);
4079 	int i, fr;
4080 
4081 	idset_reset(precise_ids);
4082 
4083 	for (fr = bt->frame; fr >= 0; fr--) {
4084 		func = st->frame[fr];
4085 
4086 		bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
4087 		for_each_set_bit(i, mask, 32) {
4088 			reg = &func->regs[i];
4089 			if (!reg->id || reg->type != SCALAR_VALUE)
4090 				continue;
4091 			if (idset_push(precise_ids, reg->id))
4092 				return -EFAULT;
4093 		}
4094 
4095 		bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
4096 		for_each_set_bit(i, mask, 64) {
4097 			if (i >= func->allocated_stack / BPF_REG_SIZE)
4098 				break;
4099 			if (!is_spilled_scalar_reg(&func->stack[i]))
4100 				continue;
4101 			reg = &func->stack[i].spilled_ptr;
4102 			if (!reg->id)
4103 				continue;
4104 			if (idset_push(precise_ids, reg->id))
4105 				return -EFAULT;
4106 		}
4107 	}
4108 
4109 	for (fr = 0; fr <= st->curframe; ++fr) {
4110 		func = st->frame[fr];
4111 
4112 		for (i = BPF_REG_0; i < BPF_REG_10; ++i) {
4113 			reg = &func->regs[i];
4114 			if (!reg->id)
4115 				continue;
4116 			if (!idset_contains(precise_ids, reg->id))
4117 				continue;
4118 			bt_set_frame_reg(bt, fr, i);
4119 		}
4120 		for (i = 0; i < func->allocated_stack / BPF_REG_SIZE; ++i) {
4121 			if (!is_spilled_scalar_reg(&func->stack[i]))
4122 				continue;
4123 			reg = &func->stack[i].spilled_ptr;
4124 			if (!reg->id)
4125 				continue;
4126 			if (!idset_contains(precise_ids, reg->id))
4127 				continue;
4128 			bt_set_frame_slot(bt, fr, i);
4129 		}
4130 	}
4131 
4132 	return 0;
4133 }
4134 
4135 /*
4136  * __mark_chain_precision() backtracks BPF program instruction sequence and
4137  * chain of verifier states making sure that register *regno* (if regno >= 0)
4138  * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
4139  * SCALARS, as well as any other registers and slots that contribute to
4140  * a tracked state of given registers/stack slots, depending on specific BPF
4141  * assembly instructions (see backtrack_insns() for exact instruction handling
4142  * logic). This backtracking relies on recorded jmp_history and is able to
4143  * traverse entire chain of parent states. This process ends only when all the
4144  * necessary registers/slots and their transitive dependencies are marked as
4145  * precise.
4146  *
4147  * One important and subtle aspect is that precise marks *do not matter* in
4148  * the currently verified state (current state). It is important to understand
4149  * why this is the case.
4150  *
4151  * First, note that current state is the state that is not yet "checkpointed",
4152  * i.e., it is not yet put into env->explored_states, and it has no children
4153  * states as well. It's ephemeral, and can end up either a) being discarded if
4154  * compatible explored state is found at some point or BPF_EXIT instruction is
4155  * reached or b) checkpointed and put into env->explored_states, branching out
4156  * into one or more children states.
4157  *
4158  * In the former case, precise markings in current state are completely
4159  * ignored by state comparison code (see regsafe() for details). Only
4160  * checkpointed ("old") state precise markings are important, and if old
4161  * state's register/slot is precise, regsafe() assumes current state's
4162  * register/slot as precise and checks value ranges exactly and precisely. If
4163  * states turn out to be compatible, current state's necessary precise
4164  * markings and any required parent states' precise markings are enforced
4165  * after the fact with propagate_precision() logic, after the fact. But it's
4166  * important to realize that in this case, even after marking current state
4167  * registers/slots as precise, we immediately discard current state. So what
4168  * actually matters is any of the precise markings propagated into current
4169  * state's parent states, which are always checkpointed (due to b) case above).
4170  * As such, for scenario a) it doesn't matter if current state has precise
4171  * markings set or not.
4172  *
4173  * Now, for the scenario b), checkpointing and forking into child(ren)
4174  * state(s). Note that before current state gets to checkpointing step, any
4175  * processed instruction always assumes precise SCALAR register/slot
4176  * knowledge: if precise value or range is useful to prune jump branch, BPF
4177  * verifier takes this opportunity enthusiastically. Similarly, when
4178  * register's value is used to calculate offset or memory address, exact
4179  * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
4180  * what we mentioned above about state comparison ignoring precise markings
4181  * during state comparison, BPF verifier ignores and also assumes precise
4182  * markings *at will* during instruction verification process. But as verifier
4183  * assumes precision, it also propagates any precision dependencies across
4184  * parent states, which are not yet finalized, so can be further restricted
4185  * based on new knowledge gained from restrictions enforced by their children
4186  * states. This is so that once those parent states are finalized, i.e., when
4187  * they have no more active children state, state comparison logic in
4188  * is_state_visited() would enforce strict and precise SCALAR ranges, if
4189  * required for correctness.
4190  *
4191  * To build a bit more intuition, note also that once a state is checkpointed,
4192  * the path we took to get to that state is not important. This is crucial
4193  * property for state pruning. When state is checkpointed and finalized at
4194  * some instruction index, it can be correctly and safely used to "short
4195  * circuit" any *compatible* state that reaches exactly the same instruction
4196  * index. I.e., if we jumped to that instruction from a completely different
4197  * code path than original finalized state was derived from, it doesn't
4198  * matter, current state can be discarded because from that instruction
4199  * forward having a compatible state will ensure we will safely reach the
4200  * exit. States describe preconditions for further exploration, but completely
4201  * forget the history of how we got here.
4202  *
4203  * This also means that even if we needed precise SCALAR range to get to
4204  * finalized state, but from that point forward *that same* SCALAR register is
4205  * never used in a precise context (i.e., it's precise value is not needed for
4206  * correctness), it's correct and safe to mark such register as "imprecise"
4207  * (i.e., precise marking set to false). This is what we rely on when we do
4208  * not set precise marking in current state. If no child state requires
4209  * precision for any given SCALAR register, it's safe to dictate that it can
4210  * be imprecise. If any child state does require this register to be precise,
4211  * we'll mark it precise later retroactively during precise markings
4212  * propagation from child state to parent states.
4213  *
4214  * Skipping precise marking setting in current state is a mild version of
4215  * relying on the above observation. But we can utilize this property even
4216  * more aggressively by proactively forgetting any precise marking in the
4217  * current state (which we inherited from the parent state), right before we
4218  * checkpoint it and branch off into new child state. This is done by
4219  * mark_all_scalars_imprecise() to hopefully get more permissive and generic
4220  * finalized states which help in short circuiting more future states.
4221  */
4222 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno)
4223 {
4224 	struct backtrack_state *bt = &env->bt;
4225 	struct bpf_verifier_state *st = env->cur_state;
4226 	int first_idx = st->first_insn_idx;
4227 	int last_idx = env->insn_idx;
4228 	int subseq_idx = -1;
4229 	struct bpf_func_state *func;
4230 	struct bpf_reg_state *reg;
4231 	bool skip_first = true;
4232 	int i, fr, err;
4233 
4234 	if (!env->bpf_capable)
4235 		return 0;
4236 
4237 	/* set frame number from which we are starting to backtrack */
4238 	bt_init(bt, env->cur_state->curframe);
4239 
4240 	/* Do sanity checks against current state of register and/or stack
4241 	 * slot, but don't set precise flag in current state, as precision
4242 	 * tracking in the current state is unnecessary.
4243 	 */
4244 	func = st->frame[bt->frame];
4245 	if (regno >= 0) {
4246 		reg = &func->regs[regno];
4247 		if (reg->type != SCALAR_VALUE) {
4248 			WARN_ONCE(1, "backtracing misuse");
4249 			return -EFAULT;
4250 		}
4251 		bt_set_reg(bt, regno);
4252 	}
4253 
4254 	if (bt_empty(bt))
4255 		return 0;
4256 
4257 	for (;;) {
4258 		DECLARE_BITMAP(mask, 64);
4259 		u32 history = st->jmp_history_cnt;
4260 
4261 		if (env->log.level & BPF_LOG_LEVEL2) {
4262 			verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
4263 				bt->frame, last_idx, first_idx, subseq_idx);
4264 		}
4265 
4266 		/* If some register with scalar ID is marked as precise,
4267 		 * make sure that all registers sharing this ID are also precise.
4268 		 * This is needed to estimate effect of find_equal_scalars().
4269 		 * Do this at the last instruction of each state,
4270 		 * bpf_reg_state::id fields are valid for these instructions.
4271 		 *
4272 		 * Allows to track precision in situation like below:
4273 		 *
4274 		 *     r2 = unknown value
4275 		 *     ...
4276 		 *   --- state #0 ---
4277 		 *     ...
4278 		 *     r1 = r2                 // r1 and r2 now share the same ID
4279 		 *     ...
4280 		 *   --- state #1 {r1.id = A, r2.id = A} ---
4281 		 *     ...
4282 		 *     if (r2 > 10) goto exit; // find_equal_scalars() assigns range to r1
4283 		 *     ...
4284 		 *   --- state #2 {r1.id = A, r2.id = A} ---
4285 		 *     r3 = r10
4286 		 *     r3 += r1                // need to mark both r1 and r2
4287 		 */
4288 		if (mark_precise_scalar_ids(env, st))
4289 			return -EFAULT;
4290 
4291 		if (last_idx < 0) {
4292 			/* we are at the entry into subprog, which
4293 			 * is expected for global funcs, but only if
4294 			 * requested precise registers are R1-R5
4295 			 * (which are global func's input arguments)
4296 			 */
4297 			if (st->curframe == 0 &&
4298 			    st->frame[0]->subprogno > 0 &&
4299 			    st->frame[0]->callsite == BPF_MAIN_FUNC &&
4300 			    bt_stack_mask(bt) == 0 &&
4301 			    (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
4302 				bitmap_from_u64(mask, bt_reg_mask(bt));
4303 				for_each_set_bit(i, mask, 32) {
4304 					reg = &st->frame[0]->regs[i];
4305 					bt_clear_reg(bt, i);
4306 					if (reg->type == SCALAR_VALUE)
4307 						reg->precise = true;
4308 				}
4309 				return 0;
4310 			}
4311 
4312 			verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n",
4313 				st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
4314 			WARN_ONCE(1, "verifier backtracking bug");
4315 			return -EFAULT;
4316 		}
4317 
4318 		for (i = last_idx;;) {
4319 			if (skip_first) {
4320 				err = 0;
4321 				skip_first = false;
4322 			} else {
4323 				err = backtrack_insn(env, i, subseq_idx, bt);
4324 			}
4325 			if (err == -ENOTSUPP) {
4326 				mark_all_scalars_precise(env, env->cur_state);
4327 				bt_reset(bt);
4328 				return 0;
4329 			} else if (err) {
4330 				return err;
4331 			}
4332 			if (bt_empty(bt))
4333 				/* Found assignment(s) into tracked register in this state.
4334 				 * Since this state is already marked, just return.
4335 				 * Nothing to be tracked further in the parent state.
4336 				 */
4337 				return 0;
4338 			subseq_idx = i;
4339 			i = get_prev_insn_idx(st, i, &history);
4340 			if (i == -ENOENT)
4341 				break;
4342 			if (i >= env->prog->len) {
4343 				/* This can happen if backtracking reached insn 0
4344 				 * and there are still reg_mask or stack_mask
4345 				 * to backtrack.
4346 				 * It means the backtracking missed the spot where
4347 				 * particular register was initialized with a constant.
4348 				 */
4349 				verbose(env, "BUG backtracking idx %d\n", i);
4350 				WARN_ONCE(1, "verifier backtracking bug");
4351 				return -EFAULT;
4352 			}
4353 		}
4354 		st = st->parent;
4355 		if (!st)
4356 			break;
4357 
4358 		for (fr = bt->frame; fr >= 0; fr--) {
4359 			func = st->frame[fr];
4360 			bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
4361 			for_each_set_bit(i, mask, 32) {
4362 				reg = &func->regs[i];
4363 				if (reg->type != SCALAR_VALUE) {
4364 					bt_clear_frame_reg(bt, fr, i);
4365 					continue;
4366 				}
4367 				if (reg->precise)
4368 					bt_clear_frame_reg(bt, fr, i);
4369 				else
4370 					reg->precise = true;
4371 			}
4372 
4373 			bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
4374 			for_each_set_bit(i, mask, 64) {
4375 				if (i >= func->allocated_stack / BPF_REG_SIZE) {
4376 					/* the sequence of instructions:
4377 					 * 2: (bf) r3 = r10
4378 					 * 3: (7b) *(u64 *)(r3 -8) = r0
4379 					 * 4: (79) r4 = *(u64 *)(r10 -8)
4380 					 * doesn't contain jmps. It's backtracked
4381 					 * as a single block.
4382 					 * During backtracking insn 3 is not recognized as
4383 					 * stack access, so at the end of backtracking
4384 					 * stack slot fp-8 is still marked in stack_mask.
4385 					 * However the parent state may not have accessed
4386 					 * fp-8 and it's "unallocated" stack space.
4387 					 * In such case fallback to conservative.
4388 					 */
4389 					mark_all_scalars_precise(env, env->cur_state);
4390 					bt_reset(bt);
4391 					return 0;
4392 				}
4393 
4394 				if (!is_spilled_scalar_reg(&func->stack[i])) {
4395 					bt_clear_frame_slot(bt, fr, i);
4396 					continue;
4397 				}
4398 				reg = &func->stack[i].spilled_ptr;
4399 				if (reg->precise)
4400 					bt_clear_frame_slot(bt, fr, i);
4401 				else
4402 					reg->precise = true;
4403 			}
4404 			if (env->log.level & BPF_LOG_LEVEL2) {
4405 				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4406 					     bt_frame_reg_mask(bt, fr));
4407 				verbose(env, "mark_precise: frame%d: parent state regs=%s ",
4408 					fr, env->tmp_str_buf);
4409 				fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
4410 					       bt_frame_stack_mask(bt, fr));
4411 				verbose(env, "stack=%s: ", env->tmp_str_buf);
4412 				print_verifier_state(env, func, true);
4413 			}
4414 		}
4415 
4416 		if (bt_empty(bt))
4417 			return 0;
4418 
4419 		subseq_idx = first_idx;
4420 		last_idx = st->last_insn_idx;
4421 		first_idx = st->first_insn_idx;
4422 	}
4423 
4424 	/* if we still have requested precise regs or slots, we missed
4425 	 * something (e.g., stack access through non-r10 register), so
4426 	 * fallback to marking all precise
4427 	 */
4428 	if (!bt_empty(bt)) {
4429 		mark_all_scalars_precise(env, env->cur_state);
4430 		bt_reset(bt);
4431 	}
4432 
4433 	return 0;
4434 }
4435 
4436 int mark_chain_precision(struct bpf_verifier_env *env, int regno)
4437 {
4438 	return __mark_chain_precision(env, regno);
4439 }
4440 
4441 /* mark_chain_precision_batch() assumes that env->bt is set in the caller to
4442  * desired reg and stack masks across all relevant frames
4443  */
4444 static int mark_chain_precision_batch(struct bpf_verifier_env *env)
4445 {
4446 	return __mark_chain_precision(env, -1);
4447 }
4448 
4449 static bool is_spillable_regtype(enum bpf_reg_type type)
4450 {
4451 	switch (base_type(type)) {
4452 	case PTR_TO_MAP_VALUE:
4453 	case PTR_TO_STACK:
4454 	case PTR_TO_CTX:
4455 	case PTR_TO_PACKET:
4456 	case PTR_TO_PACKET_META:
4457 	case PTR_TO_PACKET_END:
4458 	case PTR_TO_FLOW_KEYS:
4459 	case CONST_PTR_TO_MAP:
4460 	case PTR_TO_SOCKET:
4461 	case PTR_TO_SOCK_COMMON:
4462 	case PTR_TO_TCP_SOCK:
4463 	case PTR_TO_XDP_SOCK:
4464 	case PTR_TO_BTF_ID:
4465 	case PTR_TO_BUF:
4466 	case PTR_TO_MEM:
4467 	case PTR_TO_FUNC:
4468 	case PTR_TO_MAP_KEY:
4469 		return true;
4470 	default:
4471 		return false;
4472 	}
4473 }
4474 
4475 /* Does this register contain a constant zero? */
4476 static bool register_is_null(struct bpf_reg_state *reg)
4477 {
4478 	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
4479 }
4480 
4481 static bool register_is_const(struct bpf_reg_state *reg)
4482 {
4483 	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
4484 }
4485 
4486 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
4487 {
4488 	return tnum_is_unknown(reg->var_off) &&
4489 	       reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
4490 	       reg->umin_value == 0 && reg->umax_value == U64_MAX &&
4491 	       reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
4492 	       reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
4493 }
4494 
4495 static bool register_is_bounded(struct bpf_reg_state *reg)
4496 {
4497 	return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
4498 }
4499 
4500 static bool __is_pointer_value(bool allow_ptr_leaks,
4501 			       const struct bpf_reg_state *reg)
4502 {
4503 	if (allow_ptr_leaks)
4504 		return false;
4505 
4506 	return reg->type != SCALAR_VALUE;
4507 }
4508 
4509 /* Copy src state preserving dst->parent and dst->live fields */
4510 static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
4511 {
4512 	struct bpf_reg_state *parent = dst->parent;
4513 	enum bpf_reg_liveness live = dst->live;
4514 
4515 	*dst = *src;
4516 	dst->parent = parent;
4517 	dst->live = live;
4518 }
4519 
4520 static void save_register_state(struct bpf_func_state *state,
4521 				int spi, struct bpf_reg_state *reg,
4522 				int size)
4523 {
4524 	int i;
4525 
4526 	copy_register_state(&state->stack[spi].spilled_ptr, reg);
4527 	if (size == BPF_REG_SIZE)
4528 		state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
4529 
4530 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
4531 		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
4532 
4533 	/* size < 8 bytes spill */
4534 	for (; i; i--)
4535 		scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]);
4536 }
4537 
4538 static bool is_bpf_st_mem(struct bpf_insn *insn)
4539 {
4540 	return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
4541 }
4542 
4543 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
4544  * stack boundary and alignment are checked in check_mem_access()
4545  */
4546 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
4547 				       /* stack frame we're writing to */
4548 				       struct bpf_func_state *state,
4549 				       int off, int size, int value_regno,
4550 				       int insn_idx)
4551 {
4552 	struct bpf_func_state *cur; /* state of the current function */
4553 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
4554 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
4555 	struct bpf_reg_state *reg = NULL;
4556 	u32 dst_reg = insn->dst_reg;
4557 
4558 	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
4559 	 * so it's aligned access and [off, off + size) are within stack limits
4560 	 */
4561 	if (!env->allow_ptr_leaks &&
4562 	    is_spilled_reg(&state->stack[spi]) &&
4563 	    size != BPF_REG_SIZE) {
4564 		verbose(env, "attempt to corrupt spilled pointer on stack\n");
4565 		return -EACCES;
4566 	}
4567 
4568 	cur = env->cur_state->frame[env->cur_state->curframe];
4569 	if (value_regno >= 0)
4570 		reg = &cur->regs[value_regno];
4571 	if (!env->bypass_spec_v4) {
4572 		bool sanitize = reg && is_spillable_regtype(reg->type);
4573 
4574 		for (i = 0; i < size; i++) {
4575 			u8 type = state->stack[spi].slot_type[i];
4576 
4577 			if (type != STACK_MISC && type != STACK_ZERO) {
4578 				sanitize = true;
4579 				break;
4580 			}
4581 		}
4582 
4583 		if (sanitize)
4584 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
4585 	}
4586 
4587 	err = destroy_if_dynptr_stack_slot(env, state, spi);
4588 	if (err)
4589 		return err;
4590 
4591 	mark_stack_slot_scratched(env, spi);
4592 	if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
4593 	    !register_is_null(reg) && env->bpf_capable) {
4594 		if (dst_reg != BPF_REG_FP) {
4595 			/* The backtracking logic can only recognize explicit
4596 			 * stack slot address like [fp - 8]. Other spill of
4597 			 * scalar via different register has to be conservative.
4598 			 * Backtrack from here and mark all registers as precise
4599 			 * that contributed into 'reg' being a constant.
4600 			 */
4601 			err = mark_chain_precision(env, value_regno);
4602 			if (err)
4603 				return err;
4604 		}
4605 		save_register_state(state, spi, reg, size);
4606 		/* Break the relation on a narrowing spill. */
4607 		if (fls64(reg->umax_value) > BITS_PER_BYTE * size)
4608 			state->stack[spi].spilled_ptr.id = 0;
4609 	} else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
4610 		   insn->imm != 0 && env->bpf_capable) {
4611 		struct bpf_reg_state fake_reg = {};
4612 
4613 		__mark_reg_known(&fake_reg, insn->imm);
4614 		fake_reg.type = SCALAR_VALUE;
4615 		save_register_state(state, spi, &fake_reg, size);
4616 	} else if (reg && is_spillable_regtype(reg->type)) {
4617 		/* register containing pointer is being spilled into stack */
4618 		if (size != BPF_REG_SIZE) {
4619 			verbose_linfo(env, insn_idx, "; ");
4620 			verbose(env, "invalid size of register spill\n");
4621 			return -EACCES;
4622 		}
4623 		if (state != cur && reg->type == PTR_TO_STACK) {
4624 			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
4625 			return -EINVAL;
4626 		}
4627 		save_register_state(state, spi, reg, size);
4628 	} else {
4629 		u8 type = STACK_MISC;
4630 
4631 		/* regular write of data into stack destroys any spilled ptr */
4632 		state->stack[spi].spilled_ptr.type = NOT_INIT;
4633 		/* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
4634 		if (is_stack_slot_special(&state->stack[spi]))
4635 			for (i = 0; i < BPF_REG_SIZE; i++)
4636 				scrub_spilled_slot(&state->stack[spi].slot_type[i]);
4637 
4638 		/* only mark the slot as written if all 8 bytes were written
4639 		 * otherwise read propagation may incorrectly stop too soon
4640 		 * when stack slots are partially written.
4641 		 * This heuristic means that read propagation will be
4642 		 * conservative, since it will add reg_live_read marks
4643 		 * to stack slots all the way to first state when programs
4644 		 * writes+reads less than 8 bytes
4645 		 */
4646 		if (size == BPF_REG_SIZE)
4647 			state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
4648 
4649 		/* when we zero initialize stack slots mark them as such */
4650 		if ((reg && register_is_null(reg)) ||
4651 		    (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
4652 			/* backtracking doesn't work for STACK_ZERO yet. */
4653 			err = mark_chain_precision(env, value_regno);
4654 			if (err)
4655 				return err;
4656 			type = STACK_ZERO;
4657 		}
4658 
4659 		/* Mark slots affected by this stack write. */
4660 		for (i = 0; i < size; i++)
4661 			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
4662 				type;
4663 	}
4664 	return 0;
4665 }
4666 
4667 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
4668  * known to contain a variable offset.
4669  * This function checks whether the write is permitted and conservatively
4670  * tracks the effects of the write, considering that each stack slot in the
4671  * dynamic range is potentially written to.
4672  *
4673  * 'off' includes 'regno->off'.
4674  * 'value_regno' can be -1, meaning that an unknown value is being written to
4675  * the stack.
4676  *
4677  * Spilled pointers in range are not marked as written because we don't know
4678  * what's going to be actually written. This means that read propagation for
4679  * future reads cannot be terminated by this write.
4680  *
4681  * For privileged programs, uninitialized stack slots are considered
4682  * initialized by this write (even though we don't know exactly what offsets
4683  * are going to be written to). The idea is that we don't want the verifier to
4684  * reject future reads that access slots written to through variable offsets.
4685  */
4686 static int check_stack_write_var_off(struct bpf_verifier_env *env,
4687 				     /* func where register points to */
4688 				     struct bpf_func_state *state,
4689 				     int ptr_regno, int off, int size,
4690 				     int value_regno, int insn_idx)
4691 {
4692 	struct bpf_func_state *cur; /* state of the current function */
4693 	int min_off, max_off;
4694 	int i, err;
4695 	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
4696 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
4697 	bool writing_zero = false;
4698 	/* set if the fact that we're writing a zero is used to let any
4699 	 * stack slots remain STACK_ZERO
4700 	 */
4701 	bool zero_used = false;
4702 
4703 	cur = env->cur_state->frame[env->cur_state->curframe];
4704 	ptr_reg = &cur->regs[ptr_regno];
4705 	min_off = ptr_reg->smin_value + off;
4706 	max_off = ptr_reg->smax_value + off + size;
4707 	if (value_regno >= 0)
4708 		value_reg = &cur->regs[value_regno];
4709 	if ((value_reg && register_is_null(value_reg)) ||
4710 	    (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0))
4711 		writing_zero = true;
4712 
4713 	for (i = min_off; i < max_off; i++) {
4714 		int spi;
4715 
4716 		spi = __get_spi(i);
4717 		err = destroy_if_dynptr_stack_slot(env, state, spi);
4718 		if (err)
4719 			return err;
4720 	}
4721 
4722 	/* Variable offset writes destroy any spilled pointers in range. */
4723 	for (i = min_off; i < max_off; i++) {
4724 		u8 new_type, *stype;
4725 		int slot, spi;
4726 
4727 		slot = -i - 1;
4728 		spi = slot / BPF_REG_SIZE;
4729 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
4730 		mark_stack_slot_scratched(env, spi);
4731 
4732 		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
4733 			/* Reject the write if range we may write to has not
4734 			 * been initialized beforehand. If we didn't reject
4735 			 * here, the ptr status would be erased below (even
4736 			 * though not all slots are actually overwritten),
4737 			 * possibly opening the door to leaks.
4738 			 *
4739 			 * We do however catch STACK_INVALID case below, and
4740 			 * only allow reading possibly uninitialized memory
4741 			 * later for CAP_PERFMON, as the write may not happen to
4742 			 * that slot.
4743 			 */
4744 			verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
4745 				insn_idx, i);
4746 			return -EINVAL;
4747 		}
4748 
4749 		/* Erase all spilled pointers. */
4750 		state->stack[spi].spilled_ptr.type = NOT_INIT;
4751 
4752 		/* Update the slot type. */
4753 		new_type = STACK_MISC;
4754 		if (writing_zero && *stype == STACK_ZERO) {
4755 			new_type = STACK_ZERO;
4756 			zero_used = true;
4757 		}
4758 		/* If the slot is STACK_INVALID, we check whether it's OK to
4759 		 * pretend that it will be initialized by this write. The slot
4760 		 * might not actually be written to, and so if we mark it as
4761 		 * initialized future reads might leak uninitialized memory.
4762 		 * For privileged programs, we will accept such reads to slots
4763 		 * that may or may not be written because, if we're reject
4764 		 * them, the error would be too confusing.
4765 		 */
4766 		if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
4767 			verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
4768 					insn_idx, i);
4769 			return -EINVAL;
4770 		}
4771 		*stype = new_type;
4772 	}
4773 	if (zero_used) {
4774 		/* backtracking doesn't work for STACK_ZERO yet. */
4775 		err = mark_chain_precision(env, value_regno);
4776 		if (err)
4777 			return err;
4778 	}
4779 	return 0;
4780 }
4781 
4782 /* When register 'dst_regno' is assigned some values from stack[min_off,
4783  * max_off), we set the register's type according to the types of the
4784  * respective stack slots. If all the stack values are known to be zeros, then
4785  * so is the destination reg. Otherwise, the register is considered to be
4786  * SCALAR. This function does not deal with register filling; the caller must
4787  * ensure that all spilled registers in the stack range have been marked as
4788  * read.
4789  */
4790 static void mark_reg_stack_read(struct bpf_verifier_env *env,
4791 				/* func where src register points to */
4792 				struct bpf_func_state *ptr_state,
4793 				int min_off, int max_off, int dst_regno)
4794 {
4795 	struct bpf_verifier_state *vstate = env->cur_state;
4796 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4797 	int i, slot, spi;
4798 	u8 *stype;
4799 	int zeros = 0;
4800 
4801 	for (i = min_off; i < max_off; i++) {
4802 		slot = -i - 1;
4803 		spi = slot / BPF_REG_SIZE;
4804 		mark_stack_slot_scratched(env, spi);
4805 		stype = ptr_state->stack[spi].slot_type;
4806 		if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
4807 			break;
4808 		zeros++;
4809 	}
4810 	if (zeros == max_off - min_off) {
4811 		/* any access_size read into register is zero extended,
4812 		 * so the whole register == const_zero
4813 		 */
4814 		__mark_reg_const_zero(&state->regs[dst_regno]);
4815 		/* backtracking doesn't support STACK_ZERO yet,
4816 		 * so mark it precise here, so that later
4817 		 * backtracking can stop here.
4818 		 * Backtracking may not need this if this register
4819 		 * doesn't participate in pointer adjustment.
4820 		 * Forward propagation of precise flag is not
4821 		 * necessary either. This mark is only to stop
4822 		 * backtracking. Any register that contributed
4823 		 * to const 0 was marked precise before spill.
4824 		 */
4825 		state->regs[dst_regno].precise = true;
4826 	} else {
4827 		/* have read misc data from the stack */
4828 		mark_reg_unknown(env, state->regs, dst_regno);
4829 	}
4830 	state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4831 }
4832 
4833 /* Read the stack at 'off' and put the results into the register indicated by
4834  * 'dst_regno'. It handles reg filling if the addressed stack slot is a
4835  * spilled reg.
4836  *
4837  * 'dst_regno' can be -1, meaning that the read value is not going to a
4838  * register.
4839  *
4840  * The access is assumed to be within the current stack bounds.
4841  */
4842 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
4843 				      /* func where src register points to */
4844 				      struct bpf_func_state *reg_state,
4845 				      int off, int size, int dst_regno)
4846 {
4847 	struct bpf_verifier_state *vstate = env->cur_state;
4848 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4849 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
4850 	struct bpf_reg_state *reg;
4851 	u8 *stype, type;
4852 
4853 	stype = reg_state->stack[spi].slot_type;
4854 	reg = &reg_state->stack[spi].spilled_ptr;
4855 
4856 	mark_stack_slot_scratched(env, spi);
4857 
4858 	if (is_spilled_reg(&reg_state->stack[spi])) {
4859 		u8 spill_size = 1;
4860 
4861 		for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
4862 			spill_size++;
4863 
4864 		if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
4865 			if (reg->type != SCALAR_VALUE) {
4866 				verbose_linfo(env, env->insn_idx, "; ");
4867 				verbose(env, "invalid size of register fill\n");
4868 				return -EACCES;
4869 			}
4870 
4871 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4872 			if (dst_regno < 0)
4873 				return 0;
4874 
4875 			if (!(off % BPF_REG_SIZE) && size == spill_size) {
4876 				/* The earlier check_reg_arg() has decided the
4877 				 * subreg_def for this insn.  Save it first.
4878 				 */
4879 				s32 subreg_def = state->regs[dst_regno].subreg_def;
4880 
4881 				copy_register_state(&state->regs[dst_regno], reg);
4882 				state->regs[dst_regno].subreg_def = subreg_def;
4883 			} else {
4884 				for (i = 0; i < size; i++) {
4885 					type = stype[(slot - i) % BPF_REG_SIZE];
4886 					if (type == STACK_SPILL)
4887 						continue;
4888 					if (type == STACK_MISC)
4889 						continue;
4890 					if (type == STACK_INVALID && env->allow_uninit_stack)
4891 						continue;
4892 					verbose(env, "invalid read from stack off %d+%d size %d\n",
4893 						off, i, size);
4894 					return -EACCES;
4895 				}
4896 				mark_reg_unknown(env, state->regs, dst_regno);
4897 			}
4898 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4899 			return 0;
4900 		}
4901 
4902 		if (dst_regno >= 0) {
4903 			/* restore register state from stack */
4904 			copy_register_state(&state->regs[dst_regno], reg);
4905 			/* mark reg as written since spilled pointer state likely
4906 			 * has its liveness marks cleared by is_state_visited()
4907 			 * which resets stack/reg liveness for state transitions
4908 			 */
4909 			state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
4910 		} else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
4911 			/* If dst_regno==-1, the caller is asking us whether
4912 			 * it is acceptable to use this value as a SCALAR_VALUE
4913 			 * (e.g. for XADD).
4914 			 * We must not allow unprivileged callers to do that
4915 			 * with spilled pointers.
4916 			 */
4917 			verbose(env, "leaking pointer from stack off %d\n",
4918 				off);
4919 			return -EACCES;
4920 		}
4921 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4922 	} else {
4923 		for (i = 0; i < size; i++) {
4924 			type = stype[(slot - i) % BPF_REG_SIZE];
4925 			if (type == STACK_MISC)
4926 				continue;
4927 			if (type == STACK_ZERO)
4928 				continue;
4929 			if (type == STACK_INVALID && env->allow_uninit_stack)
4930 				continue;
4931 			verbose(env, "invalid read from stack off %d+%d size %d\n",
4932 				off, i, size);
4933 			return -EACCES;
4934 		}
4935 		mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
4936 		if (dst_regno >= 0)
4937 			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
4938 	}
4939 	return 0;
4940 }
4941 
4942 enum bpf_access_src {
4943 	ACCESS_DIRECT = 1,  /* the access is performed by an instruction */
4944 	ACCESS_HELPER = 2,  /* the access is performed by a helper */
4945 };
4946 
4947 static int check_stack_range_initialized(struct bpf_verifier_env *env,
4948 					 int regno, int off, int access_size,
4949 					 bool zero_size_allowed,
4950 					 enum bpf_access_src type,
4951 					 struct bpf_call_arg_meta *meta);
4952 
4953 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
4954 {
4955 	return cur_regs(env) + regno;
4956 }
4957 
4958 /* Read the stack at 'ptr_regno + off' and put the result into the register
4959  * 'dst_regno'.
4960  * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
4961  * but not its variable offset.
4962  * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
4963  *
4964  * As opposed to check_stack_read_fixed_off, this function doesn't deal with
4965  * filling registers (i.e. reads of spilled register cannot be detected when
4966  * the offset is not fixed). We conservatively mark 'dst_regno' as containing
4967  * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
4968  * offset; for a fixed offset check_stack_read_fixed_off should be used
4969  * instead.
4970  */
4971 static int check_stack_read_var_off(struct bpf_verifier_env *env,
4972 				    int ptr_regno, int off, int size, int dst_regno)
4973 {
4974 	/* The state of the source register. */
4975 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
4976 	struct bpf_func_state *ptr_state = func(env, reg);
4977 	int err;
4978 	int min_off, max_off;
4979 
4980 	/* Note that we pass a NULL meta, so raw access will not be permitted.
4981 	 */
4982 	err = check_stack_range_initialized(env, ptr_regno, off, size,
4983 					    false, ACCESS_DIRECT, NULL);
4984 	if (err)
4985 		return err;
4986 
4987 	min_off = reg->smin_value + off;
4988 	max_off = reg->smax_value + off;
4989 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
4990 	return 0;
4991 }
4992 
4993 /* check_stack_read dispatches to check_stack_read_fixed_off or
4994  * check_stack_read_var_off.
4995  *
4996  * The caller must ensure that the offset falls within the allocated stack
4997  * bounds.
4998  *
4999  * 'dst_regno' is a register which will receive the value from the stack. It
5000  * can be -1, meaning that the read value is not going to a register.
5001  */
5002 static int check_stack_read(struct bpf_verifier_env *env,
5003 			    int ptr_regno, int off, int size,
5004 			    int dst_regno)
5005 {
5006 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
5007 	struct bpf_func_state *state = func(env, reg);
5008 	int err;
5009 	/* Some accesses are only permitted with a static offset. */
5010 	bool var_off = !tnum_is_const(reg->var_off);
5011 
5012 	/* The offset is required to be static when reads don't go to a
5013 	 * register, in order to not leak pointers (see
5014 	 * check_stack_read_fixed_off).
5015 	 */
5016 	if (dst_regno < 0 && var_off) {
5017 		char tn_buf[48];
5018 
5019 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5020 		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
5021 			tn_buf, off, size);
5022 		return -EACCES;
5023 	}
5024 	/* Variable offset is prohibited for unprivileged mode for simplicity
5025 	 * since it requires corresponding support in Spectre masking for stack
5026 	 * ALU. See also retrieve_ptr_limit(). The check in
5027 	 * check_stack_access_for_ptr_arithmetic() called by
5028 	 * adjust_ptr_min_max_vals() prevents users from creating stack pointers
5029 	 * with variable offsets, therefore no check is required here. Further,
5030 	 * just checking it here would be insufficient as speculative stack
5031 	 * writes could still lead to unsafe speculative behaviour.
5032 	 */
5033 	if (!var_off) {
5034 		off += reg->var_off.value;
5035 		err = check_stack_read_fixed_off(env, state, off, size,
5036 						 dst_regno);
5037 	} else {
5038 		/* Variable offset stack reads need more conservative handling
5039 		 * than fixed offset ones. Note that dst_regno >= 0 on this
5040 		 * branch.
5041 		 */
5042 		err = check_stack_read_var_off(env, ptr_regno, off, size,
5043 					       dst_regno);
5044 	}
5045 	return err;
5046 }
5047 
5048 
5049 /* check_stack_write dispatches to check_stack_write_fixed_off or
5050  * check_stack_write_var_off.
5051  *
5052  * 'ptr_regno' is the register used as a pointer into the stack.
5053  * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
5054  * 'value_regno' is the register whose value we're writing to the stack. It can
5055  * be -1, meaning that we're not writing from a register.
5056  *
5057  * The caller must ensure that the offset falls within the maximum stack size.
5058  */
5059 static int check_stack_write(struct bpf_verifier_env *env,
5060 			     int ptr_regno, int off, int size,
5061 			     int value_regno, int insn_idx)
5062 {
5063 	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
5064 	struct bpf_func_state *state = func(env, reg);
5065 	int err;
5066 
5067 	if (tnum_is_const(reg->var_off)) {
5068 		off += reg->var_off.value;
5069 		err = check_stack_write_fixed_off(env, state, off, size,
5070 						  value_regno, insn_idx);
5071 	} else {
5072 		/* Variable offset stack reads need more conservative handling
5073 		 * than fixed offset ones.
5074 		 */
5075 		err = check_stack_write_var_off(env, state,
5076 						ptr_regno, off, size,
5077 						value_regno, insn_idx);
5078 	}
5079 	return err;
5080 }
5081 
5082 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
5083 				 int off, int size, enum bpf_access_type type)
5084 {
5085 	struct bpf_reg_state *regs = cur_regs(env);
5086 	struct bpf_map *map = regs[regno].map_ptr;
5087 	u32 cap = bpf_map_flags_to_cap(map);
5088 
5089 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
5090 		verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
5091 			map->value_size, off, size);
5092 		return -EACCES;
5093 	}
5094 
5095 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
5096 		verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
5097 			map->value_size, off, size);
5098 		return -EACCES;
5099 	}
5100 
5101 	return 0;
5102 }
5103 
5104 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
5105 static int __check_mem_access(struct bpf_verifier_env *env, int regno,
5106 			      int off, int size, u32 mem_size,
5107 			      bool zero_size_allowed)
5108 {
5109 	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
5110 	struct bpf_reg_state *reg;
5111 
5112 	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
5113 		return 0;
5114 
5115 	reg = &cur_regs(env)[regno];
5116 	switch (reg->type) {
5117 	case PTR_TO_MAP_KEY:
5118 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
5119 			mem_size, off, size);
5120 		break;
5121 	case PTR_TO_MAP_VALUE:
5122 		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
5123 			mem_size, off, size);
5124 		break;
5125 	case PTR_TO_PACKET:
5126 	case PTR_TO_PACKET_META:
5127 	case PTR_TO_PACKET_END:
5128 		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
5129 			off, size, regno, reg->id, off, mem_size);
5130 		break;
5131 	case PTR_TO_MEM:
5132 	default:
5133 		verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
5134 			mem_size, off, size);
5135 	}
5136 
5137 	return -EACCES;
5138 }
5139 
5140 /* check read/write into a memory region with possible variable offset */
5141 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
5142 				   int off, int size, u32 mem_size,
5143 				   bool zero_size_allowed)
5144 {
5145 	struct bpf_verifier_state *vstate = env->cur_state;
5146 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
5147 	struct bpf_reg_state *reg = &state->regs[regno];
5148 	int err;
5149 
5150 	/* We may have adjusted the register pointing to memory region, so we
5151 	 * need to try adding each of min_value and max_value to off
5152 	 * to make sure our theoretical access will be safe.
5153 	 *
5154 	 * The minimum value is only important with signed
5155 	 * comparisons where we can't assume the floor of a
5156 	 * value is 0.  If we are using signed variables for our
5157 	 * index'es we need to make sure that whatever we use
5158 	 * will have a set floor within our range.
5159 	 */
5160 	if (reg->smin_value < 0 &&
5161 	    (reg->smin_value == S64_MIN ||
5162 	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
5163 	      reg->smin_value + off < 0)) {
5164 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5165 			regno);
5166 		return -EACCES;
5167 	}
5168 	err = __check_mem_access(env, regno, reg->smin_value + off, size,
5169 				 mem_size, zero_size_allowed);
5170 	if (err) {
5171 		verbose(env, "R%d min value is outside of the allowed memory range\n",
5172 			regno);
5173 		return err;
5174 	}
5175 
5176 	/* If we haven't set a max value then we need to bail since we can't be
5177 	 * sure we won't do bad things.
5178 	 * If reg->umax_value + off could overflow, treat that as unbounded too.
5179 	 */
5180 	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
5181 		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
5182 			regno);
5183 		return -EACCES;
5184 	}
5185 	err = __check_mem_access(env, regno, reg->umax_value + off, size,
5186 				 mem_size, zero_size_allowed);
5187 	if (err) {
5188 		verbose(env, "R%d max value is outside of the allowed memory range\n",
5189 			regno);
5190 		return err;
5191 	}
5192 
5193 	return 0;
5194 }
5195 
5196 static int __check_ptr_off_reg(struct bpf_verifier_env *env,
5197 			       const struct bpf_reg_state *reg, int regno,
5198 			       bool fixed_off_ok)
5199 {
5200 	/* Access to this pointer-typed register or passing it to a helper
5201 	 * is only allowed in its original, unmodified form.
5202 	 */
5203 
5204 	if (reg->off < 0) {
5205 		verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
5206 			reg_type_str(env, reg->type), regno, reg->off);
5207 		return -EACCES;
5208 	}
5209 
5210 	if (!fixed_off_ok && reg->off) {
5211 		verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
5212 			reg_type_str(env, reg->type), regno, reg->off);
5213 		return -EACCES;
5214 	}
5215 
5216 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
5217 		char tn_buf[48];
5218 
5219 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5220 		verbose(env, "variable %s access var_off=%s disallowed\n",
5221 			reg_type_str(env, reg->type), tn_buf);
5222 		return -EACCES;
5223 	}
5224 
5225 	return 0;
5226 }
5227 
5228 int check_ptr_off_reg(struct bpf_verifier_env *env,
5229 		      const struct bpf_reg_state *reg, int regno)
5230 {
5231 	return __check_ptr_off_reg(env, reg, regno, false);
5232 }
5233 
5234 static int map_kptr_match_type(struct bpf_verifier_env *env,
5235 			       struct btf_field *kptr_field,
5236 			       struct bpf_reg_state *reg, u32 regno)
5237 {
5238 	const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id);
5239 	int perm_flags;
5240 	const char *reg_name = "";
5241 
5242 	if (btf_is_kernel(reg->btf)) {
5243 		perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
5244 
5245 		/* Only unreferenced case accepts untrusted pointers */
5246 		if (kptr_field->type == BPF_KPTR_UNREF)
5247 			perm_flags |= PTR_UNTRUSTED;
5248 	} else {
5249 		perm_flags = PTR_MAYBE_NULL | MEM_ALLOC;
5250 	}
5251 
5252 	if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
5253 		goto bad_type;
5254 
5255 	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
5256 	reg_name = btf_type_name(reg->btf, reg->btf_id);
5257 
5258 	/* For ref_ptr case, release function check should ensure we get one
5259 	 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
5260 	 * normal store of unreferenced kptr, we must ensure var_off is zero.
5261 	 * Since ref_ptr cannot be accessed directly by BPF insns, checks for
5262 	 * reg->off and reg->ref_obj_id are not needed here.
5263 	 */
5264 	if (__check_ptr_off_reg(env, reg, regno, true))
5265 		return -EACCES;
5266 
5267 	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
5268 	 * we also need to take into account the reg->off.
5269 	 *
5270 	 * We want to support cases like:
5271 	 *
5272 	 * struct foo {
5273 	 *         struct bar br;
5274 	 *         struct baz bz;
5275 	 * };
5276 	 *
5277 	 * struct foo *v;
5278 	 * v = func();	      // PTR_TO_BTF_ID
5279 	 * val->foo = v;      // reg->off is zero, btf and btf_id match type
5280 	 * val->bar = &v->br; // reg->off is still zero, but we need to retry with
5281 	 *                    // first member type of struct after comparison fails
5282 	 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
5283 	 *                    // to match type
5284 	 *
5285 	 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
5286 	 * is zero. We must also ensure that btf_struct_ids_match does not walk
5287 	 * the struct to match type against first member of struct, i.e. reject
5288 	 * second case from above. Hence, when type is BPF_KPTR_REF, we set
5289 	 * strict mode to true for type match.
5290 	 */
5291 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
5292 				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
5293 				  kptr_field->type == BPF_KPTR_REF))
5294 		goto bad_type;
5295 	return 0;
5296 bad_type:
5297 	verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
5298 		reg_type_str(env, reg->type), reg_name);
5299 	verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
5300 	if (kptr_field->type == BPF_KPTR_UNREF)
5301 		verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
5302 			targ_name);
5303 	else
5304 		verbose(env, "\n");
5305 	return -EINVAL;
5306 }
5307 
5308 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock()
5309  * can dereference RCU protected pointers and result is PTR_TRUSTED.
5310  */
5311 static bool in_rcu_cs(struct bpf_verifier_env *env)
5312 {
5313 	return env->cur_state->active_rcu_lock ||
5314 	       env->cur_state->active_lock.ptr ||
5315 	       !env->prog->aux->sleepable;
5316 }
5317 
5318 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */
5319 BTF_SET_START(rcu_protected_types)
5320 BTF_ID(struct, prog_test_ref_kfunc)
5321 BTF_ID(struct, cgroup)
5322 BTF_ID(struct, bpf_cpumask)
5323 BTF_ID(struct, task_struct)
5324 BTF_SET_END(rcu_protected_types)
5325 
5326 static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
5327 {
5328 	if (!btf_is_kernel(btf))
5329 		return false;
5330 	return btf_id_set_contains(&rcu_protected_types, btf_id);
5331 }
5332 
5333 static bool rcu_safe_kptr(const struct btf_field *field)
5334 {
5335 	const struct btf_field_kptr *kptr = &field->kptr;
5336 
5337 	return field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id);
5338 }
5339 
5340 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
5341 				 int value_regno, int insn_idx,
5342 				 struct btf_field *kptr_field)
5343 {
5344 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
5345 	int class = BPF_CLASS(insn->code);
5346 	struct bpf_reg_state *val_reg;
5347 
5348 	/* Things we already checked for in check_map_access and caller:
5349 	 *  - Reject cases where variable offset may touch kptr
5350 	 *  - size of access (must be BPF_DW)
5351 	 *  - tnum_is_const(reg->var_off)
5352 	 *  - kptr_field->offset == off + reg->var_off.value
5353 	 */
5354 	/* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
5355 	if (BPF_MODE(insn->code) != BPF_MEM) {
5356 		verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
5357 		return -EACCES;
5358 	}
5359 
5360 	/* We only allow loading referenced kptr, since it will be marked as
5361 	 * untrusted, similar to unreferenced kptr.
5362 	 */
5363 	if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) {
5364 		verbose(env, "store to referenced kptr disallowed\n");
5365 		return -EACCES;
5366 	}
5367 
5368 	if (class == BPF_LDX) {
5369 		val_reg = reg_state(env, value_regno);
5370 		/* We can simply mark the value_regno receiving the pointer
5371 		 * value from map as PTR_TO_BTF_ID, with the correct type.
5372 		 */
5373 		mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf,
5374 				kptr_field->kptr.btf_id,
5375 				rcu_safe_kptr(kptr_field) && in_rcu_cs(env) ?
5376 				PTR_MAYBE_NULL | MEM_RCU :
5377 				PTR_MAYBE_NULL | PTR_UNTRUSTED);
5378 	} else if (class == BPF_STX) {
5379 		val_reg = reg_state(env, value_regno);
5380 		if (!register_is_null(val_reg) &&
5381 		    map_kptr_match_type(env, kptr_field, val_reg, value_regno))
5382 			return -EACCES;
5383 	} else if (class == BPF_ST) {
5384 		if (insn->imm) {
5385 			verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
5386 				kptr_field->offset);
5387 			return -EACCES;
5388 		}
5389 	} else {
5390 		verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
5391 		return -EACCES;
5392 	}
5393 	return 0;
5394 }
5395 
5396 /* check read/write into a map element with possible variable offset */
5397 static int check_map_access(struct bpf_verifier_env *env, u32 regno,
5398 			    int off, int size, bool zero_size_allowed,
5399 			    enum bpf_access_src src)
5400 {
5401 	struct bpf_verifier_state *vstate = env->cur_state;
5402 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
5403 	struct bpf_reg_state *reg = &state->regs[regno];
5404 	struct bpf_map *map = reg->map_ptr;
5405 	struct btf_record *rec;
5406 	int err, i;
5407 
5408 	err = check_mem_region_access(env, regno, off, size, map->value_size,
5409 				      zero_size_allowed);
5410 	if (err)
5411 		return err;
5412 
5413 	if (IS_ERR_OR_NULL(map->record))
5414 		return 0;
5415 	rec = map->record;
5416 	for (i = 0; i < rec->cnt; i++) {
5417 		struct btf_field *field = &rec->fields[i];
5418 		u32 p = field->offset;
5419 
5420 		/* If any part of a field  can be touched by load/store, reject
5421 		 * this program. To check that [x1, x2) overlaps with [y1, y2),
5422 		 * it is sufficient to check x1 < y2 && y1 < x2.
5423 		 */
5424 		if (reg->smin_value + off < p + btf_field_type_size(field->type) &&
5425 		    p < reg->umax_value + off + size) {
5426 			switch (field->type) {
5427 			case BPF_KPTR_UNREF:
5428 			case BPF_KPTR_REF:
5429 				if (src != ACCESS_DIRECT) {
5430 					verbose(env, "kptr cannot be accessed indirectly by helper\n");
5431 					return -EACCES;
5432 				}
5433 				if (!tnum_is_const(reg->var_off)) {
5434 					verbose(env, "kptr access cannot have variable offset\n");
5435 					return -EACCES;
5436 				}
5437 				if (p != off + reg->var_off.value) {
5438 					verbose(env, "kptr access misaligned expected=%u off=%llu\n",
5439 						p, off + reg->var_off.value);
5440 					return -EACCES;
5441 				}
5442 				if (size != bpf_size_to_bytes(BPF_DW)) {
5443 					verbose(env, "kptr access size must be BPF_DW\n");
5444 					return -EACCES;
5445 				}
5446 				break;
5447 			default:
5448 				verbose(env, "%s cannot be accessed directly by load/store\n",
5449 					btf_field_type_name(field->type));
5450 				return -EACCES;
5451 			}
5452 		}
5453 	}
5454 	return 0;
5455 }
5456 
5457 #define MAX_PACKET_OFF 0xffff
5458 
5459 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
5460 				       const struct bpf_call_arg_meta *meta,
5461 				       enum bpf_access_type t)
5462 {
5463 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
5464 
5465 	switch (prog_type) {
5466 	/* Program types only with direct read access go here! */
5467 	case BPF_PROG_TYPE_LWT_IN:
5468 	case BPF_PROG_TYPE_LWT_OUT:
5469 	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
5470 	case BPF_PROG_TYPE_SK_REUSEPORT:
5471 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
5472 	case BPF_PROG_TYPE_CGROUP_SKB:
5473 		if (t == BPF_WRITE)
5474 			return false;
5475 		fallthrough;
5476 
5477 	/* Program types with direct read + write access go here! */
5478 	case BPF_PROG_TYPE_SCHED_CLS:
5479 	case BPF_PROG_TYPE_SCHED_ACT:
5480 	case BPF_PROG_TYPE_XDP:
5481 	case BPF_PROG_TYPE_LWT_XMIT:
5482 	case BPF_PROG_TYPE_SK_SKB:
5483 	case BPF_PROG_TYPE_SK_MSG:
5484 		if (meta)
5485 			return meta->pkt_access;
5486 
5487 		env->seen_direct_write = true;
5488 		return true;
5489 
5490 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
5491 		if (t == BPF_WRITE)
5492 			env->seen_direct_write = true;
5493 
5494 		return true;
5495 
5496 	default:
5497 		return false;
5498 	}
5499 }
5500 
5501 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
5502 			       int size, bool zero_size_allowed)
5503 {
5504 	struct bpf_reg_state *regs = cur_regs(env);
5505 	struct bpf_reg_state *reg = &regs[regno];
5506 	int err;
5507 
5508 	/* We may have added a variable offset to the packet pointer; but any
5509 	 * reg->range we have comes after that.  We are only checking the fixed
5510 	 * offset.
5511 	 */
5512 
5513 	/* We don't allow negative numbers, because we aren't tracking enough
5514 	 * detail to prove they're safe.
5515 	 */
5516 	if (reg->smin_value < 0) {
5517 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5518 			regno);
5519 		return -EACCES;
5520 	}
5521 
5522 	err = reg->range < 0 ? -EINVAL :
5523 	      __check_mem_access(env, regno, off, size, reg->range,
5524 				 zero_size_allowed);
5525 	if (err) {
5526 		verbose(env, "R%d offset is outside of the packet\n", regno);
5527 		return err;
5528 	}
5529 
5530 	/* __check_mem_access has made sure "off + size - 1" is within u16.
5531 	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
5532 	 * otherwise find_good_pkt_pointers would have refused to set range info
5533 	 * that __check_mem_access would have rejected this pkt access.
5534 	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
5535 	 */
5536 	env->prog->aux->max_pkt_offset =
5537 		max_t(u32, env->prog->aux->max_pkt_offset,
5538 		      off + reg->umax_value + size - 1);
5539 
5540 	return err;
5541 }
5542 
5543 /* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
5544 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
5545 			    enum bpf_access_type t, enum bpf_reg_type *reg_type,
5546 			    struct btf **btf, u32 *btf_id)
5547 {
5548 	struct bpf_insn_access_aux info = {
5549 		.reg_type = *reg_type,
5550 		.log = &env->log,
5551 	};
5552 
5553 	if (env->ops->is_valid_access &&
5554 	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {
5555 		/* A non zero info.ctx_field_size indicates that this field is a
5556 		 * candidate for later verifier transformation to load the whole
5557 		 * field and then apply a mask when accessed with a narrower
5558 		 * access than actual ctx access size. A zero info.ctx_field_size
5559 		 * will only allow for whole field access and rejects any other
5560 		 * type of narrower access.
5561 		 */
5562 		*reg_type = info.reg_type;
5563 
5564 		if (base_type(*reg_type) == PTR_TO_BTF_ID) {
5565 			*btf = info.btf;
5566 			*btf_id = info.btf_id;
5567 		} else {
5568 			env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
5569 		}
5570 		/* remember the offset of last byte accessed in ctx */
5571 		if (env->prog->aux->max_ctx_offset < off + size)
5572 			env->prog->aux->max_ctx_offset = off + size;
5573 		return 0;
5574 	}
5575 
5576 	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
5577 	return -EACCES;
5578 }
5579 
5580 static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
5581 				  int size)
5582 {
5583 	if (size < 0 || off < 0 ||
5584 	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
5585 		verbose(env, "invalid access to flow keys off=%d size=%d\n",
5586 			off, size);
5587 		return -EACCES;
5588 	}
5589 	return 0;
5590 }
5591 
5592 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
5593 			     u32 regno, int off, int size,
5594 			     enum bpf_access_type t)
5595 {
5596 	struct bpf_reg_state *regs = cur_regs(env);
5597 	struct bpf_reg_state *reg = &regs[regno];
5598 	struct bpf_insn_access_aux info = {};
5599 	bool valid;
5600 
5601 	if (reg->smin_value < 0) {
5602 		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
5603 			regno);
5604 		return -EACCES;
5605 	}
5606 
5607 	switch (reg->type) {
5608 	case PTR_TO_SOCK_COMMON:
5609 		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
5610 		break;
5611 	case PTR_TO_SOCKET:
5612 		valid = bpf_sock_is_valid_access(off, size, t, &info);
5613 		break;
5614 	case PTR_TO_TCP_SOCK:
5615 		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
5616 		break;
5617 	case PTR_TO_XDP_SOCK:
5618 		valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
5619 		break;
5620 	default:
5621 		valid = false;
5622 	}
5623 
5624 
5625 	if (valid) {
5626 		env->insn_aux_data[insn_idx].ctx_field_size =
5627 			info.ctx_field_size;
5628 		return 0;
5629 	}
5630 
5631 	verbose(env, "R%d invalid %s access off=%d size=%d\n",
5632 		regno, reg_type_str(env, reg->type), off, size);
5633 
5634 	return -EACCES;
5635 }
5636 
5637 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
5638 {
5639 	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
5640 }
5641 
5642 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
5643 {
5644 	const struct bpf_reg_state *reg = reg_state(env, regno);
5645 
5646 	return reg->type == PTR_TO_CTX;
5647 }
5648 
5649 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
5650 {
5651 	const struct bpf_reg_state *reg = reg_state(env, regno);
5652 
5653 	return type_is_sk_pointer(reg->type);
5654 }
5655 
5656 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
5657 {
5658 	const struct bpf_reg_state *reg = reg_state(env, regno);
5659 
5660 	return type_is_pkt_pointer(reg->type);
5661 }
5662 
5663 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
5664 {
5665 	const struct bpf_reg_state *reg = reg_state(env, regno);
5666 
5667 	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
5668 	return reg->type == PTR_TO_FLOW_KEYS;
5669 }
5670 
5671 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5672 #ifdef CONFIG_NET
5673 	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5674 	[PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5675 	[PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5676 #endif
5677 	[CONST_PTR_TO_MAP] = btf_bpf_map_id,
5678 };
5679 
5680 static bool is_trusted_reg(const struct bpf_reg_state *reg)
5681 {
5682 	/* A referenced register is always trusted. */
5683 	if (reg->ref_obj_id)
5684 		return true;
5685 
5686 	/* Types listed in the reg2btf_ids are always trusted */
5687 	if (reg2btf_ids[base_type(reg->type)] &&
5688 	    !bpf_type_has_unsafe_modifiers(reg->type))
5689 		return true;
5690 
5691 	/* If a register is not referenced, it is trusted if it has the
5692 	 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the
5693 	 * other type modifiers may be safe, but we elect to take an opt-in
5694 	 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are
5695 	 * not.
5696 	 *
5697 	 * Eventually, we should make PTR_TRUSTED the single source of truth
5698 	 * for whether a register is trusted.
5699 	 */
5700 	return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS &&
5701 	       !bpf_type_has_unsafe_modifiers(reg->type);
5702 }
5703 
5704 static bool is_rcu_reg(const struct bpf_reg_state *reg)
5705 {
5706 	return reg->type & MEM_RCU;
5707 }
5708 
5709 static void clear_trusted_flags(enum bpf_type_flag *flag)
5710 {
5711 	*flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU);
5712 }
5713 
5714 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
5715 				   const struct bpf_reg_state *reg,
5716 				   int off, int size, bool strict)
5717 {
5718 	struct tnum reg_off;
5719 	int ip_align;
5720 
5721 	/* Byte size accesses are always allowed. */
5722 	if (!strict || size == 1)
5723 		return 0;
5724 
5725 	/* For platforms that do not have a Kconfig enabling
5726 	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
5727 	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
5728 	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
5729 	 * to this code only in strict mode where we want to emulate
5730 	 * the NET_IP_ALIGN==2 checking.  Therefore use an
5731 	 * unconditional IP align value of '2'.
5732 	 */
5733 	ip_align = 2;
5734 
5735 	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
5736 	if (!tnum_is_aligned(reg_off, size)) {
5737 		char tn_buf[48];
5738 
5739 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5740 		verbose(env,
5741 			"misaligned packet access off %d+%s+%d+%d size %d\n",
5742 			ip_align, tn_buf, reg->off, off, size);
5743 		return -EACCES;
5744 	}
5745 
5746 	return 0;
5747 }
5748 
5749 static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
5750 				       const struct bpf_reg_state *reg,
5751 				       const char *pointer_desc,
5752 				       int off, int size, bool strict)
5753 {
5754 	struct tnum reg_off;
5755 
5756 	/* Byte size accesses are always allowed. */
5757 	if (!strict || size == 1)
5758 		return 0;
5759 
5760 	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
5761 	if (!tnum_is_aligned(reg_off, size)) {
5762 		char tn_buf[48];
5763 
5764 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5765 		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
5766 			pointer_desc, tn_buf, reg->off, off, size);
5767 		return -EACCES;
5768 	}
5769 
5770 	return 0;
5771 }
5772 
5773 static int check_ptr_alignment(struct bpf_verifier_env *env,
5774 			       const struct bpf_reg_state *reg, int off,
5775 			       int size, bool strict_alignment_once)
5776 {
5777 	bool strict = env->strict_alignment || strict_alignment_once;
5778 	const char *pointer_desc = "";
5779 
5780 	switch (reg->type) {
5781 	case PTR_TO_PACKET:
5782 	case PTR_TO_PACKET_META:
5783 		/* Special case, because of NET_IP_ALIGN. Given metadata sits
5784 		 * right in front, treat it the very same way.
5785 		 */
5786 		return check_pkt_ptr_alignment(env, reg, off, size, strict);
5787 	case PTR_TO_FLOW_KEYS:
5788 		pointer_desc = "flow keys ";
5789 		break;
5790 	case PTR_TO_MAP_KEY:
5791 		pointer_desc = "key ";
5792 		break;
5793 	case PTR_TO_MAP_VALUE:
5794 		pointer_desc = "value ";
5795 		break;
5796 	case PTR_TO_CTX:
5797 		pointer_desc = "context ";
5798 		break;
5799 	case PTR_TO_STACK:
5800 		pointer_desc = "stack ";
5801 		/* The stack spill tracking logic in check_stack_write_fixed_off()
5802 		 * and check_stack_read_fixed_off() relies on stack accesses being
5803 		 * aligned.
5804 		 */
5805 		strict = true;
5806 		break;
5807 	case PTR_TO_SOCKET:
5808 		pointer_desc = "sock ";
5809 		break;
5810 	case PTR_TO_SOCK_COMMON:
5811 		pointer_desc = "sock_common ";
5812 		break;
5813 	case PTR_TO_TCP_SOCK:
5814 		pointer_desc = "tcp_sock ";
5815 		break;
5816 	case PTR_TO_XDP_SOCK:
5817 		pointer_desc = "xdp_sock ";
5818 		break;
5819 	default:
5820 		break;
5821 	}
5822 	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
5823 					   strict);
5824 }
5825 
5826 /* starting from main bpf function walk all instructions of the function
5827  * and recursively walk all callees that given function can call.
5828  * Ignore jump and exit insns.
5829  * Since recursion is prevented by check_cfg() this algorithm
5830  * only needs a local stack of MAX_CALL_FRAMES to remember callsites
5831  */
5832 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)
5833 {
5834 	struct bpf_subprog_info *subprog = env->subprog_info;
5835 	struct bpf_insn *insn = env->prog->insnsi;
5836 	int depth = 0, frame = 0, i, subprog_end;
5837 	bool tail_call_reachable = false;
5838 	int ret_insn[MAX_CALL_FRAMES];
5839 	int ret_prog[MAX_CALL_FRAMES];
5840 	int j;
5841 
5842 	i = subprog[idx].start;
5843 process_func:
5844 	/* protect against potential stack overflow that might happen when
5845 	 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
5846 	 * depth for such case down to 256 so that the worst case scenario
5847 	 * would result in 8k stack size (32 which is tailcall limit * 256 =
5848 	 * 8k).
5849 	 *
5850 	 * To get the idea what might happen, see an example:
5851 	 * func1 -> sub rsp, 128
5852 	 *  subfunc1 -> sub rsp, 256
5853 	 *  tailcall1 -> add rsp, 256
5854 	 *   func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
5855 	 *   subfunc2 -> sub rsp, 64
5856 	 *   subfunc22 -> sub rsp, 128
5857 	 *   tailcall2 -> add rsp, 128
5858 	 *    func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
5859 	 *
5860 	 * tailcall will unwind the current stack frame but it will not get rid
5861 	 * of caller's stack as shown on the example above.
5862 	 */
5863 	if (idx && subprog[idx].has_tail_call && depth >= 256) {
5864 		verbose(env,
5865 			"tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
5866 			depth);
5867 		return -EACCES;
5868 	}
5869 	/* round up to 32-bytes, since this is granularity
5870 	 * of interpreter stack size
5871 	 */
5872 	depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
5873 	if (depth > MAX_BPF_STACK) {
5874 		verbose(env, "combined stack size of %d calls is %d. Too large\n",
5875 			frame + 1, depth);
5876 		return -EACCES;
5877 	}
5878 continue_func:
5879 	subprog_end = subprog[idx + 1].start;
5880 	for (; i < subprog_end; i++) {
5881 		int next_insn, sidx;
5882 
5883 		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
5884 			continue;
5885 		/* remember insn and function to return to */
5886 		ret_insn[frame] = i + 1;
5887 		ret_prog[frame] = idx;
5888 
5889 		/* find the callee */
5890 		next_insn = i + insn[i].imm + 1;
5891 		sidx = find_subprog(env, next_insn);
5892 		if (sidx < 0) {
5893 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
5894 				  next_insn);
5895 			return -EFAULT;
5896 		}
5897 		if (subprog[sidx].is_async_cb) {
5898 			if (subprog[sidx].has_tail_call) {
5899 				verbose(env, "verifier bug. subprog has tail_call and async cb\n");
5900 				return -EFAULT;
5901 			}
5902 			/* async callbacks don't increase bpf prog stack size unless called directly */
5903 			if (!bpf_pseudo_call(insn + i))
5904 				continue;
5905 		}
5906 		i = next_insn;
5907 		idx = sidx;
5908 
5909 		if (subprog[idx].has_tail_call)
5910 			tail_call_reachable = true;
5911 
5912 		frame++;
5913 		if (frame >= MAX_CALL_FRAMES) {
5914 			verbose(env, "the call stack of %d frames is too deep !\n",
5915 				frame);
5916 			return -E2BIG;
5917 		}
5918 		goto process_func;
5919 	}
5920 	/* if tail call got detected across bpf2bpf calls then mark each of the
5921 	 * currently present subprog frames as tail call reachable subprogs;
5922 	 * this info will be utilized by JIT so that we will be preserving the
5923 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
5924 	 */
5925 	if (tail_call_reachable)
5926 		for (j = 0; j < frame; j++)
5927 			subprog[ret_prog[j]].tail_call_reachable = true;
5928 	if (subprog[0].tail_call_reachable)
5929 		env->prog->aux->tail_call_reachable = true;
5930 
5931 	/* end of for() loop means the last insn of the 'subprog'
5932 	 * was reached. Doesn't matter whether it was JA or EXIT
5933 	 */
5934 	if (frame == 0)
5935 		return 0;
5936 	depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
5937 	frame--;
5938 	i = ret_insn[frame];
5939 	idx = ret_prog[frame];
5940 	goto continue_func;
5941 }
5942 
5943 static int check_max_stack_depth(struct bpf_verifier_env *env)
5944 {
5945 	struct bpf_subprog_info *si = env->subprog_info;
5946 	int ret;
5947 
5948 	for (int i = 0; i < env->subprog_cnt; i++) {
5949 		if (!i || si[i].is_async_cb) {
5950 			ret = check_max_stack_depth_subprog(env, i);
5951 			if (ret < 0)
5952 				return ret;
5953 		}
5954 		continue;
5955 	}
5956 	return 0;
5957 }
5958 
5959 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
5960 static int get_callee_stack_depth(struct bpf_verifier_env *env,
5961 				  const struct bpf_insn *insn, int idx)
5962 {
5963 	int start = idx + insn->imm + 1, subprog;
5964 
5965 	subprog = find_subprog(env, start);
5966 	if (subprog < 0) {
5967 		WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
5968 			  start);
5969 		return -EFAULT;
5970 	}
5971 	return env->subprog_info[subprog].stack_depth;
5972 }
5973 #endif
5974 
5975 static int __check_buffer_access(struct bpf_verifier_env *env,
5976 				 const char *buf_info,
5977 				 const struct bpf_reg_state *reg,
5978 				 int regno, int off, int size)
5979 {
5980 	if (off < 0) {
5981 		verbose(env,
5982 			"R%d invalid %s buffer access: off=%d, size=%d\n",
5983 			regno, buf_info, off, size);
5984 		return -EACCES;
5985 	}
5986 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
5987 		char tn_buf[48];
5988 
5989 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5990 		verbose(env,
5991 			"R%d invalid variable buffer offset: off=%d, var_off=%s\n",
5992 			regno, off, tn_buf);
5993 		return -EACCES;
5994 	}
5995 
5996 	return 0;
5997 }
5998 
5999 static int check_tp_buffer_access(struct bpf_verifier_env *env,
6000 				  const struct bpf_reg_state *reg,
6001 				  int regno, int off, int size)
6002 {
6003 	int err;
6004 
6005 	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
6006 	if (err)
6007 		return err;
6008 
6009 	if (off + size > env->prog->aux->max_tp_access)
6010 		env->prog->aux->max_tp_access = off + size;
6011 
6012 	return 0;
6013 }
6014 
6015 static int check_buffer_access(struct bpf_verifier_env *env,
6016 			       const struct bpf_reg_state *reg,
6017 			       int regno, int off, int size,
6018 			       bool zero_size_allowed,
6019 			       u32 *max_access)
6020 {
6021 	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
6022 	int err;
6023 
6024 	err = __check_buffer_access(env, buf_info, reg, regno, off, size);
6025 	if (err)
6026 		return err;
6027 
6028 	if (off + size > *max_access)
6029 		*max_access = off + size;
6030 
6031 	return 0;
6032 }
6033 
6034 /* BPF architecture zero extends alu32 ops into 64-bit registesr */
6035 static void zext_32_to_64(struct bpf_reg_state *reg)
6036 {
6037 	reg->var_off = tnum_subreg(reg->var_off);
6038 	__reg_assign_32_into_64(reg);
6039 }
6040 
6041 /* truncate register to smaller size (in bytes)
6042  * must be called with size < BPF_REG_SIZE
6043  */
6044 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
6045 {
6046 	u64 mask;
6047 
6048 	/* clear high bits in bit representation */
6049 	reg->var_off = tnum_cast(reg->var_off, size);
6050 
6051 	/* fix arithmetic bounds */
6052 	mask = ((u64)1 << (size * 8)) - 1;
6053 	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
6054 		reg->umin_value &= mask;
6055 		reg->umax_value &= mask;
6056 	} else {
6057 		reg->umin_value = 0;
6058 		reg->umax_value = mask;
6059 	}
6060 	reg->smin_value = reg->umin_value;
6061 	reg->smax_value = reg->umax_value;
6062 
6063 	/* If size is smaller than 32bit register the 32bit register
6064 	 * values are also truncated so we push 64-bit bounds into
6065 	 * 32-bit bounds. Above were truncated < 32-bits already.
6066 	 */
6067 	if (size >= 4)
6068 		return;
6069 	__reg_combine_64_into_32(reg);
6070 }
6071 
6072 static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
6073 {
6074 	if (size == 1) {
6075 		reg->smin_value = reg->s32_min_value = S8_MIN;
6076 		reg->smax_value = reg->s32_max_value = S8_MAX;
6077 	} else if (size == 2) {
6078 		reg->smin_value = reg->s32_min_value = S16_MIN;
6079 		reg->smax_value = reg->s32_max_value = S16_MAX;
6080 	} else {
6081 		/* size == 4 */
6082 		reg->smin_value = reg->s32_min_value = S32_MIN;
6083 		reg->smax_value = reg->s32_max_value = S32_MAX;
6084 	}
6085 	reg->umin_value = reg->u32_min_value = 0;
6086 	reg->umax_value = U64_MAX;
6087 	reg->u32_max_value = U32_MAX;
6088 	reg->var_off = tnum_unknown;
6089 }
6090 
6091 static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
6092 {
6093 	s64 init_s64_max, init_s64_min, s64_max, s64_min, u64_cval;
6094 	u64 top_smax_value, top_smin_value;
6095 	u64 num_bits = size * 8;
6096 
6097 	if (tnum_is_const(reg->var_off)) {
6098 		u64_cval = reg->var_off.value;
6099 		if (size == 1)
6100 			reg->var_off = tnum_const((s8)u64_cval);
6101 		else if (size == 2)
6102 			reg->var_off = tnum_const((s16)u64_cval);
6103 		else
6104 			/* size == 4 */
6105 			reg->var_off = tnum_const((s32)u64_cval);
6106 
6107 		u64_cval = reg->var_off.value;
6108 		reg->smax_value = reg->smin_value = u64_cval;
6109 		reg->umax_value = reg->umin_value = u64_cval;
6110 		reg->s32_max_value = reg->s32_min_value = u64_cval;
6111 		reg->u32_max_value = reg->u32_min_value = u64_cval;
6112 		return;
6113 	}
6114 
6115 	top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits;
6116 	top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits;
6117 
6118 	if (top_smax_value != top_smin_value)
6119 		goto out;
6120 
6121 	/* find the s64_min and s64_min after sign extension */
6122 	if (size == 1) {
6123 		init_s64_max = (s8)reg->smax_value;
6124 		init_s64_min = (s8)reg->smin_value;
6125 	} else if (size == 2) {
6126 		init_s64_max = (s16)reg->smax_value;
6127 		init_s64_min = (s16)reg->smin_value;
6128 	} else {
6129 		init_s64_max = (s32)reg->smax_value;
6130 		init_s64_min = (s32)reg->smin_value;
6131 	}
6132 
6133 	s64_max = max(init_s64_max, init_s64_min);
6134 	s64_min = min(init_s64_max, init_s64_min);
6135 
6136 	/* both of s64_max/s64_min positive or negative */
6137 	if ((s64_max >= 0) == (s64_min >= 0)) {
6138 		reg->smin_value = reg->s32_min_value = s64_min;
6139 		reg->smax_value = reg->s32_max_value = s64_max;
6140 		reg->umin_value = reg->u32_min_value = s64_min;
6141 		reg->umax_value = reg->u32_max_value = s64_max;
6142 		reg->var_off = tnum_range(s64_min, s64_max);
6143 		return;
6144 	}
6145 
6146 out:
6147 	set_sext64_default_val(reg, size);
6148 }
6149 
6150 static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
6151 {
6152 	if (size == 1) {
6153 		reg->s32_min_value = S8_MIN;
6154 		reg->s32_max_value = S8_MAX;
6155 	} else {
6156 		/* size == 2 */
6157 		reg->s32_min_value = S16_MIN;
6158 		reg->s32_max_value = S16_MAX;
6159 	}
6160 	reg->u32_min_value = 0;
6161 	reg->u32_max_value = U32_MAX;
6162 }
6163 
6164 static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
6165 {
6166 	s32 init_s32_max, init_s32_min, s32_max, s32_min, u32_val;
6167 	u32 top_smax_value, top_smin_value;
6168 	u32 num_bits = size * 8;
6169 
6170 	if (tnum_is_const(reg->var_off)) {
6171 		u32_val = reg->var_off.value;
6172 		if (size == 1)
6173 			reg->var_off = tnum_const((s8)u32_val);
6174 		else
6175 			reg->var_off = tnum_const((s16)u32_val);
6176 
6177 		u32_val = reg->var_off.value;
6178 		reg->s32_min_value = reg->s32_max_value = u32_val;
6179 		reg->u32_min_value = reg->u32_max_value = u32_val;
6180 		return;
6181 	}
6182 
6183 	top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits;
6184 	top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits;
6185 
6186 	if (top_smax_value != top_smin_value)
6187 		goto out;
6188 
6189 	/* find the s32_min and s32_min after sign extension */
6190 	if (size == 1) {
6191 		init_s32_max = (s8)reg->s32_max_value;
6192 		init_s32_min = (s8)reg->s32_min_value;
6193 	} else {
6194 		/* size == 2 */
6195 		init_s32_max = (s16)reg->s32_max_value;
6196 		init_s32_min = (s16)reg->s32_min_value;
6197 	}
6198 	s32_max = max(init_s32_max, init_s32_min);
6199 	s32_min = min(init_s32_max, init_s32_min);
6200 
6201 	if ((s32_min >= 0) == (s32_max >= 0)) {
6202 		reg->s32_min_value = s32_min;
6203 		reg->s32_max_value = s32_max;
6204 		reg->u32_min_value = (u32)s32_min;
6205 		reg->u32_max_value = (u32)s32_max;
6206 		return;
6207 	}
6208 
6209 out:
6210 	set_sext32_default_val(reg, size);
6211 }
6212 
6213 static bool bpf_map_is_rdonly(const struct bpf_map *map)
6214 {
6215 	/* A map is considered read-only if the following condition are true:
6216 	 *
6217 	 * 1) BPF program side cannot change any of the map content. The
6218 	 *    BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
6219 	 *    and was set at map creation time.
6220 	 * 2) The map value(s) have been initialized from user space by a
6221 	 *    loader and then "frozen", such that no new map update/delete
6222 	 *    operations from syscall side are possible for the rest of
6223 	 *    the map's lifetime from that point onwards.
6224 	 * 3) Any parallel/pending map update/delete operations from syscall
6225 	 *    side have been completed. Only after that point, it's safe to
6226 	 *    assume that map value(s) are immutable.
6227 	 */
6228 	return (map->map_flags & BPF_F_RDONLY_PROG) &&
6229 	       READ_ONCE(map->frozen) &&
6230 	       !bpf_map_write_active(map);
6231 }
6232 
6233 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
6234 			       bool is_ldsx)
6235 {
6236 	void *ptr;
6237 	u64 addr;
6238 	int err;
6239 
6240 	err = map->ops->map_direct_value_addr(map, &addr, off);
6241 	if (err)
6242 		return err;
6243 	ptr = (void *)(long)addr + off;
6244 
6245 	switch (size) {
6246 	case sizeof(u8):
6247 		*val = is_ldsx ? (s64)*(s8 *)ptr : (u64)*(u8 *)ptr;
6248 		break;
6249 	case sizeof(u16):
6250 		*val = is_ldsx ? (s64)*(s16 *)ptr : (u64)*(u16 *)ptr;
6251 		break;
6252 	case sizeof(u32):
6253 		*val = is_ldsx ? (s64)*(s32 *)ptr : (u64)*(u32 *)ptr;
6254 		break;
6255 	case sizeof(u64):
6256 		*val = *(u64 *)ptr;
6257 		break;
6258 	default:
6259 		return -EINVAL;
6260 	}
6261 	return 0;
6262 }
6263 
6264 #define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
6265 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
6266 #define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
6267 #define BTF_TYPE_SAFE_TRUSTED_OR_NULL(__type)  __PASTE(__type, __safe_trusted_or_null)
6268 
6269 /*
6270  * Allow list few fields as RCU trusted or full trusted.
6271  * This logic doesn't allow mix tagging and will be removed once GCC supports
6272  * btf_type_tag.
6273  */
6274 
6275 /* RCU trusted: these fields are trusted in RCU CS and never NULL */
6276 BTF_TYPE_SAFE_RCU(struct task_struct) {
6277 	const cpumask_t *cpus_ptr;
6278 	struct css_set __rcu *cgroups;
6279 	struct task_struct __rcu *real_parent;
6280 	struct task_struct *group_leader;
6281 };
6282 
6283 BTF_TYPE_SAFE_RCU(struct cgroup) {
6284 	/* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */
6285 	struct kernfs_node *kn;
6286 };
6287 
6288 BTF_TYPE_SAFE_RCU(struct css_set) {
6289 	struct cgroup *dfl_cgrp;
6290 };
6291 
6292 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */
6293 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) {
6294 	struct file __rcu *exe_file;
6295 };
6296 
6297 /* skb->sk, req->sk are not RCU protected, but we mark them as such
6298  * because bpf prog accessible sockets are SOCK_RCU_FREE.
6299  */
6300 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) {
6301 	struct sock *sk;
6302 };
6303 
6304 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) {
6305 	struct sock *sk;
6306 };
6307 
6308 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */
6309 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) {
6310 	struct seq_file *seq;
6311 };
6312 
6313 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) {
6314 	struct bpf_iter_meta *meta;
6315 	struct task_struct *task;
6316 };
6317 
6318 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) {
6319 	struct file *file;
6320 };
6321 
6322 BTF_TYPE_SAFE_TRUSTED(struct file) {
6323 	struct inode *f_inode;
6324 };
6325 
6326 BTF_TYPE_SAFE_TRUSTED(struct dentry) {
6327 	/* no negative dentry-s in places where bpf can see it */
6328 	struct inode *d_inode;
6329 };
6330 
6331 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
6332 	struct sock *sk;
6333 };
6334 
6335 static bool type_is_rcu(struct bpf_verifier_env *env,
6336 			struct bpf_reg_state *reg,
6337 			const char *field_name, u32 btf_id)
6338 {
6339 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct));
6340 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup));
6341 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set));
6342 
6343 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu");
6344 }
6345 
6346 static bool type_is_rcu_or_null(struct bpf_verifier_env *env,
6347 				struct bpf_reg_state *reg,
6348 				const char *field_name, u32 btf_id)
6349 {
6350 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct));
6351 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff));
6352 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock));
6353 
6354 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null");
6355 }
6356 
6357 static bool type_is_trusted(struct bpf_verifier_env *env,
6358 			    struct bpf_reg_state *reg,
6359 			    const char *field_name, u32 btf_id)
6360 {
6361 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta));
6362 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
6363 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
6364 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
6365 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct dentry));
6366 
6367 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
6368 }
6369 
6370 static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
6371 				    struct bpf_reg_state *reg,
6372 				    const char *field_name, u32 btf_id)
6373 {
6374 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));
6375 
6376 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
6377 					  "__safe_trusted_or_null");
6378 }
6379 
6380 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
6381 				   struct bpf_reg_state *regs,
6382 				   int regno, int off, int size,
6383 				   enum bpf_access_type atype,
6384 				   int value_regno)
6385 {
6386 	struct bpf_reg_state *reg = regs + regno;
6387 	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
6388 	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
6389 	const char *field_name = NULL;
6390 	enum bpf_type_flag flag = 0;
6391 	u32 btf_id = 0;
6392 	int ret;
6393 
6394 	if (!env->allow_ptr_leaks) {
6395 		verbose(env,
6396 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6397 			tname);
6398 		return -EPERM;
6399 	}
6400 	if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) {
6401 		verbose(env,
6402 			"Cannot access kernel 'struct %s' from non-GPL compatible program\n",
6403 			tname);
6404 		return -EINVAL;
6405 	}
6406 	if (off < 0) {
6407 		verbose(env,
6408 			"R%d is ptr_%s invalid negative access: off=%d\n",
6409 			regno, tname, off);
6410 		return -EACCES;
6411 	}
6412 	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
6413 		char tn_buf[48];
6414 
6415 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6416 		verbose(env,
6417 			"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
6418 			regno, tname, off, tn_buf);
6419 		return -EACCES;
6420 	}
6421 
6422 	if (reg->type & MEM_USER) {
6423 		verbose(env,
6424 			"R%d is ptr_%s access user memory: off=%d\n",
6425 			regno, tname, off);
6426 		return -EACCES;
6427 	}
6428 
6429 	if (reg->type & MEM_PERCPU) {
6430 		verbose(env,
6431 			"R%d is ptr_%s access percpu memory: off=%d\n",
6432 			regno, tname, off);
6433 		return -EACCES;
6434 	}
6435 
6436 	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
6437 		if (!btf_is_kernel(reg->btf)) {
6438 			verbose(env, "verifier internal error: reg->btf must be kernel btf\n");
6439 			return -EFAULT;
6440 		}
6441 		ret = env->ops->btf_struct_access(&env->log, reg, off, size);
6442 	} else {
6443 		/* Writes are permitted with default btf_struct_access for
6444 		 * program allocated objects (which always have ref_obj_id > 0),
6445 		 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
6446 		 */
6447 		if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
6448 			verbose(env, "only read is supported\n");
6449 			return -EACCES;
6450 		}
6451 
6452 		if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
6453 		    !reg->ref_obj_id) {
6454 			verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n");
6455 			return -EFAULT;
6456 		}
6457 
6458 		ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name);
6459 	}
6460 
6461 	if (ret < 0)
6462 		return ret;
6463 
6464 	if (ret != PTR_TO_BTF_ID) {
6465 		/* just mark; */
6466 
6467 	} else if (type_flag(reg->type) & PTR_UNTRUSTED) {
6468 		/* If this is an untrusted pointer, all pointers formed by walking it
6469 		 * also inherit the untrusted flag.
6470 		 */
6471 		flag = PTR_UNTRUSTED;
6472 
6473 	} else if (is_trusted_reg(reg) || is_rcu_reg(reg)) {
6474 		/* By default any pointer obtained from walking a trusted pointer is no
6475 		 * longer trusted, unless the field being accessed has explicitly been
6476 		 * marked as inheriting its parent's state of trust (either full or RCU).
6477 		 * For example:
6478 		 * 'cgroups' pointer is untrusted if task->cgroups dereference
6479 		 * happened in a sleepable program outside of bpf_rcu_read_lock()
6480 		 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU).
6481 		 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED.
6482 		 *
6483 		 * A regular RCU-protected pointer with __rcu tag can also be deemed
6484 		 * trusted if we are in an RCU CS. Such pointer can be NULL.
6485 		 */
6486 		if (type_is_trusted(env, reg, field_name, btf_id)) {
6487 			flag |= PTR_TRUSTED;
6488 		} else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) {
6489 			flag |= PTR_TRUSTED | PTR_MAYBE_NULL;
6490 		} else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) {
6491 			if (type_is_rcu(env, reg, field_name, btf_id)) {
6492 				/* ignore __rcu tag and mark it MEM_RCU */
6493 				flag |= MEM_RCU;
6494 			} else if (flag & MEM_RCU ||
6495 				   type_is_rcu_or_null(env, reg, field_name, btf_id)) {
6496 				/* __rcu tagged pointers can be NULL */
6497 				flag |= MEM_RCU | PTR_MAYBE_NULL;
6498 
6499 				/* We always trust them */
6500 				if (type_is_rcu_or_null(env, reg, field_name, btf_id) &&
6501 				    flag & PTR_UNTRUSTED)
6502 					flag &= ~PTR_UNTRUSTED;
6503 			} else if (flag & (MEM_PERCPU | MEM_USER)) {
6504 				/* keep as-is */
6505 			} else {
6506 				/* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */
6507 				clear_trusted_flags(&flag);
6508 			}
6509 		} else {
6510 			/*
6511 			 * If not in RCU CS or MEM_RCU pointer can be NULL then
6512 			 * aggressively mark as untrusted otherwise such
6513 			 * pointers will be plain PTR_TO_BTF_ID without flags
6514 			 * and will be allowed to be passed into helpers for
6515 			 * compat reasons.
6516 			 */
6517 			flag = PTR_UNTRUSTED;
6518 		}
6519 	} else {
6520 		/* Old compat. Deprecated */
6521 		clear_trusted_flags(&flag);
6522 	}
6523 
6524 	if (atype == BPF_READ && value_regno >= 0)
6525 		mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
6526 
6527 	return 0;
6528 }
6529 
6530 static int check_ptr_to_map_access(struct bpf_verifier_env *env,
6531 				   struct bpf_reg_state *regs,
6532 				   int regno, int off, int size,
6533 				   enum bpf_access_type atype,
6534 				   int value_regno)
6535 {
6536 	struct bpf_reg_state *reg = regs + regno;
6537 	struct bpf_map *map = reg->map_ptr;
6538 	struct bpf_reg_state map_reg;
6539 	enum bpf_type_flag flag = 0;
6540 	const struct btf_type *t;
6541 	const char *tname;
6542 	u32 btf_id;
6543 	int ret;
6544 
6545 	if (!btf_vmlinux) {
6546 		verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
6547 		return -ENOTSUPP;
6548 	}
6549 
6550 	if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
6551 		verbose(env, "map_ptr access not supported for map type %d\n",
6552 			map->map_type);
6553 		return -ENOTSUPP;
6554 	}
6555 
6556 	t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
6557 	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
6558 
6559 	if (!env->allow_ptr_leaks) {
6560 		verbose(env,
6561 			"'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
6562 			tname);
6563 		return -EPERM;
6564 	}
6565 
6566 	if (off < 0) {
6567 		verbose(env, "R%d is %s invalid negative access: off=%d\n",
6568 			regno, tname, off);
6569 		return -EACCES;
6570 	}
6571 
6572 	if (atype != BPF_READ) {
6573 		verbose(env, "only read from %s is supported\n", tname);
6574 		return -EACCES;
6575 	}
6576 
6577 	/* Simulate access to a PTR_TO_BTF_ID */
6578 	memset(&map_reg, 0, sizeof(map_reg));
6579 	mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0);
6580 	ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL);
6581 	if (ret < 0)
6582 		return ret;
6583 
6584 	if (value_regno >= 0)
6585 		mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);
6586 
6587 	return 0;
6588 }
6589 
6590 /* Check that the stack access at the given offset is within bounds. The
6591  * maximum valid offset is -1.
6592  *
6593  * The minimum valid offset is -MAX_BPF_STACK for writes, and
6594  * -state->allocated_stack for reads.
6595  */
6596 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
6597                                           s64 off,
6598                                           struct bpf_func_state *state,
6599                                           enum bpf_access_type t)
6600 {
6601 	int min_valid_off;
6602 
6603 	if (t == BPF_WRITE || env->allow_uninit_stack)
6604 		min_valid_off = -MAX_BPF_STACK;
6605 	else
6606 		min_valid_off = -state->allocated_stack;
6607 
6608 	if (off < min_valid_off || off > -1)
6609 		return -EACCES;
6610 	return 0;
6611 }
6612 
6613 /* Check that the stack access at 'regno + off' falls within the maximum stack
6614  * bounds.
6615  *
6616  * 'off' includes `regno->offset`, but not its dynamic part (if any).
6617  */
6618 static int check_stack_access_within_bounds(
6619 		struct bpf_verifier_env *env,
6620 		int regno, int off, int access_size,
6621 		enum bpf_access_src src, enum bpf_access_type type)
6622 {
6623 	struct bpf_reg_state *regs = cur_regs(env);
6624 	struct bpf_reg_state *reg = regs + regno;
6625 	struct bpf_func_state *state = func(env, reg);
6626 	s64 min_off, max_off;
6627 	int err;
6628 	char *err_extra;
6629 
6630 	if (src == ACCESS_HELPER)
6631 		/* We don't know if helpers are reading or writing (or both). */
6632 		err_extra = " indirect access to";
6633 	else if (type == BPF_READ)
6634 		err_extra = " read from";
6635 	else
6636 		err_extra = " write to";
6637 
6638 	if (tnum_is_const(reg->var_off)) {
6639 		min_off = (s64)reg->var_off.value + off;
6640 		max_off = min_off + access_size;
6641 	} else {
6642 		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
6643 		    reg->smin_value <= -BPF_MAX_VAR_OFF) {
6644 			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
6645 				err_extra, regno);
6646 			return -EACCES;
6647 		}
6648 		min_off = reg->smin_value + off;
6649 		max_off = reg->smax_value + off + access_size;
6650 	}
6651 
6652 	err = check_stack_slot_within_bounds(env, min_off, state, type);
6653 	if (!err && max_off > 0)
6654 		err = -EINVAL; /* out of stack access into non-negative offsets */
6655 	if (!err && access_size < 0)
6656 		/* access_size should not be negative (or overflow an int); others checks
6657 		 * along the way should have prevented such an access.
6658 		 */
6659 		err = -EFAULT; /* invalid negative access size; integer overflow? */
6660 
6661 	if (err) {
6662 		if (tnum_is_const(reg->var_off)) {
6663 			verbose(env, "invalid%s stack R%d off=%d size=%d\n",
6664 				err_extra, regno, off, access_size);
6665 		} else {
6666 			char tn_buf[48];
6667 
6668 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6669 			verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
6670 				err_extra, regno, tn_buf, access_size);
6671 		}
6672 		return err;
6673 	}
6674 
6675 	return grow_stack_state(env, state, round_up(-min_off, BPF_REG_SIZE));
6676 }
6677 
6678 /* check whether memory at (regno + off) is accessible for t = (read | write)
6679  * if t==write, value_regno is a register which value is stored into memory
6680  * if t==read, value_regno is a register which will receive the value from memory
6681  * if t==write && value_regno==-1, some unknown value is stored into memory
6682  * if t==read && value_regno==-1, don't care what we read from memory
6683  */
6684 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
6685 			    int off, int bpf_size, enum bpf_access_type t,
6686 			    int value_regno, bool strict_alignment_once, bool is_ldsx)
6687 {
6688 	struct bpf_reg_state *regs = cur_regs(env);
6689 	struct bpf_reg_state *reg = regs + regno;
6690 	int size, err = 0;
6691 
6692 	size = bpf_size_to_bytes(bpf_size);
6693 	if (size < 0)
6694 		return size;
6695 
6696 	/* alignment checks will add in reg->off themselves */
6697 	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
6698 	if (err)
6699 		return err;
6700 
6701 	/* for access checks, reg->off is just part of off */
6702 	off += reg->off;
6703 
6704 	if (reg->type == PTR_TO_MAP_KEY) {
6705 		if (t == BPF_WRITE) {
6706 			verbose(env, "write to change key R%d not allowed\n", regno);
6707 			return -EACCES;
6708 		}
6709 
6710 		err = check_mem_region_access(env, regno, off, size,
6711 					      reg->map_ptr->key_size, false);
6712 		if (err)
6713 			return err;
6714 		if (value_regno >= 0)
6715 			mark_reg_unknown(env, regs, value_regno);
6716 	} else if (reg->type == PTR_TO_MAP_VALUE) {
6717 		struct btf_field *kptr_field = NULL;
6718 
6719 		if (t == BPF_WRITE && value_regno >= 0 &&
6720 		    is_pointer_value(env, value_regno)) {
6721 			verbose(env, "R%d leaks addr into map\n", value_regno);
6722 			return -EACCES;
6723 		}
6724 		err = check_map_access_type(env, regno, off, size, t);
6725 		if (err)
6726 			return err;
6727 		err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
6728 		if (err)
6729 			return err;
6730 		if (tnum_is_const(reg->var_off))
6731 			kptr_field = btf_record_find(reg->map_ptr->record,
6732 						     off + reg->var_off.value, BPF_KPTR);
6733 		if (kptr_field) {
6734 			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
6735 		} else if (t == BPF_READ && value_regno >= 0) {
6736 			struct bpf_map *map = reg->map_ptr;
6737 
6738 			/* if map is read-only, track its contents as scalars */
6739 			if (tnum_is_const(reg->var_off) &&
6740 			    bpf_map_is_rdonly(map) &&
6741 			    map->ops->map_direct_value_addr) {
6742 				int map_off = off + reg->var_off.value;
6743 				u64 val = 0;
6744 
6745 				err = bpf_map_direct_read(map, map_off, size,
6746 							  &val, is_ldsx);
6747 				if (err)
6748 					return err;
6749 
6750 				regs[value_regno].type = SCALAR_VALUE;
6751 				__mark_reg_known(&regs[value_regno], val);
6752 			} else {
6753 				mark_reg_unknown(env, regs, value_regno);
6754 			}
6755 		}
6756 	} else if (base_type(reg->type) == PTR_TO_MEM) {
6757 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
6758 
6759 		if (type_may_be_null(reg->type)) {
6760 			verbose(env, "R%d invalid mem access '%s'\n", regno,
6761 				reg_type_str(env, reg->type));
6762 			return -EACCES;
6763 		}
6764 
6765 		if (t == BPF_WRITE && rdonly_mem) {
6766 			verbose(env, "R%d cannot write into %s\n",
6767 				regno, reg_type_str(env, reg->type));
6768 			return -EACCES;
6769 		}
6770 
6771 		if (t == BPF_WRITE && value_regno >= 0 &&
6772 		    is_pointer_value(env, value_regno)) {
6773 			verbose(env, "R%d leaks addr into mem\n", value_regno);
6774 			return -EACCES;
6775 		}
6776 
6777 		err = check_mem_region_access(env, regno, off, size,
6778 					      reg->mem_size, false);
6779 		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
6780 			mark_reg_unknown(env, regs, value_regno);
6781 	} else if (reg->type == PTR_TO_CTX) {
6782 		enum bpf_reg_type reg_type = SCALAR_VALUE;
6783 		struct btf *btf = NULL;
6784 		u32 btf_id = 0;
6785 
6786 		if (t == BPF_WRITE && value_regno >= 0 &&
6787 		    is_pointer_value(env, value_regno)) {
6788 			verbose(env, "R%d leaks addr into ctx\n", value_regno);
6789 			return -EACCES;
6790 		}
6791 
6792 		err = check_ptr_off_reg(env, reg, regno);
6793 		if (err < 0)
6794 			return err;
6795 
6796 		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,
6797 				       &btf_id);
6798 		if (err)
6799 			verbose_linfo(env, insn_idx, "; ");
6800 		if (!err && t == BPF_READ && value_regno >= 0) {
6801 			/* ctx access returns either a scalar, or a
6802 			 * PTR_TO_PACKET[_META,_END]. In the latter
6803 			 * case, we know the offset is zero.
6804 			 */
6805 			if (reg_type == SCALAR_VALUE) {
6806 				mark_reg_unknown(env, regs, value_regno);
6807 			} else {
6808 				mark_reg_known_zero(env, regs,
6809 						    value_regno);
6810 				if (type_may_be_null(reg_type))
6811 					regs[value_regno].id = ++env->id_gen;
6812 				/* A load of ctx field could have different
6813 				 * actual load size with the one encoded in the
6814 				 * insn. When the dst is PTR, it is for sure not
6815 				 * a sub-register.
6816 				 */
6817 				regs[value_regno].subreg_def = DEF_NOT_SUBREG;
6818 				if (base_type(reg_type) == PTR_TO_BTF_ID) {
6819 					regs[value_regno].btf = btf;
6820 					regs[value_regno].btf_id = btf_id;
6821 				}
6822 			}
6823 			regs[value_regno].type = reg_type;
6824 		}
6825 
6826 	} else if (reg->type == PTR_TO_STACK) {
6827 		/* Basic bounds checks. */
6828 		err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
6829 		if (err)
6830 			return err;
6831 
6832 		if (t == BPF_READ)
6833 			err = check_stack_read(env, regno, off, size,
6834 					       value_regno);
6835 		else
6836 			err = check_stack_write(env, regno, off, size,
6837 						value_regno, insn_idx);
6838 	} else if (reg_is_pkt_pointer(reg)) {
6839 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
6840 			verbose(env, "cannot write into packet\n");
6841 			return -EACCES;
6842 		}
6843 		if (t == BPF_WRITE && value_regno >= 0 &&
6844 		    is_pointer_value(env, value_regno)) {
6845 			verbose(env, "R%d leaks addr into packet\n",
6846 				value_regno);
6847 			return -EACCES;
6848 		}
6849 		err = check_packet_access(env, regno, off, size, false);
6850 		if (!err && t == BPF_READ && value_regno >= 0)
6851 			mark_reg_unknown(env, regs, value_regno);
6852 	} else if (reg->type == PTR_TO_FLOW_KEYS) {
6853 		if (t == BPF_WRITE && value_regno >= 0 &&
6854 		    is_pointer_value(env, value_regno)) {
6855 			verbose(env, "R%d leaks addr into flow keys\n",
6856 				value_regno);
6857 			return -EACCES;
6858 		}
6859 
6860 		err = check_flow_keys_access(env, off, size);
6861 		if (!err && t == BPF_READ && value_regno >= 0)
6862 			mark_reg_unknown(env, regs, value_regno);
6863 	} else if (type_is_sk_pointer(reg->type)) {
6864 		if (t == BPF_WRITE) {
6865 			verbose(env, "R%d cannot write into %s\n",
6866 				regno, reg_type_str(env, reg->type));
6867 			return -EACCES;
6868 		}
6869 		err = check_sock_access(env, insn_idx, regno, off, size, t);
6870 		if (!err && value_regno >= 0)
6871 			mark_reg_unknown(env, regs, value_regno);
6872 	} else if (reg->type == PTR_TO_TP_BUFFER) {
6873 		err = check_tp_buffer_access(env, reg, regno, off, size);
6874 		if (!err && t == BPF_READ && value_regno >= 0)
6875 			mark_reg_unknown(env, regs, value_regno);
6876 	} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
6877 		   !type_may_be_null(reg->type)) {
6878 		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
6879 					      value_regno);
6880 	} else if (reg->type == CONST_PTR_TO_MAP) {
6881 		err = check_ptr_to_map_access(env, regs, regno, off, size, t,
6882 					      value_regno);
6883 	} else if (base_type(reg->type) == PTR_TO_BUF) {
6884 		bool rdonly_mem = type_is_rdonly_mem(reg->type);
6885 		u32 *max_access;
6886 
6887 		if (rdonly_mem) {
6888 			if (t == BPF_WRITE) {
6889 				verbose(env, "R%d cannot write into %s\n",
6890 					regno, reg_type_str(env, reg->type));
6891 				return -EACCES;
6892 			}
6893 			max_access = &env->prog->aux->max_rdonly_access;
6894 		} else {
6895 			max_access = &env->prog->aux->max_rdwr_access;
6896 		}
6897 
6898 		err = check_buffer_access(env, reg, regno, off, size, false,
6899 					  max_access);
6900 
6901 		if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
6902 			mark_reg_unknown(env, regs, value_regno);
6903 	} else {
6904 		verbose(env, "R%d invalid mem access '%s'\n", regno,
6905 			reg_type_str(env, reg->type));
6906 		return -EACCES;
6907 	}
6908 
6909 	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
6910 	    regs[value_regno].type == SCALAR_VALUE) {
6911 		if (!is_ldsx)
6912 			/* b/h/w load zero-extends, mark upper bits as known 0 */
6913 			coerce_reg_to_size(&regs[value_regno], size);
6914 		else
6915 			coerce_reg_to_size_sx(&regs[value_regno], size);
6916 	}
6917 	return err;
6918 }
6919 
6920 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
6921 {
6922 	int load_reg;
6923 	int err;
6924 
6925 	switch (insn->imm) {
6926 	case BPF_ADD:
6927 	case BPF_ADD | BPF_FETCH:
6928 	case BPF_AND:
6929 	case BPF_AND | BPF_FETCH:
6930 	case BPF_OR:
6931 	case BPF_OR | BPF_FETCH:
6932 	case BPF_XOR:
6933 	case BPF_XOR | BPF_FETCH:
6934 	case BPF_XCHG:
6935 	case BPF_CMPXCHG:
6936 		break;
6937 	default:
6938 		verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
6939 		return -EINVAL;
6940 	}
6941 
6942 	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
6943 		verbose(env, "invalid atomic operand size\n");
6944 		return -EINVAL;
6945 	}
6946 
6947 	/* check src1 operand */
6948 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
6949 	if (err)
6950 		return err;
6951 
6952 	/* check src2 operand */
6953 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
6954 	if (err)
6955 		return err;
6956 
6957 	if (insn->imm == BPF_CMPXCHG) {
6958 		/* Check comparison of R0 with memory location */
6959 		const u32 aux_reg = BPF_REG_0;
6960 
6961 		err = check_reg_arg(env, aux_reg, SRC_OP);
6962 		if (err)
6963 			return err;
6964 
6965 		if (is_pointer_value(env, aux_reg)) {
6966 			verbose(env, "R%d leaks addr into mem\n", aux_reg);
6967 			return -EACCES;
6968 		}
6969 	}
6970 
6971 	if (is_pointer_value(env, insn->src_reg)) {
6972 		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
6973 		return -EACCES;
6974 	}
6975 
6976 	if (is_ctx_reg(env, insn->dst_reg) ||
6977 	    is_pkt_reg(env, insn->dst_reg) ||
6978 	    is_flow_key_reg(env, insn->dst_reg) ||
6979 	    is_sk_reg(env, insn->dst_reg)) {
6980 		verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
6981 			insn->dst_reg,
6982 			reg_type_str(env, reg_state(env, insn->dst_reg)->type));
6983 		return -EACCES;
6984 	}
6985 
6986 	if (insn->imm & BPF_FETCH) {
6987 		if (insn->imm == BPF_CMPXCHG)
6988 			load_reg = BPF_REG_0;
6989 		else
6990 			load_reg = insn->src_reg;
6991 
6992 		/* check and record load of old value */
6993 		err = check_reg_arg(env, load_reg, DST_OP);
6994 		if (err)
6995 			return err;
6996 	} else {
6997 		/* This instruction accesses a memory location but doesn't
6998 		 * actually load it into a register.
6999 		 */
7000 		load_reg = -1;
7001 	}
7002 
7003 	/* Check whether we can read the memory, with second call for fetch
7004 	 * case to simulate the register fill.
7005 	 */
7006 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7007 			       BPF_SIZE(insn->code), BPF_READ, -1, true, false);
7008 	if (!err && load_reg >= 0)
7009 		err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7010 				       BPF_SIZE(insn->code), BPF_READ, load_reg,
7011 				       true, false);
7012 	if (err)
7013 		return err;
7014 
7015 	/* Check whether we can write into the same memory. */
7016 	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
7017 			       BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
7018 	if (err)
7019 		return err;
7020 
7021 	return 0;
7022 }
7023 
7024 /* When register 'regno' is used to read the stack (either directly or through
7025  * a helper function) make sure that it's within stack boundary and, depending
7026  * on the access type and privileges, that all elements of the stack are
7027  * initialized.
7028  *
7029  * 'off' includes 'regno->off', but not its dynamic part (if any).
7030  *
7031  * All registers that have been spilled on the stack in the slots within the
7032  * read offsets are marked as read.
7033  */
7034 static int check_stack_range_initialized(
7035 		struct bpf_verifier_env *env, int regno, int off,
7036 		int access_size, bool zero_size_allowed,
7037 		enum bpf_access_src type, struct bpf_call_arg_meta *meta)
7038 {
7039 	struct bpf_reg_state *reg = reg_state(env, regno);
7040 	struct bpf_func_state *state = func(env, reg);
7041 	int err, min_off, max_off, i, j, slot, spi;
7042 	char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
7043 	enum bpf_access_type bounds_check_type;
7044 	/* Some accesses can write anything into the stack, others are
7045 	 * read-only.
7046 	 */
7047 	bool clobber = false;
7048 
7049 	if (access_size == 0 && !zero_size_allowed) {
7050 		verbose(env, "invalid zero-sized read\n");
7051 		return -EACCES;
7052 	}
7053 
7054 	if (type == ACCESS_HELPER) {
7055 		/* The bounds checks for writes are more permissive than for
7056 		 * reads. However, if raw_mode is not set, we'll do extra
7057 		 * checks below.
7058 		 */
7059 		bounds_check_type = BPF_WRITE;
7060 		clobber = true;
7061 	} else {
7062 		bounds_check_type = BPF_READ;
7063 	}
7064 	err = check_stack_access_within_bounds(env, regno, off, access_size,
7065 					       type, bounds_check_type);
7066 	if (err)
7067 		return err;
7068 
7069 
7070 	if (tnum_is_const(reg->var_off)) {
7071 		min_off = max_off = reg->var_off.value + off;
7072 	} else {
7073 		/* Variable offset is prohibited for unprivileged mode for
7074 		 * simplicity since it requires corresponding support in
7075 		 * Spectre masking for stack ALU.
7076 		 * See also retrieve_ptr_limit().
7077 		 */
7078 		if (!env->bypass_spec_v1) {
7079 			char tn_buf[48];
7080 
7081 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7082 			verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
7083 				regno, err_extra, tn_buf);
7084 			return -EACCES;
7085 		}
7086 		/* Only initialized buffer on stack is allowed to be accessed
7087 		 * with variable offset. With uninitialized buffer it's hard to
7088 		 * guarantee that whole memory is marked as initialized on
7089 		 * helper return since specific bounds are unknown what may
7090 		 * cause uninitialized stack leaking.
7091 		 */
7092 		if (meta && meta->raw_mode)
7093 			meta = NULL;
7094 
7095 		min_off = reg->smin_value + off;
7096 		max_off = reg->smax_value + off;
7097 	}
7098 
7099 	if (meta && meta->raw_mode) {
7100 		/* Ensure we won't be overwriting dynptrs when simulating byte
7101 		 * by byte access in check_helper_call using meta.access_size.
7102 		 * This would be a problem if we have a helper in the future
7103 		 * which takes:
7104 		 *
7105 		 *	helper(uninit_mem, len, dynptr)
7106 		 *
7107 		 * Now, uninint_mem may overlap with dynptr pointer. Hence, it
7108 		 * may end up writing to dynptr itself when touching memory from
7109 		 * arg 1. This can be relaxed on a case by case basis for known
7110 		 * safe cases, but reject due to the possibilitiy of aliasing by
7111 		 * default.
7112 		 */
7113 		for (i = min_off; i < max_off + access_size; i++) {
7114 			int stack_off = -i - 1;
7115 
7116 			spi = __get_spi(i);
7117 			/* raw_mode may write past allocated_stack */
7118 			if (state->allocated_stack <= stack_off)
7119 				continue;
7120 			if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) {
7121 				verbose(env, "potential write to dynptr at off=%d disallowed\n", i);
7122 				return -EACCES;
7123 			}
7124 		}
7125 		meta->access_size = access_size;
7126 		meta->regno = regno;
7127 		return 0;
7128 	}
7129 
7130 	for (i = min_off; i < max_off + access_size; i++) {
7131 		u8 *stype;
7132 
7133 		slot = -i - 1;
7134 		spi = slot / BPF_REG_SIZE;
7135 		if (state->allocated_stack <= slot) {
7136 			verbose(env, "verifier bug: allocated_stack too small");
7137 			return -EFAULT;
7138 		}
7139 
7140 		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
7141 		if (*stype == STACK_MISC)
7142 			goto mark;
7143 		if ((*stype == STACK_ZERO) ||
7144 		    (*stype == STACK_INVALID && env->allow_uninit_stack)) {
7145 			if (clobber) {
7146 				/* helper can write anything into the stack */
7147 				*stype = STACK_MISC;
7148 			}
7149 			goto mark;
7150 		}
7151 
7152 		if (is_spilled_reg(&state->stack[spi]) &&
7153 		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
7154 		     env->allow_ptr_leaks)) {
7155 			if (clobber) {
7156 				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
7157 				for (j = 0; j < BPF_REG_SIZE; j++)
7158 					scrub_spilled_slot(&state->stack[spi].slot_type[j]);
7159 			}
7160 			goto mark;
7161 		}
7162 
7163 		if (tnum_is_const(reg->var_off)) {
7164 			verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
7165 				err_extra, regno, min_off, i - min_off, access_size);
7166 		} else {
7167 			char tn_buf[48];
7168 
7169 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
7170 			verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
7171 				err_extra, regno, tn_buf, i - min_off, access_size);
7172 		}
7173 		return -EACCES;
7174 mark:
7175 		/* reading any byte out of 8-byte 'spill_slot' will cause
7176 		 * the whole slot to be marked as 'read'
7177 		 */
7178 		mark_reg_read(env, &state->stack[spi].spilled_ptr,
7179 			      state->stack[spi].spilled_ptr.parent,
7180 			      REG_LIVE_READ64);
7181 		/* We do not set REG_LIVE_WRITTEN for stack slot, as we can not
7182 		 * be sure that whether stack slot is written to or not. Hence,
7183 		 * we must still conservatively propagate reads upwards even if
7184 		 * helper may write to the entire memory range.
7185 		 */
7186 	}
7187 	return 0;
7188 }
7189 
7190 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
7191 				   int access_size, bool zero_size_allowed,
7192 				   struct bpf_call_arg_meta *meta)
7193 {
7194 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7195 	u32 *max_access;
7196 
7197 	switch (base_type(reg->type)) {
7198 	case PTR_TO_PACKET:
7199 	case PTR_TO_PACKET_META:
7200 		return check_packet_access(env, regno, reg->off, access_size,
7201 					   zero_size_allowed);
7202 	case PTR_TO_MAP_KEY:
7203 		if (meta && meta->raw_mode) {
7204 			verbose(env, "R%d cannot write into %s\n", regno,
7205 				reg_type_str(env, reg->type));
7206 			return -EACCES;
7207 		}
7208 		return check_mem_region_access(env, regno, reg->off, access_size,
7209 					       reg->map_ptr->key_size, false);
7210 	case PTR_TO_MAP_VALUE:
7211 		if (check_map_access_type(env, regno, reg->off, access_size,
7212 					  meta && meta->raw_mode ? BPF_WRITE :
7213 					  BPF_READ))
7214 			return -EACCES;
7215 		return check_map_access(env, regno, reg->off, access_size,
7216 					zero_size_allowed, ACCESS_HELPER);
7217 	case PTR_TO_MEM:
7218 		if (type_is_rdonly_mem(reg->type)) {
7219 			if (meta && meta->raw_mode) {
7220 				verbose(env, "R%d cannot write into %s\n", regno,
7221 					reg_type_str(env, reg->type));
7222 				return -EACCES;
7223 			}
7224 		}
7225 		return check_mem_region_access(env, regno, reg->off,
7226 					       access_size, reg->mem_size,
7227 					       zero_size_allowed);
7228 	case PTR_TO_BUF:
7229 		if (type_is_rdonly_mem(reg->type)) {
7230 			if (meta && meta->raw_mode) {
7231 				verbose(env, "R%d cannot write into %s\n", regno,
7232 					reg_type_str(env, reg->type));
7233 				return -EACCES;
7234 			}
7235 
7236 			max_access = &env->prog->aux->max_rdonly_access;
7237 		} else {
7238 			max_access = &env->prog->aux->max_rdwr_access;
7239 		}
7240 		return check_buffer_access(env, reg, regno, reg->off,
7241 					   access_size, zero_size_allowed,
7242 					   max_access);
7243 	case PTR_TO_STACK:
7244 		return check_stack_range_initialized(
7245 				env,
7246 				regno, reg->off, access_size,
7247 				zero_size_allowed, ACCESS_HELPER, meta);
7248 	case PTR_TO_BTF_ID:
7249 		return check_ptr_to_btf_access(env, regs, regno, reg->off,
7250 					       access_size, BPF_READ, -1);
7251 	case PTR_TO_CTX:
7252 		/* in case the function doesn't know how to access the context,
7253 		 * (because we are in a program of type SYSCALL for example), we
7254 		 * can not statically check its size.
7255 		 * Dynamically check it now.
7256 		 */
7257 		if (!env->ops->convert_ctx_access) {
7258 			enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ;
7259 			int offset = access_size - 1;
7260 
7261 			/* Allow zero-byte read from PTR_TO_CTX */
7262 			if (access_size == 0)
7263 				return zero_size_allowed ? 0 : -EACCES;
7264 
7265 			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
7266 						atype, -1, false, false);
7267 		}
7268 
7269 		fallthrough;
7270 	default: /* scalar_value or invalid ptr */
7271 		/* Allow zero-byte read from NULL, regardless of pointer type */
7272 		if (zero_size_allowed && access_size == 0 &&
7273 		    register_is_null(reg))
7274 			return 0;
7275 
7276 		verbose(env, "R%d type=%s ", regno,
7277 			reg_type_str(env, reg->type));
7278 		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
7279 		return -EACCES;
7280 	}
7281 }
7282 
7283 static int check_mem_size_reg(struct bpf_verifier_env *env,
7284 			      struct bpf_reg_state *reg, u32 regno,
7285 			      bool zero_size_allowed,
7286 			      struct bpf_call_arg_meta *meta)
7287 {
7288 	int err;
7289 
7290 	/* This is used to refine r0 return value bounds for helpers
7291 	 * that enforce this value as an upper bound on return values.
7292 	 * See do_refine_retval_range() for helpers that can refine
7293 	 * the return value. C type of helper is u32 so we pull register
7294 	 * bound from umax_value however, if negative verifier errors
7295 	 * out. Only upper bounds can be learned because retval is an
7296 	 * int type and negative retvals are allowed.
7297 	 */
7298 	meta->msize_max_value = reg->umax_value;
7299 
7300 	/* The register is SCALAR_VALUE; the access check
7301 	 * happens using its boundaries.
7302 	 */
7303 	if (!tnum_is_const(reg->var_off))
7304 		/* For unprivileged variable accesses, disable raw
7305 		 * mode so that the program is required to
7306 		 * initialize all the memory that the helper could
7307 		 * just partially fill up.
7308 		 */
7309 		meta = NULL;
7310 
7311 	if (reg->smin_value < 0) {
7312 		verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
7313 			regno);
7314 		return -EACCES;
7315 	}
7316 
7317 	if (reg->umin_value == 0) {
7318 		err = check_helper_mem_access(env, regno - 1, 0,
7319 					      zero_size_allowed,
7320 					      meta);
7321 		if (err)
7322 			return err;
7323 	}
7324 
7325 	if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
7326 		verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
7327 			regno);
7328 		return -EACCES;
7329 	}
7330 	err = check_helper_mem_access(env, regno - 1,
7331 				      reg->umax_value,
7332 				      zero_size_allowed, meta);
7333 	if (!err)
7334 		err = mark_chain_precision(env, regno);
7335 	return err;
7336 }
7337 
7338 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
7339 		   u32 regno, u32 mem_size)
7340 {
7341 	bool may_be_null = type_may_be_null(reg->type);
7342 	struct bpf_reg_state saved_reg;
7343 	struct bpf_call_arg_meta meta;
7344 	int err;
7345 
7346 	if (register_is_null(reg))
7347 		return 0;
7348 
7349 	memset(&meta, 0, sizeof(meta));
7350 	/* Assuming that the register contains a value check if the memory
7351 	 * access is safe. Temporarily save and restore the register's state as
7352 	 * the conversion shouldn't be visible to a caller.
7353 	 */
7354 	if (may_be_null) {
7355 		saved_reg = *reg;
7356 		mark_ptr_not_null_reg(reg);
7357 	}
7358 
7359 	err = check_helper_mem_access(env, regno, mem_size, true, &meta);
7360 	/* Check access for BPF_WRITE */
7361 	meta.raw_mode = true;
7362 	err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta);
7363 
7364 	if (may_be_null)
7365 		*reg = saved_reg;
7366 
7367 	return err;
7368 }
7369 
7370 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
7371 				    u32 regno)
7372 {
7373 	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
7374 	bool may_be_null = type_may_be_null(mem_reg->type);
7375 	struct bpf_reg_state saved_reg;
7376 	struct bpf_call_arg_meta meta;
7377 	int err;
7378 
7379 	WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);
7380 
7381 	memset(&meta, 0, sizeof(meta));
7382 
7383 	if (may_be_null) {
7384 		saved_reg = *mem_reg;
7385 		mark_ptr_not_null_reg(mem_reg);
7386 	}
7387 
7388 	err = check_mem_size_reg(env, reg, regno, true, &meta);
7389 	/* Check access for BPF_WRITE */
7390 	meta.raw_mode = true;
7391 	err = err ?: check_mem_size_reg(env, reg, regno, true, &meta);
7392 
7393 	if (may_be_null)
7394 		*mem_reg = saved_reg;
7395 	return err;
7396 }
7397 
7398 /* Implementation details:
7399  * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL.
7400  * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL.
7401  * Two bpf_map_lookups (even with the same key) will have different reg->id.
7402  * Two separate bpf_obj_new will also have different reg->id.
7403  * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier
7404  * clears reg->id after value_or_null->value transition, since the verifier only
7405  * cares about the range of access to valid map value pointer and doesn't care
7406  * about actual address of the map element.
7407  * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
7408  * reg->id > 0 after value_or_null->value transition. By doing so
7409  * two bpf_map_lookups will be considered two different pointers that
7410  * point to different bpf_spin_locks. Likewise for pointers to allocated objects
7411  * returned from bpf_obj_new.
7412  * The verifier allows taking only one bpf_spin_lock at a time to avoid
7413  * dead-locks.
7414  * Since only one bpf_spin_lock is allowed the checks are simpler than
7415  * reg_is_refcounted() logic. The verifier needs to remember only
7416  * one spin_lock instead of array of acquired_refs.
7417  * cur_state->active_lock remembers which map value element or allocated
7418  * object got locked and clears it after bpf_spin_unlock.
7419  */
7420 static int process_spin_lock(struct bpf_verifier_env *env, int regno,
7421 			     bool is_lock)
7422 {
7423 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7424 	struct bpf_verifier_state *cur = env->cur_state;
7425 	bool is_const = tnum_is_const(reg->var_off);
7426 	u64 val = reg->var_off.value;
7427 	struct bpf_map *map = NULL;
7428 	struct btf *btf = NULL;
7429 	struct btf_record *rec;
7430 
7431 	if (!is_const) {
7432 		verbose(env,
7433 			"R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
7434 			regno);
7435 		return -EINVAL;
7436 	}
7437 	if (reg->type == PTR_TO_MAP_VALUE) {
7438 		map = reg->map_ptr;
7439 		if (!map->btf) {
7440 			verbose(env,
7441 				"map '%s' has to have BTF in order to use bpf_spin_lock\n",
7442 				map->name);
7443 			return -EINVAL;
7444 		}
7445 	} else {
7446 		btf = reg->btf;
7447 	}
7448 
7449 	rec = reg_btf_record(reg);
7450 	if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) {
7451 		verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local",
7452 			map ? map->name : "kptr");
7453 		return -EINVAL;
7454 	}
7455 	if (rec->spin_lock_off != val + reg->off) {
7456 		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n",
7457 			val + reg->off, rec->spin_lock_off);
7458 		return -EINVAL;
7459 	}
7460 	if (is_lock) {
7461 		if (cur->active_lock.ptr) {
7462 			verbose(env,
7463 				"Locking two bpf_spin_locks are not allowed\n");
7464 			return -EINVAL;
7465 		}
7466 		if (map)
7467 			cur->active_lock.ptr = map;
7468 		else
7469 			cur->active_lock.ptr = btf;
7470 		cur->active_lock.id = reg->id;
7471 	} else {
7472 		void *ptr;
7473 
7474 		if (map)
7475 			ptr = map;
7476 		else
7477 			ptr = btf;
7478 
7479 		if (!cur->active_lock.ptr) {
7480 			verbose(env, "bpf_spin_unlock without taking a lock\n");
7481 			return -EINVAL;
7482 		}
7483 		if (cur->active_lock.ptr != ptr ||
7484 		    cur->active_lock.id != reg->id) {
7485 			verbose(env, "bpf_spin_unlock of different lock\n");
7486 			return -EINVAL;
7487 		}
7488 
7489 		invalidate_non_owning_refs(env);
7490 
7491 		cur->active_lock.ptr = NULL;
7492 		cur->active_lock.id = 0;
7493 	}
7494 	return 0;
7495 }
7496 
7497 static int process_timer_func(struct bpf_verifier_env *env, int regno,
7498 			      struct bpf_call_arg_meta *meta)
7499 {
7500 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7501 	bool is_const = tnum_is_const(reg->var_off);
7502 	struct bpf_map *map = reg->map_ptr;
7503 	u64 val = reg->var_off.value;
7504 
7505 	if (!is_const) {
7506 		verbose(env,
7507 			"R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n",
7508 			regno);
7509 		return -EINVAL;
7510 	}
7511 	if (!map->btf) {
7512 		verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",
7513 			map->name);
7514 		return -EINVAL;
7515 	}
7516 	if (!btf_record_has_field(map->record, BPF_TIMER)) {
7517 		verbose(env, "map '%s' has no valid bpf_timer\n", map->name);
7518 		return -EINVAL;
7519 	}
7520 	if (map->record->timer_off != val + reg->off) {
7521 		verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",
7522 			val + reg->off, map->record->timer_off);
7523 		return -EINVAL;
7524 	}
7525 	if (meta->map_ptr) {
7526 		verbose(env, "verifier bug. Two map pointers in a timer helper\n");
7527 		return -EFAULT;
7528 	}
7529 	meta->map_uid = reg->map_uid;
7530 	meta->map_ptr = map;
7531 	return 0;
7532 }
7533 
7534 static int process_kptr_func(struct bpf_verifier_env *env, int regno,
7535 			     struct bpf_call_arg_meta *meta)
7536 {
7537 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7538 	struct bpf_map *map_ptr = reg->map_ptr;
7539 	struct btf_field *kptr_field;
7540 	u32 kptr_off;
7541 
7542 	if (!tnum_is_const(reg->var_off)) {
7543 		verbose(env,
7544 			"R%d doesn't have constant offset. kptr has to be at the constant offset\n",
7545 			regno);
7546 		return -EINVAL;
7547 	}
7548 	if (!map_ptr->btf) {
7549 		verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
7550 			map_ptr->name);
7551 		return -EINVAL;
7552 	}
7553 	if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) {
7554 		verbose(env, "map '%s' has no valid kptr\n", map_ptr->name);
7555 		return -EINVAL;
7556 	}
7557 
7558 	meta->map_ptr = map_ptr;
7559 	kptr_off = reg->off + reg->var_off.value;
7560 	kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR);
7561 	if (!kptr_field) {
7562 		verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
7563 		return -EACCES;
7564 	}
7565 	if (kptr_field->type != BPF_KPTR_REF) {
7566 		verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
7567 		return -EACCES;
7568 	}
7569 	meta->kptr_field = kptr_field;
7570 	return 0;
7571 }
7572 
7573 /* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
7574  * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
7575  *
7576  * In both cases we deal with the first 8 bytes, but need to mark the next 8
7577  * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
7578  * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
7579  *
7580  * Mutability of bpf_dynptr is at two levels, one is at the level of struct
7581  * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct
7582  * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can
7583  * mutate the view of the dynptr and also possibly destroy it. In the latter
7584  * case, it cannot mutate the bpf_dynptr itself but it can still mutate the
7585  * memory that dynptr points to.
7586  *
7587  * The verifier will keep track both levels of mutation (bpf_dynptr's in
7588  * reg->type and the memory's in reg->dynptr.type), but there is no support for
7589  * readonly dynptr view yet, hence only the first case is tracked and checked.
7590  *
7591  * This is consistent with how C applies the const modifier to a struct object,
7592  * where the pointer itself inside bpf_dynptr becomes const but not what it
7593  * points to.
7594  *
7595  * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
7596  * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
7597  */
7598 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
7599 			       enum bpf_arg_type arg_type, int clone_ref_obj_id)
7600 {
7601 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7602 	int err;
7603 
7604 	/* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
7605 	 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
7606 	 */
7607 	if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) {
7608 		verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n");
7609 		return -EFAULT;
7610 	}
7611 
7612 	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
7613 	 *		 constructing a mutable bpf_dynptr object.
7614 	 *
7615 	 *		 Currently, this is only possible with PTR_TO_STACK
7616 	 *		 pointing to a region of at least 16 bytes which doesn't
7617 	 *		 contain an existing bpf_dynptr.
7618 	 *
7619 	 *  MEM_RDONLY - Points to a initialized bpf_dynptr that will not be
7620 	 *		 mutated or destroyed. However, the memory it points to
7621 	 *		 may be mutated.
7622 	 *
7623 	 *  None       - Points to a initialized dynptr that can be mutated and
7624 	 *		 destroyed, including mutation of the memory it points
7625 	 *		 to.
7626 	 */
7627 	if (arg_type & MEM_UNINIT) {
7628 		int i;
7629 
7630 		if (!is_dynptr_reg_valid_uninit(env, reg)) {
7631 			verbose(env, "Dynptr has to be an uninitialized dynptr\n");
7632 			return -EINVAL;
7633 		}
7634 
7635 		/* we write BPF_DW bits (8 bytes) at a time */
7636 		for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
7637 			err = check_mem_access(env, insn_idx, regno,
7638 					       i, BPF_DW, BPF_WRITE, -1, false, false);
7639 			if (err)
7640 				return err;
7641 		}
7642 
7643 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
7644 	} else /* MEM_RDONLY and None case from above */ {
7645 		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
7646 		if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
7647 			verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
7648 			return -EINVAL;
7649 		}
7650 
7651 		if (!is_dynptr_reg_valid_init(env, reg)) {
7652 			verbose(env,
7653 				"Expected an initialized dynptr as arg #%d\n",
7654 				regno);
7655 			return -EINVAL;
7656 		}
7657 
7658 		/* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */
7659 		if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {
7660 			verbose(env,
7661 				"Expected a dynptr of type %s as arg #%d\n",
7662 				dynptr_type_str(arg_to_dynptr_type(arg_type)), regno);
7663 			return -EINVAL;
7664 		}
7665 
7666 		err = mark_dynptr_read(env, reg);
7667 	}
7668 	return err;
7669 }
7670 
7671 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
7672 {
7673 	struct bpf_func_state *state = func(env, reg);
7674 
7675 	return state->stack[spi].spilled_ptr.ref_obj_id;
7676 }
7677 
7678 static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7679 {
7680 	return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
7681 }
7682 
7683 static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7684 {
7685 	return meta->kfunc_flags & KF_ITER_NEW;
7686 }
7687 
7688 static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7689 {
7690 	return meta->kfunc_flags & KF_ITER_NEXT;
7691 }
7692 
7693 static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
7694 {
7695 	return meta->kfunc_flags & KF_ITER_DESTROY;
7696 }
7697 
7698 static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg)
7699 {
7700 	/* btf_check_iter_kfuncs() guarantees that first argument of any iter
7701 	 * kfunc is iter state pointer
7702 	 */
7703 	return arg == 0 && is_iter_kfunc(meta);
7704 }
7705 
7706 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,
7707 			    struct bpf_kfunc_call_arg_meta *meta)
7708 {
7709 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7710 	const struct btf_type *t;
7711 	const struct btf_param *arg;
7712 	int spi, err, i, nr_slots;
7713 	u32 btf_id;
7714 
7715 	/* btf_check_iter_kfuncs() ensures we don't need to validate anything here */
7716 	arg = &btf_params(meta->func_proto)[0];
7717 	t = btf_type_skip_modifiers(meta->btf, arg->type, NULL);	/* PTR */
7718 	t = btf_type_skip_modifiers(meta->btf, t->type, &btf_id);	/* STRUCT */
7719 	nr_slots = t->size / BPF_REG_SIZE;
7720 
7721 	if (is_iter_new_kfunc(meta)) {
7722 		/* bpf_iter_<type>_new() expects pointer to uninit iter state */
7723 		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
7724 			verbose(env, "expected uninitialized iter_%s as arg #%d\n",
7725 				iter_type_str(meta->btf, btf_id), regno);
7726 			return -EINVAL;
7727 		}
7728 
7729 		for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
7730 			err = check_mem_access(env, insn_idx, regno,
7731 					       i, BPF_DW, BPF_WRITE, -1, false, false);
7732 			if (err)
7733 				return err;
7734 		}
7735 
7736 		err = mark_stack_slots_iter(env, reg, insn_idx, meta->btf, btf_id, nr_slots);
7737 		if (err)
7738 			return err;
7739 	} else {
7740 		/* iter_next() or iter_destroy() expect initialized iter state*/
7741 		if (!is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots)) {
7742 			verbose(env, "expected an initialized iter_%s as arg #%d\n",
7743 				iter_type_str(meta->btf, btf_id), regno);
7744 			return -EINVAL;
7745 		}
7746 
7747 		spi = iter_get_spi(env, reg, nr_slots);
7748 		if (spi < 0)
7749 			return spi;
7750 
7751 		err = mark_iter_read(env, reg, spi, nr_slots);
7752 		if (err)
7753 			return err;
7754 
7755 		/* remember meta->iter info for process_iter_next_call() */
7756 		meta->iter.spi = spi;
7757 		meta->iter.frameno = reg->frameno;
7758 		meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);
7759 
7760 		if (is_iter_destroy_kfunc(meta)) {
7761 			err = unmark_stack_slots_iter(env, reg, nr_slots);
7762 			if (err)
7763 				return err;
7764 		}
7765 	}
7766 
7767 	return 0;
7768 }
7769 
7770 /* Look for a previous loop entry at insn_idx: nearest parent state
7771  * stopped at insn_idx with callsites matching those in cur->frame.
7772  */
7773 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env,
7774 						  struct bpf_verifier_state *cur,
7775 						  int insn_idx)
7776 {
7777 	struct bpf_verifier_state_list *sl;
7778 	struct bpf_verifier_state *st;
7779 
7780 	/* Explored states are pushed in stack order, most recent states come first */
7781 	sl = *explored_state(env, insn_idx);
7782 	for (; sl; sl = sl->next) {
7783 		/* If st->branches != 0 state is a part of current DFS verification path,
7784 		 * hence cur & st for a loop.
7785 		 */
7786 		st = &sl->state;
7787 		if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&
7788 		    st->dfs_depth < cur->dfs_depth)
7789 			return st;
7790 	}
7791 
7792 	return NULL;
7793 }
7794 
7795 static void reset_idmap_scratch(struct bpf_verifier_env *env);
7796 static bool regs_exact(const struct bpf_reg_state *rold,
7797 		       const struct bpf_reg_state *rcur,
7798 		       struct bpf_idmap *idmap);
7799 
7800 static void maybe_widen_reg(struct bpf_verifier_env *env,
7801 			    struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
7802 			    struct bpf_idmap *idmap)
7803 {
7804 	if (rold->type != SCALAR_VALUE)
7805 		return;
7806 	if (rold->type != rcur->type)
7807 		return;
7808 	if (rold->precise || rcur->precise || regs_exact(rold, rcur, idmap))
7809 		return;
7810 	__mark_reg_unknown(env, rcur);
7811 }
7812 
7813 static int widen_imprecise_scalars(struct bpf_verifier_env *env,
7814 				   struct bpf_verifier_state *old,
7815 				   struct bpf_verifier_state *cur)
7816 {
7817 	struct bpf_func_state *fold, *fcur;
7818 	int i, fr;
7819 
7820 	reset_idmap_scratch(env);
7821 	for (fr = old->curframe; fr >= 0; fr--) {
7822 		fold = old->frame[fr];
7823 		fcur = cur->frame[fr];
7824 
7825 		for (i = 0; i < MAX_BPF_REG; i++)
7826 			maybe_widen_reg(env,
7827 					&fold->regs[i],
7828 					&fcur->regs[i],
7829 					&env->idmap_scratch);
7830 
7831 		for (i = 0; i < fold->allocated_stack / BPF_REG_SIZE; i++) {
7832 			if (!is_spilled_reg(&fold->stack[i]) ||
7833 			    !is_spilled_reg(&fcur->stack[i]))
7834 				continue;
7835 
7836 			maybe_widen_reg(env,
7837 					&fold->stack[i].spilled_ptr,
7838 					&fcur->stack[i].spilled_ptr,
7839 					&env->idmap_scratch);
7840 		}
7841 	}
7842 	return 0;
7843 }
7844 
7845 /* process_iter_next_call() is called when verifier gets to iterator's next
7846  * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer
7847  * to it as just "iter_next()" in comments below.
7848  *
7849  * BPF verifier relies on a crucial contract for any iter_next()
7850  * implementation: it should *eventually* return NULL, and once that happens
7851  * it should keep returning NULL. That is, once iterator exhausts elements to
7852  * iterate, it should never reset or spuriously return new elements.
7853  *
7854  * With the assumption of such contract, process_iter_next_call() simulates
7855  * a fork in the verifier state to validate loop logic correctness and safety
7856  * without having to simulate infinite amount of iterations.
7857  *
7858  * In current state, we first assume that iter_next() returned NULL and
7859  * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such
7860  * conditions we should not form an infinite loop and should eventually reach
7861  * exit.
7862  *
7863  * Besides that, we also fork current state and enqueue it for later
7864  * verification. In a forked state we keep iterator state as ACTIVE
7865  * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We
7866  * also bump iteration depth to prevent erroneous infinite loop detection
7867  * later on (see iter_active_depths_differ() comment for details). In this
7868  * state we assume that we'll eventually loop back to another iter_next()
7869  * calls (it could be in exactly same location or in some other instruction,
7870  * it doesn't matter, we don't make any unnecessary assumptions about this,
7871  * everything revolves around iterator state in a stack slot, not which
7872  * instruction is calling iter_next()). When that happens, we either will come
7873  * to iter_next() with equivalent state and can conclude that next iteration
7874  * will proceed in exactly the same way as we just verified, so it's safe to
7875  * assume that loop converges. If not, we'll go on another iteration
7876  * simulation with a different input state, until all possible starting states
7877  * are validated or we reach maximum number of instructions limit.
7878  *
7879  * This way, we will either exhaustively discover all possible input states
7880  * that iterator loop can start with and eventually will converge, or we'll
7881  * effectively regress into bounded loop simulation logic and either reach
7882  * maximum number of instructions if loop is not provably convergent, or there
7883  * is some statically known limit on number of iterations (e.g., if there is
7884  * an explicit `if n > 100 then break;` statement somewhere in the loop).
7885  *
7886  * Iteration convergence logic in is_state_visited() relies on exact
7887  * states comparison, which ignores read and precision marks.
7888  * This is necessary because read and precision marks are not finalized
7889  * while in the loop. Exact comparison might preclude convergence for
7890  * simple programs like below:
7891  *
7892  *     i = 0;
7893  *     while(iter_next(&it))
7894  *       i++;
7895  *
7896  * At each iteration step i++ would produce a new distinct state and
7897  * eventually instruction processing limit would be reached.
7898  *
7899  * To avoid such behavior speculatively forget (widen) range for
7900  * imprecise scalar registers, if those registers were not precise at the
7901  * end of the previous iteration and do not match exactly.
7902  *
7903  * This is a conservative heuristic that allows to verify wide range of programs,
7904  * however it precludes verification of programs that conjure an
7905  * imprecise value on the first loop iteration and use it as precise on a second.
7906  * For example, the following safe program would fail to verify:
7907  *
7908  *     struct bpf_num_iter it;
7909  *     int arr[10];
7910  *     int i = 0, a = 0;
7911  *     bpf_iter_num_new(&it, 0, 10);
7912  *     while (bpf_iter_num_next(&it)) {
7913  *       if (a == 0) {
7914  *         a = 1;
7915  *         i = 7; // Because i changed verifier would forget
7916  *                // it's range on second loop entry.
7917  *       } else {
7918  *         arr[i] = 42; // This would fail to verify.
7919  *       }
7920  *     }
7921  *     bpf_iter_num_destroy(&it);
7922  */
7923 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
7924 				  struct bpf_kfunc_call_arg_meta *meta)
7925 {
7926 	struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
7927 	struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
7928 	struct bpf_reg_state *cur_iter, *queued_iter;
7929 	int iter_frameno = meta->iter.frameno;
7930 	int iter_spi = meta->iter.spi;
7931 
7932 	BTF_TYPE_EMIT(struct bpf_iter);
7933 
7934 	cur_iter = &env->cur_state->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
7935 
7936 	if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE &&
7937 	    cur_iter->iter.state != BPF_ITER_STATE_DRAINED) {
7938 		verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n",
7939 			cur_iter->iter.state, iter_state_str(cur_iter->iter.state));
7940 		return -EFAULT;
7941 	}
7942 
7943 	if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) {
7944 		/* Because iter_next() call is a checkpoint is_state_visitied()
7945 		 * should guarantee parent state with same call sites and insn_idx.
7946 		 */
7947 		if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||
7948 		    !same_callsites(cur_st->parent, cur_st)) {
7949 			verbose(env, "bug: bad parent state for iter next call");
7950 			return -EFAULT;
7951 		}
7952 		/* Note cur_st->parent in the call below, it is necessary to skip
7953 		 * checkpoint created for cur_st by is_state_visited()
7954 		 * right at this instruction.
7955 		 */
7956 		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);
7957 		/* branch out active iter state */
7958 		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);
7959 		if (!queued_st)
7960 			return -ENOMEM;
7961 
7962 		queued_iter = &queued_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
7963 		queued_iter->iter.state = BPF_ITER_STATE_ACTIVE;
7964 		queued_iter->iter.depth++;
7965 		if (prev_st)
7966 			widen_imprecise_scalars(env, prev_st, queued_st);
7967 
7968 		queued_fr = queued_st->frame[queued_st->curframe];
7969 		mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]);
7970 	}
7971 
7972 	/* switch to DRAINED state, but keep the depth unchanged */
7973 	/* mark current iter state as drained and assume returned NULL */
7974 	cur_iter->iter.state = BPF_ITER_STATE_DRAINED;
7975 	__mark_reg_const_zero(&cur_fr->regs[BPF_REG_0]);
7976 
7977 	return 0;
7978 }
7979 
7980 static bool arg_type_is_mem_size(enum bpf_arg_type type)
7981 {
7982 	return type == ARG_CONST_SIZE ||
7983 	       type == ARG_CONST_SIZE_OR_ZERO;
7984 }
7985 
7986 static bool arg_type_is_release(enum bpf_arg_type type)
7987 {
7988 	return type & OBJ_RELEASE;
7989 }
7990 
7991 static bool arg_type_is_dynptr(enum bpf_arg_type type)
7992 {
7993 	return base_type(type) == ARG_PTR_TO_DYNPTR;
7994 }
7995 
7996 static int int_ptr_type_to_size(enum bpf_arg_type type)
7997 {
7998 	if (type == ARG_PTR_TO_INT)
7999 		return sizeof(u32);
8000 	else if (type == ARG_PTR_TO_LONG)
8001 		return sizeof(u64);
8002 
8003 	return -EINVAL;
8004 }
8005 
8006 static int resolve_map_arg_type(struct bpf_verifier_env *env,
8007 				 const struct bpf_call_arg_meta *meta,
8008 				 enum bpf_arg_type *arg_type)
8009 {
8010 	if (!meta->map_ptr) {
8011 		/* kernel subsystem misconfigured verifier */
8012 		verbose(env, "invalid map_ptr to access map->type\n");
8013 		return -EACCES;
8014 	}
8015 
8016 	switch (meta->map_ptr->map_type) {
8017 	case BPF_MAP_TYPE_SOCKMAP:
8018 	case BPF_MAP_TYPE_SOCKHASH:
8019 		if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
8020 			*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
8021 		} else {
8022 			verbose(env, "invalid arg_type for sockmap/sockhash\n");
8023 			return -EINVAL;
8024 		}
8025 		break;
8026 	case BPF_MAP_TYPE_BLOOM_FILTER:
8027 		if (meta->func_id == BPF_FUNC_map_peek_elem)
8028 			*arg_type = ARG_PTR_TO_MAP_VALUE;
8029 		break;
8030 	default:
8031 		break;
8032 	}
8033 	return 0;
8034 }
8035 
8036 struct bpf_reg_types {
8037 	const enum bpf_reg_type types[10];
8038 	u32 *btf_id;
8039 };
8040 
8041 static const struct bpf_reg_types sock_types = {
8042 	.types = {
8043 		PTR_TO_SOCK_COMMON,
8044 		PTR_TO_SOCKET,
8045 		PTR_TO_TCP_SOCK,
8046 		PTR_TO_XDP_SOCK,
8047 	},
8048 };
8049 
8050 #ifdef CONFIG_NET
8051 static const struct bpf_reg_types btf_id_sock_common_types = {
8052 	.types = {
8053 		PTR_TO_SOCK_COMMON,
8054 		PTR_TO_SOCKET,
8055 		PTR_TO_TCP_SOCK,
8056 		PTR_TO_XDP_SOCK,
8057 		PTR_TO_BTF_ID,
8058 		PTR_TO_BTF_ID | PTR_TRUSTED,
8059 	},
8060 	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
8061 };
8062 #endif
8063 
8064 static const struct bpf_reg_types mem_types = {
8065 	.types = {
8066 		PTR_TO_STACK,
8067 		PTR_TO_PACKET,
8068 		PTR_TO_PACKET_META,
8069 		PTR_TO_MAP_KEY,
8070 		PTR_TO_MAP_VALUE,
8071 		PTR_TO_MEM,
8072 		PTR_TO_MEM | MEM_RINGBUF,
8073 		PTR_TO_BUF,
8074 		PTR_TO_BTF_ID | PTR_TRUSTED,
8075 	},
8076 };
8077 
8078 static const struct bpf_reg_types int_ptr_types = {
8079 	.types = {
8080 		PTR_TO_STACK,
8081 		PTR_TO_PACKET,
8082 		PTR_TO_PACKET_META,
8083 		PTR_TO_MAP_KEY,
8084 		PTR_TO_MAP_VALUE,
8085 	},
8086 };
8087 
8088 static const struct bpf_reg_types spin_lock_types = {
8089 	.types = {
8090 		PTR_TO_MAP_VALUE,
8091 		PTR_TO_BTF_ID | MEM_ALLOC,
8092 	}
8093 };
8094 
8095 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
8096 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
8097 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
8098 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } };
8099 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
8100 static const struct bpf_reg_types btf_ptr_types = {
8101 	.types = {
8102 		PTR_TO_BTF_ID,
8103 		PTR_TO_BTF_ID | PTR_TRUSTED,
8104 		PTR_TO_BTF_ID | MEM_RCU,
8105 	},
8106 };
8107 static const struct bpf_reg_types percpu_btf_ptr_types = {
8108 	.types = {
8109 		PTR_TO_BTF_ID | MEM_PERCPU,
8110 		PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED,
8111 	}
8112 };
8113 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
8114 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
8115 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
8116 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
8117 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } };
8118 static const struct bpf_reg_types dynptr_types = {
8119 	.types = {
8120 		PTR_TO_STACK,
8121 		CONST_PTR_TO_DYNPTR,
8122 	}
8123 };
8124 
8125 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
8126 	[ARG_PTR_TO_MAP_KEY]		= &mem_types,
8127 	[ARG_PTR_TO_MAP_VALUE]		= &mem_types,
8128 	[ARG_CONST_SIZE]		= &scalar_types,
8129 	[ARG_CONST_SIZE_OR_ZERO]	= &scalar_types,
8130 	[ARG_CONST_ALLOC_SIZE_OR_ZERO]	= &scalar_types,
8131 	[ARG_CONST_MAP_PTR]		= &const_map_ptr_types,
8132 	[ARG_PTR_TO_CTX]		= &context_types,
8133 	[ARG_PTR_TO_SOCK_COMMON]	= &sock_types,
8134 #ifdef CONFIG_NET
8135 	[ARG_PTR_TO_BTF_ID_SOCK_COMMON]	= &btf_id_sock_common_types,
8136 #endif
8137 	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
8138 	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
8139 	[ARG_PTR_TO_SPIN_LOCK]		= &spin_lock_types,
8140 	[ARG_PTR_TO_MEM]		= &mem_types,
8141 	[ARG_PTR_TO_RINGBUF_MEM]	= &ringbuf_mem_types,
8142 	[ARG_PTR_TO_INT]		= &int_ptr_types,
8143 	[ARG_PTR_TO_LONG]		= &int_ptr_types,
8144 	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
8145 	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
8146 	[ARG_PTR_TO_STACK]		= &stack_ptr_types,
8147 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
8148 	[ARG_PTR_TO_TIMER]		= &timer_types,
8149 	[ARG_PTR_TO_KPTR]		= &kptr_types,
8150 	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
8151 };
8152 
8153 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
8154 			  enum bpf_arg_type arg_type,
8155 			  const u32 *arg_btf_id,
8156 			  struct bpf_call_arg_meta *meta)
8157 {
8158 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
8159 	enum bpf_reg_type expected, type = reg->type;
8160 	const struct bpf_reg_types *compatible;
8161 	int i, j;
8162 
8163 	compatible = compatible_reg_types[base_type(arg_type)];
8164 	if (!compatible) {
8165 		verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
8166 		return -EFAULT;
8167 	}
8168 
8169 	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
8170 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
8171 	 *
8172 	 * Same for MAYBE_NULL:
8173 	 *
8174 	 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
8175 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
8176 	 *
8177 	 * ARG_PTR_TO_MEM is compatible with PTR_TO_MEM that is tagged with a dynptr type.
8178 	 *
8179 	 * Therefore we fold these flags depending on the arg_type before comparison.
8180 	 */
8181 	if (arg_type & MEM_RDONLY)
8182 		type &= ~MEM_RDONLY;
8183 	if (arg_type & PTR_MAYBE_NULL)
8184 		type &= ~PTR_MAYBE_NULL;
8185 	if (base_type(arg_type) == ARG_PTR_TO_MEM)
8186 		type &= ~DYNPTR_TYPE_FLAG_MASK;
8187 
8188 	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type))
8189 		type &= ~MEM_ALLOC;
8190 
8191 	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
8192 		expected = compatible->types[i];
8193 		if (expected == NOT_INIT)
8194 			break;
8195 
8196 		if (type == expected)
8197 			goto found;
8198 	}
8199 
8200 	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
8201 	for (j = 0; j + 1 < i; j++)
8202 		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
8203 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
8204 	return -EACCES;
8205 
8206 found:
8207 	if (base_type(reg->type) != PTR_TO_BTF_ID)
8208 		return 0;
8209 
8210 	if (compatible == &mem_types) {
8211 		if (!(arg_type & MEM_RDONLY)) {
8212 			verbose(env,
8213 				"%s() may write into memory pointed by R%d type=%s\n",
8214 				func_id_name(meta->func_id),
8215 				regno, reg_type_str(env, reg->type));
8216 			return -EACCES;
8217 		}
8218 		return 0;
8219 	}
8220 
8221 	switch ((int)reg->type) {
8222 	case PTR_TO_BTF_ID:
8223 	case PTR_TO_BTF_ID | PTR_TRUSTED:
8224 	case PTR_TO_BTF_ID | MEM_RCU:
8225 	case PTR_TO_BTF_ID | PTR_MAYBE_NULL:
8226 	case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU:
8227 	{
8228 		/* For bpf_sk_release, it needs to match against first member
8229 		 * 'struct sock_common', hence make an exception for it. This
8230 		 * allows bpf_sk_release to work for multiple socket types.
8231 		 */
8232 		bool strict_type_match = arg_type_is_release(arg_type) &&
8233 					 meta->func_id != BPF_FUNC_sk_release;
8234 
8235 		if (type_may_be_null(reg->type) &&
8236 		    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
8237 			verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);
8238 			return -EACCES;
8239 		}
8240 
8241 		if (!arg_btf_id) {
8242 			if (!compatible->btf_id) {
8243 				verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
8244 				return -EFAULT;
8245 			}
8246 			arg_btf_id = compatible->btf_id;
8247 		}
8248 
8249 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
8250 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
8251 				return -EACCES;
8252 		} else {
8253 			if (arg_btf_id == BPF_PTR_POISON) {
8254 				verbose(env, "verifier internal error:");
8255 				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
8256 					regno);
8257 				return -EACCES;
8258 			}
8259 
8260 			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
8261 						  btf_vmlinux, *arg_btf_id,
8262 						  strict_type_match)) {
8263 				verbose(env, "R%d is of type %s but %s is expected\n",
8264 					regno, btf_type_name(reg->btf, reg->btf_id),
8265 					btf_type_name(btf_vmlinux, *arg_btf_id));
8266 				return -EACCES;
8267 			}
8268 		}
8269 		break;
8270 	}
8271 	case PTR_TO_BTF_ID | MEM_ALLOC:
8272 		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
8273 		    meta->func_id != BPF_FUNC_kptr_xchg) {
8274 			verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n");
8275 			return -EFAULT;
8276 		}
8277 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
8278 			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
8279 				return -EACCES;
8280 		}
8281 		break;
8282 	case PTR_TO_BTF_ID | MEM_PERCPU:
8283 	case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED:
8284 		/* Handled by helper specific checks */
8285 		break;
8286 	default:
8287 		verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n");
8288 		return -EFAULT;
8289 	}
8290 	return 0;
8291 }
8292 
8293 static struct btf_field *
8294 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
8295 {
8296 	struct btf_field *field;
8297 	struct btf_record *rec;
8298 
8299 	rec = reg_btf_record(reg);
8300 	if (!rec)
8301 		return NULL;
8302 
8303 	field = btf_record_find(rec, off, fields);
8304 	if (!field)
8305 		return NULL;
8306 
8307 	return field;
8308 }
8309 
8310 int check_func_arg_reg_off(struct bpf_verifier_env *env,
8311 			   const struct bpf_reg_state *reg, int regno,
8312 			   enum bpf_arg_type arg_type)
8313 {
8314 	u32 type = reg->type;
8315 
8316 	/* When referenced register is passed to release function, its fixed
8317 	 * offset must be 0.
8318 	 *
8319 	 * We will check arg_type_is_release reg has ref_obj_id when storing
8320 	 * meta->release_regno.
8321 	 */
8322 	if (arg_type_is_release(arg_type)) {
8323 		/* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it
8324 		 * may not directly point to the object being released, but to
8325 		 * dynptr pointing to such object, which might be at some offset
8326 		 * on the stack. In that case, we simply to fallback to the
8327 		 * default handling.
8328 		 */
8329 		if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK)
8330 			return 0;
8331 
8332 		/* Doing check_ptr_off_reg check for the offset will catch this
8333 		 * because fixed_off_ok is false, but checking here allows us
8334 		 * to give the user a better error message.
8335 		 */
8336 		if (reg->off) {
8337 			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
8338 				regno);
8339 			return -EINVAL;
8340 		}
8341 		return __check_ptr_off_reg(env, reg, regno, false);
8342 	}
8343 
8344 	switch (type) {
8345 	/* Pointer types where both fixed and variable offset is explicitly allowed: */
8346 	case PTR_TO_STACK:
8347 	case PTR_TO_PACKET:
8348 	case PTR_TO_PACKET_META:
8349 	case PTR_TO_MAP_KEY:
8350 	case PTR_TO_MAP_VALUE:
8351 	case PTR_TO_MEM:
8352 	case PTR_TO_MEM | MEM_RDONLY:
8353 	case PTR_TO_MEM | MEM_RINGBUF:
8354 	case PTR_TO_BUF:
8355 	case PTR_TO_BUF | MEM_RDONLY:
8356 	case SCALAR_VALUE:
8357 		return 0;
8358 	/* All the rest must be rejected, except PTR_TO_BTF_ID which allows
8359 	 * fixed offset.
8360 	 */
8361 	case PTR_TO_BTF_ID:
8362 	case PTR_TO_BTF_ID | MEM_ALLOC:
8363 	case PTR_TO_BTF_ID | PTR_TRUSTED:
8364 	case PTR_TO_BTF_ID | MEM_RCU:
8365 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
8366 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
8367 		/* When referenced PTR_TO_BTF_ID is passed to release function,
8368 		 * its fixed offset must be 0. In the other cases, fixed offset
8369 		 * can be non-zero. This was already checked above. So pass
8370 		 * fixed_off_ok as true to allow fixed offset for all other
8371 		 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
8372 		 * still need to do checks instead of returning.
8373 		 */
8374 		return __check_ptr_off_reg(env, reg, regno, true);
8375 	default:
8376 		return __check_ptr_off_reg(env, reg, regno, false);
8377 	}
8378 }
8379 
8380 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env,
8381 						const struct bpf_func_proto *fn,
8382 						struct bpf_reg_state *regs)
8383 {
8384 	struct bpf_reg_state *state = NULL;
8385 	int i;
8386 
8387 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
8388 		if (arg_type_is_dynptr(fn->arg_type[i])) {
8389 			if (state) {
8390 				verbose(env, "verifier internal error: multiple dynptr args\n");
8391 				return NULL;
8392 			}
8393 			state = &regs[BPF_REG_1 + i];
8394 		}
8395 
8396 	if (!state)
8397 		verbose(env, "verifier internal error: no dynptr arg found\n");
8398 
8399 	return state;
8400 }
8401 
8402 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
8403 {
8404 	struct bpf_func_state *state = func(env, reg);
8405 	int spi;
8406 
8407 	if (reg->type == CONST_PTR_TO_DYNPTR)
8408 		return reg->id;
8409 	spi = dynptr_get_spi(env, reg);
8410 	if (spi < 0)
8411 		return spi;
8412 	return state->stack[spi].spilled_ptr.id;
8413 }
8414 
8415 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
8416 {
8417 	struct bpf_func_state *state = func(env, reg);
8418 	int spi;
8419 
8420 	if (reg->type == CONST_PTR_TO_DYNPTR)
8421 		return reg->ref_obj_id;
8422 	spi = dynptr_get_spi(env, reg);
8423 	if (spi < 0)
8424 		return spi;
8425 	return state->stack[spi].spilled_ptr.ref_obj_id;
8426 }
8427 
8428 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
8429 					    struct bpf_reg_state *reg)
8430 {
8431 	struct bpf_func_state *state = func(env, reg);
8432 	int spi;
8433 
8434 	if (reg->type == CONST_PTR_TO_DYNPTR)
8435 		return reg->dynptr.type;
8436 
8437 	spi = __get_spi(reg->off);
8438 	if (spi < 0) {
8439 		verbose(env, "verifier internal error: invalid spi when querying dynptr type\n");
8440 		return BPF_DYNPTR_TYPE_INVALID;
8441 	}
8442 
8443 	return state->stack[spi].spilled_ptr.dynptr.type;
8444 }
8445 
8446 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
8447 			  struct bpf_call_arg_meta *meta,
8448 			  const struct bpf_func_proto *fn,
8449 			  int insn_idx)
8450 {
8451 	u32 regno = BPF_REG_1 + arg;
8452 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
8453 	enum bpf_arg_type arg_type = fn->arg_type[arg];
8454 	enum bpf_reg_type type = reg->type;
8455 	u32 *arg_btf_id = NULL;
8456 	int err = 0;
8457 
8458 	if (arg_type == ARG_DONTCARE)
8459 		return 0;
8460 
8461 	err = check_reg_arg(env, regno, SRC_OP);
8462 	if (err)
8463 		return err;
8464 
8465 	if (arg_type == ARG_ANYTHING) {
8466 		if (is_pointer_value(env, regno)) {
8467 			verbose(env, "R%d leaks addr into helper function\n",
8468 				regno);
8469 			return -EACCES;
8470 		}
8471 		return 0;
8472 	}
8473 
8474 	if (type_is_pkt_pointer(type) &&
8475 	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {
8476 		verbose(env, "helper access to the packet is not allowed\n");
8477 		return -EACCES;
8478 	}
8479 
8480 	if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
8481 		err = resolve_map_arg_type(env, meta, &arg_type);
8482 		if (err)
8483 			return err;
8484 	}
8485 
8486 	if (register_is_null(reg) && type_may_be_null(arg_type))
8487 		/* A NULL register has a SCALAR_VALUE type, so skip
8488 		 * type checking.
8489 		 */
8490 		goto skip_type_check;
8491 
8492 	/* arg_btf_id and arg_size are in a union. */
8493 	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
8494 	    base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
8495 		arg_btf_id = fn->arg_btf_id[arg];
8496 
8497 	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
8498 	if (err)
8499 		return err;
8500 
8501 	err = check_func_arg_reg_off(env, reg, regno, arg_type);
8502 	if (err)
8503 		return err;
8504 
8505 skip_type_check:
8506 	if (arg_type_is_release(arg_type)) {
8507 		if (arg_type_is_dynptr(arg_type)) {
8508 			struct bpf_func_state *state = func(env, reg);
8509 			int spi;
8510 
8511 			/* Only dynptr created on stack can be released, thus
8512 			 * the get_spi and stack state checks for spilled_ptr
8513 			 * should only be done before process_dynptr_func for
8514 			 * PTR_TO_STACK.
8515 			 */
8516 			if (reg->type == PTR_TO_STACK) {
8517 				spi = dynptr_get_spi(env, reg);
8518 				if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) {
8519 					verbose(env, "arg %d is an unacquired reference\n", regno);
8520 					return -EINVAL;
8521 				}
8522 			} else {
8523 				verbose(env, "cannot release unowned const bpf_dynptr\n");
8524 				return -EINVAL;
8525 			}
8526 		} else if (!reg->ref_obj_id && !register_is_null(reg)) {
8527 			verbose(env, "R%d must be referenced when passed to release function\n",
8528 				regno);
8529 			return -EINVAL;
8530 		}
8531 		if (meta->release_regno) {
8532 			verbose(env, "verifier internal error: more than one release argument\n");
8533 			return -EFAULT;
8534 		}
8535 		meta->release_regno = regno;
8536 	}
8537 
8538 	if (reg->ref_obj_id) {
8539 		if (meta->ref_obj_id) {
8540 			verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
8541 				regno, reg->ref_obj_id,
8542 				meta->ref_obj_id);
8543 			return -EFAULT;
8544 		}
8545 		meta->ref_obj_id = reg->ref_obj_id;
8546 	}
8547 
8548 	switch (base_type(arg_type)) {
8549 	case ARG_CONST_MAP_PTR:
8550 		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
8551 		if (meta->map_ptr) {
8552 			/* Use map_uid (which is unique id of inner map) to reject:
8553 			 * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
8554 			 * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
8555 			 * if (inner_map1 && inner_map2) {
8556 			 *     timer = bpf_map_lookup_elem(inner_map1);
8557 			 *     if (timer)
8558 			 *         // mismatch would have been allowed
8559 			 *         bpf_timer_init(timer, inner_map2);
8560 			 * }
8561 			 *
8562 			 * Comparing map_ptr is enough to distinguish normal and outer maps.
8563 			 */
8564 			if (meta->map_ptr != reg->map_ptr ||
8565 			    meta->map_uid != reg->map_uid) {
8566 				verbose(env,
8567 					"timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
8568 					meta->map_uid, reg->map_uid);
8569 				return -EINVAL;
8570 			}
8571 		}
8572 		meta->map_ptr = reg->map_ptr;
8573 		meta->map_uid = reg->map_uid;
8574 		break;
8575 	case ARG_PTR_TO_MAP_KEY:
8576 		/* bpf_map_xxx(..., map_ptr, ..., key) call:
8577 		 * check that [key, key + map->key_size) are within
8578 		 * stack limits and initialized
8579 		 */
8580 		if (!meta->map_ptr) {
8581 			/* in function declaration map_ptr must come before
8582 			 * map_key, so that it's verified and known before
8583 			 * we have to check map_key here. Otherwise it means
8584 			 * that kernel subsystem misconfigured verifier
8585 			 */
8586 			verbose(env, "invalid map_ptr to access map->key\n");
8587 			return -EACCES;
8588 		}
8589 		err = check_helper_mem_access(env, regno,
8590 					      meta->map_ptr->key_size, false,
8591 					      NULL);
8592 		break;
8593 	case ARG_PTR_TO_MAP_VALUE:
8594 		if (type_may_be_null(arg_type) && register_is_null(reg))
8595 			return 0;
8596 
8597 		/* bpf_map_xxx(..., map_ptr, ..., value) call:
8598 		 * check [value, value + map->value_size) validity
8599 		 */
8600 		if (!meta->map_ptr) {
8601 			/* kernel subsystem misconfigured verifier */
8602 			verbose(env, "invalid map_ptr to access map->value\n");
8603 			return -EACCES;
8604 		}
8605 		meta->raw_mode = arg_type & MEM_UNINIT;
8606 		err = check_helper_mem_access(env, regno,
8607 					      meta->map_ptr->value_size, false,
8608 					      meta);
8609 		break;
8610 	case ARG_PTR_TO_PERCPU_BTF_ID:
8611 		if (!reg->btf_id) {
8612 			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
8613 			return -EACCES;
8614 		}
8615 		meta->ret_btf = reg->btf;
8616 		meta->ret_btf_id = reg->btf_id;
8617 		break;
8618 	case ARG_PTR_TO_SPIN_LOCK:
8619 		if (in_rbtree_lock_required_cb(env)) {
8620 			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
8621 			return -EACCES;
8622 		}
8623 		if (meta->func_id == BPF_FUNC_spin_lock) {
8624 			err = process_spin_lock(env, regno, true);
8625 			if (err)
8626 				return err;
8627 		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
8628 			err = process_spin_lock(env, regno, false);
8629 			if (err)
8630 				return err;
8631 		} else {
8632 			verbose(env, "verifier internal error\n");
8633 			return -EFAULT;
8634 		}
8635 		break;
8636 	case ARG_PTR_TO_TIMER:
8637 		err = process_timer_func(env, regno, meta);
8638 		if (err)
8639 			return err;
8640 		break;
8641 	case ARG_PTR_TO_FUNC:
8642 		meta->subprogno = reg->subprogno;
8643 		break;
8644 	case ARG_PTR_TO_MEM:
8645 		/* The access to this pointer is only checked when we hit the
8646 		 * next is_mem_size argument below.
8647 		 */
8648 		meta->raw_mode = arg_type & MEM_UNINIT;
8649 		if (arg_type & MEM_FIXED_SIZE) {
8650 			err = check_helper_mem_access(env, regno,
8651 						      fn->arg_size[arg], false,
8652 						      meta);
8653 		}
8654 		break;
8655 	case ARG_CONST_SIZE:
8656 		err = check_mem_size_reg(env, reg, regno, false, meta);
8657 		break;
8658 	case ARG_CONST_SIZE_OR_ZERO:
8659 		err = check_mem_size_reg(env, reg, regno, true, meta);
8660 		break;
8661 	case ARG_PTR_TO_DYNPTR:
8662 		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);
8663 		if (err)
8664 			return err;
8665 		break;
8666 	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
8667 		if (!tnum_is_const(reg->var_off)) {
8668 			verbose(env, "R%d is not a known constant'\n",
8669 				regno);
8670 			return -EACCES;
8671 		}
8672 		meta->mem_size = reg->var_off.value;
8673 		err = mark_chain_precision(env, regno);
8674 		if (err)
8675 			return err;
8676 		break;
8677 	case ARG_PTR_TO_INT:
8678 	case ARG_PTR_TO_LONG:
8679 	{
8680 		int size = int_ptr_type_to_size(arg_type);
8681 
8682 		err = check_helper_mem_access(env, regno, size, false, meta);
8683 		if (err)
8684 			return err;
8685 		err = check_ptr_alignment(env, reg, 0, size, true);
8686 		break;
8687 	}
8688 	case ARG_PTR_TO_CONST_STR:
8689 	{
8690 		struct bpf_map *map = reg->map_ptr;
8691 		int map_off;
8692 		u64 map_addr;
8693 		char *str_ptr;
8694 
8695 		if (!bpf_map_is_rdonly(map)) {
8696 			verbose(env, "R%d does not point to a readonly map'\n", regno);
8697 			return -EACCES;
8698 		}
8699 
8700 		if (!tnum_is_const(reg->var_off)) {
8701 			verbose(env, "R%d is not a constant address'\n", regno);
8702 			return -EACCES;
8703 		}
8704 
8705 		if (!map->ops->map_direct_value_addr) {
8706 			verbose(env, "no direct value access support for this map type\n");
8707 			return -EACCES;
8708 		}
8709 
8710 		err = check_map_access(env, regno, reg->off,
8711 				       map->value_size - reg->off, false,
8712 				       ACCESS_HELPER);
8713 		if (err)
8714 			return err;
8715 
8716 		map_off = reg->off + reg->var_off.value;
8717 		err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
8718 		if (err) {
8719 			verbose(env, "direct value access on string failed\n");
8720 			return err;
8721 		}
8722 
8723 		str_ptr = (char *)(long)(map_addr);
8724 		if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
8725 			verbose(env, "string is not zero-terminated\n");
8726 			return -EINVAL;
8727 		}
8728 		break;
8729 	}
8730 	case ARG_PTR_TO_KPTR:
8731 		err = process_kptr_func(env, regno, meta);
8732 		if (err)
8733 			return err;
8734 		break;
8735 	}
8736 
8737 	return err;
8738 }
8739 
8740 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
8741 {
8742 	enum bpf_attach_type eatype = env->prog->expected_attach_type;
8743 	enum bpf_prog_type type = resolve_prog_type(env->prog);
8744 
8745 	if (func_id != BPF_FUNC_map_update_elem &&
8746 	    func_id != BPF_FUNC_map_delete_elem)
8747 		return false;
8748 
8749 	/* It's not possible to get access to a locked struct sock in these
8750 	 * contexts, so updating is safe.
8751 	 */
8752 	switch (type) {
8753 	case BPF_PROG_TYPE_TRACING:
8754 		if (eatype == BPF_TRACE_ITER)
8755 			return true;
8756 		break;
8757 	case BPF_PROG_TYPE_SOCK_OPS:
8758 		/* map_update allowed only via dedicated helpers with event type checks */
8759 		if (func_id == BPF_FUNC_map_delete_elem)
8760 			return true;
8761 		break;
8762 	case BPF_PROG_TYPE_SOCKET_FILTER:
8763 	case BPF_PROG_TYPE_SCHED_CLS:
8764 	case BPF_PROG_TYPE_SCHED_ACT:
8765 	case BPF_PROG_TYPE_XDP:
8766 	case BPF_PROG_TYPE_SK_REUSEPORT:
8767 	case BPF_PROG_TYPE_FLOW_DISSECTOR:
8768 	case BPF_PROG_TYPE_SK_LOOKUP:
8769 		return true;
8770 	default:
8771 		break;
8772 	}
8773 
8774 	verbose(env, "cannot update sockmap in this context\n");
8775 	return false;
8776 }
8777 
8778 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
8779 {
8780 	return env->prog->jit_requested &&
8781 	       bpf_jit_supports_subprog_tailcalls();
8782 }
8783 
8784 static int check_map_func_compatibility(struct bpf_verifier_env *env,
8785 					struct bpf_map *map, int func_id)
8786 {
8787 	if (!map)
8788 		return 0;
8789 
8790 	/* We need a two way check, first is from map perspective ... */
8791 	switch (map->map_type) {
8792 	case BPF_MAP_TYPE_PROG_ARRAY:
8793 		if (func_id != BPF_FUNC_tail_call)
8794 			goto error;
8795 		break;
8796 	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
8797 		if (func_id != BPF_FUNC_perf_event_read &&
8798 		    func_id != BPF_FUNC_perf_event_output &&
8799 		    func_id != BPF_FUNC_skb_output &&
8800 		    func_id != BPF_FUNC_perf_event_read_value &&
8801 		    func_id != BPF_FUNC_xdp_output)
8802 			goto error;
8803 		break;
8804 	case BPF_MAP_TYPE_RINGBUF:
8805 		if (func_id != BPF_FUNC_ringbuf_output &&
8806 		    func_id != BPF_FUNC_ringbuf_reserve &&
8807 		    func_id != BPF_FUNC_ringbuf_query &&
8808 		    func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
8809 		    func_id != BPF_FUNC_ringbuf_submit_dynptr &&
8810 		    func_id != BPF_FUNC_ringbuf_discard_dynptr)
8811 			goto error;
8812 		break;
8813 	case BPF_MAP_TYPE_USER_RINGBUF:
8814 		if (func_id != BPF_FUNC_user_ringbuf_drain)
8815 			goto error;
8816 		break;
8817 	case BPF_MAP_TYPE_STACK_TRACE:
8818 		if (func_id != BPF_FUNC_get_stackid)
8819 			goto error;
8820 		break;
8821 	case BPF_MAP_TYPE_CGROUP_ARRAY:
8822 		if (func_id != BPF_FUNC_skb_under_cgroup &&
8823 		    func_id != BPF_FUNC_current_task_under_cgroup)
8824 			goto error;
8825 		break;
8826 	case BPF_MAP_TYPE_CGROUP_STORAGE:
8827 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
8828 		if (func_id != BPF_FUNC_get_local_storage)
8829 			goto error;
8830 		break;
8831 	case BPF_MAP_TYPE_DEVMAP:
8832 	case BPF_MAP_TYPE_DEVMAP_HASH:
8833 		if (func_id != BPF_FUNC_redirect_map &&
8834 		    func_id != BPF_FUNC_map_lookup_elem)
8835 			goto error;
8836 		break;
8837 	/* Restrict bpf side of cpumap and xskmap, open when use-cases
8838 	 * appear.
8839 	 */
8840 	case BPF_MAP_TYPE_CPUMAP:
8841 		if (func_id != BPF_FUNC_redirect_map)
8842 			goto error;
8843 		break;
8844 	case BPF_MAP_TYPE_XSKMAP:
8845 		if (func_id != BPF_FUNC_redirect_map &&
8846 		    func_id != BPF_FUNC_map_lookup_elem)
8847 			goto error;
8848 		break;
8849 	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
8850 	case BPF_MAP_TYPE_HASH_OF_MAPS:
8851 		if (func_id != BPF_FUNC_map_lookup_elem)
8852 			goto error;
8853 		break;
8854 	case BPF_MAP_TYPE_SOCKMAP:
8855 		if (func_id != BPF_FUNC_sk_redirect_map &&
8856 		    func_id != BPF_FUNC_sock_map_update &&
8857 		    func_id != BPF_FUNC_msg_redirect_map &&
8858 		    func_id != BPF_FUNC_sk_select_reuseport &&
8859 		    func_id != BPF_FUNC_map_lookup_elem &&
8860 		    !may_update_sockmap(env, func_id))
8861 			goto error;
8862 		break;
8863 	case BPF_MAP_TYPE_SOCKHASH:
8864 		if (func_id != BPF_FUNC_sk_redirect_hash &&
8865 		    func_id != BPF_FUNC_sock_hash_update &&
8866 		    func_id != BPF_FUNC_msg_redirect_hash &&
8867 		    func_id != BPF_FUNC_sk_select_reuseport &&
8868 		    func_id != BPF_FUNC_map_lookup_elem &&
8869 		    !may_update_sockmap(env, func_id))
8870 			goto error;
8871 		break;
8872 	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
8873 		if (func_id != BPF_FUNC_sk_select_reuseport)
8874 			goto error;
8875 		break;
8876 	case BPF_MAP_TYPE_QUEUE:
8877 	case BPF_MAP_TYPE_STACK:
8878 		if (func_id != BPF_FUNC_map_peek_elem &&
8879 		    func_id != BPF_FUNC_map_pop_elem &&
8880 		    func_id != BPF_FUNC_map_push_elem)
8881 			goto error;
8882 		break;
8883 	case BPF_MAP_TYPE_SK_STORAGE:
8884 		if (func_id != BPF_FUNC_sk_storage_get &&
8885 		    func_id != BPF_FUNC_sk_storage_delete &&
8886 		    func_id != BPF_FUNC_kptr_xchg)
8887 			goto error;
8888 		break;
8889 	case BPF_MAP_TYPE_INODE_STORAGE:
8890 		if (func_id != BPF_FUNC_inode_storage_get &&
8891 		    func_id != BPF_FUNC_inode_storage_delete &&
8892 		    func_id != BPF_FUNC_kptr_xchg)
8893 			goto error;
8894 		break;
8895 	case BPF_MAP_TYPE_TASK_STORAGE:
8896 		if (func_id != BPF_FUNC_task_storage_get &&
8897 		    func_id != BPF_FUNC_task_storage_delete &&
8898 		    func_id != BPF_FUNC_kptr_xchg)
8899 			goto error;
8900 		break;
8901 	case BPF_MAP_TYPE_CGRP_STORAGE:
8902 		if (func_id != BPF_FUNC_cgrp_storage_get &&
8903 		    func_id != BPF_FUNC_cgrp_storage_delete &&
8904 		    func_id != BPF_FUNC_kptr_xchg)
8905 			goto error;
8906 		break;
8907 	case BPF_MAP_TYPE_BLOOM_FILTER:
8908 		if (func_id != BPF_FUNC_map_peek_elem &&
8909 		    func_id != BPF_FUNC_map_push_elem)
8910 			goto error;
8911 		break;
8912 	default:
8913 		break;
8914 	}
8915 
8916 	/* ... and second from the function itself. */
8917 	switch (func_id) {
8918 	case BPF_FUNC_tail_call:
8919 		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
8920 			goto error;
8921 		if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
8922 			verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
8923 			return -EINVAL;
8924 		}
8925 		break;
8926 	case BPF_FUNC_perf_event_read:
8927 	case BPF_FUNC_perf_event_output:
8928 	case BPF_FUNC_perf_event_read_value:
8929 	case BPF_FUNC_skb_output:
8930 	case BPF_FUNC_xdp_output:
8931 		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
8932 			goto error;
8933 		break;
8934 	case BPF_FUNC_ringbuf_output:
8935 	case BPF_FUNC_ringbuf_reserve:
8936 	case BPF_FUNC_ringbuf_query:
8937 	case BPF_FUNC_ringbuf_reserve_dynptr:
8938 	case BPF_FUNC_ringbuf_submit_dynptr:
8939 	case BPF_FUNC_ringbuf_discard_dynptr:
8940 		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
8941 			goto error;
8942 		break;
8943 	case BPF_FUNC_user_ringbuf_drain:
8944 		if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
8945 			goto error;
8946 		break;
8947 	case BPF_FUNC_get_stackid:
8948 		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
8949 			goto error;
8950 		break;
8951 	case BPF_FUNC_current_task_under_cgroup:
8952 	case BPF_FUNC_skb_under_cgroup:
8953 		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
8954 			goto error;
8955 		break;
8956 	case BPF_FUNC_redirect_map:
8957 		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
8958 		    map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
8959 		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
8960 		    map->map_type != BPF_MAP_TYPE_XSKMAP)
8961 			goto error;
8962 		break;
8963 	case BPF_FUNC_sk_redirect_map:
8964 	case BPF_FUNC_msg_redirect_map:
8965 	case BPF_FUNC_sock_map_update:
8966 		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
8967 			goto error;
8968 		break;
8969 	case BPF_FUNC_sk_redirect_hash:
8970 	case BPF_FUNC_msg_redirect_hash:
8971 	case BPF_FUNC_sock_hash_update:
8972 		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
8973 			goto error;
8974 		break;
8975 	case BPF_FUNC_get_local_storage:
8976 		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
8977 		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
8978 			goto error;
8979 		break;
8980 	case BPF_FUNC_sk_select_reuseport:
8981 		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
8982 		    map->map_type != BPF_MAP_TYPE_SOCKMAP &&
8983 		    map->map_type != BPF_MAP_TYPE_SOCKHASH)
8984 			goto error;
8985 		break;
8986 	case BPF_FUNC_map_pop_elem:
8987 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
8988 		    map->map_type != BPF_MAP_TYPE_STACK)
8989 			goto error;
8990 		break;
8991 	case BPF_FUNC_map_peek_elem:
8992 	case BPF_FUNC_map_push_elem:
8993 		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
8994 		    map->map_type != BPF_MAP_TYPE_STACK &&
8995 		    map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
8996 			goto error;
8997 		break;
8998 	case BPF_FUNC_map_lookup_percpu_elem:
8999 		if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
9000 		    map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
9001 		    map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
9002 			goto error;
9003 		break;
9004 	case BPF_FUNC_sk_storage_get:
9005 	case BPF_FUNC_sk_storage_delete:
9006 		if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
9007 			goto error;
9008 		break;
9009 	case BPF_FUNC_inode_storage_get:
9010 	case BPF_FUNC_inode_storage_delete:
9011 		if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
9012 			goto error;
9013 		break;
9014 	case BPF_FUNC_task_storage_get:
9015 	case BPF_FUNC_task_storage_delete:
9016 		if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
9017 			goto error;
9018 		break;
9019 	case BPF_FUNC_cgrp_storage_get:
9020 	case BPF_FUNC_cgrp_storage_delete:
9021 		if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE)
9022 			goto error;
9023 		break;
9024 	default:
9025 		break;
9026 	}
9027 
9028 	return 0;
9029 error:
9030 	verbose(env, "cannot pass map_type %d into func %s#%d\n",
9031 		map->map_type, func_id_name(func_id), func_id);
9032 	return -EINVAL;
9033 }
9034 
9035 static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
9036 {
9037 	int count = 0;
9038 
9039 	if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
9040 		count++;
9041 	if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
9042 		count++;
9043 	if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
9044 		count++;
9045 	if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
9046 		count++;
9047 	if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
9048 		count++;
9049 
9050 	/* We only support one arg being in raw mode at the moment,
9051 	 * which is sufficient for the helper functions we have
9052 	 * right now.
9053 	 */
9054 	return count <= 1;
9055 }
9056 
9057 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
9058 {
9059 	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
9060 	bool has_size = fn->arg_size[arg] != 0;
9061 	bool is_next_size = false;
9062 
9063 	if (arg + 1 < ARRAY_SIZE(fn->arg_type))
9064 		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
9065 
9066 	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
9067 		return is_next_size;
9068 
9069 	return has_size == is_next_size || is_next_size == is_fixed;
9070 }
9071 
9072 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
9073 {
9074 	/* bpf_xxx(..., buf, len) call will access 'len'
9075 	 * bytes from memory 'buf'. Both arg types need
9076 	 * to be paired, so make sure there's no buggy
9077 	 * helper function specification.
9078 	 */
9079 	if (arg_type_is_mem_size(fn->arg1_type) ||
9080 	    check_args_pair_invalid(fn, 0) ||
9081 	    check_args_pair_invalid(fn, 1) ||
9082 	    check_args_pair_invalid(fn, 2) ||
9083 	    check_args_pair_invalid(fn, 3) ||
9084 	    check_args_pair_invalid(fn, 4))
9085 		return false;
9086 
9087 	return true;
9088 }
9089 
9090 static bool check_btf_id_ok(const struct bpf_func_proto *fn)
9091 {
9092 	int i;
9093 
9094 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
9095 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
9096 			return !!fn->arg_btf_id[i];
9097 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK)
9098 			return fn->arg_btf_id[i] == BPF_PTR_POISON;
9099 		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
9100 		    /* arg_btf_id and arg_size are in a union. */
9101 		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
9102 		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
9103 			return false;
9104 	}
9105 
9106 	return true;
9107 }
9108 
9109 static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
9110 {
9111 	return check_raw_mode_ok(fn) &&
9112 	       check_arg_pair_ok(fn) &&
9113 	       check_btf_id_ok(fn) ? 0 : -EINVAL;
9114 }
9115 
9116 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
9117  * are now invalid, so turn them into unknown SCALAR_VALUE.
9118  *
9119  * This also applies to dynptr slices belonging to skb and xdp dynptrs,
9120  * since these slices point to packet data.
9121  */
9122 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
9123 {
9124 	struct bpf_func_state *state;
9125 	struct bpf_reg_state *reg;
9126 
9127 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
9128 		if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))
9129 			mark_reg_invalid(env, reg);
9130 	}));
9131 }
9132 
9133 enum {
9134 	AT_PKT_END = -1,
9135 	BEYOND_PKT_END = -2,
9136 };
9137 
9138 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
9139 {
9140 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
9141 	struct bpf_reg_state *reg = &state->regs[regn];
9142 
9143 	if (reg->type != PTR_TO_PACKET)
9144 		/* PTR_TO_PACKET_META is not supported yet */
9145 		return;
9146 
9147 	/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
9148 	 * How far beyond pkt_end it goes is unknown.
9149 	 * if (!range_open) it's the case of pkt >= pkt_end
9150 	 * if (range_open) it's the case of pkt > pkt_end
9151 	 * hence this pointer is at least 1 byte bigger than pkt_end
9152 	 */
9153 	if (range_open)
9154 		reg->range = BEYOND_PKT_END;
9155 	else
9156 		reg->range = AT_PKT_END;
9157 }
9158 
9159 /* The pointer with the specified id has released its reference to kernel
9160  * resources. Identify all copies of the same pointer and clear the reference.
9161  */
9162 static int release_reference(struct bpf_verifier_env *env,
9163 			     int ref_obj_id)
9164 {
9165 	struct bpf_func_state *state;
9166 	struct bpf_reg_state *reg;
9167 	int err;
9168 
9169 	err = release_reference_state(cur_func(env), ref_obj_id);
9170 	if (err)
9171 		return err;
9172 
9173 	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
9174 		if (reg->ref_obj_id == ref_obj_id)
9175 			mark_reg_invalid(env, reg);
9176 	}));
9177 
9178 	return 0;
9179 }
9180 
9181 static void invalidate_non_owning_refs(struct bpf_verifier_env *env)
9182 {
9183 	struct bpf_func_state *unused;
9184 	struct bpf_reg_state *reg;
9185 
9186 	bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
9187 		if (type_is_non_owning_ref(reg->type))
9188 			mark_reg_invalid(env, reg);
9189 	}));
9190 }
9191 
9192 static void clear_caller_saved_regs(struct bpf_verifier_env *env,
9193 				    struct bpf_reg_state *regs)
9194 {
9195 	int i;
9196 
9197 	/* after the call registers r0 - r5 were scratched */
9198 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
9199 		mark_reg_not_init(env, regs, caller_saved[i]);
9200 		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
9201 	}
9202 }
9203 
9204 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
9205 				   struct bpf_func_state *caller,
9206 				   struct bpf_func_state *callee,
9207 				   int insn_idx);
9208 
9209 static int set_callee_state(struct bpf_verifier_env *env,
9210 			    struct bpf_func_state *caller,
9211 			    struct bpf_func_state *callee, int insn_idx);
9212 
9213 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
9214 			    set_callee_state_fn set_callee_state_cb,
9215 			    struct bpf_verifier_state *state)
9216 {
9217 	struct bpf_func_state *caller, *callee;
9218 	int err;
9219 
9220 	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
9221 		verbose(env, "the call stack of %d frames is too deep\n",
9222 			state->curframe + 2);
9223 		return -E2BIG;
9224 	}
9225 
9226 	if (state->frame[state->curframe + 1]) {
9227 		verbose(env, "verifier bug. Frame %d already allocated\n",
9228 			state->curframe + 1);
9229 		return -EFAULT;
9230 	}
9231 
9232 	caller = state->frame[state->curframe];
9233 	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
9234 	if (!callee)
9235 		return -ENOMEM;
9236 	state->frame[state->curframe + 1] = callee;
9237 
9238 	/* callee cannot access r0, r6 - r9 for reading and has to write
9239 	 * into its own stack before reading from it.
9240 	 * callee can read/write into caller's stack
9241 	 */
9242 	init_func_state(env, callee,
9243 			/* remember the callsite, it will be used by bpf_exit */
9244 			callsite,
9245 			state->curframe + 1 /* frameno within this callchain */,
9246 			subprog /* subprog number within this prog */);
9247 	/* Transfer references to the callee */
9248 	err = copy_reference_state(callee, caller);
9249 	err = err ?: set_callee_state_cb(env, caller, callee, callsite);
9250 	if (err)
9251 		goto err_out;
9252 
9253 	/* only increment it after check_reg_arg() finished */
9254 	state->curframe++;
9255 
9256 	return 0;
9257 
9258 err_out:
9259 	free_func_state(callee);
9260 	state->frame[state->curframe + 1] = NULL;
9261 	return err;
9262 }
9263 
9264 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
9265 			      int insn_idx, int subprog,
9266 			      set_callee_state_fn set_callee_state_cb)
9267 {
9268 	struct bpf_verifier_state *state = env->cur_state, *callback_state;
9269 	struct bpf_func_state *caller, *callee;
9270 	int err;
9271 
9272 	caller = state->frame[state->curframe];
9273 	err = btf_check_subprog_call(env, subprog, caller->regs);
9274 	if (err == -EFAULT)
9275 		return err;
9276 
9277 	/* set_callee_state is used for direct subprog calls, but we are
9278 	 * interested in validating only BPF helpers that can call subprogs as
9279 	 * callbacks
9280 	 */
9281 	if (bpf_pseudo_kfunc_call(insn) &&
9282 	    !is_sync_callback_calling_kfunc(insn->imm)) {
9283 		verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
9284 			func_id_name(insn->imm), insn->imm);
9285 		return -EFAULT;
9286 	} else if (!bpf_pseudo_kfunc_call(insn) &&
9287 		   !is_callback_calling_function(insn->imm)) { /* helper */
9288 		verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
9289 			func_id_name(insn->imm), insn->imm);
9290 		return -EFAULT;
9291 	}
9292 
9293 	if (insn->code == (BPF_JMP | BPF_CALL) &&
9294 	    insn->src_reg == 0 &&
9295 	    insn->imm == BPF_FUNC_timer_set_callback) {
9296 		struct bpf_verifier_state *async_cb;
9297 
9298 		/* there is no real recursion here. timer callbacks are async */
9299 		env->subprog_info[subprog].is_async_cb = true;
9300 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
9301 					 insn_idx, subprog);
9302 		if (!async_cb)
9303 			return -EFAULT;
9304 		callee = async_cb->frame[0];
9305 		callee->async_entry_cnt = caller->async_entry_cnt + 1;
9306 
9307 		/* Convert bpf_timer_set_callback() args into timer callback args */
9308 		err = set_callee_state_cb(env, caller, callee, insn_idx);
9309 		if (err)
9310 			return err;
9311 
9312 		return 0;
9313 	}
9314 
9315 	/* for callback functions enqueue entry to callback and
9316 	 * proceed with next instruction within current frame.
9317 	 */
9318 	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
9319 	if (!callback_state)
9320 		return -ENOMEM;
9321 
9322 	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
9323 			       callback_state);
9324 	if (err)
9325 		return err;
9326 
9327 	callback_state->callback_unroll_depth++;
9328 	callback_state->frame[callback_state->curframe - 1]->callback_depth++;
9329 	caller->callback_depth = 0;
9330 	return 0;
9331 }
9332 
9333 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
9334 			   int *insn_idx)
9335 {
9336 	struct bpf_verifier_state *state = env->cur_state;
9337 	struct bpf_func_state *caller;
9338 	int err, subprog, target_insn;
9339 
9340 	target_insn = *insn_idx + insn->imm + 1;
9341 	subprog = find_subprog(env, target_insn);
9342 	if (subprog < 0) {
9343 		verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);
9344 		return -EFAULT;
9345 	}
9346 
9347 	caller = state->frame[state->curframe];
9348 	err = btf_check_subprog_call(env, subprog, caller->regs);
9349 	if (err == -EFAULT)
9350 		return err;
9351 	if (subprog_is_global(env, subprog)) {
9352 		if (err) {
9353 			verbose(env, "Caller passes invalid args into func#%d\n", subprog);
9354 			return err;
9355 		}
9356 
9357 		if (env->log.level & BPF_LOG_LEVEL)
9358 			verbose(env, "Func#%d is global and valid. Skipping.\n", subprog);
9359 		clear_caller_saved_regs(env, caller->regs);
9360 
9361 		/* All global functions return a 64-bit SCALAR_VALUE */
9362 		mark_reg_unknown(env, caller->regs, BPF_REG_0);
9363 		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
9364 
9365 		/* continue with next insn after call */
9366 		return 0;
9367 	}
9368 
9369 	/* for regular function entry setup new frame and continue
9370 	 * from that frame.
9371 	 */
9372 	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
9373 	if (err)
9374 		return err;
9375 
9376 	clear_caller_saved_regs(env, caller->regs);
9377 
9378 	/* and go analyze first insn of the callee */
9379 	*insn_idx = env->subprog_info[subprog].start - 1;
9380 
9381 	if (env->log.level & BPF_LOG_LEVEL) {
9382 		verbose(env, "caller:\n");
9383 		print_verifier_state(env, caller, true);
9384 		verbose(env, "callee:\n");
9385 		print_verifier_state(env, state->frame[state->curframe], true);
9386 	}
9387 
9388 	return 0;
9389 }
9390 
9391 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
9392 				   struct bpf_func_state *caller,
9393 				   struct bpf_func_state *callee)
9394 {
9395 	/* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
9396 	 *      void *callback_ctx, u64 flags);
9397 	 * callback_fn(struct bpf_map *map, void *key, void *value,
9398 	 *      void *callback_ctx);
9399 	 */
9400 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
9401 
9402 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
9403 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9404 	callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
9405 
9406 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
9407 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
9408 	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
9409 
9410 	/* pointer to stack or null */
9411 	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
9412 
9413 	/* unused */
9414 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9415 	return 0;
9416 }
9417 
9418 static int set_callee_state(struct bpf_verifier_env *env,
9419 			    struct bpf_func_state *caller,
9420 			    struct bpf_func_state *callee, int insn_idx)
9421 {
9422 	int i;
9423 
9424 	/* copy r1 - r5 args that callee can access.  The copy includes parent
9425 	 * pointers, which connects us up to the liveness chain
9426 	 */
9427 	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
9428 		callee->regs[i] = caller->regs[i];
9429 	return 0;
9430 }
9431 
9432 static int set_map_elem_callback_state(struct bpf_verifier_env *env,
9433 				       struct bpf_func_state *caller,
9434 				       struct bpf_func_state *callee,
9435 				       int insn_idx)
9436 {
9437 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
9438 	struct bpf_map *map;
9439 	int err;
9440 
9441 	if (bpf_map_ptr_poisoned(insn_aux)) {
9442 		verbose(env, "tail_call abusing map_ptr\n");
9443 		return -EINVAL;
9444 	}
9445 
9446 	map = BPF_MAP_PTR(insn_aux->map_ptr_state);
9447 	if (!map->ops->map_set_for_each_callback_args ||
9448 	    !map->ops->map_for_each_callback) {
9449 		verbose(env, "callback function not allowed for map\n");
9450 		return -ENOTSUPP;
9451 	}
9452 
9453 	err = map->ops->map_set_for_each_callback_args(env, caller, callee);
9454 	if (err)
9455 		return err;
9456 
9457 	callee->in_callback_fn = true;
9458 	callee->callback_ret_range = tnum_range(0, 1);
9459 	return 0;
9460 }
9461 
9462 static int set_loop_callback_state(struct bpf_verifier_env *env,
9463 				   struct bpf_func_state *caller,
9464 				   struct bpf_func_state *callee,
9465 				   int insn_idx)
9466 {
9467 	/* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
9468 	 *	    u64 flags);
9469 	 * callback_fn(u32 index, void *callback_ctx);
9470 	 */
9471 	callee->regs[BPF_REG_1].type = SCALAR_VALUE;
9472 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
9473 
9474 	/* unused */
9475 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9476 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9477 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9478 
9479 	callee->in_callback_fn = true;
9480 	callee->callback_ret_range = tnum_range(0, 1);
9481 	return 0;
9482 }
9483 
9484 static int set_timer_callback_state(struct bpf_verifier_env *env,
9485 				    struct bpf_func_state *caller,
9486 				    struct bpf_func_state *callee,
9487 				    int insn_idx)
9488 {
9489 	struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
9490 
9491 	/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
9492 	 * callback_fn(struct bpf_map *map, void *key, void *value);
9493 	 */
9494 	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
9495 	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
9496 	callee->regs[BPF_REG_1].map_ptr = map_ptr;
9497 
9498 	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
9499 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9500 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
9501 
9502 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
9503 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
9504 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
9505 
9506 	/* unused */
9507 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9508 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9509 	callee->in_async_callback_fn = true;
9510 	callee->callback_ret_range = tnum_range(0, 1);
9511 	return 0;
9512 }
9513 
9514 static int set_find_vma_callback_state(struct bpf_verifier_env *env,
9515 				       struct bpf_func_state *caller,
9516 				       struct bpf_func_state *callee,
9517 				       int insn_idx)
9518 {
9519 	/* bpf_find_vma(struct task_struct *task, u64 addr,
9520 	 *               void *callback_fn, void *callback_ctx, u64 flags)
9521 	 * (callback_fn)(struct task_struct *task,
9522 	 *               struct vm_area_struct *vma, void *callback_ctx);
9523 	 */
9524 	callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
9525 
9526 	callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
9527 	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
9528 	callee->regs[BPF_REG_2].btf =  btf_vmlinux;
9529 	callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA],
9530 
9531 	/* pointer to stack or null */
9532 	callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
9533 
9534 	/* unused */
9535 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9536 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9537 	callee->in_callback_fn = true;
9538 	callee->callback_ret_range = tnum_range(0, 1);
9539 	return 0;
9540 }
9541 
9542 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
9543 					   struct bpf_func_state *caller,
9544 					   struct bpf_func_state *callee,
9545 					   int insn_idx)
9546 {
9547 	/* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
9548 	 *			  callback_ctx, u64 flags);
9549 	 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx);
9550 	 */
9551 	__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
9552 	mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL);
9553 	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
9554 
9555 	/* unused */
9556 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9557 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9558 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9559 
9560 	callee->in_callback_fn = true;
9561 	callee->callback_ret_range = tnum_range(0, 1);
9562 	return 0;
9563 }
9564 
9565 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env,
9566 					 struct bpf_func_state *caller,
9567 					 struct bpf_func_state *callee,
9568 					 int insn_idx)
9569 {
9570 	/* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
9571 	 *                     bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b));
9572 	 *
9573 	 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset
9574 	 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd
9575 	 * by this point, so look at 'root'
9576 	 */
9577 	struct btf_field *field;
9578 
9579 	field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off,
9580 				      BPF_RB_ROOT);
9581 	if (!field || !field->graph_root.value_btf_id)
9582 		return -EFAULT;
9583 
9584 	mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root);
9585 	ref_set_non_owning(env, &callee->regs[BPF_REG_1]);
9586 	mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root);
9587 	ref_set_non_owning(env, &callee->regs[BPF_REG_2]);
9588 
9589 	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
9590 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
9591 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
9592 	callee->in_callback_fn = true;
9593 	callee->callback_ret_range = tnum_range(0, 1);
9594 	return 0;
9595 }
9596 
9597 static bool is_rbtree_lock_required_kfunc(u32 btf_id);
9598 
9599 /* Are we currently verifying the callback for a rbtree helper that must
9600  * be called with lock held? If so, no need to complain about unreleased
9601  * lock
9602  */
9603 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
9604 {
9605 	struct bpf_verifier_state *state = env->cur_state;
9606 	struct bpf_insn *insn = env->prog->insnsi;
9607 	struct bpf_func_state *callee;
9608 	int kfunc_btf_id;
9609 
9610 	if (!state->curframe)
9611 		return false;
9612 
9613 	callee = state->frame[state->curframe];
9614 
9615 	if (!callee->in_callback_fn)
9616 		return false;
9617 
9618 	kfunc_btf_id = insn[callee->callsite].imm;
9619 	return is_rbtree_lock_required_kfunc(kfunc_btf_id);
9620 }
9621 
9622 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
9623 {
9624 	struct bpf_verifier_state *state = env->cur_state, *prev_st;
9625 	struct bpf_func_state *caller, *callee;
9626 	struct bpf_reg_state *r0;
9627 	bool in_callback_fn;
9628 	int err;
9629 
9630 	callee = state->frame[state->curframe];
9631 	r0 = &callee->regs[BPF_REG_0];
9632 	if (r0->type == PTR_TO_STACK) {
9633 		/* technically it's ok to return caller's stack pointer
9634 		 * (or caller's caller's pointer) back to the caller,
9635 		 * since these pointers are valid. Only current stack
9636 		 * pointer will be invalid as soon as function exits,
9637 		 * but let's be conservative
9638 		 */
9639 		verbose(env, "cannot return stack pointer to the caller\n");
9640 		return -EINVAL;
9641 	}
9642 
9643 	caller = state->frame[state->curframe - 1];
9644 	if (callee->in_callback_fn) {
9645 		/* enforce R0 return value range [0, 1]. */
9646 		struct tnum range = callee->callback_ret_range;
9647 
9648 		if (r0->type != SCALAR_VALUE) {
9649 			verbose(env, "R0 not a scalar value\n");
9650 			return -EACCES;
9651 		}
9652 
9653 		/* we are going to rely on register's precise value */
9654 		err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64);
9655 		err = err ?: mark_chain_precision(env, BPF_REG_0);
9656 		if (err)
9657 			return err;
9658 
9659 		if (!tnum_in(range, r0->var_off)) {
9660 			verbose_invalid_scalar(env, r0, &range, "callback return", "R0");
9661 			return -EINVAL;
9662 		}
9663 		if (!calls_callback(env, callee->callsite)) {
9664 			verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n",
9665 				*insn_idx, callee->callsite);
9666 			return -EFAULT;
9667 		}
9668 	} else {
9669 		/* return to the caller whatever r0 had in the callee */
9670 		caller->regs[BPF_REG_0] = *r0;
9671 	}
9672 
9673 	/* callback_fn frame should have released its own additions to parent's
9674 	 * reference state at this point, or check_reference_leak would
9675 	 * complain, hence it must be the same as the caller. There is no need
9676 	 * to copy it back.
9677 	 */
9678 	if (!callee->in_callback_fn) {
9679 		/* Transfer references to the caller */
9680 		err = copy_reference_state(caller, callee);
9681 		if (err)
9682 			return err;
9683 	}
9684 
9685 	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
9686 	 * there function call logic would reschedule callback visit. If iteration
9687 	 * converges is_state_visited() would prune that visit eventually.
9688 	 */
9689 	in_callback_fn = callee->in_callback_fn;
9690 	if (in_callback_fn)
9691 		*insn_idx = callee->callsite;
9692 	else
9693 		*insn_idx = callee->callsite + 1;
9694 
9695 	if (env->log.level & BPF_LOG_LEVEL) {
9696 		verbose(env, "returning from callee:\n");
9697 		print_verifier_state(env, callee, true);
9698 		verbose(env, "to caller at %d:\n", *insn_idx);
9699 		print_verifier_state(env, caller, true);
9700 	}
9701 	/* clear everything in the callee */
9702 	free_func_state(callee);
9703 	state->frame[state->curframe--] = NULL;
9704 
9705 	/* for callbacks widen imprecise scalars to make programs like below verify:
9706 	 *
9707 	 *   struct ctx { int i; }
9708 	 *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
9709 	 *   ...
9710 	 *   struct ctx = { .i = 0; }
9711 	 *   bpf_loop(100, cb, &ctx, 0);
9712 	 *
9713 	 * This is similar to what is done in process_iter_next_call() for open
9714 	 * coded iterators.
9715 	 */
9716 	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
9717 	if (prev_st) {
9718 		err = widen_imprecise_scalars(env, prev_st, state);
9719 		if (err)
9720 			return err;
9721 	}
9722 	return 0;
9723 }
9724 
9725 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
9726 				   int func_id,
9727 				   struct bpf_call_arg_meta *meta)
9728 {
9729 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
9730 
9731 	if (ret_type != RET_INTEGER)
9732 		return;
9733 
9734 	switch (func_id) {
9735 	case BPF_FUNC_get_stack:
9736 	case BPF_FUNC_get_task_stack:
9737 	case BPF_FUNC_probe_read_str:
9738 	case BPF_FUNC_probe_read_kernel_str:
9739 	case BPF_FUNC_probe_read_user_str:
9740 		ret_reg->smax_value = meta->msize_max_value;
9741 		ret_reg->s32_max_value = meta->msize_max_value;
9742 		ret_reg->smin_value = -MAX_ERRNO;
9743 		ret_reg->s32_min_value = -MAX_ERRNO;
9744 		reg_bounds_sync(ret_reg);
9745 		break;
9746 	case BPF_FUNC_get_smp_processor_id:
9747 		ret_reg->umax_value = nr_cpu_ids - 1;
9748 		ret_reg->u32_max_value = nr_cpu_ids - 1;
9749 		ret_reg->smax_value = nr_cpu_ids - 1;
9750 		ret_reg->s32_max_value = nr_cpu_ids - 1;
9751 		ret_reg->umin_value = 0;
9752 		ret_reg->u32_min_value = 0;
9753 		ret_reg->smin_value = 0;
9754 		ret_reg->s32_min_value = 0;
9755 		reg_bounds_sync(ret_reg);
9756 		break;
9757 	}
9758 }
9759 
9760 static int
9761 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
9762 		int func_id, int insn_idx)
9763 {
9764 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
9765 	struct bpf_map *map = meta->map_ptr;
9766 
9767 	if (func_id != BPF_FUNC_tail_call &&
9768 	    func_id != BPF_FUNC_map_lookup_elem &&
9769 	    func_id != BPF_FUNC_map_update_elem &&
9770 	    func_id != BPF_FUNC_map_delete_elem &&
9771 	    func_id != BPF_FUNC_map_push_elem &&
9772 	    func_id != BPF_FUNC_map_pop_elem &&
9773 	    func_id != BPF_FUNC_map_peek_elem &&
9774 	    func_id != BPF_FUNC_for_each_map_elem &&
9775 	    func_id != BPF_FUNC_redirect_map &&
9776 	    func_id != BPF_FUNC_map_lookup_percpu_elem)
9777 		return 0;
9778 
9779 	if (map == NULL) {
9780 		verbose(env, "kernel subsystem misconfigured verifier\n");
9781 		return -EINVAL;
9782 	}
9783 
9784 	/* In case of read-only, some additional restrictions
9785 	 * need to be applied in order to prevent altering the
9786 	 * state of the map from program side.
9787 	 */
9788 	if ((map->map_flags & BPF_F_RDONLY_PROG) &&
9789 	    (func_id == BPF_FUNC_map_delete_elem ||
9790 	     func_id == BPF_FUNC_map_update_elem ||
9791 	     func_id == BPF_FUNC_map_push_elem ||
9792 	     func_id == BPF_FUNC_map_pop_elem)) {
9793 		verbose(env, "write into map forbidden\n");
9794 		return -EACCES;
9795 	}
9796 
9797 	if (!BPF_MAP_PTR(aux->map_ptr_state))
9798 		bpf_map_ptr_store(aux, meta->map_ptr,
9799 				  !meta->map_ptr->bypass_spec_v1);
9800 	else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr)
9801 		bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
9802 				  !meta->map_ptr->bypass_spec_v1);
9803 	return 0;
9804 }
9805 
9806 static int
9807 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
9808 		int func_id, int insn_idx)
9809 {
9810 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
9811 	struct bpf_reg_state *regs = cur_regs(env), *reg;
9812 	struct bpf_map *map = meta->map_ptr;
9813 	u64 val, max;
9814 	int err;
9815 
9816 	if (func_id != BPF_FUNC_tail_call)
9817 		return 0;
9818 	if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
9819 		verbose(env, "kernel subsystem misconfigured verifier\n");
9820 		return -EINVAL;
9821 	}
9822 
9823 	reg = &regs[BPF_REG_3];
9824 	val = reg->var_off.value;
9825 	max = map->max_entries;
9826 
9827 	if (!(register_is_const(reg) && val < max)) {
9828 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
9829 		return 0;
9830 	}
9831 
9832 	err = mark_chain_precision(env, BPF_REG_3);
9833 	if (err)
9834 		return err;
9835 	if (bpf_map_key_unseen(aux))
9836 		bpf_map_key_store(aux, val);
9837 	else if (!bpf_map_key_poisoned(aux) &&
9838 		  bpf_map_key_immediate(aux) != val)
9839 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
9840 	return 0;
9841 }
9842 
9843 static int check_reference_leak(struct bpf_verifier_env *env)
9844 {
9845 	struct bpf_func_state *state = cur_func(env);
9846 	bool refs_lingering = false;
9847 	int i;
9848 
9849 	if (state->frameno && !state->in_callback_fn)
9850 		return 0;
9851 
9852 	for (i = 0; i < state->acquired_refs; i++) {
9853 		if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
9854 			continue;
9855 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
9856 			state->refs[i].id, state->refs[i].insn_idx);
9857 		refs_lingering = true;
9858 	}
9859 	return refs_lingering ? -EINVAL : 0;
9860 }
9861 
9862 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
9863 				   struct bpf_reg_state *regs)
9864 {
9865 	struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
9866 	struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
9867 	struct bpf_map *fmt_map = fmt_reg->map_ptr;
9868 	struct bpf_bprintf_data data = {};
9869 	int err, fmt_map_off, num_args;
9870 	u64 fmt_addr;
9871 	char *fmt;
9872 
9873 	/* data must be an array of u64 */
9874 	if (data_len_reg->var_off.value % 8)
9875 		return -EINVAL;
9876 	num_args = data_len_reg->var_off.value / 8;
9877 
9878 	/* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
9879 	 * and map_direct_value_addr is set.
9880 	 */
9881 	fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
9882 	err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
9883 						  fmt_map_off);
9884 	if (err) {
9885 		verbose(env, "verifier bug\n");
9886 		return -EFAULT;
9887 	}
9888 	fmt = (char *)(long)fmt_addr + fmt_map_off;
9889 
9890 	/* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
9891 	 * can focus on validating the format specifiers.
9892 	 */
9893 	err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data);
9894 	if (err < 0)
9895 		verbose(env, "Invalid format string\n");
9896 
9897 	return err;
9898 }
9899 
9900 static int check_get_func_ip(struct bpf_verifier_env *env)
9901 {
9902 	enum bpf_prog_type type = resolve_prog_type(env->prog);
9903 	int func_id = BPF_FUNC_get_func_ip;
9904 
9905 	if (type == BPF_PROG_TYPE_TRACING) {
9906 		if (!bpf_prog_has_trampoline(env->prog)) {
9907 			verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
9908 				func_id_name(func_id), func_id);
9909 			return -ENOTSUPP;
9910 		}
9911 		return 0;
9912 	} else if (type == BPF_PROG_TYPE_KPROBE) {
9913 		return 0;
9914 	}
9915 
9916 	verbose(env, "func %s#%d not supported for program type %d\n",
9917 		func_id_name(func_id), func_id, type);
9918 	return -ENOTSUPP;
9919 }
9920 
9921 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
9922 {
9923 	return &env->insn_aux_data[env->insn_idx];
9924 }
9925 
9926 static bool loop_flag_is_zero(struct bpf_verifier_env *env)
9927 {
9928 	struct bpf_reg_state *regs = cur_regs(env);
9929 	struct bpf_reg_state *reg = &regs[BPF_REG_4];
9930 	bool reg_is_null = register_is_null(reg);
9931 
9932 	if (reg_is_null)
9933 		mark_chain_precision(env, BPF_REG_4);
9934 
9935 	return reg_is_null;
9936 }
9937 
9938 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
9939 {
9940 	struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
9941 
9942 	if (!state->initialized) {
9943 		state->initialized = 1;
9944 		state->fit_for_inline = loop_flag_is_zero(env);
9945 		state->callback_subprogno = subprogno;
9946 		return;
9947 	}
9948 
9949 	if (!state->fit_for_inline)
9950 		return;
9951 
9952 	state->fit_for_inline = (loop_flag_is_zero(env) &&
9953 				 state->callback_subprogno == subprogno);
9954 }
9955 
9956 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
9957 			     int *insn_idx_p)
9958 {
9959 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
9960 	const struct bpf_func_proto *fn = NULL;
9961 	enum bpf_return_type ret_type;
9962 	enum bpf_type_flag ret_flag;
9963 	struct bpf_reg_state *regs;
9964 	struct bpf_call_arg_meta meta;
9965 	int insn_idx = *insn_idx_p;
9966 	bool changes_data;
9967 	int i, err, func_id;
9968 
9969 	/* find function prototype */
9970 	func_id = insn->imm;
9971 	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
9972 		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
9973 			func_id);
9974 		return -EINVAL;
9975 	}
9976 
9977 	if (env->ops->get_func_proto)
9978 		fn = env->ops->get_func_proto(func_id, env->prog);
9979 	if (!fn) {
9980 		verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
9981 			func_id);
9982 		return -EINVAL;
9983 	}
9984 
9985 	/* eBPF programs must be GPL compatible to use GPL-ed functions */
9986 	if (!env->prog->gpl_compatible && fn->gpl_only) {
9987 		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
9988 		return -EINVAL;
9989 	}
9990 
9991 	if (fn->allowed && !fn->allowed(env->prog)) {
9992 		verbose(env, "helper call is not allowed in probe\n");
9993 		return -EINVAL;
9994 	}
9995 
9996 	if (!env->prog->aux->sleepable && fn->might_sleep) {
9997 		verbose(env, "helper call might sleep in a non-sleepable prog\n");
9998 		return -EINVAL;
9999 	}
10000 
10001 	/* With LD_ABS/IND some JITs save/restore skb from r1. */
10002 	changes_data = bpf_helper_changes_pkt_data(fn->func);
10003 	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
10004 		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
10005 			func_id_name(func_id), func_id);
10006 		return -EINVAL;
10007 	}
10008 
10009 	memset(&meta, 0, sizeof(meta));
10010 	meta.pkt_access = fn->pkt_access;
10011 
10012 	err = check_func_proto(fn, func_id);
10013 	if (err) {
10014 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",
10015 			func_id_name(func_id), func_id);
10016 		return err;
10017 	}
10018 
10019 	if (env->cur_state->active_rcu_lock) {
10020 		if (fn->might_sleep) {
10021 			verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n",
10022 				func_id_name(func_id), func_id);
10023 			return -EINVAL;
10024 		}
10025 
10026 		if (env->prog->aux->sleepable && is_storage_get_function(func_id))
10027 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;
10028 	}
10029 
10030 	meta.func_id = func_id;
10031 	/* check args */
10032 	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
10033 		err = check_func_arg(env, i, &meta, fn, insn_idx);
10034 		if (err)
10035 			return err;
10036 	}
10037 
10038 	err = record_func_map(env, &meta, func_id, insn_idx);
10039 	if (err)
10040 		return err;
10041 
10042 	err = record_func_key(env, &meta, func_id, insn_idx);
10043 	if (err)
10044 		return err;
10045 
10046 	/* Mark slots with STACK_MISC in case of raw mode, stack offset
10047 	 * is inferred from register state.
10048 	 */
10049 	for (i = 0; i < meta.access_size; i++) {
10050 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
10051 				       BPF_WRITE, -1, false, false);
10052 		if (err)
10053 			return err;
10054 	}
10055 
10056 	regs = cur_regs(env);
10057 
10058 	if (meta.release_regno) {
10059 		err = -EINVAL;
10060 		/* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
10061 		 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr
10062 		 * is safe to do directly.
10063 		 */
10064 		if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
10065 			if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) {
10066 				verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n");
10067 				return -EFAULT;
10068 			}
10069 			err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
10070 		} else if (meta.ref_obj_id) {
10071 			err = release_reference(env, meta.ref_obj_id);
10072 		} else if (register_is_null(&regs[meta.release_regno])) {
10073 			/* meta.ref_obj_id can only be 0 if register that is meant to be
10074 			 * released is NULL, which must be > R0.
10075 			 */
10076 			err = 0;
10077 		}
10078 		if (err) {
10079 			verbose(env, "func %s#%d reference has not been acquired before\n",
10080 				func_id_name(func_id), func_id);
10081 			return err;
10082 		}
10083 	}
10084 
10085 	switch (func_id) {
10086 	case BPF_FUNC_tail_call:
10087 		err = check_reference_leak(env);
10088 		if (err) {
10089 			verbose(env, "tail_call would lead to reference leak\n");
10090 			return err;
10091 		}
10092 		break;
10093 	case BPF_FUNC_get_local_storage:
10094 		/* check that flags argument in get_local_storage(map, flags) is 0,
10095 		 * this is required because get_local_storage() can't return an error.
10096 		 */
10097 		if (!register_is_null(&regs[BPF_REG_2])) {
10098 			verbose(env, "get_local_storage() doesn't support non-zero flags\n");
10099 			return -EINVAL;
10100 		}
10101 		break;
10102 	case BPF_FUNC_for_each_map_elem:
10103 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10104 					 set_map_elem_callback_state);
10105 		break;
10106 	case BPF_FUNC_timer_set_callback:
10107 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10108 					 set_timer_callback_state);
10109 		break;
10110 	case BPF_FUNC_find_vma:
10111 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10112 					 set_find_vma_callback_state);
10113 		break;
10114 	case BPF_FUNC_snprintf:
10115 		err = check_bpf_snprintf_call(env, regs);
10116 		break;
10117 	case BPF_FUNC_loop:
10118 		update_loop_inline_state(env, meta.subprogno);
10119 		/* Verifier relies on R1 value to determine if bpf_loop() iteration
10120 		 * is finished, thus mark it precise.
10121 		 */
10122 		err = mark_chain_precision(env, BPF_REG_1);
10123 		if (err)
10124 			return err;
10125 		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
10126 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10127 						 set_loop_callback_state);
10128 		} else {
10129 			cur_func(env)->callback_depth = 0;
10130 			if (env->log.level & BPF_LOG_LEVEL2)
10131 				verbose(env, "frame%d bpf_loop iteration limit reached\n",
10132 					env->cur_state->curframe);
10133 		}
10134 		break;
10135 	case BPF_FUNC_dynptr_from_mem:
10136 		if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
10137 			verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
10138 				reg_type_str(env, regs[BPF_REG_1].type));
10139 			return -EACCES;
10140 		}
10141 		break;
10142 	case BPF_FUNC_set_retval:
10143 		if (prog_type == BPF_PROG_TYPE_LSM &&
10144 		    env->prog->expected_attach_type == BPF_LSM_CGROUP) {
10145 			if (!env->prog->aux->attach_func_proto->type) {
10146 				/* Make sure programs that attach to void
10147 				 * hooks don't try to modify return value.
10148 				 */
10149 				verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
10150 				return -EINVAL;
10151 			}
10152 		}
10153 		break;
10154 	case BPF_FUNC_dynptr_data:
10155 	{
10156 		struct bpf_reg_state *reg;
10157 		int id, ref_obj_id;
10158 
10159 		reg = get_dynptr_arg_reg(env, fn, regs);
10160 		if (!reg)
10161 			return -EFAULT;
10162 
10163 
10164 		if (meta.dynptr_id) {
10165 			verbose(env, "verifier internal error: meta.dynptr_id already set\n");
10166 			return -EFAULT;
10167 		}
10168 		if (meta.ref_obj_id) {
10169 			verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
10170 			return -EFAULT;
10171 		}
10172 
10173 		id = dynptr_id(env, reg);
10174 		if (id < 0) {
10175 			verbose(env, "verifier internal error: failed to obtain dynptr id\n");
10176 			return id;
10177 		}
10178 
10179 		ref_obj_id = dynptr_ref_obj_id(env, reg);
10180 		if (ref_obj_id < 0) {
10181 			verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n");
10182 			return ref_obj_id;
10183 		}
10184 
10185 		meta.dynptr_id = id;
10186 		meta.ref_obj_id = ref_obj_id;
10187 
10188 		break;
10189 	}
10190 	case BPF_FUNC_dynptr_write:
10191 	{
10192 		enum bpf_dynptr_type dynptr_type;
10193 		struct bpf_reg_state *reg;
10194 
10195 		reg = get_dynptr_arg_reg(env, fn, regs);
10196 		if (!reg)
10197 			return -EFAULT;
10198 
10199 		dynptr_type = dynptr_get_type(env, reg);
10200 		if (dynptr_type == BPF_DYNPTR_TYPE_INVALID)
10201 			return -EFAULT;
10202 
10203 		if (dynptr_type == BPF_DYNPTR_TYPE_SKB)
10204 			/* this will trigger clear_all_pkt_pointers(), which will
10205 			 * invalidate all dynptr slices associated with the skb
10206 			 */
10207 			changes_data = true;
10208 
10209 		break;
10210 	}
10211 	case BPF_FUNC_user_ringbuf_drain:
10212 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
10213 					 set_user_ringbuf_callback_state);
10214 		break;
10215 	}
10216 
10217 	if (err)
10218 		return err;
10219 
10220 	/* reset caller saved regs */
10221 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
10222 		mark_reg_not_init(env, regs, caller_saved[i]);
10223 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
10224 	}
10225 
10226 	/* helper call returns 64-bit value. */
10227 	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
10228 
10229 	/* update return register (already marked as written above) */
10230 	ret_type = fn->ret_type;
10231 	ret_flag = type_flag(ret_type);
10232 
10233 	switch (base_type(ret_type)) {
10234 	case RET_INTEGER:
10235 		/* sets type to SCALAR_VALUE */
10236 		mark_reg_unknown(env, regs, BPF_REG_0);
10237 		break;
10238 	case RET_VOID:
10239 		regs[BPF_REG_0].type = NOT_INIT;
10240 		break;
10241 	case RET_PTR_TO_MAP_VALUE:
10242 		/* There is no offset yet applied, variable or fixed */
10243 		mark_reg_known_zero(env, regs, BPF_REG_0);
10244 		/* remember map_ptr, so that check_map_access()
10245 		 * can check 'value_size' boundary of memory access
10246 		 * to map element returned from bpf_map_lookup_elem()
10247 		 */
10248 		if (meta.map_ptr == NULL) {
10249 			verbose(env,
10250 				"kernel subsystem misconfigured verifier\n");
10251 			return -EINVAL;
10252 		}
10253 		regs[BPF_REG_0].map_ptr = meta.map_ptr;
10254 		regs[BPF_REG_0].map_uid = meta.map_uid;
10255 		regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
10256 		if (!type_may_be_null(ret_type) &&
10257 		    btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) {
10258 			regs[BPF_REG_0].id = ++env->id_gen;
10259 		}
10260 		break;
10261 	case RET_PTR_TO_SOCKET:
10262 		mark_reg_known_zero(env, regs, BPF_REG_0);
10263 		regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
10264 		break;
10265 	case RET_PTR_TO_SOCK_COMMON:
10266 		mark_reg_known_zero(env, regs, BPF_REG_0);
10267 		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
10268 		break;
10269 	case RET_PTR_TO_TCP_SOCK:
10270 		mark_reg_known_zero(env, regs, BPF_REG_0);
10271 		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
10272 		break;
10273 	case RET_PTR_TO_MEM:
10274 		mark_reg_known_zero(env, regs, BPF_REG_0);
10275 		regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
10276 		regs[BPF_REG_0].mem_size = meta.mem_size;
10277 		break;
10278 	case RET_PTR_TO_MEM_OR_BTF_ID:
10279 	{
10280 		const struct btf_type *t;
10281 
10282 		mark_reg_known_zero(env, regs, BPF_REG_0);
10283 		t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
10284 		if (!btf_type_is_struct(t)) {
10285 			u32 tsize;
10286 			const struct btf_type *ret;
10287 			const char *tname;
10288 
10289 			/* resolve the type size of ksym. */
10290 			ret = btf_resolve_size(meta.ret_btf, t, &tsize);
10291 			if (IS_ERR(ret)) {
10292 				tname = btf_name_by_offset(meta.ret_btf, t->name_off);
10293 				verbose(env, "unable to resolve the size of type '%s': %ld\n",
10294 					tname, PTR_ERR(ret));
10295 				return -EINVAL;
10296 			}
10297 			regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
10298 			regs[BPF_REG_0].mem_size = tsize;
10299 		} else {
10300 			/* MEM_RDONLY may be carried from ret_flag, but it
10301 			 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
10302 			 * it will confuse the check of PTR_TO_BTF_ID in
10303 			 * check_mem_access().
10304 			 */
10305 			ret_flag &= ~MEM_RDONLY;
10306 
10307 			regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
10308 			regs[BPF_REG_0].btf = meta.ret_btf;
10309 			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
10310 		}
10311 		break;
10312 	}
10313 	case RET_PTR_TO_BTF_ID:
10314 	{
10315 		struct btf *ret_btf;
10316 		int ret_btf_id;
10317 
10318 		mark_reg_known_zero(env, regs, BPF_REG_0);
10319 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
10320 		if (func_id == BPF_FUNC_kptr_xchg) {
10321 			ret_btf = meta.kptr_field->kptr.btf;
10322 			ret_btf_id = meta.kptr_field->kptr.btf_id;
10323 			if (!btf_is_kernel(ret_btf))
10324 				regs[BPF_REG_0].type |= MEM_ALLOC;
10325 		} else {
10326 			if (fn->ret_btf_id == BPF_PTR_POISON) {
10327 				verbose(env, "verifier internal error:");
10328 				verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",
10329 					func_id_name(func_id));
10330 				return -EINVAL;
10331 			}
10332 			ret_btf = btf_vmlinux;
10333 			ret_btf_id = *fn->ret_btf_id;
10334 		}
10335 		if (ret_btf_id == 0) {
10336 			verbose(env, "invalid return type %u of func %s#%d\n",
10337 				base_type(ret_type), func_id_name(func_id),
10338 				func_id);
10339 			return -EINVAL;
10340 		}
10341 		regs[BPF_REG_0].btf = ret_btf;
10342 		regs[BPF_REG_0].btf_id = ret_btf_id;
10343 		break;
10344 	}
10345 	default:
10346 		verbose(env, "unknown return type %u of func %s#%d\n",
10347 			base_type(ret_type), func_id_name(func_id), func_id);
10348 		return -EINVAL;
10349 	}
10350 
10351 	if (type_may_be_null(regs[BPF_REG_0].type))
10352 		regs[BPF_REG_0].id = ++env->id_gen;
10353 
10354 	if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
10355 		verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",
10356 			func_id_name(func_id), func_id);
10357 		return -EFAULT;
10358 	}
10359 
10360 	if (is_dynptr_ref_function(func_id))
10361 		regs[BPF_REG_0].dynptr_id = meta.dynptr_id;
10362 
10363 	if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
10364 		/* For release_reference() */
10365 		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
10366 	} else if (is_acquire_function(func_id, meta.map_ptr)) {
10367 		int id = acquire_reference_state(env, insn_idx);
10368 
10369 		if (id < 0)
10370 			return id;
10371 		/* For mark_ptr_or_null_reg() */
10372 		regs[BPF_REG_0].id = id;
10373 		/* For release_reference() */
10374 		regs[BPF_REG_0].ref_obj_id = id;
10375 	}
10376 
10377 	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
10378 
10379 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
10380 	if (err)
10381 		return err;
10382 
10383 	if ((func_id == BPF_FUNC_get_stack ||
10384 	     func_id == BPF_FUNC_get_task_stack) &&
10385 	    !env->prog->has_callchain_buf) {
10386 		const char *err_str;
10387 
10388 #ifdef CONFIG_PERF_EVENTS
10389 		err = get_callchain_buffers(sysctl_perf_event_max_stack);
10390 		err_str = "cannot get callchain buffer for func %s#%d\n";
10391 #else
10392 		err = -ENOTSUPP;
10393 		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
10394 #endif
10395 		if (err) {
10396 			verbose(env, err_str, func_id_name(func_id), func_id);
10397 			return err;
10398 		}
10399 
10400 		env->prog->has_callchain_buf = true;
10401 	}
10402 
10403 	if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
10404 		env->prog->call_get_stack = true;
10405 
10406 	if (func_id == BPF_FUNC_get_func_ip) {
10407 		if (check_get_func_ip(env))
10408 			return -ENOTSUPP;
10409 		env->prog->call_get_func_ip = true;
10410 	}
10411 
10412 	if (changes_data)
10413 		clear_all_pkt_pointers(env);
10414 	return 0;
10415 }
10416 
10417 /* mark_btf_func_reg_size() is used when the reg size is determined by
10418  * the BTF func_proto's return value size and argument.
10419  */
10420 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
10421 				   size_t reg_size)
10422 {
10423 	struct bpf_reg_state *reg = &cur_regs(env)[regno];
10424 
10425 	if (regno == BPF_REG_0) {
10426 		/* Function return value */
10427 		reg->live |= REG_LIVE_WRITTEN;
10428 		reg->subreg_def = reg_size == sizeof(u64) ?
10429 			DEF_NOT_SUBREG : env->insn_idx + 1;
10430 	} else {
10431 		/* Function argument */
10432 		if (reg_size == sizeof(u64)) {
10433 			mark_insn_zext(env, reg);
10434 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
10435 		} else {
10436 			mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
10437 		}
10438 	}
10439 }
10440 
10441 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta)
10442 {
10443 	return meta->kfunc_flags & KF_ACQUIRE;
10444 }
10445 
10446 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta)
10447 {
10448 	return meta->kfunc_flags & KF_RELEASE;
10449 }
10450 
10451 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta)
10452 {
10453 	return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta);
10454 }
10455 
10456 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
10457 {
10458 	return meta->kfunc_flags & KF_SLEEPABLE;
10459 }
10460 
10461 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta)
10462 {
10463 	return meta->kfunc_flags & KF_DESTRUCTIVE;
10464 }
10465 
10466 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta)
10467 {
10468 	return meta->kfunc_flags & KF_RCU;
10469 }
10470 
10471 static bool __kfunc_param_match_suffix(const struct btf *btf,
10472 				       const struct btf_param *arg,
10473 				       const char *suffix)
10474 {
10475 	int suffix_len = strlen(suffix), len;
10476 	const char *param_name;
10477 
10478 	/* In the future, this can be ported to use BTF tagging */
10479 	param_name = btf_name_by_offset(btf, arg->name_off);
10480 	if (str_is_empty(param_name))
10481 		return false;
10482 	len = strlen(param_name);
10483 	if (len < suffix_len)
10484 		return false;
10485 	param_name += len - suffix_len;
10486 	return !strncmp(param_name, suffix, suffix_len);
10487 }
10488 
10489 static bool is_kfunc_arg_mem_size(const struct btf *btf,
10490 				  const struct btf_param *arg,
10491 				  const struct bpf_reg_state *reg)
10492 {
10493 	const struct btf_type *t;
10494 
10495 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10496 	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
10497 		return false;
10498 
10499 	return __kfunc_param_match_suffix(btf, arg, "__sz");
10500 }
10501 
10502 static bool is_kfunc_arg_const_mem_size(const struct btf *btf,
10503 					const struct btf_param *arg,
10504 					const struct bpf_reg_state *reg)
10505 {
10506 	const struct btf_type *t;
10507 
10508 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10509 	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
10510 		return false;
10511 
10512 	return __kfunc_param_match_suffix(btf, arg, "__szk");
10513 }
10514 
10515 static bool is_kfunc_arg_optional(const struct btf *btf, const struct btf_param *arg)
10516 {
10517 	return __kfunc_param_match_suffix(btf, arg, "__opt");
10518 }
10519 
10520 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg)
10521 {
10522 	return __kfunc_param_match_suffix(btf, arg, "__k");
10523 }
10524 
10525 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg)
10526 {
10527 	return __kfunc_param_match_suffix(btf, arg, "__ign");
10528 }
10529 
10530 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg)
10531 {
10532 	return __kfunc_param_match_suffix(btf, arg, "__alloc");
10533 }
10534 
10535 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg)
10536 {
10537 	return __kfunc_param_match_suffix(btf, arg, "__uninit");
10538 }
10539 
10540 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg)
10541 {
10542 	return __kfunc_param_match_suffix(btf, arg, "__refcounted_kptr");
10543 }
10544 
10545 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
10546 					  const struct btf_param *arg,
10547 					  const char *name)
10548 {
10549 	int len, target_len = strlen(name);
10550 	const char *param_name;
10551 
10552 	param_name = btf_name_by_offset(btf, arg->name_off);
10553 	if (str_is_empty(param_name))
10554 		return false;
10555 	len = strlen(param_name);
10556 	if (len != target_len)
10557 		return false;
10558 	if (strcmp(param_name, name))
10559 		return false;
10560 
10561 	return true;
10562 }
10563 
10564 enum {
10565 	KF_ARG_DYNPTR_ID,
10566 	KF_ARG_LIST_HEAD_ID,
10567 	KF_ARG_LIST_NODE_ID,
10568 	KF_ARG_RB_ROOT_ID,
10569 	KF_ARG_RB_NODE_ID,
10570 };
10571 
10572 BTF_ID_LIST(kf_arg_btf_ids)
10573 BTF_ID(struct, bpf_dynptr_kern)
10574 BTF_ID(struct, bpf_list_head)
10575 BTF_ID(struct, bpf_list_node)
10576 BTF_ID(struct, bpf_rb_root)
10577 BTF_ID(struct, bpf_rb_node)
10578 
10579 static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
10580 				    const struct btf_param *arg, int type)
10581 {
10582 	const struct btf_type *t;
10583 	u32 res_id;
10584 
10585 	t = btf_type_skip_modifiers(btf, arg->type, NULL);
10586 	if (!t)
10587 		return false;
10588 	if (!btf_type_is_ptr(t))
10589 		return false;
10590 	t = btf_type_skip_modifiers(btf, t->type, &res_id);
10591 	if (!t)
10592 		return false;
10593 	return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]);
10594 }
10595 
10596 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg)
10597 {
10598 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID);
10599 }
10600 
10601 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg)
10602 {
10603 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID);
10604 }
10605 
10606 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg)
10607 {
10608 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID);
10609 }
10610 
10611 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg)
10612 {
10613 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID);
10614 }
10615 
10616 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg)
10617 {
10618 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID);
10619 }
10620 
10621 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf,
10622 				  const struct btf_param *arg)
10623 {
10624 	const struct btf_type *t;
10625 
10626 	t = btf_type_resolve_func_ptr(btf, arg->type, NULL);
10627 	if (!t)
10628 		return false;
10629 
10630 	return true;
10631 }
10632 
10633 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
10634 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
10635 					const struct btf *btf,
10636 					const struct btf_type *t, int rec)
10637 {
10638 	const struct btf_type *member_type;
10639 	const struct btf_member *member;
10640 	u32 i;
10641 
10642 	if (!btf_type_is_struct(t))
10643 		return false;
10644 
10645 	for_each_member(i, t, member) {
10646 		const struct btf_array *array;
10647 
10648 		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
10649 		if (btf_type_is_struct(member_type)) {
10650 			if (rec >= 3) {
10651 				verbose(env, "max struct nesting depth exceeded\n");
10652 				return false;
10653 			}
10654 			if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
10655 				return false;
10656 			continue;
10657 		}
10658 		if (btf_type_is_array(member_type)) {
10659 			array = btf_array(member_type);
10660 			if (!array->nelems)
10661 				return false;
10662 			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
10663 			if (!btf_type_is_scalar(member_type))
10664 				return false;
10665 			continue;
10666 		}
10667 		if (!btf_type_is_scalar(member_type))
10668 			return false;
10669 	}
10670 	return true;
10671 }
10672 
10673 enum kfunc_ptr_arg_type {
10674 	KF_ARG_PTR_TO_CTX,
10675 	KF_ARG_PTR_TO_ALLOC_BTF_ID,    /* Allocated object */
10676 	KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
10677 	KF_ARG_PTR_TO_DYNPTR,
10678 	KF_ARG_PTR_TO_ITER,
10679 	KF_ARG_PTR_TO_LIST_HEAD,
10680 	KF_ARG_PTR_TO_LIST_NODE,
10681 	KF_ARG_PTR_TO_BTF_ID,	       /* Also covers reg2btf_ids conversions */
10682 	KF_ARG_PTR_TO_MEM,
10683 	KF_ARG_PTR_TO_MEM_SIZE,	       /* Size derived from next argument, skip it */
10684 	KF_ARG_PTR_TO_CALLBACK,
10685 	KF_ARG_PTR_TO_RB_ROOT,
10686 	KF_ARG_PTR_TO_RB_NODE,
10687 };
10688 
10689 enum special_kfunc_type {
10690 	KF_bpf_obj_new_impl,
10691 	KF_bpf_obj_drop_impl,
10692 	KF_bpf_refcount_acquire_impl,
10693 	KF_bpf_list_push_front_impl,
10694 	KF_bpf_list_push_back_impl,
10695 	KF_bpf_list_pop_front,
10696 	KF_bpf_list_pop_back,
10697 	KF_bpf_cast_to_kern_ctx,
10698 	KF_bpf_rdonly_cast,
10699 	KF_bpf_rcu_read_lock,
10700 	KF_bpf_rcu_read_unlock,
10701 	KF_bpf_rbtree_remove,
10702 	KF_bpf_rbtree_add_impl,
10703 	KF_bpf_rbtree_first,
10704 	KF_bpf_dynptr_from_skb,
10705 	KF_bpf_dynptr_from_xdp,
10706 	KF_bpf_dynptr_slice,
10707 	KF_bpf_dynptr_slice_rdwr,
10708 	KF_bpf_dynptr_clone,
10709 };
10710 
10711 BTF_SET_START(special_kfunc_set)
10712 BTF_ID(func, bpf_obj_new_impl)
10713 BTF_ID(func, bpf_obj_drop_impl)
10714 BTF_ID(func, bpf_refcount_acquire_impl)
10715 BTF_ID(func, bpf_list_push_front_impl)
10716 BTF_ID(func, bpf_list_push_back_impl)
10717 BTF_ID(func, bpf_list_pop_front)
10718 BTF_ID(func, bpf_list_pop_back)
10719 BTF_ID(func, bpf_cast_to_kern_ctx)
10720 BTF_ID(func, bpf_rdonly_cast)
10721 BTF_ID(func, bpf_rbtree_remove)
10722 BTF_ID(func, bpf_rbtree_add_impl)
10723 BTF_ID(func, bpf_rbtree_first)
10724 BTF_ID(func, bpf_dynptr_from_skb)
10725 BTF_ID(func, bpf_dynptr_from_xdp)
10726 BTF_ID(func, bpf_dynptr_slice)
10727 BTF_ID(func, bpf_dynptr_slice_rdwr)
10728 BTF_ID(func, bpf_dynptr_clone)
10729 BTF_SET_END(special_kfunc_set)
10730 
10731 BTF_ID_LIST(special_kfunc_list)
10732 BTF_ID(func, bpf_obj_new_impl)
10733 BTF_ID(func, bpf_obj_drop_impl)
10734 BTF_ID(func, bpf_refcount_acquire_impl)
10735 BTF_ID(func, bpf_list_push_front_impl)
10736 BTF_ID(func, bpf_list_push_back_impl)
10737 BTF_ID(func, bpf_list_pop_front)
10738 BTF_ID(func, bpf_list_pop_back)
10739 BTF_ID(func, bpf_cast_to_kern_ctx)
10740 BTF_ID(func, bpf_rdonly_cast)
10741 BTF_ID(func, bpf_rcu_read_lock)
10742 BTF_ID(func, bpf_rcu_read_unlock)
10743 BTF_ID(func, bpf_rbtree_remove)
10744 BTF_ID(func, bpf_rbtree_add_impl)
10745 BTF_ID(func, bpf_rbtree_first)
10746 BTF_ID(func, bpf_dynptr_from_skb)
10747 BTF_ID(func, bpf_dynptr_from_xdp)
10748 BTF_ID(func, bpf_dynptr_slice)
10749 BTF_ID(func, bpf_dynptr_slice_rdwr)
10750 BTF_ID(func, bpf_dynptr_clone)
10751 
10752 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
10753 {
10754 	if (meta->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
10755 	    meta->arg_owning_ref) {
10756 		return false;
10757 	}
10758 
10759 	return meta->kfunc_flags & KF_RET_NULL;
10760 }
10761 
10762 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
10763 {
10764 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
10765 }
10766 
10767 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta)
10768 {
10769 	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
10770 }
10771 
10772 static enum kfunc_ptr_arg_type
10773 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
10774 		       struct bpf_kfunc_call_arg_meta *meta,
10775 		       const struct btf_type *t, const struct btf_type *ref_t,
10776 		       const char *ref_tname, const struct btf_param *args,
10777 		       int argno, int nargs)
10778 {
10779 	u32 regno = argno + 1;
10780 	struct bpf_reg_state *regs = cur_regs(env);
10781 	struct bpf_reg_state *reg = &regs[regno];
10782 	bool arg_mem_size = false;
10783 
10784 	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx])
10785 		return KF_ARG_PTR_TO_CTX;
10786 
10787 	/* In this function, we verify the kfunc's BTF as per the argument type,
10788 	 * leaving the rest of the verification with respect to the register
10789 	 * type to our caller. When a set of conditions hold in the BTF type of
10790 	 * arguments, we resolve it to a known kfunc_ptr_arg_type.
10791 	 */
10792 	if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))
10793 		return KF_ARG_PTR_TO_CTX;
10794 
10795 	if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno]))
10796 		return KF_ARG_PTR_TO_ALLOC_BTF_ID;
10797 
10798 	if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno]))
10799 		return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
10800 
10801 	if (is_kfunc_arg_dynptr(meta->btf, &args[argno]))
10802 		return KF_ARG_PTR_TO_DYNPTR;
10803 
10804 	if (is_kfunc_arg_iter(meta, argno))
10805 		return KF_ARG_PTR_TO_ITER;
10806 
10807 	if (is_kfunc_arg_list_head(meta->btf, &args[argno]))
10808 		return KF_ARG_PTR_TO_LIST_HEAD;
10809 
10810 	if (is_kfunc_arg_list_node(meta->btf, &args[argno]))
10811 		return KF_ARG_PTR_TO_LIST_NODE;
10812 
10813 	if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno]))
10814 		return KF_ARG_PTR_TO_RB_ROOT;
10815 
10816 	if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
10817 		return KF_ARG_PTR_TO_RB_NODE;
10818 
10819 	if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
10820 		if (!btf_type_is_struct(ref_t)) {
10821 			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
10822 				meta->func_name, argno, btf_type_str(ref_t), ref_tname);
10823 			return -EINVAL;
10824 		}
10825 		return KF_ARG_PTR_TO_BTF_ID;
10826 	}
10827 
10828 	if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))
10829 		return KF_ARG_PTR_TO_CALLBACK;
10830 
10831 
10832 	if (argno + 1 < nargs &&
10833 	    (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1]) ||
10834 	     is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1])))
10835 		arg_mem_size = true;
10836 
10837 	/* This is the catch all argument type of register types supported by
10838 	 * check_helper_mem_access. However, we only allow when argument type is
10839 	 * pointer to scalar, or struct composed (recursively) of scalars. When
10840 	 * arg_mem_size is true, the pointer can be void *.
10841 	 */
10842 	if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
10843 	    (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
10844 		verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
10845 			argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
10846 		return -EINVAL;
10847 	}
10848 	return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
10849 }
10850 
10851 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
10852 					struct bpf_reg_state *reg,
10853 					const struct btf_type *ref_t,
10854 					const char *ref_tname, u32 ref_id,
10855 					struct bpf_kfunc_call_arg_meta *meta,
10856 					int argno)
10857 {
10858 	const struct btf_type *reg_ref_t;
10859 	bool strict_type_match = false;
10860 	const struct btf *reg_btf;
10861 	const char *reg_ref_tname;
10862 	u32 reg_ref_id;
10863 
10864 	if (base_type(reg->type) == PTR_TO_BTF_ID) {
10865 		reg_btf = reg->btf;
10866 		reg_ref_id = reg->btf_id;
10867 	} else {
10868 		reg_btf = btf_vmlinux;
10869 		reg_ref_id = *reg2btf_ids[base_type(reg->type)];
10870 	}
10871 
10872 	/* Enforce strict type matching for calls to kfuncs that are acquiring
10873 	 * or releasing a reference, or are no-cast aliases. We do _not_
10874 	 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default,
10875 	 * as we want to enable BPF programs to pass types that are bitwise
10876 	 * equivalent without forcing them to explicitly cast with something
10877 	 * like bpf_cast_to_kern_ctx().
10878 	 *
10879 	 * For example, say we had a type like the following:
10880 	 *
10881 	 * struct bpf_cpumask {
10882 	 *	cpumask_t cpumask;
10883 	 *	refcount_t usage;
10884 	 * };
10885 	 *
10886 	 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed
10887 	 * to a struct cpumask, so it would be safe to pass a struct
10888 	 * bpf_cpumask * to a kfunc expecting a struct cpumask *.
10889 	 *
10890 	 * The philosophy here is similar to how we allow scalars of different
10891 	 * types to be passed to kfuncs as long as the size is the same. The
10892 	 * only difference here is that we're simply allowing
10893 	 * btf_struct_ids_match() to walk the struct at the 0th offset, and
10894 	 * resolve types.
10895 	 */
10896 	if (is_kfunc_acquire(meta) ||
10897 	    (is_kfunc_release(meta) && reg->ref_obj_id) ||
10898 	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
10899 		strict_type_match = true;
10900 
10901 	WARN_ON_ONCE(is_kfunc_trusted_args(meta) && reg->off);
10902 
10903 	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
10904 	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
10905 	if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match)) {
10906 		verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
10907 			meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1,
10908 			btf_type_str(reg_ref_t), reg_ref_tname);
10909 		return -EINVAL;
10910 	}
10911 	return 0;
10912 }
10913 
10914 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
10915 {
10916 	struct bpf_verifier_state *state = env->cur_state;
10917 	struct btf_record *rec = reg_btf_record(reg);
10918 
10919 	if (!state->active_lock.ptr) {
10920 		verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
10921 		return -EFAULT;
10922 	}
10923 
10924 	if (type_flag(reg->type) & NON_OWN_REF) {
10925 		verbose(env, "verifier internal error: NON_OWN_REF already set\n");
10926 		return -EFAULT;
10927 	}
10928 
10929 	reg->type |= NON_OWN_REF;
10930 	if (rec->refcount_off >= 0)
10931 		reg->type |= MEM_RCU;
10932 
10933 	return 0;
10934 }
10935 
10936 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
10937 {
10938 	struct bpf_func_state *state, *unused;
10939 	struct bpf_reg_state *reg;
10940 	int i;
10941 
10942 	state = cur_func(env);
10943 
10944 	if (!ref_obj_id) {
10945 		verbose(env, "verifier internal error: ref_obj_id is zero for "
10946 			     "owning -> non-owning conversion\n");
10947 		return -EFAULT;
10948 	}
10949 
10950 	for (i = 0; i < state->acquired_refs; i++) {
10951 		if (state->refs[i].id != ref_obj_id)
10952 			continue;
10953 
10954 		/* Clear ref_obj_id here so release_reference doesn't clobber
10955 		 * the whole reg
10956 		 */
10957 		bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
10958 			if (reg->ref_obj_id == ref_obj_id) {
10959 				reg->ref_obj_id = 0;
10960 				ref_set_non_owning(env, reg);
10961 			}
10962 		}));
10963 		return 0;
10964 	}
10965 
10966 	verbose(env, "verifier internal error: ref state missing for ref_obj_id\n");
10967 	return -EFAULT;
10968 }
10969 
10970 /* Implementation details:
10971  *
10972  * Each register points to some region of memory, which we define as an
10973  * allocation. Each allocation may embed a bpf_spin_lock which protects any
10974  * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same
10975  * allocation. The lock and the data it protects are colocated in the same
10976  * memory region.
10977  *
10978  * Hence, everytime a register holds a pointer value pointing to such
10979  * allocation, the verifier preserves a unique reg->id for it.
10980  *
10981  * The verifier remembers the lock 'ptr' and the lock 'id' whenever
10982  * bpf_spin_lock is called.
10983  *
10984  * To enable this, lock state in the verifier captures two values:
10985  *	active_lock.ptr = Register's type specific pointer
10986  *	active_lock.id  = A unique ID for each register pointer value
10987  *
10988  * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two
10989  * supported register types.
10990  *
10991  * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of
10992  * allocated objects is the reg->btf pointer.
10993  *
10994  * The active_lock.id is non-unique for maps supporting direct_value_addr, as we
10995  * can establish the provenance of the map value statically for each distinct
10996  * lookup into such maps. They always contain a single map value hence unique
10997  * IDs for each pseudo load pessimizes the algorithm and rejects valid programs.
10998  *
10999  * So, in case of global variables, they use array maps with max_entries = 1,
11000  * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point
11001  * into the same map value as max_entries is 1, as described above).
11002  *
11003  * In case of inner map lookups, the inner map pointer has same map_ptr as the
11004  * outer map pointer (in verifier context), but each lookup into an inner map
11005  * assigns a fresh reg->id to the lookup, so while lookups into distinct inner
11006  * maps from the same outer map share the same map_ptr as active_lock.ptr, they
11007  * will get different reg->id assigned to each lookup, hence different
11008  * active_lock.id.
11009  *
11010  * In case of allocated objects, active_lock.ptr is the reg->btf, and the
11011  * reg->id is a unique ID preserved after the NULL pointer check on the pointer
11012  * returned from bpf_obj_new. Each allocation receives a new reg->id.
11013  */
11014 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
11015 {
11016 	void *ptr;
11017 	u32 id;
11018 
11019 	switch ((int)reg->type) {
11020 	case PTR_TO_MAP_VALUE:
11021 		ptr = reg->map_ptr;
11022 		break;
11023 	case PTR_TO_BTF_ID | MEM_ALLOC:
11024 		ptr = reg->btf;
11025 		break;
11026 	default:
11027 		verbose(env, "verifier internal error: unknown reg type for lock check\n");
11028 		return -EFAULT;
11029 	}
11030 	id = reg->id;
11031 
11032 	if (!env->cur_state->active_lock.ptr)
11033 		return -EINVAL;
11034 	if (env->cur_state->active_lock.ptr != ptr ||
11035 	    env->cur_state->active_lock.id != id) {
11036 		verbose(env, "held lock and object are not in the same allocation\n");
11037 		return -EINVAL;
11038 	}
11039 	return 0;
11040 }
11041 
11042 static bool is_bpf_list_api_kfunc(u32 btf_id)
11043 {
11044 	return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
11045 	       btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
11046 	       btf_id == special_kfunc_list[KF_bpf_list_pop_front] ||
11047 	       btf_id == special_kfunc_list[KF_bpf_list_pop_back];
11048 }
11049 
11050 static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
11051 {
11052 	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] ||
11053 	       btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
11054 	       btf_id == special_kfunc_list[KF_bpf_rbtree_first];
11055 }
11056 
11057 static bool is_bpf_graph_api_kfunc(u32 btf_id)
11058 {
11059 	return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) ||
11060 	       btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
11061 }
11062 
11063 static bool is_sync_callback_calling_kfunc(u32 btf_id)
11064 {
11065 	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
11066 }
11067 
11068 static bool is_rbtree_lock_required_kfunc(u32 btf_id)
11069 {
11070 	return is_bpf_rbtree_api_kfunc(btf_id);
11071 }
11072 
11073 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env,
11074 					  enum btf_field_type head_field_type,
11075 					  u32 kfunc_btf_id)
11076 {
11077 	bool ret;
11078 
11079 	switch (head_field_type) {
11080 	case BPF_LIST_HEAD:
11081 		ret = is_bpf_list_api_kfunc(kfunc_btf_id);
11082 		break;
11083 	case BPF_RB_ROOT:
11084 		ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id);
11085 		break;
11086 	default:
11087 		verbose(env, "verifier internal error: unexpected graph root argument type %s\n",
11088 			btf_field_type_name(head_field_type));
11089 		return false;
11090 	}
11091 
11092 	if (!ret)
11093 		verbose(env, "verifier internal error: %s head arg for unknown kfunc\n",
11094 			btf_field_type_name(head_field_type));
11095 	return ret;
11096 }
11097 
11098 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
11099 					  enum btf_field_type node_field_type,
11100 					  u32 kfunc_btf_id)
11101 {
11102 	bool ret;
11103 
11104 	switch (node_field_type) {
11105 	case BPF_LIST_NODE:
11106 		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
11107 		       kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]);
11108 		break;
11109 	case BPF_RB_NODE:
11110 		ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
11111 		       kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]);
11112 		break;
11113 	default:
11114 		verbose(env, "verifier internal error: unexpected graph node argument type %s\n",
11115 			btf_field_type_name(node_field_type));
11116 		return false;
11117 	}
11118 
11119 	if (!ret)
11120 		verbose(env, "verifier internal error: %s node arg for unknown kfunc\n",
11121 			btf_field_type_name(node_field_type));
11122 	return ret;
11123 }
11124 
11125 static int
11126 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
11127 				   struct bpf_reg_state *reg, u32 regno,
11128 				   struct bpf_kfunc_call_arg_meta *meta,
11129 				   enum btf_field_type head_field_type,
11130 				   struct btf_field **head_field)
11131 {
11132 	const char *head_type_name;
11133 	struct btf_field *field;
11134 	struct btf_record *rec;
11135 	u32 head_off;
11136 
11137 	if (meta->btf != btf_vmlinux) {
11138 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
11139 		return -EFAULT;
11140 	}
11141 
11142 	if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id))
11143 		return -EFAULT;
11144 
11145 	head_type_name = btf_field_type_name(head_field_type);
11146 	if (!tnum_is_const(reg->var_off)) {
11147 		verbose(env,
11148 			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
11149 			regno, head_type_name);
11150 		return -EINVAL;
11151 	}
11152 
11153 	rec = reg_btf_record(reg);
11154 	head_off = reg->off + reg->var_off.value;
11155 	field = btf_record_find(rec, head_off, head_field_type);
11156 	if (!field) {
11157 		verbose(env, "%s not found at offset=%u\n", head_type_name, head_off);
11158 		return -EINVAL;
11159 	}
11160 
11161 	/* All functions require bpf_list_head to be protected using a bpf_spin_lock */
11162 	if (check_reg_allocation_locked(env, reg)) {
11163 		verbose(env, "bpf_spin_lock at off=%d must be held for %s\n",
11164 			rec->spin_lock_off, head_type_name);
11165 		return -EINVAL;
11166 	}
11167 
11168 	if (*head_field) {
11169 		verbose(env, "verifier internal error: repeating %s arg\n", head_type_name);
11170 		return -EFAULT;
11171 	}
11172 	*head_field = field;
11173 	return 0;
11174 }
11175 
11176 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
11177 					   struct bpf_reg_state *reg, u32 regno,
11178 					   struct bpf_kfunc_call_arg_meta *meta)
11179 {
11180 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,
11181 							  &meta->arg_list_head.field);
11182 }
11183 
11184 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
11185 					     struct bpf_reg_state *reg, u32 regno,
11186 					     struct bpf_kfunc_call_arg_meta *meta)
11187 {
11188 	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,
11189 							  &meta->arg_rbtree_root.field);
11190 }
11191 
11192 static int
11193 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
11194 				   struct bpf_reg_state *reg, u32 regno,
11195 				   struct bpf_kfunc_call_arg_meta *meta,
11196 				   enum btf_field_type head_field_type,
11197 				   enum btf_field_type node_field_type,
11198 				   struct btf_field **node_field)
11199 {
11200 	const char *node_type_name;
11201 	const struct btf_type *et, *t;
11202 	struct btf_field *field;
11203 	u32 node_off;
11204 
11205 	if (meta->btf != btf_vmlinux) {
11206 		verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n");
11207 		return -EFAULT;
11208 	}
11209 
11210 	if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id))
11211 		return -EFAULT;
11212 
11213 	node_type_name = btf_field_type_name(node_field_type);
11214 	if (!tnum_is_const(reg->var_off)) {
11215 		verbose(env,
11216 			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
11217 			regno, node_type_name);
11218 		return -EINVAL;
11219 	}
11220 
11221 	node_off = reg->off + reg->var_off.value;
11222 	field = reg_find_field_offset(reg, node_off, node_field_type);
11223 	if (!field || field->offset != node_off) {
11224 		verbose(env, "%s not found at offset=%u\n", node_type_name, node_off);
11225 		return -EINVAL;
11226 	}
11227 
11228 	field = *node_field;
11229 
11230 	et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
11231 	t = btf_type_by_id(reg->btf, reg->btf_id);
11232 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
11233 				  field->graph_root.value_btf_id, true)) {
11234 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
11235 			"in struct %s, but arg is at offset=%d in struct %s\n",
11236 			btf_field_type_name(head_field_type),
11237 			btf_field_type_name(node_field_type),
11238 			field->graph_root.node_offset,
11239 			btf_name_by_offset(field->graph_root.btf, et->name_off),
11240 			node_off, btf_name_by_offset(reg->btf, t->name_off));
11241 		return -EINVAL;
11242 	}
11243 	meta->arg_btf = reg->btf;
11244 	meta->arg_btf_id = reg->btf_id;
11245 
11246 	if (node_off != field->graph_root.node_offset) {
11247 		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
11248 			node_off, btf_field_type_name(node_field_type),
11249 			field->graph_root.node_offset,
11250 			btf_name_by_offset(field->graph_root.btf, et->name_off));
11251 		return -EINVAL;
11252 	}
11253 
11254 	return 0;
11255 }
11256 
11257 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
11258 					   struct bpf_reg_state *reg, u32 regno,
11259 					   struct bpf_kfunc_call_arg_meta *meta)
11260 {
11261 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
11262 						  BPF_LIST_HEAD, BPF_LIST_NODE,
11263 						  &meta->arg_list_head.field);
11264 }
11265 
11266 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
11267 					     struct bpf_reg_state *reg, u32 regno,
11268 					     struct bpf_kfunc_call_arg_meta *meta)
11269 {
11270 	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
11271 						  BPF_RB_ROOT, BPF_RB_NODE,
11272 						  &meta->arg_rbtree_root.field);
11273 }
11274 
11275 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
11276 			    int insn_idx)
11277 {
11278 	const char *func_name = meta->func_name, *ref_tname;
11279 	const struct btf *btf = meta->btf;
11280 	const struct btf_param *args;
11281 	struct btf_record *rec;
11282 	u32 i, nargs;
11283 	int ret;
11284 
11285 	args = (const struct btf_param *)(meta->func_proto + 1);
11286 	nargs = btf_type_vlen(meta->func_proto);
11287 	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
11288 		verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
11289 			MAX_BPF_FUNC_REG_ARGS);
11290 		return -EINVAL;
11291 	}
11292 
11293 	/* Check that BTF function arguments match actual types that the
11294 	 * verifier sees.
11295 	 */
11296 	for (i = 0; i < nargs; i++) {
11297 		struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
11298 		const struct btf_type *t, *ref_t, *resolve_ret;
11299 		enum bpf_arg_type arg_type = ARG_DONTCARE;
11300 		u32 regno = i + 1, ref_id, type_size;
11301 		bool is_ret_buf_sz = false;
11302 		int kf_arg_type;
11303 
11304 		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
11305 
11306 		if (is_kfunc_arg_ignore(btf, &args[i]))
11307 			continue;
11308 
11309 		if (btf_type_is_scalar(t)) {
11310 			if (reg->type != SCALAR_VALUE) {
11311 				verbose(env, "R%d is not a scalar\n", regno);
11312 				return -EINVAL;
11313 			}
11314 
11315 			if (is_kfunc_arg_constant(meta->btf, &args[i])) {
11316 				if (meta->arg_constant.found) {
11317 					verbose(env, "verifier internal error: only one constant argument permitted\n");
11318 					return -EFAULT;
11319 				}
11320 				if (!tnum_is_const(reg->var_off)) {
11321 					verbose(env, "R%d must be a known constant\n", regno);
11322 					return -EINVAL;
11323 				}
11324 				ret = mark_chain_precision(env, regno);
11325 				if (ret < 0)
11326 					return ret;
11327 				meta->arg_constant.found = true;
11328 				meta->arg_constant.value = reg->var_off.value;
11329 			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) {
11330 				meta->r0_rdonly = true;
11331 				is_ret_buf_sz = true;
11332 			} else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) {
11333 				is_ret_buf_sz = true;
11334 			}
11335 
11336 			if (is_ret_buf_sz) {
11337 				if (meta->r0_size) {
11338 					verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
11339 					return -EINVAL;
11340 				}
11341 
11342 				if (!tnum_is_const(reg->var_off)) {
11343 					verbose(env, "R%d is not a const\n", regno);
11344 					return -EINVAL;
11345 				}
11346 
11347 				meta->r0_size = reg->var_off.value;
11348 				ret = mark_chain_precision(env, regno);
11349 				if (ret)
11350 					return ret;
11351 			}
11352 			continue;
11353 		}
11354 
11355 		if (!btf_type_is_ptr(t)) {
11356 			verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));
11357 			return -EINVAL;
11358 		}
11359 
11360 		if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
11361 		    (register_is_null(reg) || type_may_be_null(reg->type))) {
11362 			verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
11363 			return -EACCES;
11364 		}
11365 
11366 		if (reg->ref_obj_id) {
11367 			if (is_kfunc_release(meta) && meta->ref_obj_id) {
11368 				verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
11369 					regno, reg->ref_obj_id,
11370 					meta->ref_obj_id);
11371 				return -EFAULT;
11372 			}
11373 			meta->ref_obj_id = reg->ref_obj_id;
11374 			if (is_kfunc_release(meta))
11375 				meta->release_regno = regno;
11376 		}
11377 
11378 		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
11379 		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
11380 
11381 		kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
11382 		if (kf_arg_type < 0)
11383 			return kf_arg_type;
11384 
11385 		switch (kf_arg_type) {
11386 		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
11387 		case KF_ARG_PTR_TO_BTF_ID:
11388 			if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
11389 				break;
11390 
11391 			if (!is_trusted_reg(reg)) {
11392 				if (!is_kfunc_rcu(meta)) {
11393 					verbose(env, "R%d must be referenced or trusted\n", regno);
11394 					return -EINVAL;
11395 				}
11396 				if (!is_rcu_reg(reg)) {
11397 					verbose(env, "R%d must be a rcu pointer\n", regno);
11398 					return -EINVAL;
11399 				}
11400 			}
11401 
11402 			fallthrough;
11403 		case KF_ARG_PTR_TO_CTX:
11404 			/* Trusted arguments have the same offset checks as release arguments */
11405 			arg_type |= OBJ_RELEASE;
11406 			break;
11407 		case KF_ARG_PTR_TO_DYNPTR:
11408 		case KF_ARG_PTR_TO_ITER:
11409 		case KF_ARG_PTR_TO_LIST_HEAD:
11410 		case KF_ARG_PTR_TO_LIST_NODE:
11411 		case KF_ARG_PTR_TO_RB_ROOT:
11412 		case KF_ARG_PTR_TO_RB_NODE:
11413 		case KF_ARG_PTR_TO_MEM:
11414 		case KF_ARG_PTR_TO_MEM_SIZE:
11415 		case KF_ARG_PTR_TO_CALLBACK:
11416 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
11417 			/* Trusted by default */
11418 			break;
11419 		default:
11420 			WARN_ON_ONCE(1);
11421 			return -EFAULT;
11422 		}
11423 
11424 		if (is_kfunc_release(meta) && reg->ref_obj_id)
11425 			arg_type |= OBJ_RELEASE;
11426 		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
11427 		if (ret < 0)
11428 			return ret;
11429 
11430 		switch (kf_arg_type) {
11431 		case KF_ARG_PTR_TO_CTX:
11432 			if (reg->type != PTR_TO_CTX) {
11433 				verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t));
11434 				return -EINVAL;
11435 			}
11436 
11437 			if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
11438 				ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
11439 				if (ret < 0)
11440 					return -EINVAL;
11441 				meta->ret_btf_id  = ret;
11442 			}
11443 			break;
11444 		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
11445 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
11446 				verbose(env, "arg#%d expected pointer to allocated object\n", i);
11447 				return -EINVAL;
11448 			}
11449 			if (!reg->ref_obj_id) {
11450 				verbose(env, "allocated object must be referenced\n");
11451 				return -EINVAL;
11452 			}
11453 			if (meta->btf == btf_vmlinux &&
11454 			    meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) {
11455 				meta->arg_btf = reg->btf;
11456 				meta->arg_btf_id = reg->btf_id;
11457 			}
11458 			break;
11459 		case KF_ARG_PTR_TO_DYNPTR:
11460 		{
11461 			enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
11462 			int clone_ref_obj_id = 0;
11463 
11464 			if (reg->type != PTR_TO_STACK &&
11465 			    reg->type != CONST_PTR_TO_DYNPTR) {
11466 				verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i);
11467 				return -EINVAL;
11468 			}
11469 
11470 			if (reg->type == CONST_PTR_TO_DYNPTR)
11471 				dynptr_arg_type |= MEM_RDONLY;
11472 
11473 			if (is_kfunc_arg_uninit(btf, &args[i]))
11474 				dynptr_arg_type |= MEM_UNINIT;
11475 
11476 			if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
11477 				dynptr_arg_type |= DYNPTR_TYPE_SKB;
11478 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
11479 				dynptr_arg_type |= DYNPTR_TYPE_XDP;
11480 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
11481 				   (dynptr_arg_type & MEM_UNINIT)) {
11482 				enum bpf_dynptr_type parent_type = meta->initialized_dynptr.type;
11483 
11484 				if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
11485 					verbose(env, "verifier internal error: no dynptr type for parent of clone\n");
11486 					return -EFAULT;
11487 				}
11488 
11489 				dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
11490 				clone_ref_obj_id = meta->initialized_dynptr.ref_obj_id;
11491 				if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) {
11492 					verbose(env, "verifier internal error: missing ref obj id for parent of clone\n");
11493 					return -EFAULT;
11494 				}
11495 			}
11496 
11497 			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);
11498 			if (ret < 0)
11499 				return ret;
11500 
11501 			if (!(dynptr_arg_type & MEM_UNINIT)) {
11502 				int id = dynptr_id(env, reg);
11503 
11504 				if (id < 0) {
11505 					verbose(env, "verifier internal error: failed to obtain dynptr id\n");
11506 					return id;
11507 				}
11508 				meta->initialized_dynptr.id = id;
11509 				meta->initialized_dynptr.type = dynptr_get_type(env, reg);
11510 				meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg);
11511 			}
11512 
11513 			break;
11514 		}
11515 		case KF_ARG_PTR_TO_ITER:
11516 			ret = process_iter_arg(env, regno, insn_idx, meta);
11517 			if (ret < 0)
11518 				return ret;
11519 			break;
11520 		case KF_ARG_PTR_TO_LIST_HEAD:
11521 			if (reg->type != PTR_TO_MAP_VALUE &&
11522 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
11523 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
11524 				return -EINVAL;
11525 			}
11526 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
11527 				verbose(env, "allocated object must be referenced\n");
11528 				return -EINVAL;
11529 			}
11530 			ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);
11531 			if (ret < 0)
11532 				return ret;
11533 			break;
11534 		case KF_ARG_PTR_TO_RB_ROOT:
11535 			if (reg->type != PTR_TO_MAP_VALUE &&
11536 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
11537 				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
11538 				return -EINVAL;
11539 			}
11540 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
11541 				verbose(env, "allocated object must be referenced\n");
11542 				return -EINVAL;
11543 			}
11544 			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);
11545 			if (ret < 0)
11546 				return ret;
11547 			break;
11548 		case KF_ARG_PTR_TO_LIST_NODE:
11549 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
11550 				verbose(env, "arg#%d expected pointer to allocated object\n", i);
11551 				return -EINVAL;
11552 			}
11553 			if (!reg->ref_obj_id) {
11554 				verbose(env, "allocated object must be referenced\n");
11555 				return -EINVAL;
11556 			}
11557 			ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);
11558 			if (ret < 0)
11559 				return ret;
11560 			break;
11561 		case KF_ARG_PTR_TO_RB_NODE:
11562 			if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) {
11563 				if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) {
11564 					verbose(env, "rbtree_remove node input must be non-owning ref\n");
11565 					return -EINVAL;
11566 				}
11567 				if (in_rbtree_lock_required_cb(env)) {
11568 					verbose(env, "rbtree_remove not allowed in rbtree cb\n");
11569 					return -EINVAL;
11570 				}
11571 			} else {
11572 				if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
11573 					verbose(env, "arg#%d expected pointer to allocated object\n", i);
11574 					return -EINVAL;
11575 				}
11576 				if (!reg->ref_obj_id) {
11577 					verbose(env, "allocated object must be referenced\n");
11578 					return -EINVAL;
11579 				}
11580 			}
11581 
11582 			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);
11583 			if (ret < 0)
11584 				return ret;
11585 			break;
11586 		case KF_ARG_PTR_TO_BTF_ID:
11587 			/* Only base_type is checked, further checks are done here */
11588 			if ((base_type(reg->type) != PTR_TO_BTF_ID ||
11589 			     (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
11590 			    !reg2btf_ids[base_type(reg->type)]) {
11591 				verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));
11592 				verbose(env, "expected %s or socket\n",
11593 					reg_type_str(env, base_type(reg->type) |
11594 							  (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
11595 				return -EINVAL;
11596 			}
11597 			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
11598 			if (ret < 0)
11599 				return ret;
11600 			break;
11601 		case KF_ARG_PTR_TO_MEM:
11602 			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
11603 			if (IS_ERR(resolve_ret)) {
11604 				verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
11605 					i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret));
11606 				return -EINVAL;
11607 			}
11608 			ret = check_mem_reg(env, reg, regno, type_size);
11609 			if (ret < 0)
11610 				return ret;
11611 			break;
11612 		case KF_ARG_PTR_TO_MEM_SIZE:
11613 		{
11614 			struct bpf_reg_state *buff_reg = &regs[regno];
11615 			const struct btf_param *buff_arg = &args[i];
11616 			struct bpf_reg_state *size_reg = &regs[regno + 1];
11617 			const struct btf_param *size_arg = &args[i + 1];
11618 
11619 			if (!register_is_null(buff_reg) || !is_kfunc_arg_optional(meta->btf, buff_arg)) {
11620 				ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);
11621 				if (ret < 0) {
11622 					verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);
11623 					return ret;
11624 				}
11625 			}
11626 
11627 			if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) {
11628 				if (meta->arg_constant.found) {
11629 					verbose(env, "verifier internal error: only one constant argument permitted\n");
11630 					return -EFAULT;
11631 				}
11632 				if (!tnum_is_const(size_reg->var_off)) {
11633 					verbose(env, "R%d must be a known constant\n", regno + 1);
11634 					return -EINVAL;
11635 				}
11636 				meta->arg_constant.found = true;
11637 				meta->arg_constant.value = size_reg->var_off.value;
11638 			}
11639 
11640 			/* Skip next '__sz' or '__szk' argument */
11641 			i++;
11642 			break;
11643 		}
11644 		case KF_ARG_PTR_TO_CALLBACK:
11645 			if (reg->type != PTR_TO_FUNC) {
11646 				verbose(env, "arg%d expected pointer to func\n", i);
11647 				return -EINVAL;
11648 			}
11649 			meta->subprogno = reg->subprogno;
11650 			break;
11651 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
11652 			if (!type_is_ptr_alloc_obj(reg->type)) {
11653 				verbose(env, "arg#%d is neither owning or non-owning ref\n", i);
11654 				return -EINVAL;
11655 			}
11656 			if (!type_is_non_owning_ref(reg->type))
11657 				meta->arg_owning_ref = true;
11658 
11659 			rec = reg_btf_record(reg);
11660 			if (!rec) {
11661 				verbose(env, "verifier internal error: Couldn't find btf_record\n");
11662 				return -EFAULT;
11663 			}
11664 
11665 			if (rec->refcount_off < 0) {
11666 				verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);
11667 				return -EINVAL;
11668 			}
11669 
11670 			meta->arg_btf = reg->btf;
11671 			meta->arg_btf_id = reg->btf_id;
11672 			break;
11673 		}
11674 	}
11675 
11676 	if (is_kfunc_release(meta) && !meta->release_regno) {
11677 		verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n",
11678 			func_name);
11679 		return -EINVAL;
11680 	}
11681 
11682 	return 0;
11683 }
11684 
11685 static int fetch_kfunc_meta(struct bpf_verifier_env *env,
11686 			    struct bpf_insn *insn,
11687 			    struct bpf_kfunc_call_arg_meta *meta,
11688 			    const char **kfunc_name)
11689 {
11690 	const struct btf_type *func, *func_proto;
11691 	u32 func_id, *kfunc_flags;
11692 	const char *func_name;
11693 	struct btf *desc_btf;
11694 
11695 	if (kfunc_name)
11696 		*kfunc_name = NULL;
11697 
11698 	if (!insn->imm)
11699 		return -EINVAL;
11700 
11701 	desc_btf = find_kfunc_desc_btf(env, insn->off);
11702 	if (IS_ERR(desc_btf))
11703 		return PTR_ERR(desc_btf);
11704 
11705 	func_id = insn->imm;
11706 	func = btf_type_by_id(desc_btf, func_id);
11707 	func_name = btf_name_by_offset(desc_btf, func->name_off);
11708 	if (kfunc_name)
11709 		*kfunc_name = func_name;
11710 	func_proto = btf_type_by_id(desc_btf, func->type);
11711 
11712 	kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog);
11713 	if (!kfunc_flags) {
11714 		return -EACCES;
11715 	}
11716 
11717 	memset(meta, 0, sizeof(*meta));
11718 	meta->btf = desc_btf;
11719 	meta->func_id = func_id;
11720 	meta->kfunc_flags = *kfunc_flags;
11721 	meta->func_proto = func_proto;
11722 	meta->func_name = func_name;
11723 
11724 	return 0;
11725 }
11726 
11727 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
11728 			    int *insn_idx_p)
11729 {
11730 	const struct btf_type *t, *ptr_type;
11731 	u32 i, nargs, ptr_type_id, release_ref_obj_id;
11732 	struct bpf_reg_state *regs = cur_regs(env);
11733 	const char *func_name, *ptr_type_name;
11734 	bool sleepable, rcu_lock, rcu_unlock;
11735 	struct bpf_kfunc_call_arg_meta meta;
11736 	struct bpf_insn_aux_data *insn_aux;
11737 	int err, insn_idx = *insn_idx_p;
11738 	const struct btf_param *args;
11739 	const struct btf_type *ret_t;
11740 	struct btf *desc_btf;
11741 
11742 	/* skip for now, but return error when we find this in fixup_kfunc_call */
11743 	if (!insn->imm)
11744 		return 0;
11745 
11746 	err = fetch_kfunc_meta(env, insn, &meta, &func_name);
11747 	if (err == -EACCES && func_name)
11748 		verbose(env, "calling kernel function %s is not allowed\n", func_name);
11749 	if (err)
11750 		return err;
11751 	desc_btf = meta.btf;
11752 	insn_aux = &env->insn_aux_data[insn_idx];
11753 
11754 	insn_aux->is_iter_next = is_iter_next_kfunc(&meta);
11755 
11756 	if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) {
11757 		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n");
11758 		return -EACCES;
11759 	}
11760 
11761 	sleepable = is_kfunc_sleepable(&meta);
11762 	if (sleepable && !env->prog->aux->sleepable) {
11763 		verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);
11764 		return -EACCES;
11765 	}
11766 
11767 	/* Check the arguments */
11768 	err = check_kfunc_args(env, &meta, insn_idx);
11769 	if (err < 0)
11770 		return err;
11771 
11772 	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
11773 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
11774 					 set_rbtree_add_callback_state);
11775 		if (err) {
11776 			verbose(env, "kfunc %s#%d failed callback verification\n",
11777 				func_name, meta.func_id);
11778 			return err;
11779 		}
11780 	}
11781 
11782 	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
11783 	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
11784 
11785 	if (env->cur_state->active_rcu_lock) {
11786 		struct bpf_func_state *state;
11787 		struct bpf_reg_state *reg;
11788 
11789 		if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
11790 			verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
11791 			return -EACCES;
11792 		}
11793 
11794 		if (rcu_lock) {
11795 			verbose(env, "nested rcu read lock (kernel function %s)\n", func_name);
11796 			return -EINVAL;
11797 		} else if (rcu_unlock) {
11798 			bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
11799 				if (reg->type & MEM_RCU) {
11800 					reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
11801 					reg->type |= PTR_UNTRUSTED;
11802 				}
11803 			}));
11804 			env->cur_state->active_rcu_lock = false;
11805 		} else if (sleepable) {
11806 			verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name);
11807 			return -EACCES;
11808 		}
11809 	} else if (rcu_lock) {
11810 		env->cur_state->active_rcu_lock = true;
11811 	} else if (rcu_unlock) {
11812 		verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name);
11813 		return -EINVAL;
11814 	}
11815 
11816 	/* In case of release function, we get register number of refcounted
11817 	 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
11818 	 */
11819 	if (meta.release_regno) {
11820 		err = release_reference(env, regs[meta.release_regno].ref_obj_id);
11821 		if (err) {
11822 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",
11823 				func_name, meta.func_id);
11824 			return err;
11825 		}
11826 	}
11827 
11828 	if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
11829 	    meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
11830 	    meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
11831 		release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
11832 		insn_aux->insert_off = regs[BPF_REG_2].off;
11833 		insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
11834 		err = ref_convert_owning_non_owning(env, release_ref_obj_id);
11835 		if (err) {
11836 			verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
11837 				func_name, meta.func_id);
11838 			return err;
11839 		}
11840 
11841 		err = release_reference(env, release_ref_obj_id);
11842 		if (err) {
11843 			verbose(env, "kfunc %s#%d reference has not been acquired before\n",
11844 				func_name, meta.func_id);
11845 			return err;
11846 		}
11847 	}
11848 
11849 	for (i = 0; i < CALLER_SAVED_REGS; i++)
11850 		mark_reg_not_init(env, regs, caller_saved[i]);
11851 
11852 	/* Check return type */
11853 	t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
11854 
11855 	if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) {
11856 		/* Only exception is bpf_obj_new_impl */
11857 		if (meta.btf != btf_vmlinux ||
11858 		    (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] &&
11859 		     meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) {
11860 			verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
11861 			return -EINVAL;
11862 		}
11863 	}
11864 
11865 	if (btf_type_is_scalar(t)) {
11866 		mark_reg_unknown(env, regs, BPF_REG_0);
11867 		mark_btf_func_reg_size(env, BPF_REG_0, t->size);
11868 	} else if (btf_type_is_ptr(t)) {
11869 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
11870 
11871 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
11872 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) {
11873 				struct btf *ret_btf;
11874 				u32 ret_btf_id;
11875 
11876 				if (unlikely(!bpf_global_ma_set))
11877 					return -ENOMEM;
11878 
11879 				if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) {
11880 					verbose(env, "local type ID argument must be in range [0, U32_MAX]\n");
11881 					return -EINVAL;
11882 				}
11883 
11884 				ret_btf = env->prog->aux->btf;
11885 				ret_btf_id = meta.arg_constant.value;
11886 
11887 				/* This may be NULL due to user not supplying a BTF */
11888 				if (!ret_btf) {
11889 					verbose(env, "bpf_obj_new requires prog BTF\n");
11890 					return -EINVAL;
11891 				}
11892 
11893 				ret_t = btf_type_by_id(ret_btf, ret_btf_id);
11894 				if (!ret_t || !__btf_type_is_struct(ret_t)) {
11895 					verbose(env, "bpf_obj_new type ID argument must be of a struct\n");
11896 					return -EINVAL;
11897 				}
11898 
11899 				mark_reg_known_zero(env, regs, BPF_REG_0);
11900 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
11901 				regs[BPF_REG_0].btf = ret_btf;
11902 				regs[BPF_REG_0].btf_id = ret_btf_id;
11903 
11904 				insn_aux->obj_new_size = ret_t->size;
11905 				insn_aux->kptr_struct_meta =
11906 					btf_find_struct_meta(ret_btf, ret_btf_id);
11907 			} else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
11908 				mark_reg_known_zero(env, regs, BPF_REG_0);
11909 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC;
11910 				regs[BPF_REG_0].btf = meta.arg_btf;
11911 				regs[BPF_REG_0].btf_id = meta.arg_btf_id;
11912 
11913 				insn_aux->kptr_struct_meta =
11914 					btf_find_struct_meta(meta.arg_btf,
11915 							     meta.arg_btf_id);
11916 			} else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] ||
11917 				   meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) {
11918 				struct btf_field *field = meta.arg_list_head.field;
11919 
11920 				mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
11921 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] ||
11922 				   meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
11923 				struct btf_field *field = meta.arg_rbtree_root.field;
11924 
11925 				mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
11926 			} else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
11927 				mark_reg_known_zero(env, regs, BPF_REG_0);
11928 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
11929 				regs[BPF_REG_0].btf = desc_btf;
11930 				regs[BPF_REG_0].btf_id = meta.ret_btf_id;
11931 			} else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
11932 				ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value);
11933 				if (!ret_t || !btf_type_is_struct(ret_t)) {
11934 					verbose(env,
11935 						"kfunc bpf_rdonly_cast type ID argument must be of a struct\n");
11936 					return -EINVAL;
11937 				}
11938 
11939 				mark_reg_known_zero(env, regs, BPF_REG_0);
11940 				regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
11941 				regs[BPF_REG_0].btf = desc_btf;
11942 				regs[BPF_REG_0].btf_id = meta.arg_constant.value;
11943 			} else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
11944 				   meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
11945 				enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type);
11946 
11947 				mark_reg_known_zero(env, regs, BPF_REG_0);
11948 
11949 				if (!meta.arg_constant.found) {
11950 					verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n");
11951 					return -EFAULT;
11952 				}
11953 
11954 				regs[BPF_REG_0].mem_size = meta.arg_constant.value;
11955 
11956 				/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
11957 				regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
11958 
11959 				if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
11960 					regs[BPF_REG_0].type |= MEM_RDONLY;
11961 				} else {
11962 					/* this will set env->seen_direct_write to true */
11963 					if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) {
11964 						verbose(env, "the prog does not allow writes to packet data\n");
11965 						return -EINVAL;
11966 					}
11967 				}
11968 
11969 				if (!meta.initialized_dynptr.id) {
11970 					verbose(env, "verifier internal error: no dynptr id\n");
11971 					return -EFAULT;
11972 				}
11973 				regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id;
11974 
11975 				/* we don't need to set BPF_REG_0's ref obj id
11976 				 * because packet slices are not refcounted (see
11977 				 * dynptr_type_refcounted)
11978 				 */
11979 			} else {
11980 				verbose(env, "kernel function %s unhandled dynamic return type\n",
11981 					meta.func_name);
11982 				return -EFAULT;
11983 			}
11984 		} else if (!__btf_type_is_struct(ptr_type)) {
11985 			if (!meta.r0_size) {
11986 				__u32 sz;
11987 
11988 				if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
11989 					meta.r0_size = sz;
11990 					meta.r0_rdonly = true;
11991 				}
11992 			}
11993 			if (!meta.r0_size) {
11994 				ptr_type_name = btf_name_by_offset(desc_btf,
11995 								   ptr_type->name_off);
11996 				verbose(env,
11997 					"kernel function %s returns pointer type %s %s is not supported\n",
11998 					func_name,
11999 					btf_type_str(ptr_type),
12000 					ptr_type_name);
12001 				return -EINVAL;
12002 			}
12003 
12004 			mark_reg_known_zero(env, regs, BPF_REG_0);
12005 			regs[BPF_REG_0].type = PTR_TO_MEM;
12006 			regs[BPF_REG_0].mem_size = meta.r0_size;
12007 
12008 			if (meta.r0_rdonly)
12009 				regs[BPF_REG_0].type |= MEM_RDONLY;
12010 
12011 			/* Ensures we don't access the memory after a release_reference() */
12012 			if (meta.ref_obj_id)
12013 				regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
12014 		} else {
12015 			mark_reg_known_zero(env, regs, BPF_REG_0);
12016 			regs[BPF_REG_0].btf = desc_btf;
12017 			regs[BPF_REG_0].type = PTR_TO_BTF_ID;
12018 			regs[BPF_REG_0].btf_id = ptr_type_id;
12019 		}
12020 
12021 		if (is_kfunc_ret_null(&meta)) {
12022 			regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
12023 			/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
12024 			regs[BPF_REG_0].id = ++env->id_gen;
12025 		}
12026 		mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
12027 		if (is_kfunc_acquire(&meta)) {
12028 			int id = acquire_reference_state(env, insn_idx);
12029 
12030 			if (id < 0)
12031 				return id;
12032 			if (is_kfunc_ret_null(&meta))
12033 				regs[BPF_REG_0].id = id;
12034 			regs[BPF_REG_0].ref_obj_id = id;
12035 		} else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) {
12036 			ref_set_non_owning(env, &regs[BPF_REG_0]);
12037 		}
12038 
12039 		if (reg_may_point_to_spin_lock(&regs[BPF_REG_0]) && !regs[BPF_REG_0].id)
12040 			regs[BPF_REG_0].id = ++env->id_gen;
12041 	} else if (btf_type_is_void(t)) {
12042 		if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) {
12043 			if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) {
12044 				insn_aux->kptr_struct_meta =
12045 					btf_find_struct_meta(meta.arg_btf,
12046 							     meta.arg_btf_id);
12047 			}
12048 		}
12049 	}
12050 
12051 	nargs = btf_type_vlen(meta.func_proto);
12052 	args = (const struct btf_param *)(meta.func_proto + 1);
12053 	for (i = 0; i < nargs; i++) {
12054 		u32 regno = i + 1;
12055 
12056 		t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
12057 		if (btf_type_is_ptr(t))
12058 			mark_btf_func_reg_size(env, regno, sizeof(void *));
12059 		else
12060 			/* scalar. ensured by btf_check_kfunc_arg_match() */
12061 			mark_btf_func_reg_size(env, regno, t->size);
12062 	}
12063 
12064 	if (is_iter_next_kfunc(&meta)) {
12065 		err = process_iter_next_call(env, insn_idx, &meta);
12066 		if (err)
12067 			return err;
12068 	}
12069 
12070 	return 0;
12071 }
12072 
12073 static bool signed_add_overflows(s64 a, s64 b)
12074 {
12075 	/* Do the add in u64, where overflow is well-defined */
12076 	s64 res = (s64)((u64)a + (u64)b);
12077 
12078 	if (b < 0)
12079 		return res > a;
12080 	return res < a;
12081 }
12082 
12083 static bool signed_add32_overflows(s32 a, s32 b)
12084 {
12085 	/* Do the add in u32, where overflow is well-defined */
12086 	s32 res = (s32)((u32)a + (u32)b);
12087 
12088 	if (b < 0)
12089 		return res > a;
12090 	return res < a;
12091 }
12092 
12093 static bool signed_sub_overflows(s64 a, s64 b)
12094 {
12095 	/* Do the sub in u64, where overflow is well-defined */
12096 	s64 res = (s64)((u64)a - (u64)b);
12097 
12098 	if (b < 0)
12099 		return res < a;
12100 	return res > a;
12101 }
12102 
12103 static bool signed_sub32_overflows(s32 a, s32 b)
12104 {
12105 	/* Do the sub in u32, where overflow is well-defined */
12106 	s32 res = (s32)((u32)a - (u32)b);
12107 
12108 	if (b < 0)
12109 		return res < a;
12110 	return res > a;
12111 }
12112 
12113 static bool check_reg_sane_offset(struct bpf_verifier_env *env,
12114 				  const struct bpf_reg_state *reg,
12115 				  enum bpf_reg_type type)
12116 {
12117 	bool known = tnum_is_const(reg->var_off);
12118 	s64 val = reg->var_off.value;
12119 	s64 smin = reg->smin_value;
12120 
12121 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
12122 		verbose(env, "math between %s pointer and %lld is not allowed\n",
12123 			reg_type_str(env, type), val);
12124 		return false;
12125 	}
12126 
12127 	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
12128 		verbose(env, "%s pointer offset %d is not allowed\n",
12129 			reg_type_str(env, type), reg->off);
12130 		return false;
12131 	}
12132 
12133 	if (smin == S64_MIN) {
12134 		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
12135 			reg_type_str(env, type));
12136 		return false;
12137 	}
12138 
12139 	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
12140 		verbose(env, "value %lld makes %s pointer be out of bounds\n",
12141 			smin, reg_type_str(env, type));
12142 		return false;
12143 	}
12144 
12145 	return true;
12146 }
12147 
12148 enum {
12149 	REASON_BOUNDS	= -1,
12150 	REASON_TYPE	= -2,
12151 	REASON_PATHS	= -3,
12152 	REASON_LIMIT	= -4,
12153 	REASON_STACK	= -5,
12154 };
12155 
12156 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
12157 			      u32 *alu_limit, bool mask_to_left)
12158 {
12159 	u32 max = 0, ptr_limit = 0;
12160 
12161 	switch (ptr_reg->type) {
12162 	case PTR_TO_STACK:
12163 		/* Offset 0 is out-of-bounds, but acceptable start for the
12164 		 * left direction, see BPF_REG_FP. Also, unknown scalar
12165 		 * offset where we would need to deal with min/max bounds is
12166 		 * currently prohibited for unprivileged.
12167 		 */
12168 		max = MAX_BPF_STACK + mask_to_left;
12169 		ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
12170 		break;
12171 	case PTR_TO_MAP_VALUE:
12172 		max = ptr_reg->map_ptr->value_size;
12173 		ptr_limit = (mask_to_left ?
12174 			     ptr_reg->smin_value :
12175 			     ptr_reg->umax_value) + ptr_reg->off;
12176 		break;
12177 	default:
12178 		return REASON_TYPE;
12179 	}
12180 
12181 	if (ptr_limit >= max)
12182 		return REASON_LIMIT;
12183 	*alu_limit = ptr_limit;
12184 	return 0;
12185 }
12186 
12187 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
12188 				    const struct bpf_insn *insn)
12189 {
12190 	return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
12191 }
12192 
12193 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
12194 				       u32 alu_state, u32 alu_limit)
12195 {
12196 	/* If we arrived here from different branches with different
12197 	 * state or limits to sanitize, then this won't work.
12198 	 */
12199 	if (aux->alu_state &&
12200 	    (aux->alu_state != alu_state ||
12201 	     aux->alu_limit != alu_limit))
12202 		return REASON_PATHS;
12203 
12204 	/* Corresponding fixup done in do_misc_fixups(). */
12205 	aux->alu_state = alu_state;
12206 	aux->alu_limit = alu_limit;
12207 	return 0;
12208 }
12209 
12210 static int sanitize_val_alu(struct bpf_verifier_env *env,
12211 			    struct bpf_insn *insn)
12212 {
12213 	struct bpf_insn_aux_data *aux = cur_aux(env);
12214 
12215 	if (can_skip_alu_sanitation(env, insn))
12216 		return 0;
12217 
12218 	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
12219 }
12220 
12221 static bool sanitize_needed(u8 opcode)
12222 {
12223 	return opcode == BPF_ADD || opcode == BPF_SUB;
12224 }
12225 
12226 struct bpf_sanitize_info {
12227 	struct bpf_insn_aux_data aux;
12228 	bool mask_to_left;
12229 };
12230 
12231 static struct bpf_verifier_state *
12232 sanitize_speculative_path(struct bpf_verifier_env *env,
12233 			  const struct bpf_insn *insn,
12234 			  u32 next_idx, u32 curr_idx)
12235 {
12236 	struct bpf_verifier_state *branch;
12237 	struct bpf_reg_state *regs;
12238 
12239 	branch = push_stack(env, next_idx, curr_idx, true);
12240 	if (branch && insn) {
12241 		regs = branch->frame[branch->curframe]->regs;
12242 		if (BPF_SRC(insn->code) == BPF_K) {
12243 			mark_reg_unknown(env, regs, insn->dst_reg);
12244 		} else if (BPF_SRC(insn->code) == BPF_X) {
12245 			mark_reg_unknown(env, regs, insn->dst_reg);
12246 			mark_reg_unknown(env, regs, insn->src_reg);
12247 		}
12248 	}
12249 	return branch;
12250 }
12251 
12252 static int sanitize_ptr_alu(struct bpf_verifier_env *env,
12253 			    struct bpf_insn *insn,
12254 			    const struct bpf_reg_state *ptr_reg,
12255 			    const struct bpf_reg_state *off_reg,
12256 			    struct bpf_reg_state *dst_reg,
12257 			    struct bpf_sanitize_info *info,
12258 			    const bool commit_window)
12259 {
12260 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
12261 	struct bpf_verifier_state *vstate = env->cur_state;
12262 	bool off_is_imm = tnum_is_const(off_reg->var_off);
12263 	bool off_is_neg = off_reg->smin_value < 0;
12264 	bool ptr_is_dst_reg = ptr_reg == dst_reg;
12265 	u8 opcode = BPF_OP(insn->code);
12266 	u32 alu_state, alu_limit;
12267 	struct bpf_reg_state tmp;
12268 	bool ret;
12269 	int err;
12270 
12271 	if (can_skip_alu_sanitation(env, insn))
12272 		return 0;
12273 
12274 	/* We already marked aux for masking from non-speculative
12275 	 * paths, thus we got here in the first place. We only care
12276 	 * to explore bad access from here.
12277 	 */
12278 	if (vstate->speculative)
12279 		goto do_sim;
12280 
12281 	if (!commit_window) {
12282 		if (!tnum_is_const(off_reg->var_off) &&
12283 		    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
12284 			return REASON_BOUNDS;
12285 
12286 		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
12287 				     (opcode == BPF_SUB && !off_is_neg);
12288 	}
12289 
12290 	err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
12291 	if (err < 0)
12292 		return err;
12293 
12294 	if (commit_window) {
12295 		/* In commit phase we narrow the masking window based on
12296 		 * the observed pointer move after the simulated operation.
12297 		 */
12298 		alu_state = info->aux.alu_state;
12299 		alu_limit = abs(info->aux.alu_limit - alu_limit);
12300 	} else {
12301 		alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
12302 		alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
12303 		alu_state |= ptr_is_dst_reg ?
12304 			     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
12305 
12306 		/* Limit pruning on unknown scalars to enable deep search for
12307 		 * potential masking differences from other program paths.
12308 		 */
12309 		if (!off_is_imm)
12310 			env->explore_alu_limits = true;
12311 	}
12312 
12313 	err = update_alu_sanitation_state(aux, alu_state, alu_limit);
12314 	if (err < 0)
12315 		return err;
12316 do_sim:
12317 	/* If we're in commit phase, we're done here given we already
12318 	 * pushed the truncated dst_reg into the speculative verification
12319 	 * stack.
12320 	 *
12321 	 * Also, when register is a known constant, we rewrite register-based
12322 	 * operation to immediate-based, and thus do not need masking (and as
12323 	 * a consequence, do not need to simulate the zero-truncation either).
12324 	 */
12325 	if (commit_window || off_is_imm)
12326 		return 0;
12327 
12328 	/* Simulate and find potential out-of-bounds access under
12329 	 * speculative execution from truncation as a result of
12330 	 * masking when off was not within expected range. If off
12331 	 * sits in dst, then we temporarily need to move ptr there
12332 	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
12333 	 * for cases where we use K-based arithmetic in one direction
12334 	 * and truncated reg-based in the other in order to explore
12335 	 * bad access.
12336 	 */
12337 	if (!ptr_is_dst_reg) {
12338 		tmp = *dst_reg;
12339 		copy_register_state(dst_reg, ptr_reg);
12340 	}
12341 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
12342 					env->insn_idx);
12343 	if (!ptr_is_dst_reg && ret)
12344 		*dst_reg = tmp;
12345 	return !ret ? REASON_STACK : 0;
12346 }
12347 
12348 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
12349 {
12350 	struct bpf_verifier_state *vstate = env->cur_state;
12351 
12352 	/* If we simulate paths under speculation, we don't update the
12353 	 * insn as 'seen' such that when we verify unreachable paths in
12354 	 * the non-speculative domain, sanitize_dead_code() can still
12355 	 * rewrite/sanitize them.
12356 	 */
12357 	if (!vstate->speculative)
12358 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
12359 }
12360 
12361 static int sanitize_err(struct bpf_verifier_env *env,
12362 			const struct bpf_insn *insn, int reason,
12363 			const struct bpf_reg_state *off_reg,
12364 			const struct bpf_reg_state *dst_reg)
12365 {
12366 	static const char *err = "pointer arithmetic with it prohibited for !root";
12367 	const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
12368 	u32 dst = insn->dst_reg, src = insn->src_reg;
12369 
12370 	switch (reason) {
12371 	case REASON_BOUNDS:
12372 		verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
12373 			off_reg == dst_reg ? dst : src, err);
12374 		break;
12375 	case REASON_TYPE:
12376 		verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
12377 			off_reg == dst_reg ? src : dst, err);
12378 		break;
12379 	case REASON_PATHS:
12380 		verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
12381 			dst, op, err);
12382 		break;
12383 	case REASON_LIMIT:
12384 		verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
12385 			dst, op, err);
12386 		break;
12387 	case REASON_STACK:
12388 		verbose(env, "R%d could not be pushed for speculative verification, %s\n",
12389 			dst, err);
12390 		break;
12391 	default:
12392 		verbose(env, "verifier internal error: unknown reason (%d)\n",
12393 			reason);
12394 		break;
12395 	}
12396 
12397 	return -EACCES;
12398 }
12399 
12400 /* check that stack access falls within stack limits and that 'reg' doesn't
12401  * have a variable offset.
12402  *
12403  * Variable offset is prohibited for unprivileged mode for simplicity since it
12404  * requires corresponding support in Spectre masking for stack ALU.  See also
12405  * retrieve_ptr_limit().
12406  *
12407  *
12408  * 'off' includes 'reg->off'.
12409  */
12410 static int check_stack_access_for_ptr_arithmetic(
12411 				struct bpf_verifier_env *env,
12412 				int regno,
12413 				const struct bpf_reg_state *reg,
12414 				int off)
12415 {
12416 	if (!tnum_is_const(reg->var_off)) {
12417 		char tn_buf[48];
12418 
12419 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
12420 		verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
12421 			regno, tn_buf, off);
12422 		return -EACCES;
12423 	}
12424 
12425 	if (off >= 0 || off < -MAX_BPF_STACK) {
12426 		verbose(env, "R%d stack pointer arithmetic goes out of range, "
12427 			"prohibited for !root; off=%d\n", regno, off);
12428 		return -EACCES;
12429 	}
12430 
12431 	return 0;
12432 }
12433 
12434 static int sanitize_check_bounds(struct bpf_verifier_env *env,
12435 				 const struct bpf_insn *insn,
12436 				 const struct bpf_reg_state *dst_reg)
12437 {
12438 	u32 dst = insn->dst_reg;
12439 
12440 	/* For unprivileged we require that resulting offset must be in bounds
12441 	 * in order to be able to sanitize access later on.
12442 	 */
12443 	if (env->bypass_spec_v1)
12444 		return 0;
12445 
12446 	switch (dst_reg->type) {
12447 	case PTR_TO_STACK:
12448 		if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
12449 					dst_reg->off + dst_reg->var_off.value))
12450 			return -EACCES;
12451 		break;
12452 	case PTR_TO_MAP_VALUE:
12453 		if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
12454 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
12455 				"prohibited for !root\n", dst);
12456 			return -EACCES;
12457 		}
12458 		break;
12459 	default:
12460 		break;
12461 	}
12462 
12463 	return 0;
12464 }
12465 
12466 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
12467  * Caller should also handle BPF_MOV case separately.
12468  * If we return -EACCES, caller may want to try again treating pointer as a
12469  * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
12470  */
12471 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
12472 				   struct bpf_insn *insn,
12473 				   const struct bpf_reg_state *ptr_reg,
12474 				   const struct bpf_reg_state *off_reg)
12475 {
12476 	struct bpf_verifier_state *vstate = env->cur_state;
12477 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
12478 	struct bpf_reg_state *regs = state->regs, *dst_reg;
12479 	bool known = tnum_is_const(off_reg->var_off);
12480 	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
12481 	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
12482 	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
12483 	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
12484 	struct bpf_sanitize_info info = {};
12485 	u8 opcode = BPF_OP(insn->code);
12486 	u32 dst = insn->dst_reg;
12487 	int ret;
12488 
12489 	dst_reg = &regs[dst];
12490 
12491 	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
12492 	    smin_val > smax_val || umin_val > umax_val) {
12493 		/* Taint dst register if offset had invalid bounds derived from
12494 		 * e.g. dead branches.
12495 		 */
12496 		__mark_reg_unknown(env, dst_reg);
12497 		return 0;
12498 	}
12499 
12500 	if (BPF_CLASS(insn->code) != BPF_ALU64) {
12501 		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
12502 		if (opcode == BPF_SUB && env->allow_ptr_leaks) {
12503 			__mark_reg_unknown(env, dst_reg);
12504 			return 0;
12505 		}
12506 
12507 		verbose(env,
12508 			"R%d 32-bit pointer arithmetic prohibited\n",
12509 			dst);
12510 		return -EACCES;
12511 	}
12512 
12513 	if (ptr_reg->type & PTR_MAYBE_NULL) {
12514 		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
12515 			dst, reg_type_str(env, ptr_reg->type));
12516 		return -EACCES;
12517 	}
12518 
12519 	switch (base_type(ptr_reg->type)) {
12520 	case PTR_TO_FLOW_KEYS:
12521 		if (known)
12522 			break;
12523 		fallthrough;
12524 	case CONST_PTR_TO_MAP:
12525 		/* smin_val represents the known value */
12526 		if (known && smin_val == 0 && opcode == BPF_ADD)
12527 			break;
12528 		fallthrough;
12529 	case PTR_TO_PACKET_END:
12530 	case PTR_TO_SOCKET:
12531 	case PTR_TO_SOCK_COMMON:
12532 	case PTR_TO_TCP_SOCK:
12533 	case PTR_TO_XDP_SOCK:
12534 		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
12535 			dst, reg_type_str(env, ptr_reg->type));
12536 		return -EACCES;
12537 	default:
12538 		break;
12539 	}
12540 
12541 	/* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
12542 	 * The id may be overwritten later if we create a new variable offset.
12543 	 */
12544 	dst_reg->type = ptr_reg->type;
12545 	dst_reg->id = ptr_reg->id;
12546 
12547 	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
12548 	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
12549 		return -EINVAL;
12550 
12551 	/* pointer types do not carry 32-bit bounds at the moment. */
12552 	__mark_reg32_unbounded(dst_reg);
12553 
12554 	if (sanitize_needed(opcode)) {
12555 		ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
12556 				       &info, false);
12557 		if (ret < 0)
12558 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
12559 	}
12560 
12561 	switch (opcode) {
12562 	case BPF_ADD:
12563 		/* We can take a fixed offset as long as it doesn't overflow
12564 		 * the s32 'off' field
12565 		 */
12566 		if (known && (ptr_reg->off + smin_val ==
12567 			      (s64)(s32)(ptr_reg->off + smin_val))) {
12568 			/* pointer += K.  Accumulate it into fixed offset */
12569 			dst_reg->smin_value = smin_ptr;
12570 			dst_reg->smax_value = smax_ptr;
12571 			dst_reg->umin_value = umin_ptr;
12572 			dst_reg->umax_value = umax_ptr;
12573 			dst_reg->var_off = ptr_reg->var_off;
12574 			dst_reg->off = ptr_reg->off + smin_val;
12575 			dst_reg->raw = ptr_reg->raw;
12576 			break;
12577 		}
12578 		/* A new variable offset is created.  Note that off_reg->off
12579 		 * == 0, since it's a scalar.
12580 		 * dst_reg gets the pointer type and since some positive
12581 		 * integer value was added to the pointer, give it a new 'id'
12582 		 * if it's a PTR_TO_PACKET.
12583 		 * this creates a new 'base' pointer, off_reg (variable) gets
12584 		 * added into the variable offset, and we copy the fixed offset
12585 		 * from ptr_reg.
12586 		 */
12587 		if (signed_add_overflows(smin_ptr, smin_val) ||
12588 		    signed_add_overflows(smax_ptr, smax_val)) {
12589 			dst_reg->smin_value = S64_MIN;
12590 			dst_reg->smax_value = S64_MAX;
12591 		} else {
12592 			dst_reg->smin_value = smin_ptr + smin_val;
12593 			dst_reg->smax_value = smax_ptr + smax_val;
12594 		}
12595 		if (umin_ptr + umin_val < umin_ptr ||
12596 		    umax_ptr + umax_val < umax_ptr) {
12597 			dst_reg->umin_value = 0;
12598 			dst_reg->umax_value = U64_MAX;
12599 		} else {
12600 			dst_reg->umin_value = umin_ptr + umin_val;
12601 			dst_reg->umax_value = umax_ptr + umax_val;
12602 		}
12603 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
12604 		dst_reg->off = ptr_reg->off;
12605 		dst_reg->raw = ptr_reg->raw;
12606 		if (reg_is_pkt_pointer(ptr_reg)) {
12607 			dst_reg->id = ++env->id_gen;
12608 			/* something was added to pkt_ptr, set range to zero */
12609 			memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
12610 		}
12611 		break;
12612 	case BPF_SUB:
12613 		if (dst_reg == off_reg) {
12614 			/* scalar -= pointer.  Creates an unknown scalar */
12615 			verbose(env, "R%d tried to subtract pointer from scalar\n",
12616 				dst);
12617 			return -EACCES;
12618 		}
12619 		/* We don't allow subtraction from FP, because (according to
12620 		 * test_verifier.c test "invalid fp arithmetic", JITs might not
12621 		 * be able to deal with it.
12622 		 */
12623 		if (ptr_reg->type == PTR_TO_STACK) {
12624 			verbose(env, "R%d subtraction from stack pointer prohibited\n",
12625 				dst);
12626 			return -EACCES;
12627 		}
12628 		if (known && (ptr_reg->off - smin_val ==
12629 			      (s64)(s32)(ptr_reg->off - smin_val))) {
12630 			/* pointer -= K.  Subtract it from fixed offset */
12631 			dst_reg->smin_value = smin_ptr;
12632 			dst_reg->smax_value = smax_ptr;
12633 			dst_reg->umin_value = umin_ptr;
12634 			dst_reg->umax_value = umax_ptr;
12635 			dst_reg->var_off = ptr_reg->var_off;
12636 			dst_reg->id = ptr_reg->id;
12637 			dst_reg->off = ptr_reg->off - smin_val;
12638 			dst_reg->raw = ptr_reg->raw;
12639 			break;
12640 		}
12641 		/* A new variable offset is created.  If the subtrahend is known
12642 		 * nonnegative, then any reg->range we had before is still good.
12643 		 */
12644 		if (signed_sub_overflows(smin_ptr, smax_val) ||
12645 		    signed_sub_overflows(smax_ptr, smin_val)) {
12646 			/* Overflow possible, we know nothing */
12647 			dst_reg->smin_value = S64_MIN;
12648 			dst_reg->smax_value = S64_MAX;
12649 		} else {
12650 			dst_reg->smin_value = smin_ptr - smax_val;
12651 			dst_reg->smax_value = smax_ptr - smin_val;
12652 		}
12653 		if (umin_ptr < umax_val) {
12654 			/* Overflow possible, we know nothing */
12655 			dst_reg->umin_value = 0;
12656 			dst_reg->umax_value = U64_MAX;
12657 		} else {
12658 			/* Cannot overflow (as long as bounds are consistent) */
12659 			dst_reg->umin_value = umin_ptr - umax_val;
12660 			dst_reg->umax_value = umax_ptr - umin_val;
12661 		}
12662 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
12663 		dst_reg->off = ptr_reg->off;
12664 		dst_reg->raw = ptr_reg->raw;
12665 		if (reg_is_pkt_pointer(ptr_reg)) {
12666 			dst_reg->id = ++env->id_gen;
12667 			/* something was added to pkt_ptr, set range to zero */
12668 			if (smin_val < 0)
12669 				memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
12670 		}
12671 		break;
12672 	case BPF_AND:
12673 	case BPF_OR:
12674 	case BPF_XOR:
12675 		/* bitwise ops on pointers are troublesome, prohibit. */
12676 		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
12677 			dst, bpf_alu_string[opcode >> 4]);
12678 		return -EACCES;
12679 	default:
12680 		/* other operators (e.g. MUL,LSH) produce non-pointer results */
12681 		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
12682 			dst, bpf_alu_string[opcode >> 4]);
12683 		return -EACCES;
12684 	}
12685 
12686 	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
12687 		return -EINVAL;
12688 	reg_bounds_sync(dst_reg);
12689 	if (sanitize_check_bounds(env, insn, dst_reg) < 0)
12690 		return -EACCES;
12691 	if (sanitize_needed(opcode)) {
12692 		ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
12693 				       &info, true);
12694 		if (ret < 0)
12695 			return sanitize_err(env, insn, ret, off_reg, dst_reg);
12696 	}
12697 
12698 	return 0;
12699 }
12700 
12701 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
12702 				 struct bpf_reg_state *src_reg)
12703 {
12704 	s32 smin_val = src_reg->s32_min_value;
12705 	s32 smax_val = src_reg->s32_max_value;
12706 	u32 umin_val = src_reg->u32_min_value;
12707 	u32 umax_val = src_reg->u32_max_value;
12708 
12709 	if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) ||
12710 	    signed_add32_overflows(dst_reg->s32_max_value, smax_val)) {
12711 		dst_reg->s32_min_value = S32_MIN;
12712 		dst_reg->s32_max_value = S32_MAX;
12713 	} else {
12714 		dst_reg->s32_min_value += smin_val;
12715 		dst_reg->s32_max_value += smax_val;
12716 	}
12717 	if (dst_reg->u32_min_value + umin_val < umin_val ||
12718 	    dst_reg->u32_max_value + umax_val < umax_val) {
12719 		dst_reg->u32_min_value = 0;
12720 		dst_reg->u32_max_value = U32_MAX;
12721 	} else {
12722 		dst_reg->u32_min_value += umin_val;
12723 		dst_reg->u32_max_value += umax_val;
12724 	}
12725 }
12726 
12727 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
12728 			       struct bpf_reg_state *src_reg)
12729 {
12730 	s64 smin_val = src_reg->smin_value;
12731 	s64 smax_val = src_reg->smax_value;
12732 	u64 umin_val = src_reg->umin_value;
12733 	u64 umax_val = src_reg->umax_value;
12734 
12735 	if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
12736 	    signed_add_overflows(dst_reg->smax_value, smax_val)) {
12737 		dst_reg->smin_value = S64_MIN;
12738 		dst_reg->smax_value = S64_MAX;
12739 	} else {
12740 		dst_reg->smin_value += smin_val;
12741 		dst_reg->smax_value += smax_val;
12742 	}
12743 	if (dst_reg->umin_value + umin_val < umin_val ||
12744 	    dst_reg->umax_value + umax_val < umax_val) {
12745 		dst_reg->umin_value = 0;
12746 		dst_reg->umax_value = U64_MAX;
12747 	} else {
12748 		dst_reg->umin_value += umin_val;
12749 		dst_reg->umax_value += umax_val;
12750 	}
12751 }
12752 
12753 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
12754 				 struct bpf_reg_state *src_reg)
12755 {
12756 	s32 smin_val = src_reg->s32_min_value;
12757 	s32 smax_val = src_reg->s32_max_value;
12758 	u32 umin_val = src_reg->u32_min_value;
12759 	u32 umax_val = src_reg->u32_max_value;
12760 
12761 	if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) ||
12762 	    signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) {
12763 		/* Overflow possible, we know nothing */
12764 		dst_reg->s32_min_value = S32_MIN;
12765 		dst_reg->s32_max_value = S32_MAX;
12766 	} else {
12767 		dst_reg->s32_min_value -= smax_val;
12768 		dst_reg->s32_max_value -= smin_val;
12769 	}
12770 	if (dst_reg->u32_min_value < umax_val) {
12771 		/* Overflow possible, we know nothing */
12772 		dst_reg->u32_min_value = 0;
12773 		dst_reg->u32_max_value = U32_MAX;
12774 	} else {
12775 		/* Cannot overflow (as long as bounds are consistent) */
12776 		dst_reg->u32_min_value -= umax_val;
12777 		dst_reg->u32_max_value -= umin_val;
12778 	}
12779 }
12780 
12781 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
12782 			       struct bpf_reg_state *src_reg)
12783 {
12784 	s64 smin_val = src_reg->smin_value;
12785 	s64 smax_val = src_reg->smax_value;
12786 	u64 umin_val = src_reg->umin_value;
12787 	u64 umax_val = src_reg->umax_value;
12788 
12789 	if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
12790 	    signed_sub_overflows(dst_reg->smax_value, smin_val)) {
12791 		/* Overflow possible, we know nothing */
12792 		dst_reg->smin_value = S64_MIN;
12793 		dst_reg->smax_value = S64_MAX;
12794 	} else {
12795 		dst_reg->smin_value -= smax_val;
12796 		dst_reg->smax_value -= smin_val;
12797 	}
12798 	if (dst_reg->umin_value < umax_val) {
12799 		/* Overflow possible, we know nothing */
12800 		dst_reg->umin_value = 0;
12801 		dst_reg->umax_value = U64_MAX;
12802 	} else {
12803 		/* Cannot overflow (as long as bounds are consistent) */
12804 		dst_reg->umin_value -= umax_val;
12805 		dst_reg->umax_value -= umin_val;
12806 	}
12807 }
12808 
12809 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
12810 				 struct bpf_reg_state *src_reg)
12811 {
12812 	s32 smin_val = src_reg->s32_min_value;
12813 	u32 umin_val = src_reg->u32_min_value;
12814 	u32 umax_val = src_reg->u32_max_value;
12815 
12816 	if (smin_val < 0 || dst_reg->s32_min_value < 0) {
12817 		/* Ain't nobody got time to multiply that sign */
12818 		__mark_reg32_unbounded(dst_reg);
12819 		return;
12820 	}
12821 	/* Both values are positive, so we can work with unsigned and
12822 	 * copy the result to signed (unless it exceeds S32_MAX).
12823 	 */
12824 	if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) {
12825 		/* Potential overflow, we know nothing */
12826 		__mark_reg32_unbounded(dst_reg);
12827 		return;
12828 	}
12829 	dst_reg->u32_min_value *= umin_val;
12830 	dst_reg->u32_max_value *= umax_val;
12831 	if (dst_reg->u32_max_value > S32_MAX) {
12832 		/* Overflow possible, we know nothing */
12833 		dst_reg->s32_min_value = S32_MIN;
12834 		dst_reg->s32_max_value = S32_MAX;
12835 	} else {
12836 		dst_reg->s32_min_value = dst_reg->u32_min_value;
12837 		dst_reg->s32_max_value = dst_reg->u32_max_value;
12838 	}
12839 }
12840 
12841 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
12842 			       struct bpf_reg_state *src_reg)
12843 {
12844 	s64 smin_val = src_reg->smin_value;
12845 	u64 umin_val = src_reg->umin_value;
12846 	u64 umax_val = src_reg->umax_value;
12847 
12848 	if (smin_val < 0 || dst_reg->smin_value < 0) {
12849 		/* Ain't nobody got time to multiply that sign */
12850 		__mark_reg64_unbounded(dst_reg);
12851 		return;
12852 	}
12853 	/* Both values are positive, so we can work with unsigned and
12854 	 * copy the result to signed (unless it exceeds S64_MAX).
12855 	 */
12856 	if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
12857 		/* Potential overflow, we know nothing */
12858 		__mark_reg64_unbounded(dst_reg);
12859 		return;
12860 	}
12861 	dst_reg->umin_value *= umin_val;
12862 	dst_reg->umax_value *= umax_val;
12863 	if (dst_reg->umax_value > S64_MAX) {
12864 		/* Overflow possible, we know nothing */
12865 		dst_reg->smin_value = S64_MIN;
12866 		dst_reg->smax_value = S64_MAX;
12867 	} else {
12868 		dst_reg->smin_value = dst_reg->umin_value;
12869 		dst_reg->smax_value = dst_reg->umax_value;
12870 	}
12871 }
12872 
12873 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
12874 				 struct bpf_reg_state *src_reg)
12875 {
12876 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
12877 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
12878 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
12879 	s32 smin_val = src_reg->s32_min_value;
12880 	u32 umax_val = src_reg->u32_max_value;
12881 
12882 	if (src_known && dst_known) {
12883 		__mark_reg32_known(dst_reg, var32_off.value);
12884 		return;
12885 	}
12886 
12887 	/* We get our minimum from the var_off, since that's inherently
12888 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
12889 	 */
12890 	dst_reg->u32_min_value = var32_off.value;
12891 	dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
12892 	if (dst_reg->s32_min_value < 0 || smin_val < 0) {
12893 		/* Lose signed bounds when ANDing negative numbers,
12894 		 * ain't nobody got time for that.
12895 		 */
12896 		dst_reg->s32_min_value = S32_MIN;
12897 		dst_reg->s32_max_value = S32_MAX;
12898 	} else {
12899 		/* ANDing two positives gives a positive, so safe to
12900 		 * cast result into s64.
12901 		 */
12902 		dst_reg->s32_min_value = dst_reg->u32_min_value;
12903 		dst_reg->s32_max_value = dst_reg->u32_max_value;
12904 	}
12905 }
12906 
12907 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
12908 			       struct bpf_reg_state *src_reg)
12909 {
12910 	bool src_known = tnum_is_const(src_reg->var_off);
12911 	bool dst_known = tnum_is_const(dst_reg->var_off);
12912 	s64 smin_val = src_reg->smin_value;
12913 	u64 umax_val = src_reg->umax_value;
12914 
12915 	if (src_known && dst_known) {
12916 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
12917 		return;
12918 	}
12919 
12920 	/* We get our minimum from the var_off, since that's inherently
12921 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
12922 	 */
12923 	dst_reg->umin_value = dst_reg->var_off.value;
12924 	dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
12925 	if (dst_reg->smin_value < 0 || smin_val < 0) {
12926 		/* Lose signed bounds when ANDing negative numbers,
12927 		 * ain't nobody got time for that.
12928 		 */
12929 		dst_reg->smin_value = S64_MIN;
12930 		dst_reg->smax_value = S64_MAX;
12931 	} else {
12932 		/* ANDing two positives gives a positive, so safe to
12933 		 * cast result into s64.
12934 		 */
12935 		dst_reg->smin_value = dst_reg->umin_value;
12936 		dst_reg->smax_value = dst_reg->umax_value;
12937 	}
12938 	/* We may learn something more from the var_off */
12939 	__update_reg_bounds(dst_reg);
12940 }
12941 
12942 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
12943 				struct bpf_reg_state *src_reg)
12944 {
12945 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
12946 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
12947 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
12948 	s32 smin_val = src_reg->s32_min_value;
12949 	u32 umin_val = src_reg->u32_min_value;
12950 
12951 	if (src_known && dst_known) {
12952 		__mark_reg32_known(dst_reg, var32_off.value);
12953 		return;
12954 	}
12955 
12956 	/* We get our maximum from the var_off, and our minimum is the
12957 	 * maximum of the operands' minima
12958 	 */
12959 	dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
12960 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
12961 	if (dst_reg->s32_min_value < 0 || smin_val < 0) {
12962 		/* Lose signed bounds when ORing negative numbers,
12963 		 * ain't nobody got time for that.
12964 		 */
12965 		dst_reg->s32_min_value = S32_MIN;
12966 		dst_reg->s32_max_value = S32_MAX;
12967 	} else {
12968 		/* ORing two positives gives a positive, so safe to
12969 		 * cast result into s64.
12970 		 */
12971 		dst_reg->s32_min_value = dst_reg->u32_min_value;
12972 		dst_reg->s32_max_value = dst_reg->u32_max_value;
12973 	}
12974 }
12975 
12976 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
12977 			      struct bpf_reg_state *src_reg)
12978 {
12979 	bool src_known = tnum_is_const(src_reg->var_off);
12980 	bool dst_known = tnum_is_const(dst_reg->var_off);
12981 	s64 smin_val = src_reg->smin_value;
12982 	u64 umin_val = src_reg->umin_value;
12983 
12984 	if (src_known && dst_known) {
12985 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
12986 		return;
12987 	}
12988 
12989 	/* We get our maximum from the var_off, and our minimum is the
12990 	 * maximum of the operands' minima
12991 	 */
12992 	dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
12993 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
12994 	if (dst_reg->smin_value < 0 || smin_val < 0) {
12995 		/* Lose signed bounds when ORing negative numbers,
12996 		 * ain't nobody got time for that.
12997 		 */
12998 		dst_reg->smin_value = S64_MIN;
12999 		dst_reg->smax_value = S64_MAX;
13000 	} else {
13001 		/* ORing two positives gives a positive, so safe to
13002 		 * cast result into s64.
13003 		 */
13004 		dst_reg->smin_value = dst_reg->umin_value;
13005 		dst_reg->smax_value = dst_reg->umax_value;
13006 	}
13007 	/* We may learn something more from the var_off */
13008 	__update_reg_bounds(dst_reg);
13009 }
13010 
13011 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
13012 				 struct bpf_reg_state *src_reg)
13013 {
13014 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
13015 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
13016 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
13017 	s32 smin_val = src_reg->s32_min_value;
13018 
13019 	if (src_known && dst_known) {
13020 		__mark_reg32_known(dst_reg, var32_off.value);
13021 		return;
13022 	}
13023 
13024 	/* We get both minimum and maximum from the var32_off. */
13025 	dst_reg->u32_min_value = var32_off.value;
13026 	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
13027 
13028 	if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
13029 		/* XORing two positive sign numbers gives a positive,
13030 		 * so safe to cast u32 result into s32.
13031 		 */
13032 		dst_reg->s32_min_value = dst_reg->u32_min_value;
13033 		dst_reg->s32_max_value = dst_reg->u32_max_value;
13034 	} else {
13035 		dst_reg->s32_min_value = S32_MIN;
13036 		dst_reg->s32_max_value = S32_MAX;
13037 	}
13038 }
13039 
13040 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
13041 			       struct bpf_reg_state *src_reg)
13042 {
13043 	bool src_known = tnum_is_const(src_reg->var_off);
13044 	bool dst_known = tnum_is_const(dst_reg->var_off);
13045 	s64 smin_val = src_reg->smin_value;
13046 
13047 	if (src_known && dst_known) {
13048 		/* dst_reg->var_off.value has been updated earlier */
13049 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
13050 		return;
13051 	}
13052 
13053 	/* We get both minimum and maximum from the var_off. */
13054 	dst_reg->umin_value = dst_reg->var_off.value;
13055 	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
13056 
13057 	if (dst_reg->smin_value >= 0 && smin_val >= 0) {
13058 		/* XORing two positive sign numbers gives a positive,
13059 		 * so safe to cast u64 result into s64.
13060 		 */
13061 		dst_reg->smin_value = dst_reg->umin_value;
13062 		dst_reg->smax_value = dst_reg->umax_value;
13063 	} else {
13064 		dst_reg->smin_value = S64_MIN;
13065 		dst_reg->smax_value = S64_MAX;
13066 	}
13067 
13068 	__update_reg_bounds(dst_reg);
13069 }
13070 
13071 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
13072 				   u64 umin_val, u64 umax_val)
13073 {
13074 	/* We lose all sign bit information (except what we can pick
13075 	 * up from var_off)
13076 	 */
13077 	dst_reg->s32_min_value = S32_MIN;
13078 	dst_reg->s32_max_value = S32_MAX;
13079 	/* If we might shift our top bit out, then we know nothing */
13080 	if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
13081 		dst_reg->u32_min_value = 0;
13082 		dst_reg->u32_max_value = U32_MAX;
13083 	} else {
13084 		dst_reg->u32_min_value <<= umin_val;
13085 		dst_reg->u32_max_value <<= umax_val;
13086 	}
13087 }
13088 
13089 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
13090 				 struct bpf_reg_state *src_reg)
13091 {
13092 	u32 umax_val = src_reg->u32_max_value;
13093 	u32 umin_val = src_reg->u32_min_value;
13094 	/* u32 alu operation will zext upper bits */
13095 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
13096 
13097 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
13098 	dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
13099 	/* Not required but being careful mark reg64 bounds as unknown so
13100 	 * that we are forced to pick them up from tnum and zext later and
13101 	 * if some path skips this step we are still safe.
13102 	 */
13103 	__mark_reg64_unbounded(dst_reg);
13104 	__update_reg32_bounds(dst_reg);
13105 }
13106 
13107 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
13108 				   u64 umin_val, u64 umax_val)
13109 {
13110 	/* Special case <<32 because it is a common compiler pattern to sign
13111 	 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
13112 	 * positive we know this shift will also be positive so we can track
13113 	 * bounds correctly. Otherwise we lose all sign bit information except
13114 	 * what we can pick up from var_off. Perhaps we can generalize this
13115 	 * later to shifts of any length.
13116 	 */
13117 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
13118 		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
13119 	else
13120 		dst_reg->smax_value = S64_MAX;
13121 
13122 	if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
13123 		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
13124 	else
13125 		dst_reg->smin_value = S64_MIN;
13126 
13127 	/* If we might shift our top bit out, then we know nothing */
13128 	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
13129 		dst_reg->umin_value = 0;
13130 		dst_reg->umax_value = U64_MAX;
13131 	} else {
13132 		dst_reg->umin_value <<= umin_val;
13133 		dst_reg->umax_value <<= umax_val;
13134 	}
13135 }
13136 
13137 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
13138 			       struct bpf_reg_state *src_reg)
13139 {
13140 	u64 umax_val = src_reg->umax_value;
13141 	u64 umin_val = src_reg->umin_value;
13142 
13143 	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
13144 	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
13145 	__scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
13146 
13147 	dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
13148 	/* We may learn something more from the var_off */
13149 	__update_reg_bounds(dst_reg);
13150 }
13151 
13152 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
13153 				 struct bpf_reg_state *src_reg)
13154 {
13155 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
13156 	u32 umax_val = src_reg->u32_max_value;
13157 	u32 umin_val = src_reg->u32_min_value;
13158 
13159 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
13160 	 * be negative, then either:
13161 	 * 1) src_reg might be zero, so the sign bit of the result is
13162 	 *    unknown, so we lose our signed bounds
13163 	 * 2) it's known negative, thus the unsigned bounds capture the
13164 	 *    signed bounds
13165 	 * 3) the signed bounds cross zero, so they tell us nothing
13166 	 *    about the result
13167 	 * If the value in dst_reg is known nonnegative, then again the
13168 	 * unsigned bounds capture the signed bounds.
13169 	 * Thus, in all cases it suffices to blow away our signed bounds
13170 	 * and rely on inferring new ones from the unsigned bounds and
13171 	 * var_off of the result.
13172 	 */
13173 	dst_reg->s32_min_value = S32_MIN;
13174 	dst_reg->s32_max_value = S32_MAX;
13175 
13176 	dst_reg->var_off = tnum_rshift(subreg, umin_val);
13177 	dst_reg->u32_min_value >>= umax_val;
13178 	dst_reg->u32_max_value >>= umin_val;
13179 
13180 	__mark_reg64_unbounded(dst_reg);
13181 	__update_reg32_bounds(dst_reg);
13182 }
13183 
13184 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
13185 			       struct bpf_reg_state *src_reg)
13186 {
13187 	u64 umax_val = src_reg->umax_value;
13188 	u64 umin_val = src_reg->umin_value;
13189 
13190 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
13191 	 * be negative, then either:
13192 	 * 1) src_reg might be zero, so the sign bit of the result is
13193 	 *    unknown, so we lose our signed bounds
13194 	 * 2) it's known negative, thus the unsigned bounds capture the
13195 	 *    signed bounds
13196 	 * 3) the signed bounds cross zero, so they tell us nothing
13197 	 *    about the result
13198 	 * If the value in dst_reg is known nonnegative, then again the
13199 	 * unsigned bounds capture the signed bounds.
13200 	 * Thus, in all cases it suffices to blow away our signed bounds
13201 	 * and rely on inferring new ones from the unsigned bounds and
13202 	 * var_off of the result.
13203 	 */
13204 	dst_reg->smin_value = S64_MIN;
13205 	dst_reg->smax_value = S64_MAX;
13206 	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
13207 	dst_reg->umin_value >>= umax_val;
13208 	dst_reg->umax_value >>= umin_val;
13209 
13210 	/* Its not easy to operate on alu32 bounds here because it depends
13211 	 * on bits being shifted in. Take easy way out and mark unbounded
13212 	 * so we can recalculate later from tnum.
13213 	 */
13214 	__mark_reg32_unbounded(dst_reg);
13215 	__update_reg_bounds(dst_reg);
13216 }
13217 
13218 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
13219 				  struct bpf_reg_state *src_reg)
13220 {
13221 	u64 umin_val = src_reg->u32_min_value;
13222 
13223 	/* Upon reaching here, src_known is true and
13224 	 * umax_val is equal to umin_val.
13225 	 */
13226 	dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
13227 	dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
13228 
13229 	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
13230 
13231 	/* blow away the dst_reg umin_value/umax_value and rely on
13232 	 * dst_reg var_off to refine the result.
13233 	 */
13234 	dst_reg->u32_min_value = 0;
13235 	dst_reg->u32_max_value = U32_MAX;
13236 
13237 	__mark_reg64_unbounded(dst_reg);
13238 	__update_reg32_bounds(dst_reg);
13239 }
13240 
13241 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
13242 				struct bpf_reg_state *src_reg)
13243 {
13244 	u64 umin_val = src_reg->umin_value;
13245 
13246 	/* Upon reaching here, src_known is true and umax_val is equal
13247 	 * to umin_val.
13248 	 */
13249 	dst_reg->smin_value >>= umin_val;
13250 	dst_reg->smax_value >>= umin_val;
13251 
13252 	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
13253 
13254 	/* blow away the dst_reg umin_value/umax_value and rely on
13255 	 * dst_reg var_off to refine the result.
13256 	 */
13257 	dst_reg->umin_value = 0;
13258 	dst_reg->umax_value = U64_MAX;
13259 
13260 	/* Its not easy to operate on alu32 bounds here because it depends
13261 	 * on bits being shifted in from upper 32-bits. Take easy way out
13262 	 * and mark unbounded so we can recalculate later from tnum.
13263 	 */
13264 	__mark_reg32_unbounded(dst_reg);
13265 	__update_reg_bounds(dst_reg);
13266 }
13267 
13268 /* WARNING: This function does calculations on 64-bit values, but the actual
13269  * execution may occur on 32-bit values. Therefore, things like bitshifts
13270  * need extra checks in the 32-bit case.
13271  */
13272 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
13273 				      struct bpf_insn *insn,
13274 				      struct bpf_reg_state *dst_reg,
13275 				      struct bpf_reg_state src_reg)
13276 {
13277 	struct bpf_reg_state *regs = cur_regs(env);
13278 	u8 opcode = BPF_OP(insn->code);
13279 	bool src_known;
13280 	s64 smin_val, smax_val;
13281 	u64 umin_val, umax_val;
13282 	s32 s32_min_val, s32_max_val;
13283 	u32 u32_min_val, u32_max_val;
13284 	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
13285 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
13286 	int ret;
13287 
13288 	smin_val = src_reg.smin_value;
13289 	smax_val = src_reg.smax_value;
13290 	umin_val = src_reg.umin_value;
13291 	umax_val = src_reg.umax_value;
13292 
13293 	s32_min_val = src_reg.s32_min_value;
13294 	s32_max_val = src_reg.s32_max_value;
13295 	u32_min_val = src_reg.u32_min_value;
13296 	u32_max_val = src_reg.u32_max_value;
13297 
13298 	if (alu32) {
13299 		src_known = tnum_subreg_is_const(src_reg.var_off);
13300 		if ((src_known &&
13301 		     (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) ||
13302 		    s32_min_val > s32_max_val || u32_min_val > u32_max_val) {
13303 			/* Taint dst register if offset had invalid bounds
13304 			 * derived from e.g. dead branches.
13305 			 */
13306 			__mark_reg_unknown(env, dst_reg);
13307 			return 0;
13308 		}
13309 	} else {
13310 		src_known = tnum_is_const(src_reg.var_off);
13311 		if ((src_known &&
13312 		     (smin_val != smax_val || umin_val != umax_val)) ||
13313 		    smin_val > smax_val || umin_val > umax_val) {
13314 			/* Taint dst register if offset had invalid bounds
13315 			 * derived from e.g. dead branches.
13316 			 */
13317 			__mark_reg_unknown(env, dst_reg);
13318 			return 0;
13319 		}
13320 	}
13321 
13322 	if (!src_known &&
13323 	    opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
13324 		__mark_reg_unknown(env, dst_reg);
13325 		return 0;
13326 	}
13327 
13328 	if (sanitize_needed(opcode)) {
13329 		ret = sanitize_val_alu(env, insn);
13330 		if (ret < 0)
13331 			return sanitize_err(env, insn, ret, NULL, NULL);
13332 	}
13333 
13334 	/* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
13335 	 * There are two classes of instructions: The first class we track both
13336 	 * alu32 and alu64 sign/unsigned bounds independently this provides the
13337 	 * greatest amount of precision when alu operations are mixed with jmp32
13338 	 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
13339 	 * and BPF_OR. This is possible because these ops have fairly easy to
13340 	 * understand and calculate behavior in both 32-bit and 64-bit alu ops.
13341 	 * See alu32 verifier tests for examples. The second class of
13342 	 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
13343 	 * with regards to tracking sign/unsigned bounds because the bits may
13344 	 * cross subreg boundaries in the alu64 case. When this happens we mark
13345 	 * the reg unbounded in the subreg bound space and use the resulting
13346 	 * tnum to calculate an approximation of the sign/unsigned bounds.
13347 	 */
13348 	switch (opcode) {
13349 	case BPF_ADD:
13350 		scalar32_min_max_add(dst_reg, &src_reg);
13351 		scalar_min_max_add(dst_reg, &src_reg);
13352 		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
13353 		break;
13354 	case BPF_SUB:
13355 		scalar32_min_max_sub(dst_reg, &src_reg);
13356 		scalar_min_max_sub(dst_reg, &src_reg);
13357 		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
13358 		break;
13359 	case BPF_MUL:
13360 		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
13361 		scalar32_min_max_mul(dst_reg, &src_reg);
13362 		scalar_min_max_mul(dst_reg, &src_reg);
13363 		break;
13364 	case BPF_AND:
13365 		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
13366 		scalar32_min_max_and(dst_reg, &src_reg);
13367 		scalar_min_max_and(dst_reg, &src_reg);
13368 		break;
13369 	case BPF_OR:
13370 		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
13371 		scalar32_min_max_or(dst_reg, &src_reg);
13372 		scalar_min_max_or(dst_reg, &src_reg);
13373 		break;
13374 	case BPF_XOR:
13375 		dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
13376 		scalar32_min_max_xor(dst_reg, &src_reg);
13377 		scalar_min_max_xor(dst_reg, &src_reg);
13378 		break;
13379 	case BPF_LSH:
13380 		if (umax_val >= insn_bitness) {
13381 			/* Shifts greater than 31 or 63 are undefined.
13382 			 * This includes shifts by a negative number.
13383 			 */
13384 			mark_reg_unknown(env, regs, insn->dst_reg);
13385 			break;
13386 		}
13387 		if (alu32)
13388 			scalar32_min_max_lsh(dst_reg, &src_reg);
13389 		else
13390 			scalar_min_max_lsh(dst_reg, &src_reg);
13391 		break;
13392 	case BPF_RSH:
13393 		if (umax_val >= insn_bitness) {
13394 			/* Shifts greater than 31 or 63 are undefined.
13395 			 * This includes shifts by a negative number.
13396 			 */
13397 			mark_reg_unknown(env, regs, insn->dst_reg);
13398 			break;
13399 		}
13400 		if (alu32)
13401 			scalar32_min_max_rsh(dst_reg, &src_reg);
13402 		else
13403 			scalar_min_max_rsh(dst_reg, &src_reg);
13404 		break;
13405 	case BPF_ARSH:
13406 		if (umax_val >= insn_bitness) {
13407 			/* Shifts greater than 31 or 63 are undefined.
13408 			 * This includes shifts by a negative number.
13409 			 */
13410 			mark_reg_unknown(env, regs, insn->dst_reg);
13411 			break;
13412 		}
13413 		if (alu32)
13414 			scalar32_min_max_arsh(dst_reg, &src_reg);
13415 		else
13416 			scalar_min_max_arsh(dst_reg, &src_reg);
13417 		break;
13418 	default:
13419 		mark_reg_unknown(env, regs, insn->dst_reg);
13420 		break;
13421 	}
13422 
13423 	/* ALU32 ops are zero extended into 64bit register */
13424 	if (alu32)
13425 		zext_32_to_64(dst_reg);
13426 	reg_bounds_sync(dst_reg);
13427 	return 0;
13428 }
13429 
13430 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
13431  * and var_off.
13432  */
13433 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
13434 				   struct bpf_insn *insn)
13435 {
13436 	struct bpf_verifier_state *vstate = env->cur_state;
13437 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
13438 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
13439 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
13440 	u8 opcode = BPF_OP(insn->code);
13441 	int err;
13442 
13443 	dst_reg = &regs[insn->dst_reg];
13444 	src_reg = NULL;
13445 	if (dst_reg->type != SCALAR_VALUE)
13446 		ptr_reg = dst_reg;
13447 	else
13448 		/* Make sure ID is cleared otherwise dst_reg min/max could be
13449 		 * incorrectly propagated into other registers by find_equal_scalars()
13450 		 */
13451 		dst_reg->id = 0;
13452 	if (BPF_SRC(insn->code) == BPF_X) {
13453 		src_reg = &regs[insn->src_reg];
13454 		if (src_reg->type != SCALAR_VALUE) {
13455 			if (dst_reg->type != SCALAR_VALUE) {
13456 				/* Combining two pointers by any ALU op yields
13457 				 * an arbitrary scalar. Disallow all math except
13458 				 * pointer subtraction
13459 				 */
13460 				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
13461 					mark_reg_unknown(env, regs, insn->dst_reg);
13462 					return 0;
13463 				}
13464 				verbose(env, "R%d pointer %s pointer prohibited\n",
13465 					insn->dst_reg,
13466 					bpf_alu_string[opcode >> 4]);
13467 				return -EACCES;
13468 			} else {
13469 				/* scalar += pointer
13470 				 * This is legal, but we have to reverse our
13471 				 * src/dest handling in computing the range
13472 				 */
13473 				err = mark_chain_precision(env, insn->dst_reg);
13474 				if (err)
13475 					return err;
13476 				return adjust_ptr_min_max_vals(env, insn,
13477 							       src_reg, dst_reg);
13478 			}
13479 		} else if (ptr_reg) {
13480 			/* pointer += scalar */
13481 			err = mark_chain_precision(env, insn->src_reg);
13482 			if (err)
13483 				return err;
13484 			return adjust_ptr_min_max_vals(env, insn,
13485 						       dst_reg, src_reg);
13486 		} else if (dst_reg->precise) {
13487 			/* if dst_reg is precise, src_reg should be precise as well */
13488 			err = mark_chain_precision(env, insn->src_reg);
13489 			if (err)
13490 				return err;
13491 		}
13492 	} else {
13493 		/* Pretend the src is a reg with a known value, since we only
13494 		 * need to be able to read from this state.
13495 		 */
13496 		off_reg.type = SCALAR_VALUE;
13497 		__mark_reg_known(&off_reg, insn->imm);
13498 		src_reg = &off_reg;
13499 		if (ptr_reg) /* pointer += K */
13500 			return adjust_ptr_min_max_vals(env, insn,
13501 						       ptr_reg, src_reg);
13502 	}
13503 
13504 	/* Got here implies adding two SCALAR_VALUEs */
13505 	if (WARN_ON_ONCE(ptr_reg)) {
13506 		print_verifier_state(env, state, true);
13507 		verbose(env, "verifier internal error: unexpected ptr_reg\n");
13508 		return -EINVAL;
13509 	}
13510 	if (WARN_ON(!src_reg)) {
13511 		print_verifier_state(env, state, true);
13512 		verbose(env, "verifier internal error: no src_reg\n");
13513 		return -EINVAL;
13514 	}
13515 	return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
13516 }
13517 
13518 /* check validity of 32-bit and 64-bit arithmetic operations */
13519 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
13520 {
13521 	struct bpf_reg_state *regs = cur_regs(env);
13522 	u8 opcode = BPF_OP(insn->code);
13523 	int err;
13524 
13525 	if (opcode == BPF_END || opcode == BPF_NEG) {
13526 		if (opcode == BPF_NEG) {
13527 			if (BPF_SRC(insn->code) != BPF_K ||
13528 			    insn->src_reg != BPF_REG_0 ||
13529 			    insn->off != 0 || insn->imm != 0) {
13530 				verbose(env, "BPF_NEG uses reserved fields\n");
13531 				return -EINVAL;
13532 			}
13533 		} else {
13534 			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
13535 			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
13536 			    (BPF_CLASS(insn->code) == BPF_ALU64 &&
13537 			     BPF_SRC(insn->code) != BPF_TO_LE)) {
13538 				verbose(env, "BPF_END uses reserved fields\n");
13539 				return -EINVAL;
13540 			}
13541 		}
13542 
13543 		/* check src operand */
13544 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
13545 		if (err)
13546 			return err;
13547 
13548 		if (is_pointer_value(env, insn->dst_reg)) {
13549 			verbose(env, "R%d pointer arithmetic prohibited\n",
13550 				insn->dst_reg);
13551 			return -EACCES;
13552 		}
13553 
13554 		/* check dest operand */
13555 		err = check_reg_arg(env, insn->dst_reg, DST_OP);
13556 		if (err)
13557 			return err;
13558 
13559 	} else if (opcode == BPF_MOV) {
13560 
13561 		if (BPF_SRC(insn->code) == BPF_X) {
13562 			if (insn->imm != 0) {
13563 				verbose(env, "BPF_MOV uses reserved fields\n");
13564 				return -EINVAL;
13565 			}
13566 
13567 			if (BPF_CLASS(insn->code) == BPF_ALU) {
13568 				if (insn->off != 0 && insn->off != 8 && insn->off != 16) {
13569 					verbose(env, "BPF_MOV uses reserved fields\n");
13570 					return -EINVAL;
13571 				}
13572 			} else {
13573 				if (insn->off != 0 && insn->off != 8 && insn->off != 16 &&
13574 				    insn->off != 32) {
13575 					verbose(env, "BPF_MOV uses reserved fields\n");
13576 					return -EINVAL;
13577 				}
13578 			}
13579 
13580 			/* check src operand */
13581 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
13582 			if (err)
13583 				return err;
13584 		} else {
13585 			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
13586 				verbose(env, "BPF_MOV uses reserved fields\n");
13587 				return -EINVAL;
13588 			}
13589 		}
13590 
13591 		/* check dest operand, mark as required later */
13592 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
13593 		if (err)
13594 			return err;
13595 
13596 		if (BPF_SRC(insn->code) == BPF_X) {
13597 			struct bpf_reg_state *src_reg = regs + insn->src_reg;
13598 			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
13599 			bool need_id = src_reg->type == SCALAR_VALUE && !src_reg->id &&
13600 				       !tnum_is_const(src_reg->var_off);
13601 
13602 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
13603 				if (insn->off == 0) {
13604 					/* case: R1 = R2
13605 					 * copy register state to dest reg
13606 					 */
13607 					if (need_id)
13608 						/* Assign src and dst registers the same ID
13609 						 * that will be used by find_equal_scalars()
13610 						 * to propagate min/max range.
13611 						 */
13612 						src_reg->id = ++env->id_gen;
13613 					copy_register_state(dst_reg, src_reg);
13614 					dst_reg->live |= REG_LIVE_WRITTEN;
13615 					dst_reg->subreg_def = DEF_NOT_SUBREG;
13616 				} else {
13617 					/* case: R1 = (s8, s16 s32)R2 */
13618 					if (is_pointer_value(env, insn->src_reg)) {
13619 						verbose(env,
13620 							"R%d sign-extension part of pointer\n",
13621 							insn->src_reg);
13622 						return -EACCES;
13623 					} else if (src_reg->type == SCALAR_VALUE) {
13624 						bool no_sext;
13625 
13626 						no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
13627 						if (no_sext && need_id)
13628 							src_reg->id = ++env->id_gen;
13629 						copy_register_state(dst_reg, src_reg);
13630 						if (!no_sext)
13631 							dst_reg->id = 0;
13632 						coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
13633 						dst_reg->live |= REG_LIVE_WRITTEN;
13634 						dst_reg->subreg_def = DEF_NOT_SUBREG;
13635 					} else {
13636 						mark_reg_unknown(env, regs, insn->dst_reg);
13637 					}
13638 				}
13639 			} else {
13640 				/* R1 = (u32) R2 */
13641 				if (is_pointer_value(env, insn->src_reg)) {
13642 					verbose(env,
13643 						"R%d partial copy of pointer\n",
13644 						insn->src_reg);
13645 					return -EACCES;
13646 				} else if (src_reg->type == SCALAR_VALUE) {
13647 					if (insn->off == 0) {
13648 						bool is_src_reg_u32 = src_reg->umax_value <= U32_MAX;
13649 
13650 						if (is_src_reg_u32 && need_id)
13651 							src_reg->id = ++env->id_gen;
13652 						copy_register_state(dst_reg, src_reg);
13653 						/* Make sure ID is cleared if src_reg is not in u32
13654 						 * range otherwise dst_reg min/max could be incorrectly
13655 						 * propagated into src_reg by find_equal_scalars()
13656 						 */
13657 						if (!is_src_reg_u32)
13658 							dst_reg->id = 0;
13659 						dst_reg->live |= REG_LIVE_WRITTEN;
13660 						dst_reg->subreg_def = env->insn_idx + 1;
13661 					} else {
13662 						/* case: W1 = (s8, s16)W2 */
13663 						bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
13664 
13665 						if (no_sext && need_id)
13666 							src_reg->id = ++env->id_gen;
13667 						copy_register_state(dst_reg, src_reg);
13668 						if (!no_sext)
13669 							dst_reg->id = 0;
13670 						dst_reg->live |= REG_LIVE_WRITTEN;
13671 						dst_reg->subreg_def = env->insn_idx + 1;
13672 						coerce_subreg_to_size_sx(dst_reg, insn->off >> 3);
13673 					}
13674 				} else {
13675 					mark_reg_unknown(env, regs,
13676 							 insn->dst_reg);
13677 				}
13678 				zext_32_to_64(dst_reg);
13679 				reg_bounds_sync(dst_reg);
13680 			}
13681 		} else {
13682 			/* case: R = imm
13683 			 * remember the value we stored into this reg
13684 			 */
13685 			/* clear any state __mark_reg_known doesn't set */
13686 			mark_reg_unknown(env, regs, insn->dst_reg);
13687 			regs[insn->dst_reg].type = SCALAR_VALUE;
13688 			if (BPF_CLASS(insn->code) == BPF_ALU64) {
13689 				__mark_reg_known(regs + insn->dst_reg,
13690 						 insn->imm);
13691 			} else {
13692 				__mark_reg_known(regs + insn->dst_reg,
13693 						 (u32)insn->imm);
13694 			}
13695 		}
13696 
13697 	} else if (opcode > BPF_END) {
13698 		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
13699 		return -EINVAL;
13700 
13701 	} else {	/* all other ALU ops: and, sub, xor, add, ... */
13702 
13703 		if (BPF_SRC(insn->code) == BPF_X) {
13704 			if (insn->imm != 0 || insn->off > 1 ||
13705 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
13706 				verbose(env, "BPF_ALU uses reserved fields\n");
13707 				return -EINVAL;
13708 			}
13709 			/* check src1 operand */
13710 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
13711 			if (err)
13712 				return err;
13713 		} else {
13714 			if (insn->src_reg != BPF_REG_0 || insn->off > 1 ||
13715 			    (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
13716 				verbose(env, "BPF_ALU uses reserved fields\n");
13717 				return -EINVAL;
13718 			}
13719 		}
13720 
13721 		/* check src2 operand */
13722 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
13723 		if (err)
13724 			return err;
13725 
13726 		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
13727 		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
13728 			verbose(env, "div by zero\n");
13729 			return -EINVAL;
13730 		}
13731 
13732 		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
13733 		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
13734 			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
13735 
13736 			if (insn->imm < 0 || insn->imm >= size) {
13737 				verbose(env, "invalid shift %d\n", insn->imm);
13738 				return -EINVAL;
13739 			}
13740 		}
13741 
13742 		/* check dest operand */
13743 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
13744 		if (err)
13745 			return err;
13746 
13747 		return adjust_reg_min_max_vals(env, insn);
13748 	}
13749 
13750 	return 0;
13751 }
13752 
13753 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
13754 				   struct bpf_reg_state *dst_reg,
13755 				   enum bpf_reg_type type,
13756 				   bool range_right_open)
13757 {
13758 	struct bpf_func_state *state;
13759 	struct bpf_reg_state *reg;
13760 	int new_range;
13761 
13762 	if (dst_reg->off < 0 ||
13763 	    (dst_reg->off == 0 && range_right_open))
13764 		/* This doesn't give us any range */
13765 		return;
13766 
13767 	if (dst_reg->umax_value > MAX_PACKET_OFF ||
13768 	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
13769 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
13770 		 * than pkt_end, but that's because it's also less than pkt.
13771 		 */
13772 		return;
13773 
13774 	new_range = dst_reg->off;
13775 	if (range_right_open)
13776 		new_range++;
13777 
13778 	/* Examples for register markings:
13779 	 *
13780 	 * pkt_data in dst register:
13781 	 *
13782 	 *   r2 = r3;
13783 	 *   r2 += 8;
13784 	 *   if (r2 > pkt_end) goto <handle exception>
13785 	 *   <access okay>
13786 	 *
13787 	 *   r2 = r3;
13788 	 *   r2 += 8;
13789 	 *   if (r2 < pkt_end) goto <access okay>
13790 	 *   <handle exception>
13791 	 *
13792 	 *   Where:
13793 	 *     r2 == dst_reg, pkt_end == src_reg
13794 	 *     r2=pkt(id=n,off=8,r=0)
13795 	 *     r3=pkt(id=n,off=0,r=0)
13796 	 *
13797 	 * pkt_data in src register:
13798 	 *
13799 	 *   r2 = r3;
13800 	 *   r2 += 8;
13801 	 *   if (pkt_end >= r2) goto <access okay>
13802 	 *   <handle exception>
13803 	 *
13804 	 *   r2 = r3;
13805 	 *   r2 += 8;
13806 	 *   if (pkt_end <= r2) goto <handle exception>
13807 	 *   <access okay>
13808 	 *
13809 	 *   Where:
13810 	 *     pkt_end == dst_reg, r2 == src_reg
13811 	 *     r2=pkt(id=n,off=8,r=0)
13812 	 *     r3=pkt(id=n,off=0,r=0)
13813 	 *
13814 	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
13815 	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
13816 	 * and [r3, r3 + 8-1) respectively is safe to access depending on
13817 	 * the check.
13818 	 */
13819 
13820 	/* If our ids match, then we must have the same max_value.  And we
13821 	 * don't care about the other reg's fixed offset, since if it's too big
13822 	 * the range won't allow anything.
13823 	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
13824 	 */
13825 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
13826 		if (reg->type == type && reg->id == dst_reg->id)
13827 			/* keep the maximum range already checked */
13828 			reg->range = max(reg->range, new_range);
13829 	}));
13830 }
13831 
13832 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
13833 {
13834 	struct tnum subreg = tnum_subreg(reg->var_off);
13835 	s32 sval = (s32)val;
13836 
13837 	switch (opcode) {
13838 	case BPF_JEQ:
13839 		if (tnum_is_const(subreg))
13840 			return !!tnum_equals_const(subreg, val);
13841 		else if (val < reg->u32_min_value || val > reg->u32_max_value)
13842 			return 0;
13843 		break;
13844 	case BPF_JNE:
13845 		if (tnum_is_const(subreg))
13846 			return !tnum_equals_const(subreg, val);
13847 		else if (val < reg->u32_min_value || val > reg->u32_max_value)
13848 			return 1;
13849 		break;
13850 	case BPF_JSET:
13851 		if ((~subreg.mask & subreg.value) & val)
13852 			return 1;
13853 		if (!((subreg.mask | subreg.value) & val))
13854 			return 0;
13855 		break;
13856 	case BPF_JGT:
13857 		if (reg->u32_min_value > val)
13858 			return 1;
13859 		else if (reg->u32_max_value <= val)
13860 			return 0;
13861 		break;
13862 	case BPF_JSGT:
13863 		if (reg->s32_min_value > sval)
13864 			return 1;
13865 		else if (reg->s32_max_value <= sval)
13866 			return 0;
13867 		break;
13868 	case BPF_JLT:
13869 		if (reg->u32_max_value < val)
13870 			return 1;
13871 		else if (reg->u32_min_value >= val)
13872 			return 0;
13873 		break;
13874 	case BPF_JSLT:
13875 		if (reg->s32_max_value < sval)
13876 			return 1;
13877 		else if (reg->s32_min_value >= sval)
13878 			return 0;
13879 		break;
13880 	case BPF_JGE:
13881 		if (reg->u32_min_value >= val)
13882 			return 1;
13883 		else if (reg->u32_max_value < val)
13884 			return 0;
13885 		break;
13886 	case BPF_JSGE:
13887 		if (reg->s32_min_value >= sval)
13888 			return 1;
13889 		else if (reg->s32_max_value < sval)
13890 			return 0;
13891 		break;
13892 	case BPF_JLE:
13893 		if (reg->u32_max_value <= val)
13894 			return 1;
13895 		else if (reg->u32_min_value > val)
13896 			return 0;
13897 		break;
13898 	case BPF_JSLE:
13899 		if (reg->s32_max_value <= sval)
13900 			return 1;
13901 		else if (reg->s32_min_value > sval)
13902 			return 0;
13903 		break;
13904 	}
13905 
13906 	return -1;
13907 }
13908 
13909 
13910 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
13911 {
13912 	s64 sval = (s64)val;
13913 
13914 	switch (opcode) {
13915 	case BPF_JEQ:
13916 		if (tnum_is_const(reg->var_off))
13917 			return !!tnum_equals_const(reg->var_off, val);
13918 		else if (val < reg->umin_value || val > reg->umax_value)
13919 			return 0;
13920 		break;
13921 	case BPF_JNE:
13922 		if (tnum_is_const(reg->var_off))
13923 			return !tnum_equals_const(reg->var_off, val);
13924 		else if (val < reg->umin_value || val > reg->umax_value)
13925 			return 1;
13926 		break;
13927 	case BPF_JSET:
13928 		if ((~reg->var_off.mask & reg->var_off.value) & val)
13929 			return 1;
13930 		if (!((reg->var_off.mask | reg->var_off.value) & val))
13931 			return 0;
13932 		break;
13933 	case BPF_JGT:
13934 		if (reg->umin_value > val)
13935 			return 1;
13936 		else if (reg->umax_value <= val)
13937 			return 0;
13938 		break;
13939 	case BPF_JSGT:
13940 		if (reg->smin_value > sval)
13941 			return 1;
13942 		else if (reg->smax_value <= sval)
13943 			return 0;
13944 		break;
13945 	case BPF_JLT:
13946 		if (reg->umax_value < val)
13947 			return 1;
13948 		else if (reg->umin_value >= val)
13949 			return 0;
13950 		break;
13951 	case BPF_JSLT:
13952 		if (reg->smax_value < sval)
13953 			return 1;
13954 		else if (reg->smin_value >= sval)
13955 			return 0;
13956 		break;
13957 	case BPF_JGE:
13958 		if (reg->umin_value >= val)
13959 			return 1;
13960 		else if (reg->umax_value < val)
13961 			return 0;
13962 		break;
13963 	case BPF_JSGE:
13964 		if (reg->smin_value >= sval)
13965 			return 1;
13966 		else if (reg->smax_value < sval)
13967 			return 0;
13968 		break;
13969 	case BPF_JLE:
13970 		if (reg->umax_value <= val)
13971 			return 1;
13972 		else if (reg->umin_value > val)
13973 			return 0;
13974 		break;
13975 	case BPF_JSLE:
13976 		if (reg->smax_value <= sval)
13977 			return 1;
13978 		else if (reg->smin_value > sval)
13979 			return 0;
13980 		break;
13981 	}
13982 
13983 	return -1;
13984 }
13985 
13986 /* compute branch direction of the expression "if (reg opcode val) goto target;"
13987  * and return:
13988  *  1 - branch will be taken and "goto target" will be executed
13989  *  0 - branch will not be taken and fall-through to next insn
13990  * -1 - unknown. Example: "if (reg < 5)" is unknown when register value
13991  *      range [0,10]
13992  */
13993 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
13994 			   bool is_jmp32)
13995 {
13996 	if (__is_pointer_value(false, reg)) {
13997 		if (!reg_not_null(reg))
13998 			return -1;
13999 
14000 		/* If pointer is valid tests against zero will fail so we can
14001 		 * use this to direct branch taken.
14002 		 */
14003 		if (val != 0)
14004 			return -1;
14005 
14006 		switch (opcode) {
14007 		case BPF_JEQ:
14008 			return 0;
14009 		case BPF_JNE:
14010 			return 1;
14011 		default:
14012 			return -1;
14013 		}
14014 	}
14015 
14016 	if (is_jmp32)
14017 		return is_branch32_taken(reg, val, opcode);
14018 	return is_branch64_taken(reg, val, opcode);
14019 }
14020 
14021 static int flip_opcode(u32 opcode)
14022 {
14023 	/* How can we transform "a <op> b" into "b <op> a"? */
14024 	static const u8 opcode_flip[16] = {
14025 		/* these stay the same */
14026 		[BPF_JEQ  >> 4] = BPF_JEQ,
14027 		[BPF_JNE  >> 4] = BPF_JNE,
14028 		[BPF_JSET >> 4] = BPF_JSET,
14029 		/* these swap "lesser" and "greater" (L and G in the opcodes) */
14030 		[BPF_JGE  >> 4] = BPF_JLE,
14031 		[BPF_JGT  >> 4] = BPF_JLT,
14032 		[BPF_JLE  >> 4] = BPF_JGE,
14033 		[BPF_JLT  >> 4] = BPF_JGT,
14034 		[BPF_JSGE >> 4] = BPF_JSLE,
14035 		[BPF_JSGT >> 4] = BPF_JSLT,
14036 		[BPF_JSLE >> 4] = BPF_JSGE,
14037 		[BPF_JSLT >> 4] = BPF_JSGT
14038 	};
14039 	return opcode_flip[opcode >> 4];
14040 }
14041 
14042 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
14043 				   struct bpf_reg_state *src_reg,
14044 				   u8 opcode)
14045 {
14046 	struct bpf_reg_state *pkt;
14047 
14048 	if (src_reg->type == PTR_TO_PACKET_END) {
14049 		pkt = dst_reg;
14050 	} else if (dst_reg->type == PTR_TO_PACKET_END) {
14051 		pkt = src_reg;
14052 		opcode = flip_opcode(opcode);
14053 	} else {
14054 		return -1;
14055 	}
14056 
14057 	if (pkt->range >= 0)
14058 		return -1;
14059 
14060 	switch (opcode) {
14061 	case BPF_JLE:
14062 		/* pkt <= pkt_end */
14063 		fallthrough;
14064 	case BPF_JGT:
14065 		/* pkt > pkt_end */
14066 		if (pkt->range == BEYOND_PKT_END)
14067 			/* pkt has at last one extra byte beyond pkt_end */
14068 			return opcode == BPF_JGT;
14069 		break;
14070 	case BPF_JLT:
14071 		/* pkt < pkt_end */
14072 		fallthrough;
14073 	case BPF_JGE:
14074 		/* pkt >= pkt_end */
14075 		if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
14076 			return opcode == BPF_JGE;
14077 		break;
14078 	}
14079 	return -1;
14080 }
14081 
14082 /* Adjusts the register min/max values in the case that the dst_reg is the
14083  * variable register that we are working on, and src_reg is a constant or we're
14084  * simply doing a BPF_K check.
14085  * In JEQ/JNE cases we also adjust the var_off values.
14086  */
14087 static void reg_set_min_max(struct bpf_reg_state *true_reg,
14088 			    struct bpf_reg_state *false_reg,
14089 			    u64 val, u32 val32,
14090 			    u8 opcode, bool is_jmp32)
14091 {
14092 	struct tnum false_32off = tnum_subreg(false_reg->var_off);
14093 	struct tnum false_64off = false_reg->var_off;
14094 	struct tnum true_32off = tnum_subreg(true_reg->var_off);
14095 	struct tnum true_64off = true_reg->var_off;
14096 	s64 sval = (s64)val;
14097 	s32 sval32 = (s32)val32;
14098 
14099 	/* If the dst_reg is a pointer, we can't learn anything about its
14100 	 * variable offset from the compare (unless src_reg were a pointer into
14101 	 * the same object, but we don't bother with that.
14102 	 * Since false_reg and true_reg have the same type by construction, we
14103 	 * only need to check one of them for pointerness.
14104 	 */
14105 	if (__is_pointer_value(false, false_reg))
14106 		return;
14107 
14108 	switch (opcode) {
14109 	/* JEQ/JNE comparison doesn't change the register equivalence.
14110 	 *
14111 	 * r1 = r2;
14112 	 * if (r1 == 42) goto label;
14113 	 * ...
14114 	 * label: // here both r1 and r2 are known to be 42.
14115 	 *
14116 	 * Hence when marking register as known preserve it's ID.
14117 	 */
14118 	case BPF_JEQ:
14119 		if (is_jmp32) {
14120 			__mark_reg32_known(true_reg, val32);
14121 			true_32off = tnum_subreg(true_reg->var_off);
14122 		} else {
14123 			___mark_reg_known(true_reg, val);
14124 			true_64off = true_reg->var_off;
14125 		}
14126 		break;
14127 	case BPF_JNE:
14128 		if (is_jmp32) {
14129 			__mark_reg32_known(false_reg, val32);
14130 			false_32off = tnum_subreg(false_reg->var_off);
14131 		} else {
14132 			___mark_reg_known(false_reg, val);
14133 			false_64off = false_reg->var_off;
14134 		}
14135 		break;
14136 	case BPF_JSET:
14137 		if (is_jmp32) {
14138 			false_32off = tnum_and(false_32off, tnum_const(~val32));
14139 			if (is_power_of_2(val32))
14140 				true_32off = tnum_or(true_32off,
14141 						     tnum_const(val32));
14142 		} else {
14143 			false_64off = tnum_and(false_64off, tnum_const(~val));
14144 			if (is_power_of_2(val))
14145 				true_64off = tnum_or(true_64off,
14146 						     tnum_const(val));
14147 		}
14148 		break;
14149 	case BPF_JGE:
14150 	case BPF_JGT:
14151 	{
14152 		if (is_jmp32) {
14153 			u32 false_umax = opcode == BPF_JGT ? val32  : val32 - 1;
14154 			u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32;
14155 
14156 			false_reg->u32_max_value = min(false_reg->u32_max_value,
14157 						       false_umax);
14158 			true_reg->u32_min_value = max(true_reg->u32_min_value,
14159 						      true_umin);
14160 		} else {
14161 			u64 false_umax = opcode == BPF_JGT ? val    : val - 1;
14162 			u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
14163 
14164 			false_reg->umax_value = min(false_reg->umax_value, false_umax);
14165 			true_reg->umin_value = max(true_reg->umin_value, true_umin);
14166 		}
14167 		break;
14168 	}
14169 	case BPF_JSGE:
14170 	case BPF_JSGT:
14171 	{
14172 		if (is_jmp32) {
14173 			s32 false_smax = opcode == BPF_JSGT ? sval32    : sval32 - 1;
14174 			s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32;
14175 
14176 			false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax);
14177 			true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin);
14178 		} else {
14179 			s64 false_smax = opcode == BPF_JSGT ? sval    : sval - 1;
14180 			s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
14181 
14182 			false_reg->smax_value = min(false_reg->smax_value, false_smax);
14183 			true_reg->smin_value = max(true_reg->smin_value, true_smin);
14184 		}
14185 		break;
14186 	}
14187 	case BPF_JLE:
14188 	case BPF_JLT:
14189 	{
14190 		if (is_jmp32) {
14191 			u32 false_umin = opcode == BPF_JLT ? val32  : val32 + 1;
14192 			u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32;
14193 
14194 			false_reg->u32_min_value = max(false_reg->u32_min_value,
14195 						       false_umin);
14196 			true_reg->u32_max_value = min(true_reg->u32_max_value,
14197 						      true_umax);
14198 		} else {
14199 			u64 false_umin = opcode == BPF_JLT ? val    : val + 1;
14200 			u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
14201 
14202 			false_reg->umin_value = max(false_reg->umin_value, false_umin);
14203 			true_reg->umax_value = min(true_reg->umax_value, true_umax);
14204 		}
14205 		break;
14206 	}
14207 	case BPF_JSLE:
14208 	case BPF_JSLT:
14209 	{
14210 		if (is_jmp32) {
14211 			s32 false_smin = opcode == BPF_JSLT ? sval32    : sval32 + 1;
14212 			s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32;
14213 
14214 			false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin);
14215 			true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax);
14216 		} else {
14217 			s64 false_smin = opcode == BPF_JSLT ? sval    : sval + 1;
14218 			s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
14219 
14220 			false_reg->smin_value = max(false_reg->smin_value, false_smin);
14221 			true_reg->smax_value = min(true_reg->smax_value, true_smax);
14222 		}
14223 		break;
14224 	}
14225 	default:
14226 		return;
14227 	}
14228 
14229 	if (is_jmp32) {
14230 		false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off),
14231 					     tnum_subreg(false_32off));
14232 		true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off),
14233 					    tnum_subreg(true_32off));
14234 		__reg_combine_32_into_64(false_reg);
14235 		__reg_combine_32_into_64(true_reg);
14236 	} else {
14237 		false_reg->var_off = false_64off;
14238 		true_reg->var_off = true_64off;
14239 		__reg_combine_64_into_32(false_reg);
14240 		__reg_combine_64_into_32(true_reg);
14241 	}
14242 }
14243 
14244 /* Same as above, but for the case that dst_reg holds a constant and src_reg is
14245  * the variable reg.
14246  */
14247 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
14248 				struct bpf_reg_state *false_reg,
14249 				u64 val, u32 val32,
14250 				u8 opcode, bool is_jmp32)
14251 {
14252 	opcode = flip_opcode(opcode);
14253 	/* This uses zero as "not present in table"; luckily the zero opcode,
14254 	 * BPF_JA, can't get here.
14255 	 */
14256 	if (opcode)
14257 		reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32);
14258 }
14259 
14260 /* Regs are known to be equal, so intersect their min/max/var_off */
14261 static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
14262 				  struct bpf_reg_state *dst_reg)
14263 {
14264 	src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
14265 							dst_reg->umin_value);
14266 	src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
14267 							dst_reg->umax_value);
14268 	src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
14269 							dst_reg->smin_value);
14270 	src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
14271 							dst_reg->smax_value);
14272 	src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
14273 							     dst_reg->var_off);
14274 	reg_bounds_sync(src_reg);
14275 	reg_bounds_sync(dst_reg);
14276 }
14277 
14278 static void reg_combine_min_max(struct bpf_reg_state *true_src,
14279 				struct bpf_reg_state *true_dst,
14280 				struct bpf_reg_state *false_src,
14281 				struct bpf_reg_state *false_dst,
14282 				u8 opcode)
14283 {
14284 	switch (opcode) {
14285 	case BPF_JEQ:
14286 		__reg_combine_min_max(true_src, true_dst);
14287 		break;
14288 	case BPF_JNE:
14289 		__reg_combine_min_max(false_src, false_dst);
14290 		break;
14291 	}
14292 }
14293 
14294 static void mark_ptr_or_null_reg(struct bpf_func_state *state,
14295 				 struct bpf_reg_state *reg, u32 id,
14296 				 bool is_null)
14297 {
14298 	if (type_may_be_null(reg->type) && reg->id == id &&
14299 	    (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) {
14300 		/* Old offset (both fixed and variable parts) should have been
14301 		 * known-zero, because we don't allow pointer arithmetic on
14302 		 * pointers that might be NULL. If we see this happening, don't
14303 		 * convert the register.
14304 		 *
14305 		 * But in some cases, some helpers that return local kptrs
14306 		 * advance offset for the returned pointer. In those cases, it
14307 		 * is fine to expect to see reg->off.
14308 		 */
14309 		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0)))
14310 			return;
14311 		if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) &&
14312 		    WARN_ON_ONCE(reg->off))
14313 			return;
14314 
14315 		if (is_null) {
14316 			reg->type = SCALAR_VALUE;
14317 			/* We don't need id and ref_obj_id from this point
14318 			 * onwards anymore, thus we should better reset it,
14319 			 * so that state pruning has chances to take effect.
14320 			 */
14321 			reg->id = 0;
14322 			reg->ref_obj_id = 0;
14323 
14324 			return;
14325 		}
14326 
14327 		mark_ptr_not_null_reg(reg);
14328 
14329 		if (!reg_may_point_to_spin_lock(reg)) {
14330 			/* For not-NULL ptr, reg->ref_obj_id will be reset
14331 			 * in release_reference().
14332 			 *
14333 			 * reg->id is still used by spin_lock ptr. Other
14334 			 * than spin_lock ptr type, reg->id can be reset.
14335 			 */
14336 			reg->id = 0;
14337 		}
14338 	}
14339 }
14340 
14341 /* The logic is similar to find_good_pkt_pointers(), both could eventually
14342  * be folded together at some point.
14343  */
14344 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
14345 				  bool is_null)
14346 {
14347 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
14348 	struct bpf_reg_state *regs = state->regs, *reg;
14349 	u32 ref_obj_id = regs[regno].ref_obj_id;
14350 	u32 id = regs[regno].id;
14351 
14352 	if (ref_obj_id && ref_obj_id == id && is_null)
14353 		/* regs[regno] is in the " == NULL" branch.
14354 		 * No one could have freed the reference state before
14355 		 * doing the NULL check.
14356 		 */
14357 		WARN_ON_ONCE(release_reference_state(state, id));
14358 
14359 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
14360 		mark_ptr_or_null_reg(state, reg, id, is_null);
14361 	}));
14362 }
14363 
14364 static bool try_match_pkt_pointers(const struct bpf_insn *insn,
14365 				   struct bpf_reg_state *dst_reg,
14366 				   struct bpf_reg_state *src_reg,
14367 				   struct bpf_verifier_state *this_branch,
14368 				   struct bpf_verifier_state *other_branch)
14369 {
14370 	if (BPF_SRC(insn->code) != BPF_X)
14371 		return false;
14372 
14373 	/* Pointers are always 64-bit. */
14374 	if (BPF_CLASS(insn->code) == BPF_JMP32)
14375 		return false;
14376 
14377 	switch (BPF_OP(insn->code)) {
14378 	case BPF_JGT:
14379 		if ((dst_reg->type == PTR_TO_PACKET &&
14380 		     src_reg->type == PTR_TO_PACKET_END) ||
14381 		    (dst_reg->type == PTR_TO_PACKET_META &&
14382 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
14383 			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
14384 			find_good_pkt_pointers(this_branch, dst_reg,
14385 					       dst_reg->type, false);
14386 			mark_pkt_end(other_branch, insn->dst_reg, true);
14387 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
14388 			    src_reg->type == PTR_TO_PACKET) ||
14389 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
14390 			    src_reg->type == PTR_TO_PACKET_META)) {
14391 			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
14392 			find_good_pkt_pointers(other_branch, src_reg,
14393 					       src_reg->type, true);
14394 			mark_pkt_end(this_branch, insn->src_reg, false);
14395 		} else {
14396 			return false;
14397 		}
14398 		break;
14399 	case BPF_JLT:
14400 		if ((dst_reg->type == PTR_TO_PACKET &&
14401 		     src_reg->type == PTR_TO_PACKET_END) ||
14402 		    (dst_reg->type == PTR_TO_PACKET_META &&
14403 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
14404 			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
14405 			find_good_pkt_pointers(other_branch, dst_reg,
14406 					       dst_reg->type, true);
14407 			mark_pkt_end(this_branch, insn->dst_reg, false);
14408 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
14409 			    src_reg->type == PTR_TO_PACKET) ||
14410 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
14411 			    src_reg->type == PTR_TO_PACKET_META)) {
14412 			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
14413 			find_good_pkt_pointers(this_branch, src_reg,
14414 					       src_reg->type, false);
14415 			mark_pkt_end(other_branch, insn->src_reg, true);
14416 		} else {
14417 			return false;
14418 		}
14419 		break;
14420 	case BPF_JGE:
14421 		if ((dst_reg->type == PTR_TO_PACKET &&
14422 		     src_reg->type == PTR_TO_PACKET_END) ||
14423 		    (dst_reg->type == PTR_TO_PACKET_META &&
14424 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
14425 			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
14426 			find_good_pkt_pointers(this_branch, dst_reg,
14427 					       dst_reg->type, true);
14428 			mark_pkt_end(other_branch, insn->dst_reg, false);
14429 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
14430 			    src_reg->type == PTR_TO_PACKET) ||
14431 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
14432 			    src_reg->type == PTR_TO_PACKET_META)) {
14433 			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
14434 			find_good_pkt_pointers(other_branch, src_reg,
14435 					       src_reg->type, false);
14436 			mark_pkt_end(this_branch, insn->src_reg, true);
14437 		} else {
14438 			return false;
14439 		}
14440 		break;
14441 	case BPF_JLE:
14442 		if ((dst_reg->type == PTR_TO_PACKET &&
14443 		     src_reg->type == PTR_TO_PACKET_END) ||
14444 		    (dst_reg->type == PTR_TO_PACKET_META &&
14445 		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
14446 			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
14447 			find_good_pkt_pointers(other_branch, dst_reg,
14448 					       dst_reg->type, false);
14449 			mark_pkt_end(this_branch, insn->dst_reg, true);
14450 		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
14451 			    src_reg->type == PTR_TO_PACKET) ||
14452 			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
14453 			    src_reg->type == PTR_TO_PACKET_META)) {
14454 			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
14455 			find_good_pkt_pointers(this_branch, src_reg,
14456 					       src_reg->type, true);
14457 			mark_pkt_end(other_branch, insn->src_reg, false);
14458 		} else {
14459 			return false;
14460 		}
14461 		break;
14462 	default:
14463 		return false;
14464 	}
14465 
14466 	return true;
14467 }
14468 
14469 static void find_equal_scalars(struct bpf_verifier_state *vstate,
14470 			       struct bpf_reg_state *known_reg)
14471 {
14472 	struct bpf_func_state *state;
14473 	struct bpf_reg_state *reg;
14474 
14475 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
14476 		if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
14477 			copy_register_state(reg, known_reg);
14478 	}));
14479 }
14480 
14481 static int check_cond_jmp_op(struct bpf_verifier_env *env,
14482 			     struct bpf_insn *insn, int *insn_idx)
14483 {
14484 	struct bpf_verifier_state *this_branch = env->cur_state;
14485 	struct bpf_verifier_state *other_branch;
14486 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
14487 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
14488 	struct bpf_reg_state *eq_branch_regs;
14489 	u8 opcode = BPF_OP(insn->code);
14490 	bool is_jmp32;
14491 	int pred = -1;
14492 	int err;
14493 
14494 	/* Only conditional jumps are expected to reach here. */
14495 	if (opcode == BPF_JA || opcode > BPF_JSLE) {
14496 		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
14497 		return -EINVAL;
14498 	}
14499 
14500 	/* check src2 operand */
14501 	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
14502 	if (err)
14503 		return err;
14504 
14505 	dst_reg = &regs[insn->dst_reg];
14506 	if (BPF_SRC(insn->code) == BPF_X) {
14507 		if (insn->imm != 0) {
14508 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
14509 			return -EINVAL;
14510 		}
14511 
14512 		/* check src1 operand */
14513 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
14514 		if (err)
14515 			return err;
14516 
14517 		src_reg = &regs[insn->src_reg];
14518 		if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
14519 		    is_pointer_value(env, insn->src_reg)) {
14520 			verbose(env, "R%d pointer comparison prohibited\n",
14521 				insn->src_reg);
14522 			return -EACCES;
14523 		}
14524 	} else {
14525 		if (insn->src_reg != BPF_REG_0) {
14526 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
14527 			return -EINVAL;
14528 		}
14529 	}
14530 
14531 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
14532 
14533 	if (BPF_SRC(insn->code) == BPF_K) {
14534 		pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32);
14535 	} else if (src_reg->type == SCALAR_VALUE &&
14536 		   is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) {
14537 		pred = is_branch_taken(dst_reg,
14538 				       tnum_subreg(src_reg->var_off).value,
14539 				       opcode,
14540 				       is_jmp32);
14541 	} else if (src_reg->type == SCALAR_VALUE &&
14542 		   !is_jmp32 && tnum_is_const(src_reg->var_off)) {
14543 		pred = is_branch_taken(dst_reg,
14544 				       src_reg->var_off.value,
14545 				       opcode,
14546 				       is_jmp32);
14547 	} else if (dst_reg->type == SCALAR_VALUE &&
14548 		   is_jmp32 && tnum_is_const(tnum_subreg(dst_reg->var_off))) {
14549 		pred = is_branch_taken(src_reg,
14550 				       tnum_subreg(dst_reg->var_off).value,
14551 				       flip_opcode(opcode),
14552 				       is_jmp32);
14553 	} else if (dst_reg->type == SCALAR_VALUE &&
14554 		   !is_jmp32 && tnum_is_const(dst_reg->var_off)) {
14555 		pred = is_branch_taken(src_reg,
14556 				       dst_reg->var_off.value,
14557 				       flip_opcode(opcode),
14558 				       is_jmp32);
14559 	} else if (reg_is_pkt_pointer_any(dst_reg) &&
14560 		   reg_is_pkt_pointer_any(src_reg) &&
14561 		   !is_jmp32) {
14562 		pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
14563 	}
14564 
14565 	if (pred >= 0) {
14566 		/* If we get here with a dst_reg pointer type it is because
14567 		 * above is_branch_taken() special cased the 0 comparison.
14568 		 */
14569 		if (!__is_pointer_value(false, dst_reg))
14570 			err = mark_chain_precision(env, insn->dst_reg);
14571 		if (BPF_SRC(insn->code) == BPF_X && !err &&
14572 		    !__is_pointer_value(false, src_reg))
14573 			err = mark_chain_precision(env, insn->src_reg);
14574 		if (err)
14575 			return err;
14576 	}
14577 
14578 	if (pred == 1) {
14579 		/* Only follow the goto, ignore fall-through. If needed, push
14580 		 * the fall-through branch for simulation under speculative
14581 		 * execution.
14582 		 */
14583 		if (!env->bypass_spec_v1 &&
14584 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,
14585 					       *insn_idx))
14586 			return -EFAULT;
14587 		if (env->log.level & BPF_LOG_LEVEL)
14588 			print_insn_state(env, this_branch->frame[this_branch->curframe]);
14589 		*insn_idx += insn->off;
14590 		return 0;
14591 	} else if (pred == 0) {
14592 		/* Only follow the fall-through branch, since that's where the
14593 		 * program will go. If needed, push the goto branch for
14594 		 * simulation under speculative execution.
14595 		 */
14596 		if (!env->bypass_spec_v1 &&
14597 		    !sanitize_speculative_path(env, insn,
14598 					       *insn_idx + insn->off + 1,
14599 					       *insn_idx))
14600 			return -EFAULT;
14601 		if (env->log.level & BPF_LOG_LEVEL)
14602 			print_insn_state(env, this_branch->frame[this_branch->curframe]);
14603 		return 0;
14604 	}
14605 
14606 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
14607 				  false);
14608 	if (!other_branch)
14609 		return -EFAULT;
14610 	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
14611 
14612 	/* detect if we are comparing against a constant value so we can adjust
14613 	 * our min/max values for our dst register.
14614 	 * this is only legit if both are scalars (or pointers to the same
14615 	 * object, I suppose, see the PTR_MAYBE_NULL related if block below),
14616 	 * because otherwise the different base pointers mean the offsets aren't
14617 	 * comparable.
14618 	 */
14619 	if (BPF_SRC(insn->code) == BPF_X) {
14620 		struct bpf_reg_state *src_reg = &regs[insn->src_reg];
14621 
14622 		if (dst_reg->type == SCALAR_VALUE &&
14623 		    src_reg->type == SCALAR_VALUE) {
14624 			if (tnum_is_const(src_reg->var_off) ||
14625 			    (is_jmp32 &&
14626 			     tnum_is_const(tnum_subreg(src_reg->var_off))))
14627 				reg_set_min_max(&other_branch_regs[insn->dst_reg],
14628 						dst_reg,
14629 						src_reg->var_off.value,
14630 						tnum_subreg(src_reg->var_off).value,
14631 						opcode, is_jmp32);
14632 			else if (tnum_is_const(dst_reg->var_off) ||
14633 				 (is_jmp32 &&
14634 				  tnum_is_const(tnum_subreg(dst_reg->var_off))))
14635 				reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
14636 						    src_reg,
14637 						    dst_reg->var_off.value,
14638 						    tnum_subreg(dst_reg->var_off).value,
14639 						    opcode, is_jmp32);
14640 			else if (!is_jmp32 &&
14641 				 (opcode == BPF_JEQ || opcode == BPF_JNE))
14642 				/* Comparing for equality, we can combine knowledge */
14643 				reg_combine_min_max(&other_branch_regs[insn->src_reg],
14644 						    &other_branch_regs[insn->dst_reg],
14645 						    src_reg, dst_reg, opcode);
14646 			if (src_reg->id &&
14647 			    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
14648 				find_equal_scalars(this_branch, src_reg);
14649 				find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
14650 			}
14651 
14652 		}
14653 	} else if (dst_reg->type == SCALAR_VALUE) {
14654 		reg_set_min_max(&other_branch_regs[insn->dst_reg],
14655 					dst_reg, insn->imm, (u32)insn->imm,
14656 					opcode, is_jmp32);
14657 	}
14658 
14659 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
14660 	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
14661 		find_equal_scalars(this_branch, dst_reg);
14662 		find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
14663 	}
14664 
14665 	/* if one pointer register is compared to another pointer
14666 	 * register check if PTR_MAYBE_NULL could be lifted.
14667 	 * E.g. register A - maybe null
14668 	 *      register B - not null
14669 	 * for JNE A, B, ... - A is not null in the false branch;
14670 	 * for JEQ A, B, ... - A is not null in the true branch.
14671 	 *
14672 	 * Since PTR_TO_BTF_ID points to a kernel struct that does
14673 	 * not need to be null checked by the BPF program, i.e.,
14674 	 * could be null even without PTR_MAYBE_NULL marking, so
14675 	 * only propagate nullness when neither reg is that type.
14676 	 */
14677 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
14678 	    __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
14679 	    type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
14680 	    base_type(src_reg->type) != PTR_TO_BTF_ID &&
14681 	    base_type(dst_reg->type) != PTR_TO_BTF_ID) {
14682 		eq_branch_regs = NULL;
14683 		switch (opcode) {
14684 		case BPF_JEQ:
14685 			eq_branch_regs = other_branch_regs;
14686 			break;
14687 		case BPF_JNE:
14688 			eq_branch_regs = regs;
14689 			break;
14690 		default:
14691 			/* do nothing */
14692 			break;
14693 		}
14694 		if (eq_branch_regs) {
14695 			if (type_may_be_null(src_reg->type))
14696 				mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
14697 			else
14698 				mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
14699 		}
14700 	}
14701 
14702 	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
14703 	 * NOTE: these optimizations below are related with pointer comparison
14704 	 *       which will never be JMP32.
14705 	 */
14706 	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
14707 	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
14708 	    type_may_be_null(dst_reg->type)) {
14709 		/* Mark all identical registers in each branch as either
14710 		 * safe or unknown depending R == 0 or R != 0 conditional.
14711 		 */
14712 		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
14713 				      opcode == BPF_JNE);
14714 		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
14715 				      opcode == BPF_JEQ);
14716 	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
14717 					   this_branch, other_branch) &&
14718 		   is_pointer_value(env, insn->dst_reg)) {
14719 		verbose(env, "R%d pointer comparison prohibited\n",
14720 			insn->dst_reg);
14721 		return -EACCES;
14722 	}
14723 	if (env->log.level & BPF_LOG_LEVEL)
14724 		print_insn_state(env, this_branch->frame[this_branch->curframe]);
14725 	return 0;
14726 }
14727 
14728 /* verify BPF_LD_IMM64 instruction */
14729 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
14730 {
14731 	struct bpf_insn_aux_data *aux = cur_aux(env);
14732 	struct bpf_reg_state *regs = cur_regs(env);
14733 	struct bpf_reg_state *dst_reg;
14734 	struct bpf_map *map;
14735 	int err;
14736 
14737 	if (BPF_SIZE(insn->code) != BPF_DW) {
14738 		verbose(env, "invalid BPF_LD_IMM insn\n");
14739 		return -EINVAL;
14740 	}
14741 	if (insn->off != 0) {
14742 		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
14743 		return -EINVAL;
14744 	}
14745 
14746 	err = check_reg_arg(env, insn->dst_reg, DST_OP);
14747 	if (err)
14748 		return err;
14749 
14750 	dst_reg = &regs[insn->dst_reg];
14751 	if (insn->src_reg == 0) {
14752 		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
14753 
14754 		dst_reg->type = SCALAR_VALUE;
14755 		__mark_reg_known(&regs[insn->dst_reg], imm);
14756 		return 0;
14757 	}
14758 
14759 	/* All special src_reg cases are listed below. From this point onwards
14760 	 * we either succeed and assign a corresponding dst_reg->type after
14761 	 * zeroing the offset, or fail and reject the program.
14762 	 */
14763 	mark_reg_known_zero(env, regs, insn->dst_reg);
14764 
14765 	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
14766 		dst_reg->type = aux->btf_var.reg_type;
14767 		switch (base_type(dst_reg->type)) {
14768 		case PTR_TO_MEM:
14769 			dst_reg->mem_size = aux->btf_var.mem_size;
14770 			break;
14771 		case PTR_TO_BTF_ID:
14772 			dst_reg->btf = aux->btf_var.btf;
14773 			dst_reg->btf_id = aux->btf_var.btf_id;
14774 			break;
14775 		default:
14776 			verbose(env, "bpf verifier is misconfigured\n");
14777 			return -EFAULT;
14778 		}
14779 		return 0;
14780 	}
14781 
14782 	if (insn->src_reg == BPF_PSEUDO_FUNC) {
14783 		struct bpf_prog_aux *aux = env->prog->aux;
14784 		u32 subprogno = find_subprog(env,
14785 					     env->insn_idx + insn->imm + 1);
14786 
14787 		if (!aux->func_info) {
14788 			verbose(env, "missing btf func_info\n");
14789 			return -EINVAL;
14790 		}
14791 		if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
14792 			verbose(env, "callback function not static\n");
14793 			return -EINVAL;
14794 		}
14795 
14796 		dst_reg->type = PTR_TO_FUNC;
14797 		dst_reg->subprogno = subprogno;
14798 		return 0;
14799 	}
14800 
14801 	map = env->used_maps[aux->map_index];
14802 	dst_reg->map_ptr = map;
14803 
14804 	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
14805 	    insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
14806 		dst_reg->type = PTR_TO_MAP_VALUE;
14807 		dst_reg->off = aux->map_off;
14808 		WARN_ON_ONCE(map->max_entries != 1);
14809 		/* We want reg->id to be same (0) as map_value is not distinct */
14810 	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
14811 		   insn->src_reg == BPF_PSEUDO_MAP_IDX) {
14812 		dst_reg->type = CONST_PTR_TO_MAP;
14813 	} else {
14814 		verbose(env, "bpf verifier is misconfigured\n");
14815 		return -EINVAL;
14816 	}
14817 
14818 	return 0;
14819 }
14820 
14821 static bool may_access_skb(enum bpf_prog_type type)
14822 {
14823 	switch (type) {
14824 	case BPF_PROG_TYPE_SOCKET_FILTER:
14825 	case BPF_PROG_TYPE_SCHED_CLS:
14826 	case BPF_PROG_TYPE_SCHED_ACT:
14827 		return true;
14828 	default:
14829 		return false;
14830 	}
14831 }
14832 
14833 /* verify safety of LD_ABS|LD_IND instructions:
14834  * - they can only appear in the programs where ctx == skb
14835  * - since they are wrappers of function calls, they scratch R1-R5 registers,
14836  *   preserve R6-R9, and store return value into R0
14837  *
14838  * Implicit input:
14839  *   ctx == skb == R6 == CTX
14840  *
14841  * Explicit input:
14842  *   SRC == any register
14843  *   IMM == 32-bit immediate
14844  *
14845  * Output:
14846  *   R0 - 8/16/32-bit skb data converted to cpu endianness
14847  */
14848 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
14849 {
14850 	struct bpf_reg_state *regs = cur_regs(env);
14851 	static const int ctx_reg = BPF_REG_6;
14852 	u8 mode = BPF_MODE(insn->code);
14853 	int i, err;
14854 
14855 	if (!may_access_skb(resolve_prog_type(env->prog))) {
14856 		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
14857 		return -EINVAL;
14858 	}
14859 
14860 	if (!env->ops->gen_ld_abs) {
14861 		verbose(env, "bpf verifier is misconfigured\n");
14862 		return -EINVAL;
14863 	}
14864 
14865 	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
14866 	    BPF_SIZE(insn->code) == BPF_DW ||
14867 	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
14868 		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
14869 		return -EINVAL;
14870 	}
14871 
14872 	/* check whether implicit source operand (register R6) is readable */
14873 	err = check_reg_arg(env, ctx_reg, SRC_OP);
14874 	if (err)
14875 		return err;
14876 
14877 	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
14878 	 * gen_ld_abs() may terminate the program at runtime, leading to
14879 	 * reference leak.
14880 	 */
14881 	err = check_reference_leak(env);
14882 	if (err) {
14883 		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
14884 		return err;
14885 	}
14886 
14887 	if (env->cur_state->active_lock.ptr) {
14888 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
14889 		return -EINVAL;
14890 	}
14891 
14892 	if (env->cur_state->active_rcu_lock) {
14893 		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n");
14894 		return -EINVAL;
14895 	}
14896 
14897 	if (regs[ctx_reg].type != PTR_TO_CTX) {
14898 		verbose(env,
14899 			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
14900 		return -EINVAL;
14901 	}
14902 
14903 	if (mode == BPF_IND) {
14904 		/* check explicit source operand */
14905 		err = check_reg_arg(env, insn->src_reg, SRC_OP);
14906 		if (err)
14907 			return err;
14908 	}
14909 
14910 	err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
14911 	if (err < 0)
14912 		return err;
14913 
14914 	/* reset caller saved regs to unreadable */
14915 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
14916 		mark_reg_not_init(env, regs, caller_saved[i]);
14917 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
14918 	}
14919 
14920 	/* mark destination R0 register as readable, since it contains
14921 	 * the value fetched from the packet.
14922 	 * Already marked as written above.
14923 	 */
14924 	mark_reg_unknown(env, regs, BPF_REG_0);
14925 	/* ld_abs load up to 32-bit skb data. */
14926 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
14927 	return 0;
14928 }
14929 
14930 static int check_return_code(struct bpf_verifier_env *env)
14931 {
14932 	struct tnum enforce_attach_type_range = tnum_unknown;
14933 	const struct bpf_prog *prog = env->prog;
14934 	struct bpf_reg_state *reg;
14935 	struct tnum range = tnum_range(0, 1), const_0 = tnum_const(0);
14936 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
14937 	int err;
14938 	struct bpf_func_state *frame = env->cur_state->frame[0];
14939 	const bool is_subprog = frame->subprogno;
14940 
14941 	/* LSM and struct_ops func-ptr's return type could be "void" */
14942 	if (!is_subprog) {
14943 		switch (prog_type) {
14944 		case BPF_PROG_TYPE_LSM:
14945 			if (prog->expected_attach_type == BPF_LSM_CGROUP)
14946 				/* See below, can be 0 or 0-1 depending on hook. */
14947 				break;
14948 			fallthrough;
14949 		case BPF_PROG_TYPE_STRUCT_OPS:
14950 			if (!prog->aux->attach_func_proto->type)
14951 				return 0;
14952 			break;
14953 		default:
14954 			break;
14955 		}
14956 	}
14957 
14958 	/* eBPF calling convention is such that R0 is used
14959 	 * to return the value from eBPF program.
14960 	 * Make sure that it's readable at this time
14961 	 * of bpf_exit, which means that program wrote
14962 	 * something into it earlier
14963 	 */
14964 	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
14965 	if (err)
14966 		return err;
14967 
14968 	if (is_pointer_value(env, BPF_REG_0)) {
14969 		verbose(env, "R0 leaks addr as return value\n");
14970 		return -EACCES;
14971 	}
14972 
14973 	reg = cur_regs(env) + BPF_REG_0;
14974 
14975 	if (frame->in_async_callback_fn) {
14976 		/* enforce return zero from async callbacks like timer */
14977 		if (reg->type != SCALAR_VALUE) {
14978 			verbose(env, "In async callback the register R0 is not a known value (%s)\n",
14979 				reg_type_str(env, reg->type));
14980 			return -EINVAL;
14981 		}
14982 
14983 		if (!tnum_in(const_0, reg->var_off)) {
14984 			verbose_invalid_scalar(env, reg, &const_0, "async callback", "R0");
14985 			return -EINVAL;
14986 		}
14987 		return 0;
14988 	}
14989 
14990 	if (is_subprog) {
14991 		if (reg->type != SCALAR_VALUE) {
14992 			verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
14993 				reg_type_str(env, reg->type));
14994 			return -EINVAL;
14995 		}
14996 		return 0;
14997 	}
14998 
14999 	switch (prog_type) {
15000 	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
15001 		if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
15002 		    env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
15003 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
15004 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
15005 		    env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
15006 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
15007 			range = tnum_range(1, 1);
15008 		if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
15009 		    env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
15010 			range = tnum_range(0, 3);
15011 		break;
15012 	case BPF_PROG_TYPE_CGROUP_SKB:
15013 		if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
15014 			range = tnum_range(0, 3);
15015 			enforce_attach_type_range = tnum_range(2, 3);
15016 		}
15017 		break;
15018 	case BPF_PROG_TYPE_CGROUP_SOCK:
15019 	case BPF_PROG_TYPE_SOCK_OPS:
15020 	case BPF_PROG_TYPE_CGROUP_DEVICE:
15021 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
15022 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
15023 		break;
15024 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
15025 		if (!env->prog->aux->attach_btf_id)
15026 			return 0;
15027 		range = tnum_const(0);
15028 		break;
15029 	case BPF_PROG_TYPE_TRACING:
15030 		switch (env->prog->expected_attach_type) {
15031 		case BPF_TRACE_FENTRY:
15032 		case BPF_TRACE_FEXIT:
15033 			range = tnum_const(0);
15034 			break;
15035 		case BPF_TRACE_RAW_TP:
15036 		case BPF_MODIFY_RETURN:
15037 			return 0;
15038 		case BPF_TRACE_ITER:
15039 			break;
15040 		default:
15041 			return -ENOTSUPP;
15042 		}
15043 		break;
15044 	case BPF_PROG_TYPE_SK_LOOKUP:
15045 		range = tnum_range(SK_DROP, SK_PASS);
15046 		break;
15047 
15048 	case BPF_PROG_TYPE_LSM:
15049 		if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
15050 			/* Regular BPF_PROG_TYPE_LSM programs can return
15051 			 * any value.
15052 			 */
15053 			return 0;
15054 		}
15055 		if (!env->prog->aux->attach_func_proto->type) {
15056 			/* Make sure programs that attach to void
15057 			 * hooks don't try to modify return value.
15058 			 */
15059 			range = tnum_range(1, 1);
15060 		}
15061 		break;
15062 
15063 	case BPF_PROG_TYPE_NETFILTER:
15064 		range = tnum_range(NF_DROP, NF_ACCEPT);
15065 		break;
15066 	case BPF_PROG_TYPE_EXT:
15067 		/* freplace program can return anything as its return value
15068 		 * depends on the to-be-replaced kernel func or bpf program.
15069 		 */
15070 	default:
15071 		return 0;
15072 	}
15073 
15074 	if (reg->type != SCALAR_VALUE) {
15075 		verbose(env, "At program exit the register R0 is not a known value (%s)\n",
15076 			reg_type_str(env, reg->type));
15077 		return -EINVAL;
15078 	}
15079 
15080 	if (!tnum_in(range, reg->var_off)) {
15081 		verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
15082 		if (prog->expected_attach_type == BPF_LSM_CGROUP &&
15083 		    prog_type == BPF_PROG_TYPE_LSM &&
15084 		    !prog->aux->attach_func_proto->type)
15085 			verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
15086 		return -EINVAL;
15087 	}
15088 
15089 	if (!tnum_is_unknown(enforce_attach_type_range) &&
15090 	    tnum_in(enforce_attach_type_range, reg->var_off))
15091 		env->prog->enforce_expected_attach_type = 1;
15092 	return 0;
15093 }
15094 
15095 /* non-recursive DFS pseudo code
15096  * 1  procedure DFS-iterative(G,v):
15097  * 2      label v as discovered
15098  * 3      let S be a stack
15099  * 4      S.push(v)
15100  * 5      while S is not empty
15101  * 6            t <- S.peek()
15102  * 7            if t is what we're looking for:
15103  * 8                return t
15104  * 9            for all edges e in G.adjacentEdges(t) do
15105  * 10               if edge e is already labelled
15106  * 11                   continue with the next edge
15107  * 12               w <- G.adjacentVertex(t,e)
15108  * 13               if vertex w is not discovered and not explored
15109  * 14                   label e as tree-edge
15110  * 15                   label w as discovered
15111  * 16                   S.push(w)
15112  * 17                   continue at 5
15113  * 18               else if vertex w is discovered
15114  * 19                   label e as back-edge
15115  * 20               else
15116  * 21                   // vertex w is explored
15117  * 22                   label e as forward- or cross-edge
15118  * 23           label t as explored
15119  * 24           S.pop()
15120  *
15121  * convention:
15122  * 0x10 - discovered
15123  * 0x11 - discovered and fall-through edge labelled
15124  * 0x12 - discovered and fall-through and branch edges labelled
15125  * 0x20 - explored
15126  */
15127 
15128 enum {
15129 	DISCOVERED = 0x10,
15130 	EXPLORED = 0x20,
15131 	FALLTHROUGH = 1,
15132 	BRANCH = 2,
15133 };
15134 
15135 static void mark_prune_point(struct bpf_verifier_env *env, int idx)
15136 {
15137 	env->insn_aux_data[idx].prune_point = true;
15138 }
15139 
15140 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)
15141 {
15142 	return env->insn_aux_data[insn_idx].prune_point;
15143 }
15144 
15145 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
15146 {
15147 	env->insn_aux_data[idx].force_checkpoint = true;
15148 }
15149 
15150 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
15151 {
15152 	return env->insn_aux_data[insn_idx].force_checkpoint;
15153 }
15154 
15155 static void mark_calls_callback(struct bpf_verifier_env *env, int idx)
15156 {
15157 	env->insn_aux_data[idx].calls_callback = true;
15158 }
15159 
15160 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)
15161 {
15162 	return env->insn_aux_data[insn_idx].calls_callback;
15163 }
15164 
15165 enum {
15166 	DONE_EXPLORING = 0,
15167 	KEEP_EXPLORING = 1,
15168 };
15169 
15170 /* t, w, e - match pseudo-code above:
15171  * t - index of current instruction
15172  * w - next instruction
15173  * e - edge
15174  */
15175 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
15176 {
15177 	int *insn_stack = env->cfg.insn_stack;
15178 	int *insn_state = env->cfg.insn_state;
15179 
15180 	if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
15181 		return DONE_EXPLORING;
15182 
15183 	if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
15184 		return DONE_EXPLORING;
15185 
15186 	if (w < 0 || w >= env->prog->len) {
15187 		verbose_linfo(env, t, "%d: ", t);
15188 		verbose(env, "jump out of range from insn %d to %d\n", t, w);
15189 		return -EINVAL;
15190 	}
15191 
15192 	if (e == BRANCH) {
15193 		/* mark branch target for state pruning */
15194 		mark_prune_point(env, w);
15195 		mark_jmp_point(env, w);
15196 	}
15197 
15198 	if (insn_state[w] == 0) {
15199 		/* tree-edge */
15200 		insn_state[t] = DISCOVERED | e;
15201 		insn_state[w] = DISCOVERED;
15202 		if (env->cfg.cur_stack >= env->prog->len)
15203 			return -E2BIG;
15204 		insn_stack[env->cfg.cur_stack++] = w;
15205 		return KEEP_EXPLORING;
15206 	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
15207 		if (env->bpf_capable)
15208 			return DONE_EXPLORING;
15209 		verbose_linfo(env, t, "%d: ", t);
15210 		verbose_linfo(env, w, "%d: ", w);
15211 		verbose(env, "back-edge from insn %d to %d\n", t, w);
15212 		return -EINVAL;
15213 	} else if (insn_state[w] == EXPLORED) {
15214 		/* forward- or cross-edge */
15215 		insn_state[t] = DISCOVERED | e;
15216 	} else {
15217 		verbose(env, "insn state internal bug\n");
15218 		return -EFAULT;
15219 	}
15220 	return DONE_EXPLORING;
15221 }
15222 
15223 static int visit_func_call_insn(int t, struct bpf_insn *insns,
15224 				struct bpf_verifier_env *env,
15225 				bool visit_callee)
15226 {
15227 	int ret, insn_sz;
15228 
15229 	insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
15230 	ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
15231 	if (ret)
15232 		return ret;
15233 
15234 	mark_prune_point(env, t + insn_sz);
15235 	/* when we exit from subprog, we need to record non-linear history */
15236 	mark_jmp_point(env, t + insn_sz);
15237 
15238 	if (visit_callee) {
15239 		mark_prune_point(env, t);
15240 		ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
15241 	}
15242 	return ret;
15243 }
15244 
15245 /* Visits the instruction at index t and returns one of the following:
15246  *  < 0 - an error occurred
15247  *  DONE_EXPLORING - the instruction was fully explored
15248  *  KEEP_EXPLORING - there is still work to be done before it is fully explored
15249  */
15250 static int visit_insn(int t, struct bpf_verifier_env *env)
15251 {
15252 	struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
15253 	int ret, off, insn_sz;
15254 
15255 	if (bpf_pseudo_func(insn))
15256 		return visit_func_call_insn(t, insns, env, true);
15257 
15258 	/* All non-branch instructions have a single fall-through edge. */
15259 	if (BPF_CLASS(insn->code) != BPF_JMP &&
15260 	    BPF_CLASS(insn->code) != BPF_JMP32) {
15261 		insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
15262 		return push_insn(t, t + insn_sz, FALLTHROUGH, env);
15263 	}
15264 
15265 	switch (BPF_OP(insn->code)) {
15266 	case BPF_EXIT:
15267 		return DONE_EXPLORING;
15268 
15269 	case BPF_CALL:
15270 		if (insn->src_reg == 0 && insn->imm == BPF_FUNC_timer_set_callback)
15271 			/* Mark this call insn as a prune point to trigger
15272 			 * is_state_visited() check before call itself is
15273 			 * processed by __check_func_call(). Otherwise new
15274 			 * async state will be pushed for further exploration.
15275 			 */
15276 			mark_prune_point(env, t);
15277 		/* For functions that invoke callbacks it is not known how many times
15278 		 * callback would be called. Verifier models callback calling functions
15279 		 * by repeatedly visiting callback bodies and returning to origin call
15280 		 * instruction.
15281 		 * In order to stop such iteration verifier needs to identify when a
15282 		 * state identical some state from a previous iteration is reached.
15283 		 * Check below forces creation of checkpoint before callback calling
15284 		 * instruction to allow search for such identical states.
15285 		 */
15286 		if (is_sync_callback_calling_insn(insn)) {
15287 			mark_calls_callback(env, t);
15288 			mark_force_checkpoint(env, t);
15289 			mark_prune_point(env, t);
15290 			mark_jmp_point(env, t);
15291 		}
15292 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
15293 			struct bpf_kfunc_call_arg_meta meta;
15294 
15295 			ret = fetch_kfunc_meta(env, insn, &meta, NULL);
15296 			if (ret == 0 && is_iter_next_kfunc(&meta)) {
15297 				mark_prune_point(env, t);
15298 				/* Checking and saving state checkpoints at iter_next() call
15299 				 * is crucial for fast convergence of open-coded iterator loop
15300 				 * logic, so we need to force it. If we don't do that,
15301 				 * is_state_visited() might skip saving a checkpoint, causing
15302 				 * unnecessarily long sequence of not checkpointed
15303 				 * instructions and jumps, leading to exhaustion of jump
15304 				 * history buffer, and potentially other undesired outcomes.
15305 				 * It is expected that with correct open-coded iterators
15306 				 * convergence will happen quickly, so we don't run a risk of
15307 				 * exhausting memory.
15308 				 */
15309 				mark_force_checkpoint(env, t);
15310 			}
15311 		}
15312 		return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
15313 
15314 	case BPF_JA:
15315 		if (BPF_SRC(insn->code) != BPF_K)
15316 			return -EINVAL;
15317 
15318 		if (BPF_CLASS(insn->code) == BPF_JMP)
15319 			off = insn->off;
15320 		else
15321 			off = insn->imm;
15322 
15323 		/* unconditional jump with single edge */
15324 		ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
15325 		if (ret)
15326 			return ret;
15327 
15328 		mark_prune_point(env, t + off + 1);
15329 		mark_jmp_point(env, t + off + 1);
15330 
15331 		return ret;
15332 
15333 	default:
15334 		/* conditional jump with two edges */
15335 		mark_prune_point(env, t);
15336 
15337 		ret = push_insn(t, t + 1, FALLTHROUGH, env);
15338 		if (ret)
15339 			return ret;
15340 
15341 		return push_insn(t, t + insn->off + 1, BRANCH, env);
15342 	}
15343 }
15344 
15345 /* non-recursive depth-first-search to detect loops in BPF program
15346  * loop == back-edge in directed graph
15347  */
15348 static int check_cfg(struct bpf_verifier_env *env)
15349 {
15350 	int insn_cnt = env->prog->len;
15351 	int *insn_stack, *insn_state;
15352 	int ret = 0;
15353 	int i;
15354 
15355 	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
15356 	if (!insn_state)
15357 		return -ENOMEM;
15358 
15359 	insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
15360 	if (!insn_stack) {
15361 		kvfree(insn_state);
15362 		return -ENOMEM;
15363 	}
15364 
15365 	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
15366 	insn_stack[0] = 0; /* 0 is the first instruction */
15367 	env->cfg.cur_stack = 1;
15368 
15369 	while (env->cfg.cur_stack > 0) {
15370 		int t = insn_stack[env->cfg.cur_stack - 1];
15371 
15372 		ret = visit_insn(t, env);
15373 		switch (ret) {
15374 		case DONE_EXPLORING:
15375 			insn_state[t] = EXPLORED;
15376 			env->cfg.cur_stack--;
15377 			break;
15378 		case KEEP_EXPLORING:
15379 			break;
15380 		default:
15381 			if (ret > 0) {
15382 				verbose(env, "visit_insn internal bug\n");
15383 				ret = -EFAULT;
15384 			}
15385 			goto err_free;
15386 		}
15387 	}
15388 
15389 	if (env->cfg.cur_stack < 0) {
15390 		verbose(env, "pop stack internal bug\n");
15391 		ret = -EFAULT;
15392 		goto err_free;
15393 	}
15394 
15395 	for (i = 0; i < insn_cnt; i++) {
15396 		struct bpf_insn *insn = &env->prog->insnsi[i];
15397 
15398 		if (insn_state[i] != EXPLORED) {
15399 			verbose(env, "unreachable insn %d\n", i);
15400 			ret = -EINVAL;
15401 			goto err_free;
15402 		}
15403 		if (bpf_is_ldimm64(insn)) {
15404 			if (insn_state[i + 1] != 0) {
15405 				verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
15406 				ret = -EINVAL;
15407 				goto err_free;
15408 			}
15409 			i++; /* skip second half of ldimm64 */
15410 		}
15411 	}
15412 	ret = 0; /* cfg looks good */
15413 
15414 err_free:
15415 	kvfree(insn_state);
15416 	kvfree(insn_stack);
15417 	env->cfg.insn_state = env->cfg.insn_stack = NULL;
15418 	return ret;
15419 }
15420 
15421 static int check_abnormal_return(struct bpf_verifier_env *env)
15422 {
15423 	int i;
15424 
15425 	for (i = 1; i < env->subprog_cnt; i++) {
15426 		if (env->subprog_info[i].has_ld_abs) {
15427 			verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
15428 			return -EINVAL;
15429 		}
15430 		if (env->subprog_info[i].has_tail_call) {
15431 			verbose(env, "tail_call is not allowed in subprogs without BTF\n");
15432 			return -EINVAL;
15433 		}
15434 	}
15435 	return 0;
15436 }
15437 
15438 /* The minimum supported BTF func info size */
15439 #define MIN_BPF_FUNCINFO_SIZE	8
15440 #define MAX_FUNCINFO_REC_SIZE	252
15441 
15442 static int check_btf_func(struct bpf_verifier_env *env,
15443 			  const union bpf_attr *attr,
15444 			  bpfptr_t uattr)
15445 {
15446 	const struct btf_type *type, *func_proto, *ret_type;
15447 	u32 i, nfuncs, urec_size, min_size;
15448 	u32 krec_size = sizeof(struct bpf_func_info);
15449 	struct bpf_func_info *krecord;
15450 	struct bpf_func_info_aux *info_aux = NULL;
15451 	struct bpf_prog *prog;
15452 	const struct btf *btf;
15453 	bpfptr_t urecord;
15454 	u32 prev_offset = 0;
15455 	bool scalar_return;
15456 	int ret = -ENOMEM;
15457 
15458 	nfuncs = attr->func_info_cnt;
15459 	if (!nfuncs) {
15460 		if (check_abnormal_return(env))
15461 			return -EINVAL;
15462 		return 0;
15463 	}
15464 
15465 	if (nfuncs != env->subprog_cnt) {
15466 		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
15467 		return -EINVAL;
15468 	}
15469 
15470 	urec_size = attr->func_info_rec_size;
15471 	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
15472 	    urec_size > MAX_FUNCINFO_REC_SIZE ||
15473 	    urec_size % sizeof(u32)) {
15474 		verbose(env, "invalid func info rec size %u\n", urec_size);
15475 		return -EINVAL;
15476 	}
15477 
15478 	prog = env->prog;
15479 	btf = prog->aux->btf;
15480 
15481 	urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
15482 	min_size = min_t(u32, krec_size, urec_size);
15483 
15484 	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
15485 	if (!krecord)
15486 		return -ENOMEM;
15487 	info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
15488 	if (!info_aux)
15489 		goto err_free;
15490 
15491 	for (i = 0; i < nfuncs; i++) {
15492 		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
15493 		if (ret) {
15494 			if (ret == -E2BIG) {
15495 				verbose(env, "nonzero tailing record in func info");
15496 				/* set the size kernel expects so loader can zero
15497 				 * out the rest of the record.
15498 				 */
15499 				if (copy_to_bpfptr_offset(uattr,
15500 							  offsetof(union bpf_attr, func_info_rec_size),
15501 							  &min_size, sizeof(min_size)))
15502 					ret = -EFAULT;
15503 			}
15504 			goto err_free;
15505 		}
15506 
15507 		if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
15508 			ret = -EFAULT;
15509 			goto err_free;
15510 		}
15511 
15512 		/* check insn_off */
15513 		ret = -EINVAL;
15514 		if (i == 0) {
15515 			if (krecord[i].insn_off) {
15516 				verbose(env,
15517 					"nonzero insn_off %u for the first func info record",
15518 					krecord[i].insn_off);
15519 				goto err_free;
15520 			}
15521 		} else if (krecord[i].insn_off <= prev_offset) {
15522 			verbose(env,
15523 				"same or smaller insn offset (%u) than previous func info record (%u)",
15524 				krecord[i].insn_off, prev_offset);
15525 			goto err_free;
15526 		}
15527 
15528 		if (env->subprog_info[i].start != krecord[i].insn_off) {
15529 			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
15530 			goto err_free;
15531 		}
15532 
15533 		/* check type_id */
15534 		type = btf_type_by_id(btf, krecord[i].type_id);
15535 		if (!type || !btf_type_is_func(type)) {
15536 			verbose(env, "invalid type id %d in func info",
15537 				krecord[i].type_id);
15538 			goto err_free;
15539 		}
15540 		info_aux[i].linkage = BTF_INFO_VLEN(type->info);
15541 
15542 		func_proto = btf_type_by_id(btf, type->type);
15543 		if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
15544 			/* btf_func_check() already verified it during BTF load */
15545 			goto err_free;
15546 		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
15547 		scalar_return =
15548 			btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
15549 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
15550 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
15551 			goto err_free;
15552 		}
15553 		if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
15554 			verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
15555 			goto err_free;
15556 		}
15557 
15558 		prev_offset = krecord[i].insn_off;
15559 		bpfptr_add(&urecord, urec_size);
15560 	}
15561 
15562 	prog->aux->func_info = krecord;
15563 	prog->aux->func_info_cnt = nfuncs;
15564 	prog->aux->func_info_aux = info_aux;
15565 	return 0;
15566 
15567 err_free:
15568 	kvfree(krecord);
15569 	kfree(info_aux);
15570 	return ret;
15571 }
15572 
15573 static void adjust_btf_func(struct bpf_verifier_env *env)
15574 {
15575 	struct bpf_prog_aux *aux = env->prog->aux;
15576 	int i;
15577 
15578 	if (!aux->func_info)
15579 		return;
15580 
15581 	for (i = 0; i < env->subprog_cnt; i++)
15582 		aux->func_info[i].insn_off = env->subprog_info[i].start;
15583 }
15584 
15585 #define MIN_BPF_LINEINFO_SIZE	offsetofend(struct bpf_line_info, line_col)
15586 #define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE
15587 
15588 static int check_btf_line(struct bpf_verifier_env *env,
15589 			  const union bpf_attr *attr,
15590 			  bpfptr_t uattr)
15591 {
15592 	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
15593 	struct bpf_subprog_info *sub;
15594 	struct bpf_line_info *linfo;
15595 	struct bpf_prog *prog;
15596 	const struct btf *btf;
15597 	bpfptr_t ulinfo;
15598 	int err;
15599 
15600 	nr_linfo = attr->line_info_cnt;
15601 	if (!nr_linfo)
15602 		return 0;
15603 	if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
15604 		return -EINVAL;
15605 
15606 	rec_size = attr->line_info_rec_size;
15607 	if (rec_size < MIN_BPF_LINEINFO_SIZE ||
15608 	    rec_size > MAX_LINEINFO_REC_SIZE ||
15609 	    rec_size & (sizeof(u32) - 1))
15610 		return -EINVAL;
15611 
15612 	/* Need to zero it in case the userspace may
15613 	 * pass in a smaller bpf_line_info object.
15614 	 */
15615 	linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
15616 			 GFP_KERNEL | __GFP_NOWARN);
15617 	if (!linfo)
15618 		return -ENOMEM;
15619 
15620 	prog = env->prog;
15621 	btf = prog->aux->btf;
15622 
15623 	s = 0;
15624 	sub = env->subprog_info;
15625 	ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
15626 	expected_size = sizeof(struct bpf_line_info);
15627 	ncopy = min_t(u32, expected_size, rec_size);
15628 	for (i = 0; i < nr_linfo; i++) {
15629 		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
15630 		if (err) {
15631 			if (err == -E2BIG) {
15632 				verbose(env, "nonzero tailing record in line_info");
15633 				if (copy_to_bpfptr_offset(uattr,
15634 							  offsetof(union bpf_attr, line_info_rec_size),
15635 							  &expected_size, sizeof(expected_size)))
15636 					err = -EFAULT;
15637 			}
15638 			goto err_free;
15639 		}
15640 
15641 		if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
15642 			err = -EFAULT;
15643 			goto err_free;
15644 		}
15645 
15646 		/*
15647 		 * Check insn_off to ensure
15648 		 * 1) strictly increasing AND
15649 		 * 2) bounded by prog->len
15650 		 *
15651 		 * The linfo[0].insn_off == 0 check logically falls into
15652 		 * the later "missing bpf_line_info for func..." case
15653 		 * because the first linfo[0].insn_off must be the
15654 		 * first sub also and the first sub must have
15655 		 * subprog_info[0].start == 0.
15656 		 */
15657 		if ((i && linfo[i].insn_off <= prev_offset) ||
15658 		    linfo[i].insn_off >= prog->len) {
15659 			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
15660 				i, linfo[i].insn_off, prev_offset,
15661 				prog->len);
15662 			err = -EINVAL;
15663 			goto err_free;
15664 		}
15665 
15666 		if (!prog->insnsi[linfo[i].insn_off].code) {
15667 			verbose(env,
15668 				"Invalid insn code at line_info[%u].insn_off\n",
15669 				i);
15670 			err = -EINVAL;
15671 			goto err_free;
15672 		}
15673 
15674 		if (!btf_name_by_offset(btf, linfo[i].line_off) ||
15675 		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
15676 			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
15677 			err = -EINVAL;
15678 			goto err_free;
15679 		}
15680 
15681 		if (s != env->subprog_cnt) {
15682 			if (linfo[i].insn_off == sub[s].start) {
15683 				sub[s].linfo_idx = i;
15684 				s++;
15685 			} else if (sub[s].start < linfo[i].insn_off) {
15686 				verbose(env, "missing bpf_line_info for func#%u\n", s);
15687 				err = -EINVAL;
15688 				goto err_free;
15689 			}
15690 		}
15691 
15692 		prev_offset = linfo[i].insn_off;
15693 		bpfptr_add(&ulinfo, rec_size);
15694 	}
15695 
15696 	if (s != env->subprog_cnt) {
15697 		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
15698 			env->subprog_cnt - s, s);
15699 		err = -EINVAL;
15700 		goto err_free;
15701 	}
15702 
15703 	prog->aux->linfo = linfo;
15704 	prog->aux->nr_linfo = nr_linfo;
15705 
15706 	return 0;
15707 
15708 err_free:
15709 	kvfree(linfo);
15710 	return err;
15711 }
15712 
15713 #define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)
15714 #define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE
15715 
15716 static int check_core_relo(struct bpf_verifier_env *env,
15717 			   const union bpf_attr *attr,
15718 			   bpfptr_t uattr)
15719 {
15720 	u32 i, nr_core_relo, ncopy, expected_size, rec_size;
15721 	struct bpf_core_relo core_relo = {};
15722 	struct bpf_prog *prog = env->prog;
15723 	const struct btf *btf = prog->aux->btf;
15724 	struct bpf_core_ctx ctx = {
15725 		.log = &env->log,
15726 		.btf = btf,
15727 	};
15728 	bpfptr_t u_core_relo;
15729 	int err;
15730 
15731 	nr_core_relo = attr->core_relo_cnt;
15732 	if (!nr_core_relo)
15733 		return 0;
15734 	if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
15735 		return -EINVAL;
15736 
15737 	rec_size = attr->core_relo_rec_size;
15738 	if (rec_size < MIN_CORE_RELO_SIZE ||
15739 	    rec_size > MAX_CORE_RELO_SIZE ||
15740 	    rec_size % sizeof(u32))
15741 		return -EINVAL;
15742 
15743 	u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
15744 	expected_size = sizeof(struct bpf_core_relo);
15745 	ncopy = min_t(u32, expected_size, rec_size);
15746 
15747 	/* Unlike func_info and line_info, copy and apply each CO-RE
15748 	 * relocation record one at a time.
15749 	 */
15750 	for (i = 0; i < nr_core_relo; i++) {
15751 		/* future proofing when sizeof(bpf_core_relo) changes */
15752 		err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
15753 		if (err) {
15754 			if (err == -E2BIG) {
15755 				verbose(env, "nonzero tailing record in core_relo");
15756 				if (copy_to_bpfptr_offset(uattr,
15757 							  offsetof(union bpf_attr, core_relo_rec_size),
15758 							  &expected_size, sizeof(expected_size)))
15759 					err = -EFAULT;
15760 			}
15761 			break;
15762 		}
15763 
15764 		if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
15765 			err = -EFAULT;
15766 			break;
15767 		}
15768 
15769 		if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
15770 			verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
15771 				i, core_relo.insn_off, prog->len);
15772 			err = -EINVAL;
15773 			break;
15774 		}
15775 
15776 		err = bpf_core_apply(&ctx, &core_relo, i,
15777 				     &prog->insnsi[core_relo.insn_off / 8]);
15778 		if (err)
15779 			break;
15780 		bpfptr_add(&u_core_relo, rec_size);
15781 	}
15782 	return err;
15783 }
15784 
15785 static int check_btf_info(struct bpf_verifier_env *env,
15786 			  const union bpf_attr *attr,
15787 			  bpfptr_t uattr)
15788 {
15789 	struct btf *btf;
15790 	int err;
15791 
15792 	if (!attr->func_info_cnt && !attr->line_info_cnt) {
15793 		if (check_abnormal_return(env))
15794 			return -EINVAL;
15795 		return 0;
15796 	}
15797 
15798 	btf = btf_get_by_fd(attr->prog_btf_fd);
15799 	if (IS_ERR(btf))
15800 		return PTR_ERR(btf);
15801 	if (btf_is_kernel(btf)) {
15802 		btf_put(btf);
15803 		return -EACCES;
15804 	}
15805 	env->prog->aux->btf = btf;
15806 
15807 	err = check_btf_func(env, attr, uattr);
15808 	if (err)
15809 		return err;
15810 
15811 	err = check_btf_line(env, attr, uattr);
15812 	if (err)
15813 		return err;
15814 
15815 	err = check_core_relo(env, attr, uattr);
15816 	if (err)
15817 		return err;
15818 
15819 	return 0;
15820 }
15821 
15822 /* check %cur's range satisfies %old's */
15823 static bool range_within(struct bpf_reg_state *old,
15824 			 struct bpf_reg_state *cur)
15825 {
15826 	return old->umin_value <= cur->umin_value &&
15827 	       old->umax_value >= cur->umax_value &&
15828 	       old->smin_value <= cur->smin_value &&
15829 	       old->smax_value >= cur->smax_value &&
15830 	       old->u32_min_value <= cur->u32_min_value &&
15831 	       old->u32_max_value >= cur->u32_max_value &&
15832 	       old->s32_min_value <= cur->s32_min_value &&
15833 	       old->s32_max_value >= cur->s32_max_value;
15834 }
15835 
15836 /* If in the old state two registers had the same id, then they need to have
15837  * the same id in the new state as well.  But that id could be different from
15838  * the old state, so we need to track the mapping from old to new ids.
15839  * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
15840  * regs with old id 5 must also have new id 9 for the new state to be safe.  But
15841  * regs with a different old id could still have new id 9, we don't care about
15842  * that.
15843  * So we look through our idmap to see if this old id has been seen before.  If
15844  * so, we require the new id to match; otherwise, we add the id pair to the map.
15845  */
15846 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
15847 {
15848 	struct bpf_id_pair *map = idmap->map;
15849 	unsigned int i;
15850 
15851 	/* either both IDs should be set or both should be zero */
15852 	if (!!old_id != !!cur_id)
15853 		return false;
15854 
15855 	if (old_id == 0) /* cur_id == 0 as well */
15856 		return true;
15857 
15858 	for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
15859 		if (!map[i].old) {
15860 			/* Reached an empty slot; haven't seen this id before */
15861 			map[i].old = old_id;
15862 			map[i].cur = cur_id;
15863 			return true;
15864 		}
15865 		if (map[i].old == old_id)
15866 			return map[i].cur == cur_id;
15867 		if (map[i].cur == cur_id)
15868 			return false;
15869 	}
15870 	/* We ran out of idmap slots, which should be impossible */
15871 	WARN_ON_ONCE(1);
15872 	return false;
15873 }
15874 
15875 /* Similar to check_ids(), but allocate a unique temporary ID
15876  * for 'old_id' or 'cur_id' of zero.
15877  * This makes pairs like '0 vs unique ID', 'unique ID vs 0' valid.
15878  */
15879 static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap)
15880 {
15881 	old_id = old_id ? old_id : ++idmap->tmp_id_gen;
15882 	cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen;
15883 
15884 	return check_ids(old_id, cur_id, idmap);
15885 }
15886 
15887 static void clean_func_state(struct bpf_verifier_env *env,
15888 			     struct bpf_func_state *st)
15889 {
15890 	enum bpf_reg_liveness live;
15891 	int i, j;
15892 
15893 	for (i = 0; i < BPF_REG_FP; i++) {
15894 		live = st->regs[i].live;
15895 		/* liveness must not touch this register anymore */
15896 		st->regs[i].live |= REG_LIVE_DONE;
15897 		if (!(live & REG_LIVE_READ))
15898 			/* since the register is unused, clear its state
15899 			 * to make further comparison simpler
15900 			 */
15901 			__mark_reg_not_init(env, &st->regs[i]);
15902 	}
15903 
15904 	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
15905 		live = st->stack[i].spilled_ptr.live;
15906 		/* liveness must not touch this stack slot anymore */
15907 		st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
15908 		if (!(live & REG_LIVE_READ)) {
15909 			__mark_reg_not_init(env, &st->stack[i].spilled_ptr);
15910 			for (j = 0; j < BPF_REG_SIZE; j++)
15911 				st->stack[i].slot_type[j] = STACK_INVALID;
15912 		}
15913 	}
15914 }
15915 
15916 static void clean_verifier_state(struct bpf_verifier_env *env,
15917 				 struct bpf_verifier_state *st)
15918 {
15919 	int i;
15920 
15921 	if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
15922 		/* all regs in this state in all frames were already marked */
15923 		return;
15924 
15925 	for (i = 0; i <= st->curframe; i++)
15926 		clean_func_state(env, st->frame[i]);
15927 }
15928 
15929 /* the parentage chains form a tree.
15930  * the verifier states are added to state lists at given insn and
15931  * pushed into state stack for future exploration.
15932  * when the verifier reaches bpf_exit insn some of the verifer states
15933  * stored in the state lists have their final liveness state already,
15934  * but a lot of states will get revised from liveness point of view when
15935  * the verifier explores other branches.
15936  * Example:
15937  * 1: r0 = 1
15938  * 2: if r1 == 100 goto pc+1
15939  * 3: r0 = 2
15940  * 4: exit
15941  * when the verifier reaches exit insn the register r0 in the state list of
15942  * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
15943  * of insn 2 and goes exploring further. At the insn 4 it will walk the
15944  * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
15945  *
15946  * Since the verifier pushes the branch states as it sees them while exploring
15947  * the program the condition of walking the branch instruction for the second
15948  * time means that all states below this branch were already explored and
15949  * their final liveness marks are already propagated.
15950  * Hence when the verifier completes the search of state list in is_state_visited()
15951  * we can call this clean_live_states() function to mark all liveness states
15952  * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
15953  * will not be used.
15954  * This function also clears the registers and stack for states that !READ
15955  * to simplify state merging.
15956  *
15957  * Important note here that walking the same branch instruction in the callee
15958  * doesn't meant that the states are DONE. The verifier has to compare
15959  * the callsites
15960  */
15961 static void clean_live_states(struct bpf_verifier_env *env, int insn,
15962 			      struct bpf_verifier_state *cur)
15963 {
15964 	struct bpf_verifier_state_list *sl;
15965 
15966 	sl = *explored_state(env, insn);
15967 	while (sl) {
15968 		if (sl->state.branches)
15969 			goto next;
15970 		if (sl->state.insn_idx != insn ||
15971 		    !same_callsites(&sl->state, cur))
15972 			goto next;
15973 		clean_verifier_state(env, &sl->state);
15974 next:
15975 		sl = sl->next;
15976 	}
15977 }
15978 
15979 static bool regs_exact(const struct bpf_reg_state *rold,
15980 		       const struct bpf_reg_state *rcur,
15981 		       struct bpf_idmap *idmap)
15982 {
15983 	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
15984 	       check_ids(rold->id, rcur->id, idmap) &&
15985 	       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
15986 }
15987 
15988 /* Returns true if (rold safe implies rcur safe) */
15989 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
15990 		    struct bpf_reg_state *rcur, struct bpf_idmap *idmap, bool exact)
15991 {
15992 	if (exact)
15993 		return regs_exact(rold, rcur, idmap);
15994 
15995 	if (!(rold->live & REG_LIVE_READ))
15996 		/* explored state didn't use this */
15997 		return true;
15998 	if (rold->type == NOT_INIT)
15999 		/* explored state can't have used this */
16000 		return true;
16001 	if (rcur->type == NOT_INIT)
16002 		return false;
16003 
16004 	/* Enforce that register types have to match exactly, including their
16005 	 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general
16006 	 * rule.
16007 	 *
16008 	 * One can make a point that using a pointer register as unbounded
16009 	 * SCALAR would be technically acceptable, but this could lead to
16010 	 * pointer leaks because scalars are allowed to leak while pointers
16011 	 * are not. We could make this safe in special cases if root is
16012 	 * calling us, but it's probably not worth the hassle.
16013 	 *
16014 	 * Also, register types that are *not* MAYBE_NULL could technically be
16015 	 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE
16016 	 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point
16017 	 * to the same map).
16018 	 * However, if the old MAYBE_NULL register then got NULL checked,
16019 	 * doing so could have affected others with the same id, and we can't
16020 	 * check for that because we lost the id when we converted to
16021 	 * a non-MAYBE_NULL variant.
16022 	 * So, as a general rule we don't allow mixing MAYBE_NULL and
16023 	 * non-MAYBE_NULL registers as well.
16024 	 */
16025 	if (rold->type != rcur->type)
16026 		return false;
16027 
16028 	switch (base_type(rold->type)) {
16029 	case SCALAR_VALUE:
16030 		if (env->explore_alu_limits) {
16031 			/* explore_alu_limits disables tnum_in() and range_within()
16032 			 * logic and requires everything to be strict
16033 			 */
16034 			return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
16035 			       check_scalar_ids(rold->id, rcur->id, idmap);
16036 		}
16037 		if (!rold->precise)
16038 			return true;
16039 		/* Why check_ids() for scalar registers?
16040 		 *
16041 		 * Consider the following BPF code:
16042 		 *   1: r6 = ... unbound scalar, ID=a ...
16043 		 *   2: r7 = ... unbound scalar, ID=b ...
16044 		 *   3: if (r6 > r7) goto +1
16045 		 *   4: r6 = r7
16046 		 *   5: if (r6 > X) goto ...
16047 		 *   6: ... memory operation using r7 ...
16048 		 *
16049 		 * First verification path is [1-6]:
16050 		 * - at (4) same bpf_reg_state::id (b) would be assigned to r6 and r7;
16051 		 * - at (5) r6 would be marked <= X, find_equal_scalars() would also mark
16052 		 *   r7 <= X, because r6 and r7 share same id.
16053 		 * Next verification path is [1-4, 6].
16054 		 *
16055 		 * Instruction (6) would be reached in two states:
16056 		 *   I.  r6{.id=b}, r7{.id=b} via path 1-6;
16057 		 *   II. r6{.id=a}, r7{.id=b} via path 1-4, 6.
16058 		 *
16059 		 * Use check_ids() to distinguish these states.
16060 		 * ---
16061 		 * Also verify that new value satisfies old value range knowledge.
16062 		 */
16063 		return range_within(rold, rcur) &&
16064 		       tnum_in(rold->var_off, rcur->var_off) &&
16065 		       check_scalar_ids(rold->id, rcur->id, idmap);
16066 	case PTR_TO_MAP_KEY:
16067 	case PTR_TO_MAP_VALUE:
16068 	case PTR_TO_MEM:
16069 	case PTR_TO_BUF:
16070 	case PTR_TO_TP_BUFFER:
16071 		/* If the new min/max/var_off satisfy the old ones and
16072 		 * everything else matches, we are OK.
16073 		 */
16074 		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 &&
16075 		       range_within(rold, rcur) &&
16076 		       tnum_in(rold->var_off, rcur->var_off) &&
16077 		       check_ids(rold->id, rcur->id, idmap) &&
16078 		       check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap);
16079 	case PTR_TO_PACKET_META:
16080 	case PTR_TO_PACKET:
16081 		/* We must have at least as much range as the old ptr
16082 		 * did, so that any accesses which were safe before are
16083 		 * still safe.  This is true even if old range < old off,
16084 		 * since someone could have accessed through (ptr - k), or
16085 		 * even done ptr -= k in a register, to get a safe access.
16086 		 */
16087 		if (rold->range > rcur->range)
16088 			return false;
16089 		/* If the offsets don't match, we can't trust our alignment;
16090 		 * nor can we be sure that we won't fall out of range.
16091 		 */
16092 		if (rold->off != rcur->off)
16093 			return false;
16094 		/* id relations must be preserved */
16095 		if (!check_ids(rold->id, rcur->id, idmap))
16096 			return false;
16097 		/* new val must satisfy old val knowledge */
16098 		return range_within(rold, rcur) &&
16099 		       tnum_in(rold->var_off, rcur->var_off);
16100 	case PTR_TO_STACK:
16101 		/* two stack pointers are equal only if they're pointing to
16102 		 * the same stack frame, since fp-8 in foo != fp-8 in bar
16103 		 */
16104 		return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
16105 	default:
16106 		return regs_exact(rold, rcur, idmap);
16107 	}
16108 }
16109 
16110 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
16111 		      struct bpf_func_state *cur, struct bpf_idmap *idmap, bool exact)
16112 {
16113 	int i, spi;
16114 
16115 	/* walk slots of the explored stack and ignore any additional
16116 	 * slots in the current stack, since explored(safe) state
16117 	 * didn't use them
16118 	 */
16119 	for (i = 0; i < old->allocated_stack; i++) {
16120 		struct bpf_reg_state *old_reg, *cur_reg;
16121 
16122 		spi = i / BPF_REG_SIZE;
16123 
16124 		if (exact &&
16125 		    old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
16126 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
16127 			return false;
16128 
16129 		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ) && !exact) {
16130 			i += BPF_REG_SIZE - 1;
16131 			/* explored state didn't use this */
16132 			continue;
16133 		}
16134 
16135 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
16136 			continue;
16137 
16138 		if (env->allow_uninit_stack &&
16139 		    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC)
16140 			continue;
16141 
16142 		/* explored stack has more populated slots than current stack
16143 		 * and these slots were used
16144 		 */
16145 		if (i >= cur->allocated_stack)
16146 			return false;
16147 
16148 		/* if old state was safe with misc data in the stack
16149 		 * it will be safe with zero-initialized stack.
16150 		 * The opposite is not true
16151 		 */
16152 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
16153 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
16154 			continue;
16155 		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
16156 		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
16157 			/* Ex: old explored (safe) state has STACK_SPILL in
16158 			 * this stack slot, but current has STACK_MISC ->
16159 			 * this verifier states are not equivalent,
16160 			 * return false to continue verification of this path
16161 			 */
16162 			return false;
16163 		if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
16164 			continue;
16165 		/* Both old and cur are having same slot_type */
16166 		switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) {
16167 		case STACK_SPILL:
16168 			/* when explored and current stack slot are both storing
16169 			 * spilled registers, check that stored pointers types
16170 			 * are the same as well.
16171 			 * Ex: explored safe path could have stored
16172 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
16173 			 * but current path has stored:
16174 			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
16175 			 * such verifier states are not equivalent.
16176 			 * return false to continue verification of this path
16177 			 */
16178 			if (!regsafe(env, &old->stack[spi].spilled_ptr,
16179 				     &cur->stack[spi].spilled_ptr, idmap, exact))
16180 				return false;
16181 			break;
16182 		case STACK_DYNPTR:
16183 			old_reg = &old->stack[spi].spilled_ptr;
16184 			cur_reg = &cur->stack[spi].spilled_ptr;
16185 			if (old_reg->dynptr.type != cur_reg->dynptr.type ||
16186 			    old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
16187 			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
16188 				return false;
16189 			break;
16190 		case STACK_ITER:
16191 			old_reg = &old->stack[spi].spilled_ptr;
16192 			cur_reg = &cur->stack[spi].spilled_ptr;
16193 			/* iter.depth is not compared between states as it
16194 			 * doesn't matter for correctness and would otherwise
16195 			 * prevent convergence; we maintain it only to prevent
16196 			 * infinite loop check triggering, see
16197 			 * iter_active_depths_differ()
16198 			 */
16199 			if (old_reg->iter.btf != cur_reg->iter.btf ||
16200 			    old_reg->iter.btf_id != cur_reg->iter.btf_id ||
16201 			    old_reg->iter.state != cur_reg->iter.state ||
16202 			    /* ignore {old_reg,cur_reg}->iter.depth, see above */
16203 			    !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap))
16204 				return false;
16205 			break;
16206 		case STACK_MISC:
16207 		case STACK_ZERO:
16208 		case STACK_INVALID:
16209 			continue;
16210 		/* Ensure that new unhandled slot types return false by default */
16211 		default:
16212 			return false;
16213 		}
16214 	}
16215 	return true;
16216 }
16217 
16218 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur,
16219 		    struct bpf_idmap *idmap)
16220 {
16221 	int i;
16222 
16223 	if (old->acquired_refs != cur->acquired_refs)
16224 		return false;
16225 
16226 	for (i = 0; i < old->acquired_refs; i++) {
16227 		if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap))
16228 			return false;
16229 	}
16230 
16231 	return true;
16232 }
16233 
16234 /* compare two verifier states
16235  *
16236  * all states stored in state_list are known to be valid, since
16237  * verifier reached 'bpf_exit' instruction through them
16238  *
16239  * this function is called when verifier exploring different branches of
16240  * execution popped from the state stack. If it sees an old state that has
16241  * more strict register state and more strict stack state then this execution
16242  * branch doesn't need to be explored further, since verifier already
16243  * concluded that more strict state leads to valid finish.
16244  *
16245  * Therefore two states are equivalent if register state is more conservative
16246  * and explored stack state is more conservative than the current one.
16247  * Example:
16248  *       explored                   current
16249  * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
16250  * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
16251  *
16252  * In other words if current stack state (one being explored) has more
16253  * valid slots than old one that already passed validation, it means
16254  * the verifier can stop exploring and conclude that current state is valid too
16255  *
16256  * Similarly with registers. If explored state has register type as invalid
16257  * whereas register type in current state is meaningful, it means that
16258  * the current state will reach 'bpf_exit' instruction safely
16259  */
16260 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
16261 			      struct bpf_func_state *cur, bool exact)
16262 {
16263 	int i;
16264 
16265 	if (old->callback_depth > cur->callback_depth)
16266 		return false;
16267 
16268 	for (i = 0; i < MAX_BPF_REG; i++)
16269 		if (!regsafe(env, &old->regs[i], &cur->regs[i],
16270 			     &env->idmap_scratch, exact))
16271 			return false;
16272 
16273 	if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))
16274 		return false;
16275 
16276 	if (!refsafe(old, cur, &env->idmap_scratch))
16277 		return false;
16278 
16279 	return true;
16280 }
16281 
16282 static void reset_idmap_scratch(struct bpf_verifier_env *env)
16283 {
16284 	env->idmap_scratch.tmp_id_gen = env->id_gen;
16285 	memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map));
16286 }
16287 
16288 static bool states_equal(struct bpf_verifier_env *env,
16289 			 struct bpf_verifier_state *old,
16290 			 struct bpf_verifier_state *cur,
16291 			 bool exact)
16292 {
16293 	int i;
16294 
16295 	if (old->curframe != cur->curframe)
16296 		return false;
16297 
16298 	reset_idmap_scratch(env);
16299 
16300 	/* Verification state from speculative execution simulation
16301 	 * must never prune a non-speculative execution one.
16302 	 */
16303 	if (old->speculative && !cur->speculative)
16304 		return false;
16305 
16306 	if (old->active_lock.ptr != cur->active_lock.ptr)
16307 		return false;
16308 
16309 	/* Old and cur active_lock's have to be either both present
16310 	 * or both absent.
16311 	 */
16312 	if (!!old->active_lock.id != !!cur->active_lock.id)
16313 		return false;
16314 
16315 	if (old->active_lock.id &&
16316 	    !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch))
16317 		return false;
16318 
16319 	if (old->active_rcu_lock != cur->active_rcu_lock)
16320 		return false;
16321 
16322 	/* for states to be equal callsites have to be the same
16323 	 * and all frame states need to be equivalent
16324 	 */
16325 	for (i = 0; i <= old->curframe; i++) {
16326 		if (old->frame[i]->callsite != cur->frame[i]->callsite)
16327 			return false;
16328 		if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))
16329 			return false;
16330 	}
16331 	return true;
16332 }
16333 
16334 /* Return 0 if no propagation happened. Return negative error code if error
16335  * happened. Otherwise, return the propagated bit.
16336  */
16337 static int propagate_liveness_reg(struct bpf_verifier_env *env,
16338 				  struct bpf_reg_state *reg,
16339 				  struct bpf_reg_state *parent_reg)
16340 {
16341 	u8 parent_flag = parent_reg->live & REG_LIVE_READ;
16342 	u8 flag = reg->live & REG_LIVE_READ;
16343 	int err;
16344 
16345 	/* When comes here, read flags of PARENT_REG or REG could be any of
16346 	 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need
16347 	 * of propagation if PARENT_REG has strongest REG_LIVE_READ64.
16348 	 */
16349 	if (parent_flag == REG_LIVE_READ64 ||
16350 	    /* Or if there is no read flag from REG. */
16351 	    !flag ||
16352 	    /* Or if the read flag from REG is the same as PARENT_REG. */
16353 	    parent_flag == flag)
16354 		return 0;
16355 
16356 	err = mark_reg_read(env, reg, parent_reg, flag);
16357 	if (err)
16358 		return err;
16359 
16360 	return flag;
16361 }
16362 
16363 /* A write screens off any subsequent reads; but write marks come from the
16364  * straight-line code between a state and its parent.  When we arrive at an
16365  * equivalent state (jump target or such) we didn't arrive by the straight-line
16366  * code, so read marks in the state must propagate to the parent regardless
16367  * of the state's write marks. That's what 'parent == state->parent' comparison
16368  * in mark_reg_read() is for.
16369  */
16370 static int propagate_liveness(struct bpf_verifier_env *env,
16371 			      const struct bpf_verifier_state *vstate,
16372 			      struct bpf_verifier_state *vparent)
16373 {
16374 	struct bpf_reg_state *state_reg, *parent_reg;
16375 	struct bpf_func_state *state, *parent;
16376 	int i, frame, err = 0;
16377 
16378 	if (vparent->curframe != vstate->curframe) {
16379 		WARN(1, "propagate_live: parent frame %d current frame %d\n",
16380 		     vparent->curframe, vstate->curframe);
16381 		return -EFAULT;
16382 	}
16383 	/* Propagate read liveness of registers... */
16384 	BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
16385 	for (frame = 0; frame <= vstate->curframe; frame++) {
16386 		parent = vparent->frame[frame];
16387 		state = vstate->frame[frame];
16388 		parent_reg = parent->regs;
16389 		state_reg = state->regs;
16390 		/* We don't need to worry about FP liveness, it's read-only */
16391 		for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
16392 			err = propagate_liveness_reg(env, &state_reg[i],
16393 						     &parent_reg[i]);
16394 			if (err < 0)
16395 				return err;
16396 			if (err == REG_LIVE_READ64)
16397 				mark_insn_zext(env, &parent_reg[i]);
16398 		}
16399 
16400 		/* Propagate stack slots. */
16401 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
16402 			    i < parent->allocated_stack / BPF_REG_SIZE; i++) {
16403 			parent_reg = &parent->stack[i].spilled_ptr;
16404 			state_reg = &state->stack[i].spilled_ptr;
16405 			err = propagate_liveness_reg(env, state_reg,
16406 						     parent_reg);
16407 			if (err < 0)
16408 				return err;
16409 		}
16410 	}
16411 	return 0;
16412 }
16413 
16414 /* find precise scalars in the previous equivalent state and
16415  * propagate them into the current state
16416  */
16417 static int propagate_precision(struct bpf_verifier_env *env,
16418 			       const struct bpf_verifier_state *old)
16419 {
16420 	struct bpf_reg_state *state_reg;
16421 	struct bpf_func_state *state;
16422 	int i, err = 0, fr;
16423 	bool first;
16424 
16425 	for (fr = old->curframe; fr >= 0; fr--) {
16426 		state = old->frame[fr];
16427 		state_reg = state->regs;
16428 		first = true;
16429 		for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
16430 			if (state_reg->type != SCALAR_VALUE ||
16431 			    !state_reg->precise ||
16432 			    !(state_reg->live & REG_LIVE_READ))
16433 				continue;
16434 			if (env->log.level & BPF_LOG_LEVEL2) {
16435 				if (first)
16436 					verbose(env, "frame %d: propagating r%d", fr, i);
16437 				else
16438 					verbose(env, ",r%d", i);
16439 			}
16440 			bt_set_frame_reg(&env->bt, fr, i);
16441 			first = false;
16442 		}
16443 
16444 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
16445 			if (!is_spilled_reg(&state->stack[i]))
16446 				continue;
16447 			state_reg = &state->stack[i].spilled_ptr;
16448 			if (state_reg->type != SCALAR_VALUE ||
16449 			    !state_reg->precise ||
16450 			    !(state_reg->live & REG_LIVE_READ))
16451 				continue;
16452 			if (env->log.level & BPF_LOG_LEVEL2) {
16453 				if (first)
16454 					verbose(env, "frame %d: propagating fp%d",
16455 						fr, (-i - 1) * BPF_REG_SIZE);
16456 				else
16457 					verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE);
16458 			}
16459 			bt_set_frame_slot(&env->bt, fr, i);
16460 			first = false;
16461 		}
16462 		if (!first)
16463 			verbose(env, "\n");
16464 	}
16465 
16466 	err = mark_chain_precision_batch(env);
16467 	if (err < 0)
16468 		return err;
16469 
16470 	return 0;
16471 }
16472 
16473 static bool states_maybe_looping(struct bpf_verifier_state *old,
16474 				 struct bpf_verifier_state *cur)
16475 {
16476 	struct bpf_func_state *fold, *fcur;
16477 	int i, fr = cur->curframe;
16478 
16479 	if (old->curframe != fr)
16480 		return false;
16481 
16482 	fold = old->frame[fr];
16483 	fcur = cur->frame[fr];
16484 	for (i = 0; i < MAX_BPF_REG; i++)
16485 		if (memcmp(&fold->regs[i], &fcur->regs[i],
16486 			   offsetof(struct bpf_reg_state, parent)))
16487 			return false;
16488 	return true;
16489 }
16490 
16491 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)
16492 {
16493 	return env->insn_aux_data[insn_idx].is_iter_next;
16494 }
16495 
16496 /* is_state_visited() handles iter_next() (see process_iter_next_call() for
16497  * terminology) calls specially: as opposed to bounded BPF loops, it *expects*
16498  * states to match, which otherwise would look like an infinite loop. So while
16499  * iter_next() calls are taken care of, we still need to be careful and
16500  * prevent erroneous and too eager declaration of "ininite loop", when
16501  * iterators are involved.
16502  *
16503  * Here's a situation in pseudo-BPF assembly form:
16504  *
16505  *   0: again:                          ; set up iter_next() call args
16506  *   1:   r1 = &it                      ; <CHECKPOINT HERE>
16507  *   2:   call bpf_iter_num_next        ; this is iter_next() call
16508  *   3:   if r0 == 0 goto done
16509  *   4:   ... something useful here ...
16510  *   5:   goto again                    ; another iteration
16511  *   6: done:
16512  *   7:   r1 = &it
16513  *   8:   call bpf_iter_num_destroy     ; clean up iter state
16514  *   9:   exit
16515  *
16516  * This is a typical loop. Let's assume that we have a prune point at 1:,
16517  * before we get to `call bpf_iter_num_next` (e.g., because of that `goto
16518  * again`, assuming other heuristics don't get in a way).
16519  *
16520  * When we first time come to 1:, let's say we have some state X. We proceed
16521  * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit.
16522  * Now we come back to validate that forked ACTIVE state. We proceed through
16523  * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we
16524  * are converging. But the problem is that we don't know that yet, as this
16525  * convergence has to happen at iter_next() call site only. So if nothing is
16526  * done, at 1: verifier will use bounded loop logic and declare infinite
16527  * looping (and would be *technically* correct, if not for iterator's
16528  * "eventual sticky NULL" contract, see process_iter_next_call()). But we
16529  * don't want that. So what we do in process_iter_next_call() when we go on
16530  * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's
16531  * a different iteration. So when we suspect an infinite loop, we additionally
16532  * check if any of the *ACTIVE* iterator states depths differ. If yes, we
16533  * pretend we are not looping and wait for next iter_next() call.
16534  *
16535  * This only applies to ACTIVE state. In DRAINED state we don't expect to
16536  * loop, because that would actually mean infinite loop, as DRAINED state is
16537  * "sticky", and so we'll keep returning into the same instruction with the
16538  * same state (at least in one of possible code paths).
16539  *
16540  * This approach allows to keep infinite loop heuristic even in the face of
16541  * active iterator. E.g., C snippet below is and will be detected as
16542  * inifintely looping:
16543  *
16544  *   struct bpf_iter_num it;
16545  *   int *p, x;
16546  *
16547  *   bpf_iter_num_new(&it, 0, 10);
16548  *   while ((p = bpf_iter_num_next(&t))) {
16549  *       x = p;
16550  *       while (x--) {} // <<-- infinite loop here
16551  *   }
16552  *
16553  */
16554 static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur)
16555 {
16556 	struct bpf_reg_state *slot, *cur_slot;
16557 	struct bpf_func_state *state;
16558 	int i, fr;
16559 
16560 	for (fr = old->curframe; fr >= 0; fr--) {
16561 		state = old->frame[fr];
16562 		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
16563 			if (state->stack[i].slot_type[0] != STACK_ITER)
16564 				continue;
16565 
16566 			slot = &state->stack[i].spilled_ptr;
16567 			if (slot->iter.state != BPF_ITER_STATE_ACTIVE)
16568 				continue;
16569 
16570 			cur_slot = &cur->frame[fr]->stack[i].spilled_ptr;
16571 			if (cur_slot->iter.depth != slot->iter.depth)
16572 				return true;
16573 		}
16574 	}
16575 	return false;
16576 }
16577 
16578 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
16579 {
16580 	struct bpf_verifier_state_list *new_sl;
16581 	struct bpf_verifier_state_list *sl, **pprev;
16582 	struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry;
16583 	int i, j, n, err, states_cnt = 0;
16584 	bool force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx);
16585 	bool add_new_state = force_new_state;
16586 	bool force_exact;
16587 
16588 	/* bpf progs typically have pruning point every 4 instructions
16589 	 * http://vger.kernel.org/bpfconf2019.html#session-1
16590 	 * Do not add new state for future pruning if the verifier hasn't seen
16591 	 * at least 2 jumps and at least 8 instructions.
16592 	 * This heuristics helps decrease 'total_states' and 'peak_states' metric.
16593 	 * In tests that amounts to up to 50% reduction into total verifier
16594 	 * memory consumption and 20% verifier time speedup.
16595 	 */
16596 	if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
16597 	    env->insn_processed - env->prev_insn_processed >= 8)
16598 		add_new_state = true;
16599 
16600 	pprev = explored_state(env, insn_idx);
16601 	sl = *pprev;
16602 
16603 	clean_live_states(env, insn_idx, cur);
16604 
16605 	while (sl) {
16606 		states_cnt++;
16607 		if (sl->state.insn_idx != insn_idx)
16608 			goto next;
16609 
16610 		if (sl->state.branches) {
16611 			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
16612 
16613 			if (frame->in_async_callback_fn &&
16614 			    frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
16615 				/* Different async_entry_cnt means that the verifier is
16616 				 * processing another entry into async callback.
16617 				 * Seeing the same state is not an indication of infinite
16618 				 * loop or infinite recursion.
16619 				 * But finding the same state doesn't mean that it's safe
16620 				 * to stop processing the current state. The previous state
16621 				 * hasn't yet reached bpf_exit, since state.branches > 0.
16622 				 * Checking in_async_callback_fn alone is not enough either.
16623 				 * Since the verifier still needs to catch infinite loops
16624 				 * inside async callbacks.
16625 				 */
16626 				goto skip_inf_loop_check;
16627 			}
16628 			/* BPF open-coded iterators loop detection is special.
16629 			 * states_maybe_looping() logic is too simplistic in detecting
16630 			 * states that *might* be equivalent, because it doesn't know
16631 			 * about ID remapping, so don't even perform it.
16632 			 * See process_iter_next_call() and iter_active_depths_differ()
16633 			 * for overview of the logic. When current and one of parent
16634 			 * states are detected as equivalent, it's a good thing: we prove
16635 			 * convergence and can stop simulating further iterations.
16636 			 * It's safe to assume that iterator loop will finish, taking into
16637 			 * account iter_next() contract of eventually returning
16638 			 * sticky NULL result.
16639 			 *
16640 			 * Note, that states have to be compared exactly in this case because
16641 			 * read and precision marks might not be finalized inside the loop.
16642 			 * E.g. as in the program below:
16643 			 *
16644 			 *     1. r7 = -16
16645 			 *     2. r6 = bpf_get_prandom_u32()
16646 			 *     3. while (bpf_iter_num_next(&fp[-8])) {
16647 			 *     4.   if (r6 != 42) {
16648 			 *     5.     r7 = -32
16649 			 *     6.     r6 = bpf_get_prandom_u32()
16650 			 *     7.     continue
16651 			 *     8.   }
16652 			 *     9.   r0 = r10
16653 			 *    10.   r0 += r7
16654 			 *    11.   r8 = *(u64 *)(r0 + 0)
16655 			 *    12.   r6 = bpf_get_prandom_u32()
16656 			 *    13. }
16657 			 *
16658 			 * Here verifier would first visit path 1-3, create a checkpoint at 3
16659 			 * with r7=-16, continue to 4-7,3. Existing checkpoint at 3 does
16660 			 * not have read or precision mark for r7 yet, thus inexact states
16661 			 * comparison would discard current state with r7=-32
16662 			 * => unsafe memory access at 11 would not be caught.
16663 			 */
16664 			if (is_iter_next_insn(env, insn_idx)) {
16665 				if (states_equal(env, &sl->state, cur, true)) {
16666 					struct bpf_func_state *cur_frame;
16667 					struct bpf_reg_state *iter_state, *iter_reg;
16668 					int spi;
16669 
16670 					cur_frame = cur->frame[cur->curframe];
16671 					/* btf_check_iter_kfuncs() enforces that
16672 					 * iter state pointer is always the first arg
16673 					 */
16674 					iter_reg = &cur_frame->regs[BPF_REG_1];
16675 					/* current state is valid due to states_equal(),
16676 					 * so we can assume valid iter and reg state,
16677 					 * no need for extra (re-)validations
16678 					 */
16679 					spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
16680 					iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
16681 					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
16682 						update_loop_entry(cur, &sl->state);
16683 						goto hit;
16684 					}
16685 				}
16686 				goto skip_inf_loop_check;
16687 			}
16688 			if (calls_callback(env, insn_idx)) {
16689 				if (states_equal(env, &sl->state, cur, true))
16690 					goto hit;
16691 				goto skip_inf_loop_check;
16692 			}
16693 			/* attempt to detect infinite loop to avoid unnecessary doomed work */
16694 			if (states_maybe_looping(&sl->state, cur) &&
16695 			    states_equal(env, &sl->state, cur, false) &&
16696 			    !iter_active_depths_differ(&sl->state, cur) &&
16697 			    sl->state.callback_unroll_depth == cur->callback_unroll_depth) {
16698 				verbose_linfo(env, insn_idx, "; ");
16699 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);
16700 				verbose(env, "cur state:");
16701 				print_verifier_state(env, cur->frame[cur->curframe], true);
16702 				verbose(env, "old state:");
16703 				print_verifier_state(env, sl->state.frame[cur->curframe], true);
16704 				return -EINVAL;
16705 			}
16706 			/* if the verifier is processing a loop, avoid adding new state
16707 			 * too often, since different loop iterations have distinct
16708 			 * states and may not help future pruning.
16709 			 * This threshold shouldn't be too low to make sure that
16710 			 * a loop with large bound will be rejected quickly.
16711 			 * The most abusive loop will be:
16712 			 * r1 += 1
16713 			 * if r1 < 1000000 goto pc-2
16714 			 * 1M insn_procssed limit / 100 == 10k peak states.
16715 			 * This threshold shouldn't be too high either, since states
16716 			 * at the end of the loop are likely to be useful in pruning.
16717 			 */
16718 skip_inf_loop_check:
16719 			if (!force_new_state &&
16720 			    env->jmps_processed - env->prev_jmps_processed < 20 &&
16721 			    env->insn_processed - env->prev_insn_processed < 100)
16722 				add_new_state = false;
16723 			goto miss;
16724 		}
16725 		/* If sl->state is a part of a loop and this loop's entry is a part of
16726 		 * current verification path then states have to be compared exactly.
16727 		 * 'force_exact' is needed to catch the following case:
16728 		 *
16729 		 *                initial     Here state 'succ' was processed first,
16730 		 *                  |         it was eventually tracked to produce a
16731 		 *                  V         state identical to 'hdr'.
16732 		 *     .---------> hdr        All branches from 'succ' had been explored
16733 		 *     |            |         and thus 'succ' has its .branches == 0.
16734 		 *     |            V
16735 		 *     |    .------...        Suppose states 'cur' and 'succ' correspond
16736 		 *     |    |       |         to the same instruction + callsites.
16737 		 *     |    V       V         In such case it is necessary to check
16738 		 *     |   ...     ...        if 'succ' and 'cur' are states_equal().
16739 		 *     |    |       |         If 'succ' and 'cur' are a part of the
16740 		 *     |    V       V         same loop exact flag has to be set.
16741 		 *     |   succ <- cur        To check if that is the case, verify
16742 		 *     |    |                 if loop entry of 'succ' is in current
16743 		 *     |    V                 DFS path.
16744 		 *     |   ...
16745 		 *     |    |
16746 		 *     '----'
16747 		 *
16748 		 * Additional details are in the comment before get_loop_entry().
16749 		 */
16750 		loop_entry = get_loop_entry(&sl->state);
16751 		force_exact = loop_entry && loop_entry->branches > 0;
16752 		if (states_equal(env, &sl->state, cur, force_exact)) {
16753 			if (force_exact)
16754 				update_loop_entry(cur, loop_entry);
16755 hit:
16756 			sl->hit_cnt++;
16757 			/* reached equivalent register/stack state,
16758 			 * prune the search.
16759 			 * Registers read by the continuation are read by us.
16760 			 * If we have any write marks in env->cur_state, they
16761 			 * will prevent corresponding reads in the continuation
16762 			 * from reaching our parent (an explored_state).  Our
16763 			 * own state will get the read marks recorded, but
16764 			 * they'll be immediately forgotten as we're pruning
16765 			 * this state and will pop a new one.
16766 			 */
16767 			err = propagate_liveness(env, &sl->state, cur);
16768 
16769 			/* if previous state reached the exit with precision and
16770 			 * current state is equivalent to it (except precsion marks)
16771 			 * the precision needs to be propagated back in
16772 			 * the current state.
16773 			 */
16774 			err = err ? : push_jmp_history(env, cur);
16775 			err = err ? : propagate_precision(env, &sl->state);
16776 			if (err)
16777 				return err;
16778 			return 1;
16779 		}
16780 miss:
16781 		/* when new state is not going to be added do not increase miss count.
16782 		 * Otherwise several loop iterations will remove the state
16783 		 * recorded earlier. The goal of these heuristics is to have
16784 		 * states from some iterations of the loop (some in the beginning
16785 		 * and some at the end) to help pruning.
16786 		 */
16787 		if (add_new_state)
16788 			sl->miss_cnt++;
16789 		/* heuristic to determine whether this state is beneficial
16790 		 * to keep checking from state equivalence point of view.
16791 		 * Higher numbers increase max_states_per_insn and verification time,
16792 		 * but do not meaningfully decrease insn_processed.
16793 		 * 'n' controls how many times state could miss before eviction.
16794 		 * Use bigger 'n' for checkpoints because evicting checkpoint states
16795 		 * too early would hinder iterator convergence.
16796 		 */
16797 		n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;
16798 		if (sl->miss_cnt > sl->hit_cnt * n + n) {
16799 			/* the state is unlikely to be useful. Remove it to
16800 			 * speed up verification
16801 			 */
16802 			*pprev = sl->next;
16803 			if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE &&
16804 			    !sl->state.used_as_loop_entry) {
16805 				u32 br = sl->state.branches;
16806 
16807 				WARN_ONCE(br,
16808 					  "BUG live_done but branches_to_explore %d\n",
16809 					  br);
16810 				free_verifier_state(&sl->state, false);
16811 				kfree(sl);
16812 				env->peak_states--;
16813 			} else {
16814 				/* cannot free this state, since parentage chain may
16815 				 * walk it later. Add it for free_list instead to
16816 				 * be freed at the end of verification
16817 				 */
16818 				sl->next = env->free_list;
16819 				env->free_list = sl;
16820 			}
16821 			sl = *pprev;
16822 			continue;
16823 		}
16824 next:
16825 		pprev = &sl->next;
16826 		sl = *pprev;
16827 	}
16828 
16829 	if (env->max_states_per_insn < states_cnt)
16830 		env->max_states_per_insn = states_cnt;
16831 
16832 	if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
16833 		return 0;
16834 
16835 	if (!add_new_state)
16836 		return 0;
16837 
16838 	/* There were no equivalent states, remember the current one.
16839 	 * Technically the current state is not proven to be safe yet,
16840 	 * but it will either reach outer most bpf_exit (which means it's safe)
16841 	 * or it will be rejected. When there are no loops the verifier won't be
16842 	 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
16843 	 * again on the way to bpf_exit.
16844 	 * When looping the sl->state.branches will be > 0 and this state
16845 	 * will not be considered for equivalence until branches == 0.
16846 	 */
16847 	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
16848 	if (!new_sl)
16849 		return -ENOMEM;
16850 	env->total_states++;
16851 	env->peak_states++;
16852 	env->prev_jmps_processed = env->jmps_processed;
16853 	env->prev_insn_processed = env->insn_processed;
16854 
16855 	/* forget precise markings we inherited, see __mark_chain_precision */
16856 	if (env->bpf_capable)
16857 		mark_all_scalars_imprecise(env, cur);
16858 
16859 	/* add new state to the head of linked list */
16860 	new = &new_sl->state;
16861 	err = copy_verifier_state(new, cur);
16862 	if (err) {
16863 		free_verifier_state(new, false);
16864 		kfree(new_sl);
16865 		return err;
16866 	}
16867 	new->insn_idx = insn_idx;
16868 	WARN_ONCE(new->branches != 1,
16869 		  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
16870 
16871 	cur->parent = new;
16872 	cur->first_insn_idx = insn_idx;
16873 	cur->dfs_depth = new->dfs_depth + 1;
16874 	clear_jmp_history(cur);
16875 	new_sl->next = *explored_state(env, insn_idx);
16876 	*explored_state(env, insn_idx) = new_sl;
16877 	/* connect new state to parentage chain. Current frame needs all
16878 	 * registers connected. Only r6 - r9 of the callers are alive (pushed
16879 	 * to the stack implicitly by JITs) so in callers' frames connect just
16880 	 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
16881 	 * the state of the call instruction (with WRITTEN set), and r0 comes
16882 	 * from callee with its full parentage chain, anyway.
16883 	 */
16884 	/* clear write marks in current state: the writes we did are not writes
16885 	 * our child did, so they don't screen off its reads from us.
16886 	 * (There are no read marks in current state, because reads always mark
16887 	 * their parent and current state never has children yet.  Only
16888 	 * explored_states can get read marks.)
16889 	 */
16890 	for (j = 0; j <= cur->curframe; j++) {
16891 		for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
16892 			cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
16893 		for (i = 0; i < BPF_REG_FP; i++)
16894 			cur->frame[j]->regs[i].live = REG_LIVE_NONE;
16895 	}
16896 
16897 	/* all stack frames are accessible from callee, clear them all */
16898 	for (j = 0; j <= cur->curframe; j++) {
16899 		struct bpf_func_state *frame = cur->frame[j];
16900 		struct bpf_func_state *newframe = new->frame[j];
16901 
16902 		for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
16903 			frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
16904 			frame->stack[i].spilled_ptr.parent =
16905 						&newframe->stack[i].spilled_ptr;
16906 		}
16907 	}
16908 	return 0;
16909 }
16910 
16911 /* Return true if it's OK to have the same insn return a different type. */
16912 static bool reg_type_mismatch_ok(enum bpf_reg_type type)
16913 {
16914 	switch (base_type(type)) {
16915 	case PTR_TO_CTX:
16916 	case PTR_TO_SOCKET:
16917 	case PTR_TO_SOCK_COMMON:
16918 	case PTR_TO_TCP_SOCK:
16919 	case PTR_TO_XDP_SOCK:
16920 	case PTR_TO_BTF_ID:
16921 		return false;
16922 	default:
16923 		return true;
16924 	}
16925 }
16926 
16927 /* If an instruction was previously used with particular pointer types, then we
16928  * need to be careful to avoid cases such as the below, where it may be ok
16929  * for one branch accessing the pointer, but not ok for the other branch:
16930  *
16931  * R1 = sock_ptr
16932  * goto X;
16933  * ...
16934  * R1 = some_other_valid_ptr;
16935  * goto X;
16936  * ...
16937  * R2 = *(u32 *)(R1 + 0);
16938  */
16939 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
16940 {
16941 	return src != prev && (!reg_type_mismatch_ok(src) ||
16942 			       !reg_type_mismatch_ok(prev));
16943 }
16944 
16945 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
16946 			     bool allow_trust_missmatch)
16947 {
16948 	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;
16949 
16950 	if (*prev_type == NOT_INIT) {
16951 		/* Saw a valid insn
16952 		 * dst_reg = *(u32 *)(src_reg + off)
16953 		 * save type to validate intersecting paths
16954 		 */
16955 		*prev_type = type;
16956 	} else if (reg_type_mismatch(type, *prev_type)) {
16957 		/* Abuser program is trying to use the same insn
16958 		 * dst_reg = *(u32*) (src_reg + off)
16959 		 * with different pointer types:
16960 		 * src_reg == ctx in one branch and
16961 		 * src_reg == stack|map in some other branch.
16962 		 * Reject it.
16963 		 */
16964 		if (allow_trust_missmatch &&
16965 		    base_type(type) == PTR_TO_BTF_ID &&
16966 		    base_type(*prev_type) == PTR_TO_BTF_ID) {
16967 			/*
16968 			 * Have to support a use case when one path through
16969 			 * the program yields TRUSTED pointer while another
16970 			 * is UNTRUSTED. Fallback to UNTRUSTED to generate
16971 			 * BPF_PROBE_MEM/BPF_PROBE_MEMSX.
16972 			 */
16973 			*prev_type = PTR_TO_BTF_ID | PTR_UNTRUSTED;
16974 		} else {
16975 			verbose(env, "same insn cannot be used with different pointers\n");
16976 			return -EINVAL;
16977 		}
16978 	}
16979 
16980 	return 0;
16981 }
16982 
16983 static int do_check(struct bpf_verifier_env *env)
16984 {
16985 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
16986 	struct bpf_verifier_state *state = env->cur_state;
16987 	struct bpf_insn *insns = env->prog->insnsi;
16988 	struct bpf_reg_state *regs;
16989 	int insn_cnt = env->prog->len;
16990 	bool do_print_state = false;
16991 	int prev_insn_idx = -1;
16992 
16993 	for (;;) {
16994 		struct bpf_insn *insn;
16995 		u8 class;
16996 		int err;
16997 
16998 		env->prev_insn_idx = prev_insn_idx;
16999 		if (env->insn_idx >= insn_cnt) {
17000 			verbose(env, "invalid insn idx %d insn_cnt %d\n",
17001 				env->insn_idx, insn_cnt);
17002 			return -EFAULT;
17003 		}
17004 
17005 		insn = &insns[env->insn_idx];
17006 		class = BPF_CLASS(insn->code);
17007 
17008 		if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
17009 			verbose(env,
17010 				"BPF program is too large. Processed %d insn\n",
17011 				env->insn_processed);
17012 			return -E2BIG;
17013 		}
17014 
17015 		state->last_insn_idx = env->prev_insn_idx;
17016 
17017 		if (is_prune_point(env, env->insn_idx)) {
17018 			err = is_state_visited(env, env->insn_idx);
17019 			if (err < 0)
17020 				return err;
17021 			if (err == 1) {
17022 				/* found equivalent state, can prune the search */
17023 				if (env->log.level & BPF_LOG_LEVEL) {
17024 					if (do_print_state)
17025 						verbose(env, "\nfrom %d to %d%s: safe\n",
17026 							env->prev_insn_idx, env->insn_idx,
17027 							env->cur_state->speculative ?
17028 							" (speculative execution)" : "");
17029 					else
17030 						verbose(env, "%d: safe\n", env->insn_idx);
17031 				}
17032 				goto process_bpf_exit;
17033 			}
17034 		}
17035 
17036 		if (is_jmp_point(env, env->insn_idx)) {
17037 			err = push_jmp_history(env, state);
17038 			if (err)
17039 				return err;
17040 		}
17041 
17042 		if (signal_pending(current))
17043 			return -EAGAIN;
17044 
17045 		if (need_resched())
17046 			cond_resched();
17047 
17048 		if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
17049 			verbose(env, "\nfrom %d to %d%s:",
17050 				env->prev_insn_idx, env->insn_idx,
17051 				env->cur_state->speculative ?
17052 				" (speculative execution)" : "");
17053 			print_verifier_state(env, state->frame[state->curframe], true);
17054 			do_print_state = false;
17055 		}
17056 
17057 		if (env->log.level & BPF_LOG_LEVEL) {
17058 			const struct bpf_insn_cbs cbs = {
17059 				.cb_call	= disasm_kfunc_name,
17060 				.cb_print	= verbose,
17061 				.private_data	= env,
17062 			};
17063 
17064 			if (verifier_state_scratched(env))
17065 				print_insn_state(env, state->frame[state->curframe]);
17066 
17067 			verbose_linfo(env, env->insn_idx, "; ");
17068 			env->prev_log_pos = env->log.end_pos;
17069 			verbose(env, "%d: ", env->insn_idx);
17070 			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
17071 			env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
17072 			env->prev_log_pos = env->log.end_pos;
17073 		}
17074 
17075 		if (bpf_prog_is_offloaded(env->prog->aux)) {
17076 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
17077 							   env->prev_insn_idx);
17078 			if (err)
17079 				return err;
17080 		}
17081 
17082 		regs = cur_regs(env);
17083 		sanitize_mark_insn_seen(env);
17084 		prev_insn_idx = env->insn_idx;
17085 
17086 		if (class == BPF_ALU || class == BPF_ALU64) {
17087 			err = check_alu_op(env, insn);
17088 			if (err)
17089 				return err;
17090 
17091 		} else if (class == BPF_LDX) {
17092 			enum bpf_reg_type src_reg_type;
17093 
17094 			/* check for reserved fields is already done */
17095 
17096 			/* check src operand */
17097 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
17098 			if (err)
17099 				return err;
17100 
17101 			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
17102 			if (err)
17103 				return err;
17104 
17105 			src_reg_type = regs[insn->src_reg].type;
17106 
17107 			/* check that memory (src_reg + off) is readable,
17108 			 * the state of dst_reg will be updated by this func
17109 			 */
17110 			err = check_mem_access(env, env->insn_idx, insn->src_reg,
17111 					       insn->off, BPF_SIZE(insn->code),
17112 					       BPF_READ, insn->dst_reg, false,
17113 					       BPF_MODE(insn->code) == BPF_MEMSX);
17114 			if (err)
17115 				return err;
17116 
17117 			err = save_aux_ptr_type(env, src_reg_type, true);
17118 			if (err)
17119 				return err;
17120 		} else if (class == BPF_STX) {
17121 			enum bpf_reg_type dst_reg_type;
17122 
17123 			if (BPF_MODE(insn->code) == BPF_ATOMIC) {
17124 				err = check_atomic(env, env->insn_idx, insn);
17125 				if (err)
17126 					return err;
17127 				env->insn_idx++;
17128 				continue;
17129 			}
17130 
17131 			if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
17132 				verbose(env, "BPF_STX uses reserved fields\n");
17133 				return -EINVAL;
17134 			}
17135 
17136 			/* check src1 operand */
17137 			err = check_reg_arg(env, insn->src_reg, SRC_OP);
17138 			if (err)
17139 				return err;
17140 			/* check src2 operand */
17141 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17142 			if (err)
17143 				return err;
17144 
17145 			dst_reg_type = regs[insn->dst_reg].type;
17146 
17147 			/* check that memory (dst_reg + off) is writeable */
17148 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
17149 					       insn->off, BPF_SIZE(insn->code),
17150 					       BPF_WRITE, insn->src_reg, false, false);
17151 			if (err)
17152 				return err;
17153 
17154 			err = save_aux_ptr_type(env, dst_reg_type, false);
17155 			if (err)
17156 				return err;
17157 		} else if (class == BPF_ST) {
17158 			enum bpf_reg_type dst_reg_type;
17159 
17160 			if (BPF_MODE(insn->code) != BPF_MEM ||
17161 			    insn->src_reg != BPF_REG_0) {
17162 				verbose(env, "BPF_ST uses reserved fields\n");
17163 				return -EINVAL;
17164 			}
17165 			/* check src operand */
17166 			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
17167 			if (err)
17168 				return err;
17169 
17170 			dst_reg_type = regs[insn->dst_reg].type;
17171 
17172 			/* check that memory (dst_reg + off) is writeable */
17173 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
17174 					       insn->off, BPF_SIZE(insn->code),
17175 					       BPF_WRITE, -1, false, false);
17176 			if (err)
17177 				return err;
17178 
17179 			err = save_aux_ptr_type(env, dst_reg_type, false);
17180 			if (err)
17181 				return err;
17182 		} else if (class == BPF_JMP || class == BPF_JMP32) {
17183 			u8 opcode = BPF_OP(insn->code);
17184 
17185 			env->jmps_processed++;
17186 			if (opcode == BPF_CALL) {
17187 				if (BPF_SRC(insn->code) != BPF_K ||
17188 				    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL
17189 				     && insn->off != 0) ||
17190 				    (insn->src_reg != BPF_REG_0 &&
17191 				     insn->src_reg != BPF_PSEUDO_CALL &&
17192 				     insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
17193 				    insn->dst_reg != BPF_REG_0 ||
17194 				    class == BPF_JMP32) {
17195 					verbose(env, "BPF_CALL uses reserved fields\n");
17196 					return -EINVAL;
17197 				}
17198 
17199 				if (env->cur_state->active_lock.ptr) {
17200 					if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||
17201 					    (insn->src_reg == BPF_PSEUDO_CALL) ||
17202 					    (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
17203 					     (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) {
17204 						verbose(env, "function calls are not allowed while holding a lock\n");
17205 						return -EINVAL;
17206 					}
17207 				}
17208 				if (insn->src_reg == BPF_PSEUDO_CALL)
17209 					err = check_func_call(env, insn, &env->insn_idx);
17210 				else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
17211 					err = check_kfunc_call(env, insn, &env->insn_idx);
17212 				else
17213 					err = check_helper_call(env, insn, &env->insn_idx);
17214 				if (err)
17215 					return err;
17216 
17217 				mark_reg_scratched(env, BPF_REG_0);
17218 			} else if (opcode == BPF_JA) {
17219 				if (BPF_SRC(insn->code) != BPF_K ||
17220 				    insn->src_reg != BPF_REG_0 ||
17221 				    insn->dst_reg != BPF_REG_0 ||
17222 				    (class == BPF_JMP && insn->imm != 0) ||
17223 				    (class == BPF_JMP32 && insn->off != 0)) {
17224 					verbose(env, "BPF_JA uses reserved fields\n");
17225 					return -EINVAL;
17226 				}
17227 
17228 				if (class == BPF_JMP)
17229 					env->insn_idx += insn->off + 1;
17230 				else
17231 					env->insn_idx += insn->imm + 1;
17232 				continue;
17233 
17234 			} else if (opcode == BPF_EXIT) {
17235 				if (BPF_SRC(insn->code) != BPF_K ||
17236 				    insn->imm != 0 ||
17237 				    insn->src_reg != BPF_REG_0 ||
17238 				    insn->dst_reg != BPF_REG_0 ||
17239 				    class == BPF_JMP32) {
17240 					verbose(env, "BPF_EXIT uses reserved fields\n");
17241 					return -EINVAL;
17242 				}
17243 
17244 				if (env->cur_state->active_lock.ptr &&
17245 				    !in_rbtree_lock_required_cb(env)) {
17246 					verbose(env, "bpf_spin_unlock is missing\n");
17247 					return -EINVAL;
17248 				}
17249 
17250 				if (env->cur_state->active_rcu_lock &&
17251 				    !in_rbtree_lock_required_cb(env)) {
17252 					verbose(env, "bpf_rcu_read_unlock is missing\n");
17253 					return -EINVAL;
17254 				}
17255 
17256 				/* We must do check_reference_leak here before
17257 				 * prepare_func_exit to handle the case when
17258 				 * state->curframe > 0, it may be a callback
17259 				 * function, for which reference_state must
17260 				 * match caller reference state when it exits.
17261 				 */
17262 				err = check_reference_leak(env);
17263 				if (err)
17264 					return err;
17265 
17266 				if (state->curframe) {
17267 					/* exit from nested function */
17268 					err = prepare_func_exit(env, &env->insn_idx);
17269 					if (err)
17270 						return err;
17271 					do_print_state = true;
17272 					continue;
17273 				}
17274 
17275 				err = check_return_code(env);
17276 				if (err)
17277 					return err;
17278 process_bpf_exit:
17279 				mark_verifier_state_scratched(env);
17280 				update_branch_counts(env, env->cur_state);
17281 				err = pop_stack(env, &prev_insn_idx,
17282 						&env->insn_idx, pop_log);
17283 				if (err < 0) {
17284 					if (err != -ENOENT)
17285 						return err;
17286 					break;
17287 				} else {
17288 					do_print_state = true;
17289 					continue;
17290 				}
17291 			} else {
17292 				err = check_cond_jmp_op(env, insn, &env->insn_idx);
17293 				if (err)
17294 					return err;
17295 			}
17296 		} else if (class == BPF_LD) {
17297 			u8 mode = BPF_MODE(insn->code);
17298 
17299 			if (mode == BPF_ABS || mode == BPF_IND) {
17300 				err = check_ld_abs(env, insn);
17301 				if (err)
17302 					return err;
17303 
17304 			} else if (mode == BPF_IMM) {
17305 				err = check_ld_imm(env, insn);
17306 				if (err)
17307 					return err;
17308 
17309 				env->insn_idx++;
17310 				sanitize_mark_insn_seen(env);
17311 			} else {
17312 				verbose(env, "invalid BPF_LD mode\n");
17313 				return -EINVAL;
17314 			}
17315 		} else {
17316 			verbose(env, "unknown insn class %d\n", class);
17317 			return -EINVAL;
17318 		}
17319 
17320 		env->insn_idx++;
17321 	}
17322 
17323 	return 0;
17324 }
17325 
17326 static int find_btf_percpu_datasec(struct btf *btf)
17327 {
17328 	const struct btf_type *t;
17329 	const char *tname;
17330 	int i, n;
17331 
17332 	/*
17333 	 * Both vmlinux and module each have their own ".data..percpu"
17334 	 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
17335 	 * types to look at only module's own BTF types.
17336 	 */
17337 	n = btf_nr_types(btf);
17338 	if (btf_is_module(btf))
17339 		i = btf_nr_types(btf_vmlinux);
17340 	else
17341 		i = 1;
17342 
17343 	for(; i < n; i++) {
17344 		t = btf_type_by_id(btf, i);
17345 		if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
17346 			continue;
17347 
17348 		tname = btf_name_by_offset(btf, t->name_off);
17349 		if (!strcmp(tname, ".data..percpu"))
17350 			return i;
17351 	}
17352 
17353 	return -ENOENT;
17354 }
17355 
17356 /* replace pseudo btf_id with kernel symbol address */
17357 static int check_pseudo_btf_id(struct bpf_verifier_env *env,
17358 			       struct bpf_insn *insn,
17359 			       struct bpf_insn_aux_data *aux)
17360 {
17361 	const struct btf_var_secinfo *vsi;
17362 	const struct btf_type *datasec;
17363 	struct btf_mod_pair *btf_mod;
17364 	const struct btf_type *t;
17365 	const char *sym_name;
17366 	bool percpu = false;
17367 	u32 type, id = insn->imm;
17368 	struct btf *btf;
17369 	s32 datasec_id;
17370 	u64 addr;
17371 	int i, btf_fd, err;
17372 
17373 	btf_fd = insn[1].imm;
17374 	if (btf_fd) {
17375 		btf = btf_get_by_fd(btf_fd);
17376 		if (IS_ERR(btf)) {
17377 			verbose(env, "invalid module BTF object FD specified.\n");
17378 			return -EINVAL;
17379 		}
17380 	} else {
17381 		if (!btf_vmlinux) {
17382 			verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
17383 			return -EINVAL;
17384 		}
17385 		btf = btf_vmlinux;
17386 		btf_get(btf);
17387 	}
17388 
17389 	t = btf_type_by_id(btf, id);
17390 	if (!t) {
17391 		verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
17392 		err = -ENOENT;
17393 		goto err_put;
17394 	}
17395 
17396 	if (!btf_type_is_var(t) && !btf_type_is_func(t)) {
17397 		verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id);
17398 		err = -EINVAL;
17399 		goto err_put;
17400 	}
17401 
17402 	sym_name = btf_name_by_offset(btf, t->name_off);
17403 	addr = kallsyms_lookup_name(sym_name);
17404 	if (!addr) {
17405 		verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
17406 			sym_name);
17407 		err = -ENOENT;
17408 		goto err_put;
17409 	}
17410 	insn[0].imm = (u32)addr;
17411 	insn[1].imm = addr >> 32;
17412 
17413 	if (btf_type_is_func(t)) {
17414 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
17415 		aux->btf_var.mem_size = 0;
17416 		goto check_btf;
17417 	}
17418 
17419 	datasec_id = find_btf_percpu_datasec(btf);
17420 	if (datasec_id > 0) {
17421 		datasec = btf_type_by_id(btf, datasec_id);
17422 		for_each_vsi(i, datasec, vsi) {
17423 			if (vsi->type == id) {
17424 				percpu = true;
17425 				break;
17426 			}
17427 		}
17428 	}
17429 
17430 	type = t->type;
17431 	t = btf_type_skip_modifiers(btf, type, NULL);
17432 	if (percpu) {
17433 		aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
17434 		aux->btf_var.btf = btf;
17435 		aux->btf_var.btf_id = type;
17436 	} else if (!btf_type_is_struct(t)) {
17437 		const struct btf_type *ret;
17438 		const char *tname;
17439 		u32 tsize;
17440 
17441 		/* resolve the type size of ksym. */
17442 		ret = btf_resolve_size(btf, t, &tsize);
17443 		if (IS_ERR(ret)) {
17444 			tname = btf_name_by_offset(btf, t->name_off);
17445 			verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
17446 				tname, PTR_ERR(ret));
17447 			err = -EINVAL;
17448 			goto err_put;
17449 		}
17450 		aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
17451 		aux->btf_var.mem_size = tsize;
17452 	} else {
17453 		aux->btf_var.reg_type = PTR_TO_BTF_ID;
17454 		aux->btf_var.btf = btf;
17455 		aux->btf_var.btf_id = type;
17456 	}
17457 check_btf:
17458 	/* check whether we recorded this BTF (and maybe module) already */
17459 	for (i = 0; i < env->used_btf_cnt; i++) {
17460 		if (env->used_btfs[i].btf == btf) {
17461 			btf_put(btf);
17462 			return 0;
17463 		}
17464 	}
17465 
17466 	if (env->used_btf_cnt >= MAX_USED_BTFS) {
17467 		err = -E2BIG;
17468 		goto err_put;
17469 	}
17470 
17471 	btf_mod = &env->used_btfs[env->used_btf_cnt];
17472 	btf_mod->btf = btf;
17473 	btf_mod->module = NULL;
17474 
17475 	/* if we reference variables from kernel module, bump its refcount */
17476 	if (btf_is_module(btf)) {
17477 		btf_mod->module = btf_try_get_module(btf);
17478 		if (!btf_mod->module) {
17479 			err = -ENXIO;
17480 			goto err_put;
17481 		}
17482 	}
17483 
17484 	env->used_btf_cnt++;
17485 
17486 	return 0;
17487 err_put:
17488 	btf_put(btf);
17489 	return err;
17490 }
17491 
17492 static bool is_tracing_prog_type(enum bpf_prog_type type)
17493 {
17494 	switch (type) {
17495 	case BPF_PROG_TYPE_KPROBE:
17496 	case BPF_PROG_TYPE_TRACEPOINT:
17497 	case BPF_PROG_TYPE_PERF_EVENT:
17498 	case BPF_PROG_TYPE_RAW_TRACEPOINT:
17499 	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
17500 		return true;
17501 	default:
17502 		return false;
17503 	}
17504 }
17505 
17506 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
17507 					struct bpf_map *map,
17508 					struct bpf_prog *prog)
17509 
17510 {
17511 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
17512 
17513 	if (btf_record_has_field(map->record, BPF_LIST_HEAD) ||
17514 	    btf_record_has_field(map->record, BPF_RB_ROOT)) {
17515 		if (is_tracing_prog_type(prog_type)) {
17516 			verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n");
17517 			return -EINVAL;
17518 		}
17519 	}
17520 
17521 	if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
17522 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
17523 			verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
17524 			return -EINVAL;
17525 		}
17526 
17527 		if (is_tracing_prog_type(prog_type)) {
17528 			verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
17529 			return -EINVAL;
17530 		}
17531 	}
17532 
17533 	if (btf_record_has_field(map->record, BPF_TIMER)) {
17534 		if (is_tracing_prog_type(prog_type)) {
17535 			verbose(env, "tracing progs cannot use bpf_timer yet\n");
17536 			return -EINVAL;
17537 		}
17538 	}
17539 
17540 	if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) &&
17541 	    !bpf_offload_prog_map_match(prog, map)) {
17542 		verbose(env, "offload device mismatch between prog and map\n");
17543 		return -EINVAL;
17544 	}
17545 
17546 	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
17547 		verbose(env, "bpf_struct_ops map cannot be used in prog\n");
17548 		return -EINVAL;
17549 	}
17550 
17551 	if (prog->aux->sleepable)
17552 		switch (map->map_type) {
17553 		case BPF_MAP_TYPE_HASH:
17554 		case BPF_MAP_TYPE_LRU_HASH:
17555 		case BPF_MAP_TYPE_ARRAY:
17556 		case BPF_MAP_TYPE_PERCPU_HASH:
17557 		case BPF_MAP_TYPE_PERCPU_ARRAY:
17558 		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
17559 		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
17560 		case BPF_MAP_TYPE_HASH_OF_MAPS:
17561 		case BPF_MAP_TYPE_RINGBUF:
17562 		case BPF_MAP_TYPE_USER_RINGBUF:
17563 		case BPF_MAP_TYPE_INODE_STORAGE:
17564 		case BPF_MAP_TYPE_SK_STORAGE:
17565 		case BPF_MAP_TYPE_TASK_STORAGE:
17566 		case BPF_MAP_TYPE_CGRP_STORAGE:
17567 			break;
17568 		default:
17569 			verbose(env,
17570 				"Sleepable programs can only use array, hash, ringbuf and local storage maps\n");
17571 			return -EINVAL;
17572 		}
17573 
17574 	return 0;
17575 }
17576 
17577 static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
17578 {
17579 	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
17580 		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
17581 }
17582 
17583 /* find and rewrite pseudo imm in ld_imm64 instructions:
17584  *
17585  * 1. if it accesses map FD, replace it with actual map pointer.
17586  * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
17587  *
17588  * NOTE: btf_vmlinux is required for converting pseudo btf_id.
17589  */
17590 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
17591 {
17592 	struct bpf_insn *insn = env->prog->insnsi;
17593 	int insn_cnt = env->prog->len;
17594 	int i, j, err;
17595 
17596 	err = bpf_prog_calc_tag(env->prog);
17597 	if (err)
17598 		return err;
17599 
17600 	for (i = 0; i < insn_cnt; i++, insn++) {
17601 		if (BPF_CLASS(insn->code) == BPF_LDX &&
17602 		    ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
17603 		    insn->imm != 0)) {
17604 			verbose(env, "BPF_LDX uses reserved fields\n");
17605 			return -EINVAL;
17606 		}
17607 
17608 		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
17609 			struct bpf_insn_aux_data *aux;
17610 			struct bpf_map *map;
17611 			struct fd f;
17612 			u64 addr;
17613 			u32 fd;
17614 
17615 			if (i == insn_cnt - 1 || insn[1].code != 0 ||
17616 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
17617 			    insn[1].off != 0) {
17618 				verbose(env, "invalid bpf_ld_imm64 insn\n");
17619 				return -EINVAL;
17620 			}
17621 
17622 			if (insn[0].src_reg == 0)
17623 				/* valid generic load 64-bit imm */
17624 				goto next_insn;
17625 
17626 			if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
17627 				aux = &env->insn_aux_data[i];
17628 				err = check_pseudo_btf_id(env, insn, aux);
17629 				if (err)
17630 					return err;
17631 				goto next_insn;
17632 			}
17633 
17634 			if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
17635 				aux = &env->insn_aux_data[i];
17636 				aux->ptr_type = PTR_TO_FUNC;
17637 				goto next_insn;
17638 			}
17639 
17640 			/* In final convert_pseudo_ld_imm64() step, this is
17641 			 * converted into regular 64-bit imm load insn.
17642 			 */
17643 			switch (insn[0].src_reg) {
17644 			case BPF_PSEUDO_MAP_VALUE:
17645 			case BPF_PSEUDO_MAP_IDX_VALUE:
17646 				break;
17647 			case BPF_PSEUDO_MAP_FD:
17648 			case BPF_PSEUDO_MAP_IDX:
17649 				if (insn[1].imm == 0)
17650 					break;
17651 				fallthrough;
17652 			default:
17653 				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
17654 				return -EINVAL;
17655 			}
17656 
17657 			switch (insn[0].src_reg) {
17658 			case BPF_PSEUDO_MAP_IDX_VALUE:
17659 			case BPF_PSEUDO_MAP_IDX:
17660 				if (bpfptr_is_null(env->fd_array)) {
17661 					verbose(env, "fd_idx without fd_array is invalid\n");
17662 					return -EPROTO;
17663 				}
17664 				if (copy_from_bpfptr_offset(&fd, env->fd_array,
17665 							    insn[0].imm * sizeof(fd),
17666 							    sizeof(fd)))
17667 					return -EFAULT;
17668 				break;
17669 			default:
17670 				fd = insn[0].imm;
17671 				break;
17672 			}
17673 
17674 			f = fdget(fd);
17675 			map = __bpf_map_get(f);
17676 			if (IS_ERR(map)) {
17677 				verbose(env, "fd %d is not pointing to valid bpf_map\n", fd);
17678 				return PTR_ERR(map);
17679 			}
17680 
17681 			err = check_map_prog_compatibility(env, map, env->prog);
17682 			if (err) {
17683 				fdput(f);
17684 				return err;
17685 			}
17686 
17687 			aux = &env->insn_aux_data[i];
17688 			if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
17689 			    insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
17690 				addr = (unsigned long)map;
17691 			} else {
17692 				u32 off = insn[1].imm;
17693 
17694 				if (off >= BPF_MAX_VAR_OFF) {
17695 					verbose(env, "direct value offset of %u is not allowed\n", off);
17696 					fdput(f);
17697 					return -EINVAL;
17698 				}
17699 
17700 				if (!map->ops->map_direct_value_addr) {
17701 					verbose(env, "no direct value access support for this map type\n");
17702 					fdput(f);
17703 					return -EINVAL;
17704 				}
17705 
17706 				err = map->ops->map_direct_value_addr(map, &addr, off);
17707 				if (err) {
17708 					verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
17709 						map->value_size, off);
17710 					fdput(f);
17711 					return err;
17712 				}
17713 
17714 				aux->map_off = off;
17715 				addr += off;
17716 			}
17717 
17718 			insn[0].imm = (u32)addr;
17719 			insn[1].imm = addr >> 32;
17720 
17721 			/* check whether we recorded this map already */
17722 			for (j = 0; j < env->used_map_cnt; j++) {
17723 				if (env->used_maps[j] == map) {
17724 					aux->map_index = j;
17725 					fdput(f);
17726 					goto next_insn;
17727 				}
17728 			}
17729 
17730 			if (env->used_map_cnt >= MAX_USED_MAPS) {
17731 				fdput(f);
17732 				return -E2BIG;
17733 			}
17734 
17735 			if (env->prog->aux->sleepable)
17736 				atomic64_inc(&map->sleepable_refcnt);
17737 			/* hold the map. If the program is rejected by verifier,
17738 			 * the map will be released by release_maps() or it
17739 			 * will be used by the valid program until it's unloaded
17740 			 * and all maps are released in bpf_free_used_maps()
17741 			 */
17742 			bpf_map_inc(map);
17743 
17744 			aux->map_index = env->used_map_cnt;
17745 			env->used_maps[env->used_map_cnt++] = map;
17746 
17747 			if (bpf_map_is_cgroup_storage(map) &&
17748 			    bpf_cgroup_storage_assign(env->prog->aux, map)) {
17749 				verbose(env, "only one cgroup storage of each type is allowed\n");
17750 				fdput(f);
17751 				return -EBUSY;
17752 			}
17753 
17754 			fdput(f);
17755 next_insn:
17756 			insn++;
17757 			i++;
17758 			continue;
17759 		}
17760 
17761 		/* Basic sanity check before we invest more work here. */
17762 		if (!bpf_opcode_in_insntable(insn->code)) {
17763 			verbose(env, "unknown opcode %02x\n", insn->code);
17764 			return -EINVAL;
17765 		}
17766 	}
17767 
17768 	/* now all pseudo BPF_LD_IMM64 instructions load valid
17769 	 * 'struct bpf_map *' into a register instead of user map_fd.
17770 	 * These pointers will be used later by verifier to validate map access.
17771 	 */
17772 	return 0;
17773 }
17774 
17775 /* drop refcnt of maps used by the rejected program */
17776 static void release_maps(struct bpf_verifier_env *env)
17777 {
17778 	__bpf_free_used_maps(env->prog->aux, env->used_maps,
17779 			     env->used_map_cnt);
17780 }
17781 
17782 /* drop refcnt of maps used by the rejected program */
17783 static void release_btfs(struct bpf_verifier_env *env)
17784 {
17785 	__bpf_free_used_btfs(env->prog->aux, env->used_btfs,
17786 			     env->used_btf_cnt);
17787 }
17788 
17789 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
17790 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
17791 {
17792 	struct bpf_insn *insn = env->prog->insnsi;
17793 	int insn_cnt = env->prog->len;
17794 	int i;
17795 
17796 	for (i = 0; i < insn_cnt; i++, insn++) {
17797 		if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
17798 			continue;
17799 		if (insn->src_reg == BPF_PSEUDO_FUNC)
17800 			continue;
17801 		insn->src_reg = 0;
17802 	}
17803 }
17804 
17805 /* single env->prog->insni[off] instruction was replaced with the range
17806  * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
17807  * [0, off) and [off, end) to new locations, so the patched range stays zero
17808  */
17809 static void adjust_insn_aux_data(struct bpf_verifier_env *env,
17810 				 struct bpf_insn_aux_data *new_data,
17811 				 struct bpf_prog *new_prog, u32 off, u32 cnt)
17812 {
17813 	struct bpf_insn_aux_data *old_data = env->insn_aux_data;
17814 	struct bpf_insn *insn = new_prog->insnsi;
17815 	u32 old_seen = old_data[off].seen;
17816 	u32 prog_len;
17817 	int i;
17818 
17819 	/* aux info at OFF always needs adjustment, no matter fast path
17820 	 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
17821 	 * original insn at old prog.
17822 	 */
17823 	old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
17824 
17825 	if (cnt == 1)
17826 		return;
17827 	prog_len = new_prog->len;
17828 
17829 	memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
17830 	memcpy(new_data + off + cnt - 1, old_data + off,
17831 	       sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
17832 	for (i = off; i < off + cnt - 1; i++) {
17833 		/* Expand insni[off]'s seen count to the patched range. */
17834 		new_data[i].seen = old_seen;
17835 		new_data[i].zext_dst = insn_has_def32(env, insn + i);
17836 	}
17837 	env->insn_aux_data = new_data;
17838 	vfree(old_data);
17839 }
17840 
17841 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
17842 {
17843 	int i;
17844 
17845 	if (len == 1)
17846 		return;
17847 	/* NOTE: fake 'exit' subprog should be updated as well. */
17848 	for (i = 0; i <= env->subprog_cnt; i++) {
17849 		if (env->subprog_info[i].start <= off)
17850 			continue;
17851 		env->subprog_info[i].start += len - 1;
17852 	}
17853 }
17854 
17855 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
17856 {
17857 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
17858 	int i, sz = prog->aux->size_poke_tab;
17859 	struct bpf_jit_poke_descriptor *desc;
17860 
17861 	for (i = 0; i < sz; i++) {
17862 		desc = &tab[i];
17863 		if (desc->insn_idx <= off)
17864 			continue;
17865 		desc->insn_idx += len - 1;
17866 	}
17867 }
17868 
17869 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
17870 					    const struct bpf_insn *patch, u32 len)
17871 {
17872 	struct bpf_prog *new_prog;
17873 	struct bpf_insn_aux_data *new_data = NULL;
17874 
17875 	if (len > 1) {
17876 		new_data = vzalloc(array_size(env->prog->len + len - 1,
17877 					      sizeof(struct bpf_insn_aux_data)));
17878 		if (!new_data)
17879 			return NULL;
17880 	}
17881 
17882 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
17883 	if (IS_ERR(new_prog)) {
17884 		if (PTR_ERR(new_prog) == -ERANGE)
17885 			verbose(env,
17886 				"insn %d cannot be patched due to 16-bit range\n",
17887 				env->insn_aux_data[off].orig_idx);
17888 		vfree(new_data);
17889 		return NULL;
17890 	}
17891 	adjust_insn_aux_data(env, new_data, new_prog, off, len);
17892 	adjust_subprog_starts(env, off, len);
17893 	adjust_poke_descs(new_prog, off, len);
17894 	return new_prog;
17895 }
17896 
17897 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
17898 					      u32 off, u32 cnt)
17899 {
17900 	int i, j;
17901 
17902 	/* find first prog starting at or after off (first to remove) */
17903 	for (i = 0; i < env->subprog_cnt; i++)
17904 		if (env->subprog_info[i].start >= off)
17905 			break;
17906 	/* find first prog starting at or after off + cnt (first to stay) */
17907 	for (j = i; j < env->subprog_cnt; j++)
17908 		if (env->subprog_info[j].start >= off + cnt)
17909 			break;
17910 	/* if j doesn't start exactly at off + cnt, we are just removing
17911 	 * the front of previous prog
17912 	 */
17913 	if (env->subprog_info[j].start != off + cnt)
17914 		j--;
17915 
17916 	if (j > i) {
17917 		struct bpf_prog_aux *aux = env->prog->aux;
17918 		int move;
17919 
17920 		/* move fake 'exit' subprog as well */
17921 		move = env->subprog_cnt + 1 - j;
17922 
17923 		memmove(env->subprog_info + i,
17924 			env->subprog_info + j,
17925 			sizeof(*env->subprog_info) * move);
17926 		env->subprog_cnt -= j - i;
17927 
17928 		/* remove func_info */
17929 		if (aux->func_info) {
17930 			move = aux->func_info_cnt - j;
17931 
17932 			memmove(aux->func_info + i,
17933 				aux->func_info + j,
17934 				sizeof(*aux->func_info) * move);
17935 			aux->func_info_cnt -= j - i;
17936 			/* func_info->insn_off is set after all code rewrites,
17937 			 * in adjust_btf_func() - no need to adjust
17938 			 */
17939 		}
17940 	} else {
17941 		/* convert i from "first prog to remove" to "first to adjust" */
17942 		if (env->subprog_info[i].start == off)
17943 			i++;
17944 	}
17945 
17946 	/* update fake 'exit' subprog as well */
17947 	for (; i <= env->subprog_cnt; i++)
17948 		env->subprog_info[i].start -= cnt;
17949 
17950 	return 0;
17951 }
17952 
17953 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
17954 				      u32 cnt)
17955 {
17956 	struct bpf_prog *prog = env->prog;
17957 	u32 i, l_off, l_cnt, nr_linfo;
17958 	struct bpf_line_info *linfo;
17959 
17960 	nr_linfo = prog->aux->nr_linfo;
17961 	if (!nr_linfo)
17962 		return 0;
17963 
17964 	linfo = prog->aux->linfo;
17965 
17966 	/* find first line info to remove, count lines to be removed */
17967 	for (i = 0; i < nr_linfo; i++)
17968 		if (linfo[i].insn_off >= off)
17969 			break;
17970 
17971 	l_off = i;
17972 	l_cnt = 0;
17973 	for (; i < nr_linfo; i++)
17974 		if (linfo[i].insn_off < off + cnt)
17975 			l_cnt++;
17976 		else
17977 			break;
17978 
17979 	/* First live insn doesn't match first live linfo, it needs to "inherit"
17980 	 * last removed linfo.  prog is already modified, so prog->len == off
17981 	 * means no live instructions after (tail of the program was removed).
17982 	 */
17983 	if (prog->len != off && l_cnt &&
17984 	    (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
17985 		l_cnt--;
17986 		linfo[--i].insn_off = off + cnt;
17987 	}
17988 
17989 	/* remove the line info which refer to the removed instructions */
17990 	if (l_cnt) {
17991 		memmove(linfo + l_off, linfo + i,
17992 			sizeof(*linfo) * (nr_linfo - i));
17993 
17994 		prog->aux->nr_linfo -= l_cnt;
17995 		nr_linfo = prog->aux->nr_linfo;
17996 	}
17997 
17998 	/* pull all linfo[i].insn_off >= off + cnt in by cnt */
17999 	for (i = l_off; i < nr_linfo; i++)
18000 		linfo[i].insn_off -= cnt;
18001 
18002 	/* fix up all subprogs (incl. 'exit') which start >= off */
18003 	for (i = 0; i <= env->subprog_cnt; i++)
18004 		if (env->subprog_info[i].linfo_idx > l_off) {
18005 			/* program may have started in the removed region but
18006 			 * may not be fully removed
18007 			 */
18008 			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
18009 				env->subprog_info[i].linfo_idx -= l_cnt;
18010 			else
18011 				env->subprog_info[i].linfo_idx = l_off;
18012 		}
18013 
18014 	return 0;
18015 }
18016 
18017 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
18018 {
18019 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18020 	unsigned int orig_prog_len = env->prog->len;
18021 	int err;
18022 
18023 	if (bpf_prog_is_offloaded(env->prog->aux))
18024 		bpf_prog_offload_remove_insns(env, off, cnt);
18025 
18026 	err = bpf_remove_insns(env->prog, off, cnt);
18027 	if (err)
18028 		return err;
18029 
18030 	err = adjust_subprog_starts_after_remove(env, off, cnt);
18031 	if (err)
18032 		return err;
18033 
18034 	err = bpf_adj_linfo_after_remove(env, off, cnt);
18035 	if (err)
18036 		return err;
18037 
18038 	memmove(aux_data + off,	aux_data + off + cnt,
18039 		sizeof(*aux_data) * (orig_prog_len - off - cnt));
18040 
18041 	return 0;
18042 }
18043 
18044 /* The verifier does more data flow analysis than llvm and will not
18045  * explore branches that are dead at run time. Malicious programs can
18046  * have dead code too. Therefore replace all dead at-run-time code
18047  * with 'ja -1'.
18048  *
18049  * Just nops are not optimal, e.g. if they would sit at the end of the
18050  * program and through another bug we would manage to jump there, then
18051  * we'd execute beyond program memory otherwise. Returning exception
18052  * code also wouldn't work since we can have subprogs where the dead
18053  * code could be located.
18054  */
18055 static void sanitize_dead_code(struct bpf_verifier_env *env)
18056 {
18057 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18058 	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
18059 	struct bpf_insn *insn = env->prog->insnsi;
18060 	const int insn_cnt = env->prog->len;
18061 	int i;
18062 
18063 	for (i = 0; i < insn_cnt; i++) {
18064 		if (aux_data[i].seen)
18065 			continue;
18066 		memcpy(insn + i, &trap, sizeof(trap));
18067 		aux_data[i].zext_dst = false;
18068 	}
18069 }
18070 
18071 static bool insn_is_cond_jump(u8 code)
18072 {
18073 	u8 op;
18074 
18075 	op = BPF_OP(code);
18076 	if (BPF_CLASS(code) == BPF_JMP32)
18077 		return op != BPF_JA;
18078 
18079 	if (BPF_CLASS(code) != BPF_JMP)
18080 		return false;
18081 
18082 	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
18083 }
18084 
18085 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
18086 {
18087 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18088 	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
18089 	struct bpf_insn *insn = env->prog->insnsi;
18090 	const int insn_cnt = env->prog->len;
18091 	int i;
18092 
18093 	for (i = 0; i < insn_cnt; i++, insn++) {
18094 		if (!insn_is_cond_jump(insn->code))
18095 			continue;
18096 
18097 		if (!aux_data[i + 1].seen)
18098 			ja.off = insn->off;
18099 		else if (!aux_data[i + 1 + insn->off].seen)
18100 			ja.off = 0;
18101 		else
18102 			continue;
18103 
18104 		if (bpf_prog_is_offloaded(env->prog->aux))
18105 			bpf_prog_offload_replace_insn(env, i, &ja);
18106 
18107 		memcpy(insn, &ja, sizeof(ja));
18108 	}
18109 }
18110 
18111 static int opt_remove_dead_code(struct bpf_verifier_env *env)
18112 {
18113 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
18114 	int insn_cnt = env->prog->len;
18115 	int i, err;
18116 
18117 	for (i = 0; i < insn_cnt; i++) {
18118 		int j;
18119 
18120 		j = 0;
18121 		while (i + j < insn_cnt && !aux_data[i + j].seen)
18122 			j++;
18123 		if (!j)
18124 			continue;
18125 
18126 		err = verifier_remove_insns(env, i, j);
18127 		if (err)
18128 			return err;
18129 		insn_cnt = env->prog->len;
18130 	}
18131 
18132 	return 0;
18133 }
18134 
18135 static int opt_remove_nops(struct bpf_verifier_env *env)
18136 {
18137 	const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
18138 	struct bpf_insn *insn = env->prog->insnsi;
18139 	int insn_cnt = env->prog->len;
18140 	int i, err;
18141 
18142 	for (i = 0; i < insn_cnt; i++) {
18143 		if (memcmp(&insn[i], &ja, sizeof(ja)))
18144 			continue;
18145 
18146 		err = verifier_remove_insns(env, i, 1);
18147 		if (err)
18148 			return err;
18149 		insn_cnt--;
18150 		i--;
18151 	}
18152 
18153 	return 0;
18154 }
18155 
18156 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
18157 					 const union bpf_attr *attr)
18158 {
18159 	struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
18160 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
18161 	int i, patch_len, delta = 0, len = env->prog->len;
18162 	struct bpf_insn *insns = env->prog->insnsi;
18163 	struct bpf_prog *new_prog;
18164 	bool rnd_hi32;
18165 
18166 	rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
18167 	zext_patch[1] = BPF_ZEXT_REG(0);
18168 	rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
18169 	rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
18170 	rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
18171 	for (i = 0; i < len; i++) {
18172 		int adj_idx = i + delta;
18173 		struct bpf_insn insn;
18174 		int load_reg;
18175 
18176 		insn = insns[adj_idx];
18177 		load_reg = insn_def_regno(&insn);
18178 		if (!aux[adj_idx].zext_dst) {
18179 			u8 code, class;
18180 			u32 imm_rnd;
18181 
18182 			if (!rnd_hi32)
18183 				continue;
18184 
18185 			code = insn.code;
18186 			class = BPF_CLASS(code);
18187 			if (load_reg == -1)
18188 				continue;
18189 
18190 			/* NOTE: arg "reg" (the fourth one) is only used for
18191 			 *       BPF_STX + SRC_OP, so it is safe to pass NULL
18192 			 *       here.
18193 			 */
18194 			if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
18195 				if (class == BPF_LD &&
18196 				    BPF_MODE(code) == BPF_IMM)
18197 					i++;
18198 				continue;
18199 			}
18200 
18201 			/* ctx load could be transformed into wider load. */
18202 			if (class == BPF_LDX &&
18203 			    aux[adj_idx].ptr_type == PTR_TO_CTX)
18204 				continue;
18205 
18206 			imm_rnd = get_random_u32();
18207 			rnd_hi32_patch[0] = insn;
18208 			rnd_hi32_patch[1].imm = imm_rnd;
18209 			rnd_hi32_patch[3].dst_reg = load_reg;
18210 			patch = rnd_hi32_patch;
18211 			patch_len = 4;
18212 			goto apply_patch_buffer;
18213 		}
18214 
18215 		/* Add in an zero-extend instruction if a) the JIT has requested
18216 		 * it or b) it's a CMPXCHG.
18217 		 *
18218 		 * The latter is because: BPF_CMPXCHG always loads a value into
18219 		 * R0, therefore always zero-extends. However some archs'
18220 		 * equivalent instruction only does this load when the
18221 		 * comparison is successful. This detail of CMPXCHG is
18222 		 * orthogonal to the general zero-extension behaviour of the
18223 		 * CPU, so it's treated independently of bpf_jit_needs_zext.
18224 		 */
18225 		if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
18226 			continue;
18227 
18228 		/* Zero-extension is done by the caller. */
18229 		if (bpf_pseudo_kfunc_call(&insn))
18230 			continue;
18231 
18232 		if (WARN_ON(load_reg == -1)) {
18233 			verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
18234 			return -EFAULT;
18235 		}
18236 
18237 		zext_patch[0] = insn;
18238 		zext_patch[1].dst_reg = load_reg;
18239 		zext_patch[1].src_reg = load_reg;
18240 		patch = zext_patch;
18241 		patch_len = 2;
18242 apply_patch_buffer:
18243 		new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
18244 		if (!new_prog)
18245 			return -ENOMEM;
18246 		env->prog = new_prog;
18247 		insns = new_prog->insnsi;
18248 		aux = env->insn_aux_data;
18249 		delta += patch_len - 1;
18250 	}
18251 
18252 	return 0;
18253 }
18254 
18255 /* convert load instructions that access fields of a context type into a
18256  * sequence of instructions that access fields of the underlying structure:
18257  *     struct __sk_buff    -> struct sk_buff
18258  *     struct bpf_sock_ops -> struct sock
18259  */
18260 static int convert_ctx_accesses(struct bpf_verifier_env *env)
18261 {
18262 	const struct bpf_verifier_ops *ops = env->ops;
18263 	int i, cnt, size, ctx_field_size, delta = 0;
18264 	const int insn_cnt = env->prog->len;
18265 	struct bpf_insn insn_buf[16], *insn;
18266 	u32 target_size, size_default, off;
18267 	struct bpf_prog *new_prog;
18268 	enum bpf_access_type type;
18269 	bool is_narrower_load;
18270 
18271 	if (ops->gen_prologue || env->seen_direct_write) {
18272 		if (!ops->gen_prologue) {
18273 			verbose(env, "bpf verifier is misconfigured\n");
18274 			return -EINVAL;
18275 		}
18276 		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
18277 					env->prog);
18278 		if (cnt >= ARRAY_SIZE(insn_buf)) {
18279 			verbose(env, "bpf verifier is misconfigured\n");
18280 			return -EINVAL;
18281 		} else if (cnt) {
18282 			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
18283 			if (!new_prog)
18284 				return -ENOMEM;
18285 
18286 			env->prog = new_prog;
18287 			delta += cnt - 1;
18288 		}
18289 	}
18290 
18291 	if (bpf_prog_is_offloaded(env->prog->aux))
18292 		return 0;
18293 
18294 	insn = env->prog->insnsi + delta;
18295 
18296 	for (i = 0; i < insn_cnt; i++, insn++) {
18297 		bpf_convert_ctx_access_t convert_ctx_access;
18298 		u8 mode;
18299 
18300 		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
18301 		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
18302 		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
18303 		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW) ||
18304 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_B) ||
18305 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_H) ||
18306 		    insn->code == (BPF_LDX | BPF_MEMSX | BPF_W)) {
18307 			type = BPF_READ;
18308 		} else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
18309 			   insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
18310 			   insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
18311 			   insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
18312 			   insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
18313 			   insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
18314 			   insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
18315 			   insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
18316 			type = BPF_WRITE;
18317 		} else {
18318 			continue;
18319 		}
18320 
18321 		if (type == BPF_WRITE &&
18322 		    env->insn_aux_data[i + delta].sanitize_stack_spill) {
18323 			struct bpf_insn patch[] = {
18324 				*insn,
18325 				BPF_ST_NOSPEC(),
18326 			};
18327 
18328 			cnt = ARRAY_SIZE(patch);
18329 			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
18330 			if (!new_prog)
18331 				return -ENOMEM;
18332 
18333 			delta    += cnt - 1;
18334 			env->prog = new_prog;
18335 			insn      = new_prog->insnsi + i + delta;
18336 			continue;
18337 		}
18338 
18339 		switch ((int)env->insn_aux_data[i + delta].ptr_type) {
18340 		case PTR_TO_CTX:
18341 			if (!ops->convert_ctx_access)
18342 				continue;
18343 			convert_ctx_access = ops->convert_ctx_access;
18344 			break;
18345 		case PTR_TO_SOCKET:
18346 		case PTR_TO_SOCK_COMMON:
18347 			convert_ctx_access = bpf_sock_convert_ctx_access;
18348 			break;
18349 		case PTR_TO_TCP_SOCK:
18350 			convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
18351 			break;
18352 		case PTR_TO_XDP_SOCK:
18353 			convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
18354 			break;
18355 		case PTR_TO_BTF_ID:
18356 		case PTR_TO_BTF_ID | PTR_UNTRUSTED:
18357 		/* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike
18358 		 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot
18359 		 * be said once it is marked PTR_UNTRUSTED, hence we must handle
18360 		 * any faults for loads into such types. BPF_WRITE is disallowed
18361 		 * for this case.
18362 		 */
18363 		case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
18364 			if (type == BPF_READ) {
18365 				if (BPF_MODE(insn->code) == BPF_MEM)
18366 					insn->code = BPF_LDX | BPF_PROBE_MEM |
18367 						     BPF_SIZE((insn)->code);
18368 				else
18369 					insn->code = BPF_LDX | BPF_PROBE_MEMSX |
18370 						     BPF_SIZE((insn)->code);
18371 				env->prog->aux->num_exentries++;
18372 			}
18373 			continue;
18374 		default:
18375 			continue;
18376 		}
18377 
18378 		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
18379 		size = BPF_LDST_BYTES(insn);
18380 		mode = BPF_MODE(insn->code);
18381 
18382 		/* If the read access is a narrower load of the field,
18383 		 * convert to a 4/8-byte load, to minimum program type specific
18384 		 * convert_ctx_access changes. If conversion is successful,
18385 		 * we will apply proper mask to the result.
18386 		 */
18387 		is_narrower_load = size < ctx_field_size;
18388 		size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
18389 		off = insn->off;
18390 		if (is_narrower_load) {
18391 			u8 size_code;
18392 
18393 			if (type == BPF_WRITE) {
18394 				verbose(env, "bpf verifier narrow ctx access misconfigured\n");
18395 				return -EINVAL;
18396 			}
18397 
18398 			size_code = BPF_H;
18399 			if (ctx_field_size == 4)
18400 				size_code = BPF_W;
18401 			else if (ctx_field_size == 8)
18402 				size_code = BPF_DW;
18403 
18404 			insn->off = off & ~(size_default - 1);
18405 			insn->code = BPF_LDX | BPF_MEM | size_code;
18406 		}
18407 
18408 		target_size = 0;
18409 		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
18410 					 &target_size);
18411 		if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
18412 		    (ctx_field_size && !target_size)) {
18413 			verbose(env, "bpf verifier is misconfigured\n");
18414 			return -EINVAL;
18415 		}
18416 
18417 		if (is_narrower_load && size < target_size) {
18418 			u8 shift = bpf_ctx_narrow_access_offset(
18419 				off, size, size_default) * 8;
18420 			if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) {
18421 				verbose(env, "bpf verifier narrow ctx load misconfigured\n");
18422 				return -EINVAL;
18423 			}
18424 			if (ctx_field_size <= 4) {
18425 				if (shift)
18426 					insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
18427 									insn->dst_reg,
18428 									shift);
18429 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
18430 								(1 << size * 8) - 1);
18431 			} else {
18432 				if (shift)
18433 					insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
18434 									insn->dst_reg,
18435 									shift);
18436 				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
18437 								(1ULL << size * 8) - 1);
18438 			}
18439 		}
18440 		if (mode == BPF_MEMSX)
18441 			insn_buf[cnt++] = BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X,
18442 						       insn->dst_reg, insn->dst_reg,
18443 						       size * 8, 0);
18444 
18445 		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
18446 		if (!new_prog)
18447 			return -ENOMEM;
18448 
18449 		delta += cnt - 1;
18450 
18451 		/* keep walking new program and skip insns we just inserted */
18452 		env->prog = new_prog;
18453 		insn      = new_prog->insnsi + i + delta;
18454 	}
18455 
18456 	return 0;
18457 }
18458 
18459 static int jit_subprogs(struct bpf_verifier_env *env)
18460 {
18461 	struct bpf_prog *prog = env->prog, **func, *tmp;
18462 	int i, j, subprog_start, subprog_end = 0, len, subprog;
18463 	struct bpf_map *map_ptr;
18464 	struct bpf_insn *insn;
18465 	void *old_bpf_func;
18466 	int err, num_exentries;
18467 
18468 	if (env->subprog_cnt <= 1)
18469 		return 0;
18470 
18471 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
18472 		if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
18473 			continue;
18474 
18475 		/* Upon error here we cannot fall back to interpreter but
18476 		 * need a hard reject of the program. Thus -EFAULT is
18477 		 * propagated in any case.
18478 		 */
18479 		subprog = find_subprog(env, i + insn->imm + 1);
18480 		if (subprog < 0) {
18481 			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
18482 				  i + insn->imm + 1);
18483 			return -EFAULT;
18484 		}
18485 		/* temporarily remember subprog id inside insn instead of
18486 		 * aux_data, since next loop will split up all insns into funcs
18487 		 */
18488 		insn->off = subprog;
18489 		/* remember original imm in case JIT fails and fallback
18490 		 * to interpreter will be needed
18491 		 */
18492 		env->insn_aux_data[i].call_imm = insn->imm;
18493 		/* point imm to __bpf_call_base+1 from JITs point of view */
18494 		insn->imm = 1;
18495 		if (bpf_pseudo_func(insn))
18496 			/* jit (e.g. x86_64) may emit fewer instructions
18497 			 * if it learns a u32 imm is the same as a u64 imm.
18498 			 * Force a non zero here.
18499 			 */
18500 			insn[1].imm = 1;
18501 	}
18502 
18503 	err = bpf_prog_alloc_jited_linfo(prog);
18504 	if (err)
18505 		goto out_undo_insn;
18506 
18507 	err = -ENOMEM;
18508 	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
18509 	if (!func)
18510 		goto out_undo_insn;
18511 
18512 	for (i = 0; i < env->subprog_cnt; i++) {
18513 		subprog_start = subprog_end;
18514 		subprog_end = env->subprog_info[i + 1].start;
18515 
18516 		len = subprog_end - subprog_start;
18517 		/* bpf_prog_run() doesn't call subprogs directly,
18518 		 * hence main prog stats include the runtime of subprogs.
18519 		 * subprogs don't have IDs and not reachable via prog_get_next_id
18520 		 * func[i]->stats will never be accessed and stays NULL
18521 		 */
18522 		func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
18523 		if (!func[i])
18524 			goto out_free;
18525 		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
18526 		       len * sizeof(struct bpf_insn));
18527 		func[i]->type = prog->type;
18528 		func[i]->len = len;
18529 		if (bpf_prog_calc_tag(func[i]))
18530 			goto out_free;
18531 		func[i]->is_func = 1;
18532 		func[i]->aux->func_idx = i;
18533 		/* Below members will be freed only at prog->aux */
18534 		func[i]->aux->btf = prog->aux->btf;
18535 		func[i]->aux->func_info = prog->aux->func_info;
18536 		func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
18537 		func[i]->aux->poke_tab = prog->aux->poke_tab;
18538 		func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
18539 
18540 		for (j = 0; j < prog->aux->size_poke_tab; j++) {
18541 			struct bpf_jit_poke_descriptor *poke;
18542 
18543 			poke = &prog->aux->poke_tab[j];
18544 			if (poke->insn_idx < subprog_end &&
18545 			    poke->insn_idx >= subprog_start)
18546 				poke->aux = func[i]->aux;
18547 		}
18548 
18549 		func[i]->aux->name[0] = 'F';
18550 		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
18551 		func[i]->jit_requested = 1;
18552 		func[i]->blinding_requested = prog->blinding_requested;
18553 		func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
18554 		func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
18555 		func[i]->aux->linfo = prog->aux->linfo;
18556 		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
18557 		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
18558 		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
18559 		num_exentries = 0;
18560 		insn = func[i]->insnsi;
18561 		for (j = 0; j < func[i]->len; j++, insn++) {
18562 			if (BPF_CLASS(insn->code) == BPF_LDX &&
18563 			    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
18564 			     BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
18565 				num_exentries++;
18566 		}
18567 		func[i]->aux->num_exentries = num_exentries;
18568 		func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
18569 		func[i] = bpf_int_jit_compile(func[i]);
18570 		if (!func[i]->jited) {
18571 			err = -ENOTSUPP;
18572 			goto out_free;
18573 		}
18574 		cond_resched();
18575 	}
18576 
18577 	/* at this point all bpf functions were successfully JITed
18578 	 * now populate all bpf_calls with correct addresses and
18579 	 * run last pass of JIT
18580 	 */
18581 	for (i = 0; i < env->subprog_cnt; i++) {
18582 		insn = func[i]->insnsi;
18583 		for (j = 0; j < func[i]->len; j++, insn++) {
18584 			if (bpf_pseudo_func(insn)) {
18585 				subprog = insn->off;
18586 				insn[0].imm = (u32)(long)func[subprog]->bpf_func;
18587 				insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
18588 				continue;
18589 			}
18590 			if (!bpf_pseudo_call(insn))
18591 				continue;
18592 			subprog = insn->off;
18593 			insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
18594 		}
18595 
18596 		/* we use the aux data to keep a list of the start addresses
18597 		 * of the JITed images for each function in the program
18598 		 *
18599 		 * for some architectures, such as powerpc64, the imm field
18600 		 * might not be large enough to hold the offset of the start
18601 		 * address of the callee's JITed image from __bpf_call_base
18602 		 *
18603 		 * in such cases, we can lookup the start address of a callee
18604 		 * by using its subprog id, available from the off field of
18605 		 * the call instruction, as an index for this list
18606 		 */
18607 		func[i]->aux->func = func;
18608 		func[i]->aux->func_cnt = env->subprog_cnt;
18609 	}
18610 	for (i = 0; i < env->subprog_cnt; i++) {
18611 		old_bpf_func = func[i]->bpf_func;
18612 		tmp = bpf_int_jit_compile(func[i]);
18613 		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
18614 			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
18615 			err = -ENOTSUPP;
18616 			goto out_free;
18617 		}
18618 		cond_resched();
18619 	}
18620 
18621 	/* finally lock prog and jit images for all functions and
18622 	 * populate kallsysm. Begin at the first subprogram, since
18623 	 * bpf_prog_load will add the kallsyms for the main program.
18624 	 */
18625 	for (i = 1; i < env->subprog_cnt; i++) {
18626 		bpf_prog_lock_ro(func[i]);
18627 		bpf_prog_kallsyms_add(func[i]);
18628 	}
18629 
18630 	/* Last step: make now unused interpreter insns from main
18631 	 * prog consistent for later dump requests, so they can
18632 	 * later look the same as if they were interpreted only.
18633 	 */
18634 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
18635 		if (bpf_pseudo_func(insn)) {
18636 			insn[0].imm = env->insn_aux_data[i].call_imm;
18637 			insn[1].imm = insn->off;
18638 			insn->off = 0;
18639 			continue;
18640 		}
18641 		if (!bpf_pseudo_call(insn))
18642 			continue;
18643 		insn->off = env->insn_aux_data[i].call_imm;
18644 		subprog = find_subprog(env, i + insn->off + 1);
18645 		insn->imm = subprog;
18646 	}
18647 
18648 	prog->jited = 1;
18649 	prog->bpf_func = func[0]->bpf_func;
18650 	prog->jited_len = func[0]->jited_len;
18651 	prog->aux->extable = func[0]->aux->extable;
18652 	prog->aux->num_exentries = func[0]->aux->num_exentries;
18653 	prog->aux->func = func;
18654 	prog->aux->func_cnt = env->subprog_cnt;
18655 	bpf_prog_jit_attempt_done(prog);
18656 	return 0;
18657 out_free:
18658 	/* We failed JIT'ing, so at this point we need to unregister poke
18659 	 * descriptors from subprogs, so that kernel is not attempting to
18660 	 * patch it anymore as we're freeing the subprog JIT memory.
18661 	 */
18662 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
18663 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
18664 		map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
18665 	}
18666 	/* At this point we're guaranteed that poke descriptors are not
18667 	 * live anymore. We can just unlink its descriptor table as it's
18668 	 * released with the main prog.
18669 	 */
18670 	for (i = 0; i < env->subprog_cnt; i++) {
18671 		if (!func[i])
18672 			continue;
18673 		func[i]->aux->poke_tab = NULL;
18674 		bpf_jit_free(func[i]);
18675 	}
18676 	kfree(func);
18677 out_undo_insn:
18678 	/* cleanup main prog to be interpreted */
18679 	prog->jit_requested = 0;
18680 	prog->blinding_requested = 0;
18681 	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
18682 		if (!bpf_pseudo_call(insn))
18683 			continue;
18684 		insn->off = 0;
18685 		insn->imm = env->insn_aux_data[i].call_imm;
18686 	}
18687 	bpf_prog_jit_attempt_done(prog);
18688 	return err;
18689 }
18690 
18691 static int fixup_call_args(struct bpf_verifier_env *env)
18692 {
18693 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
18694 	struct bpf_prog *prog = env->prog;
18695 	struct bpf_insn *insn = prog->insnsi;
18696 	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
18697 	int i, depth;
18698 #endif
18699 	int err = 0;
18700 
18701 	if (env->prog->jit_requested &&
18702 	    !bpf_prog_is_offloaded(env->prog->aux)) {
18703 		err = jit_subprogs(env);
18704 		if (err == 0)
18705 			return 0;
18706 		if (err == -EFAULT)
18707 			return err;
18708 	}
18709 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
18710 	if (has_kfunc_call) {
18711 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
18712 		return -EINVAL;
18713 	}
18714 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
18715 		/* When JIT fails the progs with bpf2bpf calls and tail_calls
18716 		 * have to be rejected, since interpreter doesn't support them yet.
18717 		 */
18718 		verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
18719 		return -EINVAL;
18720 	}
18721 	for (i = 0; i < prog->len; i++, insn++) {
18722 		if (bpf_pseudo_func(insn)) {
18723 			/* When JIT fails the progs with callback calls
18724 			 * have to be rejected, since interpreter doesn't support them yet.
18725 			 */
18726 			verbose(env, "callbacks are not allowed in non-JITed programs\n");
18727 			return -EINVAL;
18728 		}
18729 
18730 		if (!bpf_pseudo_call(insn))
18731 			continue;
18732 		depth = get_callee_stack_depth(env, insn, i);
18733 		if (depth < 0)
18734 			return depth;
18735 		bpf_patch_call_args(insn, depth);
18736 	}
18737 	err = 0;
18738 #endif
18739 	return err;
18740 }
18741 
18742 /* replace a generic kfunc with a specialized version if necessary */
18743 static void specialize_kfunc(struct bpf_verifier_env *env,
18744 			     u32 func_id, u16 offset, unsigned long *addr)
18745 {
18746 	struct bpf_prog *prog = env->prog;
18747 	bool seen_direct_write;
18748 	void *xdp_kfunc;
18749 	bool is_rdonly;
18750 
18751 	if (bpf_dev_bound_kfunc_id(func_id)) {
18752 		xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id);
18753 		if (xdp_kfunc) {
18754 			*addr = (unsigned long)xdp_kfunc;
18755 			return;
18756 		}
18757 		/* fallback to default kfunc when not supported by netdev */
18758 	}
18759 
18760 	if (offset)
18761 		return;
18762 
18763 	if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
18764 		seen_direct_write = env->seen_direct_write;
18765 		is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE);
18766 
18767 		if (is_rdonly)
18768 			*addr = (unsigned long)bpf_dynptr_from_skb_rdonly;
18769 
18770 		/* restore env->seen_direct_write to its original value, since
18771 		 * may_access_direct_pkt_data mutates it
18772 		 */
18773 		env->seen_direct_write = seen_direct_write;
18774 	}
18775 }
18776 
18777 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
18778 					    u16 struct_meta_reg,
18779 					    u16 node_offset_reg,
18780 					    struct bpf_insn *insn,
18781 					    struct bpf_insn *insn_buf,
18782 					    int *cnt)
18783 {
18784 	struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta;
18785 	struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) };
18786 
18787 	insn_buf[0] = addr[0];
18788 	insn_buf[1] = addr[1];
18789 	insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off);
18790 	insn_buf[3] = *insn;
18791 	*cnt = 4;
18792 }
18793 
18794 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
18795 			    struct bpf_insn *insn_buf, int insn_idx, int *cnt)
18796 {
18797 	const struct bpf_kfunc_desc *desc;
18798 
18799 	if (!insn->imm) {
18800 		verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
18801 		return -EINVAL;
18802 	}
18803 
18804 	*cnt = 0;
18805 
18806 	/* insn->imm has the btf func_id. Replace it with an offset relative to
18807 	 * __bpf_call_base, unless the JIT needs to call functions that are
18808 	 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()).
18809 	 */
18810 	desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
18811 	if (!desc) {
18812 		verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
18813 			insn->imm);
18814 		return -EFAULT;
18815 	}
18816 
18817 	if (!bpf_jit_supports_far_kfunc_call())
18818 		insn->imm = BPF_CALL_IMM(desc->addr);
18819 	if (insn->off)
18820 		return 0;
18821 	if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) {
18822 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
18823 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
18824 		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;
18825 
18826 		insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size);
18827 		insn_buf[1] = addr[0];
18828 		insn_buf[2] = addr[1];
18829 		insn_buf[3] = *insn;
18830 		*cnt = 4;
18831 	} else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] ||
18832 		   desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) {
18833 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
18834 		struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) };
18835 
18836 		if (desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl] &&
18837 		    !kptr_struct_meta) {
18838 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
18839 				insn_idx);
18840 			return -EFAULT;
18841 		}
18842 
18843 		insn_buf[0] = addr[0];
18844 		insn_buf[1] = addr[1];
18845 		insn_buf[2] = *insn;
18846 		*cnt = 3;
18847 	} else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] ||
18848 		   desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] ||
18849 		   desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
18850 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;
18851 		int struct_meta_reg = BPF_REG_3;
18852 		int node_offset_reg = BPF_REG_4;
18853 
18854 		/* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */
18855 		if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
18856 			struct_meta_reg = BPF_REG_4;
18857 			node_offset_reg = BPF_REG_5;
18858 		}
18859 
18860 		if (!kptr_struct_meta) {
18861 			verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n",
18862 				insn_idx);
18863 			return -EFAULT;
18864 		}
18865 
18866 		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,
18867 						node_offset_reg, insn, insn_buf, cnt);
18868 	} else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
18869 		   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
18870 		insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
18871 		*cnt = 1;
18872 	}
18873 	return 0;
18874 }
18875 
18876 /* Do various post-verification rewrites in a single program pass.
18877  * These rewrites simplify JIT and interpreter implementations.
18878  */
18879 static int do_misc_fixups(struct bpf_verifier_env *env)
18880 {
18881 	struct bpf_prog *prog = env->prog;
18882 	enum bpf_attach_type eatype = prog->expected_attach_type;
18883 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
18884 	struct bpf_insn *insn = prog->insnsi;
18885 	const struct bpf_func_proto *fn;
18886 	const int insn_cnt = prog->len;
18887 	const struct bpf_map_ops *ops;
18888 	struct bpf_insn_aux_data *aux;
18889 	struct bpf_insn insn_buf[16];
18890 	struct bpf_prog *new_prog;
18891 	struct bpf_map *map_ptr;
18892 	int i, ret, cnt, delta = 0;
18893 
18894 	for (i = 0; i < insn_cnt; i++, insn++) {
18895 		/* Make divide-by-zero exceptions impossible. */
18896 		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
18897 		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
18898 		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
18899 		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
18900 			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
18901 			bool isdiv = BPF_OP(insn->code) == BPF_DIV;
18902 			struct bpf_insn *patchlet;
18903 			struct bpf_insn chk_and_div[] = {
18904 				/* [R,W]x div 0 -> 0 */
18905 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
18906 					     BPF_JNE | BPF_K, insn->src_reg,
18907 					     0, 2, 0),
18908 				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
18909 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
18910 				*insn,
18911 			};
18912 			struct bpf_insn chk_and_mod[] = {
18913 				/* [R,W]x mod 0 -> [R,W]x */
18914 				BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
18915 					     BPF_JEQ | BPF_K, insn->src_reg,
18916 					     0, 1 + (is64 ? 0 : 1), 0),
18917 				*insn,
18918 				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
18919 				BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
18920 			};
18921 
18922 			patchlet = isdiv ? chk_and_div : chk_and_mod;
18923 			cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
18924 				      ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
18925 
18926 			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
18927 			if (!new_prog)
18928 				return -ENOMEM;
18929 
18930 			delta    += cnt - 1;
18931 			env->prog = prog = new_prog;
18932 			insn      = new_prog->insnsi + i + delta;
18933 			continue;
18934 		}
18935 
18936 		/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
18937 		if (BPF_CLASS(insn->code) == BPF_LD &&
18938 		    (BPF_MODE(insn->code) == BPF_ABS ||
18939 		     BPF_MODE(insn->code) == BPF_IND)) {
18940 			cnt = env->ops->gen_ld_abs(insn, insn_buf);
18941 			if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
18942 				verbose(env, "bpf verifier is misconfigured\n");
18943 				return -EINVAL;
18944 			}
18945 
18946 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
18947 			if (!new_prog)
18948 				return -ENOMEM;
18949 
18950 			delta    += cnt - 1;
18951 			env->prog = prog = new_prog;
18952 			insn      = new_prog->insnsi + i + delta;
18953 			continue;
18954 		}
18955 
18956 		/* Rewrite pointer arithmetic to mitigate speculation attacks. */
18957 		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
18958 		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
18959 			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
18960 			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
18961 			struct bpf_insn *patch = &insn_buf[0];
18962 			bool issrc, isneg, isimm;
18963 			u32 off_reg;
18964 
18965 			aux = &env->insn_aux_data[i + delta];
18966 			if (!aux->alu_state ||
18967 			    aux->alu_state == BPF_ALU_NON_POINTER)
18968 				continue;
18969 
18970 			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
18971 			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
18972 				BPF_ALU_SANITIZE_SRC;
18973 			isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
18974 
18975 			off_reg = issrc ? insn->src_reg : insn->dst_reg;
18976 			if (isimm) {
18977 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
18978 			} else {
18979 				if (isneg)
18980 					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
18981 				*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
18982 				*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
18983 				*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
18984 				*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
18985 				*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
18986 				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
18987 			}
18988 			if (!issrc)
18989 				*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
18990 			insn->src_reg = BPF_REG_AX;
18991 			if (isneg)
18992 				insn->code = insn->code == code_add ?
18993 					     code_sub : code_add;
18994 			*patch++ = *insn;
18995 			if (issrc && isneg && !isimm)
18996 				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
18997 			cnt = patch - insn_buf;
18998 
18999 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19000 			if (!new_prog)
19001 				return -ENOMEM;
19002 
19003 			delta    += cnt - 1;
19004 			env->prog = prog = new_prog;
19005 			insn      = new_prog->insnsi + i + delta;
19006 			continue;
19007 		}
19008 
19009 		if (insn->code != (BPF_JMP | BPF_CALL))
19010 			continue;
19011 		if (insn->src_reg == BPF_PSEUDO_CALL)
19012 			continue;
19013 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
19014 			ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
19015 			if (ret)
19016 				return ret;
19017 			if (cnt == 0)
19018 				continue;
19019 
19020 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19021 			if (!new_prog)
19022 				return -ENOMEM;
19023 
19024 			delta	 += cnt - 1;
19025 			env->prog = prog = new_prog;
19026 			insn	  = new_prog->insnsi + i + delta;
19027 			continue;
19028 		}
19029 
19030 		if (insn->imm == BPF_FUNC_get_route_realm)
19031 			prog->dst_needed = 1;
19032 		if (insn->imm == BPF_FUNC_get_prandom_u32)
19033 			bpf_user_rnd_init_once();
19034 		if (insn->imm == BPF_FUNC_override_return)
19035 			prog->kprobe_override = 1;
19036 		if (insn->imm == BPF_FUNC_tail_call) {
19037 			/* If we tail call into other programs, we
19038 			 * cannot make any assumptions since they can
19039 			 * be replaced dynamically during runtime in
19040 			 * the program array.
19041 			 */
19042 			prog->cb_access = 1;
19043 			if (!allow_tail_call_in_subprogs(env))
19044 				prog->aux->stack_depth = MAX_BPF_STACK;
19045 			prog->aux->max_pkt_offset = MAX_PACKET_OFF;
19046 
19047 			/* mark bpf_tail_call as different opcode to avoid
19048 			 * conditional branch in the interpreter for every normal
19049 			 * call and to prevent accidental JITing by JIT compiler
19050 			 * that doesn't support bpf_tail_call yet
19051 			 */
19052 			insn->imm = 0;
19053 			insn->code = BPF_JMP | BPF_TAIL_CALL;
19054 
19055 			aux = &env->insn_aux_data[i + delta];
19056 			if (env->bpf_capable && !prog->blinding_requested &&
19057 			    prog->jit_requested &&
19058 			    !bpf_map_key_poisoned(aux) &&
19059 			    !bpf_map_ptr_poisoned(aux) &&
19060 			    !bpf_map_ptr_unpriv(aux)) {
19061 				struct bpf_jit_poke_descriptor desc = {
19062 					.reason = BPF_POKE_REASON_TAIL_CALL,
19063 					.tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
19064 					.tail_call.key = bpf_map_key_immediate(aux),
19065 					.insn_idx = i + delta,
19066 				};
19067 
19068 				ret = bpf_jit_add_poke_descriptor(prog, &desc);
19069 				if (ret < 0) {
19070 					verbose(env, "adding tail call poke descriptor failed\n");
19071 					return ret;
19072 				}
19073 
19074 				insn->imm = ret + 1;
19075 				continue;
19076 			}
19077 
19078 			if (!bpf_map_ptr_unpriv(aux))
19079 				continue;
19080 
19081 			/* instead of changing every JIT dealing with tail_call
19082 			 * emit two extra insns:
19083 			 * if (index >= max_entries) goto out;
19084 			 * index &= array->index_mask;
19085 			 * to avoid out-of-bounds cpu speculation
19086 			 */
19087 			if (bpf_map_ptr_poisoned(aux)) {
19088 				verbose(env, "tail_call abusing map_ptr\n");
19089 				return -EINVAL;
19090 			}
19091 
19092 			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
19093 			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
19094 						  map_ptr->max_entries, 2);
19095 			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
19096 						    container_of(map_ptr,
19097 								 struct bpf_array,
19098 								 map)->index_mask);
19099 			insn_buf[2] = *insn;
19100 			cnt = 3;
19101 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19102 			if (!new_prog)
19103 				return -ENOMEM;
19104 
19105 			delta    += cnt - 1;
19106 			env->prog = prog = new_prog;
19107 			insn      = new_prog->insnsi + i + delta;
19108 			continue;
19109 		}
19110 
19111 		if (insn->imm == BPF_FUNC_timer_set_callback) {
19112 			/* The verifier will process callback_fn as many times as necessary
19113 			 * with different maps and the register states prepared by
19114 			 * set_timer_callback_state will be accurate.
19115 			 *
19116 			 * The following use case is valid:
19117 			 *   map1 is shared by prog1, prog2, prog3.
19118 			 *   prog1 calls bpf_timer_init for some map1 elements
19119 			 *   prog2 calls bpf_timer_set_callback for some map1 elements.
19120 			 *     Those that were not bpf_timer_init-ed will return -EINVAL.
19121 			 *   prog3 calls bpf_timer_start for some map1 elements.
19122 			 *     Those that were not both bpf_timer_init-ed and
19123 			 *     bpf_timer_set_callback-ed will return -EINVAL.
19124 			 */
19125 			struct bpf_insn ld_addrs[2] = {
19126 				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
19127 			};
19128 
19129 			insn_buf[0] = ld_addrs[0];
19130 			insn_buf[1] = ld_addrs[1];
19131 			insn_buf[2] = *insn;
19132 			cnt = 3;
19133 
19134 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19135 			if (!new_prog)
19136 				return -ENOMEM;
19137 
19138 			delta    += cnt - 1;
19139 			env->prog = prog = new_prog;
19140 			insn      = new_prog->insnsi + i + delta;
19141 			goto patch_call_imm;
19142 		}
19143 
19144 		if (is_storage_get_function(insn->imm)) {
19145 			if (!env->prog->aux->sleepable ||
19146 			    env->insn_aux_data[i + delta].storage_get_func_atomic)
19147 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
19148 			else
19149 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
19150 			insn_buf[1] = *insn;
19151 			cnt = 2;
19152 
19153 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19154 			if (!new_prog)
19155 				return -ENOMEM;
19156 
19157 			delta += cnt - 1;
19158 			env->prog = prog = new_prog;
19159 			insn = new_prog->insnsi + i + delta;
19160 			goto patch_call_imm;
19161 		}
19162 
19163 		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
19164 		 * and other inlining handlers are currently limited to 64 bit
19165 		 * only.
19166 		 */
19167 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
19168 		    (insn->imm == BPF_FUNC_map_lookup_elem ||
19169 		     insn->imm == BPF_FUNC_map_update_elem ||
19170 		     insn->imm == BPF_FUNC_map_delete_elem ||
19171 		     insn->imm == BPF_FUNC_map_push_elem   ||
19172 		     insn->imm == BPF_FUNC_map_pop_elem    ||
19173 		     insn->imm == BPF_FUNC_map_peek_elem   ||
19174 		     insn->imm == BPF_FUNC_redirect_map    ||
19175 		     insn->imm == BPF_FUNC_for_each_map_elem ||
19176 		     insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
19177 			aux = &env->insn_aux_data[i + delta];
19178 			if (bpf_map_ptr_poisoned(aux))
19179 				goto patch_call_imm;
19180 
19181 			map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
19182 			ops = map_ptr->ops;
19183 			if (insn->imm == BPF_FUNC_map_lookup_elem &&
19184 			    ops->map_gen_lookup) {
19185 				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
19186 				if (cnt == -EOPNOTSUPP)
19187 					goto patch_map_ops_generic;
19188 				if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
19189 					verbose(env, "bpf verifier is misconfigured\n");
19190 					return -EINVAL;
19191 				}
19192 
19193 				new_prog = bpf_patch_insn_data(env, i + delta,
19194 							       insn_buf, cnt);
19195 				if (!new_prog)
19196 					return -ENOMEM;
19197 
19198 				delta    += cnt - 1;
19199 				env->prog = prog = new_prog;
19200 				insn      = new_prog->insnsi + i + delta;
19201 				continue;
19202 			}
19203 
19204 			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
19205 				     (void *(*)(struct bpf_map *map, void *key))NULL));
19206 			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
19207 				     (long (*)(struct bpf_map *map, void *key))NULL));
19208 			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
19209 				     (long (*)(struct bpf_map *map, void *key, void *value,
19210 					      u64 flags))NULL));
19211 			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
19212 				     (long (*)(struct bpf_map *map, void *value,
19213 					      u64 flags))NULL));
19214 			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
19215 				     (long (*)(struct bpf_map *map, void *value))NULL));
19216 			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
19217 				     (long (*)(struct bpf_map *map, void *value))NULL));
19218 			BUILD_BUG_ON(!__same_type(ops->map_redirect,
19219 				     (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL));
19220 			BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
19221 				     (long (*)(struct bpf_map *map,
19222 					      bpf_callback_t callback_fn,
19223 					      void *callback_ctx,
19224 					      u64 flags))NULL));
19225 			BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
19226 				     (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
19227 
19228 patch_map_ops_generic:
19229 			switch (insn->imm) {
19230 			case BPF_FUNC_map_lookup_elem:
19231 				insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
19232 				continue;
19233 			case BPF_FUNC_map_update_elem:
19234 				insn->imm = BPF_CALL_IMM(ops->map_update_elem);
19235 				continue;
19236 			case BPF_FUNC_map_delete_elem:
19237 				insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
19238 				continue;
19239 			case BPF_FUNC_map_push_elem:
19240 				insn->imm = BPF_CALL_IMM(ops->map_push_elem);
19241 				continue;
19242 			case BPF_FUNC_map_pop_elem:
19243 				insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
19244 				continue;
19245 			case BPF_FUNC_map_peek_elem:
19246 				insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
19247 				continue;
19248 			case BPF_FUNC_redirect_map:
19249 				insn->imm = BPF_CALL_IMM(ops->map_redirect);
19250 				continue;
19251 			case BPF_FUNC_for_each_map_elem:
19252 				insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
19253 				continue;
19254 			case BPF_FUNC_map_lookup_percpu_elem:
19255 				insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
19256 				continue;
19257 			}
19258 
19259 			goto patch_call_imm;
19260 		}
19261 
19262 		/* Implement bpf_jiffies64 inline. */
19263 		if (prog->jit_requested && BITS_PER_LONG == 64 &&
19264 		    insn->imm == BPF_FUNC_jiffies64) {
19265 			struct bpf_insn ld_jiffies_addr[2] = {
19266 				BPF_LD_IMM64(BPF_REG_0,
19267 					     (unsigned long)&jiffies),
19268 			};
19269 
19270 			insn_buf[0] = ld_jiffies_addr[0];
19271 			insn_buf[1] = ld_jiffies_addr[1];
19272 			insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
19273 						  BPF_REG_0, 0);
19274 			cnt = 3;
19275 
19276 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
19277 						       cnt);
19278 			if (!new_prog)
19279 				return -ENOMEM;
19280 
19281 			delta    += cnt - 1;
19282 			env->prog = prog = new_prog;
19283 			insn      = new_prog->insnsi + i + delta;
19284 			continue;
19285 		}
19286 
19287 		/* Implement bpf_get_func_arg inline. */
19288 		if (prog_type == BPF_PROG_TYPE_TRACING &&
19289 		    insn->imm == BPF_FUNC_get_func_arg) {
19290 			/* Load nr_args from ctx - 8 */
19291 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
19292 			insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
19293 			insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
19294 			insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
19295 			insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
19296 			insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
19297 			insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
19298 			insn_buf[7] = BPF_JMP_A(1);
19299 			insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
19300 			cnt = 9;
19301 
19302 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19303 			if (!new_prog)
19304 				return -ENOMEM;
19305 
19306 			delta    += cnt - 1;
19307 			env->prog = prog = new_prog;
19308 			insn      = new_prog->insnsi + i + delta;
19309 			continue;
19310 		}
19311 
19312 		/* Implement bpf_get_func_ret inline. */
19313 		if (prog_type == BPF_PROG_TYPE_TRACING &&
19314 		    insn->imm == BPF_FUNC_get_func_ret) {
19315 			if (eatype == BPF_TRACE_FEXIT ||
19316 			    eatype == BPF_MODIFY_RETURN) {
19317 				/* Load nr_args from ctx - 8 */
19318 				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
19319 				insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
19320 				insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
19321 				insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
19322 				insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
19323 				insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
19324 				cnt = 6;
19325 			} else {
19326 				insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
19327 				cnt = 1;
19328 			}
19329 
19330 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
19331 			if (!new_prog)
19332 				return -ENOMEM;
19333 
19334 			delta    += cnt - 1;
19335 			env->prog = prog = new_prog;
19336 			insn      = new_prog->insnsi + i + delta;
19337 			continue;
19338 		}
19339 
19340 		/* Implement get_func_arg_cnt inline. */
19341 		if (prog_type == BPF_PROG_TYPE_TRACING &&
19342 		    insn->imm == BPF_FUNC_get_func_arg_cnt) {
19343 			/* Load nr_args from ctx - 8 */
19344 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
19345 
19346 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
19347 			if (!new_prog)
19348 				return -ENOMEM;
19349 
19350 			env->prog = prog = new_prog;
19351 			insn      = new_prog->insnsi + i + delta;
19352 			continue;
19353 		}
19354 
19355 		/* Implement bpf_get_func_ip inline. */
19356 		if (prog_type == BPF_PROG_TYPE_TRACING &&
19357 		    insn->imm == BPF_FUNC_get_func_ip) {
19358 			/* Load IP address from ctx - 16 */
19359 			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
19360 
19361 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
19362 			if (!new_prog)
19363 				return -ENOMEM;
19364 
19365 			env->prog = prog = new_prog;
19366 			insn      = new_prog->insnsi + i + delta;
19367 			continue;
19368 		}
19369 
19370 patch_call_imm:
19371 		fn = env->ops->get_func_proto(insn->imm, env->prog);
19372 		/* all functions that have prototype and verifier allowed
19373 		 * programs to call them, must be real in-kernel functions
19374 		 */
19375 		if (!fn->func) {
19376 			verbose(env,
19377 				"kernel subsystem misconfigured func %s#%d\n",
19378 				func_id_name(insn->imm), insn->imm);
19379 			return -EFAULT;
19380 		}
19381 		insn->imm = fn->func - __bpf_call_base;
19382 	}
19383 
19384 	/* Since poke tab is now finalized, publish aux to tracker. */
19385 	for (i = 0; i < prog->aux->size_poke_tab; i++) {
19386 		map_ptr = prog->aux->poke_tab[i].tail_call.map;
19387 		if (!map_ptr->ops->map_poke_track ||
19388 		    !map_ptr->ops->map_poke_untrack ||
19389 		    !map_ptr->ops->map_poke_run) {
19390 			verbose(env, "bpf verifier is misconfigured\n");
19391 			return -EINVAL;
19392 		}
19393 
19394 		ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
19395 		if (ret < 0) {
19396 			verbose(env, "tracking tail call prog failed\n");
19397 			return ret;
19398 		}
19399 	}
19400 
19401 	sort_kfunc_descs_by_imm_off(env->prog);
19402 
19403 	return 0;
19404 }
19405 
19406 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,
19407 					int position,
19408 					s32 stack_base,
19409 					u32 callback_subprogno,
19410 					u32 *cnt)
19411 {
19412 	s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;
19413 	s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
19414 	s32 r8_offset = stack_base + 2 * BPF_REG_SIZE;
19415 	int reg_loop_max = BPF_REG_6;
19416 	int reg_loop_cnt = BPF_REG_7;
19417 	int reg_loop_ctx = BPF_REG_8;
19418 
19419 	struct bpf_prog *new_prog;
19420 	u32 callback_start;
19421 	u32 call_insn_offset;
19422 	s32 callback_offset;
19423 
19424 	/* This represents an inlined version of bpf_iter.c:bpf_loop,
19425 	 * be careful to modify this code in sync.
19426 	 */
19427 	struct bpf_insn insn_buf[] = {
19428 		/* Return error and jump to the end of the patch if
19429 		 * expected number of iterations is too big.
19430 		 */
19431 		BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2),
19432 		BPF_MOV32_IMM(BPF_REG_0, -E2BIG),
19433 		BPF_JMP_IMM(BPF_JA, 0, 0, 16),
19434 		/* spill R6, R7, R8 to use these as loop vars */
19435 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset),
19436 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset),
19437 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset),
19438 		/* initialize loop vars */
19439 		BPF_MOV64_REG(reg_loop_max, BPF_REG_1),
19440 		BPF_MOV32_IMM(reg_loop_cnt, 0),
19441 		BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3),
19442 		/* loop header,
19443 		 * if reg_loop_cnt >= reg_loop_max skip the loop body
19444 		 */
19445 		BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5),
19446 		/* callback call,
19447 		 * correct callback offset would be set after patching
19448 		 */
19449 		BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt),
19450 		BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx),
19451 		BPF_CALL_REL(0),
19452 		/* increment loop counter */
19453 		BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1),
19454 		/* jump to loop header if callback returned 0 */
19455 		BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6),
19456 		/* return value of bpf_loop,
19457 		 * set R0 to the number of iterations
19458 		 */
19459 		BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt),
19460 		/* restore original values of R6, R7, R8 */
19461 		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset),
19462 		BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset),
19463 		BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset),
19464 	};
19465 
19466 	*cnt = ARRAY_SIZE(insn_buf);
19467 	new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt);
19468 	if (!new_prog)
19469 		return new_prog;
19470 
19471 	/* callback start is known only after patching */
19472 	callback_start = env->subprog_info[callback_subprogno].start;
19473 	/* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */
19474 	call_insn_offset = position + 12;
19475 	callback_offset = callback_start - call_insn_offset - 1;
19476 	new_prog->insnsi[call_insn_offset].imm = callback_offset;
19477 
19478 	return new_prog;
19479 }
19480 
19481 static bool is_bpf_loop_call(struct bpf_insn *insn)
19482 {
19483 	return insn->code == (BPF_JMP | BPF_CALL) &&
19484 		insn->src_reg == 0 &&
19485 		insn->imm == BPF_FUNC_loop;
19486 }
19487 
19488 /* For all sub-programs in the program (including main) check
19489  * insn_aux_data to see if there are bpf_loop calls that require
19490  * inlining. If such calls are found the calls are replaced with a
19491  * sequence of instructions produced by `inline_bpf_loop` function and
19492  * subprog stack_depth is increased by the size of 3 registers.
19493  * This stack space is used to spill values of the R6, R7, R8.  These
19494  * registers are used to store the loop bound, counter and context
19495  * variables.
19496  */
19497 static int optimize_bpf_loop(struct bpf_verifier_env *env)
19498 {
19499 	struct bpf_subprog_info *subprogs = env->subprog_info;
19500 	int i, cur_subprog = 0, cnt, delta = 0;
19501 	struct bpf_insn *insn = env->prog->insnsi;
19502 	int insn_cnt = env->prog->len;
19503 	u16 stack_depth = subprogs[cur_subprog].stack_depth;
19504 	u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
19505 	u16 stack_depth_extra = 0;
19506 
19507 	for (i = 0; i < insn_cnt; i++, insn++) {
19508 		struct bpf_loop_inline_state *inline_state =
19509 			&env->insn_aux_data[i + delta].loop_inline_state;
19510 
19511 		if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) {
19512 			struct bpf_prog *new_prog;
19513 
19514 			stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup;
19515 			new_prog = inline_bpf_loop(env,
19516 						   i + delta,
19517 						   -(stack_depth + stack_depth_extra),
19518 						   inline_state->callback_subprogno,
19519 						   &cnt);
19520 			if (!new_prog)
19521 				return -ENOMEM;
19522 
19523 			delta     += cnt - 1;
19524 			env->prog  = new_prog;
19525 			insn       = new_prog->insnsi + i + delta;
19526 		}
19527 
19528 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
19529 			subprogs[cur_subprog].stack_depth += stack_depth_extra;
19530 			cur_subprog++;
19531 			stack_depth = subprogs[cur_subprog].stack_depth;
19532 			stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
19533 			stack_depth_extra = 0;
19534 		}
19535 	}
19536 
19537 	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
19538 
19539 	return 0;
19540 }
19541 
19542 static void free_states(struct bpf_verifier_env *env)
19543 {
19544 	struct bpf_verifier_state_list *sl, *sln;
19545 	int i;
19546 
19547 	sl = env->free_list;
19548 	while (sl) {
19549 		sln = sl->next;
19550 		free_verifier_state(&sl->state, false);
19551 		kfree(sl);
19552 		sl = sln;
19553 	}
19554 	env->free_list = NULL;
19555 
19556 	if (!env->explored_states)
19557 		return;
19558 
19559 	for (i = 0; i < state_htab_size(env); i++) {
19560 		sl = env->explored_states[i];
19561 
19562 		while (sl) {
19563 			sln = sl->next;
19564 			free_verifier_state(&sl->state, false);
19565 			kfree(sl);
19566 			sl = sln;
19567 		}
19568 		env->explored_states[i] = NULL;
19569 	}
19570 }
19571 
19572 static int do_check_common(struct bpf_verifier_env *env, int subprog)
19573 {
19574 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
19575 	struct bpf_verifier_state *state;
19576 	struct bpf_reg_state *regs;
19577 	int ret, i;
19578 
19579 	env->prev_linfo = NULL;
19580 	env->pass_cnt++;
19581 
19582 	state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
19583 	if (!state)
19584 		return -ENOMEM;
19585 	state->curframe = 0;
19586 	state->speculative = false;
19587 	state->branches = 1;
19588 	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
19589 	if (!state->frame[0]) {
19590 		kfree(state);
19591 		return -ENOMEM;
19592 	}
19593 	env->cur_state = state;
19594 	init_func_state(env, state->frame[0],
19595 			BPF_MAIN_FUNC /* callsite */,
19596 			0 /* frameno */,
19597 			subprog);
19598 	state->first_insn_idx = env->subprog_info[subprog].start;
19599 	state->last_insn_idx = -1;
19600 
19601 	regs = state->frame[state->curframe]->regs;
19602 	if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
19603 		ret = btf_prepare_func_args(env, subprog, regs);
19604 		if (ret)
19605 			goto out;
19606 		for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
19607 			if (regs[i].type == PTR_TO_CTX)
19608 				mark_reg_known_zero(env, regs, i);
19609 			else if (regs[i].type == SCALAR_VALUE)
19610 				mark_reg_unknown(env, regs, i);
19611 			else if (base_type(regs[i].type) == PTR_TO_MEM) {
19612 				const u32 mem_size = regs[i].mem_size;
19613 
19614 				mark_reg_known_zero(env, regs, i);
19615 				regs[i].mem_size = mem_size;
19616 				regs[i].id = ++env->id_gen;
19617 			}
19618 		}
19619 	} else {
19620 		/* 1st arg to a function */
19621 		regs[BPF_REG_1].type = PTR_TO_CTX;
19622 		mark_reg_known_zero(env, regs, BPF_REG_1);
19623 		ret = btf_check_subprog_arg_match(env, subprog, regs);
19624 		if (ret == -EFAULT)
19625 			/* unlikely verifier bug. abort.
19626 			 * ret == 0 and ret < 0 are sadly acceptable for
19627 			 * main() function due to backward compatibility.
19628 			 * Like socket filter program may be written as:
19629 			 * int bpf_prog(struct pt_regs *ctx)
19630 			 * and never dereference that ctx in the program.
19631 			 * 'struct pt_regs' is a type mismatch for socket
19632 			 * filter that should be using 'struct __sk_buff'.
19633 			 */
19634 			goto out;
19635 	}
19636 
19637 	ret = do_check(env);
19638 out:
19639 	/* check for NULL is necessary, since cur_state can be freed inside
19640 	 * do_check() under memory pressure.
19641 	 */
19642 	if (env->cur_state) {
19643 		free_verifier_state(env->cur_state, true);
19644 		env->cur_state = NULL;
19645 	}
19646 	while (!pop_stack(env, NULL, NULL, false));
19647 	if (!ret && pop_log)
19648 		bpf_vlog_reset(&env->log, 0);
19649 	free_states(env);
19650 	return ret;
19651 }
19652 
19653 /* Verify all global functions in a BPF program one by one based on their BTF.
19654  * All global functions must pass verification. Otherwise the whole program is rejected.
19655  * Consider:
19656  * int bar(int);
19657  * int foo(int f)
19658  * {
19659  *    return bar(f);
19660  * }
19661  * int bar(int b)
19662  * {
19663  *    ...
19664  * }
19665  * foo() will be verified first for R1=any_scalar_value. During verification it
19666  * will be assumed that bar() already verified successfully and call to bar()
19667  * from foo() will be checked for type match only. Later bar() will be verified
19668  * independently to check that it's safe for R1=any_scalar_value.
19669  */
19670 static int do_check_subprogs(struct bpf_verifier_env *env)
19671 {
19672 	struct bpf_prog_aux *aux = env->prog->aux;
19673 	int i, ret;
19674 
19675 	if (!aux->func_info)
19676 		return 0;
19677 
19678 	for (i = 1; i < env->subprog_cnt; i++) {
19679 		if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL)
19680 			continue;
19681 		env->insn_idx = env->subprog_info[i].start;
19682 		WARN_ON_ONCE(env->insn_idx == 0);
19683 		ret = do_check_common(env, i);
19684 		if (ret) {
19685 			return ret;
19686 		} else if (env->log.level & BPF_LOG_LEVEL) {
19687 			verbose(env,
19688 				"Func#%d is safe for any args that match its prototype\n",
19689 				i);
19690 		}
19691 	}
19692 	return 0;
19693 }
19694 
19695 static int do_check_main(struct bpf_verifier_env *env)
19696 {
19697 	int ret;
19698 
19699 	env->insn_idx = 0;
19700 	ret = do_check_common(env, 0);
19701 	if (!ret)
19702 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
19703 	return ret;
19704 }
19705 
19706 
19707 static void print_verification_stats(struct bpf_verifier_env *env)
19708 {
19709 	int i;
19710 
19711 	if (env->log.level & BPF_LOG_STATS) {
19712 		verbose(env, "verification time %lld usec\n",
19713 			div_u64(env->verification_time, 1000));
19714 		verbose(env, "stack depth ");
19715 		for (i = 0; i < env->subprog_cnt; i++) {
19716 			u32 depth = env->subprog_info[i].stack_depth;
19717 
19718 			verbose(env, "%d", depth);
19719 			if (i + 1 < env->subprog_cnt)
19720 				verbose(env, "+");
19721 		}
19722 		verbose(env, "\n");
19723 	}
19724 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
19725 		"total_states %d peak_states %d mark_read %d\n",
19726 		env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
19727 		env->max_states_per_insn, env->total_states,
19728 		env->peak_states, env->longest_mark_read_walk);
19729 }
19730 
19731 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
19732 {
19733 	const struct btf_type *t, *func_proto;
19734 	const struct bpf_struct_ops *st_ops;
19735 	const struct btf_member *member;
19736 	struct bpf_prog *prog = env->prog;
19737 	u32 btf_id, member_idx;
19738 	const char *mname;
19739 
19740 	if (!prog->gpl_compatible) {
19741 		verbose(env, "struct ops programs must have a GPL compatible license\n");
19742 		return -EINVAL;
19743 	}
19744 
19745 	btf_id = prog->aux->attach_btf_id;
19746 	st_ops = bpf_struct_ops_find(btf_id);
19747 	if (!st_ops) {
19748 		verbose(env, "attach_btf_id %u is not a supported struct\n",
19749 			btf_id);
19750 		return -ENOTSUPP;
19751 	}
19752 
19753 	t = st_ops->type;
19754 	member_idx = prog->expected_attach_type;
19755 	if (member_idx >= btf_type_vlen(t)) {
19756 		verbose(env, "attach to invalid member idx %u of struct %s\n",
19757 			member_idx, st_ops->name);
19758 		return -EINVAL;
19759 	}
19760 
19761 	member = &btf_type_member(t)[member_idx];
19762 	mname = btf_name_by_offset(btf_vmlinux, member->name_off);
19763 	func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
19764 					       NULL);
19765 	if (!func_proto) {
19766 		verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
19767 			mname, member_idx, st_ops->name);
19768 		return -EINVAL;
19769 	}
19770 
19771 	if (st_ops->check_member) {
19772 		int err = st_ops->check_member(t, member, prog);
19773 
19774 		if (err) {
19775 			verbose(env, "attach to unsupported member %s of struct %s\n",
19776 				mname, st_ops->name);
19777 			return err;
19778 		}
19779 	}
19780 
19781 	prog->aux->attach_func_proto = func_proto;
19782 	prog->aux->attach_func_name = mname;
19783 	env->ops = st_ops->verifier_ops;
19784 
19785 	return 0;
19786 }
19787 #define SECURITY_PREFIX "security_"
19788 
19789 static int check_attach_modify_return(unsigned long addr, const char *func_name)
19790 {
19791 	if (within_error_injection_list(addr) ||
19792 	    !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
19793 		return 0;
19794 
19795 	return -EINVAL;
19796 }
19797 
19798 /* list of non-sleepable functions that are otherwise on
19799  * ALLOW_ERROR_INJECTION list
19800  */
19801 BTF_SET_START(btf_non_sleepable_error_inject)
19802 /* Three functions below can be called from sleepable and non-sleepable context.
19803  * Assume non-sleepable from bpf safety point of view.
19804  */
19805 BTF_ID(func, __filemap_add_folio)
19806 BTF_ID(func, should_fail_alloc_page)
19807 BTF_ID(func, should_failslab)
19808 BTF_SET_END(btf_non_sleepable_error_inject)
19809 
19810 static int check_non_sleepable_error_inject(u32 btf_id)
19811 {
19812 	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
19813 }
19814 
19815 int bpf_check_attach_target(struct bpf_verifier_log *log,
19816 			    const struct bpf_prog *prog,
19817 			    const struct bpf_prog *tgt_prog,
19818 			    u32 btf_id,
19819 			    struct bpf_attach_target_info *tgt_info)
19820 {
19821 	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
19822 	const char prefix[] = "btf_trace_";
19823 	int ret = 0, subprog = -1, i;
19824 	const struct btf_type *t;
19825 	bool conservative = true;
19826 	const char *tname;
19827 	struct btf *btf;
19828 	long addr = 0;
19829 	struct module *mod = NULL;
19830 
19831 	if (!btf_id) {
19832 		bpf_log(log, "Tracing programs must provide btf_id\n");
19833 		return -EINVAL;
19834 	}
19835 	btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
19836 	if (!btf) {
19837 		bpf_log(log,
19838 			"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
19839 		return -EINVAL;
19840 	}
19841 	t = btf_type_by_id(btf, btf_id);
19842 	if (!t) {
19843 		bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
19844 		return -EINVAL;
19845 	}
19846 	tname = btf_name_by_offset(btf, t->name_off);
19847 	if (!tname) {
19848 		bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
19849 		return -EINVAL;
19850 	}
19851 	if (tgt_prog) {
19852 		struct bpf_prog_aux *aux = tgt_prog->aux;
19853 
19854 		if (bpf_prog_is_dev_bound(prog->aux) &&
19855 		    !bpf_prog_dev_bound_match(prog, tgt_prog)) {
19856 			bpf_log(log, "Target program bound device mismatch");
19857 			return -EINVAL;
19858 		}
19859 
19860 		for (i = 0; i < aux->func_info_cnt; i++)
19861 			if (aux->func_info[i].type_id == btf_id) {
19862 				subprog = i;
19863 				break;
19864 			}
19865 		if (subprog == -1) {
19866 			bpf_log(log, "Subprog %s doesn't exist\n", tname);
19867 			return -EINVAL;
19868 		}
19869 		conservative = aux->func_info_aux[subprog].unreliable;
19870 		if (prog_extension) {
19871 			if (conservative) {
19872 				bpf_log(log,
19873 					"Cannot replace static functions\n");
19874 				return -EINVAL;
19875 			}
19876 			if (!prog->jit_requested) {
19877 				bpf_log(log,
19878 					"Extension programs should be JITed\n");
19879 				return -EINVAL;
19880 			}
19881 		}
19882 		if (!tgt_prog->jited) {
19883 			bpf_log(log, "Can attach to only JITed progs\n");
19884 			return -EINVAL;
19885 		}
19886 		if (tgt_prog->type == prog->type) {
19887 			/* Cannot fentry/fexit another fentry/fexit program.
19888 			 * Cannot attach program extension to another extension.
19889 			 * It's ok to attach fentry/fexit to extension program.
19890 			 */
19891 			bpf_log(log, "Cannot recursively attach\n");
19892 			return -EINVAL;
19893 		}
19894 		if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
19895 		    prog_extension &&
19896 		    (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
19897 		     tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
19898 			/* Program extensions can extend all program types
19899 			 * except fentry/fexit. The reason is the following.
19900 			 * The fentry/fexit programs are used for performance
19901 			 * analysis, stats and can be attached to any program
19902 			 * type except themselves. When extension program is
19903 			 * replacing XDP function it is necessary to allow
19904 			 * performance analysis of all functions. Both original
19905 			 * XDP program and its program extension. Hence
19906 			 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is
19907 			 * allowed. If extending of fentry/fexit was allowed it
19908 			 * would be possible to create long call chain
19909 			 * fentry->extension->fentry->extension beyond
19910 			 * reasonable stack size. Hence extending fentry is not
19911 			 * allowed.
19912 			 */
19913 			bpf_log(log, "Cannot extend fentry/fexit\n");
19914 			return -EINVAL;
19915 		}
19916 	} else {
19917 		if (prog_extension) {
19918 			bpf_log(log, "Cannot replace kernel functions\n");
19919 			return -EINVAL;
19920 		}
19921 	}
19922 
19923 	switch (prog->expected_attach_type) {
19924 	case BPF_TRACE_RAW_TP:
19925 		if (tgt_prog) {
19926 			bpf_log(log,
19927 				"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
19928 			return -EINVAL;
19929 		}
19930 		if (!btf_type_is_typedef(t)) {
19931 			bpf_log(log, "attach_btf_id %u is not a typedef\n",
19932 				btf_id);
19933 			return -EINVAL;
19934 		}
19935 		if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
19936 			bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
19937 				btf_id, tname);
19938 			return -EINVAL;
19939 		}
19940 		tname += sizeof(prefix) - 1;
19941 		t = btf_type_by_id(btf, t->type);
19942 		if (!btf_type_is_ptr(t))
19943 			/* should never happen in valid vmlinux build */
19944 			return -EINVAL;
19945 		t = btf_type_by_id(btf, t->type);
19946 		if (!btf_type_is_func_proto(t))
19947 			/* should never happen in valid vmlinux build */
19948 			return -EINVAL;
19949 
19950 		break;
19951 	case BPF_TRACE_ITER:
19952 		if (!btf_type_is_func(t)) {
19953 			bpf_log(log, "attach_btf_id %u is not a function\n",
19954 				btf_id);
19955 			return -EINVAL;
19956 		}
19957 		t = btf_type_by_id(btf, t->type);
19958 		if (!btf_type_is_func_proto(t))
19959 			return -EINVAL;
19960 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
19961 		if (ret)
19962 			return ret;
19963 		break;
19964 	default:
19965 		if (!prog_extension)
19966 			return -EINVAL;
19967 		fallthrough;
19968 	case BPF_MODIFY_RETURN:
19969 	case BPF_LSM_MAC:
19970 	case BPF_LSM_CGROUP:
19971 	case BPF_TRACE_FENTRY:
19972 	case BPF_TRACE_FEXIT:
19973 		if (!btf_type_is_func(t)) {
19974 			bpf_log(log, "attach_btf_id %u is not a function\n",
19975 				btf_id);
19976 			return -EINVAL;
19977 		}
19978 		if (prog_extension &&
19979 		    btf_check_type_match(log, prog, btf, t))
19980 			return -EINVAL;
19981 		t = btf_type_by_id(btf, t->type);
19982 		if (!btf_type_is_func_proto(t))
19983 			return -EINVAL;
19984 
19985 		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
19986 		    (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
19987 		     prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
19988 			return -EINVAL;
19989 
19990 		if (tgt_prog && conservative)
19991 			t = NULL;
19992 
19993 		ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
19994 		if (ret < 0)
19995 			return ret;
19996 
19997 		if (tgt_prog) {
19998 			if (subprog == 0)
19999 				addr = (long) tgt_prog->bpf_func;
20000 			else
20001 				addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
20002 		} else {
20003 			if (btf_is_module(btf)) {
20004 				mod = btf_try_get_module(btf);
20005 				if (mod)
20006 					addr = find_kallsyms_symbol_value(mod, tname);
20007 				else
20008 					addr = 0;
20009 			} else {
20010 				addr = kallsyms_lookup_name(tname);
20011 			}
20012 			if (!addr) {
20013 				module_put(mod);
20014 				bpf_log(log,
20015 					"The address of function %s cannot be found\n",
20016 					tname);
20017 				return -ENOENT;
20018 			}
20019 		}
20020 
20021 		if (prog->aux->sleepable) {
20022 			ret = -EINVAL;
20023 			switch (prog->type) {
20024 			case BPF_PROG_TYPE_TRACING:
20025 
20026 				/* fentry/fexit/fmod_ret progs can be sleepable if they are
20027 				 * attached to ALLOW_ERROR_INJECTION and are not in denylist.
20028 				 */
20029 				if (!check_non_sleepable_error_inject(btf_id) &&
20030 				    within_error_injection_list(addr))
20031 					ret = 0;
20032 				/* fentry/fexit/fmod_ret progs can also be sleepable if they are
20033 				 * in the fmodret id set with the KF_SLEEPABLE flag.
20034 				 */
20035 				else {
20036 					u32 *flags = btf_kfunc_is_modify_return(btf, btf_id,
20037 										prog);
20038 
20039 					if (flags && (*flags & KF_SLEEPABLE))
20040 						ret = 0;
20041 				}
20042 				break;
20043 			case BPF_PROG_TYPE_LSM:
20044 				/* LSM progs check that they are attached to bpf_lsm_*() funcs.
20045 				 * Only some of them are sleepable.
20046 				 */
20047 				if (bpf_lsm_is_sleepable_hook(btf_id))
20048 					ret = 0;
20049 				break;
20050 			default:
20051 				break;
20052 			}
20053 			if (ret) {
20054 				module_put(mod);
20055 				bpf_log(log, "%s is not sleepable\n", tname);
20056 				return ret;
20057 			}
20058 		} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
20059 			if (tgt_prog) {
20060 				module_put(mod);
20061 				bpf_log(log, "can't modify return codes of BPF programs\n");
20062 				return -EINVAL;
20063 			}
20064 			ret = -EINVAL;
20065 			if (btf_kfunc_is_modify_return(btf, btf_id, prog) ||
20066 			    !check_attach_modify_return(addr, tname))
20067 				ret = 0;
20068 			if (ret) {
20069 				module_put(mod);
20070 				bpf_log(log, "%s() is not modifiable\n", tname);
20071 				return ret;
20072 			}
20073 		}
20074 
20075 		break;
20076 	}
20077 	tgt_info->tgt_addr = addr;
20078 	tgt_info->tgt_name = tname;
20079 	tgt_info->tgt_type = t;
20080 	tgt_info->tgt_mod = mod;
20081 	return 0;
20082 }
20083 
20084 BTF_SET_START(btf_id_deny)
20085 BTF_ID_UNUSED
20086 #ifdef CONFIG_SMP
20087 BTF_ID(func, migrate_disable)
20088 BTF_ID(func, migrate_enable)
20089 #endif
20090 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
20091 BTF_ID(func, rcu_read_unlock_strict)
20092 #endif
20093 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
20094 BTF_ID(func, preempt_count_add)
20095 BTF_ID(func, preempt_count_sub)
20096 #endif
20097 #ifdef CONFIG_PREEMPT_RCU
20098 BTF_ID(func, __rcu_read_lock)
20099 BTF_ID(func, __rcu_read_unlock)
20100 #endif
20101 BTF_SET_END(btf_id_deny)
20102 
20103 static bool can_be_sleepable(struct bpf_prog *prog)
20104 {
20105 	if (prog->type == BPF_PROG_TYPE_TRACING) {
20106 		switch (prog->expected_attach_type) {
20107 		case BPF_TRACE_FENTRY:
20108 		case BPF_TRACE_FEXIT:
20109 		case BPF_MODIFY_RETURN:
20110 		case BPF_TRACE_ITER:
20111 			return true;
20112 		default:
20113 			return false;
20114 		}
20115 	}
20116 	return prog->type == BPF_PROG_TYPE_LSM ||
20117 	       prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
20118 	       prog->type == BPF_PROG_TYPE_STRUCT_OPS;
20119 }
20120 
20121 static int check_attach_btf_id(struct bpf_verifier_env *env)
20122 {
20123 	struct bpf_prog *prog = env->prog;
20124 	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
20125 	struct bpf_attach_target_info tgt_info = {};
20126 	u32 btf_id = prog->aux->attach_btf_id;
20127 	struct bpf_trampoline *tr;
20128 	int ret;
20129 	u64 key;
20130 
20131 	if (prog->type == BPF_PROG_TYPE_SYSCALL) {
20132 		if (prog->aux->sleepable)
20133 			/* attach_btf_id checked to be zero already */
20134 			return 0;
20135 		verbose(env, "Syscall programs can only be sleepable\n");
20136 		return -EINVAL;
20137 	}
20138 
20139 	if (prog->aux->sleepable && !can_be_sleepable(prog)) {
20140 		verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
20141 		return -EINVAL;
20142 	}
20143 
20144 	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
20145 		return check_struct_ops_btf_id(env);
20146 
20147 	if (prog->type != BPF_PROG_TYPE_TRACING &&
20148 	    prog->type != BPF_PROG_TYPE_LSM &&
20149 	    prog->type != BPF_PROG_TYPE_EXT)
20150 		return 0;
20151 
20152 	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
20153 	if (ret)
20154 		return ret;
20155 
20156 	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
20157 		/* to make freplace equivalent to their targets, they need to
20158 		 * inherit env->ops and expected_attach_type for the rest of the
20159 		 * verification
20160 		 */
20161 		env->ops = bpf_verifier_ops[tgt_prog->type];
20162 		prog->expected_attach_type = tgt_prog->expected_attach_type;
20163 	}
20164 
20165 	/* store info about the attachment target that will be used later */
20166 	prog->aux->attach_func_proto = tgt_info.tgt_type;
20167 	prog->aux->attach_func_name = tgt_info.tgt_name;
20168 	prog->aux->mod = tgt_info.tgt_mod;
20169 
20170 	if (tgt_prog) {
20171 		prog->aux->saved_dst_prog_type = tgt_prog->type;
20172 		prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
20173 	}
20174 
20175 	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
20176 		prog->aux->attach_btf_trace = true;
20177 		return 0;
20178 	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
20179 		if (!bpf_iter_prog_supported(prog))
20180 			return -EINVAL;
20181 		return 0;
20182 	}
20183 
20184 	if (prog->type == BPF_PROG_TYPE_LSM) {
20185 		ret = bpf_lsm_verify_prog(&env->log, prog);
20186 		if (ret < 0)
20187 			return ret;
20188 	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
20189 		   btf_id_set_contains(&btf_id_deny, btf_id)) {
20190 		return -EINVAL;
20191 	}
20192 
20193 	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
20194 	tr = bpf_trampoline_get(key, &tgt_info);
20195 	if (!tr)
20196 		return -ENOMEM;
20197 
20198 	if (tgt_prog && tgt_prog->aux->tail_call_reachable)
20199 		tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;
20200 
20201 	prog->aux->dst_trampoline = tr;
20202 	return 0;
20203 }
20204 
20205 struct btf *bpf_get_btf_vmlinux(void)
20206 {
20207 	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
20208 		mutex_lock(&bpf_verifier_lock);
20209 		if (!btf_vmlinux)
20210 			btf_vmlinux = btf_parse_vmlinux();
20211 		mutex_unlock(&bpf_verifier_lock);
20212 	}
20213 	return btf_vmlinux;
20214 }
20215 
20216 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
20217 {
20218 	u64 start_time = ktime_get_ns();
20219 	struct bpf_verifier_env *env;
20220 	int i, len, ret = -EINVAL, err;
20221 	u32 log_true_size;
20222 	bool is_priv;
20223 
20224 	/* no program is valid */
20225 	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
20226 		return -EINVAL;
20227 
20228 	/* 'struct bpf_verifier_env' can be global, but since it's not small,
20229 	 * allocate/free it every time bpf_check() is called
20230 	 */
20231 	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
20232 	if (!env)
20233 		return -ENOMEM;
20234 
20235 	env->bt.env = env;
20236 
20237 	len = (*prog)->len;
20238 	env->insn_aux_data =
20239 		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
20240 	ret = -ENOMEM;
20241 	if (!env->insn_aux_data)
20242 		goto err_free_env;
20243 	for (i = 0; i < len; i++)
20244 		env->insn_aux_data[i].orig_idx = i;
20245 	env->prog = *prog;
20246 	env->ops = bpf_verifier_ops[env->prog->type];
20247 	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
20248 	is_priv = bpf_capable();
20249 
20250 	bpf_get_btf_vmlinux();
20251 
20252 	/* grab the mutex to protect few globals used by verifier */
20253 	if (!is_priv)
20254 		mutex_lock(&bpf_verifier_lock);
20255 
20256 	/* user could have requested verbose verifier output
20257 	 * and supplied buffer to store the verification trace
20258 	 */
20259 	ret = bpf_vlog_init(&env->log, attr->log_level,
20260 			    (char __user *) (unsigned long) attr->log_buf,
20261 			    attr->log_size);
20262 	if (ret)
20263 		goto err_unlock;
20264 
20265 	mark_verifier_state_clean(env);
20266 
20267 	if (IS_ERR(btf_vmlinux)) {
20268 		/* Either gcc or pahole or kernel are broken. */
20269 		verbose(env, "in-kernel BTF is malformed\n");
20270 		ret = PTR_ERR(btf_vmlinux);
20271 		goto skip_full_check;
20272 	}
20273 
20274 	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
20275 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
20276 		env->strict_alignment = true;
20277 	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
20278 		env->strict_alignment = false;
20279 
20280 	env->allow_ptr_leaks = bpf_allow_ptr_leaks();
20281 	env->allow_uninit_stack = bpf_allow_uninit_stack();
20282 	env->bypass_spec_v1 = bpf_bypass_spec_v1();
20283 	env->bypass_spec_v4 = bpf_bypass_spec_v4();
20284 	env->bpf_capable = bpf_capable();
20285 
20286 	if (is_priv)
20287 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
20288 
20289 	env->explored_states = kvcalloc(state_htab_size(env),
20290 				       sizeof(struct bpf_verifier_state_list *),
20291 				       GFP_USER);
20292 	ret = -ENOMEM;
20293 	if (!env->explored_states)
20294 		goto skip_full_check;
20295 
20296 	ret = add_subprog_and_kfunc(env);
20297 	if (ret < 0)
20298 		goto skip_full_check;
20299 
20300 	ret = check_subprogs(env);
20301 	if (ret < 0)
20302 		goto skip_full_check;
20303 
20304 	ret = check_btf_info(env, attr, uattr);
20305 	if (ret < 0)
20306 		goto skip_full_check;
20307 
20308 	ret = check_attach_btf_id(env);
20309 	if (ret)
20310 		goto skip_full_check;
20311 
20312 	ret = resolve_pseudo_ldimm64(env);
20313 	if (ret < 0)
20314 		goto skip_full_check;
20315 
20316 	if (bpf_prog_is_offloaded(env->prog->aux)) {
20317 		ret = bpf_prog_offload_verifier_prep(env->prog);
20318 		if (ret)
20319 			goto skip_full_check;
20320 	}
20321 
20322 	ret = check_cfg(env);
20323 	if (ret < 0)
20324 		goto skip_full_check;
20325 
20326 	ret = do_check_subprogs(env);
20327 	ret = ret ?: do_check_main(env);
20328 
20329 	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
20330 		ret = bpf_prog_offload_finalize(env);
20331 
20332 skip_full_check:
20333 	kvfree(env->explored_states);
20334 
20335 	if (ret == 0)
20336 		ret = check_max_stack_depth(env);
20337 
20338 	/* instruction rewrites happen after this point */
20339 	if (ret == 0)
20340 		ret = optimize_bpf_loop(env);
20341 
20342 	if (is_priv) {
20343 		if (ret == 0)
20344 			opt_hard_wire_dead_code_branches(env);
20345 		if (ret == 0)
20346 			ret = opt_remove_dead_code(env);
20347 		if (ret == 0)
20348 			ret = opt_remove_nops(env);
20349 	} else {
20350 		if (ret == 0)
20351 			sanitize_dead_code(env);
20352 	}
20353 
20354 	if (ret == 0)
20355 		/* program is valid, convert *(u32*)(ctx + off) accesses */
20356 		ret = convert_ctx_accesses(env);
20357 
20358 	if (ret == 0)
20359 		ret = do_misc_fixups(env);
20360 
20361 	/* do 32-bit optimization after insn patching has done so those patched
20362 	 * insns could be handled correctly.
20363 	 */
20364 	if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) {
20365 		ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
20366 		env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
20367 								     : false;
20368 	}
20369 
20370 	if (ret == 0)
20371 		ret = fixup_call_args(env);
20372 
20373 	env->verification_time = ktime_get_ns() - start_time;
20374 	print_verification_stats(env);
20375 	env->prog->aux->verified_insns = env->insn_processed;
20376 
20377 	/* preserve original error even if log finalization is successful */
20378 	err = bpf_vlog_finalize(&env->log, &log_true_size);
20379 	if (err)
20380 		ret = err;
20381 
20382 	if (uattr_size >= offsetofend(union bpf_attr, log_true_size) &&
20383 	    copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size),
20384 				  &log_true_size, sizeof(log_true_size))) {
20385 		ret = -EFAULT;
20386 		goto err_release_maps;
20387 	}
20388 
20389 	if (ret)
20390 		goto err_release_maps;
20391 
20392 	if (env->used_map_cnt) {
20393 		/* if program passed verifier, update used_maps in bpf_prog_info */
20394 		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
20395 							  sizeof(env->used_maps[0]),
20396 							  GFP_KERNEL);
20397 
20398 		if (!env->prog->aux->used_maps) {
20399 			ret = -ENOMEM;
20400 			goto err_release_maps;
20401 		}
20402 
20403 		memcpy(env->prog->aux->used_maps, env->used_maps,
20404 		       sizeof(env->used_maps[0]) * env->used_map_cnt);
20405 		env->prog->aux->used_map_cnt = env->used_map_cnt;
20406 	}
20407 	if (env->used_btf_cnt) {
20408 		/* if program passed verifier, update used_btfs in bpf_prog_aux */
20409 		env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
20410 							  sizeof(env->used_btfs[0]),
20411 							  GFP_KERNEL);
20412 		if (!env->prog->aux->used_btfs) {
20413 			ret = -ENOMEM;
20414 			goto err_release_maps;
20415 		}
20416 
20417 		memcpy(env->prog->aux->used_btfs, env->used_btfs,
20418 		       sizeof(env->used_btfs[0]) * env->used_btf_cnt);
20419 		env->prog->aux->used_btf_cnt = env->used_btf_cnt;
20420 	}
20421 	if (env->used_map_cnt || env->used_btf_cnt) {
20422 		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
20423 		 * bpf_ld_imm64 instructions
20424 		 */
20425 		convert_pseudo_ld_imm64(env);
20426 	}
20427 
20428 	adjust_btf_func(env);
20429 
20430 err_release_maps:
20431 	if (!env->prog->aux->used_maps)
20432 		/* if we didn't copy map pointers into bpf_prog_info, release
20433 		 * them now. Otherwise free_used_maps() will release them.
20434 		 */
20435 		release_maps(env);
20436 	if (!env->prog->aux->used_btfs)
20437 		release_btfs(env);
20438 
20439 	/* extension progs temporarily inherit the attach_type of their targets
20440 	   for verification purposes, so set it back to zero before returning
20441 	 */
20442 	if (env->prog->type == BPF_PROG_TYPE_EXT)
20443 		env->prog->expected_attach_type = 0;
20444 
20445 	*prog = env->prog;
20446 err_unlock:
20447 	if (!is_priv)
20448 		mutex_unlock(&bpf_verifier_lock);
20449 	vfree(env->insn_aux_data);
20450 err_free_env:
20451 	kfree(env);
20452 	return ret;
20453 }
20454