1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 #include <linux/poison.h> 27 #include <linux/module.h> 28 29 #include "disasm.h" 30 31 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 32 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 33 [_id] = & _name ## _verifier_ops, 34 #define BPF_MAP_TYPE(_id, _ops) 35 #define BPF_LINK_TYPE(_id, _name) 36 #include <linux/bpf_types.h> 37 #undef BPF_PROG_TYPE 38 #undef BPF_MAP_TYPE 39 #undef BPF_LINK_TYPE 40 }; 41 42 /* bpf_check() is a static code analyzer that walks eBPF program 43 * instruction by instruction and updates register/stack state. 44 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 45 * 46 * The first pass is depth-first-search to check that the program is a DAG. 47 * It rejects the following programs: 48 * - larger than BPF_MAXINSNS insns 49 * - if loop is present (detected via back-edge) 50 * - unreachable insns exist (shouldn't be a forest. program = one function) 51 * - out of bounds or malformed jumps 52 * The second pass is all possible path descent from the 1st insn. 53 * Since it's analyzing all paths through the program, the length of the 54 * analysis is limited to 64k insn, which may be hit even if total number of 55 * insn is less then 4K, but there are too many branches that change stack/regs. 56 * Number of 'branches to be analyzed' is limited to 1k 57 * 58 * On entry to each instruction, each register has a type, and the instruction 59 * changes the types of the registers depending on instruction semantics. 60 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 61 * copied to R1. 62 * 63 * All registers are 64-bit. 64 * R0 - return register 65 * R1-R5 argument passing registers 66 * R6-R9 callee saved registers 67 * R10 - frame pointer read-only 68 * 69 * At the start of BPF program the register R1 contains a pointer to bpf_context 70 * and has type PTR_TO_CTX. 71 * 72 * Verifier tracks arithmetic operations on pointers in case: 73 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 74 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 75 * 1st insn copies R10 (which has FRAME_PTR) type into R1 76 * and 2nd arithmetic instruction is pattern matched to recognize 77 * that it wants to construct a pointer to some element within stack. 78 * So after 2nd insn, the register R1 has type PTR_TO_STACK 79 * (and -20 constant is saved for further stack bounds checking). 80 * Meaning that this reg is a pointer to stack plus known immediate constant. 81 * 82 * Most of the time the registers have SCALAR_VALUE type, which 83 * means the register has some value, but it's not a valid pointer. 84 * (like pointer plus pointer becomes SCALAR_VALUE type) 85 * 86 * When verifier sees load or store instructions the type of base register 87 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 88 * four pointer types recognized by check_mem_access() function. 89 * 90 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 91 * and the range of [ptr, ptr + map's value_size) is accessible. 92 * 93 * registers used to pass values to function calls are checked against 94 * function argument constraints. 95 * 96 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 97 * It means that the register type passed to this function must be 98 * PTR_TO_STACK and it will be used inside the function as 99 * 'pointer to map element key' 100 * 101 * For example the argument constraints for bpf_map_lookup_elem(): 102 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 103 * .arg1_type = ARG_CONST_MAP_PTR, 104 * .arg2_type = ARG_PTR_TO_MAP_KEY, 105 * 106 * ret_type says that this function returns 'pointer to map elem value or null' 107 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 108 * 2nd argument should be a pointer to stack, which will be used inside 109 * the helper function as a pointer to map element key. 110 * 111 * On the kernel side the helper function looks like: 112 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 113 * { 114 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 115 * void *key = (void *) (unsigned long) r2; 116 * void *value; 117 * 118 * here kernel can access 'key' and 'map' pointers safely, knowing that 119 * [key, key + map->key_size) bytes are valid and were initialized on 120 * the stack of eBPF program. 121 * } 122 * 123 * Corresponding eBPF program may look like: 124 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 125 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 126 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 127 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 128 * here verifier looks at prototype of map_lookup_elem() and sees: 129 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 130 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 131 * 132 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 133 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 134 * and were initialized prior to this call. 135 * If it's ok, then verifier allows this BPF_CALL insn and looks at 136 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 137 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 138 * returns either pointer to map value or NULL. 139 * 140 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 141 * insn, the register holding that pointer in the true branch changes state to 142 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 143 * branch. See check_cond_jmp_op(). 144 * 145 * After the call R0 is set to return type of the function and registers R1-R5 146 * are set to NOT_INIT to indicate that they are no longer readable. 147 * 148 * The following reference types represent a potential reference to a kernel 149 * resource which, after first being allocated, must be checked and freed by 150 * the BPF program: 151 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 152 * 153 * When the verifier sees a helper call return a reference type, it allocates a 154 * pointer id for the reference and stores it in the current function state. 155 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 156 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 157 * passes through a NULL-check conditional. For the branch wherein the state is 158 * changed to CONST_IMM, the verifier releases the reference. 159 * 160 * For each helper function that allocates a reference, such as 161 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 162 * bpf_sk_release(). When a reference type passes into the release function, 163 * the verifier also releases the reference. If any unchecked or unreleased 164 * reference remains at the end of the program, the verifier rejects it. 165 */ 166 167 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 168 struct bpf_verifier_stack_elem { 169 /* verifer state is 'st' 170 * before processing instruction 'insn_idx' 171 * and after processing instruction 'prev_insn_idx' 172 */ 173 struct bpf_verifier_state st; 174 int insn_idx; 175 int prev_insn_idx; 176 struct bpf_verifier_stack_elem *next; 177 /* length of verifier log at the time this state was pushed on stack */ 178 u32 log_pos; 179 }; 180 181 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 182 #define BPF_COMPLEXITY_LIMIT_STATES 64 183 184 #define BPF_MAP_KEY_POISON (1ULL << 63) 185 #define BPF_MAP_KEY_SEEN (1ULL << 62) 186 187 #define BPF_MAP_PTR_UNPRIV 1UL 188 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 189 POISON_POINTER_DELTA)) 190 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 191 192 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 193 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 194 static void invalidate_non_owning_refs(struct bpf_verifier_env *env); 195 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); 196 static int ref_set_non_owning(struct bpf_verifier_env *env, 197 struct bpf_reg_state *reg); 198 static void specialize_kfunc(struct bpf_verifier_env *env, 199 u32 func_id, u16 offset, unsigned long *addr); 200 201 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 202 { 203 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 204 } 205 206 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 207 { 208 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 209 } 210 211 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 212 const struct bpf_map *map, bool unpriv) 213 { 214 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 215 unpriv |= bpf_map_ptr_unpriv(aux); 216 aux->map_ptr_state = (unsigned long)map | 217 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 218 } 219 220 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 221 { 222 return aux->map_key_state & BPF_MAP_KEY_POISON; 223 } 224 225 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 226 { 227 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 228 } 229 230 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 231 { 232 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 233 } 234 235 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 236 { 237 bool poisoned = bpf_map_key_poisoned(aux); 238 239 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 240 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 241 } 242 243 static bool bpf_pseudo_call(const struct bpf_insn *insn) 244 { 245 return insn->code == (BPF_JMP | BPF_CALL) && 246 insn->src_reg == BPF_PSEUDO_CALL; 247 } 248 249 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 250 { 251 return insn->code == (BPF_JMP | BPF_CALL) && 252 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 253 } 254 255 struct bpf_call_arg_meta { 256 struct bpf_map *map_ptr; 257 bool raw_mode; 258 bool pkt_access; 259 u8 release_regno; 260 int regno; 261 int access_size; 262 int mem_size; 263 u64 msize_max_value; 264 int ref_obj_id; 265 int dynptr_id; 266 int map_uid; 267 int func_id; 268 struct btf *btf; 269 u32 btf_id; 270 struct btf *ret_btf; 271 u32 ret_btf_id; 272 u32 subprogno; 273 struct btf_field *kptr_field; 274 }; 275 276 struct btf_and_id { 277 struct btf *btf; 278 u32 btf_id; 279 }; 280 281 struct bpf_kfunc_call_arg_meta { 282 /* In parameters */ 283 struct btf *btf; 284 u32 func_id; 285 u32 kfunc_flags; 286 const struct btf_type *func_proto; 287 const char *func_name; 288 /* Out parameters */ 289 u32 ref_obj_id; 290 u8 release_regno; 291 bool r0_rdonly; 292 u32 ret_btf_id; 293 u64 r0_size; 294 u32 subprogno; 295 struct { 296 u64 value; 297 bool found; 298 } arg_constant; 299 union { 300 struct btf_and_id arg_obj_drop; 301 struct btf_and_id arg_refcount_acquire; 302 }; 303 struct { 304 struct btf_field *field; 305 } arg_list_head; 306 struct { 307 struct btf_field *field; 308 } arg_rbtree_root; 309 struct { 310 enum bpf_dynptr_type type; 311 u32 id; 312 } initialized_dynptr; 313 struct { 314 u8 spi; 315 u8 frameno; 316 } iter; 317 u64 mem_size; 318 }; 319 320 struct btf *btf_vmlinux; 321 322 static DEFINE_MUTEX(bpf_verifier_lock); 323 324 static const struct bpf_line_info * 325 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 326 { 327 const struct bpf_line_info *linfo; 328 const struct bpf_prog *prog; 329 u32 i, nr_linfo; 330 331 prog = env->prog; 332 nr_linfo = prog->aux->nr_linfo; 333 334 if (!nr_linfo || insn_off >= prog->len) 335 return NULL; 336 337 linfo = prog->aux->linfo; 338 for (i = 1; i < nr_linfo; i++) 339 if (insn_off < linfo[i].insn_off) 340 break; 341 342 return &linfo[i - 1]; 343 } 344 345 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 346 { 347 struct bpf_verifier_env *env = private_data; 348 va_list args; 349 350 if (!bpf_verifier_log_needed(&env->log)) 351 return; 352 353 va_start(args, fmt); 354 bpf_verifier_vlog(&env->log, fmt, args); 355 va_end(args); 356 } 357 358 static const char *ltrim(const char *s) 359 { 360 while (isspace(*s)) 361 s++; 362 363 return s; 364 } 365 366 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 367 u32 insn_off, 368 const char *prefix_fmt, ...) 369 { 370 const struct bpf_line_info *linfo; 371 372 if (!bpf_verifier_log_needed(&env->log)) 373 return; 374 375 linfo = find_linfo(env, insn_off); 376 if (!linfo || linfo == env->prev_linfo) 377 return; 378 379 if (prefix_fmt) { 380 va_list args; 381 382 va_start(args, prefix_fmt); 383 bpf_verifier_vlog(&env->log, prefix_fmt, args); 384 va_end(args); 385 } 386 387 verbose(env, "%s\n", 388 ltrim(btf_name_by_offset(env->prog->aux->btf, 389 linfo->line_off))); 390 391 env->prev_linfo = linfo; 392 } 393 394 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 395 struct bpf_reg_state *reg, 396 struct tnum *range, const char *ctx, 397 const char *reg_name) 398 { 399 char tn_buf[48]; 400 401 verbose(env, "At %s the register %s ", ctx, reg_name); 402 if (!tnum_is_unknown(reg->var_off)) { 403 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 404 verbose(env, "has value %s", tn_buf); 405 } else { 406 verbose(env, "has unknown scalar value"); 407 } 408 tnum_strn(tn_buf, sizeof(tn_buf), *range); 409 verbose(env, " should have been in %s\n", tn_buf); 410 } 411 412 static bool type_is_pkt_pointer(enum bpf_reg_type type) 413 { 414 type = base_type(type); 415 return type == PTR_TO_PACKET || 416 type == PTR_TO_PACKET_META; 417 } 418 419 static bool type_is_sk_pointer(enum bpf_reg_type type) 420 { 421 return type == PTR_TO_SOCKET || 422 type == PTR_TO_SOCK_COMMON || 423 type == PTR_TO_TCP_SOCK || 424 type == PTR_TO_XDP_SOCK; 425 } 426 427 static bool type_may_be_null(u32 type) 428 { 429 return type & PTR_MAYBE_NULL; 430 } 431 432 static bool reg_type_not_null(enum bpf_reg_type type) 433 { 434 if (type_may_be_null(type)) 435 return false; 436 437 type = base_type(type); 438 return type == PTR_TO_SOCKET || 439 type == PTR_TO_TCP_SOCK || 440 type == PTR_TO_MAP_VALUE || 441 type == PTR_TO_MAP_KEY || 442 type == PTR_TO_SOCK_COMMON || 443 type == PTR_TO_MEM; 444 } 445 446 static bool type_is_ptr_alloc_obj(u32 type) 447 { 448 return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; 449 } 450 451 static bool type_is_non_owning_ref(u32 type) 452 { 453 return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; 454 } 455 456 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 457 { 458 struct btf_record *rec = NULL; 459 struct btf_struct_meta *meta; 460 461 if (reg->type == PTR_TO_MAP_VALUE) { 462 rec = reg->map_ptr->record; 463 } else if (type_is_ptr_alloc_obj(reg->type)) { 464 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 465 if (meta) 466 rec = meta->record; 467 } 468 return rec; 469 } 470 471 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 472 { 473 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); 474 } 475 476 static bool type_is_rdonly_mem(u32 type) 477 { 478 return type & MEM_RDONLY; 479 } 480 481 static bool is_acquire_function(enum bpf_func_id func_id, 482 const struct bpf_map *map) 483 { 484 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 485 486 if (func_id == BPF_FUNC_sk_lookup_tcp || 487 func_id == BPF_FUNC_sk_lookup_udp || 488 func_id == BPF_FUNC_skc_lookup_tcp || 489 func_id == BPF_FUNC_ringbuf_reserve || 490 func_id == BPF_FUNC_kptr_xchg) 491 return true; 492 493 if (func_id == BPF_FUNC_map_lookup_elem && 494 (map_type == BPF_MAP_TYPE_SOCKMAP || 495 map_type == BPF_MAP_TYPE_SOCKHASH)) 496 return true; 497 498 return false; 499 } 500 501 static bool is_ptr_cast_function(enum bpf_func_id func_id) 502 { 503 return func_id == BPF_FUNC_tcp_sock || 504 func_id == BPF_FUNC_sk_fullsock || 505 func_id == BPF_FUNC_skc_to_tcp_sock || 506 func_id == BPF_FUNC_skc_to_tcp6_sock || 507 func_id == BPF_FUNC_skc_to_udp6_sock || 508 func_id == BPF_FUNC_skc_to_mptcp_sock || 509 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 510 func_id == BPF_FUNC_skc_to_tcp_request_sock; 511 } 512 513 static bool is_dynptr_ref_function(enum bpf_func_id func_id) 514 { 515 return func_id == BPF_FUNC_dynptr_data; 516 } 517 518 static bool is_callback_calling_function(enum bpf_func_id func_id) 519 { 520 return func_id == BPF_FUNC_for_each_map_elem || 521 func_id == BPF_FUNC_timer_set_callback || 522 func_id == BPF_FUNC_find_vma || 523 func_id == BPF_FUNC_loop || 524 func_id == BPF_FUNC_user_ringbuf_drain; 525 } 526 527 static bool is_storage_get_function(enum bpf_func_id func_id) 528 { 529 return func_id == BPF_FUNC_sk_storage_get || 530 func_id == BPF_FUNC_inode_storage_get || 531 func_id == BPF_FUNC_task_storage_get || 532 func_id == BPF_FUNC_cgrp_storage_get; 533 } 534 535 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, 536 const struct bpf_map *map) 537 { 538 int ref_obj_uses = 0; 539 540 if (is_ptr_cast_function(func_id)) 541 ref_obj_uses++; 542 if (is_acquire_function(func_id, map)) 543 ref_obj_uses++; 544 if (is_dynptr_ref_function(func_id)) 545 ref_obj_uses++; 546 547 return ref_obj_uses > 1; 548 } 549 550 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 551 { 552 return BPF_CLASS(insn->code) == BPF_STX && 553 BPF_MODE(insn->code) == BPF_ATOMIC && 554 insn->imm == BPF_CMPXCHG; 555 } 556 557 /* string representation of 'enum bpf_reg_type' 558 * 559 * Note that reg_type_str() can not appear more than once in a single verbose() 560 * statement. 561 */ 562 static const char *reg_type_str(struct bpf_verifier_env *env, 563 enum bpf_reg_type type) 564 { 565 char postfix[16] = {0}, prefix[64] = {0}; 566 static const char * const str[] = { 567 [NOT_INIT] = "?", 568 [SCALAR_VALUE] = "scalar", 569 [PTR_TO_CTX] = "ctx", 570 [CONST_PTR_TO_MAP] = "map_ptr", 571 [PTR_TO_MAP_VALUE] = "map_value", 572 [PTR_TO_STACK] = "fp", 573 [PTR_TO_PACKET] = "pkt", 574 [PTR_TO_PACKET_META] = "pkt_meta", 575 [PTR_TO_PACKET_END] = "pkt_end", 576 [PTR_TO_FLOW_KEYS] = "flow_keys", 577 [PTR_TO_SOCKET] = "sock", 578 [PTR_TO_SOCK_COMMON] = "sock_common", 579 [PTR_TO_TCP_SOCK] = "tcp_sock", 580 [PTR_TO_TP_BUFFER] = "tp_buffer", 581 [PTR_TO_XDP_SOCK] = "xdp_sock", 582 [PTR_TO_BTF_ID] = "ptr_", 583 [PTR_TO_MEM] = "mem", 584 [PTR_TO_BUF] = "buf", 585 [PTR_TO_FUNC] = "func", 586 [PTR_TO_MAP_KEY] = "map_key", 587 [CONST_PTR_TO_DYNPTR] = "dynptr_ptr", 588 }; 589 590 if (type & PTR_MAYBE_NULL) { 591 if (base_type(type) == PTR_TO_BTF_ID) 592 strncpy(postfix, "or_null_", 16); 593 else 594 strncpy(postfix, "_or_null", 16); 595 } 596 597 snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", 598 type & MEM_RDONLY ? "rdonly_" : "", 599 type & MEM_RINGBUF ? "ringbuf_" : "", 600 type & MEM_USER ? "user_" : "", 601 type & MEM_PERCPU ? "percpu_" : "", 602 type & MEM_RCU ? "rcu_" : "", 603 type & PTR_UNTRUSTED ? "untrusted_" : "", 604 type & PTR_TRUSTED ? "trusted_" : "" 605 ); 606 607 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", 608 prefix, str[base_type(type)], postfix); 609 return env->type_str_buf; 610 } 611 612 static char slot_type_char[] = { 613 [STACK_INVALID] = '?', 614 [STACK_SPILL] = 'r', 615 [STACK_MISC] = 'm', 616 [STACK_ZERO] = '0', 617 [STACK_DYNPTR] = 'd', 618 [STACK_ITER] = 'i', 619 }; 620 621 static void print_liveness(struct bpf_verifier_env *env, 622 enum bpf_reg_liveness live) 623 { 624 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 625 verbose(env, "_"); 626 if (live & REG_LIVE_READ) 627 verbose(env, "r"); 628 if (live & REG_LIVE_WRITTEN) 629 verbose(env, "w"); 630 if (live & REG_LIVE_DONE) 631 verbose(env, "D"); 632 } 633 634 static int __get_spi(s32 off) 635 { 636 return (-off - 1) / BPF_REG_SIZE; 637 } 638 639 static struct bpf_func_state *func(struct bpf_verifier_env *env, 640 const struct bpf_reg_state *reg) 641 { 642 struct bpf_verifier_state *cur = env->cur_state; 643 644 return cur->frame[reg->frameno]; 645 } 646 647 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 648 { 649 int allocated_slots = state->allocated_stack / BPF_REG_SIZE; 650 651 /* We need to check that slots between [spi - nr_slots + 1, spi] are 652 * within [0, allocated_stack). 653 * 654 * Please note that the spi grows downwards. For example, a dynptr 655 * takes the size of two stack slots; the first slot will be at 656 * spi and the second slot will be at spi - 1. 657 */ 658 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 659 } 660 661 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 662 const char *obj_kind, int nr_slots) 663 { 664 int off, spi; 665 666 if (!tnum_is_const(reg->var_off)) { 667 verbose(env, "%s has to be at a constant offset\n", obj_kind); 668 return -EINVAL; 669 } 670 671 off = reg->off + reg->var_off.value; 672 if (off % BPF_REG_SIZE) { 673 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 674 return -EINVAL; 675 } 676 677 spi = __get_spi(off); 678 if (spi + 1 < nr_slots) { 679 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); 680 return -EINVAL; 681 } 682 683 if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots)) 684 return -ERANGE; 685 return spi; 686 } 687 688 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 689 { 690 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); 691 } 692 693 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) 694 { 695 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); 696 } 697 698 static const char *btf_type_name(const struct btf *btf, u32 id) 699 { 700 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 701 } 702 703 static const char *dynptr_type_str(enum bpf_dynptr_type type) 704 { 705 switch (type) { 706 case BPF_DYNPTR_TYPE_LOCAL: 707 return "local"; 708 case BPF_DYNPTR_TYPE_RINGBUF: 709 return "ringbuf"; 710 case BPF_DYNPTR_TYPE_SKB: 711 return "skb"; 712 case BPF_DYNPTR_TYPE_XDP: 713 return "xdp"; 714 case BPF_DYNPTR_TYPE_INVALID: 715 return "<invalid>"; 716 default: 717 WARN_ONCE(1, "unknown dynptr type %d\n", type); 718 return "<unknown>"; 719 } 720 } 721 722 static const char *iter_type_str(const struct btf *btf, u32 btf_id) 723 { 724 if (!btf || btf_id == 0) 725 return "<invalid>"; 726 727 /* we already validated that type is valid and has conforming name */ 728 return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1; 729 } 730 731 static const char *iter_state_str(enum bpf_iter_state state) 732 { 733 switch (state) { 734 case BPF_ITER_STATE_ACTIVE: 735 return "active"; 736 case BPF_ITER_STATE_DRAINED: 737 return "drained"; 738 case BPF_ITER_STATE_INVALID: 739 return "<invalid>"; 740 default: 741 WARN_ONCE(1, "unknown iter state %d\n", state); 742 return "<unknown>"; 743 } 744 } 745 746 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) 747 { 748 env->scratched_regs |= 1U << regno; 749 } 750 751 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) 752 { 753 env->scratched_stack_slots |= 1ULL << spi; 754 } 755 756 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) 757 { 758 return (env->scratched_regs >> regno) & 1; 759 } 760 761 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) 762 { 763 return (env->scratched_stack_slots >> regno) & 1; 764 } 765 766 static bool verifier_state_scratched(const struct bpf_verifier_env *env) 767 { 768 return env->scratched_regs || env->scratched_stack_slots; 769 } 770 771 static void mark_verifier_state_clean(struct bpf_verifier_env *env) 772 { 773 env->scratched_regs = 0U; 774 env->scratched_stack_slots = 0ULL; 775 } 776 777 /* Used for printing the entire verifier state. */ 778 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) 779 { 780 env->scratched_regs = ~0U; 781 env->scratched_stack_slots = ~0ULL; 782 } 783 784 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 785 { 786 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 787 case DYNPTR_TYPE_LOCAL: 788 return BPF_DYNPTR_TYPE_LOCAL; 789 case DYNPTR_TYPE_RINGBUF: 790 return BPF_DYNPTR_TYPE_RINGBUF; 791 case DYNPTR_TYPE_SKB: 792 return BPF_DYNPTR_TYPE_SKB; 793 case DYNPTR_TYPE_XDP: 794 return BPF_DYNPTR_TYPE_XDP; 795 default: 796 return BPF_DYNPTR_TYPE_INVALID; 797 } 798 } 799 800 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type) 801 { 802 switch (type) { 803 case BPF_DYNPTR_TYPE_LOCAL: 804 return DYNPTR_TYPE_LOCAL; 805 case BPF_DYNPTR_TYPE_RINGBUF: 806 return DYNPTR_TYPE_RINGBUF; 807 case BPF_DYNPTR_TYPE_SKB: 808 return DYNPTR_TYPE_SKB; 809 case BPF_DYNPTR_TYPE_XDP: 810 return DYNPTR_TYPE_XDP; 811 default: 812 return 0; 813 } 814 } 815 816 static bool dynptr_type_refcounted(enum bpf_dynptr_type type) 817 { 818 return type == BPF_DYNPTR_TYPE_RINGBUF; 819 } 820 821 static void __mark_dynptr_reg(struct bpf_reg_state *reg, 822 enum bpf_dynptr_type type, 823 bool first_slot, int dynptr_id); 824 825 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 826 struct bpf_reg_state *reg); 827 828 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, 829 struct bpf_reg_state *sreg1, 830 struct bpf_reg_state *sreg2, 831 enum bpf_dynptr_type type) 832 { 833 int id = ++env->id_gen; 834 835 __mark_dynptr_reg(sreg1, type, true, id); 836 __mark_dynptr_reg(sreg2, type, false, id); 837 } 838 839 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, 840 struct bpf_reg_state *reg, 841 enum bpf_dynptr_type type) 842 { 843 __mark_dynptr_reg(reg, type, true, ++env->id_gen); 844 } 845 846 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 847 struct bpf_func_state *state, int spi); 848 849 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 850 enum bpf_arg_type arg_type, int insn_idx) 851 { 852 struct bpf_func_state *state = func(env, reg); 853 enum bpf_dynptr_type type; 854 int spi, i, id, err; 855 856 spi = dynptr_get_spi(env, reg); 857 if (spi < 0) 858 return spi; 859 860 /* We cannot assume both spi and spi - 1 belong to the same dynptr, 861 * hence we need to call destroy_if_dynptr_stack_slot twice for both, 862 * to ensure that for the following example: 863 * [d1][d1][d2][d2] 864 * spi 3 2 1 0 865 * So marking spi = 2 should lead to destruction of both d1 and d2. In 866 * case they do belong to same dynptr, second call won't see slot_type 867 * as STACK_DYNPTR and will simply skip destruction. 868 */ 869 err = destroy_if_dynptr_stack_slot(env, state, spi); 870 if (err) 871 return err; 872 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); 873 if (err) 874 return err; 875 876 for (i = 0; i < BPF_REG_SIZE; i++) { 877 state->stack[spi].slot_type[i] = STACK_DYNPTR; 878 state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; 879 } 880 881 type = arg_to_dynptr_type(arg_type); 882 if (type == BPF_DYNPTR_TYPE_INVALID) 883 return -EINVAL; 884 885 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, 886 &state->stack[spi - 1].spilled_ptr, type); 887 888 if (dynptr_type_refcounted(type)) { 889 /* The id is used to track proper releasing */ 890 id = acquire_reference_state(env, insn_idx); 891 if (id < 0) 892 return id; 893 894 state->stack[spi].spilled_ptr.ref_obj_id = id; 895 state->stack[spi - 1].spilled_ptr.ref_obj_id = id; 896 } 897 898 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 899 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 900 901 return 0; 902 } 903 904 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 905 { 906 struct bpf_func_state *state = func(env, reg); 907 int spi, i; 908 909 spi = dynptr_get_spi(env, reg); 910 if (spi < 0) 911 return spi; 912 913 for (i = 0; i < BPF_REG_SIZE; i++) { 914 state->stack[spi].slot_type[i] = STACK_INVALID; 915 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 916 } 917 918 /* Invalidate any slices associated with this dynptr */ 919 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) 920 WARN_ON_ONCE(release_reference(env, state->stack[spi].spilled_ptr.ref_obj_id)); 921 922 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 923 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 924 925 /* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot? 926 * 927 * While we don't allow reading STACK_INVALID, it is still possible to 928 * do <8 byte writes marking some but not all slots as STACK_MISC. Then, 929 * helpers or insns can do partial read of that part without failing, 930 * but check_stack_range_initialized, check_stack_read_var_off, and 931 * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of 932 * the slot conservatively. Hence we need to prevent those liveness 933 * marking walks. 934 * 935 * This was not a problem before because STACK_INVALID is only set by 936 * default (where the default reg state has its reg->parent as NULL), or 937 * in clean_live_states after REG_LIVE_DONE (at which point 938 * mark_reg_read won't walk reg->parent chain), but not randomly during 939 * verifier state exploration (like we did above). Hence, for our case 940 * parentage chain will still be live (i.e. reg->parent may be 941 * non-NULL), while earlier reg->parent was NULL, so we need 942 * REG_LIVE_WRITTEN to screen off read marker propagation when it is 943 * done later on reads or by mark_dynptr_read as well to unnecessary 944 * mark registers in verifier state. 945 */ 946 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 947 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 948 949 return 0; 950 } 951 952 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 953 struct bpf_reg_state *reg); 954 955 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 956 { 957 if (!env->allow_ptr_leaks) 958 __mark_reg_not_init(env, reg); 959 else 960 __mark_reg_unknown(env, reg); 961 } 962 963 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 964 struct bpf_func_state *state, int spi) 965 { 966 struct bpf_func_state *fstate; 967 struct bpf_reg_state *dreg; 968 int i, dynptr_id; 969 970 /* We always ensure that STACK_DYNPTR is never set partially, 971 * hence just checking for slot_type[0] is enough. This is 972 * different for STACK_SPILL, where it may be only set for 973 * 1 byte, so code has to use is_spilled_reg. 974 */ 975 if (state->stack[spi].slot_type[0] != STACK_DYNPTR) 976 return 0; 977 978 /* Reposition spi to first slot */ 979 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 980 spi = spi + 1; 981 982 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 983 verbose(env, "cannot overwrite referenced dynptr\n"); 984 return -EINVAL; 985 } 986 987 mark_stack_slot_scratched(env, spi); 988 mark_stack_slot_scratched(env, spi - 1); 989 990 /* Writing partially to one dynptr stack slot destroys both. */ 991 for (i = 0; i < BPF_REG_SIZE; i++) { 992 state->stack[spi].slot_type[i] = STACK_INVALID; 993 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 994 } 995 996 dynptr_id = state->stack[spi].spilled_ptr.id; 997 /* Invalidate any slices associated with this dynptr */ 998 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ 999 /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */ 1000 if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM) 1001 continue; 1002 if (dreg->dynptr_id == dynptr_id) 1003 mark_reg_invalid(env, dreg); 1004 })); 1005 1006 /* Do not release reference state, we are destroying dynptr on stack, 1007 * not using some helper to release it. Just reset register. 1008 */ 1009 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 1010 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 1011 1012 /* Same reason as unmark_stack_slots_dynptr above */ 1013 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 1014 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 1015 1016 return 0; 1017 } 1018 1019 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1020 { 1021 int spi; 1022 1023 if (reg->type == CONST_PTR_TO_DYNPTR) 1024 return false; 1025 1026 spi = dynptr_get_spi(env, reg); 1027 1028 /* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an 1029 * error because this just means the stack state hasn't been updated yet. 1030 * We will do check_mem_access to check and update stack bounds later. 1031 */ 1032 if (spi < 0 && spi != -ERANGE) 1033 return false; 1034 1035 /* We don't need to check if the stack slots are marked by previous 1036 * dynptr initializations because we allow overwriting existing unreferenced 1037 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls 1038 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are 1039 * touching are completely destructed before we reinitialize them for a new 1040 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early 1041 * instead of delaying it until the end where the user will get "Unreleased 1042 * reference" error. 1043 */ 1044 return true; 1045 } 1046 1047 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1048 { 1049 struct bpf_func_state *state = func(env, reg); 1050 int i, spi; 1051 1052 /* This already represents first slot of initialized bpf_dynptr. 1053 * 1054 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to 1055 * check_func_arg_reg_off's logic, so we don't need to check its 1056 * offset and alignment. 1057 */ 1058 if (reg->type == CONST_PTR_TO_DYNPTR) 1059 return true; 1060 1061 spi = dynptr_get_spi(env, reg); 1062 if (spi < 0) 1063 return false; 1064 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 1065 return false; 1066 1067 for (i = 0; i < BPF_REG_SIZE; i++) { 1068 if (state->stack[spi].slot_type[i] != STACK_DYNPTR || 1069 state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) 1070 return false; 1071 } 1072 1073 return true; 1074 } 1075 1076 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1077 enum bpf_arg_type arg_type) 1078 { 1079 struct bpf_func_state *state = func(env, reg); 1080 enum bpf_dynptr_type dynptr_type; 1081 int spi; 1082 1083 /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ 1084 if (arg_type == ARG_PTR_TO_DYNPTR) 1085 return true; 1086 1087 dynptr_type = arg_to_dynptr_type(arg_type); 1088 if (reg->type == CONST_PTR_TO_DYNPTR) { 1089 return reg->dynptr.type == dynptr_type; 1090 } else { 1091 spi = dynptr_get_spi(env, reg); 1092 if (spi < 0) 1093 return false; 1094 return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; 1095 } 1096 } 1097 1098 static void __mark_reg_known_zero(struct bpf_reg_state *reg); 1099 1100 static int mark_stack_slots_iter(struct bpf_verifier_env *env, 1101 struct bpf_reg_state *reg, int insn_idx, 1102 struct btf *btf, u32 btf_id, int nr_slots) 1103 { 1104 struct bpf_func_state *state = func(env, reg); 1105 int spi, i, j, id; 1106 1107 spi = iter_get_spi(env, reg, nr_slots); 1108 if (spi < 0) 1109 return spi; 1110 1111 id = acquire_reference_state(env, insn_idx); 1112 if (id < 0) 1113 return id; 1114 1115 for (i = 0; i < nr_slots; i++) { 1116 struct bpf_stack_state *slot = &state->stack[spi - i]; 1117 struct bpf_reg_state *st = &slot->spilled_ptr; 1118 1119 __mark_reg_known_zero(st); 1120 st->type = PTR_TO_STACK; /* we don't have dedicated reg type */ 1121 st->live |= REG_LIVE_WRITTEN; 1122 st->ref_obj_id = i == 0 ? id : 0; 1123 st->iter.btf = btf; 1124 st->iter.btf_id = btf_id; 1125 st->iter.state = BPF_ITER_STATE_ACTIVE; 1126 st->iter.depth = 0; 1127 1128 for (j = 0; j < BPF_REG_SIZE; j++) 1129 slot->slot_type[j] = STACK_ITER; 1130 1131 mark_stack_slot_scratched(env, spi - i); 1132 } 1133 1134 return 0; 1135 } 1136 1137 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, 1138 struct bpf_reg_state *reg, int nr_slots) 1139 { 1140 struct bpf_func_state *state = func(env, reg); 1141 int spi, i, j; 1142 1143 spi = iter_get_spi(env, reg, nr_slots); 1144 if (spi < 0) 1145 return spi; 1146 1147 for (i = 0; i < nr_slots; i++) { 1148 struct bpf_stack_state *slot = &state->stack[spi - i]; 1149 struct bpf_reg_state *st = &slot->spilled_ptr; 1150 1151 if (i == 0) 1152 WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); 1153 1154 __mark_reg_not_init(env, st); 1155 1156 /* see unmark_stack_slots_dynptr() for why we need to set REG_LIVE_WRITTEN */ 1157 st->live |= REG_LIVE_WRITTEN; 1158 1159 for (j = 0; j < BPF_REG_SIZE; j++) 1160 slot->slot_type[j] = STACK_INVALID; 1161 1162 mark_stack_slot_scratched(env, spi - i); 1163 } 1164 1165 return 0; 1166 } 1167 1168 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, 1169 struct bpf_reg_state *reg, int nr_slots) 1170 { 1171 struct bpf_func_state *state = func(env, reg); 1172 int spi, i, j; 1173 1174 /* For -ERANGE (i.e. spi not falling into allocated stack slots), we 1175 * will do check_mem_access to check and update stack bounds later, so 1176 * return true for that case. 1177 */ 1178 spi = iter_get_spi(env, reg, nr_slots); 1179 if (spi == -ERANGE) 1180 return true; 1181 if (spi < 0) 1182 return false; 1183 1184 for (i = 0; i < nr_slots; i++) { 1185 struct bpf_stack_state *slot = &state->stack[spi - i]; 1186 1187 for (j = 0; j < BPF_REG_SIZE; j++) 1188 if (slot->slot_type[j] == STACK_ITER) 1189 return false; 1190 } 1191 1192 return true; 1193 } 1194 1195 static bool is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1196 struct btf *btf, u32 btf_id, int nr_slots) 1197 { 1198 struct bpf_func_state *state = func(env, reg); 1199 int spi, i, j; 1200 1201 spi = iter_get_spi(env, reg, nr_slots); 1202 if (spi < 0) 1203 return false; 1204 1205 for (i = 0; i < nr_slots; i++) { 1206 struct bpf_stack_state *slot = &state->stack[spi - i]; 1207 struct bpf_reg_state *st = &slot->spilled_ptr; 1208 1209 /* only main (first) slot has ref_obj_id set */ 1210 if (i == 0 && !st->ref_obj_id) 1211 return false; 1212 if (i != 0 && st->ref_obj_id) 1213 return false; 1214 if (st->iter.btf != btf || st->iter.btf_id != btf_id) 1215 return false; 1216 1217 for (j = 0; j < BPF_REG_SIZE; j++) 1218 if (slot->slot_type[j] != STACK_ITER) 1219 return false; 1220 } 1221 1222 return true; 1223 } 1224 1225 /* Check if given stack slot is "special": 1226 * - spilled register state (STACK_SPILL); 1227 * - dynptr state (STACK_DYNPTR); 1228 * - iter state (STACK_ITER). 1229 */ 1230 static bool is_stack_slot_special(const struct bpf_stack_state *stack) 1231 { 1232 enum bpf_stack_slot_type type = stack->slot_type[BPF_REG_SIZE - 1]; 1233 1234 switch (type) { 1235 case STACK_SPILL: 1236 case STACK_DYNPTR: 1237 case STACK_ITER: 1238 return true; 1239 case STACK_INVALID: 1240 case STACK_MISC: 1241 case STACK_ZERO: 1242 return false; 1243 default: 1244 WARN_ONCE(1, "unknown stack slot type %d\n", type); 1245 return true; 1246 } 1247 } 1248 1249 /* The reg state of a pointer or a bounded scalar was saved when 1250 * it was spilled to the stack. 1251 */ 1252 static bool is_spilled_reg(const struct bpf_stack_state *stack) 1253 { 1254 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 1255 } 1256 1257 static void scrub_spilled_slot(u8 *stype) 1258 { 1259 if (*stype != STACK_INVALID) 1260 *stype = STACK_MISC; 1261 } 1262 1263 static void print_verifier_state(struct bpf_verifier_env *env, 1264 const struct bpf_func_state *state, 1265 bool print_all) 1266 { 1267 const struct bpf_reg_state *reg; 1268 enum bpf_reg_type t; 1269 int i; 1270 1271 if (state->frameno) 1272 verbose(env, " frame%d:", state->frameno); 1273 for (i = 0; i < MAX_BPF_REG; i++) { 1274 reg = &state->regs[i]; 1275 t = reg->type; 1276 if (t == NOT_INIT) 1277 continue; 1278 if (!print_all && !reg_scratched(env, i)) 1279 continue; 1280 verbose(env, " R%d", i); 1281 print_liveness(env, reg->live); 1282 verbose(env, "="); 1283 if (t == SCALAR_VALUE && reg->precise) 1284 verbose(env, "P"); 1285 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 1286 tnum_is_const(reg->var_off)) { 1287 /* reg->off should be 0 for SCALAR_VALUE */ 1288 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 1289 verbose(env, "%lld", reg->var_off.value + reg->off); 1290 } else { 1291 const char *sep = ""; 1292 1293 verbose(env, "%s", reg_type_str(env, t)); 1294 if (base_type(t) == PTR_TO_BTF_ID) 1295 verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id)); 1296 verbose(env, "("); 1297 /* 1298 * _a stands for append, was shortened to avoid multiline statements below. 1299 * This macro is used to output a comma separated list of attributes. 1300 */ 1301 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) 1302 1303 if (reg->id) 1304 verbose_a("id=%d", reg->id); 1305 if (reg->ref_obj_id) 1306 verbose_a("ref_obj_id=%d", reg->ref_obj_id); 1307 if (type_is_non_owning_ref(reg->type)) 1308 verbose_a("%s", "non_own_ref"); 1309 if (t != SCALAR_VALUE) 1310 verbose_a("off=%d", reg->off); 1311 if (type_is_pkt_pointer(t)) 1312 verbose_a("r=%d", reg->range); 1313 else if (base_type(t) == CONST_PTR_TO_MAP || 1314 base_type(t) == PTR_TO_MAP_KEY || 1315 base_type(t) == PTR_TO_MAP_VALUE) 1316 verbose_a("ks=%d,vs=%d", 1317 reg->map_ptr->key_size, 1318 reg->map_ptr->value_size); 1319 if (tnum_is_const(reg->var_off)) { 1320 /* Typically an immediate SCALAR_VALUE, but 1321 * could be a pointer whose offset is too big 1322 * for reg->off 1323 */ 1324 verbose_a("imm=%llx", reg->var_off.value); 1325 } else { 1326 if (reg->smin_value != reg->umin_value && 1327 reg->smin_value != S64_MIN) 1328 verbose_a("smin=%lld", (long long)reg->smin_value); 1329 if (reg->smax_value != reg->umax_value && 1330 reg->smax_value != S64_MAX) 1331 verbose_a("smax=%lld", (long long)reg->smax_value); 1332 if (reg->umin_value != 0) 1333 verbose_a("umin=%llu", (unsigned long long)reg->umin_value); 1334 if (reg->umax_value != U64_MAX) 1335 verbose_a("umax=%llu", (unsigned long long)reg->umax_value); 1336 if (!tnum_is_unknown(reg->var_off)) { 1337 char tn_buf[48]; 1338 1339 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1340 verbose_a("var_off=%s", tn_buf); 1341 } 1342 if (reg->s32_min_value != reg->smin_value && 1343 reg->s32_min_value != S32_MIN) 1344 verbose_a("s32_min=%d", (int)(reg->s32_min_value)); 1345 if (reg->s32_max_value != reg->smax_value && 1346 reg->s32_max_value != S32_MAX) 1347 verbose_a("s32_max=%d", (int)(reg->s32_max_value)); 1348 if (reg->u32_min_value != reg->umin_value && 1349 reg->u32_min_value != U32_MIN) 1350 verbose_a("u32_min=%d", (int)(reg->u32_min_value)); 1351 if (reg->u32_max_value != reg->umax_value && 1352 reg->u32_max_value != U32_MAX) 1353 verbose_a("u32_max=%d", (int)(reg->u32_max_value)); 1354 } 1355 #undef verbose_a 1356 1357 verbose(env, ")"); 1358 } 1359 } 1360 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 1361 char types_buf[BPF_REG_SIZE + 1]; 1362 bool valid = false; 1363 int j; 1364 1365 for (j = 0; j < BPF_REG_SIZE; j++) { 1366 if (state->stack[i].slot_type[j] != STACK_INVALID) 1367 valid = true; 1368 types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; 1369 } 1370 types_buf[BPF_REG_SIZE] = 0; 1371 if (!valid) 1372 continue; 1373 if (!print_all && !stack_slot_scratched(env, i)) 1374 continue; 1375 switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) { 1376 case STACK_SPILL: 1377 reg = &state->stack[i].spilled_ptr; 1378 t = reg->type; 1379 1380 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 1381 print_liveness(env, reg->live); 1382 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 1383 if (t == SCALAR_VALUE && reg->precise) 1384 verbose(env, "P"); 1385 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 1386 verbose(env, "%lld", reg->var_off.value + reg->off); 1387 break; 1388 case STACK_DYNPTR: 1389 i += BPF_DYNPTR_NR_SLOTS - 1; 1390 reg = &state->stack[i].spilled_ptr; 1391 1392 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 1393 print_liveness(env, reg->live); 1394 verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type)); 1395 if (reg->ref_obj_id) 1396 verbose(env, "(ref_id=%d)", reg->ref_obj_id); 1397 break; 1398 case STACK_ITER: 1399 /* only main slot has ref_obj_id set; skip others */ 1400 reg = &state->stack[i].spilled_ptr; 1401 if (!reg->ref_obj_id) 1402 continue; 1403 1404 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 1405 print_liveness(env, reg->live); 1406 verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)", 1407 iter_type_str(reg->iter.btf, reg->iter.btf_id), 1408 reg->ref_obj_id, iter_state_str(reg->iter.state), 1409 reg->iter.depth); 1410 break; 1411 case STACK_MISC: 1412 case STACK_ZERO: 1413 default: 1414 reg = &state->stack[i].spilled_ptr; 1415 1416 for (j = 0; j < BPF_REG_SIZE; j++) 1417 types_buf[j] = slot_type_char[state->stack[i].slot_type[j]]; 1418 types_buf[BPF_REG_SIZE] = 0; 1419 1420 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 1421 print_liveness(env, reg->live); 1422 verbose(env, "=%s", types_buf); 1423 break; 1424 } 1425 } 1426 if (state->acquired_refs && state->refs[0].id) { 1427 verbose(env, " refs=%d", state->refs[0].id); 1428 for (i = 1; i < state->acquired_refs; i++) 1429 if (state->refs[i].id) 1430 verbose(env, ",%d", state->refs[i].id); 1431 } 1432 if (state->in_callback_fn) 1433 verbose(env, " cb"); 1434 if (state->in_async_callback_fn) 1435 verbose(env, " async_cb"); 1436 verbose(env, "\n"); 1437 mark_verifier_state_clean(env); 1438 } 1439 1440 static inline u32 vlog_alignment(u32 pos) 1441 { 1442 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), 1443 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 1444 } 1445 1446 static void print_insn_state(struct bpf_verifier_env *env, 1447 const struct bpf_func_state *state) 1448 { 1449 if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { 1450 /* remove new line character */ 1451 bpf_vlog_reset(&env->log, env->prev_log_pos - 1); 1452 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' '); 1453 } else { 1454 verbose(env, "%d:", env->insn_idx); 1455 } 1456 print_verifier_state(env, state, false); 1457 } 1458 1459 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1460 * small to hold src. This is different from krealloc since we don't want to preserve 1461 * the contents of dst. 1462 * 1463 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1464 * not be allocated. 1465 */ 1466 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1467 { 1468 size_t alloc_bytes; 1469 void *orig = dst; 1470 size_t bytes; 1471 1472 if (ZERO_OR_NULL_PTR(src)) 1473 goto out; 1474 1475 if (unlikely(check_mul_overflow(n, size, &bytes))) 1476 return NULL; 1477 1478 alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes)); 1479 dst = krealloc(orig, alloc_bytes, flags); 1480 if (!dst) { 1481 kfree(orig); 1482 return NULL; 1483 } 1484 1485 memcpy(dst, src, bytes); 1486 out: 1487 return dst ? dst : ZERO_SIZE_PTR; 1488 } 1489 1490 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1491 * small to hold new_n items. new items are zeroed out if the array grows. 1492 * 1493 * Contrary to krealloc_array, does not free arr if new_n is zero. 1494 */ 1495 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1496 { 1497 size_t alloc_size; 1498 void *new_arr; 1499 1500 if (!new_n || old_n == new_n) 1501 goto out; 1502 1503 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1504 new_arr = krealloc(arr, alloc_size, GFP_KERNEL); 1505 if (!new_arr) { 1506 kfree(arr); 1507 return NULL; 1508 } 1509 arr = new_arr; 1510 1511 if (new_n > old_n) 1512 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1513 1514 out: 1515 return arr ? arr : ZERO_SIZE_PTR; 1516 } 1517 1518 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1519 { 1520 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1521 sizeof(struct bpf_reference_state), GFP_KERNEL); 1522 if (!dst->refs) 1523 return -ENOMEM; 1524 1525 dst->acquired_refs = src->acquired_refs; 1526 return 0; 1527 } 1528 1529 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1530 { 1531 size_t n = src->allocated_stack / BPF_REG_SIZE; 1532 1533 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1534 GFP_KERNEL); 1535 if (!dst->stack) 1536 return -ENOMEM; 1537 1538 dst->allocated_stack = src->allocated_stack; 1539 return 0; 1540 } 1541 1542 static int resize_reference_state(struct bpf_func_state *state, size_t n) 1543 { 1544 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1545 sizeof(struct bpf_reference_state)); 1546 if (!state->refs) 1547 return -ENOMEM; 1548 1549 state->acquired_refs = n; 1550 return 0; 1551 } 1552 1553 static int grow_stack_state(struct bpf_func_state *state, int size) 1554 { 1555 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 1556 1557 if (old_n >= n) 1558 return 0; 1559 1560 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1561 if (!state->stack) 1562 return -ENOMEM; 1563 1564 state->allocated_stack = size; 1565 return 0; 1566 } 1567 1568 /* Acquire a pointer id from the env and update the state->refs to include 1569 * this new pointer reference. 1570 * On success, returns a valid pointer id to associate with the register 1571 * On failure, returns a negative errno. 1572 */ 1573 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1574 { 1575 struct bpf_func_state *state = cur_func(env); 1576 int new_ofs = state->acquired_refs; 1577 int id, err; 1578 1579 err = resize_reference_state(state, state->acquired_refs + 1); 1580 if (err) 1581 return err; 1582 id = ++env->id_gen; 1583 state->refs[new_ofs].id = id; 1584 state->refs[new_ofs].insn_idx = insn_idx; 1585 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; 1586 1587 return id; 1588 } 1589 1590 /* release function corresponding to acquire_reference_state(). Idempotent. */ 1591 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1592 { 1593 int i, last_idx; 1594 1595 last_idx = state->acquired_refs - 1; 1596 for (i = 0; i < state->acquired_refs; i++) { 1597 if (state->refs[i].id == ptr_id) { 1598 /* Cannot release caller references in callbacks */ 1599 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 1600 return -EINVAL; 1601 if (last_idx && i != last_idx) 1602 memcpy(&state->refs[i], &state->refs[last_idx], 1603 sizeof(*state->refs)); 1604 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1605 state->acquired_refs--; 1606 return 0; 1607 } 1608 } 1609 return -EINVAL; 1610 } 1611 1612 static void free_func_state(struct bpf_func_state *state) 1613 { 1614 if (!state) 1615 return; 1616 kfree(state->refs); 1617 kfree(state->stack); 1618 kfree(state); 1619 } 1620 1621 static void clear_jmp_history(struct bpf_verifier_state *state) 1622 { 1623 kfree(state->jmp_history); 1624 state->jmp_history = NULL; 1625 state->jmp_history_cnt = 0; 1626 } 1627 1628 static void free_verifier_state(struct bpf_verifier_state *state, 1629 bool free_self) 1630 { 1631 int i; 1632 1633 for (i = 0; i <= state->curframe; i++) { 1634 free_func_state(state->frame[i]); 1635 state->frame[i] = NULL; 1636 } 1637 clear_jmp_history(state); 1638 if (free_self) 1639 kfree(state); 1640 } 1641 1642 /* copy verifier state from src to dst growing dst stack space 1643 * when necessary to accommodate larger src stack 1644 */ 1645 static int copy_func_state(struct bpf_func_state *dst, 1646 const struct bpf_func_state *src) 1647 { 1648 int err; 1649 1650 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1651 err = copy_reference_state(dst, src); 1652 if (err) 1653 return err; 1654 return copy_stack_state(dst, src); 1655 } 1656 1657 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1658 const struct bpf_verifier_state *src) 1659 { 1660 struct bpf_func_state *dst; 1661 int i, err; 1662 1663 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1664 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 1665 GFP_USER); 1666 if (!dst_state->jmp_history) 1667 return -ENOMEM; 1668 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1669 1670 /* if dst has more stack frames then src frame, free them */ 1671 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1672 free_func_state(dst_state->frame[i]); 1673 dst_state->frame[i] = NULL; 1674 } 1675 dst_state->speculative = src->speculative; 1676 dst_state->active_rcu_lock = src->active_rcu_lock; 1677 dst_state->curframe = src->curframe; 1678 dst_state->active_lock.ptr = src->active_lock.ptr; 1679 dst_state->active_lock.id = src->active_lock.id; 1680 dst_state->branches = src->branches; 1681 dst_state->parent = src->parent; 1682 dst_state->first_insn_idx = src->first_insn_idx; 1683 dst_state->last_insn_idx = src->last_insn_idx; 1684 for (i = 0; i <= src->curframe; i++) { 1685 dst = dst_state->frame[i]; 1686 if (!dst) { 1687 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1688 if (!dst) 1689 return -ENOMEM; 1690 dst_state->frame[i] = dst; 1691 } 1692 err = copy_func_state(dst, src->frame[i]); 1693 if (err) 1694 return err; 1695 } 1696 return 0; 1697 } 1698 1699 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1700 { 1701 while (st) { 1702 u32 br = --st->branches; 1703 1704 /* WARN_ON(br > 1) technically makes sense here, 1705 * but see comment in push_stack(), hence: 1706 */ 1707 WARN_ONCE((int)br < 0, 1708 "BUG update_branch_counts:branches_to_explore=%d\n", 1709 br); 1710 if (br) 1711 break; 1712 st = st->parent; 1713 } 1714 } 1715 1716 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1717 int *insn_idx, bool pop_log) 1718 { 1719 struct bpf_verifier_state *cur = env->cur_state; 1720 struct bpf_verifier_stack_elem *elem, *head = env->head; 1721 int err; 1722 1723 if (env->head == NULL) 1724 return -ENOENT; 1725 1726 if (cur) { 1727 err = copy_verifier_state(cur, &head->st); 1728 if (err) 1729 return err; 1730 } 1731 if (pop_log) 1732 bpf_vlog_reset(&env->log, head->log_pos); 1733 if (insn_idx) 1734 *insn_idx = head->insn_idx; 1735 if (prev_insn_idx) 1736 *prev_insn_idx = head->prev_insn_idx; 1737 elem = head->next; 1738 free_verifier_state(&head->st, false); 1739 kfree(head); 1740 env->head = elem; 1741 env->stack_size--; 1742 return 0; 1743 } 1744 1745 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1746 int insn_idx, int prev_insn_idx, 1747 bool speculative) 1748 { 1749 struct bpf_verifier_state *cur = env->cur_state; 1750 struct bpf_verifier_stack_elem *elem; 1751 int err; 1752 1753 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1754 if (!elem) 1755 goto err; 1756 1757 elem->insn_idx = insn_idx; 1758 elem->prev_insn_idx = prev_insn_idx; 1759 elem->next = env->head; 1760 elem->log_pos = env->log.end_pos; 1761 env->head = elem; 1762 env->stack_size++; 1763 err = copy_verifier_state(&elem->st, cur); 1764 if (err) 1765 goto err; 1766 elem->st.speculative |= speculative; 1767 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1768 verbose(env, "The sequence of %d jumps is too complex.\n", 1769 env->stack_size); 1770 goto err; 1771 } 1772 if (elem->st.parent) { 1773 ++elem->st.parent->branches; 1774 /* WARN_ON(branches > 2) technically makes sense here, 1775 * but 1776 * 1. speculative states will bump 'branches' for non-branch 1777 * instructions 1778 * 2. is_state_visited() heuristics may decide not to create 1779 * a new state for a sequence of branches and all such current 1780 * and cloned states will be pointing to a single parent state 1781 * which might have large 'branches' count. 1782 */ 1783 } 1784 return &elem->st; 1785 err: 1786 free_verifier_state(env->cur_state, true); 1787 env->cur_state = NULL; 1788 /* pop all elements and return */ 1789 while (!pop_stack(env, NULL, NULL, false)); 1790 return NULL; 1791 } 1792 1793 #define CALLER_SAVED_REGS 6 1794 static const int caller_saved[CALLER_SAVED_REGS] = { 1795 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1796 }; 1797 1798 /* This helper doesn't clear reg->id */ 1799 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1800 { 1801 reg->var_off = tnum_const(imm); 1802 reg->smin_value = (s64)imm; 1803 reg->smax_value = (s64)imm; 1804 reg->umin_value = imm; 1805 reg->umax_value = imm; 1806 1807 reg->s32_min_value = (s32)imm; 1808 reg->s32_max_value = (s32)imm; 1809 reg->u32_min_value = (u32)imm; 1810 reg->u32_max_value = (u32)imm; 1811 } 1812 1813 /* Mark the unknown part of a register (variable offset or scalar value) as 1814 * known to have the value @imm. 1815 */ 1816 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1817 { 1818 /* Clear off and union(map_ptr, range) */ 1819 memset(((u8 *)reg) + sizeof(reg->type), 0, 1820 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1821 reg->id = 0; 1822 reg->ref_obj_id = 0; 1823 ___mark_reg_known(reg, imm); 1824 } 1825 1826 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1827 { 1828 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1829 reg->s32_min_value = (s32)imm; 1830 reg->s32_max_value = (s32)imm; 1831 reg->u32_min_value = (u32)imm; 1832 reg->u32_max_value = (u32)imm; 1833 } 1834 1835 /* Mark the 'variable offset' part of a register as zero. This should be 1836 * used only on registers holding a pointer type. 1837 */ 1838 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1839 { 1840 __mark_reg_known(reg, 0); 1841 } 1842 1843 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1844 { 1845 __mark_reg_known(reg, 0); 1846 reg->type = SCALAR_VALUE; 1847 } 1848 1849 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1850 struct bpf_reg_state *regs, u32 regno) 1851 { 1852 if (WARN_ON(regno >= MAX_BPF_REG)) { 1853 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1854 /* Something bad happened, let's kill all regs */ 1855 for (regno = 0; regno < MAX_BPF_REG; regno++) 1856 __mark_reg_not_init(env, regs + regno); 1857 return; 1858 } 1859 __mark_reg_known_zero(regs + regno); 1860 } 1861 1862 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, 1863 bool first_slot, int dynptr_id) 1864 { 1865 /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for 1866 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply 1867 * set it unconditionally as it is ignored for STACK_DYNPTR anyway. 1868 */ 1869 __mark_reg_known_zero(reg); 1870 reg->type = CONST_PTR_TO_DYNPTR; 1871 /* Give each dynptr a unique id to uniquely associate slices to it. */ 1872 reg->id = dynptr_id; 1873 reg->dynptr.type = type; 1874 reg->dynptr.first_slot = first_slot; 1875 } 1876 1877 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1878 { 1879 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1880 const struct bpf_map *map = reg->map_ptr; 1881 1882 if (map->inner_map_meta) { 1883 reg->type = CONST_PTR_TO_MAP; 1884 reg->map_ptr = map->inner_map_meta; 1885 /* transfer reg's id which is unique for every map_lookup_elem 1886 * as UID of the inner map. 1887 */ 1888 if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER)) 1889 reg->map_uid = reg->id; 1890 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1891 reg->type = PTR_TO_XDP_SOCK; 1892 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1893 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1894 reg->type = PTR_TO_SOCKET; 1895 } else { 1896 reg->type = PTR_TO_MAP_VALUE; 1897 } 1898 return; 1899 } 1900 1901 reg->type &= ~PTR_MAYBE_NULL; 1902 } 1903 1904 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno, 1905 struct btf_field_graph_root *ds_head) 1906 { 1907 __mark_reg_known_zero(®s[regno]); 1908 regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC; 1909 regs[regno].btf = ds_head->btf; 1910 regs[regno].btf_id = ds_head->value_btf_id; 1911 regs[regno].off = ds_head->node_offset; 1912 } 1913 1914 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1915 { 1916 return type_is_pkt_pointer(reg->type); 1917 } 1918 1919 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1920 { 1921 return reg_is_pkt_pointer(reg) || 1922 reg->type == PTR_TO_PACKET_END; 1923 } 1924 1925 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg) 1926 { 1927 return base_type(reg->type) == PTR_TO_MEM && 1928 (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP); 1929 } 1930 1931 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1932 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1933 enum bpf_reg_type which) 1934 { 1935 /* The register can already have a range from prior markings. 1936 * This is fine as long as it hasn't been advanced from its 1937 * origin. 1938 */ 1939 return reg->type == which && 1940 reg->id == 0 && 1941 reg->off == 0 && 1942 tnum_equals_const(reg->var_off, 0); 1943 } 1944 1945 /* Reset the min/max bounds of a register */ 1946 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1947 { 1948 reg->smin_value = S64_MIN; 1949 reg->smax_value = S64_MAX; 1950 reg->umin_value = 0; 1951 reg->umax_value = U64_MAX; 1952 1953 reg->s32_min_value = S32_MIN; 1954 reg->s32_max_value = S32_MAX; 1955 reg->u32_min_value = 0; 1956 reg->u32_max_value = U32_MAX; 1957 } 1958 1959 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1960 { 1961 reg->smin_value = S64_MIN; 1962 reg->smax_value = S64_MAX; 1963 reg->umin_value = 0; 1964 reg->umax_value = U64_MAX; 1965 } 1966 1967 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1968 { 1969 reg->s32_min_value = S32_MIN; 1970 reg->s32_max_value = S32_MAX; 1971 reg->u32_min_value = 0; 1972 reg->u32_max_value = U32_MAX; 1973 } 1974 1975 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1976 { 1977 struct tnum var32_off = tnum_subreg(reg->var_off); 1978 1979 /* min signed is max(sign bit) | min(other bits) */ 1980 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1981 var32_off.value | (var32_off.mask & S32_MIN)); 1982 /* max signed is min(sign bit) | max(other bits) */ 1983 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1984 var32_off.value | (var32_off.mask & S32_MAX)); 1985 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1986 reg->u32_max_value = min(reg->u32_max_value, 1987 (u32)(var32_off.value | var32_off.mask)); 1988 } 1989 1990 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1991 { 1992 /* min signed is max(sign bit) | min(other bits) */ 1993 reg->smin_value = max_t(s64, reg->smin_value, 1994 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1995 /* max signed is min(sign bit) | max(other bits) */ 1996 reg->smax_value = min_t(s64, reg->smax_value, 1997 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1998 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1999 reg->umax_value = min(reg->umax_value, 2000 reg->var_off.value | reg->var_off.mask); 2001 } 2002 2003 static void __update_reg_bounds(struct bpf_reg_state *reg) 2004 { 2005 __update_reg32_bounds(reg); 2006 __update_reg64_bounds(reg); 2007 } 2008 2009 /* Uses signed min/max values to inform unsigned, and vice-versa */ 2010 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 2011 { 2012 /* Learn sign from signed bounds. 2013 * If we cannot cross the sign boundary, then signed and unsigned bounds 2014 * are the same, so combine. This works even in the negative case, e.g. 2015 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 2016 */ 2017 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 2018 reg->s32_min_value = reg->u32_min_value = 2019 max_t(u32, reg->s32_min_value, reg->u32_min_value); 2020 reg->s32_max_value = reg->u32_max_value = 2021 min_t(u32, reg->s32_max_value, reg->u32_max_value); 2022 return; 2023 } 2024 /* Learn sign from unsigned bounds. Signed bounds cross the sign 2025 * boundary, so we must be careful. 2026 */ 2027 if ((s32)reg->u32_max_value >= 0) { 2028 /* Positive. We can't learn anything from the smin, but smax 2029 * is positive, hence safe. 2030 */ 2031 reg->s32_min_value = reg->u32_min_value; 2032 reg->s32_max_value = reg->u32_max_value = 2033 min_t(u32, reg->s32_max_value, reg->u32_max_value); 2034 } else if ((s32)reg->u32_min_value < 0) { 2035 /* Negative. We can't learn anything from the smax, but smin 2036 * is negative, hence safe. 2037 */ 2038 reg->s32_min_value = reg->u32_min_value = 2039 max_t(u32, reg->s32_min_value, reg->u32_min_value); 2040 reg->s32_max_value = reg->u32_max_value; 2041 } 2042 } 2043 2044 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 2045 { 2046 /* Learn sign from signed bounds. 2047 * If we cannot cross the sign boundary, then signed and unsigned bounds 2048 * are the same, so combine. This works even in the negative case, e.g. 2049 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 2050 */ 2051 if (reg->smin_value >= 0 || reg->smax_value < 0) { 2052 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 2053 reg->umin_value); 2054 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 2055 reg->umax_value); 2056 return; 2057 } 2058 /* Learn sign from unsigned bounds. Signed bounds cross the sign 2059 * boundary, so we must be careful. 2060 */ 2061 if ((s64)reg->umax_value >= 0) { 2062 /* Positive. We can't learn anything from the smin, but smax 2063 * is positive, hence safe. 2064 */ 2065 reg->smin_value = reg->umin_value; 2066 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 2067 reg->umax_value); 2068 } else if ((s64)reg->umin_value < 0) { 2069 /* Negative. We can't learn anything from the smax, but smin 2070 * is negative, hence safe. 2071 */ 2072 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 2073 reg->umin_value); 2074 reg->smax_value = reg->umax_value; 2075 } 2076 } 2077 2078 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 2079 { 2080 __reg32_deduce_bounds(reg); 2081 __reg64_deduce_bounds(reg); 2082 } 2083 2084 /* Attempts to improve var_off based on unsigned min/max information */ 2085 static void __reg_bound_offset(struct bpf_reg_state *reg) 2086 { 2087 struct tnum var64_off = tnum_intersect(reg->var_off, 2088 tnum_range(reg->umin_value, 2089 reg->umax_value)); 2090 struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off), 2091 tnum_range(reg->u32_min_value, 2092 reg->u32_max_value)); 2093 2094 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 2095 } 2096 2097 static void reg_bounds_sync(struct bpf_reg_state *reg) 2098 { 2099 /* We might have learned new bounds from the var_off. */ 2100 __update_reg_bounds(reg); 2101 /* We might have learned something about the sign bit. */ 2102 __reg_deduce_bounds(reg); 2103 /* We might have learned some bits from the bounds. */ 2104 __reg_bound_offset(reg); 2105 /* Intersecting with the old var_off might have improved our bounds 2106 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 2107 * then new var_off is (0; 0x7f...fc) which improves our umax. 2108 */ 2109 __update_reg_bounds(reg); 2110 } 2111 2112 static bool __reg32_bound_s64(s32 a) 2113 { 2114 return a >= 0 && a <= S32_MAX; 2115 } 2116 2117 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 2118 { 2119 reg->umin_value = reg->u32_min_value; 2120 reg->umax_value = reg->u32_max_value; 2121 2122 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 2123 * be positive otherwise set to worse case bounds and refine later 2124 * from tnum. 2125 */ 2126 if (__reg32_bound_s64(reg->s32_min_value) && 2127 __reg32_bound_s64(reg->s32_max_value)) { 2128 reg->smin_value = reg->s32_min_value; 2129 reg->smax_value = reg->s32_max_value; 2130 } else { 2131 reg->smin_value = 0; 2132 reg->smax_value = U32_MAX; 2133 } 2134 } 2135 2136 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 2137 { 2138 /* special case when 64-bit register has upper 32-bit register 2139 * zeroed. Typically happens after zext or <<32, >>32 sequence 2140 * allowing us to use 32-bit bounds directly, 2141 */ 2142 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 2143 __reg_assign_32_into_64(reg); 2144 } else { 2145 /* Otherwise the best we can do is push lower 32bit known and 2146 * unknown bits into register (var_off set from jmp logic) 2147 * then learn as much as possible from the 64-bit tnum 2148 * known and unknown bits. The previous smin/smax bounds are 2149 * invalid here because of jmp32 compare so mark them unknown 2150 * so they do not impact tnum bounds calculation. 2151 */ 2152 __mark_reg64_unbounded(reg); 2153 } 2154 reg_bounds_sync(reg); 2155 } 2156 2157 static bool __reg64_bound_s32(s64 a) 2158 { 2159 return a >= S32_MIN && a <= S32_MAX; 2160 } 2161 2162 static bool __reg64_bound_u32(u64 a) 2163 { 2164 return a >= U32_MIN && a <= U32_MAX; 2165 } 2166 2167 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 2168 { 2169 __mark_reg32_unbounded(reg); 2170 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 2171 reg->s32_min_value = (s32)reg->smin_value; 2172 reg->s32_max_value = (s32)reg->smax_value; 2173 } 2174 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 2175 reg->u32_min_value = (u32)reg->umin_value; 2176 reg->u32_max_value = (u32)reg->umax_value; 2177 } 2178 reg_bounds_sync(reg); 2179 } 2180 2181 /* Mark a register as having a completely unknown (scalar) value. */ 2182 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 2183 struct bpf_reg_state *reg) 2184 { 2185 /* 2186 * Clear type, off, and union(map_ptr, range) and 2187 * padding between 'type' and union 2188 */ 2189 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 2190 reg->type = SCALAR_VALUE; 2191 reg->id = 0; 2192 reg->ref_obj_id = 0; 2193 reg->var_off = tnum_unknown; 2194 reg->frameno = 0; 2195 reg->precise = !env->bpf_capable; 2196 __mark_reg_unbounded(reg); 2197 } 2198 2199 static void mark_reg_unknown(struct bpf_verifier_env *env, 2200 struct bpf_reg_state *regs, u32 regno) 2201 { 2202 if (WARN_ON(regno >= MAX_BPF_REG)) { 2203 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 2204 /* Something bad happened, let's kill all regs except FP */ 2205 for (regno = 0; regno < BPF_REG_FP; regno++) 2206 __mark_reg_not_init(env, regs + regno); 2207 return; 2208 } 2209 __mark_reg_unknown(env, regs + regno); 2210 } 2211 2212 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 2213 struct bpf_reg_state *reg) 2214 { 2215 __mark_reg_unknown(env, reg); 2216 reg->type = NOT_INIT; 2217 } 2218 2219 static void mark_reg_not_init(struct bpf_verifier_env *env, 2220 struct bpf_reg_state *regs, u32 regno) 2221 { 2222 if (WARN_ON(regno >= MAX_BPF_REG)) { 2223 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 2224 /* Something bad happened, let's kill all regs except FP */ 2225 for (regno = 0; regno < BPF_REG_FP; regno++) 2226 __mark_reg_not_init(env, regs + regno); 2227 return; 2228 } 2229 __mark_reg_not_init(env, regs + regno); 2230 } 2231 2232 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 2233 struct bpf_reg_state *regs, u32 regno, 2234 enum bpf_reg_type reg_type, 2235 struct btf *btf, u32 btf_id, 2236 enum bpf_type_flag flag) 2237 { 2238 if (reg_type == SCALAR_VALUE) { 2239 mark_reg_unknown(env, regs, regno); 2240 return; 2241 } 2242 mark_reg_known_zero(env, regs, regno); 2243 regs[regno].type = PTR_TO_BTF_ID | flag; 2244 regs[regno].btf = btf; 2245 regs[regno].btf_id = btf_id; 2246 } 2247 2248 #define DEF_NOT_SUBREG (0) 2249 static void init_reg_state(struct bpf_verifier_env *env, 2250 struct bpf_func_state *state) 2251 { 2252 struct bpf_reg_state *regs = state->regs; 2253 int i; 2254 2255 for (i = 0; i < MAX_BPF_REG; i++) { 2256 mark_reg_not_init(env, regs, i); 2257 regs[i].live = REG_LIVE_NONE; 2258 regs[i].parent = NULL; 2259 regs[i].subreg_def = DEF_NOT_SUBREG; 2260 } 2261 2262 /* frame pointer */ 2263 regs[BPF_REG_FP].type = PTR_TO_STACK; 2264 mark_reg_known_zero(env, regs, BPF_REG_FP); 2265 regs[BPF_REG_FP].frameno = state->frameno; 2266 } 2267 2268 #define BPF_MAIN_FUNC (-1) 2269 static void init_func_state(struct bpf_verifier_env *env, 2270 struct bpf_func_state *state, 2271 int callsite, int frameno, int subprogno) 2272 { 2273 state->callsite = callsite; 2274 state->frameno = frameno; 2275 state->subprogno = subprogno; 2276 state->callback_ret_range = tnum_range(0, 0); 2277 init_reg_state(env, state); 2278 mark_verifier_state_scratched(env); 2279 } 2280 2281 /* Similar to push_stack(), but for async callbacks */ 2282 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 2283 int insn_idx, int prev_insn_idx, 2284 int subprog) 2285 { 2286 struct bpf_verifier_stack_elem *elem; 2287 struct bpf_func_state *frame; 2288 2289 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 2290 if (!elem) 2291 goto err; 2292 2293 elem->insn_idx = insn_idx; 2294 elem->prev_insn_idx = prev_insn_idx; 2295 elem->next = env->head; 2296 elem->log_pos = env->log.end_pos; 2297 env->head = elem; 2298 env->stack_size++; 2299 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 2300 verbose(env, 2301 "The sequence of %d jumps is too complex for async cb.\n", 2302 env->stack_size); 2303 goto err; 2304 } 2305 /* Unlike push_stack() do not copy_verifier_state(). 2306 * The caller state doesn't matter. 2307 * This is async callback. It starts in a fresh stack. 2308 * Initialize it similar to do_check_common(). 2309 */ 2310 elem->st.branches = 1; 2311 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 2312 if (!frame) 2313 goto err; 2314 init_func_state(env, frame, 2315 BPF_MAIN_FUNC /* callsite */, 2316 0 /* frameno within this callchain */, 2317 subprog /* subprog number within this prog */); 2318 elem->st.frame[0] = frame; 2319 return &elem->st; 2320 err: 2321 free_verifier_state(env->cur_state, true); 2322 env->cur_state = NULL; 2323 /* pop all elements and return */ 2324 while (!pop_stack(env, NULL, NULL, false)); 2325 return NULL; 2326 } 2327 2328 2329 enum reg_arg_type { 2330 SRC_OP, /* register is used as source operand */ 2331 DST_OP, /* register is used as destination operand */ 2332 DST_OP_NO_MARK /* same as above, check only, don't mark */ 2333 }; 2334 2335 static int cmp_subprogs(const void *a, const void *b) 2336 { 2337 return ((struct bpf_subprog_info *)a)->start - 2338 ((struct bpf_subprog_info *)b)->start; 2339 } 2340 2341 static int find_subprog(struct bpf_verifier_env *env, int off) 2342 { 2343 struct bpf_subprog_info *p; 2344 2345 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 2346 sizeof(env->subprog_info[0]), cmp_subprogs); 2347 if (!p) 2348 return -ENOENT; 2349 return p - env->subprog_info; 2350 2351 } 2352 2353 static int add_subprog(struct bpf_verifier_env *env, int off) 2354 { 2355 int insn_cnt = env->prog->len; 2356 int ret; 2357 2358 if (off >= insn_cnt || off < 0) { 2359 verbose(env, "call to invalid destination\n"); 2360 return -EINVAL; 2361 } 2362 ret = find_subprog(env, off); 2363 if (ret >= 0) 2364 return ret; 2365 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 2366 verbose(env, "too many subprograms\n"); 2367 return -E2BIG; 2368 } 2369 /* determine subprog starts. The end is one before the next starts */ 2370 env->subprog_info[env->subprog_cnt++].start = off; 2371 sort(env->subprog_info, env->subprog_cnt, 2372 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 2373 return env->subprog_cnt - 1; 2374 } 2375 2376 #define MAX_KFUNC_DESCS 256 2377 #define MAX_KFUNC_BTFS 256 2378 2379 struct bpf_kfunc_desc { 2380 struct btf_func_model func_model; 2381 u32 func_id; 2382 s32 imm; 2383 u16 offset; 2384 unsigned long addr; 2385 }; 2386 2387 struct bpf_kfunc_btf { 2388 struct btf *btf; 2389 struct module *module; 2390 u16 offset; 2391 }; 2392 2393 struct bpf_kfunc_desc_tab { 2394 /* Sorted by func_id (BTF ID) and offset (fd_array offset) during 2395 * verification. JITs do lookups by bpf_insn, where func_id may not be 2396 * available, therefore at the end of verification do_misc_fixups() 2397 * sorts this by imm and offset. 2398 */ 2399 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 2400 u32 nr_descs; 2401 }; 2402 2403 struct bpf_kfunc_btf_tab { 2404 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 2405 u32 nr_descs; 2406 }; 2407 2408 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 2409 { 2410 const struct bpf_kfunc_desc *d0 = a; 2411 const struct bpf_kfunc_desc *d1 = b; 2412 2413 /* func_id is not greater than BTF_MAX_TYPE */ 2414 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 2415 } 2416 2417 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 2418 { 2419 const struct bpf_kfunc_btf *d0 = a; 2420 const struct bpf_kfunc_btf *d1 = b; 2421 2422 return d0->offset - d1->offset; 2423 } 2424 2425 static const struct bpf_kfunc_desc * 2426 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 2427 { 2428 struct bpf_kfunc_desc desc = { 2429 .func_id = func_id, 2430 .offset = offset, 2431 }; 2432 struct bpf_kfunc_desc_tab *tab; 2433 2434 tab = prog->aux->kfunc_tab; 2435 return bsearch(&desc, tab->descs, tab->nr_descs, 2436 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 2437 } 2438 2439 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, 2440 u16 btf_fd_idx, u8 **func_addr) 2441 { 2442 const struct bpf_kfunc_desc *desc; 2443 2444 desc = find_kfunc_desc(prog, func_id, btf_fd_idx); 2445 if (!desc) 2446 return -EFAULT; 2447 2448 *func_addr = (u8 *)desc->addr; 2449 return 0; 2450 } 2451 2452 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 2453 s16 offset) 2454 { 2455 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 2456 struct bpf_kfunc_btf_tab *tab; 2457 struct bpf_kfunc_btf *b; 2458 struct module *mod; 2459 struct btf *btf; 2460 int btf_fd; 2461 2462 tab = env->prog->aux->kfunc_btf_tab; 2463 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 2464 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 2465 if (!b) { 2466 if (tab->nr_descs == MAX_KFUNC_BTFS) { 2467 verbose(env, "too many different module BTFs\n"); 2468 return ERR_PTR(-E2BIG); 2469 } 2470 2471 if (bpfptr_is_null(env->fd_array)) { 2472 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 2473 return ERR_PTR(-EPROTO); 2474 } 2475 2476 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 2477 offset * sizeof(btf_fd), 2478 sizeof(btf_fd))) 2479 return ERR_PTR(-EFAULT); 2480 2481 btf = btf_get_by_fd(btf_fd); 2482 if (IS_ERR(btf)) { 2483 verbose(env, "invalid module BTF fd specified\n"); 2484 return btf; 2485 } 2486 2487 if (!btf_is_module(btf)) { 2488 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 2489 btf_put(btf); 2490 return ERR_PTR(-EINVAL); 2491 } 2492 2493 mod = btf_try_get_module(btf); 2494 if (!mod) { 2495 btf_put(btf); 2496 return ERR_PTR(-ENXIO); 2497 } 2498 2499 b = &tab->descs[tab->nr_descs++]; 2500 b->btf = btf; 2501 b->module = mod; 2502 b->offset = offset; 2503 2504 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2505 kfunc_btf_cmp_by_off, NULL); 2506 } 2507 return b->btf; 2508 } 2509 2510 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2511 { 2512 if (!tab) 2513 return; 2514 2515 while (tab->nr_descs--) { 2516 module_put(tab->descs[tab->nr_descs].module); 2517 btf_put(tab->descs[tab->nr_descs].btf); 2518 } 2519 kfree(tab); 2520 } 2521 2522 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2523 { 2524 if (offset) { 2525 if (offset < 0) { 2526 /* In the future, this can be allowed to increase limit 2527 * of fd index into fd_array, interpreted as u16. 2528 */ 2529 verbose(env, "negative offset disallowed for kernel module function call\n"); 2530 return ERR_PTR(-EINVAL); 2531 } 2532 2533 return __find_kfunc_desc_btf(env, offset); 2534 } 2535 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2536 } 2537 2538 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 2539 { 2540 const struct btf_type *func, *func_proto; 2541 struct bpf_kfunc_btf_tab *btf_tab; 2542 struct bpf_kfunc_desc_tab *tab; 2543 struct bpf_prog_aux *prog_aux; 2544 struct bpf_kfunc_desc *desc; 2545 const char *func_name; 2546 struct btf *desc_btf; 2547 unsigned long call_imm; 2548 unsigned long addr; 2549 int err; 2550 2551 prog_aux = env->prog->aux; 2552 tab = prog_aux->kfunc_tab; 2553 btf_tab = prog_aux->kfunc_btf_tab; 2554 if (!tab) { 2555 if (!btf_vmlinux) { 2556 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 2557 return -ENOTSUPP; 2558 } 2559 2560 if (!env->prog->jit_requested) { 2561 verbose(env, "JIT is required for calling kernel function\n"); 2562 return -ENOTSUPP; 2563 } 2564 2565 if (!bpf_jit_supports_kfunc_call()) { 2566 verbose(env, "JIT does not support calling kernel function\n"); 2567 return -ENOTSUPP; 2568 } 2569 2570 if (!env->prog->gpl_compatible) { 2571 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 2572 return -EINVAL; 2573 } 2574 2575 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 2576 if (!tab) 2577 return -ENOMEM; 2578 prog_aux->kfunc_tab = tab; 2579 } 2580 2581 /* func_id == 0 is always invalid, but instead of returning an error, be 2582 * conservative and wait until the code elimination pass before returning 2583 * error, so that invalid calls that get pruned out can be in BPF programs 2584 * loaded from userspace. It is also required that offset be untouched 2585 * for such calls. 2586 */ 2587 if (!func_id && !offset) 2588 return 0; 2589 2590 if (!btf_tab && offset) { 2591 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 2592 if (!btf_tab) 2593 return -ENOMEM; 2594 prog_aux->kfunc_btf_tab = btf_tab; 2595 } 2596 2597 desc_btf = find_kfunc_desc_btf(env, offset); 2598 if (IS_ERR(desc_btf)) { 2599 verbose(env, "failed to find BTF for kernel function\n"); 2600 return PTR_ERR(desc_btf); 2601 } 2602 2603 if (find_kfunc_desc(env->prog, func_id, offset)) 2604 return 0; 2605 2606 if (tab->nr_descs == MAX_KFUNC_DESCS) { 2607 verbose(env, "too many different kernel function calls\n"); 2608 return -E2BIG; 2609 } 2610 2611 func = btf_type_by_id(desc_btf, func_id); 2612 if (!func || !btf_type_is_func(func)) { 2613 verbose(env, "kernel btf_id %u is not a function\n", 2614 func_id); 2615 return -EINVAL; 2616 } 2617 func_proto = btf_type_by_id(desc_btf, func->type); 2618 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 2619 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 2620 func_id); 2621 return -EINVAL; 2622 } 2623 2624 func_name = btf_name_by_offset(desc_btf, func->name_off); 2625 addr = kallsyms_lookup_name(func_name); 2626 if (!addr) { 2627 verbose(env, "cannot find address for kernel function %s\n", 2628 func_name); 2629 return -EINVAL; 2630 } 2631 specialize_kfunc(env, func_id, offset, &addr); 2632 2633 if (bpf_jit_supports_far_kfunc_call()) { 2634 call_imm = func_id; 2635 } else { 2636 call_imm = BPF_CALL_IMM(addr); 2637 /* Check whether the relative offset overflows desc->imm */ 2638 if ((unsigned long)(s32)call_imm != call_imm) { 2639 verbose(env, "address of kernel function %s is out of range\n", 2640 func_name); 2641 return -EINVAL; 2642 } 2643 } 2644 2645 if (bpf_dev_bound_kfunc_id(func_id)) { 2646 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); 2647 if (err) 2648 return err; 2649 } 2650 2651 desc = &tab->descs[tab->nr_descs++]; 2652 desc->func_id = func_id; 2653 desc->imm = call_imm; 2654 desc->offset = offset; 2655 desc->addr = addr; 2656 err = btf_distill_func_proto(&env->log, desc_btf, 2657 func_proto, func_name, 2658 &desc->func_model); 2659 if (!err) 2660 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2661 kfunc_desc_cmp_by_id_off, NULL); 2662 return err; 2663 } 2664 2665 static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) 2666 { 2667 const struct bpf_kfunc_desc *d0 = a; 2668 const struct bpf_kfunc_desc *d1 = b; 2669 2670 if (d0->imm != d1->imm) 2671 return d0->imm < d1->imm ? -1 : 1; 2672 if (d0->offset != d1->offset) 2673 return d0->offset < d1->offset ? -1 : 1; 2674 return 0; 2675 } 2676 2677 static void sort_kfunc_descs_by_imm_off(struct bpf_prog *prog) 2678 { 2679 struct bpf_kfunc_desc_tab *tab; 2680 2681 tab = prog->aux->kfunc_tab; 2682 if (!tab) 2683 return; 2684 2685 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2686 kfunc_desc_cmp_by_imm_off, NULL); 2687 } 2688 2689 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2690 { 2691 return !!prog->aux->kfunc_tab; 2692 } 2693 2694 const struct btf_func_model * 2695 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2696 const struct bpf_insn *insn) 2697 { 2698 const struct bpf_kfunc_desc desc = { 2699 .imm = insn->imm, 2700 .offset = insn->off, 2701 }; 2702 const struct bpf_kfunc_desc *res; 2703 struct bpf_kfunc_desc_tab *tab; 2704 2705 tab = prog->aux->kfunc_tab; 2706 res = bsearch(&desc, tab->descs, tab->nr_descs, 2707 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm_off); 2708 2709 return res ? &res->func_model : NULL; 2710 } 2711 2712 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2713 { 2714 struct bpf_subprog_info *subprog = env->subprog_info; 2715 struct bpf_insn *insn = env->prog->insnsi; 2716 int i, ret, insn_cnt = env->prog->len; 2717 2718 /* Add entry function. */ 2719 ret = add_subprog(env, 0); 2720 if (ret) 2721 return ret; 2722 2723 for (i = 0; i < insn_cnt; i++, insn++) { 2724 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2725 !bpf_pseudo_kfunc_call(insn)) 2726 continue; 2727 2728 if (!env->bpf_capable) { 2729 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2730 return -EPERM; 2731 } 2732 2733 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2734 ret = add_subprog(env, i + insn->imm + 1); 2735 else 2736 ret = add_kfunc_call(env, insn->imm, insn->off); 2737 2738 if (ret < 0) 2739 return ret; 2740 } 2741 2742 /* Add a fake 'exit' subprog which could simplify subprog iteration 2743 * logic. 'subprog_cnt' should not be increased. 2744 */ 2745 subprog[env->subprog_cnt].start = insn_cnt; 2746 2747 if (env->log.level & BPF_LOG_LEVEL2) 2748 for (i = 0; i < env->subprog_cnt; i++) 2749 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2750 2751 return 0; 2752 } 2753 2754 static int check_subprogs(struct bpf_verifier_env *env) 2755 { 2756 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2757 struct bpf_subprog_info *subprog = env->subprog_info; 2758 struct bpf_insn *insn = env->prog->insnsi; 2759 int insn_cnt = env->prog->len; 2760 2761 /* now check that all jumps are within the same subprog */ 2762 subprog_start = subprog[cur_subprog].start; 2763 subprog_end = subprog[cur_subprog + 1].start; 2764 for (i = 0; i < insn_cnt; i++) { 2765 u8 code = insn[i].code; 2766 2767 if (code == (BPF_JMP | BPF_CALL) && 2768 insn[i].src_reg == 0 && 2769 insn[i].imm == BPF_FUNC_tail_call) 2770 subprog[cur_subprog].has_tail_call = true; 2771 if (BPF_CLASS(code) == BPF_LD && 2772 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2773 subprog[cur_subprog].has_ld_abs = true; 2774 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2775 goto next; 2776 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2777 goto next; 2778 off = i + insn[i].off + 1; 2779 if (off < subprog_start || off >= subprog_end) { 2780 verbose(env, "jump out of range from insn %d to %d\n", i, off); 2781 return -EINVAL; 2782 } 2783 next: 2784 if (i == subprog_end - 1) { 2785 /* to avoid fall-through from one subprog into another 2786 * the last insn of the subprog should be either exit 2787 * or unconditional jump back 2788 */ 2789 if (code != (BPF_JMP | BPF_EXIT) && 2790 code != (BPF_JMP | BPF_JA)) { 2791 verbose(env, "last insn is not an exit or jmp\n"); 2792 return -EINVAL; 2793 } 2794 subprog_start = subprog_end; 2795 cur_subprog++; 2796 if (cur_subprog < env->subprog_cnt) 2797 subprog_end = subprog[cur_subprog + 1].start; 2798 } 2799 } 2800 return 0; 2801 } 2802 2803 /* Parentage chain of this register (or stack slot) should take care of all 2804 * issues like callee-saved registers, stack slot allocation time, etc. 2805 */ 2806 static int mark_reg_read(struct bpf_verifier_env *env, 2807 const struct bpf_reg_state *state, 2808 struct bpf_reg_state *parent, u8 flag) 2809 { 2810 bool writes = parent == state->parent; /* Observe write marks */ 2811 int cnt = 0; 2812 2813 while (parent) { 2814 /* if read wasn't screened by an earlier write ... */ 2815 if (writes && state->live & REG_LIVE_WRITTEN) 2816 break; 2817 if (parent->live & REG_LIVE_DONE) { 2818 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 2819 reg_type_str(env, parent->type), 2820 parent->var_off.value, parent->off); 2821 return -EFAULT; 2822 } 2823 /* The first condition is more likely to be true than the 2824 * second, checked it first. 2825 */ 2826 if ((parent->live & REG_LIVE_READ) == flag || 2827 parent->live & REG_LIVE_READ64) 2828 /* The parentage chain never changes and 2829 * this parent was already marked as LIVE_READ. 2830 * There is no need to keep walking the chain again and 2831 * keep re-marking all parents as LIVE_READ. 2832 * This case happens when the same register is read 2833 * multiple times without writes into it in-between. 2834 * Also, if parent has the stronger REG_LIVE_READ64 set, 2835 * then no need to set the weak REG_LIVE_READ32. 2836 */ 2837 break; 2838 /* ... then we depend on parent's value */ 2839 parent->live |= flag; 2840 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 2841 if (flag == REG_LIVE_READ64) 2842 parent->live &= ~REG_LIVE_READ32; 2843 state = parent; 2844 parent = state->parent; 2845 writes = true; 2846 cnt++; 2847 } 2848 2849 if (env->longest_mark_read_walk < cnt) 2850 env->longest_mark_read_walk = cnt; 2851 return 0; 2852 } 2853 2854 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 2855 { 2856 struct bpf_func_state *state = func(env, reg); 2857 int spi, ret; 2858 2859 /* For CONST_PTR_TO_DYNPTR, it must have already been done by 2860 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in 2861 * check_kfunc_call. 2862 */ 2863 if (reg->type == CONST_PTR_TO_DYNPTR) 2864 return 0; 2865 spi = dynptr_get_spi(env, reg); 2866 if (spi < 0) 2867 return spi; 2868 /* Caller ensures dynptr is valid and initialized, which means spi is in 2869 * bounds and spi is the first dynptr slot. Simply mark stack slot as 2870 * read. 2871 */ 2872 ret = mark_reg_read(env, &state->stack[spi].spilled_ptr, 2873 state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64); 2874 if (ret) 2875 return ret; 2876 return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr, 2877 state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64); 2878 } 2879 2880 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 2881 int spi, int nr_slots) 2882 { 2883 struct bpf_func_state *state = func(env, reg); 2884 int err, i; 2885 2886 for (i = 0; i < nr_slots; i++) { 2887 struct bpf_reg_state *st = &state->stack[spi - i].spilled_ptr; 2888 2889 err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); 2890 if (err) 2891 return err; 2892 2893 mark_stack_slot_scratched(env, spi - i); 2894 } 2895 2896 return 0; 2897 } 2898 2899 /* This function is supposed to be used by the following 32-bit optimization 2900 * code only. It returns TRUE if the source or destination register operates 2901 * on 64-bit, otherwise return FALSE. 2902 */ 2903 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 2904 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 2905 { 2906 u8 code, class, op; 2907 2908 code = insn->code; 2909 class = BPF_CLASS(code); 2910 op = BPF_OP(code); 2911 if (class == BPF_JMP) { 2912 /* BPF_EXIT for "main" will reach here. Return TRUE 2913 * conservatively. 2914 */ 2915 if (op == BPF_EXIT) 2916 return true; 2917 if (op == BPF_CALL) { 2918 /* BPF to BPF call will reach here because of marking 2919 * caller saved clobber with DST_OP_NO_MARK for which we 2920 * don't care the register def because they are anyway 2921 * marked as NOT_INIT already. 2922 */ 2923 if (insn->src_reg == BPF_PSEUDO_CALL) 2924 return false; 2925 /* Helper call will reach here because of arg type 2926 * check, conservatively return TRUE. 2927 */ 2928 if (t == SRC_OP) 2929 return true; 2930 2931 return false; 2932 } 2933 } 2934 2935 if (class == BPF_ALU64 || class == BPF_JMP || 2936 /* BPF_END always use BPF_ALU class. */ 2937 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 2938 return true; 2939 2940 if (class == BPF_ALU || class == BPF_JMP32) 2941 return false; 2942 2943 if (class == BPF_LDX) { 2944 if (t != SRC_OP) 2945 return BPF_SIZE(code) == BPF_DW; 2946 /* LDX source must be ptr. */ 2947 return true; 2948 } 2949 2950 if (class == BPF_STX) { 2951 /* BPF_STX (including atomic variants) has multiple source 2952 * operands, one of which is a ptr. Check whether the caller is 2953 * asking about it. 2954 */ 2955 if (t == SRC_OP && reg->type != SCALAR_VALUE) 2956 return true; 2957 return BPF_SIZE(code) == BPF_DW; 2958 } 2959 2960 if (class == BPF_LD) { 2961 u8 mode = BPF_MODE(code); 2962 2963 /* LD_IMM64 */ 2964 if (mode == BPF_IMM) 2965 return true; 2966 2967 /* Both LD_IND and LD_ABS return 32-bit data. */ 2968 if (t != SRC_OP) 2969 return false; 2970 2971 /* Implicit ctx ptr. */ 2972 if (regno == BPF_REG_6) 2973 return true; 2974 2975 /* Explicit source could be any width. */ 2976 return true; 2977 } 2978 2979 if (class == BPF_ST) 2980 /* The only source register for BPF_ST is a ptr. */ 2981 return true; 2982 2983 /* Conservatively return true at default. */ 2984 return true; 2985 } 2986 2987 /* Return the regno defined by the insn, or -1. */ 2988 static int insn_def_regno(const struct bpf_insn *insn) 2989 { 2990 switch (BPF_CLASS(insn->code)) { 2991 case BPF_JMP: 2992 case BPF_JMP32: 2993 case BPF_ST: 2994 return -1; 2995 case BPF_STX: 2996 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2997 (insn->imm & BPF_FETCH)) { 2998 if (insn->imm == BPF_CMPXCHG) 2999 return BPF_REG_0; 3000 else 3001 return insn->src_reg; 3002 } else { 3003 return -1; 3004 } 3005 default: 3006 return insn->dst_reg; 3007 } 3008 } 3009 3010 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 3011 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 3012 { 3013 int dst_reg = insn_def_regno(insn); 3014 3015 if (dst_reg == -1) 3016 return false; 3017 3018 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 3019 } 3020 3021 static void mark_insn_zext(struct bpf_verifier_env *env, 3022 struct bpf_reg_state *reg) 3023 { 3024 s32 def_idx = reg->subreg_def; 3025 3026 if (def_idx == DEF_NOT_SUBREG) 3027 return; 3028 3029 env->insn_aux_data[def_idx - 1].zext_dst = true; 3030 /* The dst will be zero extended, so won't be sub-register anymore. */ 3031 reg->subreg_def = DEF_NOT_SUBREG; 3032 } 3033 3034 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 3035 enum reg_arg_type t) 3036 { 3037 struct bpf_verifier_state *vstate = env->cur_state; 3038 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3039 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 3040 struct bpf_reg_state *reg, *regs = state->regs; 3041 bool rw64; 3042 3043 if (regno >= MAX_BPF_REG) { 3044 verbose(env, "R%d is invalid\n", regno); 3045 return -EINVAL; 3046 } 3047 3048 mark_reg_scratched(env, regno); 3049 3050 reg = ®s[regno]; 3051 rw64 = is_reg64(env, insn, regno, reg, t); 3052 if (t == SRC_OP) { 3053 /* check whether register used as source operand can be read */ 3054 if (reg->type == NOT_INIT) { 3055 verbose(env, "R%d !read_ok\n", regno); 3056 return -EACCES; 3057 } 3058 /* We don't need to worry about FP liveness because it's read-only */ 3059 if (regno == BPF_REG_FP) 3060 return 0; 3061 3062 if (rw64) 3063 mark_insn_zext(env, reg); 3064 3065 return mark_reg_read(env, reg, reg->parent, 3066 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 3067 } else { 3068 /* check whether register used as dest operand can be written to */ 3069 if (regno == BPF_REG_FP) { 3070 verbose(env, "frame pointer is read only\n"); 3071 return -EACCES; 3072 } 3073 reg->live |= REG_LIVE_WRITTEN; 3074 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 3075 if (t == DST_OP) 3076 mark_reg_unknown(env, regs, regno); 3077 } 3078 return 0; 3079 } 3080 3081 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) 3082 { 3083 env->insn_aux_data[idx].jmp_point = true; 3084 } 3085 3086 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) 3087 { 3088 return env->insn_aux_data[insn_idx].jmp_point; 3089 } 3090 3091 /* for any branch, call, exit record the history of jmps in the given state */ 3092 static int push_jmp_history(struct bpf_verifier_env *env, 3093 struct bpf_verifier_state *cur) 3094 { 3095 u32 cnt = cur->jmp_history_cnt; 3096 struct bpf_idx_pair *p; 3097 size_t alloc_size; 3098 3099 if (!is_jmp_point(env, env->insn_idx)) 3100 return 0; 3101 3102 cnt++; 3103 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); 3104 p = krealloc(cur->jmp_history, alloc_size, GFP_USER); 3105 if (!p) 3106 return -ENOMEM; 3107 p[cnt - 1].idx = env->insn_idx; 3108 p[cnt - 1].prev_idx = env->prev_insn_idx; 3109 cur->jmp_history = p; 3110 cur->jmp_history_cnt = cnt; 3111 return 0; 3112 } 3113 3114 /* Backtrack one insn at a time. If idx is not at the top of recorded 3115 * history then previous instruction came from straight line execution. 3116 */ 3117 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 3118 u32 *history) 3119 { 3120 u32 cnt = *history; 3121 3122 if (cnt && st->jmp_history[cnt - 1].idx == i) { 3123 i = st->jmp_history[cnt - 1].prev_idx; 3124 (*history)--; 3125 } else { 3126 i--; 3127 } 3128 return i; 3129 } 3130 3131 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 3132 { 3133 const struct btf_type *func; 3134 struct btf *desc_btf; 3135 3136 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 3137 return NULL; 3138 3139 desc_btf = find_kfunc_desc_btf(data, insn->off); 3140 if (IS_ERR(desc_btf)) 3141 return "<error>"; 3142 3143 func = btf_type_by_id(desc_btf, insn->imm); 3144 return btf_name_by_offset(desc_btf, func->name_off); 3145 } 3146 3147 /* For given verifier state backtrack_insn() is called from the last insn to 3148 * the first insn. Its purpose is to compute a bitmask of registers and 3149 * stack slots that needs precision in the parent verifier state. 3150 */ 3151 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 3152 u32 *reg_mask, u64 *stack_mask) 3153 { 3154 const struct bpf_insn_cbs cbs = { 3155 .cb_call = disasm_kfunc_name, 3156 .cb_print = verbose, 3157 .private_data = env, 3158 }; 3159 struct bpf_insn *insn = env->prog->insnsi + idx; 3160 u8 class = BPF_CLASS(insn->code); 3161 u8 opcode = BPF_OP(insn->code); 3162 u8 mode = BPF_MODE(insn->code); 3163 u32 dreg = 1u << insn->dst_reg; 3164 u32 sreg = 1u << insn->src_reg; 3165 u32 spi; 3166 3167 if (insn->code == 0) 3168 return 0; 3169 if (env->log.level & BPF_LOG_LEVEL2) { 3170 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 3171 verbose(env, "%d: ", idx); 3172 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 3173 } 3174 3175 if (class == BPF_ALU || class == BPF_ALU64) { 3176 if (!(*reg_mask & dreg)) 3177 return 0; 3178 if (opcode == BPF_MOV) { 3179 if (BPF_SRC(insn->code) == BPF_X) { 3180 /* dreg = sreg 3181 * dreg needs precision after this insn 3182 * sreg needs precision before this insn 3183 */ 3184 *reg_mask &= ~dreg; 3185 *reg_mask |= sreg; 3186 } else { 3187 /* dreg = K 3188 * dreg needs precision after this insn. 3189 * Corresponding register is already marked 3190 * as precise=true in this verifier state. 3191 * No further markings in parent are necessary 3192 */ 3193 *reg_mask &= ~dreg; 3194 } 3195 } else { 3196 if (BPF_SRC(insn->code) == BPF_X) { 3197 /* dreg += sreg 3198 * both dreg and sreg need precision 3199 * before this insn 3200 */ 3201 *reg_mask |= sreg; 3202 } /* else dreg += K 3203 * dreg still needs precision before this insn 3204 */ 3205 } 3206 } else if (class == BPF_LDX) { 3207 if (!(*reg_mask & dreg)) 3208 return 0; 3209 *reg_mask &= ~dreg; 3210 3211 /* scalars can only be spilled into stack w/o losing precision. 3212 * Load from any other memory can be zero extended. 3213 * The desire to keep that precision is already indicated 3214 * by 'precise' mark in corresponding register of this state. 3215 * No further tracking necessary. 3216 */ 3217 if (insn->src_reg != BPF_REG_FP) 3218 return 0; 3219 3220 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 3221 * that [fp - off] slot contains scalar that needs to be 3222 * tracked with precision 3223 */ 3224 spi = (-insn->off - 1) / BPF_REG_SIZE; 3225 if (spi >= 64) { 3226 verbose(env, "BUG spi %d\n", spi); 3227 WARN_ONCE(1, "verifier backtracking bug"); 3228 return -EFAULT; 3229 } 3230 *stack_mask |= 1ull << spi; 3231 } else if (class == BPF_STX || class == BPF_ST) { 3232 if (*reg_mask & dreg) 3233 /* stx & st shouldn't be using _scalar_ dst_reg 3234 * to access memory. It means backtracking 3235 * encountered a case of pointer subtraction. 3236 */ 3237 return -ENOTSUPP; 3238 /* scalars can only be spilled into stack */ 3239 if (insn->dst_reg != BPF_REG_FP) 3240 return 0; 3241 spi = (-insn->off - 1) / BPF_REG_SIZE; 3242 if (spi >= 64) { 3243 verbose(env, "BUG spi %d\n", spi); 3244 WARN_ONCE(1, "verifier backtracking bug"); 3245 return -EFAULT; 3246 } 3247 if (!(*stack_mask & (1ull << spi))) 3248 return 0; 3249 *stack_mask &= ~(1ull << spi); 3250 if (class == BPF_STX) 3251 *reg_mask |= sreg; 3252 } else if (class == BPF_JMP || class == BPF_JMP32) { 3253 if (opcode == BPF_CALL) { 3254 if (insn->src_reg == BPF_PSEUDO_CALL) 3255 return -ENOTSUPP; 3256 /* BPF helpers that invoke callback subprogs are 3257 * equivalent to BPF_PSEUDO_CALL above 3258 */ 3259 if (insn->src_reg == 0 && is_callback_calling_function(insn->imm)) 3260 return -ENOTSUPP; 3261 /* kfunc with imm==0 is invalid and fixup_kfunc_call will 3262 * catch this error later. Make backtracking conservative 3263 * with ENOTSUPP. 3264 */ 3265 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) 3266 return -ENOTSUPP; 3267 /* regular helper call sets R0 */ 3268 *reg_mask &= ~1; 3269 if (*reg_mask & 0x3f) { 3270 /* if backtracing was looking for registers R1-R5 3271 * they should have been found already. 3272 */ 3273 verbose(env, "BUG regs %x\n", *reg_mask); 3274 WARN_ONCE(1, "verifier backtracking bug"); 3275 return -EFAULT; 3276 } 3277 } else if (opcode == BPF_EXIT) { 3278 return -ENOTSUPP; 3279 } else if (BPF_SRC(insn->code) == BPF_X) { 3280 if (!(*reg_mask & (dreg | sreg))) 3281 return 0; 3282 /* dreg <cond> sreg 3283 * Both dreg and sreg need precision before 3284 * this insn. If only sreg was marked precise 3285 * before it would be equally necessary to 3286 * propagate it to dreg. 3287 */ 3288 *reg_mask |= (sreg | dreg); 3289 /* else dreg <cond> K 3290 * Only dreg still needs precision before 3291 * this insn, so for the K-based conditional 3292 * there is nothing new to be marked. 3293 */ 3294 } 3295 } else if (class == BPF_LD) { 3296 if (!(*reg_mask & dreg)) 3297 return 0; 3298 *reg_mask &= ~dreg; 3299 /* It's ld_imm64 or ld_abs or ld_ind. 3300 * For ld_imm64 no further tracking of precision 3301 * into parent is necessary 3302 */ 3303 if (mode == BPF_IND || mode == BPF_ABS) 3304 /* to be analyzed */ 3305 return -ENOTSUPP; 3306 } 3307 return 0; 3308 } 3309 3310 /* the scalar precision tracking algorithm: 3311 * . at the start all registers have precise=false. 3312 * . scalar ranges are tracked as normal through alu and jmp insns. 3313 * . once precise value of the scalar register is used in: 3314 * . ptr + scalar alu 3315 * . if (scalar cond K|scalar) 3316 * . helper_call(.., scalar, ...) where ARG_CONST is expected 3317 * backtrack through the verifier states and mark all registers and 3318 * stack slots with spilled constants that these scalar regisers 3319 * should be precise. 3320 * . during state pruning two registers (or spilled stack slots) 3321 * are equivalent if both are not precise. 3322 * 3323 * Note the verifier cannot simply walk register parentage chain, 3324 * since many different registers and stack slots could have been 3325 * used to compute single precise scalar. 3326 * 3327 * The approach of starting with precise=true for all registers and then 3328 * backtrack to mark a register as not precise when the verifier detects 3329 * that program doesn't care about specific value (e.g., when helper 3330 * takes register as ARG_ANYTHING parameter) is not safe. 3331 * 3332 * It's ok to walk single parentage chain of the verifier states. 3333 * It's possible that this backtracking will go all the way till 1st insn. 3334 * All other branches will be explored for needing precision later. 3335 * 3336 * The backtracking needs to deal with cases like: 3337 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 3338 * r9 -= r8 3339 * r5 = r9 3340 * if r5 > 0x79f goto pc+7 3341 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 3342 * r5 += 1 3343 * ... 3344 * call bpf_perf_event_output#25 3345 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 3346 * 3347 * and this case: 3348 * r6 = 1 3349 * call foo // uses callee's r6 inside to compute r0 3350 * r0 += r6 3351 * if r0 == 0 goto 3352 * 3353 * to track above reg_mask/stack_mask needs to be independent for each frame. 3354 * 3355 * Also if parent's curframe > frame where backtracking started, 3356 * the verifier need to mark registers in both frames, otherwise callees 3357 * may incorrectly prune callers. This is similar to 3358 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 3359 * 3360 * For now backtracking falls back into conservative marking. 3361 */ 3362 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 3363 struct bpf_verifier_state *st) 3364 { 3365 struct bpf_func_state *func; 3366 struct bpf_reg_state *reg; 3367 int i, j; 3368 3369 /* big hammer: mark all scalars precise in this path. 3370 * pop_stack may still get !precise scalars. 3371 * We also skip current state and go straight to first parent state, 3372 * because precision markings in current non-checkpointed state are 3373 * not needed. See why in the comment in __mark_chain_precision below. 3374 */ 3375 for (st = st->parent; st; st = st->parent) { 3376 for (i = 0; i <= st->curframe; i++) { 3377 func = st->frame[i]; 3378 for (j = 0; j < BPF_REG_FP; j++) { 3379 reg = &func->regs[j]; 3380 if (reg->type != SCALAR_VALUE) 3381 continue; 3382 reg->precise = true; 3383 } 3384 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3385 if (!is_spilled_reg(&func->stack[j])) 3386 continue; 3387 reg = &func->stack[j].spilled_ptr; 3388 if (reg->type != SCALAR_VALUE) 3389 continue; 3390 reg->precise = true; 3391 } 3392 } 3393 } 3394 } 3395 3396 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 3397 { 3398 struct bpf_func_state *func; 3399 struct bpf_reg_state *reg; 3400 int i, j; 3401 3402 for (i = 0; i <= st->curframe; i++) { 3403 func = st->frame[i]; 3404 for (j = 0; j < BPF_REG_FP; j++) { 3405 reg = &func->regs[j]; 3406 if (reg->type != SCALAR_VALUE) 3407 continue; 3408 reg->precise = false; 3409 } 3410 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3411 if (!is_spilled_reg(&func->stack[j])) 3412 continue; 3413 reg = &func->stack[j].spilled_ptr; 3414 if (reg->type != SCALAR_VALUE) 3415 continue; 3416 reg->precise = false; 3417 } 3418 } 3419 } 3420 3421 /* 3422 * __mark_chain_precision() backtracks BPF program instruction sequence and 3423 * chain of verifier states making sure that register *regno* (if regno >= 0) 3424 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked 3425 * SCALARS, as well as any other registers and slots that contribute to 3426 * a tracked state of given registers/stack slots, depending on specific BPF 3427 * assembly instructions (see backtrack_insns() for exact instruction handling 3428 * logic). This backtracking relies on recorded jmp_history and is able to 3429 * traverse entire chain of parent states. This process ends only when all the 3430 * necessary registers/slots and their transitive dependencies are marked as 3431 * precise. 3432 * 3433 * One important and subtle aspect is that precise marks *do not matter* in 3434 * the currently verified state (current state). It is important to understand 3435 * why this is the case. 3436 * 3437 * First, note that current state is the state that is not yet "checkpointed", 3438 * i.e., it is not yet put into env->explored_states, and it has no children 3439 * states as well. It's ephemeral, and can end up either a) being discarded if 3440 * compatible explored state is found at some point or BPF_EXIT instruction is 3441 * reached or b) checkpointed and put into env->explored_states, branching out 3442 * into one or more children states. 3443 * 3444 * In the former case, precise markings in current state are completely 3445 * ignored by state comparison code (see regsafe() for details). Only 3446 * checkpointed ("old") state precise markings are important, and if old 3447 * state's register/slot is precise, regsafe() assumes current state's 3448 * register/slot as precise and checks value ranges exactly and precisely. If 3449 * states turn out to be compatible, current state's necessary precise 3450 * markings and any required parent states' precise markings are enforced 3451 * after the fact with propagate_precision() logic, after the fact. But it's 3452 * important to realize that in this case, even after marking current state 3453 * registers/slots as precise, we immediately discard current state. So what 3454 * actually matters is any of the precise markings propagated into current 3455 * state's parent states, which are always checkpointed (due to b) case above). 3456 * As such, for scenario a) it doesn't matter if current state has precise 3457 * markings set or not. 3458 * 3459 * Now, for the scenario b), checkpointing and forking into child(ren) 3460 * state(s). Note that before current state gets to checkpointing step, any 3461 * processed instruction always assumes precise SCALAR register/slot 3462 * knowledge: if precise value or range is useful to prune jump branch, BPF 3463 * verifier takes this opportunity enthusiastically. Similarly, when 3464 * register's value is used to calculate offset or memory address, exact 3465 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to 3466 * what we mentioned above about state comparison ignoring precise markings 3467 * during state comparison, BPF verifier ignores and also assumes precise 3468 * markings *at will* during instruction verification process. But as verifier 3469 * assumes precision, it also propagates any precision dependencies across 3470 * parent states, which are not yet finalized, so can be further restricted 3471 * based on new knowledge gained from restrictions enforced by their children 3472 * states. This is so that once those parent states are finalized, i.e., when 3473 * they have no more active children state, state comparison logic in 3474 * is_state_visited() would enforce strict and precise SCALAR ranges, if 3475 * required for correctness. 3476 * 3477 * To build a bit more intuition, note also that once a state is checkpointed, 3478 * the path we took to get to that state is not important. This is crucial 3479 * property for state pruning. When state is checkpointed and finalized at 3480 * some instruction index, it can be correctly and safely used to "short 3481 * circuit" any *compatible* state that reaches exactly the same instruction 3482 * index. I.e., if we jumped to that instruction from a completely different 3483 * code path than original finalized state was derived from, it doesn't 3484 * matter, current state can be discarded because from that instruction 3485 * forward having a compatible state will ensure we will safely reach the 3486 * exit. States describe preconditions for further exploration, but completely 3487 * forget the history of how we got here. 3488 * 3489 * This also means that even if we needed precise SCALAR range to get to 3490 * finalized state, but from that point forward *that same* SCALAR register is 3491 * never used in a precise context (i.e., it's precise value is not needed for 3492 * correctness), it's correct and safe to mark such register as "imprecise" 3493 * (i.e., precise marking set to false). This is what we rely on when we do 3494 * not set precise marking in current state. If no child state requires 3495 * precision for any given SCALAR register, it's safe to dictate that it can 3496 * be imprecise. If any child state does require this register to be precise, 3497 * we'll mark it precise later retroactively during precise markings 3498 * propagation from child state to parent states. 3499 * 3500 * Skipping precise marking setting in current state is a mild version of 3501 * relying on the above observation. But we can utilize this property even 3502 * more aggressively by proactively forgetting any precise marking in the 3503 * current state (which we inherited from the parent state), right before we 3504 * checkpoint it and branch off into new child state. This is done by 3505 * mark_all_scalars_imprecise() to hopefully get more permissive and generic 3506 * finalized states which help in short circuiting more future states. 3507 */ 3508 static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno, 3509 int spi) 3510 { 3511 struct bpf_verifier_state *st = env->cur_state; 3512 int first_idx = st->first_insn_idx; 3513 int last_idx = env->insn_idx; 3514 struct bpf_func_state *func; 3515 struct bpf_reg_state *reg; 3516 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 3517 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 3518 bool skip_first = true; 3519 bool new_marks = false; 3520 int i, err; 3521 3522 if (!env->bpf_capable) 3523 return 0; 3524 3525 /* Do sanity checks against current state of register and/or stack 3526 * slot, but don't set precise flag in current state, as precision 3527 * tracking in the current state is unnecessary. 3528 */ 3529 func = st->frame[frame]; 3530 if (regno >= 0) { 3531 reg = &func->regs[regno]; 3532 if (reg->type != SCALAR_VALUE) { 3533 WARN_ONCE(1, "backtracing misuse"); 3534 return -EFAULT; 3535 } 3536 new_marks = true; 3537 } 3538 3539 while (spi >= 0) { 3540 if (!is_spilled_reg(&func->stack[spi])) { 3541 stack_mask = 0; 3542 break; 3543 } 3544 reg = &func->stack[spi].spilled_ptr; 3545 if (reg->type != SCALAR_VALUE) { 3546 stack_mask = 0; 3547 break; 3548 } 3549 new_marks = true; 3550 break; 3551 } 3552 3553 if (!new_marks) 3554 return 0; 3555 if (!reg_mask && !stack_mask) 3556 return 0; 3557 3558 for (;;) { 3559 DECLARE_BITMAP(mask, 64); 3560 u32 history = st->jmp_history_cnt; 3561 3562 if (env->log.level & BPF_LOG_LEVEL2) 3563 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 3564 3565 if (last_idx < 0) { 3566 /* we are at the entry into subprog, which 3567 * is expected for global funcs, but only if 3568 * requested precise registers are R1-R5 3569 * (which are global func's input arguments) 3570 */ 3571 if (st->curframe == 0 && 3572 st->frame[0]->subprogno > 0 && 3573 st->frame[0]->callsite == BPF_MAIN_FUNC && 3574 stack_mask == 0 && (reg_mask & ~0x3e) == 0) { 3575 bitmap_from_u64(mask, reg_mask); 3576 for_each_set_bit(i, mask, 32) { 3577 reg = &st->frame[0]->regs[i]; 3578 if (reg->type != SCALAR_VALUE) { 3579 reg_mask &= ~(1u << i); 3580 continue; 3581 } 3582 reg->precise = true; 3583 } 3584 return 0; 3585 } 3586 3587 verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n", 3588 st->frame[0]->subprogno, reg_mask, stack_mask); 3589 WARN_ONCE(1, "verifier backtracking bug"); 3590 return -EFAULT; 3591 } 3592 3593 for (i = last_idx;;) { 3594 if (skip_first) { 3595 err = 0; 3596 skip_first = false; 3597 } else { 3598 err = backtrack_insn(env, i, ®_mask, &stack_mask); 3599 } 3600 if (err == -ENOTSUPP) { 3601 mark_all_scalars_precise(env, st); 3602 return 0; 3603 } else if (err) { 3604 return err; 3605 } 3606 if (!reg_mask && !stack_mask) 3607 /* Found assignment(s) into tracked register in this state. 3608 * Since this state is already marked, just return. 3609 * Nothing to be tracked further in the parent state. 3610 */ 3611 return 0; 3612 if (i == first_idx) 3613 break; 3614 i = get_prev_insn_idx(st, i, &history); 3615 if (i >= env->prog->len) { 3616 /* This can happen if backtracking reached insn 0 3617 * and there are still reg_mask or stack_mask 3618 * to backtrack. 3619 * It means the backtracking missed the spot where 3620 * particular register was initialized with a constant. 3621 */ 3622 verbose(env, "BUG backtracking idx %d\n", i); 3623 WARN_ONCE(1, "verifier backtracking bug"); 3624 return -EFAULT; 3625 } 3626 } 3627 st = st->parent; 3628 if (!st) 3629 break; 3630 3631 new_marks = false; 3632 func = st->frame[frame]; 3633 bitmap_from_u64(mask, reg_mask); 3634 for_each_set_bit(i, mask, 32) { 3635 reg = &func->regs[i]; 3636 if (reg->type != SCALAR_VALUE) { 3637 reg_mask &= ~(1u << i); 3638 continue; 3639 } 3640 if (!reg->precise) 3641 new_marks = true; 3642 reg->precise = true; 3643 } 3644 3645 bitmap_from_u64(mask, stack_mask); 3646 for_each_set_bit(i, mask, 64) { 3647 if (i >= func->allocated_stack / BPF_REG_SIZE) { 3648 /* the sequence of instructions: 3649 * 2: (bf) r3 = r10 3650 * 3: (7b) *(u64 *)(r3 -8) = r0 3651 * 4: (79) r4 = *(u64 *)(r10 -8) 3652 * doesn't contain jmps. It's backtracked 3653 * as a single block. 3654 * During backtracking insn 3 is not recognized as 3655 * stack access, so at the end of backtracking 3656 * stack slot fp-8 is still marked in stack_mask. 3657 * However the parent state may not have accessed 3658 * fp-8 and it's "unallocated" stack space. 3659 * In such case fallback to conservative. 3660 */ 3661 mark_all_scalars_precise(env, st); 3662 return 0; 3663 } 3664 3665 if (!is_spilled_reg(&func->stack[i])) { 3666 stack_mask &= ~(1ull << i); 3667 continue; 3668 } 3669 reg = &func->stack[i].spilled_ptr; 3670 if (reg->type != SCALAR_VALUE) { 3671 stack_mask &= ~(1ull << i); 3672 continue; 3673 } 3674 if (!reg->precise) 3675 new_marks = true; 3676 reg->precise = true; 3677 } 3678 if (env->log.level & BPF_LOG_LEVEL2) { 3679 verbose(env, "parent %s regs=%x stack=%llx marks:", 3680 new_marks ? "didn't have" : "already had", 3681 reg_mask, stack_mask); 3682 print_verifier_state(env, func, true); 3683 } 3684 3685 if (!reg_mask && !stack_mask) 3686 break; 3687 if (!new_marks) 3688 break; 3689 3690 last_idx = st->last_insn_idx; 3691 first_idx = st->first_insn_idx; 3692 } 3693 return 0; 3694 } 3695 3696 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 3697 { 3698 return __mark_chain_precision(env, env->cur_state->curframe, regno, -1); 3699 } 3700 3701 static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno) 3702 { 3703 return __mark_chain_precision(env, frame, regno, -1); 3704 } 3705 3706 static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi) 3707 { 3708 return __mark_chain_precision(env, frame, -1, spi); 3709 } 3710 3711 static bool is_spillable_regtype(enum bpf_reg_type type) 3712 { 3713 switch (base_type(type)) { 3714 case PTR_TO_MAP_VALUE: 3715 case PTR_TO_STACK: 3716 case PTR_TO_CTX: 3717 case PTR_TO_PACKET: 3718 case PTR_TO_PACKET_META: 3719 case PTR_TO_PACKET_END: 3720 case PTR_TO_FLOW_KEYS: 3721 case CONST_PTR_TO_MAP: 3722 case PTR_TO_SOCKET: 3723 case PTR_TO_SOCK_COMMON: 3724 case PTR_TO_TCP_SOCK: 3725 case PTR_TO_XDP_SOCK: 3726 case PTR_TO_BTF_ID: 3727 case PTR_TO_BUF: 3728 case PTR_TO_MEM: 3729 case PTR_TO_FUNC: 3730 case PTR_TO_MAP_KEY: 3731 return true; 3732 default: 3733 return false; 3734 } 3735 } 3736 3737 /* Does this register contain a constant zero? */ 3738 static bool register_is_null(struct bpf_reg_state *reg) 3739 { 3740 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 3741 } 3742 3743 static bool register_is_const(struct bpf_reg_state *reg) 3744 { 3745 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 3746 } 3747 3748 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 3749 { 3750 return tnum_is_unknown(reg->var_off) && 3751 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 3752 reg->umin_value == 0 && reg->umax_value == U64_MAX && 3753 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 3754 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 3755 } 3756 3757 static bool register_is_bounded(struct bpf_reg_state *reg) 3758 { 3759 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 3760 } 3761 3762 static bool __is_pointer_value(bool allow_ptr_leaks, 3763 const struct bpf_reg_state *reg) 3764 { 3765 if (allow_ptr_leaks) 3766 return false; 3767 3768 return reg->type != SCALAR_VALUE; 3769 } 3770 3771 /* Copy src state preserving dst->parent and dst->live fields */ 3772 static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src) 3773 { 3774 struct bpf_reg_state *parent = dst->parent; 3775 enum bpf_reg_liveness live = dst->live; 3776 3777 *dst = *src; 3778 dst->parent = parent; 3779 dst->live = live; 3780 } 3781 3782 static void save_register_state(struct bpf_func_state *state, 3783 int spi, struct bpf_reg_state *reg, 3784 int size) 3785 { 3786 int i; 3787 3788 copy_register_state(&state->stack[spi].spilled_ptr, reg); 3789 if (size == BPF_REG_SIZE) 3790 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3791 3792 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 3793 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 3794 3795 /* size < 8 bytes spill */ 3796 for (; i; i--) 3797 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); 3798 } 3799 3800 static bool is_bpf_st_mem(struct bpf_insn *insn) 3801 { 3802 return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM; 3803 } 3804 3805 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 3806 * stack boundary and alignment are checked in check_mem_access() 3807 */ 3808 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 3809 /* stack frame we're writing to */ 3810 struct bpf_func_state *state, 3811 int off, int size, int value_regno, 3812 int insn_idx) 3813 { 3814 struct bpf_func_state *cur; /* state of the current function */ 3815 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 3816 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3817 struct bpf_reg_state *reg = NULL; 3818 u32 dst_reg = insn->dst_reg; 3819 3820 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 3821 if (err) 3822 return err; 3823 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 3824 * so it's aligned access and [off, off + size) are within stack limits 3825 */ 3826 if (!env->allow_ptr_leaks && 3827 state->stack[spi].slot_type[0] == STACK_SPILL && 3828 size != BPF_REG_SIZE) { 3829 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 3830 return -EACCES; 3831 } 3832 3833 cur = env->cur_state->frame[env->cur_state->curframe]; 3834 if (value_regno >= 0) 3835 reg = &cur->regs[value_regno]; 3836 if (!env->bypass_spec_v4) { 3837 bool sanitize = reg && is_spillable_regtype(reg->type); 3838 3839 for (i = 0; i < size; i++) { 3840 u8 type = state->stack[spi].slot_type[i]; 3841 3842 if (type != STACK_MISC && type != STACK_ZERO) { 3843 sanitize = true; 3844 break; 3845 } 3846 } 3847 3848 if (sanitize) 3849 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 3850 } 3851 3852 err = destroy_if_dynptr_stack_slot(env, state, spi); 3853 if (err) 3854 return err; 3855 3856 mark_stack_slot_scratched(env, spi); 3857 if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && 3858 !register_is_null(reg) && env->bpf_capable) { 3859 if (dst_reg != BPF_REG_FP) { 3860 /* The backtracking logic can only recognize explicit 3861 * stack slot address like [fp - 8]. Other spill of 3862 * scalar via different register has to be conservative. 3863 * Backtrack from here and mark all registers as precise 3864 * that contributed into 'reg' being a constant. 3865 */ 3866 err = mark_chain_precision(env, value_regno); 3867 if (err) 3868 return err; 3869 } 3870 save_register_state(state, spi, reg, size); 3871 } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && 3872 insn->imm != 0 && env->bpf_capable) { 3873 struct bpf_reg_state fake_reg = {}; 3874 3875 __mark_reg_known(&fake_reg, (u32)insn->imm); 3876 fake_reg.type = SCALAR_VALUE; 3877 save_register_state(state, spi, &fake_reg, size); 3878 } else if (reg && is_spillable_regtype(reg->type)) { 3879 /* register containing pointer is being spilled into stack */ 3880 if (size != BPF_REG_SIZE) { 3881 verbose_linfo(env, insn_idx, "; "); 3882 verbose(env, "invalid size of register spill\n"); 3883 return -EACCES; 3884 } 3885 if (state != cur && reg->type == PTR_TO_STACK) { 3886 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 3887 return -EINVAL; 3888 } 3889 save_register_state(state, spi, reg, size); 3890 } else { 3891 u8 type = STACK_MISC; 3892 3893 /* regular write of data into stack destroys any spilled ptr */ 3894 state->stack[spi].spilled_ptr.type = NOT_INIT; 3895 /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ 3896 if (is_stack_slot_special(&state->stack[spi])) 3897 for (i = 0; i < BPF_REG_SIZE; i++) 3898 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 3899 3900 /* only mark the slot as written if all 8 bytes were written 3901 * otherwise read propagation may incorrectly stop too soon 3902 * when stack slots are partially written. 3903 * This heuristic means that read propagation will be 3904 * conservative, since it will add reg_live_read marks 3905 * to stack slots all the way to first state when programs 3906 * writes+reads less than 8 bytes 3907 */ 3908 if (size == BPF_REG_SIZE) 3909 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3910 3911 /* when we zero initialize stack slots mark them as such */ 3912 if ((reg && register_is_null(reg)) || 3913 (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { 3914 /* backtracking doesn't work for STACK_ZERO yet. */ 3915 err = mark_chain_precision(env, value_regno); 3916 if (err) 3917 return err; 3918 type = STACK_ZERO; 3919 } 3920 3921 /* Mark slots affected by this stack write. */ 3922 for (i = 0; i < size; i++) 3923 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 3924 type; 3925 } 3926 return 0; 3927 } 3928 3929 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 3930 * known to contain a variable offset. 3931 * This function checks whether the write is permitted and conservatively 3932 * tracks the effects of the write, considering that each stack slot in the 3933 * dynamic range is potentially written to. 3934 * 3935 * 'off' includes 'regno->off'. 3936 * 'value_regno' can be -1, meaning that an unknown value is being written to 3937 * the stack. 3938 * 3939 * Spilled pointers in range are not marked as written because we don't know 3940 * what's going to be actually written. This means that read propagation for 3941 * future reads cannot be terminated by this write. 3942 * 3943 * For privileged programs, uninitialized stack slots are considered 3944 * initialized by this write (even though we don't know exactly what offsets 3945 * are going to be written to). The idea is that we don't want the verifier to 3946 * reject future reads that access slots written to through variable offsets. 3947 */ 3948 static int check_stack_write_var_off(struct bpf_verifier_env *env, 3949 /* func where register points to */ 3950 struct bpf_func_state *state, 3951 int ptr_regno, int off, int size, 3952 int value_regno, int insn_idx) 3953 { 3954 struct bpf_func_state *cur; /* state of the current function */ 3955 int min_off, max_off; 3956 int i, err; 3957 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 3958 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3959 bool writing_zero = false; 3960 /* set if the fact that we're writing a zero is used to let any 3961 * stack slots remain STACK_ZERO 3962 */ 3963 bool zero_used = false; 3964 3965 cur = env->cur_state->frame[env->cur_state->curframe]; 3966 ptr_reg = &cur->regs[ptr_regno]; 3967 min_off = ptr_reg->smin_value + off; 3968 max_off = ptr_reg->smax_value + off + size; 3969 if (value_regno >= 0) 3970 value_reg = &cur->regs[value_regno]; 3971 if ((value_reg && register_is_null(value_reg)) || 3972 (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) 3973 writing_zero = true; 3974 3975 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 3976 if (err) 3977 return err; 3978 3979 for (i = min_off; i < max_off; i++) { 3980 int spi; 3981 3982 spi = __get_spi(i); 3983 err = destroy_if_dynptr_stack_slot(env, state, spi); 3984 if (err) 3985 return err; 3986 } 3987 3988 /* Variable offset writes destroy any spilled pointers in range. */ 3989 for (i = min_off; i < max_off; i++) { 3990 u8 new_type, *stype; 3991 int slot, spi; 3992 3993 slot = -i - 1; 3994 spi = slot / BPF_REG_SIZE; 3995 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3996 mark_stack_slot_scratched(env, spi); 3997 3998 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 3999 /* Reject the write if range we may write to has not 4000 * been initialized beforehand. If we didn't reject 4001 * here, the ptr status would be erased below (even 4002 * though not all slots are actually overwritten), 4003 * possibly opening the door to leaks. 4004 * 4005 * We do however catch STACK_INVALID case below, and 4006 * only allow reading possibly uninitialized memory 4007 * later for CAP_PERFMON, as the write may not happen to 4008 * that slot. 4009 */ 4010 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 4011 insn_idx, i); 4012 return -EINVAL; 4013 } 4014 4015 /* Erase all spilled pointers. */ 4016 state->stack[spi].spilled_ptr.type = NOT_INIT; 4017 4018 /* Update the slot type. */ 4019 new_type = STACK_MISC; 4020 if (writing_zero && *stype == STACK_ZERO) { 4021 new_type = STACK_ZERO; 4022 zero_used = true; 4023 } 4024 /* If the slot is STACK_INVALID, we check whether it's OK to 4025 * pretend that it will be initialized by this write. The slot 4026 * might not actually be written to, and so if we mark it as 4027 * initialized future reads might leak uninitialized memory. 4028 * For privileged programs, we will accept such reads to slots 4029 * that may or may not be written because, if we're reject 4030 * them, the error would be too confusing. 4031 */ 4032 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 4033 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 4034 insn_idx, i); 4035 return -EINVAL; 4036 } 4037 *stype = new_type; 4038 } 4039 if (zero_used) { 4040 /* backtracking doesn't work for STACK_ZERO yet. */ 4041 err = mark_chain_precision(env, value_regno); 4042 if (err) 4043 return err; 4044 } 4045 return 0; 4046 } 4047 4048 /* When register 'dst_regno' is assigned some values from stack[min_off, 4049 * max_off), we set the register's type according to the types of the 4050 * respective stack slots. If all the stack values are known to be zeros, then 4051 * so is the destination reg. Otherwise, the register is considered to be 4052 * SCALAR. This function does not deal with register filling; the caller must 4053 * ensure that all spilled registers in the stack range have been marked as 4054 * read. 4055 */ 4056 static void mark_reg_stack_read(struct bpf_verifier_env *env, 4057 /* func where src register points to */ 4058 struct bpf_func_state *ptr_state, 4059 int min_off, int max_off, int dst_regno) 4060 { 4061 struct bpf_verifier_state *vstate = env->cur_state; 4062 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4063 int i, slot, spi; 4064 u8 *stype; 4065 int zeros = 0; 4066 4067 for (i = min_off; i < max_off; i++) { 4068 slot = -i - 1; 4069 spi = slot / BPF_REG_SIZE; 4070 stype = ptr_state->stack[spi].slot_type; 4071 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 4072 break; 4073 zeros++; 4074 } 4075 if (zeros == max_off - min_off) { 4076 /* any access_size read into register is zero extended, 4077 * so the whole register == const_zero 4078 */ 4079 __mark_reg_const_zero(&state->regs[dst_regno]); 4080 /* backtracking doesn't support STACK_ZERO yet, 4081 * so mark it precise here, so that later 4082 * backtracking can stop here. 4083 * Backtracking may not need this if this register 4084 * doesn't participate in pointer adjustment. 4085 * Forward propagation of precise flag is not 4086 * necessary either. This mark is only to stop 4087 * backtracking. Any register that contributed 4088 * to const 0 was marked precise before spill. 4089 */ 4090 state->regs[dst_regno].precise = true; 4091 } else { 4092 /* have read misc data from the stack */ 4093 mark_reg_unknown(env, state->regs, dst_regno); 4094 } 4095 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4096 } 4097 4098 /* Read the stack at 'off' and put the results into the register indicated by 4099 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 4100 * spilled reg. 4101 * 4102 * 'dst_regno' can be -1, meaning that the read value is not going to a 4103 * register. 4104 * 4105 * The access is assumed to be within the current stack bounds. 4106 */ 4107 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 4108 /* func where src register points to */ 4109 struct bpf_func_state *reg_state, 4110 int off, int size, int dst_regno) 4111 { 4112 struct bpf_verifier_state *vstate = env->cur_state; 4113 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4114 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 4115 struct bpf_reg_state *reg; 4116 u8 *stype, type; 4117 4118 stype = reg_state->stack[spi].slot_type; 4119 reg = ®_state->stack[spi].spilled_ptr; 4120 4121 if (is_spilled_reg(®_state->stack[spi])) { 4122 u8 spill_size = 1; 4123 4124 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 4125 spill_size++; 4126 4127 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 4128 if (reg->type != SCALAR_VALUE) { 4129 verbose_linfo(env, env->insn_idx, "; "); 4130 verbose(env, "invalid size of register fill\n"); 4131 return -EACCES; 4132 } 4133 4134 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4135 if (dst_regno < 0) 4136 return 0; 4137 4138 if (!(off % BPF_REG_SIZE) && size == spill_size) { 4139 /* The earlier check_reg_arg() has decided the 4140 * subreg_def for this insn. Save it first. 4141 */ 4142 s32 subreg_def = state->regs[dst_regno].subreg_def; 4143 4144 copy_register_state(&state->regs[dst_regno], reg); 4145 state->regs[dst_regno].subreg_def = subreg_def; 4146 } else { 4147 for (i = 0; i < size; i++) { 4148 type = stype[(slot - i) % BPF_REG_SIZE]; 4149 if (type == STACK_SPILL) 4150 continue; 4151 if (type == STACK_MISC) 4152 continue; 4153 if (type == STACK_INVALID && env->allow_uninit_stack) 4154 continue; 4155 verbose(env, "invalid read from stack off %d+%d size %d\n", 4156 off, i, size); 4157 return -EACCES; 4158 } 4159 mark_reg_unknown(env, state->regs, dst_regno); 4160 } 4161 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4162 return 0; 4163 } 4164 4165 if (dst_regno >= 0) { 4166 /* restore register state from stack */ 4167 copy_register_state(&state->regs[dst_regno], reg); 4168 /* mark reg as written since spilled pointer state likely 4169 * has its liveness marks cleared by is_state_visited() 4170 * which resets stack/reg liveness for state transitions 4171 */ 4172 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 4173 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 4174 /* If dst_regno==-1, the caller is asking us whether 4175 * it is acceptable to use this value as a SCALAR_VALUE 4176 * (e.g. for XADD). 4177 * We must not allow unprivileged callers to do that 4178 * with spilled pointers. 4179 */ 4180 verbose(env, "leaking pointer from stack off %d\n", 4181 off); 4182 return -EACCES; 4183 } 4184 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4185 } else { 4186 for (i = 0; i < size; i++) { 4187 type = stype[(slot - i) % BPF_REG_SIZE]; 4188 if (type == STACK_MISC) 4189 continue; 4190 if (type == STACK_ZERO) 4191 continue; 4192 if (type == STACK_INVALID && env->allow_uninit_stack) 4193 continue; 4194 verbose(env, "invalid read from stack off %d+%d size %d\n", 4195 off, i, size); 4196 return -EACCES; 4197 } 4198 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 4199 if (dst_regno >= 0) 4200 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 4201 } 4202 return 0; 4203 } 4204 4205 enum bpf_access_src { 4206 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 4207 ACCESS_HELPER = 2, /* the access is performed by a helper */ 4208 }; 4209 4210 static int check_stack_range_initialized(struct bpf_verifier_env *env, 4211 int regno, int off, int access_size, 4212 bool zero_size_allowed, 4213 enum bpf_access_src type, 4214 struct bpf_call_arg_meta *meta); 4215 4216 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 4217 { 4218 return cur_regs(env) + regno; 4219 } 4220 4221 /* Read the stack at 'ptr_regno + off' and put the result into the register 4222 * 'dst_regno'. 4223 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 4224 * but not its variable offset. 4225 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 4226 * 4227 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 4228 * filling registers (i.e. reads of spilled register cannot be detected when 4229 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 4230 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 4231 * offset; for a fixed offset check_stack_read_fixed_off should be used 4232 * instead. 4233 */ 4234 static int check_stack_read_var_off(struct bpf_verifier_env *env, 4235 int ptr_regno, int off, int size, int dst_regno) 4236 { 4237 /* The state of the source register. */ 4238 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 4239 struct bpf_func_state *ptr_state = func(env, reg); 4240 int err; 4241 int min_off, max_off; 4242 4243 /* Note that we pass a NULL meta, so raw access will not be permitted. 4244 */ 4245 err = check_stack_range_initialized(env, ptr_regno, off, size, 4246 false, ACCESS_DIRECT, NULL); 4247 if (err) 4248 return err; 4249 4250 min_off = reg->smin_value + off; 4251 max_off = reg->smax_value + off; 4252 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 4253 return 0; 4254 } 4255 4256 /* check_stack_read dispatches to check_stack_read_fixed_off or 4257 * check_stack_read_var_off. 4258 * 4259 * The caller must ensure that the offset falls within the allocated stack 4260 * bounds. 4261 * 4262 * 'dst_regno' is a register which will receive the value from the stack. It 4263 * can be -1, meaning that the read value is not going to a register. 4264 */ 4265 static int check_stack_read(struct bpf_verifier_env *env, 4266 int ptr_regno, int off, int size, 4267 int dst_regno) 4268 { 4269 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 4270 struct bpf_func_state *state = func(env, reg); 4271 int err; 4272 /* Some accesses are only permitted with a static offset. */ 4273 bool var_off = !tnum_is_const(reg->var_off); 4274 4275 /* The offset is required to be static when reads don't go to a 4276 * register, in order to not leak pointers (see 4277 * check_stack_read_fixed_off). 4278 */ 4279 if (dst_regno < 0 && var_off) { 4280 char tn_buf[48]; 4281 4282 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4283 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 4284 tn_buf, off, size); 4285 return -EACCES; 4286 } 4287 /* Variable offset is prohibited for unprivileged mode for simplicity 4288 * since it requires corresponding support in Spectre masking for stack 4289 * ALU. See also retrieve_ptr_limit(). The check in 4290 * check_stack_access_for_ptr_arithmetic() called by 4291 * adjust_ptr_min_max_vals() prevents users from creating stack pointers 4292 * with variable offsets, therefore no check is required here. Further, 4293 * just checking it here would be insufficient as speculative stack 4294 * writes could still lead to unsafe speculative behaviour. 4295 */ 4296 if (!var_off) { 4297 off += reg->var_off.value; 4298 err = check_stack_read_fixed_off(env, state, off, size, 4299 dst_regno); 4300 } else { 4301 /* Variable offset stack reads need more conservative handling 4302 * than fixed offset ones. Note that dst_regno >= 0 on this 4303 * branch. 4304 */ 4305 err = check_stack_read_var_off(env, ptr_regno, off, size, 4306 dst_regno); 4307 } 4308 return err; 4309 } 4310 4311 4312 /* check_stack_write dispatches to check_stack_write_fixed_off or 4313 * check_stack_write_var_off. 4314 * 4315 * 'ptr_regno' is the register used as a pointer into the stack. 4316 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 4317 * 'value_regno' is the register whose value we're writing to the stack. It can 4318 * be -1, meaning that we're not writing from a register. 4319 * 4320 * The caller must ensure that the offset falls within the maximum stack size. 4321 */ 4322 static int check_stack_write(struct bpf_verifier_env *env, 4323 int ptr_regno, int off, int size, 4324 int value_regno, int insn_idx) 4325 { 4326 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 4327 struct bpf_func_state *state = func(env, reg); 4328 int err; 4329 4330 if (tnum_is_const(reg->var_off)) { 4331 off += reg->var_off.value; 4332 err = check_stack_write_fixed_off(env, state, off, size, 4333 value_regno, insn_idx); 4334 } else { 4335 /* Variable offset stack reads need more conservative handling 4336 * than fixed offset ones. 4337 */ 4338 err = check_stack_write_var_off(env, state, 4339 ptr_regno, off, size, 4340 value_regno, insn_idx); 4341 } 4342 return err; 4343 } 4344 4345 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 4346 int off, int size, enum bpf_access_type type) 4347 { 4348 struct bpf_reg_state *regs = cur_regs(env); 4349 struct bpf_map *map = regs[regno].map_ptr; 4350 u32 cap = bpf_map_flags_to_cap(map); 4351 4352 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 4353 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 4354 map->value_size, off, size); 4355 return -EACCES; 4356 } 4357 4358 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 4359 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 4360 map->value_size, off, size); 4361 return -EACCES; 4362 } 4363 4364 return 0; 4365 } 4366 4367 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 4368 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 4369 int off, int size, u32 mem_size, 4370 bool zero_size_allowed) 4371 { 4372 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 4373 struct bpf_reg_state *reg; 4374 4375 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 4376 return 0; 4377 4378 reg = &cur_regs(env)[regno]; 4379 switch (reg->type) { 4380 case PTR_TO_MAP_KEY: 4381 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 4382 mem_size, off, size); 4383 break; 4384 case PTR_TO_MAP_VALUE: 4385 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 4386 mem_size, off, size); 4387 break; 4388 case PTR_TO_PACKET: 4389 case PTR_TO_PACKET_META: 4390 case PTR_TO_PACKET_END: 4391 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 4392 off, size, regno, reg->id, off, mem_size); 4393 break; 4394 case PTR_TO_MEM: 4395 default: 4396 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 4397 mem_size, off, size); 4398 } 4399 4400 return -EACCES; 4401 } 4402 4403 /* check read/write into a memory region with possible variable offset */ 4404 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 4405 int off, int size, u32 mem_size, 4406 bool zero_size_allowed) 4407 { 4408 struct bpf_verifier_state *vstate = env->cur_state; 4409 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4410 struct bpf_reg_state *reg = &state->regs[regno]; 4411 int err; 4412 4413 /* We may have adjusted the register pointing to memory region, so we 4414 * need to try adding each of min_value and max_value to off 4415 * to make sure our theoretical access will be safe. 4416 * 4417 * The minimum value is only important with signed 4418 * comparisons where we can't assume the floor of a 4419 * value is 0. If we are using signed variables for our 4420 * index'es we need to make sure that whatever we use 4421 * will have a set floor within our range. 4422 */ 4423 if (reg->smin_value < 0 && 4424 (reg->smin_value == S64_MIN || 4425 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 4426 reg->smin_value + off < 0)) { 4427 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4428 regno); 4429 return -EACCES; 4430 } 4431 err = __check_mem_access(env, regno, reg->smin_value + off, size, 4432 mem_size, zero_size_allowed); 4433 if (err) { 4434 verbose(env, "R%d min value is outside of the allowed memory range\n", 4435 regno); 4436 return err; 4437 } 4438 4439 /* If we haven't set a max value then we need to bail since we can't be 4440 * sure we won't do bad things. 4441 * If reg->umax_value + off could overflow, treat that as unbounded too. 4442 */ 4443 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 4444 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 4445 regno); 4446 return -EACCES; 4447 } 4448 err = __check_mem_access(env, regno, reg->umax_value + off, size, 4449 mem_size, zero_size_allowed); 4450 if (err) { 4451 verbose(env, "R%d max value is outside of the allowed memory range\n", 4452 regno); 4453 return err; 4454 } 4455 4456 return 0; 4457 } 4458 4459 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 4460 const struct bpf_reg_state *reg, int regno, 4461 bool fixed_off_ok) 4462 { 4463 /* Access to this pointer-typed register or passing it to a helper 4464 * is only allowed in its original, unmodified form. 4465 */ 4466 4467 if (reg->off < 0) { 4468 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 4469 reg_type_str(env, reg->type), regno, reg->off); 4470 return -EACCES; 4471 } 4472 4473 if (!fixed_off_ok && reg->off) { 4474 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 4475 reg_type_str(env, reg->type), regno, reg->off); 4476 return -EACCES; 4477 } 4478 4479 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4480 char tn_buf[48]; 4481 4482 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4483 verbose(env, "variable %s access var_off=%s disallowed\n", 4484 reg_type_str(env, reg->type), tn_buf); 4485 return -EACCES; 4486 } 4487 4488 return 0; 4489 } 4490 4491 int check_ptr_off_reg(struct bpf_verifier_env *env, 4492 const struct bpf_reg_state *reg, int regno) 4493 { 4494 return __check_ptr_off_reg(env, reg, regno, false); 4495 } 4496 4497 static int map_kptr_match_type(struct bpf_verifier_env *env, 4498 struct btf_field *kptr_field, 4499 struct bpf_reg_state *reg, u32 regno) 4500 { 4501 const char *targ_name = btf_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 4502 int perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU; 4503 const char *reg_name = ""; 4504 4505 /* Only unreferenced case accepts untrusted pointers */ 4506 if (kptr_field->type == BPF_KPTR_UNREF) 4507 perm_flags |= PTR_UNTRUSTED; 4508 4509 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 4510 goto bad_type; 4511 4512 if (!btf_is_kernel(reg->btf)) { 4513 verbose(env, "R%d must point to kernel BTF\n", regno); 4514 return -EINVAL; 4515 } 4516 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 4517 reg_name = btf_type_name(reg->btf, reg->btf_id); 4518 4519 /* For ref_ptr case, release function check should ensure we get one 4520 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 4521 * normal store of unreferenced kptr, we must ensure var_off is zero. 4522 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 4523 * reg->off and reg->ref_obj_id are not needed here. 4524 */ 4525 if (__check_ptr_off_reg(env, reg, regno, true)) 4526 return -EACCES; 4527 4528 /* A full type match is needed, as BTF can be vmlinux or module BTF, and 4529 * we also need to take into account the reg->off. 4530 * 4531 * We want to support cases like: 4532 * 4533 * struct foo { 4534 * struct bar br; 4535 * struct baz bz; 4536 * }; 4537 * 4538 * struct foo *v; 4539 * v = func(); // PTR_TO_BTF_ID 4540 * val->foo = v; // reg->off is zero, btf and btf_id match type 4541 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 4542 * // first member type of struct after comparison fails 4543 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 4544 * // to match type 4545 * 4546 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 4547 * is zero. We must also ensure that btf_struct_ids_match does not walk 4548 * the struct to match type against first member of struct, i.e. reject 4549 * second case from above. Hence, when type is BPF_KPTR_REF, we set 4550 * strict mode to true for type match. 4551 */ 4552 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 4553 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 4554 kptr_field->type == BPF_KPTR_REF)) 4555 goto bad_type; 4556 return 0; 4557 bad_type: 4558 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 4559 reg_type_str(env, reg->type), reg_name); 4560 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 4561 if (kptr_field->type == BPF_KPTR_UNREF) 4562 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 4563 targ_name); 4564 else 4565 verbose(env, "\n"); 4566 return -EINVAL; 4567 } 4568 4569 /* The non-sleepable programs and sleepable programs with explicit bpf_rcu_read_lock() 4570 * can dereference RCU protected pointers and result is PTR_TRUSTED. 4571 */ 4572 static bool in_rcu_cs(struct bpf_verifier_env *env) 4573 { 4574 return env->cur_state->active_rcu_lock || !env->prog->aux->sleepable; 4575 } 4576 4577 /* Once GCC supports btf_type_tag the following mechanism will be replaced with tag check */ 4578 BTF_SET_START(rcu_protected_types) 4579 BTF_ID(struct, prog_test_ref_kfunc) 4580 BTF_ID(struct, cgroup) 4581 BTF_ID(struct, bpf_cpumask) 4582 BTF_ID(struct, task_struct) 4583 BTF_SET_END(rcu_protected_types) 4584 4585 static bool rcu_protected_object(const struct btf *btf, u32 btf_id) 4586 { 4587 if (!btf_is_kernel(btf)) 4588 return false; 4589 return btf_id_set_contains(&rcu_protected_types, btf_id); 4590 } 4591 4592 static bool rcu_safe_kptr(const struct btf_field *field) 4593 { 4594 const struct btf_field_kptr *kptr = &field->kptr; 4595 4596 return field->type == BPF_KPTR_REF && rcu_protected_object(kptr->btf, kptr->btf_id); 4597 } 4598 4599 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 4600 int value_regno, int insn_idx, 4601 struct btf_field *kptr_field) 4602 { 4603 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 4604 int class = BPF_CLASS(insn->code); 4605 struct bpf_reg_state *val_reg; 4606 4607 /* Things we already checked for in check_map_access and caller: 4608 * - Reject cases where variable offset may touch kptr 4609 * - size of access (must be BPF_DW) 4610 * - tnum_is_const(reg->var_off) 4611 * - kptr_field->offset == off + reg->var_off.value 4612 */ 4613 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 4614 if (BPF_MODE(insn->code) != BPF_MEM) { 4615 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 4616 return -EACCES; 4617 } 4618 4619 /* We only allow loading referenced kptr, since it will be marked as 4620 * untrusted, similar to unreferenced kptr. 4621 */ 4622 if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 4623 verbose(env, "store to referenced kptr disallowed\n"); 4624 return -EACCES; 4625 } 4626 4627 if (class == BPF_LDX) { 4628 val_reg = reg_state(env, value_regno); 4629 /* We can simply mark the value_regno receiving the pointer 4630 * value from map as PTR_TO_BTF_ID, with the correct type. 4631 */ 4632 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 4633 kptr_field->kptr.btf_id, 4634 rcu_safe_kptr(kptr_field) && in_rcu_cs(env) ? 4635 PTR_MAYBE_NULL | MEM_RCU : 4636 PTR_MAYBE_NULL | PTR_UNTRUSTED); 4637 /* For mark_ptr_or_null_reg */ 4638 val_reg->id = ++env->id_gen; 4639 } else if (class == BPF_STX) { 4640 val_reg = reg_state(env, value_regno); 4641 if (!register_is_null(val_reg) && 4642 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 4643 return -EACCES; 4644 } else if (class == BPF_ST) { 4645 if (insn->imm) { 4646 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 4647 kptr_field->offset); 4648 return -EACCES; 4649 } 4650 } else { 4651 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 4652 return -EACCES; 4653 } 4654 return 0; 4655 } 4656 4657 /* check read/write into a map element with possible variable offset */ 4658 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 4659 int off, int size, bool zero_size_allowed, 4660 enum bpf_access_src src) 4661 { 4662 struct bpf_verifier_state *vstate = env->cur_state; 4663 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4664 struct bpf_reg_state *reg = &state->regs[regno]; 4665 struct bpf_map *map = reg->map_ptr; 4666 struct btf_record *rec; 4667 int err, i; 4668 4669 err = check_mem_region_access(env, regno, off, size, map->value_size, 4670 zero_size_allowed); 4671 if (err) 4672 return err; 4673 4674 if (IS_ERR_OR_NULL(map->record)) 4675 return 0; 4676 rec = map->record; 4677 for (i = 0; i < rec->cnt; i++) { 4678 struct btf_field *field = &rec->fields[i]; 4679 u32 p = field->offset; 4680 4681 /* If any part of a field can be touched by load/store, reject 4682 * this program. To check that [x1, x2) overlaps with [y1, y2), 4683 * it is sufficient to check x1 < y2 && y1 < x2. 4684 */ 4685 if (reg->smin_value + off < p + btf_field_type_size(field->type) && 4686 p < reg->umax_value + off + size) { 4687 switch (field->type) { 4688 case BPF_KPTR_UNREF: 4689 case BPF_KPTR_REF: 4690 if (src != ACCESS_DIRECT) { 4691 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 4692 return -EACCES; 4693 } 4694 if (!tnum_is_const(reg->var_off)) { 4695 verbose(env, "kptr access cannot have variable offset\n"); 4696 return -EACCES; 4697 } 4698 if (p != off + reg->var_off.value) { 4699 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 4700 p, off + reg->var_off.value); 4701 return -EACCES; 4702 } 4703 if (size != bpf_size_to_bytes(BPF_DW)) { 4704 verbose(env, "kptr access size must be BPF_DW\n"); 4705 return -EACCES; 4706 } 4707 break; 4708 default: 4709 verbose(env, "%s cannot be accessed directly by load/store\n", 4710 btf_field_type_name(field->type)); 4711 return -EACCES; 4712 } 4713 } 4714 } 4715 return 0; 4716 } 4717 4718 #define MAX_PACKET_OFF 0xffff 4719 4720 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 4721 const struct bpf_call_arg_meta *meta, 4722 enum bpf_access_type t) 4723 { 4724 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 4725 4726 switch (prog_type) { 4727 /* Program types only with direct read access go here! */ 4728 case BPF_PROG_TYPE_LWT_IN: 4729 case BPF_PROG_TYPE_LWT_OUT: 4730 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 4731 case BPF_PROG_TYPE_SK_REUSEPORT: 4732 case BPF_PROG_TYPE_FLOW_DISSECTOR: 4733 case BPF_PROG_TYPE_CGROUP_SKB: 4734 if (t == BPF_WRITE) 4735 return false; 4736 fallthrough; 4737 4738 /* Program types with direct read + write access go here! */ 4739 case BPF_PROG_TYPE_SCHED_CLS: 4740 case BPF_PROG_TYPE_SCHED_ACT: 4741 case BPF_PROG_TYPE_XDP: 4742 case BPF_PROG_TYPE_LWT_XMIT: 4743 case BPF_PROG_TYPE_SK_SKB: 4744 case BPF_PROG_TYPE_SK_MSG: 4745 if (meta) 4746 return meta->pkt_access; 4747 4748 env->seen_direct_write = true; 4749 return true; 4750 4751 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4752 if (t == BPF_WRITE) 4753 env->seen_direct_write = true; 4754 4755 return true; 4756 4757 default: 4758 return false; 4759 } 4760 } 4761 4762 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 4763 int size, bool zero_size_allowed) 4764 { 4765 struct bpf_reg_state *regs = cur_regs(env); 4766 struct bpf_reg_state *reg = ®s[regno]; 4767 int err; 4768 4769 /* We may have added a variable offset to the packet pointer; but any 4770 * reg->range we have comes after that. We are only checking the fixed 4771 * offset. 4772 */ 4773 4774 /* We don't allow negative numbers, because we aren't tracking enough 4775 * detail to prove they're safe. 4776 */ 4777 if (reg->smin_value < 0) { 4778 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4779 regno); 4780 return -EACCES; 4781 } 4782 4783 err = reg->range < 0 ? -EINVAL : 4784 __check_mem_access(env, regno, off, size, reg->range, 4785 zero_size_allowed); 4786 if (err) { 4787 verbose(env, "R%d offset is outside of the packet\n", regno); 4788 return err; 4789 } 4790 4791 /* __check_mem_access has made sure "off + size - 1" is within u16. 4792 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 4793 * otherwise find_good_pkt_pointers would have refused to set range info 4794 * that __check_mem_access would have rejected this pkt access. 4795 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 4796 */ 4797 env->prog->aux->max_pkt_offset = 4798 max_t(u32, env->prog->aux->max_pkt_offset, 4799 off + reg->umax_value + size - 1); 4800 4801 return err; 4802 } 4803 4804 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 4805 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 4806 enum bpf_access_type t, enum bpf_reg_type *reg_type, 4807 struct btf **btf, u32 *btf_id) 4808 { 4809 struct bpf_insn_access_aux info = { 4810 .reg_type = *reg_type, 4811 .log = &env->log, 4812 }; 4813 4814 if (env->ops->is_valid_access && 4815 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 4816 /* A non zero info.ctx_field_size indicates that this field is a 4817 * candidate for later verifier transformation to load the whole 4818 * field and then apply a mask when accessed with a narrower 4819 * access than actual ctx access size. A zero info.ctx_field_size 4820 * will only allow for whole field access and rejects any other 4821 * type of narrower access. 4822 */ 4823 *reg_type = info.reg_type; 4824 4825 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 4826 *btf = info.btf; 4827 *btf_id = info.btf_id; 4828 } else { 4829 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 4830 } 4831 /* remember the offset of last byte accessed in ctx */ 4832 if (env->prog->aux->max_ctx_offset < off + size) 4833 env->prog->aux->max_ctx_offset = off + size; 4834 return 0; 4835 } 4836 4837 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 4838 return -EACCES; 4839 } 4840 4841 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 4842 int size) 4843 { 4844 if (size < 0 || off < 0 || 4845 (u64)off + size > sizeof(struct bpf_flow_keys)) { 4846 verbose(env, "invalid access to flow keys off=%d size=%d\n", 4847 off, size); 4848 return -EACCES; 4849 } 4850 return 0; 4851 } 4852 4853 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 4854 u32 regno, int off, int size, 4855 enum bpf_access_type t) 4856 { 4857 struct bpf_reg_state *regs = cur_regs(env); 4858 struct bpf_reg_state *reg = ®s[regno]; 4859 struct bpf_insn_access_aux info = {}; 4860 bool valid; 4861 4862 if (reg->smin_value < 0) { 4863 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4864 regno); 4865 return -EACCES; 4866 } 4867 4868 switch (reg->type) { 4869 case PTR_TO_SOCK_COMMON: 4870 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 4871 break; 4872 case PTR_TO_SOCKET: 4873 valid = bpf_sock_is_valid_access(off, size, t, &info); 4874 break; 4875 case PTR_TO_TCP_SOCK: 4876 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 4877 break; 4878 case PTR_TO_XDP_SOCK: 4879 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 4880 break; 4881 default: 4882 valid = false; 4883 } 4884 4885 4886 if (valid) { 4887 env->insn_aux_data[insn_idx].ctx_field_size = 4888 info.ctx_field_size; 4889 return 0; 4890 } 4891 4892 verbose(env, "R%d invalid %s access off=%d size=%d\n", 4893 regno, reg_type_str(env, reg->type), off, size); 4894 4895 return -EACCES; 4896 } 4897 4898 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 4899 { 4900 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 4901 } 4902 4903 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 4904 { 4905 const struct bpf_reg_state *reg = reg_state(env, regno); 4906 4907 return reg->type == PTR_TO_CTX; 4908 } 4909 4910 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 4911 { 4912 const struct bpf_reg_state *reg = reg_state(env, regno); 4913 4914 return type_is_sk_pointer(reg->type); 4915 } 4916 4917 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 4918 { 4919 const struct bpf_reg_state *reg = reg_state(env, regno); 4920 4921 return type_is_pkt_pointer(reg->type); 4922 } 4923 4924 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 4925 { 4926 const struct bpf_reg_state *reg = reg_state(env, regno); 4927 4928 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 4929 return reg->type == PTR_TO_FLOW_KEYS; 4930 } 4931 4932 static bool is_trusted_reg(const struct bpf_reg_state *reg) 4933 { 4934 /* A referenced register is always trusted. */ 4935 if (reg->ref_obj_id) 4936 return true; 4937 4938 /* If a register is not referenced, it is trusted if it has the 4939 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 4940 * other type modifiers may be safe, but we elect to take an opt-in 4941 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 4942 * not. 4943 * 4944 * Eventually, we should make PTR_TRUSTED the single source of truth 4945 * for whether a register is trusted. 4946 */ 4947 return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && 4948 !bpf_type_has_unsafe_modifiers(reg->type); 4949 } 4950 4951 static bool is_rcu_reg(const struct bpf_reg_state *reg) 4952 { 4953 return reg->type & MEM_RCU; 4954 } 4955 4956 static void clear_trusted_flags(enum bpf_type_flag *flag) 4957 { 4958 *flag &= ~(BPF_REG_TRUSTED_MODIFIERS | MEM_RCU); 4959 } 4960 4961 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 4962 const struct bpf_reg_state *reg, 4963 int off, int size, bool strict) 4964 { 4965 struct tnum reg_off; 4966 int ip_align; 4967 4968 /* Byte size accesses are always allowed. */ 4969 if (!strict || size == 1) 4970 return 0; 4971 4972 /* For platforms that do not have a Kconfig enabling 4973 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 4974 * NET_IP_ALIGN is universally set to '2'. And on platforms 4975 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 4976 * to this code only in strict mode where we want to emulate 4977 * the NET_IP_ALIGN==2 checking. Therefore use an 4978 * unconditional IP align value of '2'. 4979 */ 4980 ip_align = 2; 4981 4982 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 4983 if (!tnum_is_aligned(reg_off, size)) { 4984 char tn_buf[48]; 4985 4986 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4987 verbose(env, 4988 "misaligned packet access off %d+%s+%d+%d size %d\n", 4989 ip_align, tn_buf, reg->off, off, size); 4990 return -EACCES; 4991 } 4992 4993 return 0; 4994 } 4995 4996 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 4997 const struct bpf_reg_state *reg, 4998 const char *pointer_desc, 4999 int off, int size, bool strict) 5000 { 5001 struct tnum reg_off; 5002 5003 /* Byte size accesses are always allowed. */ 5004 if (!strict || size == 1) 5005 return 0; 5006 5007 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 5008 if (!tnum_is_aligned(reg_off, size)) { 5009 char tn_buf[48]; 5010 5011 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5012 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 5013 pointer_desc, tn_buf, reg->off, off, size); 5014 return -EACCES; 5015 } 5016 5017 return 0; 5018 } 5019 5020 static int check_ptr_alignment(struct bpf_verifier_env *env, 5021 const struct bpf_reg_state *reg, int off, 5022 int size, bool strict_alignment_once) 5023 { 5024 bool strict = env->strict_alignment || strict_alignment_once; 5025 const char *pointer_desc = ""; 5026 5027 switch (reg->type) { 5028 case PTR_TO_PACKET: 5029 case PTR_TO_PACKET_META: 5030 /* Special case, because of NET_IP_ALIGN. Given metadata sits 5031 * right in front, treat it the very same way. 5032 */ 5033 return check_pkt_ptr_alignment(env, reg, off, size, strict); 5034 case PTR_TO_FLOW_KEYS: 5035 pointer_desc = "flow keys "; 5036 break; 5037 case PTR_TO_MAP_KEY: 5038 pointer_desc = "key "; 5039 break; 5040 case PTR_TO_MAP_VALUE: 5041 pointer_desc = "value "; 5042 break; 5043 case PTR_TO_CTX: 5044 pointer_desc = "context "; 5045 break; 5046 case PTR_TO_STACK: 5047 pointer_desc = "stack "; 5048 /* The stack spill tracking logic in check_stack_write_fixed_off() 5049 * and check_stack_read_fixed_off() relies on stack accesses being 5050 * aligned. 5051 */ 5052 strict = true; 5053 break; 5054 case PTR_TO_SOCKET: 5055 pointer_desc = "sock "; 5056 break; 5057 case PTR_TO_SOCK_COMMON: 5058 pointer_desc = "sock_common "; 5059 break; 5060 case PTR_TO_TCP_SOCK: 5061 pointer_desc = "tcp_sock "; 5062 break; 5063 case PTR_TO_XDP_SOCK: 5064 pointer_desc = "xdp_sock "; 5065 break; 5066 default: 5067 break; 5068 } 5069 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 5070 strict); 5071 } 5072 5073 static int update_stack_depth(struct bpf_verifier_env *env, 5074 const struct bpf_func_state *func, 5075 int off) 5076 { 5077 u16 stack = env->subprog_info[func->subprogno].stack_depth; 5078 5079 if (stack >= -off) 5080 return 0; 5081 5082 /* update known max for given subprogram */ 5083 env->subprog_info[func->subprogno].stack_depth = -off; 5084 return 0; 5085 } 5086 5087 /* starting from main bpf function walk all instructions of the function 5088 * and recursively walk all callees that given function can call. 5089 * Ignore jump and exit insns. 5090 * Since recursion is prevented by check_cfg() this algorithm 5091 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 5092 */ 5093 static int check_max_stack_depth(struct bpf_verifier_env *env) 5094 { 5095 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 5096 struct bpf_subprog_info *subprog = env->subprog_info; 5097 struct bpf_insn *insn = env->prog->insnsi; 5098 bool tail_call_reachable = false; 5099 int ret_insn[MAX_CALL_FRAMES]; 5100 int ret_prog[MAX_CALL_FRAMES]; 5101 int j; 5102 5103 process_func: 5104 /* protect against potential stack overflow that might happen when 5105 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 5106 * depth for such case down to 256 so that the worst case scenario 5107 * would result in 8k stack size (32 which is tailcall limit * 256 = 5108 * 8k). 5109 * 5110 * To get the idea what might happen, see an example: 5111 * func1 -> sub rsp, 128 5112 * subfunc1 -> sub rsp, 256 5113 * tailcall1 -> add rsp, 256 5114 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 5115 * subfunc2 -> sub rsp, 64 5116 * subfunc22 -> sub rsp, 128 5117 * tailcall2 -> add rsp, 128 5118 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 5119 * 5120 * tailcall will unwind the current stack frame but it will not get rid 5121 * of caller's stack as shown on the example above. 5122 */ 5123 if (idx && subprog[idx].has_tail_call && depth >= 256) { 5124 verbose(env, 5125 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 5126 depth); 5127 return -EACCES; 5128 } 5129 /* round up to 32-bytes, since this is granularity 5130 * of interpreter stack size 5131 */ 5132 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 5133 if (depth > MAX_BPF_STACK) { 5134 verbose(env, "combined stack size of %d calls is %d. Too large\n", 5135 frame + 1, depth); 5136 return -EACCES; 5137 } 5138 continue_func: 5139 subprog_end = subprog[idx + 1].start; 5140 for (; i < subprog_end; i++) { 5141 int next_insn; 5142 5143 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 5144 continue; 5145 /* remember insn and function to return to */ 5146 ret_insn[frame] = i + 1; 5147 ret_prog[frame] = idx; 5148 5149 /* find the callee */ 5150 next_insn = i + insn[i].imm + 1; 5151 idx = find_subprog(env, next_insn); 5152 if (idx < 0) { 5153 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 5154 next_insn); 5155 return -EFAULT; 5156 } 5157 if (subprog[idx].is_async_cb) { 5158 if (subprog[idx].has_tail_call) { 5159 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 5160 return -EFAULT; 5161 } 5162 /* async callbacks don't increase bpf prog stack size */ 5163 continue; 5164 } 5165 i = next_insn; 5166 5167 if (subprog[idx].has_tail_call) 5168 tail_call_reachable = true; 5169 5170 frame++; 5171 if (frame >= MAX_CALL_FRAMES) { 5172 verbose(env, "the call stack of %d frames is too deep !\n", 5173 frame); 5174 return -E2BIG; 5175 } 5176 goto process_func; 5177 } 5178 /* if tail call got detected across bpf2bpf calls then mark each of the 5179 * currently present subprog frames as tail call reachable subprogs; 5180 * this info will be utilized by JIT so that we will be preserving the 5181 * tail call counter throughout bpf2bpf calls combined with tailcalls 5182 */ 5183 if (tail_call_reachable) 5184 for (j = 0; j < frame; j++) 5185 subprog[ret_prog[j]].tail_call_reachable = true; 5186 if (subprog[0].tail_call_reachable) 5187 env->prog->aux->tail_call_reachable = true; 5188 5189 /* end of for() loop means the last insn of the 'subprog' 5190 * was reached. Doesn't matter whether it was JA or EXIT 5191 */ 5192 if (frame == 0) 5193 return 0; 5194 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 5195 frame--; 5196 i = ret_insn[frame]; 5197 idx = ret_prog[frame]; 5198 goto continue_func; 5199 } 5200 5201 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 5202 static int get_callee_stack_depth(struct bpf_verifier_env *env, 5203 const struct bpf_insn *insn, int idx) 5204 { 5205 int start = idx + insn->imm + 1, subprog; 5206 5207 subprog = find_subprog(env, start); 5208 if (subprog < 0) { 5209 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 5210 start); 5211 return -EFAULT; 5212 } 5213 return env->subprog_info[subprog].stack_depth; 5214 } 5215 #endif 5216 5217 static int __check_buffer_access(struct bpf_verifier_env *env, 5218 const char *buf_info, 5219 const struct bpf_reg_state *reg, 5220 int regno, int off, int size) 5221 { 5222 if (off < 0) { 5223 verbose(env, 5224 "R%d invalid %s buffer access: off=%d, size=%d\n", 5225 regno, buf_info, off, size); 5226 return -EACCES; 5227 } 5228 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 5229 char tn_buf[48]; 5230 5231 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5232 verbose(env, 5233 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 5234 regno, off, tn_buf); 5235 return -EACCES; 5236 } 5237 5238 return 0; 5239 } 5240 5241 static int check_tp_buffer_access(struct bpf_verifier_env *env, 5242 const struct bpf_reg_state *reg, 5243 int regno, int off, int size) 5244 { 5245 int err; 5246 5247 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 5248 if (err) 5249 return err; 5250 5251 if (off + size > env->prog->aux->max_tp_access) 5252 env->prog->aux->max_tp_access = off + size; 5253 5254 return 0; 5255 } 5256 5257 static int check_buffer_access(struct bpf_verifier_env *env, 5258 const struct bpf_reg_state *reg, 5259 int regno, int off, int size, 5260 bool zero_size_allowed, 5261 u32 *max_access) 5262 { 5263 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 5264 int err; 5265 5266 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 5267 if (err) 5268 return err; 5269 5270 if (off + size > *max_access) 5271 *max_access = off + size; 5272 5273 return 0; 5274 } 5275 5276 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 5277 static void zext_32_to_64(struct bpf_reg_state *reg) 5278 { 5279 reg->var_off = tnum_subreg(reg->var_off); 5280 __reg_assign_32_into_64(reg); 5281 } 5282 5283 /* truncate register to smaller size (in bytes) 5284 * must be called with size < BPF_REG_SIZE 5285 */ 5286 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 5287 { 5288 u64 mask; 5289 5290 /* clear high bits in bit representation */ 5291 reg->var_off = tnum_cast(reg->var_off, size); 5292 5293 /* fix arithmetic bounds */ 5294 mask = ((u64)1 << (size * 8)) - 1; 5295 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 5296 reg->umin_value &= mask; 5297 reg->umax_value &= mask; 5298 } else { 5299 reg->umin_value = 0; 5300 reg->umax_value = mask; 5301 } 5302 reg->smin_value = reg->umin_value; 5303 reg->smax_value = reg->umax_value; 5304 5305 /* If size is smaller than 32bit register the 32bit register 5306 * values are also truncated so we push 64-bit bounds into 5307 * 32-bit bounds. Above were truncated < 32-bits already. 5308 */ 5309 if (size >= 4) 5310 return; 5311 __reg_combine_64_into_32(reg); 5312 } 5313 5314 static bool bpf_map_is_rdonly(const struct bpf_map *map) 5315 { 5316 /* A map is considered read-only if the following condition are true: 5317 * 5318 * 1) BPF program side cannot change any of the map content. The 5319 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 5320 * and was set at map creation time. 5321 * 2) The map value(s) have been initialized from user space by a 5322 * loader and then "frozen", such that no new map update/delete 5323 * operations from syscall side are possible for the rest of 5324 * the map's lifetime from that point onwards. 5325 * 3) Any parallel/pending map update/delete operations from syscall 5326 * side have been completed. Only after that point, it's safe to 5327 * assume that map value(s) are immutable. 5328 */ 5329 return (map->map_flags & BPF_F_RDONLY_PROG) && 5330 READ_ONCE(map->frozen) && 5331 !bpf_map_write_active(map); 5332 } 5333 5334 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 5335 { 5336 void *ptr; 5337 u64 addr; 5338 int err; 5339 5340 err = map->ops->map_direct_value_addr(map, &addr, off); 5341 if (err) 5342 return err; 5343 ptr = (void *)(long)addr + off; 5344 5345 switch (size) { 5346 case sizeof(u8): 5347 *val = (u64)*(u8 *)ptr; 5348 break; 5349 case sizeof(u16): 5350 *val = (u64)*(u16 *)ptr; 5351 break; 5352 case sizeof(u32): 5353 *val = (u64)*(u32 *)ptr; 5354 break; 5355 case sizeof(u64): 5356 *val = *(u64 *)ptr; 5357 break; 5358 default: 5359 return -EINVAL; 5360 } 5361 return 0; 5362 } 5363 5364 #define BTF_TYPE_SAFE_RCU(__type) __PASTE(__type, __safe_rcu) 5365 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type) __PASTE(__type, __safe_rcu_or_null) 5366 #define BTF_TYPE_SAFE_TRUSTED(__type) __PASTE(__type, __safe_trusted) 5367 5368 /* 5369 * Allow list few fields as RCU trusted or full trusted. 5370 * This logic doesn't allow mix tagging and will be removed once GCC supports 5371 * btf_type_tag. 5372 */ 5373 5374 /* RCU trusted: these fields are trusted in RCU CS and never NULL */ 5375 BTF_TYPE_SAFE_RCU(struct task_struct) { 5376 const cpumask_t *cpus_ptr; 5377 struct css_set __rcu *cgroups; 5378 struct task_struct __rcu *real_parent; 5379 struct task_struct *group_leader; 5380 }; 5381 5382 BTF_TYPE_SAFE_RCU(struct cgroup) { 5383 /* cgrp->kn is always accessible as documented in kernel/cgroup/cgroup.c */ 5384 struct kernfs_node *kn; 5385 }; 5386 5387 BTF_TYPE_SAFE_RCU(struct css_set) { 5388 struct cgroup *dfl_cgrp; 5389 }; 5390 5391 /* RCU trusted: these fields are trusted in RCU CS and can be NULL */ 5392 BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct) { 5393 struct file __rcu *exe_file; 5394 }; 5395 5396 /* skb->sk, req->sk are not RCU protected, but we mark them as such 5397 * because bpf prog accessible sockets are SOCK_RCU_FREE. 5398 */ 5399 BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff) { 5400 struct sock *sk; 5401 }; 5402 5403 BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock) { 5404 struct sock *sk; 5405 }; 5406 5407 /* full trusted: these fields are trusted even outside of RCU CS and never NULL */ 5408 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta) { 5409 struct seq_file *seq; 5410 }; 5411 5412 BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task) { 5413 struct bpf_iter_meta *meta; 5414 struct task_struct *task; 5415 }; 5416 5417 BTF_TYPE_SAFE_TRUSTED(struct linux_binprm) { 5418 struct file *file; 5419 }; 5420 5421 BTF_TYPE_SAFE_TRUSTED(struct file) { 5422 struct inode *f_inode; 5423 }; 5424 5425 BTF_TYPE_SAFE_TRUSTED(struct dentry) { 5426 /* no negative dentry-s in places where bpf can see it */ 5427 struct inode *d_inode; 5428 }; 5429 5430 BTF_TYPE_SAFE_TRUSTED(struct socket) { 5431 struct sock *sk; 5432 }; 5433 5434 static bool type_is_rcu(struct bpf_verifier_env *env, 5435 struct bpf_reg_state *reg, 5436 const char *field_name, u32 btf_id) 5437 { 5438 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct task_struct)); 5439 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct cgroup)); 5440 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU(struct css_set)); 5441 5442 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); 5443 } 5444 5445 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, 5446 struct bpf_reg_state *reg, 5447 const char *field_name, u32 btf_id) 5448 { 5449 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct mm_struct)); 5450 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct sk_buff)); 5451 BTF_TYPE_EMIT(BTF_TYPE_SAFE_RCU_OR_NULL(struct request_sock)); 5452 5453 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); 5454 } 5455 5456 static bool type_is_trusted(struct bpf_verifier_env *env, 5457 struct bpf_reg_state *reg, 5458 const char *field_name, u32 btf_id) 5459 { 5460 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter_meta)); 5461 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); 5462 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); 5463 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); 5464 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct dentry)); 5465 BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct socket)); 5466 5467 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); 5468 } 5469 5470 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 5471 struct bpf_reg_state *regs, 5472 int regno, int off, int size, 5473 enum bpf_access_type atype, 5474 int value_regno) 5475 { 5476 struct bpf_reg_state *reg = regs + regno; 5477 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 5478 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 5479 const char *field_name = NULL; 5480 enum bpf_type_flag flag = 0; 5481 u32 btf_id = 0; 5482 int ret; 5483 5484 if (!env->allow_ptr_leaks) { 5485 verbose(env, 5486 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 5487 tname); 5488 return -EPERM; 5489 } 5490 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 5491 verbose(env, 5492 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 5493 tname); 5494 return -EINVAL; 5495 } 5496 if (off < 0) { 5497 verbose(env, 5498 "R%d is ptr_%s invalid negative access: off=%d\n", 5499 regno, tname, off); 5500 return -EACCES; 5501 } 5502 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 5503 char tn_buf[48]; 5504 5505 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5506 verbose(env, 5507 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 5508 regno, tname, off, tn_buf); 5509 return -EACCES; 5510 } 5511 5512 if (reg->type & MEM_USER) { 5513 verbose(env, 5514 "R%d is ptr_%s access user memory: off=%d\n", 5515 regno, tname, off); 5516 return -EACCES; 5517 } 5518 5519 if (reg->type & MEM_PERCPU) { 5520 verbose(env, 5521 "R%d is ptr_%s access percpu memory: off=%d\n", 5522 regno, tname, off); 5523 return -EACCES; 5524 } 5525 5526 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { 5527 if (!btf_is_kernel(reg->btf)) { 5528 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); 5529 return -EFAULT; 5530 } 5531 ret = env->ops->btf_struct_access(&env->log, reg, off, size); 5532 } else { 5533 /* Writes are permitted with default btf_struct_access for 5534 * program allocated objects (which always have ref_obj_id > 0), 5535 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. 5536 */ 5537 if (atype != BPF_READ && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 5538 verbose(env, "only read is supported\n"); 5539 return -EACCES; 5540 } 5541 5542 if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && 5543 !reg->ref_obj_id) { 5544 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 5545 return -EFAULT; 5546 } 5547 5548 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); 5549 } 5550 5551 if (ret < 0) 5552 return ret; 5553 5554 if (ret != PTR_TO_BTF_ID) { 5555 /* just mark; */ 5556 5557 } else if (type_flag(reg->type) & PTR_UNTRUSTED) { 5558 /* If this is an untrusted pointer, all pointers formed by walking it 5559 * also inherit the untrusted flag. 5560 */ 5561 flag = PTR_UNTRUSTED; 5562 5563 } else if (is_trusted_reg(reg) || is_rcu_reg(reg)) { 5564 /* By default any pointer obtained from walking a trusted pointer is no 5565 * longer trusted, unless the field being accessed has explicitly been 5566 * marked as inheriting its parent's state of trust (either full or RCU). 5567 * For example: 5568 * 'cgroups' pointer is untrusted if task->cgroups dereference 5569 * happened in a sleepable program outside of bpf_rcu_read_lock() 5570 * section. In a non-sleepable program it's trusted while in RCU CS (aka MEM_RCU). 5571 * Note bpf_rcu_read_unlock() converts MEM_RCU pointers to PTR_UNTRUSTED. 5572 * 5573 * A regular RCU-protected pointer with __rcu tag can also be deemed 5574 * trusted if we are in an RCU CS. Such pointer can be NULL. 5575 */ 5576 if (type_is_trusted(env, reg, field_name, btf_id)) { 5577 flag |= PTR_TRUSTED; 5578 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { 5579 if (type_is_rcu(env, reg, field_name, btf_id)) { 5580 /* ignore __rcu tag and mark it MEM_RCU */ 5581 flag |= MEM_RCU; 5582 } else if (flag & MEM_RCU || 5583 type_is_rcu_or_null(env, reg, field_name, btf_id)) { 5584 /* __rcu tagged pointers can be NULL */ 5585 flag |= MEM_RCU | PTR_MAYBE_NULL; 5586 } else if (flag & (MEM_PERCPU | MEM_USER)) { 5587 /* keep as-is */ 5588 } else { 5589 /* walking unknown pointers yields old deprecated PTR_TO_BTF_ID */ 5590 clear_trusted_flags(&flag); 5591 } 5592 } else { 5593 /* 5594 * If not in RCU CS or MEM_RCU pointer can be NULL then 5595 * aggressively mark as untrusted otherwise such 5596 * pointers will be plain PTR_TO_BTF_ID without flags 5597 * and will be allowed to be passed into helpers for 5598 * compat reasons. 5599 */ 5600 flag = PTR_UNTRUSTED; 5601 } 5602 } else { 5603 /* Old compat. Deprecated */ 5604 clear_trusted_flags(&flag); 5605 } 5606 5607 if (atype == BPF_READ && value_regno >= 0) 5608 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 5609 5610 return 0; 5611 } 5612 5613 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 5614 struct bpf_reg_state *regs, 5615 int regno, int off, int size, 5616 enum bpf_access_type atype, 5617 int value_regno) 5618 { 5619 struct bpf_reg_state *reg = regs + regno; 5620 struct bpf_map *map = reg->map_ptr; 5621 struct bpf_reg_state map_reg; 5622 enum bpf_type_flag flag = 0; 5623 const struct btf_type *t; 5624 const char *tname; 5625 u32 btf_id; 5626 int ret; 5627 5628 if (!btf_vmlinux) { 5629 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 5630 return -ENOTSUPP; 5631 } 5632 5633 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 5634 verbose(env, "map_ptr access not supported for map type %d\n", 5635 map->map_type); 5636 return -ENOTSUPP; 5637 } 5638 5639 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 5640 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 5641 5642 if (!env->allow_ptr_leaks) { 5643 verbose(env, 5644 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 5645 tname); 5646 return -EPERM; 5647 } 5648 5649 if (off < 0) { 5650 verbose(env, "R%d is %s invalid negative access: off=%d\n", 5651 regno, tname, off); 5652 return -EACCES; 5653 } 5654 5655 if (atype != BPF_READ) { 5656 verbose(env, "only read from %s is supported\n", tname); 5657 return -EACCES; 5658 } 5659 5660 /* Simulate access to a PTR_TO_BTF_ID */ 5661 memset(&map_reg, 0, sizeof(map_reg)); 5662 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); 5663 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); 5664 if (ret < 0) 5665 return ret; 5666 5667 if (value_regno >= 0) 5668 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 5669 5670 return 0; 5671 } 5672 5673 /* Check that the stack access at the given offset is within bounds. The 5674 * maximum valid offset is -1. 5675 * 5676 * The minimum valid offset is -MAX_BPF_STACK for writes, and 5677 * -state->allocated_stack for reads. 5678 */ 5679 static int check_stack_slot_within_bounds(int off, 5680 struct bpf_func_state *state, 5681 enum bpf_access_type t) 5682 { 5683 int min_valid_off; 5684 5685 if (t == BPF_WRITE) 5686 min_valid_off = -MAX_BPF_STACK; 5687 else 5688 min_valid_off = -state->allocated_stack; 5689 5690 if (off < min_valid_off || off > -1) 5691 return -EACCES; 5692 return 0; 5693 } 5694 5695 /* Check that the stack access at 'regno + off' falls within the maximum stack 5696 * bounds. 5697 * 5698 * 'off' includes `regno->offset`, but not its dynamic part (if any). 5699 */ 5700 static int check_stack_access_within_bounds( 5701 struct bpf_verifier_env *env, 5702 int regno, int off, int access_size, 5703 enum bpf_access_src src, enum bpf_access_type type) 5704 { 5705 struct bpf_reg_state *regs = cur_regs(env); 5706 struct bpf_reg_state *reg = regs + regno; 5707 struct bpf_func_state *state = func(env, reg); 5708 int min_off, max_off; 5709 int err; 5710 char *err_extra; 5711 5712 if (src == ACCESS_HELPER) 5713 /* We don't know if helpers are reading or writing (or both). */ 5714 err_extra = " indirect access to"; 5715 else if (type == BPF_READ) 5716 err_extra = " read from"; 5717 else 5718 err_extra = " write to"; 5719 5720 if (tnum_is_const(reg->var_off)) { 5721 min_off = reg->var_off.value + off; 5722 if (access_size > 0) 5723 max_off = min_off + access_size - 1; 5724 else 5725 max_off = min_off; 5726 } else { 5727 if (reg->smax_value >= BPF_MAX_VAR_OFF || 5728 reg->smin_value <= -BPF_MAX_VAR_OFF) { 5729 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 5730 err_extra, regno); 5731 return -EACCES; 5732 } 5733 min_off = reg->smin_value + off; 5734 if (access_size > 0) 5735 max_off = reg->smax_value + off + access_size - 1; 5736 else 5737 max_off = min_off; 5738 } 5739 5740 err = check_stack_slot_within_bounds(min_off, state, type); 5741 if (!err) 5742 err = check_stack_slot_within_bounds(max_off, state, type); 5743 5744 if (err) { 5745 if (tnum_is_const(reg->var_off)) { 5746 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 5747 err_extra, regno, off, access_size); 5748 } else { 5749 char tn_buf[48]; 5750 5751 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5752 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 5753 err_extra, regno, tn_buf, access_size); 5754 } 5755 } 5756 return err; 5757 } 5758 5759 /* check whether memory at (regno + off) is accessible for t = (read | write) 5760 * if t==write, value_regno is a register which value is stored into memory 5761 * if t==read, value_regno is a register which will receive the value from memory 5762 * if t==write && value_regno==-1, some unknown value is stored into memory 5763 * if t==read && value_regno==-1, don't care what we read from memory 5764 */ 5765 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 5766 int off, int bpf_size, enum bpf_access_type t, 5767 int value_regno, bool strict_alignment_once) 5768 { 5769 struct bpf_reg_state *regs = cur_regs(env); 5770 struct bpf_reg_state *reg = regs + regno; 5771 struct bpf_func_state *state; 5772 int size, err = 0; 5773 5774 size = bpf_size_to_bytes(bpf_size); 5775 if (size < 0) 5776 return size; 5777 5778 /* alignment checks will add in reg->off themselves */ 5779 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 5780 if (err) 5781 return err; 5782 5783 /* for access checks, reg->off is just part of off */ 5784 off += reg->off; 5785 5786 if (reg->type == PTR_TO_MAP_KEY) { 5787 if (t == BPF_WRITE) { 5788 verbose(env, "write to change key R%d not allowed\n", regno); 5789 return -EACCES; 5790 } 5791 5792 err = check_mem_region_access(env, regno, off, size, 5793 reg->map_ptr->key_size, false); 5794 if (err) 5795 return err; 5796 if (value_regno >= 0) 5797 mark_reg_unknown(env, regs, value_regno); 5798 } else if (reg->type == PTR_TO_MAP_VALUE) { 5799 struct btf_field *kptr_field = NULL; 5800 5801 if (t == BPF_WRITE && value_regno >= 0 && 5802 is_pointer_value(env, value_regno)) { 5803 verbose(env, "R%d leaks addr into map\n", value_regno); 5804 return -EACCES; 5805 } 5806 err = check_map_access_type(env, regno, off, size, t); 5807 if (err) 5808 return err; 5809 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 5810 if (err) 5811 return err; 5812 if (tnum_is_const(reg->var_off)) 5813 kptr_field = btf_record_find(reg->map_ptr->record, 5814 off + reg->var_off.value, BPF_KPTR); 5815 if (kptr_field) { 5816 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 5817 } else if (t == BPF_READ && value_regno >= 0) { 5818 struct bpf_map *map = reg->map_ptr; 5819 5820 /* if map is read-only, track its contents as scalars */ 5821 if (tnum_is_const(reg->var_off) && 5822 bpf_map_is_rdonly(map) && 5823 map->ops->map_direct_value_addr) { 5824 int map_off = off + reg->var_off.value; 5825 u64 val = 0; 5826 5827 err = bpf_map_direct_read(map, map_off, size, 5828 &val); 5829 if (err) 5830 return err; 5831 5832 regs[value_regno].type = SCALAR_VALUE; 5833 __mark_reg_known(®s[value_regno], val); 5834 } else { 5835 mark_reg_unknown(env, regs, value_regno); 5836 } 5837 } 5838 } else if (base_type(reg->type) == PTR_TO_MEM) { 5839 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5840 5841 if (type_may_be_null(reg->type)) { 5842 verbose(env, "R%d invalid mem access '%s'\n", regno, 5843 reg_type_str(env, reg->type)); 5844 return -EACCES; 5845 } 5846 5847 if (t == BPF_WRITE && rdonly_mem) { 5848 verbose(env, "R%d cannot write into %s\n", 5849 regno, reg_type_str(env, reg->type)); 5850 return -EACCES; 5851 } 5852 5853 if (t == BPF_WRITE && value_regno >= 0 && 5854 is_pointer_value(env, value_regno)) { 5855 verbose(env, "R%d leaks addr into mem\n", value_regno); 5856 return -EACCES; 5857 } 5858 5859 err = check_mem_region_access(env, regno, off, size, 5860 reg->mem_size, false); 5861 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 5862 mark_reg_unknown(env, regs, value_regno); 5863 } else if (reg->type == PTR_TO_CTX) { 5864 enum bpf_reg_type reg_type = SCALAR_VALUE; 5865 struct btf *btf = NULL; 5866 u32 btf_id = 0; 5867 5868 if (t == BPF_WRITE && value_regno >= 0 && 5869 is_pointer_value(env, value_regno)) { 5870 verbose(env, "R%d leaks addr into ctx\n", value_regno); 5871 return -EACCES; 5872 } 5873 5874 err = check_ptr_off_reg(env, reg, regno); 5875 if (err < 0) 5876 return err; 5877 5878 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 5879 &btf_id); 5880 if (err) 5881 verbose_linfo(env, insn_idx, "; "); 5882 if (!err && t == BPF_READ && value_regno >= 0) { 5883 /* ctx access returns either a scalar, or a 5884 * PTR_TO_PACKET[_META,_END]. In the latter 5885 * case, we know the offset is zero. 5886 */ 5887 if (reg_type == SCALAR_VALUE) { 5888 mark_reg_unknown(env, regs, value_regno); 5889 } else { 5890 mark_reg_known_zero(env, regs, 5891 value_regno); 5892 if (type_may_be_null(reg_type)) 5893 regs[value_regno].id = ++env->id_gen; 5894 /* A load of ctx field could have different 5895 * actual load size with the one encoded in the 5896 * insn. When the dst is PTR, it is for sure not 5897 * a sub-register. 5898 */ 5899 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 5900 if (base_type(reg_type) == PTR_TO_BTF_ID) { 5901 regs[value_regno].btf = btf; 5902 regs[value_regno].btf_id = btf_id; 5903 } 5904 } 5905 regs[value_regno].type = reg_type; 5906 } 5907 5908 } else if (reg->type == PTR_TO_STACK) { 5909 /* Basic bounds checks. */ 5910 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 5911 if (err) 5912 return err; 5913 5914 state = func(env, reg); 5915 err = update_stack_depth(env, state, off); 5916 if (err) 5917 return err; 5918 5919 if (t == BPF_READ) 5920 err = check_stack_read(env, regno, off, size, 5921 value_regno); 5922 else 5923 err = check_stack_write(env, regno, off, size, 5924 value_regno, insn_idx); 5925 } else if (reg_is_pkt_pointer(reg)) { 5926 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 5927 verbose(env, "cannot write into packet\n"); 5928 return -EACCES; 5929 } 5930 if (t == BPF_WRITE && value_regno >= 0 && 5931 is_pointer_value(env, value_regno)) { 5932 verbose(env, "R%d leaks addr into packet\n", 5933 value_regno); 5934 return -EACCES; 5935 } 5936 err = check_packet_access(env, regno, off, size, false); 5937 if (!err && t == BPF_READ && value_regno >= 0) 5938 mark_reg_unknown(env, regs, value_regno); 5939 } else if (reg->type == PTR_TO_FLOW_KEYS) { 5940 if (t == BPF_WRITE && value_regno >= 0 && 5941 is_pointer_value(env, value_regno)) { 5942 verbose(env, "R%d leaks addr into flow keys\n", 5943 value_regno); 5944 return -EACCES; 5945 } 5946 5947 err = check_flow_keys_access(env, off, size); 5948 if (!err && t == BPF_READ && value_regno >= 0) 5949 mark_reg_unknown(env, regs, value_regno); 5950 } else if (type_is_sk_pointer(reg->type)) { 5951 if (t == BPF_WRITE) { 5952 verbose(env, "R%d cannot write into %s\n", 5953 regno, reg_type_str(env, reg->type)); 5954 return -EACCES; 5955 } 5956 err = check_sock_access(env, insn_idx, regno, off, size, t); 5957 if (!err && value_regno >= 0) 5958 mark_reg_unknown(env, regs, value_regno); 5959 } else if (reg->type == PTR_TO_TP_BUFFER) { 5960 err = check_tp_buffer_access(env, reg, regno, off, size); 5961 if (!err && t == BPF_READ && value_regno >= 0) 5962 mark_reg_unknown(env, regs, value_regno); 5963 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 5964 !type_may_be_null(reg->type)) { 5965 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 5966 value_regno); 5967 } else if (reg->type == CONST_PTR_TO_MAP) { 5968 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 5969 value_regno); 5970 } else if (base_type(reg->type) == PTR_TO_BUF) { 5971 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5972 u32 *max_access; 5973 5974 if (rdonly_mem) { 5975 if (t == BPF_WRITE) { 5976 verbose(env, "R%d cannot write into %s\n", 5977 regno, reg_type_str(env, reg->type)); 5978 return -EACCES; 5979 } 5980 max_access = &env->prog->aux->max_rdonly_access; 5981 } else { 5982 max_access = &env->prog->aux->max_rdwr_access; 5983 } 5984 5985 err = check_buffer_access(env, reg, regno, off, size, false, 5986 max_access); 5987 5988 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 5989 mark_reg_unknown(env, regs, value_regno); 5990 } else { 5991 verbose(env, "R%d invalid mem access '%s'\n", regno, 5992 reg_type_str(env, reg->type)); 5993 return -EACCES; 5994 } 5995 5996 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 5997 regs[value_regno].type == SCALAR_VALUE) { 5998 /* b/h/w load zero-extends, mark upper bits as known 0 */ 5999 coerce_reg_to_size(®s[value_regno], size); 6000 } 6001 return err; 6002 } 6003 6004 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 6005 { 6006 int load_reg; 6007 int err; 6008 6009 switch (insn->imm) { 6010 case BPF_ADD: 6011 case BPF_ADD | BPF_FETCH: 6012 case BPF_AND: 6013 case BPF_AND | BPF_FETCH: 6014 case BPF_OR: 6015 case BPF_OR | BPF_FETCH: 6016 case BPF_XOR: 6017 case BPF_XOR | BPF_FETCH: 6018 case BPF_XCHG: 6019 case BPF_CMPXCHG: 6020 break; 6021 default: 6022 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 6023 return -EINVAL; 6024 } 6025 6026 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 6027 verbose(env, "invalid atomic operand size\n"); 6028 return -EINVAL; 6029 } 6030 6031 /* check src1 operand */ 6032 err = check_reg_arg(env, insn->src_reg, SRC_OP); 6033 if (err) 6034 return err; 6035 6036 /* check src2 operand */ 6037 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 6038 if (err) 6039 return err; 6040 6041 if (insn->imm == BPF_CMPXCHG) { 6042 /* Check comparison of R0 with memory location */ 6043 const u32 aux_reg = BPF_REG_0; 6044 6045 err = check_reg_arg(env, aux_reg, SRC_OP); 6046 if (err) 6047 return err; 6048 6049 if (is_pointer_value(env, aux_reg)) { 6050 verbose(env, "R%d leaks addr into mem\n", aux_reg); 6051 return -EACCES; 6052 } 6053 } 6054 6055 if (is_pointer_value(env, insn->src_reg)) { 6056 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 6057 return -EACCES; 6058 } 6059 6060 if (is_ctx_reg(env, insn->dst_reg) || 6061 is_pkt_reg(env, insn->dst_reg) || 6062 is_flow_key_reg(env, insn->dst_reg) || 6063 is_sk_reg(env, insn->dst_reg)) { 6064 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 6065 insn->dst_reg, 6066 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 6067 return -EACCES; 6068 } 6069 6070 if (insn->imm & BPF_FETCH) { 6071 if (insn->imm == BPF_CMPXCHG) 6072 load_reg = BPF_REG_0; 6073 else 6074 load_reg = insn->src_reg; 6075 6076 /* check and record load of old value */ 6077 err = check_reg_arg(env, load_reg, DST_OP); 6078 if (err) 6079 return err; 6080 } else { 6081 /* This instruction accesses a memory location but doesn't 6082 * actually load it into a register. 6083 */ 6084 load_reg = -1; 6085 } 6086 6087 /* Check whether we can read the memory, with second call for fetch 6088 * case to simulate the register fill. 6089 */ 6090 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 6091 BPF_SIZE(insn->code), BPF_READ, -1, true); 6092 if (!err && load_reg >= 0) 6093 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 6094 BPF_SIZE(insn->code), BPF_READ, load_reg, 6095 true); 6096 if (err) 6097 return err; 6098 6099 /* Check whether we can write into the same memory. */ 6100 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 6101 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 6102 if (err) 6103 return err; 6104 6105 return 0; 6106 } 6107 6108 /* When register 'regno' is used to read the stack (either directly or through 6109 * a helper function) make sure that it's within stack boundary and, depending 6110 * on the access type, that all elements of the stack are initialized. 6111 * 6112 * 'off' includes 'regno->off', but not its dynamic part (if any). 6113 * 6114 * All registers that have been spilled on the stack in the slots within the 6115 * read offsets are marked as read. 6116 */ 6117 static int check_stack_range_initialized( 6118 struct bpf_verifier_env *env, int regno, int off, 6119 int access_size, bool zero_size_allowed, 6120 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 6121 { 6122 struct bpf_reg_state *reg = reg_state(env, regno); 6123 struct bpf_func_state *state = func(env, reg); 6124 int err, min_off, max_off, i, j, slot, spi; 6125 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 6126 enum bpf_access_type bounds_check_type; 6127 /* Some accesses can write anything into the stack, others are 6128 * read-only. 6129 */ 6130 bool clobber = false; 6131 6132 if (access_size == 0 && !zero_size_allowed) { 6133 verbose(env, "invalid zero-sized read\n"); 6134 return -EACCES; 6135 } 6136 6137 if (type == ACCESS_HELPER) { 6138 /* The bounds checks for writes are more permissive than for 6139 * reads. However, if raw_mode is not set, we'll do extra 6140 * checks below. 6141 */ 6142 bounds_check_type = BPF_WRITE; 6143 clobber = true; 6144 } else { 6145 bounds_check_type = BPF_READ; 6146 } 6147 err = check_stack_access_within_bounds(env, regno, off, access_size, 6148 type, bounds_check_type); 6149 if (err) 6150 return err; 6151 6152 6153 if (tnum_is_const(reg->var_off)) { 6154 min_off = max_off = reg->var_off.value + off; 6155 } else { 6156 /* Variable offset is prohibited for unprivileged mode for 6157 * simplicity since it requires corresponding support in 6158 * Spectre masking for stack ALU. 6159 * See also retrieve_ptr_limit(). 6160 */ 6161 if (!env->bypass_spec_v1) { 6162 char tn_buf[48]; 6163 6164 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6165 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 6166 regno, err_extra, tn_buf); 6167 return -EACCES; 6168 } 6169 /* Only initialized buffer on stack is allowed to be accessed 6170 * with variable offset. With uninitialized buffer it's hard to 6171 * guarantee that whole memory is marked as initialized on 6172 * helper return since specific bounds are unknown what may 6173 * cause uninitialized stack leaking. 6174 */ 6175 if (meta && meta->raw_mode) 6176 meta = NULL; 6177 6178 min_off = reg->smin_value + off; 6179 max_off = reg->smax_value + off; 6180 } 6181 6182 if (meta && meta->raw_mode) { 6183 /* Ensure we won't be overwriting dynptrs when simulating byte 6184 * by byte access in check_helper_call using meta.access_size. 6185 * This would be a problem if we have a helper in the future 6186 * which takes: 6187 * 6188 * helper(uninit_mem, len, dynptr) 6189 * 6190 * Now, uninint_mem may overlap with dynptr pointer. Hence, it 6191 * may end up writing to dynptr itself when touching memory from 6192 * arg 1. This can be relaxed on a case by case basis for known 6193 * safe cases, but reject due to the possibilitiy of aliasing by 6194 * default. 6195 */ 6196 for (i = min_off; i < max_off + access_size; i++) { 6197 int stack_off = -i - 1; 6198 6199 spi = __get_spi(i); 6200 /* raw_mode may write past allocated_stack */ 6201 if (state->allocated_stack <= stack_off) 6202 continue; 6203 if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { 6204 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); 6205 return -EACCES; 6206 } 6207 } 6208 meta->access_size = access_size; 6209 meta->regno = regno; 6210 return 0; 6211 } 6212 6213 for (i = min_off; i < max_off + access_size; i++) { 6214 u8 *stype; 6215 6216 slot = -i - 1; 6217 spi = slot / BPF_REG_SIZE; 6218 if (state->allocated_stack <= slot) 6219 goto err; 6220 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 6221 if (*stype == STACK_MISC) 6222 goto mark; 6223 if ((*stype == STACK_ZERO) || 6224 (*stype == STACK_INVALID && env->allow_uninit_stack)) { 6225 if (clobber) { 6226 /* helper can write anything into the stack */ 6227 *stype = STACK_MISC; 6228 } 6229 goto mark; 6230 } 6231 6232 if (is_spilled_reg(&state->stack[spi]) && 6233 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 6234 env->allow_ptr_leaks)) { 6235 if (clobber) { 6236 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 6237 for (j = 0; j < BPF_REG_SIZE; j++) 6238 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 6239 } 6240 goto mark; 6241 } 6242 6243 err: 6244 if (tnum_is_const(reg->var_off)) { 6245 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 6246 err_extra, regno, min_off, i - min_off, access_size); 6247 } else { 6248 char tn_buf[48]; 6249 6250 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 6251 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 6252 err_extra, regno, tn_buf, i - min_off, access_size); 6253 } 6254 return -EACCES; 6255 mark: 6256 /* reading any byte out of 8-byte 'spill_slot' will cause 6257 * the whole slot to be marked as 'read' 6258 */ 6259 mark_reg_read(env, &state->stack[spi].spilled_ptr, 6260 state->stack[spi].spilled_ptr.parent, 6261 REG_LIVE_READ64); 6262 /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not 6263 * be sure that whether stack slot is written to or not. Hence, 6264 * we must still conservatively propagate reads upwards even if 6265 * helper may write to the entire memory range. 6266 */ 6267 } 6268 return update_stack_depth(env, state, min_off); 6269 } 6270 6271 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 6272 int access_size, bool zero_size_allowed, 6273 struct bpf_call_arg_meta *meta) 6274 { 6275 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6276 u32 *max_access; 6277 6278 switch (base_type(reg->type)) { 6279 case PTR_TO_PACKET: 6280 case PTR_TO_PACKET_META: 6281 return check_packet_access(env, regno, reg->off, access_size, 6282 zero_size_allowed); 6283 case PTR_TO_MAP_KEY: 6284 if (meta && meta->raw_mode) { 6285 verbose(env, "R%d cannot write into %s\n", regno, 6286 reg_type_str(env, reg->type)); 6287 return -EACCES; 6288 } 6289 return check_mem_region_access(env, regno, reg->off, access_size, 6290 reg->map_ptr->key_size, false); 6291 case PTR_TO_MAP_VALUE: 6292 if (check_map_access_type(env, regno, reg->off, access_size, 6293 meta && meta->raw_mode ? BPF_WRITE : 6294 BPF_READ)) 6295 return -EACCES; 6296 return check_map_access(env, regno, reg->off, access_size, 6297 zero_size_allowed, ACCESS_HELPER); 6298 case PTR_TO_MEM: 6299 if (type_is_rdonly_mem(reg->type)) { 6300 if (meta && meta->raw_mode) { 6301 verbose(env, "R%d cannot write into %s\n", regno, 6302 reg_type_str(env, reg->type)); 6303 return -EACCES; 6304 } 6305 } 6306 return check_mem_region_access(env, regno, reg->off, 6307 access_size, reg->mem_size, 6308 zero_size_allowed); 6309 case PTR_TO_BUF: 6310 if (type_is_rdonly_mem(reg->type)) { 6311 if (meta && meta->raw_mode) { 6312 verbose(env, "R%d cannot write into %s\n", regno, 6313 reg_type_str(env, reg->type)); 6314 return -EACCES; 6315 } 6316 6317 max_access = &env->prog->aux->max_rdonly_access; 6318 } else { 6319 max_access = &env->prog->aux->max_rdwr_access; 6320 } 6321 return check_buffer_access(env, reg, regno, reg->off, 6322 access_size, zero_size_allowed, 6323 max_access); 6324 case PTR_TO_STACK: 6325 return check_stack_range_initialized( 6326 env, 6327 regno, reg->off, access_size, 6328 zero_size_allowed, ACCESS_HELPER, meta); 6329 case PTR_TO_BTF_ID: 6330 return check_ptr_to_btf_access(env, regs, regno, reg->off, 6331 access_size, BPF_READ, -1); 6332 case PTR_TO_CTX: 6333 /* in case the function doesn't know how to access the context, 6334 * (because we are in a program of type SYSCALL for example), we 6335 * can not statically check its size. 6336 * Dynamically check it now. 6337 */ 6338 if (!env->ops->convert_ctx_access) { 6339 enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; 6340 int offset = access_size - 1; 6341 6342 /* Allow zero-byte read from PTR_TO_CTX */ 6343 if (access_size == 0) 6344 return zero_size_allowed ? 0 : -EACCES; 6345 6346 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, 6347 atype, -1, false); 6348 } 6349 6350 fallthrough; 6351 default: /* scalar_value or invalid ptr */ 6352 /* Allow zero-byte read from NULL, regardless of pointer type */ 6353 if (zero_size_allowed && access_size == 0 && 6354 register_is_null(reg)) 6355 return 0; 6356 6357 verbose(env, "R%d type=%s ", regno, 6358 reg_type_str(env, reg->type)); 6359 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 6360 return -EACCES; 6361 } 6362 } 6363 6364 static int check_mem_size_reg(struct bpf_verifier_env *env, 6365 struct bpf_reg_state *reg, u32 regno, 6366 bool zero_size_allowed, 6367 struct bpf_call_arg_meta *meta) 6368 { 6369 int err; 6370 6371 /* This is used to refine r0 return value bounds for helpers 6372 * that enforce this value as an upper bound on return values. 6373 * See do_refine_retval_range() for helpers that can refine 6374 * the return value. C type of helper is u32 so we pull register 6375 * bound from umax_value however, if negative verifier errors 6376 * out. Only upper bounds can be learned because retval is an 6377 * int type and negative retvals are allowed. 6378 */ 6379 meta->msize_max_value = reg->umax_value; 6380 6381 /* The register is SCALAR_VALUE; the access check 6382 * happens using its boundaries. 6383 */ 6384 if (!tnum_is_const(reg->var_off)) 6385 /* For unprivileged variable accesses, disable raw 6386 * mode so that the program is required to 6387 * initialize all the memory that the helper could 6388 * just partially fill up. 6389 */ 6390 meta = NULL; 6391 6392 if (reg->smin_value < 0) { 6393 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 6394 regno); 6395 return -EACCES; 6396 } 6397 6398 if (reg->umin_value == 0) { 6399 err = check_helper_mem_access(env, regno - 1, 0, 6400 zero_size_allowed, 6401 meta); 6402 if (err) 6403 return err; 6404 } 6405 6406 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 6407 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 6408 regno); 6409 return -EACCES; 6410 } 6411 err = check_helper_mem_access(env, regno - 1, 6412 reg->umax_value, 6413 zero_size_allowed, meta); 6414 if (!err) 6415 err = mark_chain_precision(env, regno); 6416 return err; 6417 } 6418 6419 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 6420 u32 regno, u32 mem_size) 6421 { 6422 bool may_be_null = type_may_be_null(reg->type); 6423 struct bpf_reg_state saved_reg; 6424 struct bpf_call_arg_meta meta; 6425 int err; 6426 6427 if (register_is_null(reg)) 6428 return 0; 6429 6430 memset(&meta, 0, sizeof(meta)); 6431 /* Assuming that the register contains a value check if the memory 6432 * access is safe. Temporarily save and restore the register's state as 6433 * the conversion shouldn't be visible to a caller. 6434 */ 6435 if (may_be_null) { 6436 saved_reg = *reg; 6437 mark_ptr_not_null_reg(reg); 6438 } 6439 6440 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 6441 /* Check access for BPF_WRITE */ 6442 meta.raw_mode = true; 6443 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 6444 6445 if (may_be_null) 6446 *reg = saved_reg; 6447 6448 return err; 6449 } 6450 6451 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 6452 u32 regno) 6453 { 6454 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 6455 bool may_be_null = type_may_be_null(mem_reg->type); 6456 struct bpf_reg_state saved_reg; 6457 struct bpf_call_arg_meta meta; 6458 int err; 6459 6460 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 6461 6462 memset(&meta, 0, sizeof(meta)); 6463 6464 if (may_be_null) { 6465 saved_reg = *mem_reg; 6466 mark_ptr_not_null_reg(mem_reg); 6467 } 6468 6469 err = check_mem_size_reg(env, reg, regno, true, &meta); 6470 /* Check access for BPF_WRITE */ 6471 meta.raw_mode = true; 6472 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 6473 6474 if (may_be_null) 6475 *mem_reg = saved_reg; 6476 return err; 6477 } 6478 6479 /* Implementation details: 6480 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 6481 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 6482 * Two bpf_map_lookups (even with the same key) will have different reg->id. 6483 * Two separate bpf_obj_new will also have different reg->id. 6484 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 6485 * clears reg->id after value_or_null->value transition, since the verifier only 6486 * cares about the range of access to valid map value pointer and doesn't care 6487 * about actual address of the map element. 6488 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 6489 * reg->id > 0 after value_or_null->value transition. By doing so 6490 * two bpf_map_lookups will be considered two different pointers that 6491 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 6492 * returned from bpf_obj_new. 6493 * The verifier allows taking only one bpf_spin_lock at a time to avoid 6494 * dead-locks. 6495 * Since only one bpf_spin_lock is allowed the checks are simpler than 6496 * reg_is_refcounted() logic. The verifier needs to remember only 6497 * one spin_lock instead of array of acquired_refs. 6498 * cur_state->active_lock remembers which map value element or allocated 6499 * object got locked and clears it after bpf_spin_unlock. 6500 */ 6501 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 6502 bool is_lock) 6503 { 6504 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6505 struct bpf_verifier_state *cur = env->cur_state; 6506 bool is_const = tnum_is_const(reg->var_off); 6507 u64 val = reg->var_off.value; 6508 struct bpf_map *map = NULL; 6509 struct btf *btf = NULL; 6510 struct btf_record *rec; 6511 6512 if (!is_const) { 6513 verbose(env, 6514 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 6515 regno); 6516 return -EINVAL; 6517 } 6518 if (reg->type == PTR_TO_MAP_VALUE) { 6519 map = reg->map_ptr; 6520 if (!map->btf) { 6521 verbose(env, 6522 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 6523 map->name); 6524 return -EINVAL; 6525 } 6526 } else { 6527 btf = reg->btf; 6528 } 6529 6530 rec = reg_btf_record(reg); 6531 if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) { 6532 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", 6533 map ? map->name : "kptr"); 6534 return -EINVAL; 6535 } 6536 if (rec->spin_lock_off != val + reg->off) { 6537 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", 6538 val + reg->off, rec->spin_lock_off); 6539 return -EINVAL; 6540 } 6541 if (is_lock) { 6542 if (cur->active_lock.ptr) { 6543 verbose(env, 6544 "Locking two bpf_spin_locks are not allowed\n"); 6545 return -EINVAL; 6546 } 6547 if (map) 6548 cur->active_lock.ptr = map; 6549 else 6550 cur->active_lock.ptr = btf; 6551 cur->active_lock.id = reg->id; 6552 } else { 6553 void *ptr; 6554 6555 if (map) 6556 ptr = map; 6557 else 6558 ptr = btf; 6559 6560 if (!cur->active_lock.ptr) { 6561 verbose(env, "bpf_spin_unlock without taking a lock\n"); 6562 return -EINVAL; 6563 } 6564 if (cur->active_lock.ptr != ptr || 6565 cur->active_lock.id != reg->id) { 6566 verbose(env, "bpf_spin_unlock of different lock\n"); 6567 return -EINVAL; 6568 } 6569 6570 invalidate_non_owning_refs(env); 6571 6572 cur->active_lock.ptr = NULL; 6573 cur->active_lock.id = 0; 6574 } 6575 return 0; 6576 } 6577 6578 static int process_timer_func(struct bpf_verifier_env *env, int regno, 6579 struct bpf_call_arg_meta *meta) 6580 { 6581 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6582 bool is_const = tnum_is_const(reg->var_off); 6583 struct bpf_map *map = reg->map_ptr; 6584 u64 val = reg->var_off.value; 6585 6586 if (!is_const) { 6587 verbose(env, 6588 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 6589 regno); 6590 return -EINVAL; 6591 } 6592 if (!map->btf) { 6593 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 6594 map->name); 6595 return -EINVAL; 6596 } 6597 if (!btf_record_has_field(map->record, BPF_TIMER)) { 6598 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); 6599 return -EINVAL; 6600 } 6601 if (map->record->timer_off != val + reg->off) { 6602 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 6603 val + reg->off, map->record->timer_off); 6604 return -EINVAL; 6605 } 6606 if (meta->map_ptr) { 6607 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 6608 return -EFAULT; 6609 } 6610 meta->map_uid = reg->map_uid; 6611 meta->map_ptr = map; 6612 return 0; 6613 } 6614 6615 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 6616 struct bpf_call_arg_meta *meta) 6617 { 6618 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6619 struct bpf_map *map_ptr = reg->map_ptr; 6620 struct btf_field *kptr_field; 6621 u32 kptr_off; 6622 6623 if (!tnum_is_const(reg->var_off)) { 6624 verbose(env, 6625 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 6626 regno); 6627 return -EINVAL; 6628 } 6629 if (!map_ptr->btf) { 6630 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 6631 map_ptr->name); 6632 return -EINVAL; 6633 } 6634 if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 6635 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 6636 return -EINVAL; 6637 } 6638 6639 meta->map_ptr = map_ptr; 6640 kptr_off = reg->off + reg->var_off.value; 6641 kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 6642 if (!kptr_field) { 6643 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 6644 return -EACCES; 6645 } 6646 if (kptr_field->type != BPF_KPTR_REF) { 6647 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 6648 return -EACCES; 6649 } 6650 meta->kptr_field = kptr_field; 6651 return 0; 6652 } 6653 6654 /* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK 6655 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. 6656 * 6657 * In both cases we deal with the first 8 bytes, but need to mark the next 8 6658 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of 6659 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. 6660 * 6661 * Mutability of bpf_dynptr is at two levels, one is at the level of struct 6662 * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct 6663 * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can 6664 * mutate the view of the dynptr and also possibly destroy it. In the latter 6665 * case, it cannot mutate the bpf_dynptr itself but it can still mutate the 6666 * memory that dynptr points to. 6667 * 6668 * The verifier will keep track both levels of mutation (bpf_dynptr's in 6669 * reg->type and the memory's in reg->dynptr.type), but there is no support for 6670 * readonly dynptr view yet, hence only the first case is tracked and checked. 6671 * 6672 * This is consistent with how C applies the const modifier to a struct object, 6673 * where the pointer itself inside bpf_dynptr becomes const but not what it 6674 * points to. 6675 * 6676 * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument 6677 * type, and declare it as 'const struct bpf_dynptr *' in their prototype. 6678 */ 6679 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, 6680 enum bpf_arg_type arg_type) 6681 { 6682 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6683 int err; 6684 6685 /* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an 6686 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*): 6687 */ 6688 if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) { 6689 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); 6690 return -EFAULT; 6691 } 6692 6693 /* MEM_UNINIT - Points to memory that is an appropriate candidate for 6694 * constructing a mutable bpf_dynptr object. 6695 * 6696 * Currently, this is only possible with PTR_TO_STACK 6697 * pointing to a region of at least 16 bytes which doesn't 6698 * contain an existing bpf_dynptr. 6699 * 6700 * MEM_RDONLY - Points to a initialized bpf_dynptr that will not be 6701 * mutated or destroyed. However, the memory it points to 6702 * may be mutated. 6703 * 6704 * None - Points to a initialized dynptr that can be mutated and 6705 * destroyed, including mutation of the memory it points 6706 * to. 6707 */ 6708 if (arg_type & MEM_UNINIT) { 6709 int i; 6710 6711 if (!is_dynptr_reg_valid_uninit(env, reg)) { 6712 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 6713 return -EINVAL; 6714 } 6715 6716 /* we write BPF_DW bits (8 bytes) at a time */ 6717 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 6718 err = check_mem_access(env, insn_idx, regno, 6719 i, BPF_DW, BPF_WRITE, -1, false); 6720 if (err) 6721 return err; 6722 } 6723 6724 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx); 6725 } else /* MEM_RDONLY and None case from above */ { 6726 /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ 6727 if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) { 6728 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); 6729 return -EINVAL; 6730 } 6731 6732 if (!is_dynptr_reg_valid_init(env, reg)) { 6733 verbose(env, 6734 "Expected an initialized dynptr as arg #%d\n", 6735 regno); 6736 return -EINVAL; 6737 } 6738 6739 /* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */ 6740 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { 6741 verbose(env, 6742 "Expected a dynptr of type %s as arg #%d\n", 6743 dynptr_type_str(arg_to_dynptr_type(arg_type)), regno); 6744 return -EINVAL; 6745 } 6746 6747 err = mark_dynptr_read(env, reg); 6748 } 6749 return err; 6750 } 6751 6752 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) 6753 { 6754 struct bpf_func_state *state = func(env, reg); 6755 6756 return state->stack[spi].spilled_ptr.ref_obj_id; 6757 } 6758 6759 static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) 6760 { 6761 return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); 6762 } 6763 6764 static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) 6765 { 6766 return meta->kfunc_flags & KF_ITER_NEW; 6767 } 6768 6769 static bool is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta) 6770 { 6771 return meta->kfunc_flags & KF_ITER_NEXT; 6772 } 6773 6774 static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) 6775 { 6776 return meta->kfunc_flags & KF_ITER_DESTROY; 6777 } 6778 6779 static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg) 6780 { 6781 /* btf_check_iter_kfuncs() guarantees that first argument of any iter 6782 * kfunc is iter state pointer 6783 */ 6784 return arg == 0 && is_iter_kfunc(meta); 6785 } 6786 6787 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, 6788 struct bpf_kfunc_call_arg_meta *meta) 6789 { 6790 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6791 const struct btf_type *t; 6792 const struct btf_param *arg; 6793 int spi, err, i, nr_slots; 6794 u32 btf_id; 6795 6796 /* btf_check_iter_kfuncs() ensures we don't need to validate anything here */ 6797 arg = &btf_params(meta->func_proto)[0]; 6798 t = btf_type_skip_modifiers(meta->btf, arg->type, NULL); /* PTR */ 6799 t = btf_type_skip_modifiers(meta->btf, t->type, &btf_id); /* STRUCT */ 6800 nr_slots = t->size / BPF_REG_SIZE; 6801 6802 if (is_iter_new_kfunc(meta)) { 6803 /* bpf_iter_<type>_new() expects pointer to uninit iter state */ 6804 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { 6805 verbose(env, "expected uninitialized iter_%s as arg #%d\n", 6806 iter_type_str(meta->btf, btf_id), regno); 6807 return -EINVAL; 6808 } 6809 6810 for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) { 6811 err = check_mem_access(env, insn_idx, regno, 6812 i, BPF_DW, BPF_WRITE, -1, false); 6813 if (err) 6814 return err; 6815 } 6816 6817 err = mark_stack_slots_iter(env, reg, insn_idx, meta->btf, btf_id, nr_slots); 6818 if (err) 6819 return err; 6820 } else { 6821 /* iter_next() or iter_destroy() expect initialized iter state*/ 6822 if (!is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots)) { 6823 verbose(env, "expected an initialized iter_%s as arg #%d\n", 6824 iter_type_str(meta->btf, btf_id), regno); 6825 return -EINVAL; 6826 } 6827 6828 spi = iter_get_spi(env, reg, nr_slots); 6829 if (spi < 0) 6830 return spi; 6831 6832 err = mark_iter_read(env, reg, spi, nr_slots); 6833 if (err) 6834 return err; 6835 6836 /* remember meta->iter info for process_iter_next_call() */ 6837 meta->iter.spi = spi; 6838 meta->iter.frameno = reg->frameno; 6839 meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); 6840 6841 if (is_iter_destroy_kfunc(meta)) { 6842 err = unmark_stack_slots_iter(env, reg, nr_slots); 6843 if (err) 6844 return err; 6845 } 6846 } 6847 6848 return 0; 6849 } 6850 6851 /* process_iter_next_call() is called when verifier gets to iterator's next 6852 * "method" (e.g., bpf_iter_num_next() for numbers iterator) call. We'll refer 6853 * to it as just "iter_next()" in comments below. 6854 * 6855 * BPF verifier relies on a crucial contract for any iter_next() 6856 * implementation: it should *eventually* return NULL, and once that happens 6857 * it should keep returning NULL. That is, once iterator exhausts elements to 6858 * iterate, it should never reset or spuriously return new elements. 6859 * 6860 * With the assumption of such contract, process_iter_next_call() simulates 6861 * a fork in the verifier state to validate loop logic correctness and safety 6862 * without having to simulate infinite amount of iterations. 6863 * 6864 * In current state, we first assume that iter_next() returned NULL and 6865 * iterator state is set to DRAINED (BPF_ITER_STATE_DRAINED). In such 6866 * conditions we should not form an infinite loop and should eventually reach 6867 * exit. 6868 * 6869 * Besides that, we also fork current state and enqueue it for later 6870 * verification. In a forked state we keep iterator state as ACTIVE 6871 * (BPF_ITER_STATE_ACTIVE) and assume non-NULL return from iter_next(). We 6872 * also bump iteration depth to prevent erroneous infinite loop detection 6873 * later on (see iter_active_depths_differ() comment for details). In this 6874 * state we assume that we'll eventually loop back to another iter_next() 6875 * calls (it could be in exactly same location or in some other instruction, 6876 * it doesn't matter, we don't make any unnecessary assumptions about this, 6877 * everything revolves around iterator state in a stack slot, not which 6878 * instruction is calling iter_next()). When that happens, we either will come 6879 * to iter_next() with equivalent state and can conclude that next iteration 6880 * will proceed in exactly the same way as we just verified, so it's safe to 6881 * assume that loop converges. If not, we'll go on another iteration 6882 * simulation with a different input state, until all possible starting states 6883 * are validated or we reach maximum number of instructions limit. 6884 * 6885 * This way, we will either exhaustively discover all possible input states 6886 * that iterator loop can start with and eventually will converge, or we'll 6887 * effectively regress into bounded loop simulation logic and either reach 6888 * maximum number of instructions if loop is not provably convergent, or there 6889 * is some statically known limit on number of iterations (e.g., if there is 6890 * an explicit `if n > 100 then break;` statement somewhere in the loop). 6891 * 6892 * One very subtle but very important aspect is that we *always* simulate NULL 6893 * condition first (as the current state) before we simulate non-NULL case. 6894 * This has to do with intricacies of scalar precision tracking. By simulating 6895 * "exit condition" of iter_next() returning NULL first, we make sure all the 6896 * relevant precision marks *that will be set **after** we exit iterator loop* 6897 * are propagated backwards to common parent state of NULL and non-NULL 6898 * branches. Thanks to that, state equivalence checks done later in forked 6899 * state, when reaching iter_next() for ACTIVE iterator, can assume that 6900 * precision marks are finalized and won't change. Because simulating another 6901 * ACTIVE iterator iteration won't change them (because given same input 6902 * states we'll end up with exactly same output states which we are currently 6903 * comparing; and verification after the loop already propagated back what 6904 * needs to be **additionally** tracked as precise). It's subtle, grok 6905 * precision tracking for more intuitive understanding. 6906 */ 6907 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, 6908 struct bpf_kfunc_call_arg_meta *meta) 6909 { 6910 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st; 6911 struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; 6912 struct bpf_reg_state *cur_iter, *queued_iter; 6913 int iter_frameno = meta->iter.frameno; 6914 int iter_spi = meta->iter.spi; 6915 6916 BTF_TYPE_EMIT(struct bpf_iter); 6917 6918 cur_iter = &env->cur_state->frame[iter_frameno]->stack[iter_spi].spilled_ptr; 6919 6920 if (cur_iter->iter.state != BPF_ITER_STATE_ACTIVE && 6921 cur_iter->iter.state != BPF_ITER_STATE_DRAINED) { 6922 verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n", 6923 cur_iter->iter.state, iter_state_str(cur_iter->iter.state)); 6924 return -EFAULT; 6925 } 6926 6927 if (cur_iter->iter.state == BPF_ITER_STATE_ACTIVE) { 6928 /* branch out active iter state */ 6929 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); 6930 if (!queued_st) 6931 return -ENOMEM; 6932 6933 queued_iter = &queued_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr; 6934 queued_iter->iter.state = BPF_ITER_STATE_ACTIVE; 6935 queued_iter->iter.depth++; 6936 6937 queued_fr = queued_st->frame[queued_st->curframe]; 6938 mark_ptr_not_null_reg(&queued_fr->regs[BPF_REG_0]); 6939 } 6940 6941 /* switch to DRAINED state, but keep the depth unchanged */ 6942 /* mark current iter state as drained and assume returned NULL */ 6943 cur_iter->iter.state = BPF_ITER_STATE_DRAINED; 6944 __mark_reg_const_zero(&cur_fr->regs[BPF_REG_0]); 6945 6946 return 0; 6947 } 6948 6949 static bool arg_type_is_mem_size(enum bpf_arg_type type) 6950 { 6951 return type == ARG_CONST_SIZE || 6952 type == ARG_CONST_SIZE_OR_ZERO; 6953 } 6954 6955 static bool arg_type_is_release(enum bpf_arg_type type) 6956 { 6957 return type & OBJ_RELEASE; 6958 } 6959 6960 static bool arg_type_is_dynptr(enum bpf_arg_type type) 6961 { 6962 return base_type(type) == ARG_PTR_TO_DYNPTR; 6963 } 6964 6965 static int int_ptr_type_to_size(enum bpf_arg_type type) 6966 { 6967 if (type == ARG_PTR_TO_INT) 6968 return sizeof(u32); 6969 else if (type == ARG_PTR_TO_LONG) 6970 return sizeof(u64); 6971 6972 return -EINVAL; 6973 } 6974 6975 static int resolve_map_arg_type(struct bpf_verifier_env *env, 6976 const struct bpf_call_arg_meta *meta, 6977 enum bpf_arg_type *arg_type) 6978 { 6979 if (!meta->map_ptr) { 6980 /* kernel subsystem misconfigured verifier */ 6981 verbose(env, "invalid map_ptr to access map->type\n"); 6982 return -EACCES; 6983 } 6984 6985 switch (meta->map_ptr->map_type) { 6986 case BPF_MAP_TYPE_SOCKMAP: 6987 case BPF_MAP_TYPE_SOCKHASH: 6988 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 6989 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 6990 } else { 6991 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 6992 return -EINVAL; 6993 } 6994 break; 6995 case BPF_MAP_TYPE_BLOOM_FILTER: 6996 if (meta->func_id == BPF_FUNC_map_peek_elem) 6997 *arg_type = ARG_PTR_TO_MAP_VALUE; 6998 break; 6999 default: 7000 break; 7001 } 7002 return 0; 7003 } 7004 7005 struct bpf_reg_types { 7006 const enum bpf_reg_type types[10]; 7007 u32 *btf_id; 7008 }; 7009 7010 static const struct bpf_reg_types sock_types = { 7011 .types = { 7012 PTR_TO_SOCK_COMMON, 7013 PTR_TO_SOCKET, 7014 PTR_TO_TCP_SOCK, 7015 PTR_TO_XDP_SOCK, 7016 }, 7017 }; 7018 7019 #ifdef CONFIG_NET 7020 static const struct bpf_reg_types btf_id_sock_common_types = { 7021 .types = { 7022 PTR_TO_SOCK_COMMON, 7023 PTR_TO_SOCKET, 7024 PTR_TO_TCP_SOCK, 7025 PTR_TO_XDP_SOCK, 7026 PTR_TO_BTF_ID, 7027 PTR_TO_BTF_ID | PTR_TRUSTED, 7028 }, 7029 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 7030 }; 7031 #endif 7032 7033 static const struct bpf_reg_types mem_types = { 7034 .types = { 7035 PTR_TO_STACK, 7036 PTR_TO_PACKET, 7037 PTR_TO_PACKET_META, 7038 PTR_TO_MAP_KEY, 7039 PTR_TO_MAP_VALUE, 7040 PTR_TO_MEM, 7041 PTR_TO_MEM | MEM_RINGBUF, 7042 PTR_TO_BUF, 7043 PTR_TO_BTF_ID | PTR_TRUSTED, 7044 }, 7045 }; 7046 7047 static const struct bpf_reg_types int_ptr_types = { 7048 .types = { 7049 PTR_TO_STACK, 7050 PTR_TO_PACKET, 7051 PTR_TO_PACKET_META, 7052 PTR_TO_MAP_KEY, 7053 PTR_TO_MAP_VALUE, 7054 }, 7055 }; 7056 7057 static const struct bpf_reg_types spin_lock_types = { 7058 .types = { 7059 PTR_TO_MAP_VALUE, 7060 PTR_TO_BTF_ID | MEM_ALLOC, 7061 } 7062 }; 7063 7064 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 7065 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 7066 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 7067 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 7068 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 7069 static const struct bpf_reg_types btf_ptr_types = { 7070 .types = { 7071 PTR_TO_BTF_ID, 7072 PTR_TO_BTF_ID | PTR_TRUSTED, 7073 PTR_TO_BTF_ID | MEM_RCU, 7074 }, 7075 }; 7076 static const struct bpf_reg_types percpu_btf_ptr_types = { 7077 .types = { 7078 PTR_TO_BTF_ID | MEM_PERCPU, 7079 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 7080 } 7081 }; 7082 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 7083 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 7084 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 7085 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 7086 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 7087 static const struct bpf_reg_types dynptr_types = { 7088 .types = { 7089 PTR_TO_STACK, 7090 CONST_PTR_TO_DYNPTR, 7091 } 7092 }; 7093 7094 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 7095 [ARG_PTR_TO_MAP_KEY] = &mem_types, 7096 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 7097 [ARG_CONST_SIZE] = &scalar_types, 7098 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 7099 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 7100 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 7101 [ARG_PTR_TO_CTX] = &context_types, 7102 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 7103 #ifdef CONFIG_NET 7104 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 7105 #endif 7106 [ARG_PTR_TO_SOCKET] = &fullsock_types, 7107 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 7108 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 7109 [ARG_PTR_TO_MEM] = &mem_types, 7110 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 7111 [ARG_PTR_TO_INT] = &int_ptr_types, 7112 [ARG_PTR_TO_LONG] = &int_ptr_types, 7113 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 7114 [ARG_PTR_TO_FUNC] = &func_ptr_types, 7115 [ARG_PTR_TO_STACK] = &stack_ptr_types, 7116 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 7117 [ARG_PTR_TO_TIMER] = &timer_types, 7118 [ARG_PTR_TO_KPTR] = &kptr_types, 7119 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 7120 }; 7121 7122 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 7123 enum bpf_arg_type arg_type, 7124 const u32 *arg_btf_id, 7125 struct bpf_call_arg_meta *meta) 7126 { 7127 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7128 enum bpf_reg_type expected, type = reg->type; 7129 const struct bpf_reg_types *compatible; 7130 int i, j; 7131 7132 compatible = compatible_reg_types[base_type(arg_type)]; 7133 if (!compatible) { 7134 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 7135 return -EFAULT; 7136 } 7137 7138 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 7139 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 7140 * 7141 * Same for MAYBE_NULL: 7142 * 7143 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 7144 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 7145 * 7146 * Therefore we fold these flags depending on the arg_type before comparison. 7147 */ 7148 if (arg_type & MEM_RDONLY) 7149 type &= ~MEM_RDONLY; 7150 if (arg_type & PTR_MAYBE_NULL) 7151 type &= ~PTR_MAYBE_NULL; 7152 7153 if (meta->func_id == BPF_FUNC_kptr_xchg && type & MEM_ALLOC) 7154 type &= ~MEM_ALLOC; 7155 7156 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 7157 expected = compatible->types[i]; 7158 if (expected == NOT_INIT) 7159 break; 7160 7161 if (type == expected) 7162 goto found; 7163 } 7164 7165 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 7166 for (j = 0; j + 1 < i; j++) 7167 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 7168 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 7169 return -EACCES; 7170 7171 found: 7172 if (base_type(reg->type) != PTR_TO_BTF_ID) 7173 return 0; 7174 7175 if (compatible == &mem_types) { 7176 if (!(arg_type & MEM_RDONLY)) { 7177 verbose(env, 7178 "%s() may write into memory pointed by R%d type=%s\n", 7179 func_id_name(meta->func_id), 7180 regno, reg_type_str(env, reg->type)); 7181 return -EACCES; 7182 } 7183 return 0; 7184 } 7185 7186 switch ((int)reg->type) { 7187 case PTR_TO_BTF_ID: 7188 case PTR_TO_BTF_ID | PTR_TRUSTED: 7189 case PTR_TO_BTF_ID | MEM_RCU: 7190 case PTR_TO_BTF_ID | PTR_MAYBE_NULL: 7191 case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU: 7192 { 7193 /* For bpf_sk_release, it needs to match against first member 7194 * 'struct sock_common', hence make an exception for it. This 7195 * allows bpf_sk_release to work for multiple socket types. 7196 */ 7197 bool strict_type_match = arg_type_is_release(arg_type) && 7198 meta->func_id != BPF_FUNC_sk_release; 7199 7200 if (type_may_be_null(reg->type) && 7201 (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { 7202 verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); 7203 return -EACCES; 7204 } 7205 7206 if (!arg_btf_id) { 7207 if (!compatible->btf_id) { 7208 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 7209 return -EFAULT; 7210 } 7211 arg_btf_id = compatible->btf_id; 7212 } 7213 7214 if (meta->func_id == BPF_FUNC_kptr_xchg) { 7215 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 7216 return -EACCES; 7217 } else { 7218 if (arg_btf_id == BPF_PTR_POISON) { 7219 verbose(env, "verifier internal error:"); 7220 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", 7221 regno); 7222 return -EACCES; 7223 } 7224 7225 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 7226 btf_vmlinux, *arg_btf_id, 7227 strict_type_match)) { 7228 verbose(env, "R%d is of type %s but %s is expected\n", 7229 regno, btf_type_name(reg->btf, reg->btf_id), 7230 btf_type_name(btf_vmlinux, *arg_btf_id)); 7231 return -EACCES; 7232 } 7233 } 7234 break; 7235 } 7236 case PTR_TO_BTF_ID | MEM_ALLOC: 7237 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock && 7238 meta->func_id != BPF_FUNC_kptr_xchg) { 7239 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); 7240 return -EFAULT; 7241 } 7242 /* Handled by helper specific checks */ 7243 break; 7244 case PTR_TO_BTF_ID | MEM_PERCPU: 7245 case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: 7246 /* Handled by helper specific checks */ 7247 break; 7248 default: 7249 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); 7250 return -EFAULT; 7251 } 7252 return 0; 7253 } 7254 7255 static struct btf_field * 7256 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) 7257 { 7258 struct btf_field *field; 7259 struct btf_record *rec; 7260 7261 rec = reg_btf_record(reg); 7262 if (!rec) 7263 return NULL; 7264 7265 field = btf_record_find(rec, off, fields); 7266 if (!field) 7267 return NULL; 7268 7269 return field; 7270 } 7271 7272 int check_func_arg_reg_off(struct bpf_verifier_env *env, 7273 const struct bpf_reg_state *reg, int regno, 7274 enum bpf_arg_type arg_type) 7275 { 7276 u32 type = reg->type; 7277 7278 /* When referenced register is passed to release function, its fixed 7279 * offset must be 0. 7280 * 7281 * We will check arg_type_is_release reg has ref_obj_id when storing 7282 * meta->release_regno. 7283 */ 7284 if (arg_type_is_release(arg_type)) { 7285 /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it 7286 * may not directly point to the object being released, but to 7287 * dynptr pointing to such object, which might be at some offset 7288 * on the stack. In that case, we simply to fallback to the 7289 * default handling. 7290 */ 7291 if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) 7292 return 0; 7293 7294 if ((type_is_ptr_alloc_obj(type) || type_is_non_owning_ref(type)) && reg->off) { 7295 if (reg_find_field_offset(reg, reg->off, BPF_GRAPH_NODE_OR_ROOT)) 7296 return __check_ptr_off_reg(env, reg, regno, true); 7297 7298 verbose(env, "R%d must have zero offset when passed to release func\n", 7299 regno); 7300 verbose(env, "No graph node or root found at R%d type:%s off:%d\n", regno, 7301 btf_type_name(reg->btf, reg->btf_id), reg->off); 7302 return -EINVAL; 7303 } 7304 7305 /* Doing check_ptr_off_reg check for the offset will catch this 7306 * because fixed_off_ok is false, but checking here allows us 7307 * to give the user a better error message. 7308 */ 7309 if (reg->off) { 7310 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", 7311 regno); 7312 return -EINVAL; 7313 } 7314 return __check_ptr_off_reg(env, reg, regno, false); 7315 } 7316 7317 switch (type) { 7318 /* Pointer types where both fixed and variable offset is explicitly allowed: */ 7319 case PTR_TO_STACK: 7320 case PTR_TO_PACKET: 7321 case PTR_TO_PACKET_META: 7322 case PTR_TO_MAP_KEY: 7323 case PTR_TO_MAP_VALUE: 7324 case PTR_TO_MEM: 7325 case PTR_TO_MEM | MEM_RDONLY: 7326 case PTR_TO_MEM | MEM_RINGBUF: 7327 case PTR_TO_BUF: 7328 case PTR_TO_BUF | MEM_RDONLY: 7329 case SCALAR_VALUE: 7330 return 0; 7331 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 7332 * fixed offset. 7333 */ 7334 case PTR_TO_BTF_ID: 7335 case PTR_TO_BTF_ID | MEM_ALLOC: 7336 case PTR_TO_BTF_ID | PTR_TRUSTED: 7337 case PTR_TO_BTF_ID | MEM_RCU: 7338 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: 7339 /* When referenced PTR_TO_BTF_ID is passed to release function, 7340 * its fixed offset must be 0. In the other cases, fixed offset 7341 * can be non-zero. This was already checked above. So pass 7342 * fixed_off_ok as true to allow fixed offset for all other 7343 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we 7344 * still need to do checks instead of returning. 7345 */ 7346 return __check_ptr_off_reg(env, reg, regno, true); 7347 default: 7348 return __check_ptr_off_reg(env, reg, regno, false); 7349 } 7350 } 7351 7352 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, 7353 const struct bpf_func_proto *fn, 7354 struct bpf_reg_state *regs) 7355 { 7356 struct bpf_reg_state *state = NULL; 7357 int i; 7358 7359 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) 7360 if (arg_type_is_dynptr(fn->arg_type[i])) { 7361 if (state) { 7362 verbose(env, "verifier internal error: multiple dynptr args\n"); 7363 return NULL; 7364 } 7365 state = ®s[BPF_REG_1 + i]; 7366 } 7367 7368 if (!state) 7369 verbose(env, "verifier internal error: no dynptr arg found\n"); 7370 7371 return state; 7372 } 7373 7374 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 7375 { 7376 struct bpf_func_state *state = func(env, reg); 7377 int spi; 7378 7379 if (reg->type == CONST_PTR_TO_DYNPTR) 7380 return reg->id; 7381 spi = dynptr_get_spi(env, reg); 7382 if (spi < 0) 7383 return spi; 7384 return state->stack[spi].spilled_ptr.id; 7385 } 7386 7387 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 7388 { 7389 struct bpf_func_state *state = func(env, reg); 7390 int spi; 7391 7392 if (reg->type == CONST_PTR_TO_DYNPTR) 7393 return reg->ref_obj_id; 7394 spi = dynptr_get_spi(env, reg); 7395 if (spi < 0) 7396 return spi; 7397 return state->stack[spi].spilled_ptr.ref_obj_id; 7398 } 7399 7400 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, 7401 struct bpf_reg_state *reg) 7402 { 7403 struct bpf_func_state *state = func(env, reg); 7404 int spi; 7405 7406 if (reg->type == CONST_PTR_TO_DYNPTR) 7407 return reg->dynptr.type; 7408 7409 spi = __get_spi(reg->off); 7410 if (spi < 0) { 7411 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); 7412 return BPF_DYNPTR_TYPE_INVALID; 7413 } 7414 7415 return state->stack[spi].spilled_ptr.dynptr.type; 7416 } 7417 7418 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 7419 struct bpf_call_arg_meta *meta, 7420 const struct bpf_func_proto *fn, 7421 int insn_idx) 7422 { 7423 u32 regno = BPF_REG_1 + arg; 7424 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 7425 enum bpf_arg_type arg_type = fn->arg_type[arg]; 7426 enum bpf_reg_type type = reg->type; 7427 u32 *arg_btf_id = NULL; 7428 int err = 0; 7429 7430 if (arg_type == ARG_DONTCARE) 7431 return 0; 7432 7433 err = check_reg_arg(env, regno, SRC_OP); 7434 if (err) 7435 return err; 7436 7437 if (arg_type == ARG_ANYTHING) { 7438 if (is_pointer_value(env, regno)) { 7439 verbose(env, "R%d leaks addr into helper function\n", 7440 regno); 7441 return -EACCES; 7442 } 7443 return 0; 7444 } 7445 7446 if (type_is_pkt_pointer(type) && 7447 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 7448 verbose(env, "helper access to the packet is not allowed\n"); 7449 return -EACCES; 7450 } 7451 7452 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { 7453 err = resolve_map_arg_type(env, meta, &arg_type); 7454 if (err) 7455 return err; 7456 } 7457 7458 if (register_is_null(reg) && type_may_be_null(arg_type)) 7459 /* A NULL register has a SCALAR_VALUE type, so skip 7460 * type checking. 7461 */ 7462 goto skip_type_check; 7463 7464 /* arg_btf_id and arg_size are in a union. */ 7465 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || 7466 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 7467 arg_btf_id = fn->arg_btf_id[arg]; 7468 7469 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 7470 if (err) 7471 return err; 7472 7473 err = check_func_arg_reg_off(env, reg, regno, arg_type); 7474 if (err) 7475 return err; 7476 7477 skip_type_check: 7478 if (arg_type_is_release(arg_type)) { 7479 if (arg_type_is_dynptr(arg_type)) { 7480 struct bpf_func_state *state = func(env, reg); 7481 int spi; 7482 7483 /* Only dynptr created on stack can be released, thus 7484 * the get_spi and stack state checks for spilled_ptr 7485 * should only be done before process_dynptr_func for 7486 * PTR_TO_STACK. 7487 */ 7488 if (reg->type == PTR_TO_STACK) { 7489 spi = dynptr_get_spi(env, reg); 7490 if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) { 7491 verbose(env, "arg %d is an unacquired reference\n", regno); 7492 return -EINVAL; 7493 } 7494 } else { 7495 verbose(env, "cannot release unowned const bpf_dynptr\n"); 7496 return -EINVAL; 7497 } 7498 } else if (!reg->ref_obj_id && !register_is_null(reg)) { 7499 verbose(env, "R%d must be referenced when passed to release function\n", 7500 regno); 7501 return -EINVAL; 7502 } 7503 if (meta->release_regno) { 7504 verbose(env, "verifier internal error: more than one release argument\n"); 7505 return -EFAULT; 7506 } 7507 meta->release_regno = regno; 7508 } 7509 7510 if (reg->ref_obj_id) { 7511 if (meta->ref_obj_id) { 7512 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 7513 regno, reg->ref_obj_id, 7514 meta->ref_obj_id); 7515 return -EFAULT; 7516 } 7517 meta->ref_obj_id = reg->ref_obj_id; 7518 } 7519 7520 switch (base_type(arg_type)) { 7521 case ARG_CONST_MAP_PTR: 7522 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 7523 if (meta->map_ptr) { 7524 /* Use map_uid (which is unique id of inner map) to reject: 7525 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 7526 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 7527 * if (inner_map1 && inner_map2) { 7528 * timer = bpf_map_lookup_elem(inner_map1); 7529 * if (timer) 7530 * // mismatch would have been allowed 7531 * bpf_timer_init(timer, inner_map2); 7532 * } 7533 * 7534 * Comparing map_ptr is enough to distinguish normal and outer maps. 7535 */ 7536 if (meta->map_ptr != reg->map_ptr || 7537 meta->map_uid != reg->map_uid) { 7538 verbose(env, 7539 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 7540 meta->map_uid, reg->map_uid); 7541 return -EINVAL; 7542 } 7543 } 7544 meta->map_ptr = reg->map_ptr; 7545 meta->map_uid = reg->map_uid; 7546 break; 7547 case ARG_PTR_TO_MAP_KEY: 7548 /* bpf_map_xxx(..., map_ptr, ..., key) call: 7549 * check that [key, key + map->key_size) are within 7550 * stack limits and initialized 7551 */ 7552 if (!meta->map_ptr) { 7553 /* in function declaration map_ptr must come before 7554 * map_key, so that it's verified and known before 7555 * we have to check map_key here. Otherwise it means 7556 * that kernel subsystem misconfigured verifier 7557 */ 7558 verbose(env, "invalid map_ptr to access map->key\n"); 7559 return -EACCES; 7560 } 7561 err = check_helper_mem_access(env, regno, 7562 meta->map_ptr->key_size, false, 7563 NULL); 7564 break; 7565 case ARG_PTR_TO_MAP_VALUE: 7566 if (type_may_be_null(arg_type) && register_is_null(reg)) 7567 return 0; 7568 7569 /* bpf_map_xxx(..., map_ptr, ..., value) call: 7570 * check [value, value + map->value_size) validity 7571 */ 7572 if (!meta->map_ptr) { 7573 /* kernel subsystem misconfigured verifier */ 7574 verbose(env, "invalid map_ptr to access map->value\n"); 7575 return -EACCES; 7576 } 7577 meta->raw_mode = arg_type & MEM_UNINIT; 7578 err = check_helper_mem_access(env, regno, 7579 meta->map_ptr->value_size, false, 7580 meta); 7581 break; 7582 case ARG_PTR_TO_PERCPU_BTF_ID: 7583 if (!reg->btf_id) { 7584 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 7585 return -EACCES; 7586 } 7587 meta->ret_btf = reg->btf; 7588 meta->ret_btf_id = reg->btf_id; 7589 break; 7590 case ARG_PTR_TO_SPIN_LOCK: 7591 if (in_rbtree_lock_required_cb(env)) { 7592 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); 7593 return -EACCES; 7594 } 7595 if (meta->func_id == BPF_FUNC_spin_lock) { 7596 err = process_spin_lock(env, regno, true); 7597 if (err) 7598 return err; 7599 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 7600 err = process_spin_lock(env, regno, false); 7601 if (err) 7602 return err; 7603 } else { 7604 verbose(env, "verifier internal error\n"); 7605 return -EFAULT; 7606 } 7607 break; 7608 case ARG_PTR_TO_TIMER: 7609 err = process_timer_func(env, regno, meta); 7610 if (err) 7611 return err; 7612 break; 7613 case ARG_PTR_TO_FUNC: 7614 meta->subprogno = reg->subprogno; 7615 break; 7616 case ARG_PTR_TO_MEM: 7617 /* The access to this pointer is only checked when we hit the 7618 * next is_mem_size argument below. 7619 */ 7620 meta->raw_mode = arg_type & MEM_UNINIT; 7621 if (arg_type & MEM_FIXED_SIZE) { 7622 err = check_helper_mem_access(env, regno, 7623 fn->arg_size[arg], false, 7624 meta); 7625 } 7626 break; 7627 case ARG_CONST_SIZE: 7628 err = check_mem_size_reg(env, reg, regno, false, meta); 7629 break; 7630 case ARG_CONST_SIZE_OR_ZERO: 7631 err = check_mem_size_reg(env, reg, regno, true, meta); 7632 break; 7633 case ARG_PTR_TO_DYNPTR: 7634 err = process_dynptr_func(env, regno, insn_idx, arg_type); 7635 if (err) 7636 return err; 7637 break; 7638 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 7639 if (!tnum_is_const(reg->var_off)) { 7640 verbose(env, "R%d is not a known constant'\n", 7641 regno); 7642 return -EACCES; 7643 } 7644 meta->mem_size = reg->var_off.value; 7645 err = mark_chain_precision(env, regno); 7646 if (err) 7647 return err; 7648 break; 7649 case ARG_PTR_TO_INT: 7650 case ARG_PTR_TO_LONG: 7651 { 7652 int size = int_ptr_type_to_size(arg_type); 7653 7654 err = check_helper_mem_access(env, regno, size, false, meta); 7655 if (err) 7656 return err; 7657 err = check_ptr_alignment(env, reg, 0, size, true); 7658 break; 7659 } 7660 case ARG_PTR_TO_CONST_STR: 7661 { 7662 struct bpf_map *map = reg->map_ptr; 7663 int map_off; 7664 u64 map_addr; 7665 char *str_ptr; 7666 7667 if (!bpf_map_is_rdonly(map)) { 7668 verbose(env, "R%d does not point to a readonly map'\n", regno); 7669 return -EACCES; 7670 } 7671 7672 if (!tnum_is_const(reg->var_off)) { 7673 verbose(env, "R%d is not a constant address'\n", regno); 7674 return -EACCES; 7675 } 7676 7677 if (!map->ops->map_direct_value_addr) { 7678 verbose(env, "no direct value access support for this map type\n"); 7679 return -EACCES; 7680 } 7681 7682 err = check_map_access(env, regno, reg->off, 7683 map->value_size - reg->off, false, 7684 ACCESS_HELPER); 7685 if (err) 7686 return err; 7687 7688 map_off = reg->off + reg->var_off.value; 7689 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 7690 if (err) { 7691 verbose(env, "direct value access on string failed\n"); 7692 return err; 7693 } 7694 7695 str_ptr = (char *)(long)(map_addr); 7696 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 7697 verbose(env, "string is not zero-terminated\n"); 7698 return -EINVAL; 7699 } 7700 break; 7701 } 7702 case ARG_PTR_TO_KPTR: 7703 err = process_kptr_func(env, regno, meta); 7704 if (err) 7705 return err; 7706 break; 7707 } 7708 7709 return err; 7710 } 7711 7712 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 7713 { 7714 enum bpf_attach_type eatype = env->prog->expected_attach_type; 7715 enum bpf_prog_type type = resolve_prog_type(env->prog); 7716 7717 if (func_id != BPF_FUNC_map_update_elem) 7718 return false; 7719 7720 /* It's not possible to get access to a locked struct sock in these 7721 * contexts, so updating is safe. 7722 */ 7723 switch (type) { 7724 case BPF_PROG_TYPE_TRACING: 7725 if (eatype == BPF_TRACE_ITER) 7726 return true; 7727 break; 7728 case BPF_PROG_TYPE_SOCKET_FILTER: 7729 case BPF_PROG_TYPE_SCHED_CLS: 7730 case BPF_PROG_TYPE_SCHED_ACT: 7731 case BPF_PROG_TYPE_XDP: 7732 case BPF_PROG_TYPE_SK_REUSEPORT: 7733 case BPF_PROG_TYPE_FLOW_DISSECTOR: 7734 case BPF_PROG_TYPE_SK_LOOKUP: 7735 return true; 7736 default: 7737 break; 7738 } 7739 7740 verbose(env, "cannot update sockmap in this context\n"); 7741 return false; 7742 } 7743 7744 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 7745 { 7746 return env->prog->jit_requested && 7747 bpf_jit_supports_subprog_tailcalls(); 7748 } 7749 7750 static int check_map_func_compatibility(struct bpf_verifier_env *env, 7751 struct bpf_map *map, int func_id) 7752 { 7753 if (!map) 7754 return 0; 7755 7756 /* We need a two way check, first is from map perspective ... */ 7757 switch (map->map_type) { 7758 case BPF_MAP_TYPE_PROG_ARRAY: 7759 if (func_id != BPF_FUNC_tail_call) 7760 goto error; 7761 break; 7762 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 7763 if (func_id != BPF_FUNC_perf_event_read && 7764 func_id != BPF_FUNC_perf_event_output && 7765 func_id != BPF_FUNC_skb_output && 7766 func_id != BPF_FUNC_perf_event_read_value && 7767 func_id != BPF_FUNC_xdp_output) 7768 goto error; 7769 break; 7770 case BPF_MAP_TYPE_RINGBUF: 7771 if (func_id != BPF_FUNC_ringbuf_output && 7772 func_id != BPF_FUNC_ringbuf_reserve && 7773 func_id != BPF_FUNC_ringbuf_query && 7774 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 7775 func_id != BPF_FUNC_ringbuf_submit_dynptr && 7776 func_id != BPF_FUNC_ringbuf_discard_dynptr) 7777 goto error; 7778 break; 7779 case BPF_MAP_TYPE_USER_RINGBUF: 7780 if (func_id != BPF_FUNC_user_ringbuf_drain) 7781 goto error; 7782 break; 7783 case BPF_MAP_TYPE_STACK_TRACE: 7784 if (func_id != BPF_FUNC_get_stackid) 7785 goto error; 7786 break; 7787 case BPF_MAP_TYPE_CGROUP_ARRAY: 7788 if (func_id != BPF_FUNC_skb_under_cgroup && 7789 func_id != BPF_FUNC_current_task_under_cgroup) 7790 goto error; 7791 break; 7792 case BPF_MAP_TYPE_CGROUP_STORAGE: 7793 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 7794 if (func_id != BPF_FUNC_get_local_storage) 7795 goto error; 7796 break; 7797 case BPF_MAP_TYPE_DEVMAP: 7798 case BPF_MAP_TYPE_DEVMAP_HASH: 7799 if (func_id != BPF_FUNC_redirect_map && 7800 func_id != BPF_FUNC_map_lookup_elem) 7801 goto error; 7802 break; 7803 /* Restrict bpf side of cpumap and xskmap, open when use-cases 7804 * appear. 7805 */ 7806 case BPF_MAP_TYPE_CPUMAP: 7807 if (func_id != BPF_FUNC_redirect_map) 7808 goto error; 7809 break; 7810 case BPF_MAP_TYPE_XSKMAP: 7811 if (func_id != BPF_FUNC_redirect_map && 7812 func_id != BPF_FUNC_map_lookup_elem) 7813 goto error; 7814 break; 7815 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 7816 case BPF_MAP_TYPE_HASH_OF_MAPS: 7817 if (func_id != BPF_FUNC_map_lookup_elem) 7818 goto error; 7819 break; 7820 case BPF_MAP_TYPE_SOCKMAP: 7821 if (func_id != BPF_FUNC_sk_redirect_map && 7822 func_id != BPF_FUNC_sock_map_update && 7823 func_id != BPF_FUNC_map_delete_elem && 7824 func_id != BPF_FUNC_msg_redirect_map && 7825 func_id != BPF_FUNC_sk_select_reuseport && 7826 func_id != BPF_FUNC_map_lookup_elem && 7827 !may_update_sockmap(env, func_id)) 7828 goto error; 7829 break; 7830 case BPF_MAP_TYPE_SOCKHASH: 7831 if (func_id != BPF_FUNC_sk_redirect_hash && 7832 func_id != BPF_FUNC_sock_hash_update && 7833 func_id != BPF_FUNC_map_delete_elem && 7834 func_id != BPF_FUNC_msg_redirect_hash && 7835 func_id != BPF_FUNC_sk_select_reuseport && 7836 func_id != BPF_FUNC_map_lookup_elem && 7837 !may_update_sockmap(env, func_id)) 7838 goto error; 7839 break; 7840 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 7841 if (func_id != BPF_FUNC_sk_select_reuseport) 7842 goto error; 7843 break; 7844 case BPF_MAP_TYPE_QUEUE: 7845 case BPF_MAP_TYPE_STACK: 7846 if (func_id != BPF_FUNC_map_peek_elem && 7847 func_id != BPF_FUNC_map_pop_elem && 7848 func_id != BPF_FUNC_map_push_elem) 7849 goto error; 7850 break; 7851 case BPF_MAP_TYPE_SK_STORAGE: 7852 if (func_id != BPF_FUNC_sk_storage_get && 7853 func_id != BPF_FUNC_sk_storage_delete && 7854 func_id != BPF_FUNC_kptr_xchg) 7855 goto error; 7856 break; 7857 case BPF_MAP_TYPE_INODE_STORAGE: 7858 if (func_id != BPF_FUNC_inode_storage_get && 7859 func_id != BPF_FUNC_inode_storage_delete && 7860 func_id != BPF_FUNC_kptr_xchg) 7861 goto error; 7862 break; 7863 case BPF_MAP_TYPE_TASK_STORAGE: 7864 if (func_id != BPF_FUNC_task_storage_get && 7865 func_id != BPF_FUNC_task_storage_delete && 7866 func_id != BPF_FUNC_kptr_xchg) 7867 goto error; 7868 break; 7869 case BPF_MAP_TYPE_CGRP_STORAGE: 7870 if (func_id != BPF_FUNC_cgrp_storage_get && 7871 func_id != BPF_FUNC_cgrp_storage_delete && 7872 func_id != BPF_FUNC_kptr_xchg) 7873 goto error; 7874 break; 7875 case BPF_MAP_TYPE_BLOOM_FILTER: 7876 if (func_id != BPF_FUNC_map_peek_elem && 7877 func_id != BPF_FUNC_map_push_elem) 7878 goto error; 7879 break; 7880 default: 7881 break; 7882 } 7883 7884 /* ... and second from the function itself. */ 7885 switch (func_id) { 7886 case BPF_FUNC_tail_call: 7887 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 7888 goto error; 7889 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 7890 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 7891 return -EINVAL; 7892 } 7893 break; 7894 case BPF_FUNC_perf_event_read: 7895 case BPF_FUNC_perf_event_output: 7896 case BPF_FUNC_perf_event_read_value: 7897 case BPF_FUNC_skb_output: 7898 case BPF_FUNC_xdp_output: 7899 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 7900 goto error; 7901 break; 7902 case BPF_FUNC_ringbuf_output: 7903 case BPF_FUNC_ringbuf_reserve: 7904 case BPF_FUNC_ringbuf_query: 7905 case BPF_FUNC_ringbuf_reserve_dynptr: 7906 case BPF_FUNC_ringbuf_submit_dynptr: 7907 case BPF_FUNC_ringbuf_discard_dynptr: 7908 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 7909 goto error; 7910 break; 7911 case BPF_FUNC_user_ringbuf_drain: 7912 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 7913 goto error; 7914 break; 7915 case BPF_FUNC_get_stackid: 7916 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 7917 goto error; 7918 break; 7919 case BPF_FUNC_current_task_under_cgroup: 7920 case BPF_FUNC_skb_under_cgroup: 7921 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 7922 goto error; 7923 break; 7924 case BPF_FUNC_redirect_map: 7925 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 7926 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 7927 map->map_type != BPF_MAP_TYPE_CPUMAP && 7928 map->map_type != BPF_MAP_TYPE_XSKMAP) 7929 goto error; 7930 break; 7931 case BPF_FUNC_sk_redirect_map: 7932 case BPF_FUNC_msg_redirect_map: 7933 case BPF_FUNC_sock_map_update: 7934 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 7935 goto error; 7936 break; 7937 case BPF_FUNC_sk_redirect_hash: 7938 case BPF_FUNC_msg_redirect_hash: 7939 case BPF_FUNC_sock_hash_update: 7940 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 7941 goto error; 7942 break; 7943 case BPF_FUNC_get_local_storage: 7944 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7945 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 7946 goto error; 7947 break; 7948 case BPF_FUNC_sk_select_reuseport: 7949 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 7950 map->map_type != BPF_MAP_TYPE_SOCKMAP && 7951 map->map_type != BPF_MAP_TYPE_SOCKHASH) 7952 goto error; 7953 break; 7954 case BPF_FUNC_map_pop_elem: 7955 if (map->map_type != BPF_MAP_TYPE_QUEUE && 7956 map->map_type != BPF_MAP_TYPE_STACK) 7957 goto error; 7958 break; 7959 case BPF_FUNC_map_peek_elem: 7960 case BPF_FUNC_map_push_elem: 7961 if (map->map_type != BPF_MAP_TYPE_QUEUE && 7962 map->map_type != BPF_MAP_TYPE_STACK && 7963 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 7964 goto error; 7965 break; 7966 case BPF_FUNC_map_lookup_percpu_elem: 7967 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 7968 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 7969 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 7970 goto error; 7971 break; 7972 case BPF_FUNC_sk_storage_get: 7973 case BPF_FUNC_sk_storage_delete: 7974 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 7975 goto error; 7976 break; 7977 case BPF_FUNC_inode_storage_get: 7978 case BPF_FUNC_inode_storage_delete: 7979 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 7980 goto error; 7981 break; 7982 case BPF_FUNC_task_storage_get: 7983 case BPF_FUNC_task_storage_delete: 7984 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 7985 goto error; 7986 break; 7987 case BPF_FUNC_cgrp_storage_get: 7988 case BPF_FUNC_cgrp_storage_delete: 7989 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 7990 goto error; 7991 break; 7992 default: 7993 break; 7994 } 7995 7996 return 0; 7997 error: 7998 verbose(env, "cannot pass map_type %d into func %s#%d\n", 7999 map->map_type, func_id_name(func_id), func_id); 8000 return -EINVAL; 8001 } 8002 8003 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 8004 { 8005 int count = 0; 8006 8007 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 8008 count++; 8009 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 8010 count++; 8011 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 8012 count++; 8013 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 8014 count++; 8015 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 8016 count++; 8017 8018 /* We only support one arg being in raw mode at the moment, 8019 * which is sufficient for the helper functions we have 8020 * right now. 8021 */ 8022 return count <= 1; 8023 } 8024 8025 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 8026 { 8027 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 8028 bool has_size = fn->arg_size[arg] != 0; 8029 bool is_next_size = false; 8030 8031 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 8032 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 8033 8034 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 8035 return is_next_size; 8036 8037 return has_size == is_next_size || is_next_size == is_fixed; 8038 } 8039 8040 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 8041 { 8042 /* bpf_xxx(..., buf, len) call will access 'len' 8043 * bytes from memory 'buf'. Both arg types need 8044 * to be paired, so make sure there's no buggy 8045 * helper function specification. 8046 */ 8047 if (arg_type_is_mem_size(fn->arg1_type) || 8048 check_args_pair_invalid(fn, 0) || 8049 check_args_pair_invalid(fn, 1) || 8050 check_args_pair_invalid(fn, 2) || 8051 check_args_pair_invalid(fn, 3) || 8052 check_args_pair_invalid(fn, 4)) 8053 return false; 8054 8055 return true; 8056 } 8057 8058 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 8059 { 8060 int i; 8061 8062 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 8063 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 8064 return !!fn->arg_btf_id[i]; 8065 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 8066 return fn->arg_btf_id[i] == BPF_PTR_POISON; 8067 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 8068 /* arg_btf_id and arg_size are in a union. */ 8069 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 8070 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 8071 return false; 8072 } 8073 8074 return true; 8075 } 8076 8077 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 8078 { 8079 return check_raw_mode_ok(fn) && 8080 check_arg_pair_ok(fn) && 8081 check_btf_id_ok(fn) ? 0 : -EINVAL; 8082 } 8083 8084 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 8085 * are now invalid, so turn them into unknown SCALAR_VALUE. 8086 * 8087 * This also applies to dynptr slices belonging to skb and xdp dynptrs, 8088 * since these slices point to packet data. 8089 */ 8090 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 8091 { 8092 struct bpf_func_state *state; 8093 struct bpf_reg_state *reg; 8094 8095 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8096 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) 8097 mark_reg_invalid(env, reg); 8098 })); 8099 } 8100 8101 enum { 8102 AT_PKT_END = -1, 8103 BEYOND_PKT_END = -2, 8104 }; 8105 8106 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 8107 { 8108 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 8109 struct bpf_reg_state *reg = &state->regs[regn]; 8110 8111 if (reg->type != PTR_TO_PACKET) 8112 /* PTR_TO_PACKET_META is not supported yet */ 8113 return; 8114 8115 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 8116 * How far beyond pkt_end it goes is unknown. 8117 * if (!range_open) it's the case of pkt >= pkt_end 8118 * if (range_open) it's the case of pkt > pkt_end 8119 * hence this pointer is at least 1 byte bigger than pkt_end 8120 */ 8121 if (range_open) 8122 reg->range = BEYOND_PKT_END; 8123 else 8124 reg->range = AT_PKT_END; 8125 } 8126 8127 /* The pointer with the specified id has released its reference to kernel 8128 * resources. Identify all copies of the same pointer and clear the reference. 8129 */ 8130 static int release_reference(struct bpf_verifier_env *env, 8131 int ref_obj_id) 8132 { 8133 struct bpf_func_state *state; 8134 struct bpf_reg_state *reg; 8135 int err; 8136 8137 err = release_reference_state(cur_func(env), ref_obj_id); 8138 if (err) 8139 return err; 8140 8141 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8142 if (reg->ref_obj_id == ref_obj_id) 8143 mark_reg_invalid(env, reg); 8144 })); 8145 8146 return 0; 8147 } 8148 8149 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) 8150 { 8151 struct bpf_func_state *unused; 8152 struct bpf_reg_state *reg; 8153 8154 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 8155 if (type_is_non_owning_ref(reg->type)) 8156 mark_reg_invalid(env, reg); 8157 })); 8158 } 8159 8160 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 8161 struct bpf_reg_state *regs) 8162 { 8163 int i; 8164 8165 /* after the call registers r0 - r5 were scratched */ 8166 for (i = 0; i < CALLER_SAVED_REGS; i++) { 8167 mark_reg_not_init(env, regs, caller_saved[i]); 8168 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 8169 } 8170 } 8171 8172 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 8173 struct bpf_func_state *caller, 8174 struct bpf_func_state *callee, 8175 int insn_idx); 8176 8177 static int set_callee_state(struct bpf_verifier_env *env, 8178 struct bpf_func_state *caller, 8179 struct bpf_func_state *callee, int insn_idx); 8180 8181 static bool is_callback_calling_kfunc(u32 btf_id); 8182 8183 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8184 int *insn_idx, int subprog, 8185 set_callee_state_fn set_callee_state_cb) 8186 { 8187 struct bpf_verifier_state *state = env->cur_state; 8188 struct bpf_func_info_aux *func_info_aux; 8189 struct bpf_func_state *caller, *callee; 8190 int err; 8191 bool is_global = false; 8192 8193 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 8194 verbose(env, "the call stack of %d frames is too deep\n", 8195 state->curframe + 2); 8196 return -E2BIG; 8197 } 8198 8199 caller = state->frame[state->curframe]; 8200 if (state->frame[state->curframe + 1]) { 8201 verbose(env, "verifier bug. Frame %d already allocated\n", 8202 state->curframe + 1); 8203 return -EFAULT; 8204 } 8205 8206 func_info_aux = env->prog->aux->func_info_aux; 8207 if (func_info_aux) 8208 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 8209 err = btf_check_subprog_call(env, subprog, caller->regs); 8210 if (err == -EFAULT) 8211 return err; 8212 if (is_global) { 8213 if (err) { 8214 verbose(env, "Caller passes invalid args into func#%d\n", 8215 subprog); 8216 return err; 8217 } else { 8218 if (env->log.level & BPF_LOG_LEVEL) 8219 verbose(env, 8220 "Func#%d is global and valid. Skipping.\n", 8221 subprog); 8222 clear_caller_saved_regs(env, caller->regs); 8223 8224 /* All global functions return a 64-bit SCALAR_VALUE */ 8225 mark_reg_unknown(env, caller->regs, BPF_REG_0); 8226 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 8227 8228 /* continue with next insn after call */ 8229 return 0; 8230 } 8231 } 8232 8233 /* set_callee_state is used for direct subprog calls, but we are 8234 * interested in validating only BPF helpers that can call subprogs as 8235 * callbacks 8236 */ 8237 if (set_callee_state_cb != set_callee_state) { 8238 if (bpf_pseudo_kfunc_call(insn) && 8239 !is_callback_calling_kfunc(insn->imm)) { 8240 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", 8241 func_id_name(insn->imm), insn->imm); 8242 return -EFAULT; 8243 } else if (!bpf_pseudo_kfunc_call(insn) && 8244 !is_callback_calling_function(insn->imm)) { /* helper */ 8245 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", 8246 func_id_name(insn->imm), insn->imm); 8247 return -EFAULT; 8248 } 8249 } 8250 8251 if (insn->code == (BPF_JMP | BPF_CALL) && 8252 insn->src_reg == 0 && 8253 insn->imm == BPF_FUNC_timer_set_callback) { 8254 struct bpf_verifier_state *async_cb; 8255 8256 /* there is no real recursion here. timer callbacks are async */ 8257 env->subprog_info[subprog].is_async_cb = true; 8258 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 8259 *insn_idx, subprog); 8260 if (!async_cb) 8261 return -EFAULT; 8262 callee = async_cb->frame[0]; 8263 callee->async_entry_cnt = caller->async_entry_cnt + 1; 8264 8265 /* Convert bpf_timer_set_callback() args into timer callback args */ 8266 err = set_callee_state_cb(env, caller, callee, *insn_idx); 8267 if (err) 8268 return err; 8269 8270 clear_caller_saved_regs(env, caller->regs); 8271 mark_reg_unknown(env, caller->regs, BPF_REG_0); 8272 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 8273 /* continue with next insn after call */ 8274 return 0; 8275 } 8276 8277 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 8278 if (!callee) 8279 return -ENOMEM; 8280 state->frame[state->curframe + 1] = callee; 8281 8282 /* callee cannot access r0, r6 - r9 for reading and has to write 8283 * into its own stack before reading from it. 8284 * callee can read/write into caller's stack 8285 */ 8286 init_func_state(env, callee, 8287 /* remember the callsite, it will be used by bpf_exit */ 8288 *insn_idx /* callsite */, 8289 state->curframe + 1 /* frameno within this callchain */, 8290 subprog /* subprog number within this prog */); 8291 8292 /* Transfer references to the callee */ 8293 err = copy_reference_state(callee, caller); 8294 if (err) 8295 goto err_out; 8296 8297 err = set_callee_state_cb(env, caller, callee, *insn_idx); 8298 if (err) 8299 goto err_out; 8300 8301 clear_caller_saved_regs(env, caller->regs); 8302 8303 /* only increment it after check_reg_arg() finished */ 8304 state->curframe++; 8305 8306 /* and go analyze first insn of the callee */ 8307 *insn_idx = env->subprog_info[subprog].start - 1; 8308 8309 if (env->log.level & BPF_LOG_LEVEL) { 8310 verbose(env, "caller:\n"); 8311 print_verifier_state(env, caller, true); 8312 verbose(env, "callee:\n"); 8313 print_verifier_state(env, callee, true); 8314 } 8315 return 0; 8316 8317 err_out: 8318 free_func_state(callee); 8319 state->frame[state->curframe + 1] = NULL; 8320 return err; 8321 } 8322 8323 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 8324 struct bpf_func_state *caller, 8325 struct bpf_func_state *callee) 8326 { 8327 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 8328 * void *callback_ctx, u64 flags); 8329 * callback_fn(struct bpf_map *map, void *key, void *value, 8330 * void *callback_ctx); 8331 */ 8332 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 8333 8334 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 8335 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 8336 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 8337 8338 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 8339 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 8340 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 8341 8342 /* pointer to stack or null */ 8343 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 8344 8345 /* unused */ 8346 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8347 return 0; 8348 } 8349 8350 static int set_callee_state(struct bpf_verifier_env *env, 8351 struct bpf_func_state *caller, 8352 struct bpf_func_state *callee, int insn_idx) 8353 { 8354 int i; 8355 8356 /* copy r1 - r5 args that callee can access. The copy includes parent 8357 * pointers, which connects us up to the liveness chain 8358 */ 8359 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 8360 callee->regs[i] = caller->regs[i]; 8361 return 0; 8362 } 8363 8364 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8365 int *insn_idx) 8366 { 8367 int subprog, target_insn; 8368 8369 target_insn = *insn_idx + insn->imm + 1; 8370 subprog = find_subprog(env, target_insn); 8371 if (subprog < 0) { 8372 verbose(env, "verifier bug. No program starts at insn %d\n", 8373 target_insn); 8374 return -EFAULT; 8375 } 8376 8377 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 8378 } 8379 8380 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 8381 struct bpf_func_state *caller, 8382 struct bpf_func_state *callee, 8383 int insn_idx) 8384 { 8385 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 8386 struct bpf_map *map; 8387 int err; 8388 8389 if (bpf_map_ptr_poisoned(insn_aux)) { 8390 verbose(env, "tail_call abusing map_ptr\n"); 8391 return -EINVAL; 8392 } 8393 8394 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 8395 if (!map->ops->map_set_for_each_callback_args || 8396 !map->ops->map_for_each_callback) { 8397 verbose(env, "callback function not allowed for map\n"); 8398 return -ENOTSUPP; 8399 } 8400 8401 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 8402 if (err) 8403 return err; 8404 8405 callee->in_callback_fn = true; 8406 callee->callback_ret_range = tnum_range(0, 1); 8407 return 0; 8408 } 8409 8410 static int set_loop_callback_state(struct bpf_verifier_env *env, 8411 struct bpf_func_state *caller, 8412 struct bpf_func_state *callee, 8413 int insn_idx) 8414 { 8415 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 8416 * u64 flags); 8417 * callback_fn(u32 index, void *callback_ctx); 8418 */ 8419 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 8420 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 8421 8422 /* unused */ 8423 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 8424 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 8425 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8426 8427 callee->in_callback_fn = true; 8428 callee->callback_ret_range = tnum_range(0, 1); 8429 return 0; 8430 } 8431 8432 static int set_timer_callback_state(struct bpf_verifier_env *env, 8433 struct bpf_func_state *caller, 8434 struct bpf_func_state *callee, 8435 int insn_idx) 8436 { 8437 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 8438 8439 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 8440 * callback_fn(struct bpf_map *map, void *key, void *value); 8441 */ 8442 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 8443 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 8444 callee->regs[BPF_REG_1].map_ptr = map_ptr; 8445 8446 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 8447 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 8448 callee->regs[BPF_REG_2].map_ptr = map_ptr; 8449 8450 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 8451 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 8452 callee->regs[BPF_REG_3].map_ptr = map_ptr; 8453 8454 /* unused */ 8455 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 8456 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8457 callee->in_async_callback_fn = true; 8458 callee->callback_ret_range = tnum_range(0, 1); 8459 return 0; 8460 } 8461 8462 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 8463 struct bpf_func_state *caller, 8464 struct bpf_func_state *callee, 8465 int insn_idx) 8466 { 8467 /* bpf_find_vma(struct task_struct *task, u64 addr, 8468 * void *callback_fn, void *callback_ctx, u64 flags) 8469 * (callback_fn)(struct task_struct *task, 8470 * struct vm_area_struct *vma, void *callback_ctx); 8471 */ 8472 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 8473 8474 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 8475 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 8476 callee->regs[BPF_REG_2].btf = btf_vmlinux; 8477 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], 8478 8479 /* pointer to stack or null */ 8480 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 8481 8482 /* unused */ 8483 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 8484 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8485 callee->in_callback_fn = true; 8486 callee->callback_ret_range = tnum_range(0, 1); 8487 return 0; 8488 } 8489 8490 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 8491 struct bpf_func_state *caller, 8492 struct bpf_func_state *callee, 8493 int insn_idx) 8494 { 8495 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 8496 * callback_ctx, u64 flags); 8497 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx); 8498 */ 8499 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 8500 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); 8501 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 8502 8503 /* unused */ 8504 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 8505 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 8506 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8507 8508 callee->in_callback_fn = true; 8509 callee->callback_ret_range = tnum_range(0, 1); 8510 return 0; 8511 } 8512 8513 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, 8514 struct bpf_func_state *caller, 8515 struct bpf_func_state *callee, 8516 int insn_idx) 8517 { 8518 /* void bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 8519 * bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)); 8520 * 8521 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add_impl is the same PTR_TO_BTF_ID w/ offset 8522 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd 8523 * by this point, so look at 'root' 8524 */ 8525 struct btf_field *field; 8526 8527 field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off, 8528 BPF_RB_ROOT); 8529 if (!field || !field->graph_root.value_btf_id) 8530 return -EFAULT; 8531 8532 mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root); 8533 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); 8534 mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root); 8535 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); 8536 8537 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 8538 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 8539 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 8540 callee->in_callback_fn = true; 8541 callee->callback_ret_range = tnum_range(0, 1); 8542 return 0; 8543 } 8544 8545 static bool is_rbtree_lock_required_kfunc(u32 btf_id); 8546 8547 /* Are we currently verifying the callback for a rbtree helper that must 8548 * be called with lock held? If so, no need to complain about unreleased 8549 * lock 8550 */ 8551 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) 8552 { 8553 struct bpf_verifier_state *state = env->cur_state; 8554 struct bpf_insn *insn = env->prog->insnsi; 8555 struct bpf_func_state *callee; 8556 int kfunc_btf_id; 8557 8558 if (!state->curframe) 8559 return false; 8560 8561 callee = state->frame[state->curframe]; 8562 8563 if (!callee->in_callback_fn) 8564 return false; 8565 8566 kfunc_btf_id = insn[callee->callsite].imm; 8567 return is_rbtree_lock_required_kfunc(kfunc_btf_id); 8568 } 8569 8570 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 8571 { 8572 struct bpf_verifier_state *state = env->cur_state; 8573 struct bpf_func_state *caller, *callee; 8574 struct bpf_reg_state *r0; 8575 int err; 8576 8577 callee = state->frame[state->curframe]; 8578 r0 = &callee->regs[BPF_REG_0]; 8579 if (r0->type == PTR_TO_STACK) { 8580 /* technically it's ok to return caller's stack pointer 8581 * (or caller's caller's pointer) back to the caller, 8582 * since these pointers are valid. Only current stack 8583 * pointer will be invalid as soon as function exits, 8584 * but let's be conservative 8585 */ 8586 verbose(env, "cannot return stack pointer to the caller\n"); 8587 return -EINVAL; 8588 } 8589 8590 caller = state->frame[state->curframe - 1]; 8591 if (callee->in_callback_fn) { 8592 /* enforce R0 return value range [0, 1]. */ 8593 struct tnum range = callee->callback_ret_range; 8594 8595 if (r0->type != SCALAR_VALUE) { 8596 verbose(env, "R0 not a scalar value\n"); 8597 return -EACCES; 8598 } 8599 if (!tnum_in(range, r0->var_off)) { 8600 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 8601 return -EINVAL; 8602 } 8603 } else { 8604 /* return to the caller whatever r0 had in the callee */ 8605 caller->regs[BPF_REG_0] = *r0; 8606 } 8607 8608 /* callback_fn frame should have released its own additions to parent's 8609 * reference state at this point, or check_reference_leak would 8610 * complain, hence it must be the same as the caller. There is no need 8611 * to copy it back. 8612 */ 8613 if (!callee->in_callback_fn) { 8614 /* Transfer references to the caller */ 8615 err = copy_reference_state(caller, callee); 8616 if (err) 8617 return err; 8618 } 8619 8620 *insn_idx = callee->callsite + 1; 8621 if (env->log.level & BPF_LOG_LEVEL) { 8622 verbose(env, "returning from callee:\n"); 8623 print_verifier_state(env, callee, true); 8624 verbose(env, "to caller at %d:\n", *insn_idx); 8625 print_verifier_state(env, caller, true); 8626 } 8627 /* clear everything in the callee */ 8628 free_func_state(callee); 8629 state->frame[state->curframe--] = NULL; 8630 return 0; 8631 } 8632 8633 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 8634 int func_id, 8635 struct bpf_call_arg_meta *meta) 8636 { 8637 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 8638 8639 if (ret_type != RET_INTEGER || 8640 (func_id != BPF_FUNC_get_stack && 8641 func_id != BPF_FUNC_get_task_stack && 8642 func_id != BPF_FUNC_probe_read_str && 8643 func_id != BPF_FUNC_probe_read_kernel_str && 8644 func_id != BPF_FUNC_probe_read_user_str)) 8645 return; 8646 8647 ret_reg->smax_value = meta->msize_max_value; 8648 ret_reg->s32_max_value = meta->msize_max_value; 8649 ret_reg->smin_value = -MAX_ERRNO; 8650 ret_reg->s32_min_value = -MAX_ERRNO; 8651 reg_bounds_sync(ret_reg); 8652 } 8653 8654 static int 8655 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 8656 int func_id, int insn_idx) 8657 { 8658 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 8659 struct bpf_map *map = meta->map_ptr; 8660 8661 if (func_id != BPF_FUNC_tail_call && 8662 func_id != BPF_FUNC_map_lookup_elem && 8663 func_id != BPF_FUNC_map_update_elem && 8664 func_id != BPF_FUNC_map_delete_elem && 8665 func_id != BPF_FUNC_map_push_elem && 8666 func_id != BPF_FUNC_map_pop_elem && 8667 func_id != BPF_FUNC_map_peek_elem && 8668 func_id != BPF_FUNC_for_each_map_elem && 8669 func_id != BPF_FUNC_redirect_map && 8670 func_id != BPF_FUNC_map_lookup_percpu_elem) 8671 return 0; 8672 8673 if (map == NULL) { 8674 verbose(env, "kernel subsystem misconfigured verifier\n"); 8675 return -EINVAL; 8676 } 8677 8678 /* In case of read-only, some additional restrictions 8679 * need to be applied in order to prevent altering the 8680 * state of the map from program side. 8681 */ 8682 if ((map->map_flags & BPF_F_RDONLY_PROG) && 8683 (func_id == BPF_FUNC_map_delete_elem || 8684 func_id == BPF_FUNC_map_update_elem || 8685 func_id == BPF_FUNC_map_push_elem || 8686 func_id == BPF_FUNC_map_pop_elem)) { 8687 verbose(env, "write into map forbidden\n"); 8688 return -EACCES; 8689 } 8690 8691 if (!BPF_MAP_PTR(aux->map_ptr_state)) 8692 bpf_map_ptr_store(aux, meta->map_ptr, 8693 !meta->map_ptr->bypass_spec_v1); 8694 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 8695 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 8696 !meta->map_ptr->bypass_spec_v1); 8697 return 0; 8698 } 8699 8700 static int 8701 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 8702 int func_id, int insn_idx) 8703 { 8704 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 8705 struct bpf_reg_state *regs = cur_regs(env), *reg; 8706 struct bpf_map *map = meta->map_ptr; 8707 u64 val, max; 8708 int err; 8709 8710 if (func_id != BPF_FUNC_tail_call) 8711 return 0; 8712 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 8713 verbose(env, "kernel subsystem misconfigured verifier\n"); 8714 return -EINVAL; 8715 } 8716 8717 reg = ®s[BPF_REG_3]; 8718 val = reg->var_off.value; 8719 max = map->max_entries; 8720 8721 if (!(register_is_const(reg) && val < max)) { 8722 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 8723 return 0; 8724 } 8725 8726 err = mark_chain_precision(env, BPF_REG_3); 8727 if (err) 8728 return err; 8729 if (bpf_map_key_unseen(aux)) 8730 bpf_map_key_store(aux, val); 8731 else if (!bpf_map_key_poisoned(aux) && 8732 bpf_map_key_immediate(aux) != val) 8733 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 8734 return 0; 8735 } 8736 8737 static int check_reference_leak(struct bpf_verifier_env *env) 8738 { 8739 struct bpf_func_state *state = cur_func(env); 8740 bool refs_lingering = false; 8741 int i; 8742 8743 if (state->frameno && !state->in_callback_fn) 8744 return 0; 8745 8746 for (i = 0; i < state->acquired_refs; i++) { 8747 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 8748 continue; 8749 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 8750 state->refs[i].id, state->refs[i].insn_idx); 8751 refs_lingering = true; 8752 } 8753 return refs_lingering ? -EINVAL : 0; 8754 } 8755 8756 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 8757 struct bpf_reg_state *regs) 8758 { 8759 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 8760 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 8761 struct bpf_map *fmt_map = fmt_reg->map_ptr; 8762 struct bpf_bprintf_data data = {}; 8763 int err, fmt_map_off, num_args; 8764 u64 fmt_addr; 8765 char *fmt; 8766 8767 /* data must be an array of u64 */ 8768 if (data_len_reg->var_off.value % 8) 8769 return -EINVAL; 8770 num_args = data_len_reg->var_off.value / 8; 8771 8772 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 8773 * and map_direct_value_addr is set. 8774 */ 8775 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 8776 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 8777 fmt_map_off); 8778 if (err) { 8779 verbose(env, "verifier bug\n"); 8780 return -EFAULT; 8781 } 8782 fmt = (char *)(long)fmt_addr + fmt_map_off; 8783 8784 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 8785 * can focus on validating the format specifiers. 8786 */ 8787 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data); 8788 if (err < 0) 8789 verbose(env, "Invalid format string\n"); 8790 8791 return err; 8792 } 8793 8794 static int check_get_func_ip(struct bpf_verifier_env *env) 8795 { 8796 enum bpf_prog_type type = resolve_prog_type(env->prog); 8797 int func_id = BPF_FUNC_get_func_ip; 8798 8799 if (type == BPF_PROG_TYPE_TRACING) { 8800 if (!bpf_prog_has_trampoline(env->prog)) { 8801 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 8802 func_id_name(func_id), func_id); 8803 return -ENOTSUPP; 8804 } 8805 return 0; 8806 } else if (type == BPF_PROG_TYPE_KPROBE) { 8807 return 0; 8808 } 8809 8810 verbose(env, "func %s#%d not supported for program type %d\n", 8811 func_id_name(func_id), func_id, type); 8812 return -ENOTSUPP; 8813 } 8814 8815 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 8816 { 8817 return &env->insn_aux_data[env->insn_idx]; 8818 } 8819 8820 static bool loop_flag_is_zero(struct bpf_verifier_env *env) 8821 { 8822 struct bpf_reg_state *regs = cur_regs(env); 8823 struct bpf_reg_state *reg = ®s[BPF_REG_4]; 8824 bool reg_is_null = register_is_null(reg); 8825 8826 if (reg_is_null) 8827 mark_chain_precision(env, BPF_REG_4); 8828 8829 return reg_is_null; 8830 } 8831 8832 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 8833 { 8834 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 8835 8836 if (!state->initialized) { 8837 state->initialized = 1; 8838 state->fit_for_inline = loop_flag_is_zero(env); 8839 state->callback_subprogno = subprogno; 8840 return; 8841 } 8842 8843 if (!state->fit_for_inline) 8844 return; 8845 8846 state->fit_for_inline = (loop_flag_is_zero(env) && 8847 state->callback_subprogno == subprogno); 8848 } 8849 8850 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8851 int *insn_idx_p) 8852 { 8853 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 8854 const struct bpf_func_proto *fn = NULL; 8855 enum bpf_return_type ret_type; 8856 enum bpf_type_flag ret_flag; 8857 struct bpf_reg_state *regs; 8858 struct bpf_call_arg_meta meta; 8859 int insn_idx = *insn_idx_p; 8860 bool changes_data; 8861 int i, err, func_id; 8862 8863 /* find function prototype */ 8864 func_id = insn->imm; 8865 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 8866 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 8867 func_id); 8868 return -EINVAL; 8869 } 8870 8871 if (env->ops->get_func_proto) 8872 fn = env->ops->get_func_proto(func_id, env->prog); 8873 if (!fn) { 8874 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 8875 func_id); 8876 return -EINVAL; 8877 } 8878 8879 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 8880 if (!env->prog->gpl_compatible && fn->gpl_only) { 8881 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 8882 return -EINVAL; 8883 } 8884 8885 if (fn->allowed && !fn->allowed(env->prog)) { 8886 verbose(env, "helper call is not allowed in probe\n"); 8887 return -EINVAL; 8888 } 8889 8890 if (!env->prog->aux->sleepable && fn->might_sleep) { 8891 verbose(env, "helper call might sleep in a non-sleepable prog\n"); 8892 return -EINVAL; 8893 } 8894 8895 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 8896 changes_data = bpf_helper_changes_pkt_data(fn->func); 8897 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 8898 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 8899 func_id_name(func_id), func_id); 8900 return -EINVAL; 8901 } 8902 8903 memset(&meta, 0, sizeof(meta)); 8904 meta.pkt_access = fn->pkt_access; 8905 8906 err = check_func_proto(fn, func_id); 8907 if (err) { 8908 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 8909 func_id_name(func_id), func_id); 8910 return err; 8911 } 8912 8913 if (env->cur_state->active_rcu_lock) { 8914 if (fn->might_sleep) { 8915 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", 8916 func_id_name(func_id), func_id); 8917 return -EINVAL; 8918 } 8919 8920 if (env->prog->aux->sleepable && is_storage_get_function(func_id)) 8921 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 8922 } 8923 8924 meta.func_id = func_id; 8925 /* check args */ 8926 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 8927 err = check_func_arg(env, i, &meta, fn, insn_idx); 8928 if (err) 8929 return err; 8930 } 8931 8932 err = record_func_map(env, &meta, func_id, insn_idx); 8933 if (err) 8934 return err; 8935 8936 err = record_func_key(env, &meta, func_id, insn_idx); 8937 if (err) 8938 return err; 8939 8940 /* Mark slots with STACK_MISC in case of raw mode, stack offset 8941 * is inferred from register state. 8942 */ 8943 for (i = 0; i < meta.access_size; i++) { 8944 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 8945 BPF_WRITE, -1, false); 8946 if (err) 8947 return err; 8948 } 8949 8950 regs = cur_regs(env); 8951 8952 if (meta.release_regno) { 8953 err = -EINVAL; 8954 /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot 8955 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr 8956 * is safe to do directly. 8957 */ 8958 if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { 8959 if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) { 8960 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); 8961 return -EFAULT; 8962 } 8963 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); 8964 } else if (meta.ref_obj_id) { 8965 err = release_reference(env, meta.ref_obj_id); 8966 } else if (register_is_null(®s[meta.release_regno])) { 8967 /* meta.ref_obj_id can only be 0 if register that is meant to be 8968 * released is NULL, which must be > R0. 8969 */ 8970 err = 0; 8971 } 8972 if (err) { 8973 verbose(env, "func %s#%d reference has not been acquired before\n", 8974 func_id_name(func_id), func_id); 8975 return err; 8976 } 8977 } 8978 8979 switch (func_id) { 8980 case BPF_FUNC_tail_call: 8981 err = check_reference_leak(env); 8982 if (err) { 8983 verbose(env, "tail_call would lead to reference leak\n"); 8984 return err; 8985 } 8986 break; 8987 case BPF_FUNC_get_local_storage: 8988 /* check that flags argument in get_local_storage(map, flags) is 0, 8989 * this is required because get_local_storage() can't return an error. 8990 */ 8991 if (!register_is_null(®s[BPF_REG_2])) { 8992 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 8993 return -EINVAL; 8994 } 8995 break; 8996 case BPF_FUNC_for_each_map_elem: 8997 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8998 set_map_elem_callback_state); 8999 break; 9000 case BPF_FUNC_timer_set_callback: 9001 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 9002 set_timer_callback_state); 9003 break; 9004 case BPF_FUNC_find_vma: 9005 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 9006 set_find_vma_callback_state); 9007 break; 9008 case BPF_FUNC_snprintf: 9009 err = check_bpf_snprintf_call(env, regs); 9010 break; 9011 case BPF_FUNC_loop: 9012 update_loop_inline_state(env, meta.subprogno); 9013 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 9014 set_loop_callback_state); 9015 break; 9016 case BPF_FUNC_dynptr_from_mem: 9017 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 9018 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 9019 reg_type_str(env, regs[BPF_REG_1].type)); 9020 return -EACCES; 9021 } 9022 break; 9023 case BPF_FUNC_set_retval: 9024 if (prog_type == BPF_PROG_TYPE_LSM && 9025 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 9026 if (!env->prog->aux->attach_func_proto->type) { 9027 /* Make sure programs that attach to void 9028 * hooks don't try to modify return value. 9029 */ 9030 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 9031 return -EINVAL; 9032 } 9033 } 9034 break; 9035 case BPF_FUNC_dynptr_data: 9036 { 9037 struct bpf_reg_state *reg; 9038 int id, ref_obj_id; 9039 9040 reg = get_dynptr_arg_reg(env, fn, regs); 9041 if (!reg) 9042 return -EFAULT; 9043 9044 9045 if (meta.dynptr_id) { 9046 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); 9047 return -EFAULT; 9048 } 9049 if (meta.ref_obj_id) { 9050 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); 9051 return -EFAULT; 9052 } 9053 9054 id = dynptr_id(env, reg); 9055 if (id < 0) { 9056 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 9057 return id; 9058 } 9059 9060 ref_obj_id = dynptr_ref_obj_id(env, reg); 9061 if (ref_obj_id < 0) { 9062 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); 9063 return ref_obj_id; 9064 } 9065 9066 meta.dynptr_id = id; 9067 meta.ref_obj_id = ref_obj_id; 9068 9069 break; 9070 } 9071 case BPF_FUNC_dynptr_write: 9072 { 9073 enum bpf_dynptr_type dynptr_type; 9074 struct bpf_reg_state *reg; 9075 9076 reg = get_dynptr_arg_reg(env, fn, regs); 9077 if (!reg) 9078 return -EFAULT; 9079 9080 dynptr_type = dynptr_get_type(env, reg); 9081 if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) 9082 return -EFAULT; 9083 9084 if (dynptr_type == BPF_DYNPTR_TYPE_SKB) 9085 /* this will trigger clear_all_pkt_pointers(), which will 9086 * invalidate all dynptr slices associated with the skb 9087 */ 9088 changes_data = true; 9089 9090 break; 9091 } 9092 case BPF_FUNC_user_ringbuf_drain: 9093 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 9094 set_user_ringbuf_callback_state); 9095 break; 9096 } 9097 9098 if (err) 9099 return err; 9100 9101 /* reset caller saved regs */ 9102 for (i = 0; i < CALLER_SAVED_REGS; i++) { 9103 mark_reg_not_init(env, regs, caller_saved[i]); 9104 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 9105 } 9106 9107 /* helper call returns 64-bit value. */ 9108 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 9109 9110 /* update return register (already marked as written above) */ 9111 ret_type = fn->ret_type; 9112 ret_flag = type_flag(ret_type); 9113 9114 switch (base_type(ret_type)) { 9115 case RET_INTEGER: 9116 /* sets type to SCALAR_VALUE */ 9117 mark_reg_unknown(env, regs, BPF_REG_0); 9118 break; 9119 case RET_VOID: 9120 regs[BPF_REG_0].type = NOT_INIT; 9121 break; 9122 case RET_PTR_TO_MAP_VALUE: 9123 /* There is no offset yet applied, variable or fixed */ 9124 mark_reg_known_zero(env, regs, BPF_REG_0); 9125 /* remember map_ptr, so that check_map_access() 9126 * can check 'value_size' boundary of memory access 9127 * to map element returned from bpf_map_lookup_elem() 9128 */ 9129 if (meta.map_ptr == NULL) { 9130 verbose(env, 9131 "kernel subsystem misconfigured verifier\n"); 9132 return -EINVAL; 9133 } 9134 regs[BPF_REG_0].map_ptr = meta.map_ptr; 9135 regs[BPF_REG_0].map_uid = meta.map_uid; 9136 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 9137 if (!type_may_be_null(ret_type) && 9138 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 9139 regs[BPF_REG_0].id = ++env->id_gen; 9140 } 9141 break; 9142 case RET_PTR_TO_SOCKET: 9143 mark_reg_known_zero(env, regs, BPF_REG_0); 9144 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 9145 break; 9146 case RET_PTR_TO_SOCK_COMMON: 9147 mark_reg_known_zero(env, regs, BPF_REG_0); 9148 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 9149 break; 9150 case RET_PTR_TO_TCP_SOCK: 9151 mark_reg_known_zero(env, regs, BPF_REG_0); 9152 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 9153 break; 9154 case RET_PTR_TO_MEM: 9155 mark_reg_known_zero(env, regs, BPF_REG_0); 9156 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 9157 regs[BPF_REG_0].mem_size = meta.mem_size; 9158 break; 9159 case RET_PTR_TO_MEM_OR_BTF_ID: 9160 { 9161 const struct btf_type *t; 9162 9163 mark_reg_known_zero(env, regs, BPF_REG_0); 9164 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 9165 if (!btf_type_is_struct(t)) { 9166 u32 tsize; 9167 const struct btf_type *ret; 9168 const char *tname; 9169 9170 /* resolve the type size of ksym. */ 9171 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 9172 if (IS_ERR(ret)) { 9173 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 9174 verbose(env, "unable to resolve the size of type '%s': %ld\n", 9175 tname, PTR_ERR(ret)); 9176 return -EINVAL; 9177 } 9178 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 9179 regs[BPF_REG_0].mem_size = tsize; 9180 } else { 9181 /* MEM_RDONLY may be carried from ret_flag, but it 9182 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 9183 * it will confuse the check of PTR_TO_BTF_ID in 9184 * check_mem_access(). 9185 */ 9186 ret_flag &= ~MEM_RDONLY; 9187 9188 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 9189 regs[BPF_REG_0].btf = meta.ret_btf; 9190 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 9191 } 9192 break; 9193 } 9194 case RET_PTR_TO_BTF_ID: 9195 { 9196 struct btf *ret_btf; 9197 int ret_btf_id; 9198 9199 mark_reg_known_zero(env, regs, BPF_REG_0); 9200 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 9201 if (func_id == BPF_FUNC_kptr_xchg) { 9202 ret_btf = meta.kptr_field->kptr.btf; 9203 ret_btf_id = meta.kptr_field->kptr.btf_id; 9204 if (!btf_is_kernel(ret_btf)) 9205 regs[BPF_REG_0].type |= MEM_ALLOC; 9206 } else { 9207 if (fn->ret_btf_id == BPF_PTR_POISON) { 9208 verbose(env, "verifier internal error:"); 9209 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", 9210 func_id_name(func_id)); 9211 return -EINVAL; 9212 } 9213 ret_btf = btf_vmlinux; 9214 ret_btf_id = *fn->ret_btf_id; 9215 } 9216 if (ret_btf_id == 0) { 9217 verbose(env, "invalid return type %u of func %s#%d\n", 9218 base_type(ret_type), func_id_name(func_id), 9219 func_id); 9220 return -EINVAL; 9221 } 9222 regs[BPF_REG_0].btf = ret_btf; 9223 regs[BPF_REG_0].btf_id = ret_btf_id; 9224 break; 9225 } 9226 default: 9227 verbose(env, "unknown return type %u of func %s#%d\n", 9228 base_type(ret_type), func_id_name(func_id), func_id); 9229 return -EINVAL; 9230 } 9231 9232 if (type_may_be_null(regs[BPF_REG_0].type)) 9233 regs[BPF_REG_0].id = ++env->id_gen; 9234 9235 if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { 9236 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", 9237 func_id_name(func_id), func_id); 9238 return -EFAULT; 9239 } 9240 9241 if (is_dynptr_ref_function(func_id)) 9242 regs[BPF_REG_0].dynptr_id = meta.dynptr_id; 9243 9244 if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { 9245 /* For release_reference() */ 9246 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 9247 } else if (is_acquire_function(func_id, meta.map_ptr)) { 9248 int id = acquire_reference_state(env, insn_idx); 9249 9250 if (id < 0) 9251 return id; 9252 /* For mark_ptr_or_null_reg() */ 9253 regs[BPF_REG_0].id = id; 9254 /* For release_reference() */ 9255 regs[BPF_REG_0].ref_obj_id = id; 9256 } 9257 9258 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 9259 9260 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 9261 if (err) 9262 return err; 9263 9264 if ((func_id == BPF_FUNC_get_stack || 9265 func_id == BPF_FUNC_get_task_stack) && 9266 !env->prog->has_callchain_buf) { 9267 const char *err_str; 9268 9269 #ifdef CONFIG_PERF_EVENTS 9270 err = get_callchain_buffers(sysctl_perf_event_max_stack); 9271 err_str = "cannot get callchain buffer for func %s#%d\n"; 9272 #else 9273 err = -ENOTSUPP; 9274 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 9275 #endif 9276 if (err) { 9277 verbose(env, err_str, func_id_name(func_id), func_id); 9278 return err; 9279 } 9280 9281 env->prog->has_callchain_buf = true; 9282 } 9283 9284 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 9285 env->prog->call_get_stack = true; 9286 9287 if (func_id == BPF_FUNC_get_func_ip) { 9288 if (check_get_func_ip(env)) 9289 return -ENOTSUPP; 9290 env->prog->call_get_func_ip = true; 9291 } 9292 9293 if (changes_data) 9294 clear_all_pkt_pointers(env); 9295 return 0; 9296 } 9297 9298 /* mark_btf_func_reg_size() is used when the reg size is determined by 9299 * the BTF func_proto's return value size and argument. 9300 */ 9301 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 9302 size_t reg_size) 9303 { 9304 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 9305 9306 if (regno == BPF_REG_0) { 9307 /* Function return value */ 9308 reg->live |= REG_LIVE_WRITTEN; 9309 reg->subreg_def = reg_size == sizeof(u64) ? 9310 DEF_NOT_SUBREG : env->insn_idx + 1; 9311 } else { 9312 /* Function argument */ 9313 if (reg_size == sizeof(u64)) { 9314 mark_insn_zext(env, reg); 9315 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 9316 } else { 9317 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 9318 } 9319 } 9320 } 9321 9322 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) 9323 { 9324 return meta->kfunc_flags & KF_ACQUIRE; 9325 } 9326 9327 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 9328 { 9329 return meta->kfunc_flags & KF_RET_NULL; 9330 } 9331 9332 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) 9333 { 9334 return meta->kfunc_flags & KF_RELEASE; 9335 } 9336 9337 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta) 9338 { 9339 return (meta->kfunc_flags & KF_TRUSTED_ARGS) || is_kfunc_release(meta); 9340 } 9341 9342 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) 9343 { 9344 return meta->kfunc_flags & KF_SLEEPABLE; 9345 } 9346 9347 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) 9348 { 9349 return meta->kfunc_flags & KF_DESTRUCTIVE; 9350 } 9351 9352 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) 9353 { 9354 return meta->kfunc_flags & KF_RCU; 9355 } 9356 9357 static bool __kfunc_param_match_suffix(const struct btf *btf, 9358 const struct btf_param *arg, 9359 const char *suffix) 9360 { 9361 int suffix_len = strlen(suffix), len; 9362 const char *param_name; 9363 9364 /* In the future, this can be ported to use BTF tagging */ 9365 param_name = btf_name_by_offset(btf, arg->name_off); 9366 if (str_is_empty(param_name)) 9367 return false; 9368 len = strlen(param_name); 9369 if (len < suffix_len) 9370 return false; 9371 param_name += len - suffix_len; 9372 return !strncmp(param_name, suffix, suffix_len); 9373 } 9374 9375 static bool is_kfunc_arg_mem_size(const struct btf *btf, 9376 const struct btf_param *arg, 9377 const struct bpf_reg_state *reg) 9378 { 9379 const struct btf_type *t; 9380 9381 t = btf_type_skip_modifiers(btf, arg->type, NULL); 9382 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 9383 return false; 9384 9385 return __kfunc_param_match_suffix(btf, arg, "__sz"); 9386 } 9387 9388 static bool is_kfunc_arg_const_mem_size(const struct btf *btf, 9389 const struct btf_param *arg, 9390 const struct bpf_reg_state *reg) 9391 { 9392 const struct btf_type *t; 9393 9394 t = btf_type_skip_modifiers(btf, arg->type, NULL); 9395 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 9396 return false; 9397 9398 return __kfunc_param_match_suffix(btf, arg, "__szk"); 9399 } 9400 9401 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 9402 { 9403 return __kfunc_param_match_suffix(btf, arg, "__k"); 9404 } 9405 9406 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 9407 { 9408 return __kfunc_param_match_suffix(btf, arg, "__ign"); 9409 } 9410 9411 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 9412 { 9413 return __kfunc_param_match_suffix(btf, arg, "__alloc"); 9414 } 9415 9416 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg) 9417 { 9418 return __kfunc_param_match_suffix(btf, arg, "__uninit"); 9419 } 9420 9421 static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf_param *arg) 9422 { 9423 return __kfunc_param_match_suffix(btf, arg, "__refcounted_kptr"); 9424 } 9425 9426 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 9427 const struct btf_param *arg, 9428 const char *name) 9429 { 9430 int len, target_len = strlen(name); 9431 const char *param_name; 9432 9433 param_name = btf_name_by_offset(btf, arg->name_off); 9434 if (str_is_empty(param_name)) 9435 return false; 9436 len = strlen(param_name); 9437 if (len != target_len) 9438 return false; 9439 if (strcmp(param_name, name)) 9440 return false; 9441 9442 return true; 9443 } 9444 9445 enum { 9446 KF_ARG_DYNPTR_ID, 9447 KF_ARG_LIST_HEAD_ID, 9448 KF_ARG_LIST_NODE_ID, 9449 KF_ARG_RB_ROOT_ID, 9450 KF_ARG_RB_NODE_ID, 9451 }; 9452 9453 BTF_ID_LIST(kf_arg_btf_ids) 9454 BTF_ID(struct, bpf_dynptr_kern) 9455 BTF_ID(struct, bpf_list_head) 9456 BTF_ID(struct, bpf_list_node) 9457 BTF_ID(struct, bpf_rb_root) 9458 BTF_ID(struct, bpf_rb_node) 9459 9460 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 9461 const struct btf_param *arg, int type) 9462 { 9463 const struct btf_type *t; 9464 u32 res_id; 9465 9466 t = btf_type_skip_modifiers(btf, arg->type, NULL); 9467 if (!t) 9468 return false; 9469 if (!btf_type_is_ptr(t)) 9470 return false; 9471 t = btf_type_skip_modifiers(btf, t->type, &res_id); 9472 if (!t) 9473 return false; 9474 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 9475 } 9476 9477 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 9478 { 9479 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 9480 } 9481 9482 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 9483 { 9484 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 9485 } 9486 9487 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 9488 { 9489 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 9490 } 9491 9492 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg) 9493 { 9494 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID); 9495 } 9496 9497 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg) 9498 { 9499 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID); 9500 } 9501 9502 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, 9503 const struct btf_param *arg) 9504 { 9505 const struct btf_type *t; 9506 9507 t = btf_type_resolve_func_ptr(btf, arg->type, NULL); 9508 if (!t) 9509 return false; 9510 9511 return true; 9512 } 9513 9514 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 9515 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 9516 const struct btf *btf, 9517 const struct btf_type *t, int rec) 9518 { 9519 const struct btf_type *member_type; 9520 const struct btf_member *member; 9521 u32 i; 9522 9523 if (!btf_type_is_struct(t)) 9524 return false; 9525 9526 for_each_member(i, t, member) { 9527 const struct btf_array *array; 9528 9529 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 9530 if (btf_type_is_struct(member_type)) { 9531 if (rec >= 3) { 9532 verbose(env, "max struct nesting depth exceeded\n"); 9533 return false; 9534 } 9535 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) 9536 return false; 9537 continue; 9538 } 9539 if (btf_type_is_array(member_type)) { 9540 array = btf_array(member_type); 9541 if (!array->nelems) 9542 return false; 9543 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 9544 if (!btf_type_is_scalar(member_type)) 9545 return false; 9546 continue; 9547 } 9548 if (!btf_type_is_scalar(member_type)) 9549 return false; 9550 } 9551 return true; 9552 } 9553 9554 9555 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 9556 #ifdef CONFIG_NET 9557 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 9558 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 9559 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 9560 #endif 9561 }; 9562 9563 enum kfunc_ptr_arg_type { 9564 KF_ARG_PTR_TO_CTX, 9565 KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ 9566 KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ 9567 KF_ARG_PTR_TO_DYNPTR, 9568 KF_ARG_PTR_TO_ITER, 9569 KF_ARG_PTR_TO_LIST_HEAD, 9570 KF_ARG_PTR_TO_LIST_NODE, 9571 KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ 9572 KF_ARG_PTR_TO_MEM, 9573 KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ 9574 KF_ARG_PTR_TO_CALLBACK, 9575 KF_ARG_PTR_TO_RB_ROOT, 9576 KF_ARG_PTR_TO_RB_NODE, 9577 }; 9578 9579 enum special_kfunc_type { 9580 KF_bpf_obj_new_impl, 9581 KF_bpf_obj_drop_impl, 9582 KF_bpf_refcount_acquire_impl, 9583 KF_bpf_list_push_front_impl, 9584 KF_bpf_list_push_back_impl, 9585 KF_bpf_list_pop_front, 9586 KF_bpf_list_pop_back, 9587 KF_bpf_cast_to_kern_ctx, 9588 KF_bpf_rdonly_cast, 9589 KF_bpf_rcu_read_lock, 9590 KF_bpf_rcu_read_unlock, 9591 KF_bpf_rbtree_remove, 9592 KF_bpf_rbtree_add_impl, 9593 KF_bpf_rbtree_first, 9594 KF_bpf_dynptr_from_skb, 9595 KF_bpf_dynptr_from_xdp, 9596 KF_bpf_dynptr_slice, 9597 KF_bpf_dynptr_slice_rdwr, 9598 }; 9599 9600 BTF_SET_START(special_kfunc_set) 9601 BTF_ID(func, bpf_obj_new_impl) 9602 BTF_ID(func, bpf_obj_drop_impl) 9603 BTF_ID(func, bpf_refcount_acquire_impl) 9604 BTF_ID(func, bpf_list_push_front_impl) 9605 BTF_ID(func, bpf_list_push_back_impl) 9606 BTF_ID(func, bpf_list_pop_front) 9607 BTF_ID(func, bpf_list_pop_back) 9608 BTF_ID(func, bpf_cast_to_kern_ctx) 9609 BTF_ID(func, bpf_rdonly_cast) 9610 BTF_ID(func, bpf_rbtree_remove) 9611 BTF_ID(func, bpf_rbtree_add_impl) 9612 BTF_ID(func, bpf_rbtree_first) 9613 BTF_ID(func, bpf_dynptr_from_skb) 9614 BTF_ID(func, bpf_dynptr_from_xdp) 9615 BTF_ID(func, bpf_dynptr_slice) 9616 BTF_ID(func, bpf_dynptr_slice_rdwr) 9617 BTF_SET_END(special_kfunc_set) 9618 9619 BTF_ID_LIST(special_kfunc_list) 9620 BTF_ID(func, bpf_obj_new_impl) 9621 BTF_ID(func, bpf_obj_drop_impl) 9622 BTF_ID(func, bpf_refcount_acquire_impl) 9623 BTF_ID(func, bpf_list_push_front_impl) 9624 BTF_ID(func, bpf_list_push_back_impl) 9625 BTF_ID(func, bpf_list_pop_front) 9626 BTF_ID(func, bpf_list_pop_back) 9627 BTF_ID(func, bpf_cast_to_kern_ctx) 9628 BTF_ID(func, bpf_rdonly_cast) 9629 BTF_ID(func, bpf_rcu_read_lock) 9630 BTF_ID(func, bpf_rcu_read_unlock) 9631 BTF_ID(func, bpf_rbtree_remove) 9632 BTF_ID(func, bpf_rbtree_add_impl) 9633 BTF_ID(func, bpf_rbtree_first) 9634 BTF_ID(func, bpf_dynptr_from_skb) 9635 BTF_ID(func, bpf_dynptr_from_xdp) 9636 BTF_ID(func, bpf_dynptr_slice) 9637 BTF_ID(func, bpf_dynptr_slice_rdwr) 9638 9639 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) 9640 { 9641 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; 9642 } 9643 9644 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) 9645 { 9646 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; 9647 } 9648 9649 static enum kfunc_ptr_arg_type 9650 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, 9651 struct bpf_kfunc_call_arg_meta *meta, 9652 const struct btf_type *t, const struct btf_type *ref_t, 9653 const char *ref_tname, const struct btf_param *args, 9654 int argno, int nargs) 9655 { 9656 u32 regno = argno + 1; 9657 struct bpf_reg_state *regs = cur_regs(env); 9658 struct bpf_reg_state *reg = ®s[regno]; 9659 bool arg_mem_size = false; 9660 9661 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) 9662 return KF_ARG_PTR_TO_CTX; 9663 9664 /* In this function, we verify the kfunc's BTF as per the argument type, 9665 * leaving the rest of the verification with respect to the register 9666 * type to our caller. When a set of conditions hold in the BTF type of 9667 * arguments, we resolve it to a known kfunc_ptr_arg_type. 9668 */ 9669 if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) 9670 return KF_ARG_PTR_TO_CTX; 9671 9672 if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) 9673 return KF_ARG_PTR_TO_ALLOC_BTF_ID; 9674 9675 if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno])) 9676 return KF_ARG_PTR_TO_REFCOUNTED_KPTR; 9677 9678 if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) 9679 return KF_ARG_PTR_TO_DYNPTR; 9680 9681 if (is_kfunc_arg_iter(meta, argno)) 9682 return KF_ARG_PTR_TO_ITER; 9683 9684 if (is_kfunc_arg_list_head(meta->btf, &args[argno])) 9685 return KF_ARG_PTR_TO_LIST_HEAD; 9686 9687 if (is_kfunc_arg_list_node(meta->btf, &args[argno])) 9688 return KF_ARG_PTR_TO_LIST_NODE; 9689 9690 if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno])) 9691 return KF_ARG_PTR_TO_RB_ROOT; 9692 9693 if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno])) 9694 return KF_ARG_PTR_TO_RB_NODE; 9695 9696 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 9697 if (!btf_type_is_struct(ref_t)) { 9698 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", 9699 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 9700 return -EINVAL; 9701 } 9702 return KF_ARG_PTR_TO_BTF_ID; 9703 } 9704 9705 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) 9706 return KF_ARG_PTR_TO_CALLBACK; 9707 9708 9709 if (argno + 1 < nargs && 9710 (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]) || 9711 is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]))) 9712 arg_mem_size = true; 9713 9714 /* This is the catch all argument type of register types supported by 9715 * check_helper_mem_access. However, we only allow when argument type is 9716 * pointer to scalar, or struct composed (recursively) of scalars. When 9717 * arg_mem_size is true, the pointer can be void *. 9718 */ 9719 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && 9720 (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { 9721 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", 9722 argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); 9723 return -EINVAL; 9724 } 9725 return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; 9726 } 9727 9728 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, 9729 struct bpf_reg_state *reg, 9730 const struct btf_type *ref_t, 9731 const char *ref_tname, u32 ref_id, 9732 struct bpf_kfunc_call_arg_meta *meta, 9733 int argno) 9734 { 9735 const struct btf_type *reg_ref_t; 9736 bool strict_type_match = false; 9737 const struct btf *reg_btf; 9738 const char *reg_ref_tname; 9739 u32 reg_ref_id; 9740 9741 if (base_type(reg->type) == PTR_TO_BTF_ID) { 9742 reg_btf = reg->btf; 9743 reg_ref_id = reg->btf_id; 9744 } else { 9745 reg_btf = btf_vmlinux; 9746 reg_ref_id = *reg2btf_ids[base_type(reg->type)]; 9747 } 9748 9749 /* Enforce strict type matching for calls to kfuncs that are acquiring 9750 * or releasing a reference, or are no-cast aliases. We do _not_ 9751 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default, 9752 * as we want to enable BPF programs to pass types that are bitwise 9753 * equivalent without forcing them to explicitly cast with something 9754 * like bpf_cast_to_kern_ctx(). 9755 * 9756 * For example, say we had a type like the following: 9757 * 9758 * struct bpf_cpumask { 9759 * cpumask_t cpumask; 9760 * refcount_t usage; 9761 * }; 9762 * 9763 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed 9764 * to a struct cpumask, so it would be safe to pass a struct 9765 * bpf_cpumask * to a kfunc expecting a struct cpumask *. 9766 * 9767 * The philosophy here is similar to how we allow scalars of different 9768 * types to be passed to kfuncs as long as the size is the same. The 9769 * only difference here is that we're simply allowing 9770 * btf_struct_ids_match() to walk the struct at the 0th offset, and 9771 * resolve types. 9772 */ 9773 if (is_kfunc_acquire(meta) || 9774 (is_kfunc_release(meta) && reg->ref_obj_id) || 9775 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) 9776 strict_type_match = true; 9777 9778 WARN_ON_ONCE(is_kfunc_trusted_args(meta) && reg->off); 9779 9780 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); 9781 reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); 9782 if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match)) { 9783 verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", 9784 meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, 9785 btf_type_str(reg_ref_t), reg_ref_tname); 9786 return -EINVAL; 9787 } 9788 return 0; 9789 } 9790 9791 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 9792 { 9793 struct bpf_verifier_state *state = env->cur_state; 9794 9795 if (!state->active_lock.ptr) { 9796 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); 9797 return -EFAULT; 9798 } 9799 9800 if (type_flag(reg->type) & NON_OWN_REF) { 9801 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); 9802 return -EFAULT; 9803 } 9804 9805 reg->type |= NON_OWN_REF; 9806 return 0; 9807 } 9808 9809 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) 9810 { 9811 struct bpf_func_state *state, *unused; 9812 struct bpf_reg_state *reg; 9813 int i; 9814 9815 state = cur_func(env); 9816 9817 if (!ref_obj_id) { 9818 verbose(env, "verifier internal error: ref_obj_id is zero for " 9819 "owning -> non-owning conversion\n"); 9820 return -EFAULT; 9821 } 9822 9823 for (i = 0; i < state->acquired_refs; i++) { 9824 if (state->refs[i].id != ref_obj_id) 9825 continue; 9826 9827 /* Clear ref_obj_id here so release_reference doesn't clobber 9828 * the whole reg 9829 */ 9830 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 9831 if (reg->ref_obj_id == ref_obj_id) { 9832 reg->ref_obj_id = 0; 9833 ref_set_non_owning(env, reg); 9834 } 9835 })); 9836 return 0; 9837 } 9838 9839 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); 9840 return -EFAULT; 9841 } 9842 9843 /* Implementation details: 9844 * 9845 * Each register points to some region of memory, which we define as an 9846 * allocation. Each allocation may embed a bpf_spin_lock which protects any 9847 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 9848 * allocation. The lock and the data it protects are colocated in the same 9849 * memory region. 9850 * 9851 * Hence, everytime a register holds a pointer value pointing to such 9852 * allocation, the verifier preserves a unique reg->id for it. 9853 * 9854 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 9855 * bpf_spin_lock is called. 9856 * 9857 * To enable this, lock state in the verifier captures two values: 9858 * active_lock.ptr = Register's type specific pointer 9859 * active_lock.id = A unique ID for each register pointer value 9860 * 9861 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 9862 * supported register types. 9863 * 9864 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 9865 * allocated objects is the reg->btf pointer. 9866 * 9867 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 9868 * can establish the provenance of the map value statically for each distinct 9869 * lookup into such maps. They always contain a single map value hence unique 9870 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 9871 * 9872 * So, in case of global variables, they use array maps with max_entries = 1, 9873 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 9874 * into the same map value as max_entries is 1, as described above). 9875 * 9876 * In case of inner map lookups, the inner map pointer has same map_ptr as the 9877 * outer map pointer (in verifier context), but each lookup into an inner map 9878 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 9879 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 9880 * will get different reg->id assigned to each lookup, hence different 9881 * active_lock.id. 9882 * 9883 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 9884 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 9885 * returned from bpf_obj_new. Each allocation receives a new reg->id. 9886 */ 9887 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 9888 { 9889 void *ptr; 9890 u32 id; 9891 9892 switch ((int)reg->type) { 9893 case PTR_TO_MAP_VALUE: 9894 ptr = reg->map_ptr; 9895 break; 9896 case PTR_TO_BTF_ID | MEM_ALLOC: 9897 ptr = reg->btf; 9898 break; 9899 default: 9900 verbose(env, "verifier internal error: unknown reg type for lock check\n"); 9901 return -EFAULT; 9902 } 9903 id = reg->id; 9904 9905 if (!env->cur_state->active_lock.ptr) 9906 return -EINVAL; 9907 if (env->cur_state->active_lock.ptr != ptr || 9908 env->cur_state->active_lock.id != id) { 9909 verbose(env, "held lock and object are not in the same allocation\n"); 9910 return -EINVAL; 9911 } 9912 return 0; 9913 } 9914 9915 static bool is_bpf_list_api_kfunc(u32 btf_id) 9916 { 9917 return btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 9918 btf_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 9919 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 9920 btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 9921 } 9922 9923 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 9924 { 9925 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || 9926 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 9927 btf_id == special_kfunc_list[KF_bpf_rbtree_first]; 9928 } 9929 9930 static bool is_bpf_graph_api_kfunc(u32 btf_id) 9931 { 9932 return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id) || 9933 btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]; 9934 } 9935 9936 static bool is_callback_calling_kfunc(u32 btf_id) 9937 { 9938 return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; 9939 } 9940 9941 static bool is_rbtree_lock_required_kfunc(u32 btf_id) 9942 { 9943 return is_bpf_rbtree_api_kfunc(btf_id); 9944 } 9945 9946 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, 9947 enum btf_field_type head_field_type, 9948 u32 kfunc_btf_id) 9949 { 9950 bool ret; 9951 9952 switch (head_field_type) { 9953 case BPF_LIST_HEAD: 9954 ret = is_bpf_list_api_kfunc(kfunc_btf_id); 9955 break; 9956 case BPF_RB_ROOT: 9957 ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id); 9958 break; 9959 default: 9960 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", 9961 btf_field_type_name(head_field_type)); 9962 return false; 9963 } 9964 9965 if (!ret) 9966 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", 9967 btf_field_type_name(head_field_type)); 9968 return ret; 9969 } 9970 9971 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, 9972 enum btf_field_type node_field_type, 9973 u32 kfunc_btf_id) 9974 { 9975 bool ret; 9976 9977 switch (node_field_type) { 9978 case BPF_LIST_NODE: 9979 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 9980 kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back_impl]); 9981 break; 9982 case BPF_RB_NODE: 9983 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 9984 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]); 9985 break; 9986 default: 9987 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", 9988 btf_field_type_name(node_field_type)); 9989 return false; 9990 } 9991 9992 if (!ret) 9993 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", 9994 btf_field_type_name(node_field_type)); 9995 return ret; 9996 } 9997 9998 static int 9999 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, 10000 struct bpf_reg_state *reg, u32 regno, 10001 struct bpf_kfunc_call_arg_meta *meta, 10002 enum btf_field_type head_field_type, 10003 struct btf_field **head_field) 10004 { 10005 const char *head_type_name; 10006 struct btf_field *field; 10007 struct btf_record *rec; 10008 u32 head_off; 10009 10010 if (meta->btf != btf_vmlinux) { 10011 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 10012 return -EFAULT; 10013 } 10014 10015 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) 10016 return -EFAULT; 10017 10018 head_type_name = btf_field_type_name(head_field_type); 10019 if (!tnum_is_const(reg->var_off)) { 10020 verbose(env, 10021 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 10022 regno, head_type_name); 10023 return -EINVAL; 10024 } 10025 10026 rec = reg_btf_record(reg); 10027 head_off = reg->off + reg->var_off.value; 10028 field = btf_record_find(rec, head_off, head_field_type); 10029 if (!field) { 10030 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); 10031 return -EINVAL; 10032 } 10033 10034 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 10035 if (check_reg_allocation_locked(env, reg)) { 10036 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", 10037 rec->spin_lock_off, head_type_name); 10038 return -EINVAL; 10039 } 10040 10041 if (*head_field) { 10042 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); 10043 return -EFAULT; 10044 } 10045 *head_field = field; 10046 return 0; 10047 } 10048 10049 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 10050 struct bpf_reg_state *reg, u32 regno, 10051 struct bpf_kfunc_call_arg_meta *meta) 10052 { 10053 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, 10054 &meta->arg_list_head.field); 10055 } 10056 10057 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, 10058 struct bpf_reg_state *reg, u32 regno, 10059 struct bpf_kfunc_call_arg_meta *meta) 10060 { 10061 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, 10062 &meta->arg_rbtree_root.field); 10063 } 10064 10065 static int 10066 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, 10067 struct bpf_reg_state *reg, u32 regno, 10068 struct bpf_kfunc_call_arg_meta *meta, 10069 enum btf_field_type head_field_type, 10070 enum btf_field_type node_field_type, 10071 struct btf_field **node_field) 10072 { 10073 const char *node_type_name; 10074 const struct btf_type *et, *t; 10075 struct btf_field *field; 10076 u32 node_off; 10077 10078 if (meta->btf != btf_vmlinux) { 10079 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 10080 return -EFAULT; 10081 } 10082 10083 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) 10084 return -EFAULT; 10085 10086 node_type_name = btf_field_type_name(node_field_type); 10087 if (!tnum_is_const(reg->var_off)) { 10088 verbose(env, 10089 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 10090 regno, node_type_name); 10091 return -EINVAL; 10092 } 10093 10094 node_off = reg->off + reg->var_off.value; 10095 field = reg_find_field_offset(reg, node_off, node_field_type); 10096 if (!field || field->offset != node_off) { 10097 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); 10098 return -EINVAL; 10099 } 10100 10101 field = *node_field; 10102 10103 et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); 10104 t = btf_type_by_id(reg->btf, reg->btf_id); 10105 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, 10106 field->graph_root.value_btf_id, true)) { 10107 verbose(env, "operation on %s expects arg#1 %s at offset=%d " 10108 "in struct %s, but arg is at offset=%d in struct %s\n", 10109 btf_field_type_name(head_field_type), 10110 btf_field_type_name(node_field_type), 10111 field->graph_root.node_offset, 10112 btf_name_by_offset(field->graph_root.btf, et->name_off), 10113 node_off, btf_name_by_offset(reg->btf, t->name_off)); 10114 return -EINVAL; 10115 } 10116 10117 if (node_off != field->graph_root.node_offset) { 10118 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", 10119 node_off, btf_field_type_name(node_field_type), 10120 field->graph_root.node_offset, 10121 btf_name_by_offset(field->graph_root.btf, et->name_off)); 10122 return -EINVAL; 10123 } 10124 10125 return 0; 10126 } 10127 10128 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 10129 struct bpf_reg_state *reg, u32 regno, 10130 struct bpf_kfunc_call_arg_meta *meta) 10131 { 10132 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 10133 BPF_LIST_HEAD, BPF_LIST_NODE, 10134 &meta->arg_list_head.field); 10135 } 10136 10137 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, 10138 struct bpf_reg_state *reg, u32 regno, 10139 struct bpf_kfunc_call_arg_meta *meta) 10140 { 10141 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 10142 BPF_RB_ROOT, BPF_RB_NODE, 10143 &meta->arg_rbtree_root.field); 10144 } 10145 10146 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, 10147 int insn_idx) 10148 { 10149 const char *func_name = meta->func_name, *ref_tname; 10150 const struct btf *btf = meta->btf; 10151 const struct btf_param *args; 10152 struct btf_record *rec; 10153 u32 i, nargs; 10154 int ret; 10155 10156 args = (const struct btf_param *)(meta->func_proto + 1); 10157 nargs = btf_type_vlen(meta->func_proto); 10158 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 10159 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, 10160 MAX_BPF_FUNC_REG_ARGS); 10161 return -EINVAL; 10162 } 10163 10164 /* Check that BTF function arguments match actual types that the 10165 * verifier sees. 10166 */ 10167 for (i = 0; i < nargs; i++) { 10168 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; 10169 const struct btf_type *t, *ref_t, *resolve_ret; 10170 enum bpf_arg_type arg_type = ARG_DONTCARE; 10171 u32 regno = i + 1, ref_id, type_size; 10172 bool is_ret_buf_sz = false; 10173 int kf_arg_type; 10174 10175 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 10176 10177 if (is_kfunc_arg_ignore(btf, &args[i])) 10178 continue; 10179 10180 if (btf_type_is_scalar(t)) { 10181 if (reg->type != SCALAR_VALUE) { 10182 verbose(env, "R%d is not a scalar\n", regno); 10183 return -EINVAL; 10184 } 10185 10186 if (is_kfunc_arg_constant(meta->btf, &args[i])) { 10187 if (meta->arg_constant.found) { 10188 verbose(env, "verifier internal error: only one constant argument permitted\n"); 10189 return -EFAULT; 10190 } 10191 if (!tnum_is_const(reg->var_off)) { 10192 verbose(env, "R%d must be a known constant\n", regno); 10193 return -EINVAL; 10194 } 10195 ret = mark_chain_precision(env, regno); 10196 if (ret < 0) 10197 return ret; 10198 meta->arg_constant.found = true; 10199 meta->arg_constant.value = reg->var_off.value; 10200 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { 10201 meta->r0_rdonly = true; 10202 is_ret_buf_sz = true; 10203 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { 10204 is_ret_buf_sz = true; 10205 } 10206 10207 if (is_ret_buf_sz) { 10208 if (meta->r0_size) { 10209 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); 10210 return -EINVAL; 10211 } 10212 10213 if (!tnum_is_const(reg->var_off)) { 10214 verbose(env, "R%d is not a const\n", regno); 10215 return -EINVAL; 10216 } 10217 10218 meta->r0_size = reg->var_off.value; 10219 ret = mark_chain_precision(env, regno); 10220 if (ret) 10221 return ret; 10222 } 10223 continue; 10224 } 10225 10226 if (!btf_type_is_ptr(t)) { 10227 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); 10228 return -EINVAL; 10229 } 10230 10231 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) && 10232 (register_is_null(reg) || type_may_be_null(reg->type))) { 10233 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); 10234 return -EACCES; 10235 } 10236 10237 if (reg->ref_obj_id) { 10238 if (is_kfunc_release(meta) && meta->ref_obj_id) { 10239 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 10240 regno, reg->ref_obj_id, 10241 meta->ref_obj_id); 10242 return -EFAULT; 10243 } 10244 meta->ref_obj_id = reg->ref_obj_id; 10245 if (is_kfunc_release(meta)) 10246 meta->release_regno = regno; 10247 } 10248 10249 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 10250 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 10251 10252 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); 10253 if (kf_arg_type < 0) 10254 return kf_arg_type; 10255 10256 switch (kf_arg_type) { 10257 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 10258 case KF_ARG_PTR_TO_BTF_ID: 10259 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 10260 break; 10261 10262 if (!is_trusted_reg(reg)) { 10263 if (!is_kfunc_rcu(meta)) { 10264 verbose(env, "R%d must be referenced or trusted\n", regno); 10265 return -EINVAL; 10266 } 10267 if (!is_rcu_reg(reg)) { 10268 verbose(env, "R%d must be a rcu pointer\n", regno); 10269 return -EINVAL; 10270 } 10271 } 10272 10273 fallthrough; 10274 case KF_ARG_PTR_TO_CTX: 10275 /* Trusted arguments have the same offset checks as release arguments */ 10276 arg_type |= OBJ_RELEASE; 10277 break; 10278 case KF_ARG_PTR_TO_DYNPTR: 10279 case KF_ARG_PTR_TO_ITER: 10280 case KF_ARG_PTR_TO_LIST_HEAD: 10281 case KF_ARG_PTR_TO_LIST_NODE: 10282 case KF_ARG_PTR_TO_RB_ROOT: 10283 case KF_ARG_PTR_TO_RB_NODE: 10284 case KF_ARG_PTR_TO_MEM: 10285 case KF_ARG_PTR_TO_MEM_SIZE: 10286 case KF_ARG_PTR_TO_CALLBACK: 10287 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: 10288 /* Trusted by default */ 10289 break; 10290 default: 10291 WARN_ON_ONCE(1); 10292 return -EFAULT; 10293 } 10294 10295 if (is_kfunc_release(meta) && reg->ref_obj_id) 10296 arg_type |= OBJ_RELEASE; 10297 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 10298 if (ret < 0) 10299 return ret; 10300 10301 switch (kf_arg_type) { 10302 case KF_ARG_PTR_TO_CTX: 10303 if (reg->type != PTR_TO_CTX) { 10304 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); 10305 return -EINVAL; 10306 } 10307 10308 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 10309 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 10310 if (ret < 0) 10311 return -EINVAL; 10312 meta->ret_btf_id = ret; 10313 } 10314 break; 10315 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 10316 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10317 verbose(env, "arg#%d expected pointer to allocated object\n", i); 10318 return -EINVAL; 10319 } 10320 if (!reg->ref_obj_id) { 10321 verbose(env, "allocated object must be referenced\n"); 10322 return -EINVAL; 10323 } 10324 if (meta->btf == btf_vmlinux && 10325 meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 10326 meta->arg_obj_drop.btf = reg->btf; 10327 meta->arg_obj_drop.btf_id = reg->btf_id; 10328 } 10329 break; 10330 case KF_ARG_PTR_TO_DYNPTR: 10331 { 10332 enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; 10333 10334 if (reg->type != PTR_TO_STACK && 10335 reg->type != CONST_PTR_TO_DYNPTR) { 10336 verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i); 10337 return -EINVAL; 10338 } 10339 10340 if (reg->type == CONST_PTR_TO_DYNPTR) 10341 dynptr_arg_type |= MEM_RDONLY; 10342 10343 if (is_kfunc_arg_uninit(btf, &args[i])) 10344 dynptr_arg_type |= MEM_UNINIT; 10345 10346 if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) 10347 dynptr_arg_type |= DYNPTR_TYPE_SKB; 10348 else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) 10349 dynptr_arg_type |= DYNPTR_TYPE_XDP; 10350 10351 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type); 10352 if (ret < 0) 10353 return ret; 10354 10355 if (!(dynptr_arg_type & MEM_UNINIT)) { 10356 int id = dynptr_id(env, reg); 10357 10358 if (id < 0) { 10359 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 10360 return id; 10361 } 10362 meta->initialized_dynptr.id = id; 10363 meta->initialized_dynptr.type = dynptr_get_type(env, reg); 10364 } 10365 10366 break; 10367 } 10368 case KF_ARG_PTR_TO_ITER: 10369 ret = process_iter_arg(env, regno, insn_idx, meta); 10370 if (ret < 0) 10371 return ret; 10372 break; 10373 case KF_ARG_PTR_TO_LIST_HEAD: 10374 if (reg->type != PTR_TO_MAP_VALUE && 10375 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10376 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 10377 return -EINVAL; 10378 } 10379 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 10380 verbose(env, "allocated object must be referenced\n"); 10381 return -EINVAL; 10382 } 10383 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); 10384 if (ret < 0) 10385 return ret; 10386 break; 10387 case KF_ARG_PTR_TO_RB_ROOT: 10388 if (reg->type != PTR_TO_MAP_VALUE && 10389 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10390 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 10391 return -EINVAL; 10392 } 10393 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 10394 verbose(env, "allocated object must be referenced\n"); 10395 return -EINVAL; 10396 } 10397 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); 10398 if (ret < 0) 10399 return ret; 10400 break; 10401 case KF_ARG_PTR_TO_LIST_NODE: 10402 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10403 verbose(env, "arg#%d expected pointer to allocated object\n", i); 10404 return -EINVAL; 10405 } 10406 if (!reg->ref_obj_id) { 10407 verbose(env, "allocated object must be referenced\n"); 10408 return -EINVAL; 10409 } 10410 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); 10411 if (ret < 0) 10412 return ret; 10413 break; 10414 case KF_ARG_PTR_TO_RB_NODE: 10415 if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) { 10416 if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) { 10417 verbose(env, "rbtree_remove node input must be non-owning ref\n"); 10418 return -EINVAL; 10419 } 10420 if (in_rbtree_lock_required_cb(env)) { 10421 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); 10422 return -EINVAL; 10423 } 10424 } else { 10425 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 10426 verbose(env, "arg#%d expected pointer to allocated object\n", i); 10427 return -EINVAL; 10428 } 10429 if (!reg->ref_obj_id) { 10430 verbose(env, "allocated object must be referenced\n"); 10431 return -EINVAL; 10432 } 10433 } 10434 10435 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); 10436 if (ret < 0) 10437 return ret; 10438 break; 10439 case KF_ARG_PTR_TO_BTF_ID: 10440 /* Only base_type is checked, further checks are done here */ 10441 if ((base_type(reg->type) != PTR_TO_BTF_ID || 10442 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && 10443 !reg2btf_ids[base_type(reg->type)]) { 10444 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); 10445 verbose(env, "expected %s or socket\n", 10446 reg_type_str(env, base_type(reg->type) | 10447 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 10448 return -EINVAL; 10449 } 10450 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 10451 if (ret < 0) 10452 return ret; 10453 break; 10454 case KF_ARG_PTR_TO_MEM: 10455 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 10456 if (IS_ERR(resolve_ret)) { 10457 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 10458 i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); 10459 return -EINVAL; 10460 } 10461 ret = check_mem_reg(env, reg, regno, type_size); 10462 if (ret < 0) 10463 return ret; 10464 break; 10465 case KF_ARG_PTR_TO_MEM_SIZE: 10466 { 10467 struct bpf_reg_state *size_reg = ®s[regno + 1]; 10468 const struct btf_param *size_arg = &args[i + 1]; 10469 10470 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); 10471 if (ret < 0) { 10472 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); 10473 return ret; 10474 } 10475 10476 if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { 10477 if (meta->arg_constant.found) { 10478 verbose(env, "verifier internal error: only one constant argument permitted\n"); 10479 return -EFAULT; 10480 } 10481 if (!tnum_is_const(size_reg->var_off)) { 10482 verbose(env, "R%d must be a known constant\n", regno + 1); 10483 return -EINVAL; 10484 } 10485 meta->arg_constant.found = true; 10486 meta->arg_constant.value = size_reg->var_off.value; 10487 } 10488 10489 /* Skip next '__sz' or '__szk' argument */ 10490 i++; 10491 break; 10492 } 10493 case KF_ARG_PTR_TO_CALLBACK: 10494 meta->subprogno = reg->subprogno; 10495 break; 10496 case KF_ARG_PTR_TO_REFCOUNTED_KPTR: 10497 if (!type_is_ptr_alloc_obj(reg->type) && !type_is_non_owning_ref(reg->type)) { 10498 verbose(env, "arg#%d is neither owning or non-owning ref\n", i); 10499 return -EINVAL; 10500 } 10501 10502 rec = reg_btf_record(reg); 10503 if (!rec) { 10504 verbose(env, "verifier internal error: Couldn't find btf_record\n"); 10505 return -EFAULT; 10506 } 10507 10508 if (rec->refcount_off < 0) { 10509 verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); 10510 return -EINVAL; 10511 } 10512 10513 meta->arg_refcount_acquire.btf = reg->btf; 10514 meta->arg_refcount_acquire.btf_id = reg->btf_id; 10515 break; 10516 } 10517 } 10518 10519 if (is_kfunc_release(meta) && !meta->release_regno) { 10520 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", 10521 func_name); 10522 return -EINVAL; 10523 } 10524 10525 return 0; 10526 } 10527 10528 static int fetch_kfunc_meta(struct bpf_verifier_env *env, 10529 struct bpf_insn *insn, 10530 struct bpf_kfunc_call_arg_meta *meta, 10531 const char **kfunc_name) 10532 { 10533 const struct btf_type *func, *func_proto; 10534 u32 func_id, *kfunc_flags; 10535 const char *func_name; 10536 struct btf *desc_btf; 10537 10538 if (kfunc_name) 10539 *kfunc_name = NULL; 10540 10541 if (!insn->imm) 10542 return -EINVAL; 10543 10544 desc_btf = find_kfunc_desc_btf(env, insn->off); 10545 if (IS_ERR(desc_btf)) 10546 return PTR_ERR(desc_btf); 10547 10548 func_id = insn->imm; 10549 func = btf_type_by_id(desc_btf, func_id); 10550 func_name = btf_name_by_offset(desc_btf, func->name_off); 10551 if (kfunc_name) 10552 *kfunc_name = func_name; 10553 func_proto = btf_type_by_id(desc_btf, func->type); 10554 10555 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); 10556 if (!kfunc_flags) { 10557 return -EACCES; 10558 } 10559 10560 memset(meta, 0, sizeof(*meta)); 10561 meta->btf = desc_btf; 10562 meta->func_id = func_id; 10563 meta->kfunc_flags = *kfunc_flags; 10564 meta->func_proto = func_proto; 10565 meta->func_name = func_name; 10566 10567 return 0; 10568 } 10569 10570 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 10571 int *insn_idx_p) 10572 { 10573 const struct btf_type *t, *ptr_type; 10574 u32 i, nargs, ptr_type_id, release_ref_obj_id; 10575 struct bpf_reg_state *regs = cur_regs(env); 10576 const char *func_name, *ptr_type_name; 10577 bool sleepable, rcu_lock, rcu_unlock; 10578 struct bpf_kfunc_call_arg_meta meta; 10579 struct bpf_insn_aux_data *insn_aux; 10580 int err, insn_idx = *insn_idx_p; 10581 const struct btf_param *args; 10582 const struct btf_type *ret_t; 10583 struct btf *desc_btf; 10584 10585 /* skip for now, but return error when we find this in fixup_kfunc_call */ 10586 if (!insn->imm) 10587 return 0; 10588 10589 err = fetch_kfunc_meta(env, insn, &meta, &func_name); 10590 if (err == -EACCES && func_name) 10591 verbose(env, "calling kernel function %s is not allowed\n", func_name); 10592 if (err) 10593 return err; 10594 desc_btf = meta.btf; 10595 insn_aux = &env->insn_aux_data[insn_idx]; 10596 10597 insn_aux->is_iter_next = is_iter_next_kfunc(&meta); 10598 10599 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 10600 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 10601 return -EACCES; 10602 } 10603 10604 sleepable = is_kfunc_sleepable(&meta); 10605 if (sleepable && !env->prog->aux->sleepable) { 10606 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); 10607 return -EACCES; 10608 } 10609 10610 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 10611 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 10612 10613 if (env->cur_state->active_rcu_lock) { 10614 struct bpf_func_state *state; 10615 struct bpf_reg_state *reg; 10616 10617 if (rcu_lock) { 10618 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); 10619 return -EINVAL; 10620 } else if (rcu_unlock) { 10621 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 10622 if (reg->type & MEM_RCU) { 10623 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); 10624 reg->type |= PTR_UNTRUSTED; 10625 } 10626 })); 10627 env->cur_state->active_rcu_lock = false; 10628 } else if (sleepable) { 10629 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); 10630 return -EACCES; 10631 } 10632 } else if (rcu_lock) { 10633 env->cur_state->active_rcu_lock = true; 10634 } else if (rcu_unlock) { 10635 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 10636 return -EINVAL; 10637 } 10638 10639 /* Check the arguments */ 10640 err = check_kfunc_args(env, &meta, insn_idx); 10641 if (err < 0) 10642 return err; 10643 /* In case of release function, we get register number of refcounted 10644 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 10645 */ 10646 if (meta.release_regno) { 10647 err = release_reference(env, regs[meta.release_regno].ref_obj_id); 10648 if (err) { 10649 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 10650 func_name, meta.func_id); 10651 return err; 10652 } 10653 } 10654 10655 if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 10656 meta.func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 10657 meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 10658 release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; 10659 insn_aux->insert_off = regs[BPF_REG_2].off; 10660 err = ref_convert_owning_non_owning(env, release_ref_obj_id); 10661 if (err) { 10662 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", 10663 func_name, meta.func_id); 10664 return err; 10665 } 10666 10667 err = release_reference(env, release_ref_obj_id); 10668 if (err) { 10669 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 10670 func_name, meta.func_id); 10671 return err; 10672 } 10673 } 10674 10675 if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 10676 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 10677 set_rbtree_add_callback_state); 10678 if (err) { 10679 verbose(env, "kfunc %s#%d failed callback verification\n", 10680 func_name, meta.func_id); 10681 return err; 10682 } 10683 } 10684 10685 for (i = 0; i < CALLER_SAVED_REGS; i++) 10686 mark_reg_not_init(env, regs, caller_saved[i]); 10687 10688 /* Check return type */ 10689 t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL); 10690 10691 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 10692 /* Only exception is bpf_obj_new_impl */ 10693 if (meta.btf != btf_vmlinux || 10694 (meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl] && 10695 meta.func_id != special_kfunc_list[KF_bpf_refcount_acquire_impl])) { 10696 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 10697 return -EINVAL; 10698 } 10699 } 10700 10701 if (btf_type_is_scalar(t)) { 10702 mark_reg_unknown(env, regs, BPF_REG_0); 10703 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 10704 } else if (btf_type_is_ptr(t)) { 10705 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 10706 10707 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 10708 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 10709 struct btf *ret_btf; 10710 u32 ret_btf_id; 10711 10712 if (unlikely(!bpf_global_ma_set)) 10713 return -ENOMEM; 10714 10715 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { 10716 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 10717 return -EINVAL; 10718 } 10719 10720 ret_btf = env->prog->aux->btf; 10721 ret_btf_id = meta.arg_constant.value; 10722 10723 /* This may be NULL due to user not supplying a BTF */ 10724 if (!ret_btf) { 10725 verbose(env, "bpf_obj_new requires prog BTF\n"); 10726 return -EINVAL; 10727 } 10728 10729 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 10730 if (!ret_t || !__btf_type_is_struct(ret_t)) { 10731 verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); 10732 return -EINVAL; 10733 } 10734 10735 mark_reg_known_zero(env, regs, BPF_REG_0); 10736 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 10737 regs[BPF_REG_0].btf = ret_btf; 10738 regs[BPF_REG_0].btf_id = ret_btf_id; 10739 10740 insn_aux->obj_new_size = ret_t->size; 10741 insn_aux->kptr_struct_meta = 10742 btf_find_struct_meta(ret_btf, ret_btf_id); 10743 } else if (meta.func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 10744 mark_reg_known_zero(env, regs, BPF_REG_0); 10745 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 10746 regs[BPF_REG_0].btf = meta.arg_refcount_acquire.btf; 10747 regs[BPF_REG_0].btf_id = meta.arg_refcount_acquire.btf_id; 10748 10749 insn_aux->kptr_struct_meta = 10750 btf_find_struct_meta(meta.arg_refcount_acquire.btf, 10751 meta.arg_refcount_acquire.btf_id); 10752 } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 10753 meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 10754 struct btf_field *field = meta.arg_list_head.field; 10755 10756 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 10757 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] || 10758 meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 10759 struct btf_field *field = meta.arg_rbtree_root.field; 10760 10761 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 10762 } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 10763 mark_reg_known_zero(env, regs, BPF_REG_0); 10764 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 10765 regs[BPF_REG_0].btf = desc_btf; 10766 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 10767 } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 10768 ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value); 10769 if (!ret_t || !btf_type_is_struct(ret_t)) { 10770 verbose(env, 10771 "kfunc bpf_rdonly_cast type ID argument must be of a struct\n"); 10772 return -EINVAL; 10773 } 10774 10775 mark_reg_known_zero(env, regs, BPF_REG_0); 10776 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 10777 regs[BPF_REG_0].btf = desc_btf; 10778 regs[BPF_REG_0].btf_id = meta.arg_constant.value; 10779 } else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] || 10780 meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { 10781 enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type); 10782 10783 mark_reg_known_zero(env, regs, BPF_REG_0); 10784 10785 if (!meta.arg_constant.found) { 10786 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); 10787 return -EFAULT; 10788 } 10789 10790 regs[BPF_REG_0].mem_size = meta.arg_constant.value; 10791 10792 /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ 10793 regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; 10794 10795 if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { 10796 regs[BPF_REG_0].type |= MEM_RDONLY; 10797 } else { 10798 /* this will set env->seen_direct_write to true */ 10799 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { 10800 verbose(env, "the prog does not allow writes to packet data\n"); 10801 return -EINVAL; 10802 } 10803 } 10804 10805 if (!meta.initialized_dynptr.id) { 10806 verbose(env, "verifier internal error: no dynptr id\n"); 10807 return -EFAULT; 10808 } 10809 regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id; 10810 10811 /* we don't need to set BPF_REG_0's ref obj id 10812 * because packet slices are not refcounted (see 10813 * dynptr_type_refcounted) 10814 */ 10815 } else { 10816 verbose(env, "kernel function %s unhandled dynamic return type\n", 10817 meta.func_name); 10818 return -EFAULT; 10819 } 10820 } else if (!__btf_type_is_struct(ptr_type)) { 10821 if (!meta.r0_size) { 10822 __u32 sz; 10823 10824 if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { 10825 meta.r0_size = sz; 10826 meta.r0_rdonly = true; 10827 } 10828 } 10829 if (!meta.r0_size) { 10830 ptr_type_name = btf_name_by_offset(desc_btf, 10831 ptr_type->name_off); 10832 verbose(env, 10833 "kernel function %s returns pointer type %s %s is not supported\n", 10834 func_name, 10835 btf_type_str(ptr_type), 10836 ptr_type_name); 10837 return -EINVAL; 10838 } 10839 10840 mark_reg_known_zero(env, regs, BPF_REG_0); 10841 regs[BPF_REG_0].type = PTR_TO_MEM; 10842 regs[BPF_REG_0].mem_size = meta.r0_size; 10843 10844 if (meta.r0_rdonly) 10845 regs[BPF_REG_0].type |= MEM_RDONLY; 10846 10847 /* Ensures we don't access the memory after a release_reference() */ 10848 if (meta.ref_obj_id) 10849 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 10850 } else { 10851 mark_reg_known_zero(env, regs, BPF_REG_0); 10852 regs[BPF_REG_0].btf = desc_btf; 10853 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 10854 regs[BPF_REG_0].btf_id = ptr_type_id; 10855 } 10856 10857 if (is_kfunc_ret_null(&meta)) { 10858 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 10859 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 10860 regs[BPF_REG_0].id = ++env->id_gen; 10861 } 10862 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 10863 if (is_kfunc_acquire(&meta)) { 10864 int id = acquire_reference_state(env, insn_idx); 10865 10866 if (id < 0) 10867 return id; 10868 if (is_kfunc_ret_null(&meta)) 10869 regs[BPF_REG_0].id = id; 10870 regs[BPF_REG_0].ref_obj_id = id; 10871 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 10872 ref_set_non_owning(env, ®s[BPF_REG_0]); 10873 } 10874 10875 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 10876 regs[BPF_REG_0].id = ++env->id_gen; 10877 } else if (btf_type_is_void(t)) { 10878 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 10879 if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 10880 insn_aux->kptr_struct_meta = 10881 btf_find_struct_meta(meta.arg_obj_drop.btf, 10882 meta.arg_obj_drop.btf_id); 10883 } 10884 } 10885 } 10886 10887 nargs = btf_type_vlen(meta.func_proto); 10888 args = (const struct btf_param *)(meta.func_proto + 1); 10889 for (i = 0; i < nargs; i++) { 10890 u32 regno = i + 1; 10891 10892 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 10893 if (btf_type_is_ptr(t)) 10894 mark_btf_func_reg_size(env, regno, sizeof(void *)); 10895 else 10896 /* scalar. ensured by btf_check_kfunc_arg_match() */ 10897 mark_btf_func_reg_size(env, regno, t->size); 10898 } 10899 10900 if (is_iter_next_kfunc(&meta)) { 10901 err = process_iter_next_call(env, insn_idx, &meta); 10902 if (err) 10903 return err; 10904 } 10905 10906 return 0; 10907 } 10908 10909 static bool signed_add_overflows(s64 a, s64 b) 10910 { 10911 /* Do the add in u64, where overflow is well-defined */ 10912 s64 res = (s64)((u64)a + (u64)b); 10913 10914 if (b < 0) 10915 return res > a; 10916 return res < a; 10917 } 10918 10919 static bool signed_add32_overflows(s32 a, s32 b) 10920 { 10921 /* Do the add in u32, where overflow is well-defined */ 10922 s32 res = (s32)((u32)a + (u32)b); 10923 10924 if (b < 0) 10925 return res > a; 10926 return res < a; 10927 } 10928 10929 static bool signed_sub_overflows(s64 a, s64 b) 10930 { 10931 /* Do the sub in u64, where overflow is well-defined */ 10932 s64 res = (s64)((u64)a - (u64)b); 10933 10934 if (b < 0) 10935 return res < a; 10936 return res > a; 10937 } 10938 10939 static bool signed_sub32_overflows(s32 a, s32 b) 10940 { 10941 /* Do the sub in u32, where overflow is well-defined */ 10942 s32 res = (s32)((u32)a - (u32)b); 10943 10944 if (b < 0) 10945 return res < a; 10946 return res > a; 10947 } 10948 10949 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 10950 const struct bpf_reg_state *reg, 10951 enum bpf_reg_type type) 10952 { 10953 bool known = tnum_is_const(reg->var_off); 10954 s64 val = reg->var_off.value; 10955 s64 smin = reg->smin_value; 10956 10957 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 10958 verbose(env, "math between %s pointer and %lld is not allowed\n", 10959 reg_type_str(env, type), val); 10960 return false; 10961 } 10962 10963 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 10964 verbose(env, "%s pointer offset %d is not allowed\n", 10965 reg_type_str(env, type), reg->off); 10966 return false; 10967 } 10968 10969 if (smin == S64_MIN) { 10970 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 10971 reg_type_str(env, type)); 10972 return false; 10973 } 10974 10975 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 10976 verbose(env, "value %lld makes %s pointer be out of bounds\n", 10977 smin, reg_type_str(env, type)); 10978 return false; 10979 } 10980 10981 return true; 10982 } 10983 10984 enum { 10985 REASON_BOUNDS = -1, 10986 REASON_TYPE = -2, 10987 REASON_PATHS = -3, 10988 REASON_LIMIT = -4, 10989 REASON_STACK = -5, 10990 }; 10991 10992 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 10993 u32 *alu_limit, bool mask_to_left) 10994 { 10995 u32 max = 0, ptr_limit = 0; 10996 10997 switch (ptr_reg->type) { 10998 case PTR_TO_STACK: 10999 /* Offset 0 is out-of-bounds, but acceptable start for the 11000 * left direction, see BPF_REG_FP. Also, unknown scalar 11001 * offset where we would need to deal with min/max bounds is 11002 * currently prohibited for unprivileged. 11003 */ 11004 max = MAX_BPF_STACK + mask_to_left; 11005 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 11006 break; 11007 case PTR_TO_MAP_VALUE: 11008 max = ptr_reg->map_ptr->value_size; 11009 ptr_limit = (mask_to_left ? 11010 ptr_reg->smin_value : 11011 ptr_reg->umax_value) + ptr_reg->off; 11012 break; 11013 default: 11014 return REASON_TYPE; 11015 } 11016 11017 if (ptr_limit >= max) 11018 return REASON_LIMIT; 11019 *alu_limit = ptr_limit; 11020 return 0; 11021 } 11022 11023 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 11024 const struct bpf_insn *insn) 11025 { 11026 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 11027 } 11028 11029 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 11030 u32 alu_state, u32 alu_limit) 11031 { 11032 /* If we arrived here from different branches with different 11033 * state or limits to sanitize, then this won't work. 11034 */ 11035 if (aux->alu_state && 11036 (aux->alu_state != alu_state || 11037 aux->alu_limit != alu_limit)) 11038 return REASON_PATHS; 11039 11040 /* Corresponding fixup done in do_misc_fixups(). */ 11041 aux->alu_state = alu_state; 11042 aux->alu_limit = alu_limit; 11043 return 0; 11044 } 11045 11046 static int sanitize_val_alu(struct bpf_verifier_env *env, 11047 struct bpf_insn *insn) 11048 { 11049 struct bpf_insn_aux_data *aux = cur_aux(env); 11050 11051 if (can_skip_alu_sanitation(env, insn)) 11052 return 0; 11053 11054 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 11055 } 11056 11057 static bool sanitize_needed(u8 opcode) 11058 { 11059 return opcode == BPF_ADD || opcode == BPF_SUB; 11060 } 11061 11062 struct bpf_sanitize_info { 11063 struct bpf_insn_aux_data aux; 11064 bool mask_to_left; 11065 }; 11066 11067 static struct bpf_verifier_state * 11068 sanitize_speculative_path(struct bpf_verifier_env *env, 11069 const struct bpf_insn *insn, 11070 u32 next_idx, u32 curr_idx) 11071 { 11072 struct bpf_verifier_state *branch; 11073 struct bpf_reg_state *regs; 11074 11075 branch = push_stack(env, next_idx, curr_idx, true); 11076 if (branch && insn) { 11077 regs = branch->frame[branch->curframe]->regs; 11078 if (BPF_SRC(insn->code) == BPF_K) { 11079 mark_reg_unknown(env, regs, insn->dst_reg); 11080 } else if (BPF_SRC(insn->code) == BPF_X) { 11081 mark_reg_unknown(env, regs, insn->dst_reg); 11082 mark_reg_unknown(env, regs, insn->src_reg); 11083 } 11084 } 11085 return branch; 11086 } 11087 11088 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 11089 struct bpf_insn *insn, 11090 const struct bpf_reg_state *ptr_reg, 11091 const struct bpf_reg_state *off_reg, 11092 struct bpf_reg_state *dst_reg, 11093 struct bpf_sanitize_info *info, 11094 const bool commit_window) 11095 { 11096 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 11097 struct bpf_verifier_state *vstate = env->cur_state; 11098 bool off_is_imm = tnum_is_const(off_reg->var_off); 11099 bool off_is_neg = off_reg->smin_value < 0; 11100 bool ptr_is_dst_reg = ptr_reg == dst_reg; 11101 u8 opcode = BPF_OP(insn->code); 11102 u32 alu_state, alu_limit; 11103 struct bpf_reg_state tmp; 11104 bool ret; 11105 int err; 11106 11107 if (can_skip_alu_sanitation(env, insn)) 11108 return 0; 11109 11110 /* We already marked aux for masking from non-speculative 11111 * paths, thus we got here in the first place. We only care 11112 * to explore bad access from here. 11113 */ 11114 if (vstate->speculative) 11115 goto do_sim; 11116 11117 if (!commit_window) { 11118 if (!tnum_is_const(off_reg->var_off) && 11119 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 11120 return REASON_BOUNDS; 11121 11122 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 11123 (opcode == BPF_SUB && !off_is_neg); 11124 } 11125 11126 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 11127 if (err < 0) 11128 return err; 11129 11130 if (commit_window) { 11131 /* In commit phase we narrow the masking window based on 11132 * the observed pointer move after the simulated operation. 11133 */ 11134 alu_state = info->aux.alu_state; 11135 alu_limit = abs(info->aux.alu_limit - alu_limit); 11136 } else { 11137 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 11138 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 11139 alu_state |= ptr_is_dst_reg ? 11140 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 11141 11142 /* Limit pruning on unknown scalars to enable deep search for 11143 * potential masking differences from other program paths. 11144 */ 11145 if (!off_is_imm) 11146 env->explore_alu_limits = true; 11147 } 11148 11149 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 11150 if (err < 0) 11151 return err; 11152 do_sim: 11153 /* If we're in commit phase, we're done here given we already 11154 * pushed the truncated dst_reg into the speculative verification 11155 * stack. 11156 * 11157 * Also, when register is a known constant, we rewrite register-based 11158 * operation to immediate-based, and thus do not need masking (and as 11159 * a consequence, do not need to simulate the zero-truncation either). 11160 */ 11161 if (commit_window || off_is_imm) 11162 return 0; 11163 11164 /* Simulate and find potential out-of-bounds access under 11165 * speculative execution from truncation as a result of 11166 * masking when off was not within expected range. If off 11167 * sits in dst, then we temporarily need to move ptr there 11168 * to simulate dst (== 0) +/-= ptr. Needed, for example, 11169 * for cases where we use K-based arithmetic in one direction 11170 * and truncated reg-based in the other in order to explore 11171 * bad access. 11172 */ 11173 if (!ptr_is_dst_reg) { 11174 tmp = *dst_reg; 11175 copy_register_state(dst_reg, ptr_reg); 11176 } 11177 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 11178 env->insn_idx); 11179 if (!ptr_is_dst_reg && ret) 11180 *dst_reg = tmp; 11181 return !ret ? REASON_STACK : 0; 11182 } 11183 11184 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 11185 { 11186 struct bpf_verifier_state *vstate = env->cur_state; 11187 11188 /* If we simulate paths under speculation, we don't update the 11189 * insn as 'seen' such that when we verify unreachable paths in 11190 * the non-speculative domain, sanitize_dead_code() can still 11191 * rewrite/sanitize them. 11192 */ 11193 if (!vstate->speculative) 11194 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 11195 } 11196 11197 static int sanitize_err(struct bpf_verifier_env *env, 11198 const struct bpf_insn *insn, int reason, 11199 const struct bpf_reg_state *off_reg, 11200 const struct bpf_reg_state *dst_reg) 11201 { 11202 static const char *err = "pointer arithmetic with it prohibited for !root"; 11203 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 11204 u32 dst = insn->dst_reg, src = insn->src_reg; 11205 11206 switch (reason) { 11207 case REASON_BOUNDS: 11208 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 11209 off_reg == dst_reg ? dst : src, err); 11210 break; 11211 case REASON_TYPE: 11212 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 11213 off_reg == dst_reg ? src : dst, err); 11214 break; 11215 case REASON_PATHS: 11216 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 11217 dst, op, err); 11218 break; 11219 case REASON_LIMIT: 11220 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 11221 dst, op, err); 11222 break; 11223 case REASON_STACK: 11224 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 11225 dst, err); 11226 break; 11227 default: 11228 verbose(env, "verifier internal error: unknown reason (%d)\n", 11229 reason); 11230 break; 11231 } 11232 11233 return -EACCES; 11234 } 11235 11236 /* check that stack access falls within stack limits and that 'reg' doesn't 11237 * have a variable offset. 11238 * 11239 * Variable offset is prohibited for unprivileged mode for simplicity since it 11240 * requires corresponding support in Spectre masking for stack ALU. See also 11241 * retrieve_ptr_limit(). 11242 * 11243 * 11244 * 'off' includes 'reg->off'. 11245 */ 11246 static int check_stack_access_for_ptr_arithmetic( 11247 struct bpf_verifier_env *env, 11248 int regno, 11249 const struct bpf_reg_state *reg, 11250 int off) 11251 { 11252 if (!tnum_is_const(reg->var_off)) { 11253 char tn_buf[48]; 11254 11255 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 11256 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 11257 regno, tn_buf, off); 11258 return -EACCES; 11259 } 11260 11261 if (off >= 0 || off < -MAX_BPF_STACK) { 11262 verbose(env, "R%d stack pointer arithmetic goes out of range, " 11263 "prohibited for !root; off=%d\n", regno, off); 11264 return -EACCES; 11265 } 11266 11267 return 0; 11268 } 11269 11270 static int sanitize_check_bounds(struct bpf_verifier_env *env, 11271 const struct bpf_insn *insn, 11272 const struct bpf_reg_state *dst_reg) 11273 { 11274 u32 dst = insn->dst_reg; 11275 11276 /* For unprivileged we require that resulting offset must be in bounds 11277 * in order to be able to sanitize access later on. 11278 */ 11279 if (env->bypass_spec_v1) 11280 return 0; 11281 11282 switch (dst_reg->type) { 11283 case PTR_TO_STACK: 11284 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 11285 dst_reg->off + dst_reg->var_off.value)) 11286 return -EACCES; 11287 break; 11288 case PTR_TO_MAP_VALUE: 11289 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 11290 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 11291 "prohibited for !root\n", dst); 11292 return -EACCES; 11293 } 11294 break; 11295 default: 11296 break; 11297 } 11298 11299 return 0; 11300 } 11301 11302 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 11303 * Caller should also handle BPF_MOV case separately. 11304 * If we return -EACCES, caller may want to try again treating pointer as a 11305 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 11306 */ 11307 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 11308 struct bpf_insn *insn, 11309 const struct bpf_reg_state *ptr_reg, 11310 const struct bpf_reg_state *off_reg) 11311 { 11312 struct bpf_verifier_state *vstate = env->cur_state; 11313 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 11314 struct bpf_reg_state *regs = state->regs, *dst_reg; 11315 bool known = tnum_is_const(off_reg->var_off); 11316 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 11317 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 11318 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 11319 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 11320 struct bpf_sanitize_info info = {}; 11321 u8 opcode = BPF_OP(insn->code); 11322 u32 dst = insn->dst_reg; 11323 int ret; 11324 11325 dst_reg = ®s[dst]; 11326 11327 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 11328 smin_val > smax_val || umin_val > umax_val) { 11329 /* Taint dst register if offset had invalid bounds derived from 11330 * e.g. dead branches. 11331 */ 11332 __mark_reg_unknown(env, dst_reg); 11333 return 0; 11334 } 11335 11336 if (BPF_CLASS(insn->code) != BPF_ALU64) { 11337 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 11338 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 11339 __mark_reg_unknown(env, dst_reg); 11340 return 0; 11341 } 11342 11343 verbose(env, 11344 "R%d 32-bit pointer arithmetic prohibited\n", 11345 dst); 11346 return -EACCES; 11347 } 11348 11349 if (ptr_reg->type & PTR_MAYBE_NULL) { 11350 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 11351 dst, reg_type_str(env, ptr_reg->type)); 11352 return -EACCES; 11353 } 11354 11355 switch (base_type(ptr_reg->type)) { 11356 case CONST_PTR_TO_MAP: 11357 /* smin_val represents the known value */ 11358 if (known && smin_val == 0 && opcode == BPF_ADD) 11359 break; 11360 fallthrough; 11361 case PTR_TO_PACKET_END: 11362 case PTR_TO_SOCKET: 11363 case PTR_TO_SOCK_COMMON: 11364 case PTR_TO_TCP_SOCK: 11365 case PTR_TO_XDP_SOCK: 11366 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 11367 dst, reg_type_str(env, ptr_reg->type)); 11368 return -EACCES; 11369 default: 11370 break; 11371 } 11372 11373 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 11374 * The id may be overwritten later if we create a new variable offset. 11375 */ 11376 dst_reg->type = ptr_reg->type; 11377 dst_reg->id = ptr_reg->id; 11378 11379 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 11380 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 11381 return -EINVAL; 11382 11383 /* pointer types do not carry 32-bit bounds at the moment. */ 11384 __mark_reg32_unbounded(dst_reg); 11385 11386 if (sanitize_needed(opcode)) { 11387 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 11388 &info, false); 11389 if (ret < 0) 11390 return sanitize_err(env, insn, ret, off_reg, dst_reg); 11391 } 11392 11393 switch (opcode) { 11394 case BPF_ADD: 11395 /* We can take a fixed offset as long as it doesn't overflow 11396 * the s32 'off' field 11397 */ 11398 if (known && (ptr_reg->off + smin_val == 11399 (s64)(s32)(ptr_reg->off + smin_val))) { 11400 /* pointer += K. Accumulate it into fixed offset */ 11401 dst_reg->smin_value = smin_ptr; 11402 dst_reg->smax_value = smax_ptr; 11403 dst_reg->umin_value = umin_ptr; 11404 dst_reg->umax_value = umax_ptr; 11405 dst_reg->var_off = ptr_reg->var_off; 11406 dst_reg->off = ptr_reg->off + smin_val; 11407 dst_reg->raw = ptr_reg->raw; 11408 break; 11409 } 11410 /* A new variable offset is created. Note that off_reg->off 11411 * == 0, since it's a scalar. 11412 * dst_reg gets the pointer type and since some positive 11413 * integer value was added to the pointer, give it a new 'id' 11414 * if it's a PTR_TO_PACKET. 11415 * this creates a new 'base' pointer, off_reg (variable) gets 11416 * added into the variable offset, and we copy the fixed offset 11417 * from ptr_reg. 11418 */ 11419 if (signed_add_overflows(smin_ptr, smin_val) || 11420 signed_add_overflows(smax_ptr, smax_val)) { 11421 dst_reg->smin_value = S64_MIN; 11422 dst_reg->smax_value = S64_MAX; 11423 } else { 11424 dst_reg->smin_value = smin_ptr + smin_val; 11425 dst_reg->smax_value = smax_ptr + smax_val; 11426 } 11427 if (umin_ptr + umin_val < umin_ptr || 11428 umax_ptr + umax_val < umax_ptr) { 11429 dst_reg->umin_value = 0; 11430 dst_reg->umax_value = U64_MAX; 11431 } else { 11432 dst_reg->umin_value = umin_ptr + umin_val; 11433 dst_reg->umax_value = umax_ptr + umax_val; 11434 } 11435 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 11436 dst_reg->off = ptr_reg->off; 11437 dst_reg->raw = ptr_reg->raw; 11438 if (reg_is_pkt_pointer(ptr_reg)) { 11439 dst_reg->id = ++env->id_gen; 11440 /* something was added to pkt_ptr, set range to zero */ 11441 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 11442 } 11443 break; 11444 case BPF_SUB: 11445 if (dst_reg == off_reg) { 11446 /* scalar -= pointer. Creates an unknown scalar */ 11447 verbose(env, "R%d tried to subtract pointer from scalar\n", 11448 dst); 11449 return -EACCES; 11450 } 11451 /* We don't allow subtraction from FP, because (according to 11452 * test_verifier.c test "invalid fp arithmetic", JITs might not 11453 * be able to deal with it. 11454 */ 11455 if (ptr_reg->type == PTR_TO_STACK) { 11456 verbose(env, "R%d subtraction from stack pointer prohibited\n", 11457 dst); 11458 return -EACCES; 11459 } 11460 if (known && (ptr_reg->off - smin_val == 11461 (s64)(s32)(ptr_reg->off - smin_val))) { 11462 /* pointer -= K. Subtract it from fixed offset */ 11463 dst_reg->smin_value = smin_ptr; 11464 dst_reg->smax_value = smax_ptr; 11465 dst_reg->umin_value = umin_ptr; 11466 dst_reg->umax_value = umax_ptr; 11467 dst_reg->var_off = ptr_reg->var_off; 11468 dst_reg->id = ptr_reg->id; 11469 dst_reg->off = ptr_reg->off - smin_val; 11470 dst_reg->raw = ptr_reg->raw; 11471 break; 11472 } 11473 /* A new variable offset is created. If the subtrahend is known 11474 * nonnegative, then any reg->range we had before is still good. 11475 */ 11476 if (signed_sub_overflows(smin_ptr, smax_val) || 11477 signed_sub_overflows(smax_ptr, smin_val)) { 11478 /* Overflow possible, we know nothing */ 11479 dst_reg->smin_value = S64_MIN; 11480 dst_reg->smax_value = S64_MAX; 11481 } else { 11482 dst_reg->smin_value = smin_ptr - smax_val; 11483 dst_reg->smax_value = smax_ptr - smin_val; 11484 } 11485 if (umin_ptr < umax_val) { 11486 /* Overflow possible, we know nothing */ 11487 dst_reg->umin_value = 0; 11488 dst_reg->umax_value = U64_MAX; 11489 } else { 11490 /* Cannot overflow (as long as bounds are consistent) */ 11491 dst_reg->umin_value = umin_ptr - umax_val; 11492 dst_reg->umax_value = umax_ptr - umin_val; 11493 } 11494 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 11495 dst_reg->off = ptr_reg->off; 11496 dst_reg->raw = ptr_reg->raw; 11497 if (reg_is_pkt_pointer(ptr_reg)) { 11498 dst_reg->id = ++env->id_gen; 11499 /* something was added to pkt_ptr, set range to zero */ 11500 if (smin_val < 0) 11501 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 11502 } 11503 break; 11504 case BPF_AND: 11505 case BPF_OR: 11506 case BPF_XOR: 11507 /* bitwise ops on pointers are troublesome, prohibit. */ 11508 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 11509 dst, bpf_alu_string[opcode >> 4]); 11510 return -EACCES; 11511 default: 11512 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 11513 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 11514 dst, bpf_alu_string[opcode >> 4]); 11515 return -EACCES; 11516 } 11517 11518 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 11519 return -EINVAL; 11520 reg_bounds_sync(dst_reg); 11521 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 11522 return -EACCES; 11523 if (sanitize_needed(opcode)) { 11524 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 11525 &info, true); 11526 if (ret < 0) 11527 return sanitize_err(env, insn, ret, off_reg, dst_reg); 11528 } 11529 11530 return 0; 11531 } 11532 11533 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 11534 struct bpf_reg_state *src_reg) 11535 { 11536 s32 smin_val = src_reg->s32_min_value; 11537 s32 smax_val = src_reg->s32_max_value; 11538 u32 umin_val = src_reg->u32_min_value; 11539 u32 umax_val = src_reg->u32_max_value; 11540 11541 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 11542 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 11543 dst_reg->s32_min_value = S32_MIN; 11544 dst_reg->s32_max_value = S32_MAX; 11545 } else { 11546 dst_reg->s32_min_value += smin_val; 11547 dst_reg->s32_max_value += smax_val; 11548 } 11549 if (dst_reg->u32_min_value + umin_val < umin_val || 11550 dst_reg->u32_max_value + umax_val < umax_val) { 11551 dst_reg->u32_min_value = 0; 11552 dst_reg->u32_max_value = U32_MAX; 11553 } else { 11554 dst_reg->u32_min_value += umin_val; 11555 dst_reg->u32_max_value += umax_val; 11556 } 11557 } 11558 11559 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 11560 struct bpf_reg_state *src_reg) 11561 { 11562 s64 smin_val = src_reg->smin_value; 11563 s64 smax_val = src_reg->smax_value; 11564 u64 umin_val = src_reg->umin_value; 11565 u64 umax_val = src_reg->umax_value; 11566 11567 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 11568 signed_add_overflows(dst_reg->smax_value, smax_val)) { 11569 dst_reg->smin_value = S64_MIN; 11570 dst_reg->smax_value = S64_MAX; 11571 } else { 11572 dst_reg->smin_value += smin_val; 11573 dst_reg->smax_value += smax_val; 11574 } 11575 if (dst_reg->umin_value + umin_val < umin_val || 11576 dst_reg->umax_value + umax_val < umax_val) { 11577 dst_reg->umin_value = 0; 11578 dst_reg->umax_value = U64_MAX; 11579 } else { 11580 dst_reg->umin_value += umin_val; 11581 dst_reg->umax_value += umax_val; 11582 } 11583 } 11584 11585 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 11586 struct bpf_reg_state *src_reg) 11587 { 11588 s32 smin_val = src_reg->s32_min_value; 11589 s32 smax_val = src_reg->s32_max_value; 11590 u32 umin_val = src_reg->u32_min_value; 11591 u32 umax_val = src_reg->u32_max_value; 11592 11593 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 11594 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 11595 /* Overflow possible, we know nothing */ 11596 dst_reg->s32_min_value = S32_MIN; 11597 dst_reg->s32_max_value = S32_MAX; 11598 } else { 11599 dst_reg->s32_min_value -= smax_val; 11600 dst_reg->s32_max_value -= smin_val; 11601 } 11602 if (dst_reg->u32_min_value < umax_val) { 11603 /* Overflow possible, we know nothing */ 11604 dst_reg->u32_min_value = 0; 11605 dst_reg->u32_max_value = U32_MAX; 11606 } else { 11607 /* Cannot overflow (as long as bounds are consistent) */ 11608 dst_reg->u32_min_value -= umax_val; 11609 dst_reg->u32_max_value -= umin_val; 11610 } 11611 } 11612 11613 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 11614 struct bpf_reg_state *src_reg) 11615 { 11616 s64 smin_val = src_reg->smin_value; 11617 s64 smax_val = src_reg->smax_value; 11618 u64 umin_val = src_reg->umin_value; 11619 u64 umax_val = src_reg->umax_value; 11620 11621 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 11622 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 11623 /* Overflow possible, we know nothing */ 11624 dst_reg->smin_value = S64_MIN; 11625 dst_reg->smax_value = S64_MAX; 11626 } else { 11627 dst_reg->smin_value -= smax_val; 11628 dst_reg->smax_value -= smin_val; 11629 } 11630 if (dst_reg->umin_value < umax_val) { 11631 /* Overflow possible, we know nothing */ 11632 dst_reg->umin_value = 0; 11633 dst_reg->umax_value = U64_MAX; 11634 } else { 11635 /* Cannot overflow (as long as bounds are consistent) */ 11636 dst_reg->umin_value -= umax_val; 11637 dst_reg->umax_value -= umin_val; 11638 } 11639 } 11640 11641 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 11642 struct bpf_reg_state *src_reg) 11643 { 11644 s32 smin_val = src_reg->s32_min_value; 11645 u32 umin_val = src_reg->u32_min_value; 11646 u32 umax_val = src_reg->u32_max_value; 11647 11648 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 11649 /* Ain't nobody got time to multiply that sign */ 11650 __mark_reg32_unbounded(dst_reg); 11651 return; 11652 } 11653 /* Both values are positive, so we can work with unsigned and 11654 * copy the result to signed (unless it exceeds S32_MAX). 11655 */ 11656 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 11657 /* Potential overflow, we know nothing */ 11658 __mark_reg32_unbounded(dst_reg); 11659 return; 11660 } 11661 dst_reg->u32_min_value *= umin_val; 11662 dst_reg->u32_max_value *= umax_val; 11663 if (dst_reg->u32_max_value > S32_MAX) { 11664 /* Overflow possible, we know nothing */ 11665 dst_reg->s32_min_value = S32_MIN; 11666 dst_reg->s32_max_value = S32_MAX; 11667 } else { 11668 dst_reg->s32_min_value = dst_reg->u32_min_value; 11669 dst_reg->s32_max_value = dst_reg->u32_max_value; 11670 } 11671 } 11672 11673 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 11674 struct bpf_reg_state *src_reg) 11675 { 11676 s64 smin_val = src_reg->smin_value; 11677 u64 umin_val = src_reg->umin_value; 11678 u64 umax_val = src_reg->umax_value; 11679 11680 if (smin_val < 0 || dst_reg->smin_value < 0) { 11681 /* Ain't nobody got time to multiply that sign */ 11682 __mark_reg64_unbounded(dst_reg); 11683 return; 11684 } 11685 /* Both values are positive, so we can work with unsigned and 11686 * copy the result to signed (unless it exceeds S64_MAX). 11687 */ 11688 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 11689 /* Potential overflow, we know nothing */ 11690 __mark_reg64_unbounded(dst_reg); 11691 return; 11692 } 11693 dst_reg->umin_value *= umin_val; 11694 dst_reg->umax_value *= umax_val; 11695 if (dst_reg->umax_value > S64_MAX) { 11696 /* Overflow possible, we know nothing */ 11697 dst_reg->smin_value = S64_MIN; 11698 dst_reg->smax_value = S64_MAX; 11699 } else { 11700 dst_reg->smin_value = dst_reg->umin_value; 11701 dst_reg->smax_value = dst_reg->umax_value; 11702 } 11703 } 11704 11705 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 11706 struct bpf_reg_state *src_reg) 11707 { 11708 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11709 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11710 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11711 s32 smin_val = src_reg->s32_min_value; 11712 u32 umax_val = src_reg->u32_max_value; 11713 11714 if (src_known && dst_known) { 11715 __mark_reg32_known(dst_reg, var32_off.value); 11716 return; 11717 } 11718 11719 /* We get our minimum from the var_off, since that's inherently 11720 * bitwise. Our maximum is the minimum of the operands' maxima. 11721 */ 11722 dst_reg->u32_min_value = var32_off.value; 11723 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 11724 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 11725 /* Lose signed bounds when ANDing negative numbers, 11726 * ain't nobody got time for that. 11727 */ 11728 dst_reg->s32_min_value = S32_MIN; 11729 dst_reg->s32_max_value = S32_MAX; 11730 } else { 11731 /* ANDing two positives gives a positive, so safe to 11732 * cast result into s64. 11733 */ 11734 dst_reg->s32_min_value = dst_reg->u32_min_value; 11735 dst_reg->s32_max_value = dst_reg->u32_max_value; 11736 } 11737 } 11738 11739 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 11740 struct bpf_reg_state *src_reg) 11741 { 11742 bool src_known = tnum_is_const(src_reg->var_off); 11743 bool dst_known = tnum_is_const(dst_reg->var_off); 11744 s64 smin_val = src_reg->smin_value; 11745 u64 umax_val = src_reg->umax_value; 11746 11747 if (src_known && dst_known) { 11748 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11749 return; 11750 } 11751 11752 /* We get our minimum from the var_off, since that's inherently 11753 * bitwise. Our maximum is the minimum of the operands' maxima. 11754 */ 11755 dst_reg->umin_value = dst_reg->var_off.value; 11756 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 11757 if (dst_reg->smin_value < 0 || smin_val < 0) { 11758 /* Lose signed bounds when ANDing negative numbers, 11759 * ain't nobody got time for that. 11760 */ 11761 dst_reg->smin_value = S64_MIN; 11762 dst_reg->smax_value = S64_MAX; 11763 } else { 11764 /* ANDing two positives gives a positive, so safe to 11765 * cast result into s64. 11766 */ 11767 dst_reg->smin_value = dst_reg->umin_value; 11768 dst_reg->smax_value = dst_reg->umax_value; 11769 } 11770 /* We may learn something more from the var_off */ 11771 __update_reg_bounds(dst_reg); 11772 } 11773 11774 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 11775 struct bpf_reg_state *src_reg) 11776 { 11777 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11778 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11779 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11780 s32 smin_val = src_reg->s32_min_value; 11781 u32 umin_val = src_reg->u32_min_value; 11782 11783 if (src_known && dst_known) { 11784 __mark_reg32_known(dst_reg, var32_off.value); 11785 return; 11786 } 11787 11788 /* We get our maximum from the var_off, and our minimum is the 11789 * maximum of the operands' minima 11790 */ 11791 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 11792 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 11793 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 11794 /* Lose signed bounds when ORing negative numbers, 11795 * ain't nobody got time for that. 11796 */ 11797 dst_reg->s32_min_value = S32_MIN; 11798 dst_reg->s32_max_value = S32_MAX; 11799 } else { 11800 /* ORing two positives gives a positive, so safe to 11801 * cast result into s64. 11802 */ 11803 dst_reg->s32_min_value = dst_reg->u32_min_value; 11804 dst_reg->s32_max_value = dst_reg->u32_max_value; 11805 } 11806 } 11807 11808 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 11809 struct bpf_reg_state *src_reg) 11810 { 11811 bool src_known = tnum_is_const(src_reg->var_off); 11812 bool dst_known = tnum_is_const(dst_reg->var_off); 11813 s64 smin_val = src_reg->smin_value; 11814 u64 umin_val = src_reg->umin_value; 11815 11816 if (src_known && dst_known) { 11817 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11818 return; 11819 } 11820 11821 /* We get our maximum from the var_off, and our minimum is the 11822 * maximum of the operands' minima 11823 */ 11824 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 11825 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 11826 if (dst_reg->smin_value < 0 || smin_val < 0) { 11827 /* Lose signed bounds when ORing negative numbers, 11828 * ain't nobody got time for that. 11829 */ 11830 dst_reg->smin_value = S64_MIN; 11831 dst_reg->smax_value = S64_MAX; 11832 } else { 11833 /* ORing two positives gives a positive, so safe to 11834 * cast result into s64. 11835 */ 11836 dst_reg->smin_value = dst_reg->umin_value; 11837 dst_reg->smax_value = dst_reg->umax_value; 11838 } 11839 /* We may learn something more from the var_off */ 11840 __update_reg_bounds(dst_reg); 11841 } 11842 11843 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 11844 struct bpf_reg_state *src_reg) 11845 { 11846 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11847 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11848 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11849 s32 smin_val = src_reg->s32_min_value; 11850 11851 if (src_known && dst_known) { 11852 __mark_reg32_known(dst_reg, var32_off.value); 11853 return; 11854 } 11855 11856 /* We get both minimum and maximum from the var32_off. */ 11857 dst_reg->u32_min_value = var32_off.value; 11858 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 11859 11860 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 11861 /* XORing two positive sign numbers gives a positive, 11862 * so safe to cast u32 result into s32. 11863 */ 11864 dst_reg->s32_min_value = dst_reg->u32_min_value; 11865 dst_reg->s32_max_value = dst_reg->u32_max_value; 11866 } else { 11867 dst_reg->s32_min_value = S32_MIN; 11868 dst_reg->s32_max_value = S32_MAX; 11869 } 11870 } 11871 11872 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 11873 struct bpf_reg_state *src_reg) 11874 { 11875 bool src_known = tnum_is_const(src_reg->var_off); 11876 bool dst_known = tnum_is_const(dst_reg->var_off); 11877 s64 smin_val = src_reg->smin_value; 11878 11879 if (src_known && dst_known) { 11880 /* dst_reg->var_off.value has been updated earlier */ 11881 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11882 return; 11883 } 11884 11885 /* We get both minimum and maximum from the var_off. */ 11886 dst_reg->umin_value = dst_reg->var_off.value; 11887 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 11888 11889 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 11890 /* XORing two positive sign numbers gives a positive, 11891 * so safe to cast u64 result into s64. 11892 */ 11893 dst_reg->smin_value = dst_reg->umin_value; 11894 dst_reg->smax_value = dst_reg->umax_value; 11895 } else { 11896 dst_reg->smin_value = S64_MIN; 11897 dst_reg->smax_value = S64_MAX; 11898 } 11899 11900 __update_reg_bounds(dst_reg); 11901 } 11902 11903 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 11904 u64 umin_val, u64 umax_val) 11905 { 11906 /* We lose all sign bit information (except what we can pick 11907 * up from var_off) 11908 */ 11909 dst_reg->s32_min_value = S32_MIN; 11910 dst_reg->s32_max_value = S32_MAX; 11911 /* If we might shift our top bit out, then we know nothing */ 11912 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 11913 dst_reg->u32_min_value = 0; 11914 dst_reg->u32_max_value = U32_MAX; 11915 } else { 11916 dst_reg->u32_min_value <<= umin_val; 11917 dst_reg->u32_max_value <<= umax_val; 11918 } 11919 } 11920 11921 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 11922 struct bpf_reg_state *src_reg) 11923 { 11924 u32 umax_val = src_reg->u32_max_value; 11925 u32 umin_val = src_reg->u32_min_value; 11926 /* u32 alu operation will zext upper bits */ 11927 struct tnum subreg = tnum_subreg(dst_reg->var_off); 11928 11929 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 11930 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 11931 /* Not required but being careful mark reg64 bounds as unknown so 11932 * that we are forced to pick them up from tnum and zext later and 11933 * if some path skips this step we are still safe. 11934 */ 11935 __mark_reg64_unbounded(dst_reg); 11936 __update_reg32_bounds(dst_reg); 11937 } 11938 11939 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 11940 u64 umin_val, u64 umax_val) 11941 { 11942 /* Special case <<32 because it is a common compiler pattern to sign 11943 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 11944 * positive we know this shift will also be positive so we can track 11945 * bounds correctly. Otherwise we lose all sign bit information except 11946 * what we can pick up from var_off. Perhaps we can generalize this 11947 * later to shifts of any length. 11948 */ 11949 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 11950 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 11951 else 11952 dst_reg->smax_value = S64_MAX; 11953 11954 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 11955 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 11956 else 11957 dst_reg->smin_value = S64_MIN; 11958 11959 /* If we might shift our top bit out, then we know nothing */ 11960 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 11961 dst_reg->umin_value = 0; 11962 dst_reg->umax_value = U64_MAX; 11963 } else { 11964 dst_reg->umin_value <<= umin_val; 11965 dst_reg->umax_value <<= umax_val; 11966 } 11967 } 11968 11969 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 11970 struct bpf_reg_state *src_reg) 11971 { 11972 u64 umax_val = src_reg->umax_value; 11973 u64 umin_val = src_reg->umin_value; 11974 11975 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 11976 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 11977 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 11978 11979 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 11980 /* We may learn something more from the var_off */ 11981 __update_reg_bounds(dst_reg); 11982 } 11983 11984 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 11985 struct bpf_reg_state *src_reg) 11986 { 11987 struct tnum subreg = tnum_subreg(dst_reg->var_off); 11988 u32 umax_val = src_reg->u32_max_value; 11989 u32 umin_val = src_reg->u32_min_value; 11990 11991 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 11992 * be negative, then either: 11993 * 1) src_reg might be zero, so the sign bit of the result is 11994 * unknown, so we lose our signed bounds 11995 * 2) it's known negative, thus the unsigned bounds capture the 11996 * signed bounds 11997 * 3) the signed bounds cross zero, so they tell us nothing 11998 * about the result 11999 * If the value in dst_reg is known nonnegative, then again the 12000 * unsigned bounds capture the signed bounds. 12001 * Thus, in all cases it suffices to blow away our signed bounds 12002 * and rely on inferring new ones from the unsigned bounds and 12003 * var_off of the result. 12004 */ 12005 dst_reg->s32_min_value = S32_MIN; 12006 dst_reg->s32_max_value = S32_MAX; 12007 12008 dst_reg->var_off = tnum_rshift(subreg, umin_val); 12009 dst_reg->u32_min_value >>= umax_val; 12010 dst_reg->u32_max_value >>= umin_val; 12011 12012 __mark_reg64_unbounded(dst_reg); 12013 __update_reg32_bounds(dst_reg); 12014 } 12015 12016 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 12017 struct bpf_reg_state *src_reg) 12018 { 12019 u64 umax_val = src_reg->umax_value; 12020 u64 umin_val = src_reg->umin_value; 12021 12022 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 12023 * be negative, then either: 12024 * 1) src_reg might be zero, so the sign bit of the result is 12025 * unknown, so we lose our signed bounds 12026 * 2) it's known negative, thus the unsigned bounds capture the 12027 * signed bounds 12028 * 3) the signed bounds cross zero, so they tell us nothing 12029 * about the result 12030 * If the value in dst_reg is known nonnegative, then again the 12031 * unsigned bounds capture the signed bounds. 12032 * Thus, in all cases it suffices to blow away our signed bounds 12033 * and rely on inferring new ones from the unsigned bounds and 12034 * var_off of the result. 12035 */ 12036 dst_reg->smin_value = S64_MIN; 12037 dst_reg->smax_value = S64_MAX; 12038 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 12039 dst_reg->umin_value >>= umax_val; 12040 dst_reg->umax_value >>= umin_val; 12041 12042 /* Its not easy to operate on alu32 bounds here because it depends 12043 * on bits being shifted in. Take easy way out and mark unbounded 12044 * so we can recalculate later from tnum. 12045 */ 12046 __mark_reg32_unbounded(dst_reg); 12047 __update_reg_bounds(dst_reg); 12048 } 12049 12050 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 12051 struct bpf_reg_state *src_reg) 12052 { 12053 u64 umin_val = src_reg->u32_min_value; 12054 12055 /* Upon reaching here, src_known is true and 12056 * umax_val is equal to umin_val. 12057 */ 12058 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 12059 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 12060 12061 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 12062 12063 /* blow away the dst_reg umin_value/umax_value and rely on 12064 * dst_reg var_off to refine the result. 12065 */ 12066 dst_reg->u32_min_value = 0; 12067 dst_reg->u32_max_value = U32_MAX; 12068 12069 __mark_reg64_unbounded(dst_reg); 12070 __update_reg32_bounds(dst_reg); 12071 } 12072 12073 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 12074 struct bpf_reg_state *src_reg) 12075 { 12076 u64 umin_val = src_reg->umin_value; 12077 12078 /* Upon reaching here, src_known is true and umax_val is equal 12079 * to umin_val. 12080 */ 12081 dst_reg->smin_value >>= umin_val; 12082 dst_reg->smax_value >>= umin_val; 12083 12084 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 12085 12086 /* blow away the dst_reg umin_value/umax_value and rely on 12087 * dst_reg var_off to refine the result. 12088 */ 12089 dst_reg->umin_value = 0; 12090 dst_reg->umax_value = U64_MAX; 12091 12092 /* Its not easy to operate on alu32 bounds here because it depends 12093 * on bits being shifted in from upper 32-bits. Take easy way out 12094 * and mark unbounded so we can recalculate later from tnum. 12095 */ 12096 __mark_reg32_unbounded(dst_reg); 12097 __update_reg_bounds(dst_reg); 12098 } 12099 12100 /* WARNING: This function does calculations on 64-bit values, but the actual 12101 * execution may occur on 32-bit values. Therefore, things like bitshifts 12102 * need extra checks in the 32-bit case. 12103 */ 12104 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 12105 struct bpf_insn *insn, 12106 struct bpf_reg_state *dst_reg, 12107 struct bpf_reg_state src_reg) 12108 { 12109 struct bpf_reg_state *regs = cur_regs(env); 12110 u8 opcode = BPF_OP(insn->code); 12111 bool src_known; 12112 s64 smin_val, smax_val; 12113 u64 umin_val, umax_val; 12114 s32 s32_min_val, s32_max_val; 12115 u32 u32_min_val, u32_max_val; 12116 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 12117 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 12118 int ret; 12119 12120 smin_val = src_reg.smin_value; 12121 smax_val = src_reg.smax_value; 12122 umin_val = src_reg.umin_value; 12123 umax_val = src_reg.umax_value; 12124 12125 s32_min_val = src_reg.s32_min_value; 12126 s32_max_val = src_reg.s32_max_value; 12127 u32_min_val = src_reg.u32_min_value; 12128 u32_max_val = src_reg.u32_max_value; 12129 12130 if (alu32) { 12131 src_known = tnum_subreg_is_const(src_reg.var_off); 12132 if ((src_known && 12133 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 12134 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 12135 /* Taint dst register if offset had invalid bounds 12136 * derived from e.g. dead branches. 12137 */ 12138 __mark_reg_unknown(env, dst_reg); 12139 return 0; 12140 } 12141 } else { 12142 src_known = tnum_is_const(src_reg.var_off); 12143 if ((src_known && 12144 (smin_val != smax_val || umin_val != umax_val)) || 12145 smin_val > smax_val || umin_val > umax_val) { 12146 /* Taint dst register if offset had invalid bounds 12147 * derived from e.g. dead branches. 12148 */ 12149 __mark_reg_unknown(env, dst_reg); 12150 return 0; 12151 } 12152 } 12153 12154 if (!src_known && 12155 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 12156 __mark_reg_unknown(env, dst_reg); 12157 return 0; 12158 } 12159 12160 if (sanitize_needed(opcode)) { 12161 ret = sanitize_val_alu(env, insn); 12162 if (ret < 0) 12163 return sanitize_err(env, insn, ret, NULL, NULL); 12164 } 12165 12166 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 12167 * There are two classes of instructions: The first class we track both 12168 * alu32 and alu64 sign/unsigned bounds independently this provides the 12169 * greatest amount of precision when alu operations are mixed with jmp32 12170 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 12171 * and BPF_OR. This is possible because these ops have fairly easy to 12172 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 12173 * See alu32 verifier tests for examples. The second class of 12174 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 12175 * with regards to tracking sign/unsigned bounds because the bits may 12176 * cross subreg boundaries in the alu64 case. When this happens we mark 12177 * the reg unbounded in the subreg bound space and use the resulting 12178 * tnum to calculate an approximation of the sign/unsigned bounds. 12179 */ 12180 switch (opcode) { 12181 case BPF_ADD: 12182 scalar32_min_max_add(dst_reg, &src_reg); 12183 scalar_min_max_add(dst_reg, &src_reg); 12184 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 12185 break; 12186 case BPF_SUB: 12187 scalar32_min_max_sub(dst_reg, &src_reg); 12188 scalar_min_max_sub(dst_reg, &src_reg); 12189 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 12190 break; 12191 case BPF_MUL: 12192 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 12193 scalar32_min_max_mul(dst_reg, &src_reg); 12194 scalar_min_max_mul(dst_reg, &src_reg); 12195 break; 12196 case BPF_AND: 12197 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 12198 scalar32_min_max_and(dst_reg, &src_reg); 12199 scalar_min_max_and(dst_reg, &src_reg); 12200 break; 12201 case BPF_OR: 12202 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 12203 scalar32_min_max_or(dst_reg, &src_reg); 12204 scalar_min_max_or(dst_reg, &src_reg); 12205 break; 12206 case BPF_XOR: 12207 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 12208 scalar32_min_max_xor(dst_reg, &src_reg); 12209 scalar_min_max_xor(dst_reg, &src_reg); 12210 break; 12211 case BPF_LSH: 12212 if (umax_val >= insn_bitness) { 12213 /* Shifts greater than 31 or 63 are undefined. 12214 * This includes shifts by a negative number. 12215 */ 12216 mark_reg_unknown(env, regs, insn->dst_reg); 12217 break; 12218 } 12219 if (alu32) 12220 scalar32_min_max_lsh(dst_reg, &src_reg); 12221 else 12222 scalar_min_max_lsh(dst_reg, &src_reg); 12223 break; 12224 case BPF_RSH: 12225 if (umax_val >= insn_bitness) { 12226 /* Shifts greater than 31 or 63 are undefined. 12227 * This includes shifts by a negative number. 12228 */ 12229 mark_reg_unknown(env, regs, insn->dst_reg); 12230 break; 12231 } 12232 if (alu32) 12233 scalar32_min_max_rsh(dst_reg, &src_reg); 12234 else 12235 scalar_min_max_rsh(dst_reg, &src_reg); 12236 break; 12237 case BPF_ARSH: 12238 if (umax_val >= insn_bitness) { 12239 /* Shifts greater than 31 or 63 are undefined. 12240 * This includes shifts by a negative number. 12241 */ 12242 mark_reg_unknown(env, regs, insn->dst_reg); 12243 break; 12244 } 12245 if (alu32) 12246 scalar32_min_max_arsh(dst_reg, &src_reg); 12247 else 12248 scalar_min_max_arsh(dst_reg, &src_reg); 12249 break; 12250 default: 12251 mark_reg_unknown(env, regs, insn->dst_reg); 12252 break; 12253 } 12254 12255 /* ALU32 ops are zero extended into 64bit register */ 12256 if (alu32) 12257 zext_32_to_64(dst_reg); 12258 reg_bounds_sync(dst_reg); 12259 return 0; 12260 } 12261 12262 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 12263 * and var_off. 12264 */ 12265 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 12266 struct bpf_insn *insn) 12267 { 12268 struct bpf_verifier_state *vstate = env->cur_state; 12269 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 12270 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 12271 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 12272 u8 opcode = BPF_OP(insn->code); 12273 int err; 12274 12275 dst_reg = ®s[insn->dst_reg]; 12276 src_reg = NULL; 12277 if (dst_reg->type != SCALAR_VALUE) 12278 ptr_reg = dst_reg; 12279 else 12280 /* Make sure ID is cleared otherwise dst_reg min/max could be 12281 * incorrectly propagated into other registers by find_equal_scalars() 12282 */ 12283 dst_reg->id = 0; 12284 if (BPF_SRC(insn->code) == BPF_X) { 12285 src_reg = ®s[insn->src_reg]; 12286 if (src_reg->type != SCALAR_VALUE) { 12287 if (dst_reg->type != SCALAR_VALUE) { 12288 /* Combining two pointers by any ALU op yields 12289 * an arbitrary scalar. Disallow all math except 12290 * pointer subtraction 12291 */ 12292 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 12293 mark_reg_unknown(env, regs, insn->dst_reg); 12294 return 0; 12295 } 12296 verbose(env, "R%d pointer %s pointer prohibited\n", 12297 insn->dst_reg, 12298 bpf_alu_string[opcode >> 4]); 12299 return -EACCES; 12300 } else { 12301 /* scalar += pointer 12302 * This is legal, but we have to reverse our 12303 * src/dest handling in computing the range 12304 */ 12305 err = mark_chain_precision(env, insn->dst_reg); 12306 if (err) 12307 return err; 12308 return adjust_ptr_min_max_vals(env, insn, 12309 src_reg, dst_reg); 12310 } 12311 } else if (ptr_reg) { 12312 /* pointer += scalar */ 12313 err = mark_chain_precision(env, insn->src_reg); 12314 if (err) 12315 return err; 12316 return adjust_ptr_min_max_vals(env, insn, 12317 dst_reg, src_reg); 12318 } else if (dst_reg->precise) { 12319 /* if dst_reg is precise, src_reg should be precise as well */ 12320 err = mark_chain_precision(env, insn->src_reg); 12321 if (err) 12322 return err; 12323 } 12324 } else { 12325 /* Pretend the src is a reg with a known value, since we only 12326 * need to be able to read from this state. 12327 */ 12328 off_reg.type = SCALAR_VALUE; 12329 __mark_reg_known(&off_reg, insn->imm); 12330 src_reg = &off_reg; 12331 if (ptr_reg) /* pointer += K */ 12332 return adjust_ptr_min_max_vals(env, insn, 12333 ptr_reg, src_reg); 12334 } 12335 12336 /* Got here implies adding two SCALAR_VALUEs */ 12337 if (WARN_ON_ONCE(ptr_reg)) { 12338 print_verifier_state(env, state, true); 12339 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 12340 return -EINVAL; 12341 } 12342 if (WARN_ON(!src_reg)) { 12343 print_verifier_state(env, state, true); 12344 verbose(env, "verifier internal error: no src_reg\n"); 12345 return -EINVAL; 12346 } 12347 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 12348 } 12349 12350 /* check validity of 32-bit and 64-bit arithmetic operations */ 12351 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 12352 { 12353 struct bpf_reg_state *regs = cur_regs(env); 12354 u8 opcode = BPF_OP(insn->code); 12355 int err; 12356 12357 if (opcode == BPF_END || opcode == BPF_NEG) { 12358 if (opcode == BPF_NEG) { 12359 if (BPF_SRC(insn->code) != BPF_K || 12360 insn->src_reg != BPF_REG_0 || 12361 insn->off != 0 || insn->imm != 0) { 12362 verbose(env, "BPF_NEG uses reserved fields\n"); 12363 return -EINVAL; 12364 } 12365 } else { 12366 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 12367 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 12368 BPF_CLASS(insn->code) == BPF_ALU64) { 12369 verbose(env, "BPF_END uses reserved fields\n"); 12370 return -EINVAL; 12371 } 12372 } 12373 12374 /* check src operand */ 12375 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 12376 if (err) 12377 return err; 12378 12379 if (is_pointer_value(env, insn->dst_reg)) { 12380 verbose(env, "R%d pointer arithmetic prohibited\n", 12381 insn->dst_reg); 12382 return -EACCES; 12383 } 12384 12385 /* check dest operand */ 12386 err = check_reg_arg(env, insn->dst_reg, DST_OP); 12387 if (err) 12388 return err; 12389 12390 } else if (opcode == BPF_MOV) { 12391 12392 if (BPF_SRC(insn->code) == BPF_X) { 12393 if (insn->imm != 0 || insn->off != 0) { 12394 verbose(env, "BPF_MOV uses reserved fields\n"); 12395 return -EINVAL; 12396 } 12397 12398 /* check src operand */ 12399 err = check_reg_arg(env, insn->src_reg, SRC_OP); 12400 if (err) 12401 return err; 12402 } else { 12403 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 12404 verbose(env, "BPF_MOV uses reserved fields\n"); 12405 return -EINVAL; 12406 } 12407 } 12408 12409 /* check dest operand, mark as required later */ 12410 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 12411 if (err) 12412 return err; 12413 12414 if (BPF_SRC(insn->code) == BPF_X) { 12415 struct bpf_reg_state *src_reg = regs + insn->src_reg; 12416 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 12417 12418 if (BPF_CLASS(insn->code) == BPF_ALU64) { 12419 /* case: R1 = R2 12420 * copy register state to dest reg 12421 */ 12422 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 12423 /* Assign src and dst registers the same ID 12424 * that will be used by find_equal_scalars() 12425 * to propagate min/max range. 12426 */ 12427 src_reg->id = ++env->id_gen; 12428 copy_register_state(dst_reg, src_reg); 12429 dst_reg->live |= REG_LIVE_WRITTEN; 12430 dst_reg->subreg_def = DEF_NOT_SUBREG; 12431 } else { 12432 /* R1 = (u32) R2 */ 12433 if (is_pointer_value(env, insn->src_reg)) { 12434 verbose(env, 12435 "R%d partial copy of pointer\n", 12436 insn->src_reg); 12437 return -EACCES; 12438 } else if (src_reg->type == SCALAR_VALUE) { 12439 bool is_src_reg_u32 = src_reg->umax_value <= U32_MAX; 12440 12441 if (is_src_reg_u32 && !src_reg->id) 12442 src_reg->id = ++env->id_gen; 12443 copy_register_state(dst_reg, src_reg); 12444 /* Make sure ID is cleared if src_reg is not in u32 range otherwise 12445 * dst_reg min/max could be incorrectly 12446 * propagated into src_reg by find_equal_scalars() 12447 */ 12448 if (!is_src_reg_u32) 12449 dst_reg->id = 0; 12450 dst_reg->live |= REG_LIVE_WRITTEN; 12451 dst_reg->subreg_def = env->insn_idx + 1; 12452 } else { 12453 mark_reg_unknown(env, regs, 12454 insn->dst_reg); 12455 } 12456 zext_32_to_64(dst_reg); 12457 reg_bounds_sync(dst_reg); 12458 } 12459 } else { 12460 /* case: R = imm 12461 * remember the value we stored into this reg 12462 */ 12463 /* clear any state __mark_reg_known doesn't set */ 12464 mark_reg_unknown(env, regs, insn->dst_reg); 12465 regs[insn->dst_reg].type = SCALAR_VALUE; 12466 if (BPF_CLASS(insn->code) == BPF_ALU64) { 12467 __mark_reg_known(regs + insn->dst_reg, 12468 insn->imm); 12469 } else { 12470 __mark_reg_known(regs + insn->dst_reg, 12471 (u32)insn->imm); 12472 } 12473 } 12474 12475 } else if (opcode > BPF_END) { 12476 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 12477 return -EINVAL; 12478 12479 } else { /* all other ALU ops: and, sub, xor, add, ... */ 12480 12481 if (BPF_SRC(insn->code) == BPF_X) { 12482 if (insn->imm != 0 || insn->off != 0) { 12483 verbose(env, "BPF_ALU uses reserved fields\n"); 12484 return -EINVAL; 12485 } 12486 /* check src1 operand */ 12487 err = check_reg_arg(env, insn->src_reg, SRC_OP); 12488 if (err) 12489 return err; 12490 } else { 12491 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 12492 verbose(env, "BPF_ALU uses reserved fields\n"); 12493 return -EINVAL; 12494 } 12495 } 12496 12497 /* check src2 operand */ 12498 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 12499 if (err) 12500 return err; 12501 12502 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 12503 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 12504 verbose(env, "div by zero\n"); 12505 return -EINVAL; 12506 } 12507 12508 if ((opcode == BPF_LSH || opcode == BPF_RSH || 12509 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 12510 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 12511 12512 if (insn->imm < 0 || insn->imm >= size) { 12513 verbose(env, "invalid shift %d\n", insn->imm); 12514 return -EINVAL; 12515 } 12516 } 12517 12518 /* check dest operand */ 12519 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 12520 if (err) 12521 return err; 12522 12523 return adjust_reg_min_max_vals(env, insn); 12524 } 12525 12526 return 0; 12527 } 12528 12529 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 12530 struct bpf_reg_state *dst_reg, 12531 enum bpf_reg_type type, 12532 bool range_right_open) 12533 { 12534 struct bpf_func_state *state; 12535 struct bpf_reg_state *reg; 12536 int new_range; 12537 12538 if (dst_reg->off < 0 || 12539 (dst_reg->off == 0 && range_right_open)) 12540 /* This doesn't give us any range */ 12541 return; 12542 12543 if (dst_reg->umax_value > MAX_PACKET_OFF || 12544 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 12545 /* Risk of overflow. For instance, ptr + (1<<63) may be less 12546 * than pkt_end, but that's because it's also less than pkt. 12547 */ 12548 return; 12549 12550 new_range = dst_reg->off; 12551 if (range_right_open) 12552 new_range++; 12553 12554 /* Examples for register markings: 12555 * 12556 * pkt_data in dst register: 12557 * 12558 * r2 = r3; 12559 * r2 += 8; 12560 * if (r2 > pkt_end) goto <handle exception> 12561 * <access okay> 12562 * 12563 * r2 = r3; 12564 * r2 += 8; 12565 * if (r2 < pkt_end) goto <access okay> 12566 * <handle exception> 12567 * 12568 * Where: 12569 * r2 == dst_reg, pkt_end == src_reg 12570 * r2=pkt(id=n,off=8,r=0) 12571 * r3=pkt(id=n,off=0,r=0) 12572 * 12573 * pkt_data in src register: 12574 * 12575 * r2 = r3; 12576 * r2 += 8; 12577 * if (pkt_end >= r2) goto <access okay> 12578 * <handle exception> 12579 * 12580 * r2 = r3; 12581 * r2 += 8; 12582 * if (pkt_end <= r2) goto <handle exception> 12583 * <access okay> 12584 * 12585 * Where: 12586 * pkt_end == dst_reg, r2 == src_reg 12587 * r2=pkt(id=n,off=8,r=0) 12588 * r3=pkt(id=n,off=0,r=0) 12589 * 12590 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 12591 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 12592 * and [r3, r3 + 8-1) respectively is safe to access depending on 12593 * the check. 12594 */ 12595 12596 /* If our ids match, then we must have the same max_value. And we 12597 * don't care about the other reg's fixed offset, since if it's too big 12598 * the range won't allow anything. 12599 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 12600 */ 12601 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 12602 if (reg->type == type && reg->id == dst_reg->id) 12603 /* keep the maximum range already checked */ 12604 reg->range = max(reg->range, new_range); 12605 })); 12606 } 12607 12608 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 12609 { 12610 struct tnum subreg = tnum_subreg(reg->var_off); 12611 s32 sval = (s32)val; 12612 12613 switch (opcode) { 12614 case BPF_JEQ: 12615 if (tnum_is_const(subreg)) 12616 return !!tnum_equals_const(subreg, val); 12617 else if (val < reg->u32_min_value || val > reg->u32_max_value) 12618 return 0; 12619 break; 12620 case BPF_JNE: 12621 if (tnum_is_const(subreg)) 12622 return !tnum_equals_const(subreg, val); 12623 else if (val < reg->u32_min_value || val > reg->u32_max_value) 12624 return 1; 12625 break; 12626 case BPF_JSET: 12627 if ((~subreg.mask & subreg.value) & val) 12628 return 1; 12629 if (!((subreg.mask | subreg.value) & val)) 12630 return 0; 12631 break; 12632 case BPF_JGT: 12633 if (reg->u32_min_value > val) 12634 return 1; 12635 else if (reg->u32_max_value <= val) 12636 return 0; 12637 break; 12638 case BPF_JSGT: 12639 if (reg->s32_min_value > sval) 12640 return 1; 12641 else if (reg->s32_max_value <= sval) 12642 return 0; 12643 break; 12644 case BPF_JLT: 12645 if (reg->u32_max_value < val) 12646 return 1; 12647 else if (reg->u32_min_value >= val) 12648 return 0; 12649 break; 12650 case BPF_JSLT: 12651 if (reg->s32_max_value < sval) 12652 return 1; 12653 else if (reg->s32_min_value >= sval) 12654 return 0; 12655 break; 12656 case BPF_JGE: 12657 if (reg->u32_min_value >= val) 12658 return 1; 12659 else if (reg->u32_max_value < val) 12660 return 0; 12661 break; 12662 case BPF_JSGE: 12663 if (reg->s32_min_value >= sval) 12664 return 1; 12665 else if (reg->s32_max_value < sval) 12666 return 0; 12667 break; 12668 case BPF_JLE: 12669 if (reg->u32_max_value <= val) 12670 return 1; 12671 else if (reg->u32_min_value > val) 12672 return 0; 12673 break; 12674 case BPF_JSLE: 12675 if (reg->s32_max_value <= sval) 12676 return 1; 12677 else if (reg->s32_min_value > sval) 12678 return 0; 12679 break; 12680 } 12681 12682 return -1; 12683 } 12684 12685 12686 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 12687 { 12688 s64 sval = (s64)val; 12689 12690 switch (opcode) { 12691 case BPF_JEQ: 12692 if (tnum_is_const(reg->var_off)) 12693 return !!tnum_equals_const(reg->var_off, val); 12694 else if (val < reg->umin_value || val > reg->umax_value) 12695 return 0; 12696 break; 12697 case BPF_JNE: 12698 if (tnum_is_const(reg->var_off)) 12699 return !tnum_equals_const(reg->var_off, val); 12700 else if (val < reg->umin_value || val > reg->umax_value) 12701 return 1; 12702 break; 12703 case BPF_JSET: 12704 if ((~reg->var_off.mask & reg->var_off.value) & val) 12705 return 1; 12706 if (!((reg->var_off.mask | reg->var_off.value) & val)) 12707 return 0; 12708 break; 12709 case BPF_JGT: 12710 if (reg->umin_value > val) 12711 return 1; 12712 else if (reg->umax_value <= val) 12713 return 0; 12714 break; 12715 case BPF_JSGT: 12716 if (reg->smin_value > sval) 12717 return 1; 12718 else if (reg->smax_value <= sval) 12719 return 0; 12720 break; 12721 case BPF_JLT: 12722 if (reg->umax_value < val) 12723 return 1; 12724 else if (reg->umin_value >= val) 12725 return 0; 12726 break; 12727 case BPF_JSLT: 12728 if (reg->smax_value < sval) 12729 return 1; 12730 else if (reg->smin_value >= sval) 12731 return 0; 12732 break; 12733 case BPF_JGE: 12734 if (reg->umin_value >= val) 12735 return 1; 12736 else if (reg->umax_value < val) 12737 return 0; 12738 break; 12739 case BPF_JSGE: 12740 if (reg->smin_value >= sval) 12741 return 1; 12742 else if (reg->smax_value < sval) 12743 return 0; 12744 break; 12745 case BPF_JLE: 12746 if (reg->umax_value <= val) 12747 return 1; 12748 else if (reg->umin_value > val) 12749 return 0; 12750 break; 12751 case BPF_JSLE: 12752 if (reg->smax_value <= sval) 12753 return 1; 12754 else if (reg->smin_value > sval) 12755 return 0; 12756 break; 12757 } 12758 12759 return -1; 12760 } 12761 12762 /* compute branch direction of the expression "if (reg opcode val) goto target;" 12763 * and return: 12764 * 1 - branch will be taken and "goto target" will be executed 12765 * 0 - branch will not be taken and fall-through to next insn 12766 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 12767 * range [0,10] 12768 */ 12769 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 12770 bool is_jmp32) 12771 { 12772 if (__is_pointer_value(false, reg)) { 12773 if (!reg_type_not_null(reg->type)) 12774 return -1; 12775 12776 /* If pointer is valid tests against zero will fail so we can 12777 * use this to direct branch taken. 12778 */ 12779 if (val != 0) 12780 return -1; 12781 12782 switch (opcode) { 12783 case BPF_JEQ: 12784 return 0; 12785 case BPF_JNE: 12786 return 1; 12787 default: 12788 return -1; 12789 } 12790 } 12791 12792 if (is_jmp32) 12793 return is_branch32_taken(reg, val, opcode); 12794 return is_branch64_taken(reg, val, opcode); 12795 } 12796 12797 static int flip_opcode(u32 opcode) 12798 { 12799 /* How can we transform "a <op> b" into "b <op> a"? */ 12800 static const u8 opcode_flip[16] = { 12801 /* these stay the same */ 12802 [BPF_JEQ >> 4] = BPF_JEQ, 12803 [BPF_JNE >> 4] = BPF_JNE, 12804 [BPF_JSET >> 4] = BPF_JSET, 12805 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 12806 [BPF_JGE >> 4] = BPF_JLE, 12807 [BPF_JGT >> 4] = BPF_JLT, 12808 [BPF_JLE >> 4] = BPF_JGE, 12809 [BPF_JLT >> 4] = BPF_JGT, 12810 [BPF_JSGE >> 4] = BPF_JSLE, 12811 [BPF_JSGT >> 4] = BPF_JSLT, 12812 [BPF_JSLE >> 4] = BPF_JSGE, 12813 [BPF_JSLT >> 4] = BPF_JSGT 12814 }; 12815 return opcode_flip[opcode >> 4]; 12816 } 12817 12818 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 12819 struct bpf_reg_state *src_reg, 12820 u8 opcode) 12821 { 12822 struct bpf_reg_state *pkt; 12823 12824 if (src_reg->type == PTR_TO_PACKET_END) { 12825 pkt = dst_reg; 12826 } else if (dst_reg->type == PTR_TO_PACKET_END) { 12827 pkt = src_reg; 12828 opcode = flip_opcode(opcode); 12829 } else { 12830 return -1; 12831 } 12832 12833 if (pkt->range >= 0) 12834 return -1; 12835 12836 switch (opcode) { 12837 case BPF_JLE: 12838 /* pkt <= pkt_end */ 12839 fallthrough; 12840 case BPF_JGT: 12841 /* pkt > pkt_end */ 12842 if (pkt->range == BEYOND_PKT_END) 12843 /* pkt has at last one extra byte beyond pkt_end */ 12844 return opcode == BPF_JGT; 12845 break; 12846 case BPF_JLT: 12847 /* pkt < pkt_end */ 12848 fallthrough; 12849 case BPF_JGE: 12850 /* pkt >= pkt_end */ 12851 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 12852 return opcode == BPF_JGE; 12853 break; 12854 } 12855 return -1; 12856 } 12857 12858 /* Adjusts the register min/max values in the case that the dst_reg is the 12859 * variable register that we are working on, and src_reg is a constant or we're 12860 * simply doing a BPF_K check. 12861 * In JEQ/JNE cases we also adjust the var_off values. 12862 */ 12863 static void reg_set_min_max(struct bpf_reg_state *true_reg, 12864 struct bpf_reg_state *false_reg, 12865 u64 val, u32 val32, 12866 u8 opcode, bool is_jmp32) 12867 { 12868 struct tnum false_32off = tnum_subreg(false_reg->var_off); 12869 struct tnum false_64off = false_reg->var_off; 12870 struct tnum true_32off = tnum_subreg(true_reg->var_off); 12871 struct tnum true_64off = true_reg->var_off; 12872 s64 sval = (s64)val; 12873 s32 sval32 = (s32)val32; 12874 12875 /* If the dst_reg is a pointer, we can't learn anything about its 12876 * variable offset from the compare (unless src_reg were a pointer into 12877 * the same object, but we don't bother with that. 12878 * Since false_reg and true_reg have the same type by construction, we 12879 * only need to check one of them for pointerness. 12880 */ 12881 if (__is_pointer_value(false, false_reg)) 12882 return; 12883 12884 switch (opcode) { 12885 /* JEQ/JNE comparison doesn't change the register equivalence. 12886 * 12887 * r1 = r2; 12888 * if (r1 == 42) goto label; 12889 * ... 12890 * label: // here both r1 and r2 are known to be 42. 12891 * 12892 * Hence when marking register as known preserve it's ID. 12893 */ 12894 case BPF_JEQ: 12895 if (is_jmp32) { 12896 __mark_reg32_known(true_reg, val32); 12897 true_32off = tnum_subreg(true_reg->var_off); 12898 } else { 12899 ___mark_reg_known(true_reg, val); 12900 true_64off = true_reg->var_off; 12901 } 12902 break; 12903 case BPF_JNE: 12904 if (is_jmp32) { 12905 __mark_reg32_known(false_reg, val32); 12906 false_32off = tnum_subreg(false_reg->var_off); 12907 } else { 12908 ___mark_reg_known(false_reg, val); 12909 false_64off = false_reg->var_off; 12910 } 12911 break; 12912 case BPF_JSET: 12913 if (is_jmp32) { 12914 false_32off = tnum_and(false_32off, tnum_const(~val32)); 12915 if (is_power_of_2(val32)) 12916 true_32off = tnum_or(true_32off, 12917 tnum_const(val32)); 12918 } else { 12919 false_64off = tnum_and(false_64off, tnum_const(~val)); 12920 if (is_power_of_2(val)) 12921 true_64off = tnum_or(true_64off, 12922 tnum_const(val)); 12923 } 12924 break; 12925 case BPF_JGE: 12926 case BPF_JGT: 12927 { 12928 if (is_jmp32) { 12929 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 12930 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 12931 12932 false_reg->u32_max_value = min(false_reg->u32_max_value, 12933 false_umax); 12934 true_reg->u32_min_value = max(true_reg->u32_min_value, 12935 true_umin); 12936 } else { 12937 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 12938 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 12939 12940 false_reg->umax_value = min(false_reg->umax_value, false_umax); 12941 true_reg->umin_value = max(true_reg->umin_value, true_umin); 12942 } 12943 break; 12944 } 12945 case BPF_JSGE: 12946 case BPF_JSGT: 12947 { 12948 if (is_jmp32) { 12949 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 12950 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 12951 12952 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 12953 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 12954 } else { 12955 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 12956 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 12957 12958 false_reg->smax_value = min(false_reg->smax_value, false_smax); 12959 true_reg->smin_value = max(true_reg->smin_value, true_smin); 12960 } 12961 break; 12962 } 12963 case BPF_JLE: 12964 case BPF_JLT: 12965 { 12966 if (is_jmp32) { 12967 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 12968 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 12969 12970 false_reg->u32_min_value = max(false_reg->u32_min_value, 12971 false_umin); 12972 true_reg->u32_max_value = min(true_reg->u32_max_value, 12973 true_umax); 12974 } else { 12975 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 12976 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 12977 12978 false_reg->umin_value = max(false_reg->umin_value, false_umin); 12979 true_reg->umax_value = min(true_reg->umax_value, true_umax); 12980 } 12981 break; 12982 } 12983 case BPF_JSLE: 12984 case BPF_JSLT: 12985 { 12986 if (is_jmp32) { 12987 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 12988 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 12989 12990 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 12991 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 12992 } else { 12993 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 12994 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 12995 12996 false_reg->smin_value = max(false_reg->smin_value, false_smin); 12997 true_reg->smax_value = min(true_reg->smax_value, true_smax); 12998 } 12999 break; 13000 } 13001 default: 13002 return; 13003 } 13004 13005 if (is_jmp32) { 13006 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 13007 tnum_subreg(false_32off)); 13008 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 13009 tnum_subreg(true_32off)); 13010 __reg_combine_32_into_64(false_reg); 13011 __reg_combine_32_into_64(true_reg); 13012 } else { 13013 false_reg->var_off = false_64off; 13014 true_reg->var_off = true_64off; 13015 __reg_combine_64_into_32(false_reg); 13016 __reg_combine_64_into_32(true_reg); 13017 } 13018 } 13019 13020 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 13021 * the variable reg. 13022 */ 13023 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 13024 struct bpf_reg_state *false_reg, 13025 u64 val, u32 val32, 13026 u8 opcode, bool is_jmp32) 13027 { 13028 opcode = flip_opcode(opcode); 13029 /* This uses zero as "not present in table"; luckily the zero opcode, 13030 * BPF_JA, can't get here. 13031 */ 13032 if (opcode) 13033 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 13034 } 13035 13036 /* Regs are known to be equal, so intersect their min/max/var_off */ 13037 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 13038 struct bpf_reg_state *dst_reg) 13039 { 13040 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 13041 dst_reg->umin_value); 13042 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 13043 dst_reg->umax_value); 13044 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 13045 dst_reg->smin_value); 13046 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 13047 dst_reg->smax_value); 13048 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 13049 dst_reg->var_off); 13050 reg_bounds_sync(src_reg); 13051 reg_bounds_sync(dst_reg); 13052 } 13053 13054 static void reg_combine_min_max(struct bpf_reg_state *true_src, 13055 struct bpf_reg_state *true_dst, 13056 struct bpf_reg_state *false_src, 13057 struct bpf_reg_state *false_dst, 13058 u8 opcode) 13059 { 13060 switch (opcode) { 13061 case BPF_JEQ: 13062 __reg_combine_min_max(true_src, true_dst); 13063 break; 13064 case BPF_JNE: 13065 __reg_combine_min_max(false_src, false_dst); 13066 break; 13067 } 13068 } 13069 13070 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 13071 struct bpf_reg_state *reg, u32 id, 13072 bool is_null) 13073 { 13074 if (type_may_be_null(reg->type) && reg->id == id && 13075 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 13076 /* Old offset (both fixed and variable parts) should have been 13077 * known-zero, because we don't allow pointer arithmetic on 13078 * pointers that might be NULL. If we see this happening, don't 13079 * convert the register. 13080 * 13081 * But in some cases, some helpers that return local kptrs 13082 * advance offset for the returned pointer. In those cases, it 13083 * is fine to expect to see reg->off. 13084 */ 13085 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) 13086 return; 13087 if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && 13088 WARN_ON_ONCE(reg->off)) 13089 return; 13090 13091 if (is_null) { 13092 reg->type = SCALAR_VALUE; 13093 /* We don't need id and ref_obj_id from this point 13094 * onwards anymore, thus we should better reset it, 13095 * so that state pruning has chances to take effect. 13096 */ 13097 reg->id = 0; 13098 reg->ref_obj_id = 0; 13099 13100 return; 13101 } 13102 13103 mark_ptr_not_null_reg(reg); 13104 13105 if (!reg_may_point_to_spin_lock(reg)) { 13106 /* For not-NULL ptr, reg->ref_obj_id will be reset 13107 * in release_reference(). 13108 * 13109 * reg->id is still used by spin_lock ptr. Other 13110 * than spin_lock ptr type, reg->id can be reset. 13111 */ 13112 reg->id = 0; 13113 } 13114 } 13115 } 13116 13117 /* The logic is similar to find_good_pkt_pointers(), both could eventually 13118 * be folded together at some point. 13119 */ 13120 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 13121 bool is_null) 13122 { 13123 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 13124 struct bpf_reg_state *regs = state->regs, *reg; 13125 u32 ref_obj_id = regs[regno].ref_obj_id; 13126 u32 id = regs[regno].id; 13127 13128 if (ref_obj_id && ref_obj_id == id && is_null) 13129 /* regs[regno] is in the " == NULL" branch. 13130 * No one could have freed the reference state before 13131 * doing the NULL check. 13132 */ 13133 WARN_ON_ONCE(release_reference_state(state, id)); 13134 13135 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 13136 mark_ptr_or_null_reg(state, reg, id, is_null); 13137 })); 13138 } 13139 13140 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 13141 struct bpf_reg_state *dst_reg, 13142 struct bpf_reg_state *src_reg, 13143 struct bpf_verifier_state *this_branch, 13144 struct bpf_verifier_state *other_branch) 13145 { 13146 if (BPF_SRC(insn->code) != BPF_X) 13147 return false; 13148 13149 /* Pointers are always 64-bit. */ 13150 if (BPF_CLASS(insn->code) == BPF_JMP32) 13151 return false; 13152 13153 switch (BPF_OP(insn->code)) { 13154 case BPF_JGT: 13155 if ((dst_reg->type == PTR_TO_PACKET && 13156 src_reg->type == PTR_TO_PACKET_END) || 13157 (dst_reg->type == PTR_TO_PACKET_META && 13158 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 13159 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 13160 find_good_pkt_pointers(this_branch, dst_reg, 13161 dst_reg->type, false); 13162 mark_pkt_end(other_branch, insn->dst_reg, true); 13163 } else if ((dst_reg->type == PTR_TO_PACKET_END && 13164 src_reg->type == PTR_TO_PACKET) || 13165 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 13166 src_reg->type == PTR_TO_PACKET_META)) { 13167 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 13168 find_good_pkt_pointers(other_branch, src_reg, 13169 src_reg->type, true); 13170 mark_pkt_end(this_branch, insn->src_reg, false); 13171 } else { 13172 return false; 13173 } 13174 break; 13175 case BPF_JLT: 13176 if ((dst_reg->type == PTR_TO_PACKET && 13177 src_reg->type == PTR_TO_PACKET_END) || 13178 (dst_reg->type == PTR_TO_PACKET_META && 13179 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 13180 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 13181 find_good_pkt_pointers(other_branch, dst_reg, 13182 dst_reg->type, true); 13183 mark_pkt_end(this_branch, insn->dst_reg, false); 13184 } else if ((dst_reg->type == PTR_TO_PACKET_END && 13185 src_reg->type == PTR_TO_PACKET) || 13186 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 13187 src_reg->type == PTR_TO_PACKET_META)) { 13188 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 13189 find_good_pkt_pointers(this_branch, src_reg, 13190 src_reg->type, false); 13191 mark_pkt_end(other_branch, insn->src_reg, true); 13192 } else { 13193 return false; 13194 } 13195 break; 13196 case BPF_JGE: 13197 if ((dst_reg->type == PTR_TO_PACKET && 13198 src_reg->type == PTR_TO_PACKET_END) || 13199 (dst_reg->type == PTR_TO_PACKET_META && 13200 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 13201 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 13202 find_good_pkt_pointers(this_branch, dst_reg, 13203 dst_reg->type, true); 13204 mark_pkt_end(other_branch, insn->dst_reg, false); 13205 } else if ((dst_reg->type == PTR_TO_PACKET_END && 13206 src_reg->type == PTR_TO_PACKET) || 13207 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 13208 src_reg->type == PTR_TO_PACKET_META)) { 13209 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 13210 find_good_pkt_pointers(other_branch, src_reg, 13211 src_reg->type, false); 13212 mark_pkt_end(this_branch, insn->src_reg, true); 13213 } else { 13214 return false; 13215 } 13216 break; 13217 case BPF_JLE: 13218 if ((dst_reg->type == PTR_TO_PACKET && 13219 src_reg->type == PTR_TO_PACKET_END) || 13220 (dst_reg->type == PTR_TO_PACKET_META && 13221 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 13222 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 13223 find_good_pkt_pointers(other_branch, dst_reg, 13224 dst_reg->type, false); 13225 mark_pkt_end(this_branch, insn->dst_reg, true); 13226 } else if ((dst_reg->type == PTR_TO_PACKET_END && 13227 src_reg->type == PTR_TO_PACKET) || 13228 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 13229 src_reg->type == PTR_TO_PACKET_META)) { 13230 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 13231 find_good_pkt_pointers(this_branch, src_reg, 13232 src_reg->type, true); 13233 mark_pkt_end(other_branch, insn->src_reg, false); 13234 } else { 13235 return false; 13236 } 13237 break; 13238 default: 13239 return false; 13240 } 13241 13242 return true; 13243 } 13244 13245 static void find_equal_scalars(struct bpf_verifier_state *vstate, 13246 struct bpf_reg_state *known_reg) 13247 { 13248 struct bpf_func_state *state; 13249 struct bpf_reg_state *reg; 13250 13251 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 13252 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 13253 copy_register_state(reg, known_reg); 13254 })); 13255 } 13256 13257 static int check_cond_jmp_op(struct bpf_verifier_env *env, 13258 struct bpf_insn *insn, int *insn_idx) 13259 { 13260 struct bpf_verifier_state *this_branch = env->cur_state; 13261 struct bpf_verifier_state *other_branch; 13262 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 13263 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 13264 struct bpf_reg_state *eq_branch_regs; 13265 u8 opcode = BPF_OP(insn->code); 13266 bool is_jmp32; 13267 int pred = -1; 13268 int err; 13269 13270 /* Only conditional jumps are expected to reach here. */ 13271 if (opcode == BPF_JA || opcode > BPF_JSLE) { 13272 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 13273 return -EINVAL; 13274 } 13275 13276 if (BPF_SRC(insn->code) == BPF_X) { 13277 if (insn->imm != 0) { 13278 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 13279 return -EINVAL; 13280 } 13281 13282 /* check src1 operand */ 13283 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13284 if (err) 13285 return err; 13286 13287 if (is_pointer_value(env, insn->src_reg)) { 13288 verbose(env, "R%d pointer comparison prohibited\n", 13289 insn->src_reg); 13290 return -EACCES; 13291 } 13292 src_reg = ®s[insn->src_reg]; 13293 } else { 13294 if (insn->src_reg != BPF_REG_0) { 13295 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 13296 return -EINVAL; 13297 } 13298 } 13299 13300 /* check src2 operand */ 13301 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 13302 if (err) 13303 return err; 13304 13305 dst_reg = ®s[insn->dst_reg]; 13306 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 13307 13308 if (BPF_SRC(insn->code) == BPF_K) { 13309 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 13310 } else if (src_reg->type == SCALAR_VALUE && 13311 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 13312 pred = is_branch_taken(dst_reg, 13313 tnum_subreg(src_reg->var_off).value, 13314 opcode, 13315 is_jmp32); 13316 } else if (src_reg->type == SCALAR_VALUE && 13317 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 13318 pred = is_branch_taken(dst_reg, 13319 src_reg->var_off.value, 13320 opcode, 13321 is_jmp32); 13322 } else if (dst_reg->type == SCALAR_VALUE && 13323 is_jmp32 && tnum_is_const(tnum_subreg(dst_reg->var_off))) { 13324 pred = is_branch_taken(src_reg, 13325 tnum_subreg(dst_reg->var_off).value, 13326 flip_opcode(opcode), 13327 is_jmp32); 13328 } else if (dst_reg->type == SCALAR_VALUE && 13329 !is_jmp32 && tnum_is_const(dst_reg->var_off)) { 13330 pred = is_branch_taken(src_reg, 13331 dst_reg->var_off.value, 13332 flip_opcode(opcode), 13333 is_jmp32); 13334 } else if (reg_is_pkt_pointer_any(dst_reg) && 13335 reg_is_pkt_pointer_any(src_reg) && 13336 !is_jmp32) { 13337 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 13338 } 13339 13340 if (pred >= 0) { 13341 /* If we get here with a dst_reg pointer type it is because 13342 * above is_branch_taken() special cased the 0 comparison. 13343 */ 13344 if (!__is_pointer_value(false, dst_reg)) 13345 err = mark_chain_precision(env, insn->dst_reg); 13346 if (BPF_SRC(insn->code) == BPF_X && !err && 13347 !__is_pointer_value(false, src_reg)) 13348 err = mark_chain_precision(env, insn->src_reg); 13349 if (err) 13350 return err; 13351 } 13352 13353 if (pred == 1) { 13354 /* Only follow the goto, ignore fall-through. If needed, push 13355 * the fall-through branch for simulation under speculative 13356 * execution. 13357 */ 13358 if (!env->bypass_spec_v1 && 13359 !sanitize_speculative_path(env, insn, *insn_idx + 1, 13360 *insn_idx)) 13361 return -EFAULT; 13362 *insn_idx += insn->off; 13363 return 0; 13364 } else if (pred == 0) { 13365 /* Only follow the fall-through branch, since that's where the 13366 * program will go. If needed, push the goto branch for 13367 * simulation under speculative execution. 13368 */ 13369 if (!env->bypass_spec_v1 && 13370 !sanitize_speculative_path(env, insn, 13371 *insn_idx + insn->off + 1, 13372 *insn_idx)) 13373 return -EFAULT; 13374 return 0; 13375 } 13376 13377 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 13378 false); 13379 if (!other_branch) 13380 return -EFAULT; 13381 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 13382 13383 /* detect if we are comparing against a constant value so we can adjust 13384 * our min/max values for our dst register. 13385 * this is only legit if both are scalars (or pointers to the same 13386 * object, I suppose, see the PTR_MAYBE_NULL related if block below), 13387 * because otherwise the different base pointers mean the offsets aren't 13388 * comparable. 13389 */ 13390 if (BPF_SRC(insn->code) == BPF_X) { 13391 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 13392 13393 if (dst_reg->type == SCALAR_VALUE && 13394 src_reg->type == SCALAR_VALUE) { 13395 if (tnum_is_const(src_reg->var_off) || 13396 (is_jmp32 && 13397 tnum_is_const(tnum_subreg(src_reg->var_off)))) 13398 reg_set_min_max(&other_branch_regs[insn->dst_reg], 13399 dst_reg, 13400 src_reg->var_off.value, 13401 tnum_subreg(src_reg->var_off).value, 13402 opcode, is_jmp32); 13403 else if (tnum_is_const(dst_reg->var_off) || 13404 (is_jmp32 && 13405 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 13406 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 13407 src_reg, 13408 dst_reg->var_off.value, 13409 tnum_subreg(dst_reg->var_off).value, 13410 opcode, is_jmp32); 13411 else if (!is_jmp32 && 13412 (opcode == BPF_JEQ || opcode == BPF_JNE)) 13413 /* Comparing for equality, we can combine knowledge */ 13414 reg_combine_min_max(&other_branch_regs[insn->src_reg], 13415 &other_branch_regs[insn->dst_reg], 13416 src_reg, dst_reg, opcode); 13417 if (src_reg->id && 13418 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 13419 find_equal_scalars(this_branch, src_reg); 13420 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 13421 } 13422 13423 } 13424 } else if (dst_reg->type == SCALAR_VALUE) { 13425 reg_set_min_max(&other_branch_regs[insn->dst_reg], 13426 dst_reg, insn->imm, (u32)insn->imm, 13427 opcode, is_jmp32); 13428 } 13429 13430 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 13431 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 13432 find_equal_scalars(this_branch, dst_reg); 13433 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 13434 } 13435 13436 /* if one pointer register is compared to another pointer 13437 * register check if PTR_MAYBE_NULL could be lifted. 13438 * E.g. register A - maybe null 13439 * register B - not null 13440 * for JNE A, B, ... - A is not null in the false branch; 13441 * for JEQ A, B, ... - A is not null in the true branch. 13442 * 13443 * Since PTR_TO_BTF_ID points to a kernel struct that does 13444 * not need to be null checked by the BPF program, i.e., 13445 * could be null even without PTR_MAYBE_NULL marking, so 13446 * only propagate nullness when neither reg is that type. 13447 */ 13448 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 13449 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 13450 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && 13451 base_type(src_reg->type) != PTR_TO_BTF_ID && 13452 base_type(dst_reg->type) != PTR_TO_BTF_ID) { 13453 eq_branch_regs = NULL; 13454 switch (opcode) { 13455 case BPF_JEQ: 13456 eq_branch_regs = other_branch_regs; 13457 break; 13458 case BPF_JNE: 13459 eq_branch_regs = regs; 13460 break; 13461 default: 13462 /* do nothing */ 13463 break; 13464 } 13465 if (eq_branch_regs) { 13466 if (type_may_be_null(src_reg->type)) 13467 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 13468 else 13469 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 13470 } 13471 } 13472 13473 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 13474 * NOTE: these optimizations below are related with pointer comparison 13475 * which will never be JMP32. 13476 */ 13477 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 13478 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 13479 type_may_be_null(dst_reg->type)) { 13480 /* Mark all identical registers in each branch as either 13481 * safe or unknown depending R == 0 or R != 0 conditional. 13482 */ 13483 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 13484 opcode == BPF_JNE); 13485 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 13486 opcode == BPF_JEQ); 13487 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 13488 this_branch, other_branch) && 13489 is_pointer_value(env, insn->dst_reg)) { 13490 verbose(env, "R%d pointer comparison prohibited\n", 13491 insn->dst_reg); 13492 return -EACCES; 13493 } 13494 if (env->log.level & BPF_LOG_LEVEL) 13495 print_insn_state(env, this_branch->frame[this_branch->curframe]); 13496 return 0; 13497 } 13498 13499 /* verify BPF_LD_IMM64 instruction */ 13500 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 13501 { 13502 struct bpf_insn_aux_data *aux = cur_aux(env); 13503 struct bpf_reg_state *regs = cur_regs(env); 13504 struct bpf_reg_state *dst_reg; 13505 struct bpf_map *map; 13506 int err; 13507 13508 if (BPF_SIZE(insn->code) != BPF_DW) { 13509 verbose(env, "invalid BPF_LD_IMM insn\n"); 13510 return -EINVAL; 13511 } 13512 if (insn->off != 0) { 13513 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 13514 return -EINVAL; 13515 } 13516 13517 err = check_reg_arg(env, insn->dst_reg, DST_OP); 13518 if (err) 13519 return err; 13520 13521 dst_reg = ®s[insn->dst_reg]; 13522 if (insn->src_reg == 0) { 13523 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 13524 13525 dst_reg->type = SCALAR_VALUE; 13526 __mark_reg_known(®s[insn->dst_reg], imm); 13527 return 0; 13528 } 13529 13530 /* All special src_reg cases are listed below. From this point onwards 13531 * we either succeed and assign a corresponding dst_reg->type after 13532 * zeroing the offset, or fail and reject the program. 13533 */ 13534 mark_reg_known_zero(env, regs, insn->dst_reg); 13535 13536 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 13537 dst_reg->type = aux->btf_var.reg_type; 13538 switch (base_type(dst_reg->type)) { 13539 case PTR_TO_MEM: 13540 dst_reg->mem_size = aux->btf_var.mem_size; 13541 break; 13542 case PTR_TO_BTF_ID: 13543 dst_reg->btf = aux->btf_var.btf; 13544 dst_reg->btf_id = aux->btf_var.btf_id; 13545 break; 13546 default: 13547 verbose(env, "bpf verifier is misconfigured\n"); 13548 return -EFAULT; 13549 } 13550 return 0; 13551 } 13552 13553 if (insn->src_reg == BPF_PSEUDO_FUNC) { 13554 struct bpf_prog_aux *aux = env->prog->aux; 13555 u32 subprogno = find_subprog(env, 13556 env->insn_idx + insn->imm + 1); 13557 13558 if (!aux->func_info) { 13559 verbose(env, "missing btf func_info\n"); 13560 return -EINVAL; 13561 } 13562 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 13563 verbose(env, "callback function not static\n"); 13564 return -EINVAL; 13565 } 13566 13567 dst_reg->type = PTR_TO_FUNC; 13568 dst_reg->subprogno = subprogno; 13569 return 0; 13570 } 13571 13572 map = env->used_maps[aux->map_index]; 13573 dst_reg->map_ptr = map; 13574 13575 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 13576 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 13577 dst_reg->type = PTR_TO_MAP_VALUE; 13578 dst_reg->off = aux->map_off; 13579 WARN_ON_ONCE(map->max_entries != 1); 13580 /* We want reg->id to be same (0) as map_value is not distinct */ 13581 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 13582 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 13583 dst_reg->type = CONST_PTR_TO_MAP; 13584 } else { 13585 verbose(env, "bpf verifier is misconfigured\n"); 13586 return -EINVAL; 13587 } 13588 13589 return 0; 13590 } 13591 13592 static bool may_access_skb(enum bpf_prog_type type) 13593 { 13594 switch (type) { 13595 case BPF_PROG_TYPE_SOCKET_FILTER: 13596 case BPF_PROG_TYPE_SCHED_CLS: 13597 case BPF_PROG_TYPE_SCHED_ACT: 13598 return true; 13599 default: 13600 return false; 13601 } 13602 } 13603 13604 /* verify safety of LD_ABS|LD_IND instructions: 13605 * - they can only appear in the programs where ctx == skb 13606 * - since they are wrappers of function calls, they scratch R1-R5 registers, 13607 * preserve R6-R9, and store return value into R0 13608 * 13609 * Implicit input: 13610 * ctx == skb == R6 == CTX 13611 * 13612 * Explicit input: 13613 * SRC == any register 13614 * IMM == 32-bit immediate 13615 * 13616 * Output: 13617 * R0 - 8/16/32-bit skb data converted to cpu endianness 13618 */ 13619 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 13620 { 13621 struct bpf_reg_state *regs = cur_regs(env); 13622 static const int ctx_reg = BPF_REG_6; 13623 u8 mode = BPF_MODE(insn->code); 13624 int i, err; 13625 13626 if (!may_access_skb(resolve_prog_type(env->prog))) { 13627 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 13628 return -EINVAL; 13629 } 13630 13631 if (!env->ops->gen_ld_abs) { 13632 verbose(env, "bpf verifier is misconfigured\n"); 13633 return -EINVAL; 13634 } 13635 13636 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 13637 BPF_SIZE(insn->code) == BPF_DW || 13638 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 13639 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 13640 return -EINVAL; 13641 } 13642 13643 /* check whether implicit source operand (register R6) is readable */ 13644 err = check_reg_arg(env, ctx_reg, SRC_OP); 13645 if (err) 13646 return err; 13647 13648 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 13649 * gen_ld_abs() may terminate the program at runtime, leading to 13650 * reference leak. 13651 */ 13652 err = check_reference_leak(env); 13653 if (err) { 13654 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 13655 return err; 13656 } 13657 13658 if (env->cur_state->active_lock.ptr) { 13659 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 13660 return -EINVAL; 13661 } 13662 13663 if (env->cur_state->active_rcu_lock) { 13664 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); 13665 return -EINVAL; 13666 } 13667 13668 if (regs[ctx_reg].type != PTR_TO_CTX) { 13669 verbose(env, 13670 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 13671 return -EINVAL; 13672 } 13673 13674 if (mode == BPF_IND) { 13675 /* check explicit source operand */ 13676 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13677 if (err) 13678 return err; 13679 } 13680 13681 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 13682 if (err < 0) 13683 return err; 13684 13685 /* reset caller saved regs to unreadable */ 13686 for (i = 0; i < CALLER_SAVED_REGS; i++) { 13687 mark_reg_not_init(env, regs, caller_saved[i]); 13688 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 13689 } 13690 13691 /* mark destination R0 register as readable, since it contains 13692 * the value fetched from the packet. 13693 * Already marked as written above. 13694 */ 13695 mark_reg_unknown(env, regs, BPF_REG_0); 13696 /* ld_abs load up to 32-bit skb data. */ 13697 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 13698 return 0; 13699 } 13700 13701 static int check_return_code(struct bpf_verifier_env *env) 13702 { 13703 struct tnum enforce_attach_type_range = tnum_unknown; 13704 const struct bpf_prog *prog = env->prog; 13705 struct bpf_reg_state *reg; 13706 struct tnum range = tnum_range(0, 1); 13707 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 13708 int err; 13709 struct bpf_func_state *frame = env->cur_state->frame[0]; 13710 const bool is_subprog = frame->subprogno; 13711 13712 /* LSM and struct_ops func-ptr's return type could be "void" */ 13713 if (!is_subprog) { 13714 switch (prog_type) { 13715 case BPF_PROG_TYPE_LSM: 13716 if (prog->expected_attach_type == BPF_LSM_CGROUP) 13717 /* See below, can be 0 or 0-1 depending on hook. */ 13718 break; 13719 fallthrough; 13720 case BPF_PROG_TYPE_STRUCT_OPS: 13721 if (!prog->aux->attach_func_proto->type) 13722 return 0; 13723 break; 13724 default: 13725 break; 13726 } 13727 } 13728 13729 /* eBPF calling convention is such that R0 is used 13730 * to return the value from eBPF program. 13731 * Make sure that it's readable at this time 13732 * of bpf_exit, which means that program wrote 13733 * something into it earlier 13734 */ 13735 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 13736 if (err) 13737 return err; 13738 13739 if (is_pointer_value(env, BPF_REG_0)) { 13740 verbose(env, "R0 leaks addr as return value\n"); 13741 return -EACCES; 13742 } 13743 13744 reg = cur_regs(env) + BPF_REG_0; 13745 13746 if (frame->in_async_callback_fn) { 13747 /* enforce return zero from async callbacks like timer */ 13748 if (reg->type != SCALAR_VALUE) { 13749 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 13750 reg_type_str(env, reg->type)); 13751 return -EINVAL; 13752 } 13753 13754 if (!tnum_in(tnum_const(0), reg->var_off)) { 13755 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 13756 return -EINVAL; 13757 } 13758 return 0; 13759 } 13760 13761 if (is_subprog) { 13762 if (reg->type != SCALAR_VALUE) { 13763 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 13764 reg_type_str(env, reg->type)); 13765 return -EINVAL; 13766 } 13767 return 0; 13768 } 13769 13770 switch (prog_type) { 13771 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 13772 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 13773 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 13774 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 13775 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 13776 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 13777 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 13778 range = tnum_range(1, 1); 13779 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 13780 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 13781 range = tnum_range(0, 3); 13782 break; 13783 case BPF_PROG_TYPE_CGROUP_SKB: 13784 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 13785 range = tnum_range(0, 3); 13786 enforce_attach_type_range = tnum_range(2, 3); 13787 } 13788 break; 13789 case BPF_PROG_TYPE_CGROUP_SOCK: 13790 case BPF_PROG_TYPE_SOCK_OPS: 13791 case BPF_PROG_TYPE_CGROUP_DEVICE: 13792 case BPF_PROG_TYPE_CGROUP_SYSCTL: 13793 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 13794 break; 13795 case BPF_PROG_TYPE_RAW_TRACEPOINT: 13796 if (!env->prog->aux->attach_btf_id) 13797 return 0; 13798 range = tnum_const(0); 13799 break; 13800 case BPF_PROG_TYPE_TRACING: 13801 switch (env->prog->expected_attach_type) { 13802 case BPF_TRACE_FENTRY: 13803 case BPF_TRACE_FEXIT: 13804 range = tnum_const(0); 13805 break; 13806 case BPF_TRACE_RAW_TP: 13807 case BPF_MODIFY_RETURN: 13808 return 0; 13809 case BPF_TRACE_ITER: 13810 break; 13811 default: 13812 return -ENOTSUPP; 13813 } 13814 break; 13815 case BPF_PROG_TYPE_SK_LOOKUP: 13816 range = tnum_range(SK_DROP, SK_PASS); 13817 break; 13818 13819 case BPF_PROG_TYPE_LSM: 13820 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 13821 /* Regular BPF_PROG_TYPE_LSM programs can return 13822 * any value. 13823 */ 13824 return 0; 13825 } 13826 if (!env->prog->aux->attach_func_proto->type) { 13827 /* Make sure programs that attach to void 13828 * hooks don't try to modify return value. 13829 */ 13830 range = tnum_range(1, 1); 13831 } 13832 break; 13833 13834 case BPF_PROG_TYPE_NETFILTER: 13835 range = tnum_range(NF_DROP, NF_ACCEPT); 13836 break; 13837 case BPF_PROG_TYPE_EXT: 13838 /* freplace program can return anything as its return value 13839 * depends on the to-be-replaced kernel func or bpf program. 13840 */ 13841 default: 13842 return 0; 13843 } 13844 13845 if (reg->type != SCALAR_VALUE) { 13846 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 13847 reg_type_str(env, reg->type)); 13848 return -EINVAL; 13849 } 13850 13851 if (!tnum_in(range, reg->var_off)) { 13852 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 13853 if (prog->expected_attach_type == BPF_LSM_CGROUP && 13854 prog_type == BPF_PROG_TYPE_LSM && 13855 !prog->aux->attach_func_proto->type) 13856 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 13857 return -EINVAL; 13858 } 13859 13860 if (!tnum_is_unknown(enforce_attach_type_range) && 13861 tnum_in(enforce_attach_type_range, reg->var_off)) 13862 env->prog->enforce_expected_attach_type = 1; 13863 return 0; 13864 } 13865 13866 /* non-recursive DFS pseudo code 13867 * 1 procedure DFS-iterative(G,v): 13868 * 2 label v as discovered 13869 * 3 let S be a stack 13870 * 4 S.push(v) 13871 * 5 while S is not empty 13872 * 6 t <- S.peek() 13873 * 7 if t is what we're looking for: 13874 * 8 return t 13875 * 9 for all edges e in G.adjacentEdges(t) do 13876 * 10 if edge e is already labelled 13877 * 11 continue with the next edge 13878 * 12 w <- G.adjacentVertex(t,e) 13879 * 13 if vertex w is not discovered and not explored 13880 * 14 label e as tree-edge 13881 * 15 label w as discovered 13882 * 16 S.push(w) 13883 * 17 continue at 5 13884 * 18 else if vertex w is discovered 13885 * 19 label e as back-edge 13886 * 20 else 13887 * 21 // vertex w is explored 13888 * 22 label e as forward- or cross-edge 13889 * 23 label t as explored 13890 * 24 S.pop() 13891 * 13892 * convention: 13893 * 0x10 - discovered 13894 * 0x11 - discovered and fall-through edge labelled 13895 * 0x12 - discovered and fall-through and branch edges labelled 13896 * 0x20 - explored 13897 */ 13898 13899 enum { 13900 DISCOVERED = 0x10, 13901 EXPLORED = 0x20, 13902 FALLTHROUGH = 1, 13903 BRANCH = 2, 13904 }; 13905 13906 static u32 state_htab_size(struct bpf_verifier_env *env) 13907 { 13908 return env->prog->len; 13909 } 13910 13911 static struct bpf_verifier_state_list **explored_state( 13912 struct bpf_verifier_env *env, 13913 int idx) 13914 { 13915 struct bpf_verifier_state *cur = env->cur_state; 13916 struct bpf_func_state *state = cur->frame[cur->curframe]; 13917 13918 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 13919 } 13920 13921 static void mark_prune_point(struct bpf_verifier_env *env, int idx) 13922 { 13923 env->insn_aux_data[idx].prune_point = true; 13924 } 13925 13926 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) 13927 { 13928 return env->insn_aux_data[insn_idx].prune_point; 13929 } 13930 13931 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) 13932 { 13933 env->insn_aux_data[idx].force_checkpoint = true; 13934 } 13935 13936 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) 13937 { 13938 return env->insn_aux_data[insn_idx].force_checkpoint; 13939 } 13940 13941 13942 enum { 13943 DONE_EXPLORING = 0, 13944 KEEP_EXPLORING = 1, 13945 }; 13946 13947 /* t, w, e - match pseudo-code above: 13948 * t - index of current instruction 13949 * w - next instruction 13950 * e - edge 13951 */ 13952 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 13953 bool loop_ok) 13954 { 13955 int *insn_stack = env->cfg.insn_stack; 13956 int *insn_state = env->cfg.insn_state; 13957 13958 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 13959 return DONE_EXPLORING; 13960 13961 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 13962 return DONE_EXPLORING; 13963 13964 if (w < 0 || w >= env->prog->len) { 13965 verbose_linfo(env, t, "%d: ", t); 13966 verbose(env, "jump out of range from insn %d to %d\n", t, w); 13967 return -EINVAL; 13968 } 13969 13970 if (e == BRANCH) { 13971 /* mark branch target for state pruning */ 13972 mark_prune_point(env, w); 13973 mark_jmp_point(env, w); 13974 } 13975 13976 if (insn_state[w] == 0) { 13977 /* tree-edge */ 13978 insn_state[t] = DISCOVERED | e; 13979 insn_state[w] = DISCOVERED; 13980 if (env->cfg.cur_stack >= env->prog->len) 13981 return -E2BIG; 13982 insn_stack[env->cfg.cur_stack++] = w; 13983 return KEEP_EXPLORING; 13984 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 13985 if (loop_ok && env->bpf_capable) 13986 return DONE_EXPLORING; 13987 verbose_linfo(env, t, "%d: ", t); 13988 verbose_linfo(env, w, "%d: ", w); 13989 verbose(env, "back-edge from insn %d to %d\n", t, w); 13990 return -EINVAL; 13991 } else if (insn_state[w] == EXPLORED) { 13992 /* forward- or cross-edge */ 13993 insn_state[t] = DISCOVERED | e; 13994 } else { 13995 verbose(env, "insn state internal bug\n"); 13996 return -EFAULT; 13997 } 13998 return DONE_EXPLORING; 13999 } 14000 14001 static int visit_func_call_insn(int t, struct bpf_insn *insns, 14002 struct bpf_verifier_env *env, 14003 bool visit_callee) 14004 { 14005 int ret; 14006 14007 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 14008 if (ret) 14009 return ret; 14010 14011 mark_prune_point(env, t + 1); 14012 /* when we exit from subprog, we need to record non-linear history */ 14013 mark_jmp_point(env, t + 1); 14014 14015 if (visit_callee) { 14016 mark_prune_point(env, t); 14017 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 14018 /* It's ok to allow recursion from CFG point of 14019 * view. __check_func_call() will do the actual 14020 * check. 14021 */ 14022 bpf_pseudo_func(insns + t)); 14023 } 14024 return ret; 14025 } 14026 14027 /* Visits the instruction at index t and returns one of the following: 14028 * < 0 - an error occurred 14029 * DONE_EXPLORING - the instruction was fully explored 14030 * KEEP_EXPLORING - there is still work to be done before it is fully explored 14031 */ 14032 static int visit_insn(int t, struct bpf_verifier_env *env) 14033 { 14034 struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; 14035 int ret; 14036 14037 if (bpf_pseudo_func(insn)) 14038 return visit_func_call_insn(t, insns, env, true); 14039 14040 /* All non-branch instructions have a single fall-through edge. */ 14041 if (BPF_CLASS(insn->code) != BPF_JMP && 14042 BPF_CLASS(insn->code) != BPF_JMP32) 14043 return push_insn(t, t + 1, FALLTHROUGH, env, false); 14044 14045 switch (BPF_OP(insn->code)) { 14046 case BPF_EXIT: 14047 return DONE_EXPLORING; 14048 14049 case BPF_CALL: 14050 if (insn->src_reg == 0 && insn->imm == BPF_FUNC_timer_set_callback) 14051 /* Mark this call insn as a prune point to trigger 14052 * is_state_visited() check before call itself is 14053 * processed by __check_func_call(). Otherwise new 14054 * async state will be pushed for further exploration. 14055 */ 14056 mark_prune_point(env, t); 14057 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 14058 struct bpf_kfunc_call_arg_meta meta; 14059 14060 ret = fetch_kfunc_meta(env, insn, &meta, NULL); 14061 if (ret == 0 && is_iter_next_kfunc(&meta)) { 14062 mark_prune_point(env, t); 14063 /* Checking and saving state checkpoints at iter_next() call 14064 * is crucial for fast convergence of open-coded iterator loop 14065 * logic, so we need to force it. If we don't do that, 14066 * is_state_visited() might skip saving a checkpoint, causing 14067 * unnecessarily long sequence of not checkpointed 14068 * instructions and jumps, leading to exhaustion of jump 14069 * history buffer, and potentially other undesired outcomes. 14070 * It is expected that with correct open-coded iterators 14071 * convergence will happen quickly, so we don't run a risk of 14072 * exhausting memory. 14073 */ 14074 mark_force_checkpoint(env, t); 14075 } 14076 } 14077 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); 14078 14079 case BPF_JA: 14080 if (BPF_SRC(insn->code) != BPF_K) 14081 return -EINVAL; 14082 14083 /* unconditional jump with single edge */ 14084 ret = push_insn(t, t + insn->off + 1, FALLTHROUGH, env, 14085 true); 14086 if (ret) 14087 return ret; 14088 14089 mark_prune_point(env, t + insn->off + 1); 14090 mark_jmp_point(env, t + insn->off + 1); 14091 14092 return ret; 14093 14094 default: 14095 /* conditional jump with two edges */ 14096 mark_prune_point(env, t); 14097 14098 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 14099 if (ret) 14100 return ret; 14101 14102 return push_insn(t, t + insn->off + 1, BRANCH, env, true); 14103 } 14104 } 14105 14106 /* non-recursive depth-first-search to detect loops in BPF program 14107 * loop == back-edge in directed graph 14108 */ 14109 static int check_cfg(struct bpf_verifier_env *env) 14110 { 14111 int insn_cnt = env->prog->len; 14112 int *insn_stack, *insn_state; 14113 int ret = 0; 14114 int i; 14115 14116 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 14117 if (!insn_state) 14118 return -ENOMEM; 14119 14120 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 14121 if (!insn_stack) { 14122 kvfree(insn_state); 14123 return -ENOMEM; 14124 } 14125 14126 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 14127 insn_stack[0] = 0; /* 0 is the first instruction */ 14128 env->cfg.cur_stack = 1; 14129 14130 while (env->cfg.cur_stack > 0) { 14131 int t = insn_stack[env->cfg.cur_stack - 1]; 14132 14133 ret = visit_insn(t, env); 14134 switch (ret) { 14135 case DONE_EXPLORING: 14136 insn_state[t] = EXPLORED; 14137 env->cfg.cur_stack--; 14138 break; 14139 case KEEP_EXPLORING: 14140 break; 14141 default: 14142 if (ret > 0) { 14143 verbose(env, "visit_insn internal bug\n"); 14144 ret = -EFAULT; 14145 } 14146 goto err_free; 14147 } 14148 } 14149 14150 if (env->cfg.cur_stack < 0) { 14151 verbose(env, "pop stack internal bug\n"); 14152 ret = -EFAULT; 14153 goto err_free; 14154 } 14155 14156 for (i = 0; i < insn_cnt; i++) { 14157 if (insn_state[i] != EXPLORED) { 14158 verbose(env, "unreachable insn %d\n", i); 14159 ret = -EINVAL; 14160 goto err_free; 14161 } 14162 } 14163 ret = 0; /* cfg looks good */ 14164 14165 err_free: 14166 kvfree(insn_state); 14167 kvfree(insn_stack); 14168 env->cfg.insn_state = env->cfg.insn_stack = NULL; 14169 return ret; 14170 } 14171 14172 static int check_abnormal_return(struct bpf_verifier_env *env) 14173 { 14174 int i; 14175 14176 for (i = 1; i < env->subprog_cnt; i++) { 14177 if (env->subprog_info[i].has_ld_abs) { 14178 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 14179 return -EINVAL; 14180 } 14181 if (env->subprog_info[i].has_tail_call) { 14182 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 14183 return -EINVAL; 14184 } 14185 } 14186 return 0; 14187 } 14188 14189 /* The minimum supported BTF func info size */ 14190 #define MIN_BPF_FUNCINFO_SIZE 8 14191 #define MAX_FUNCINFO_REC_SIZE 252 14192 14193 static int check_btf_func(struct bpf_verifier_env *env, 14194 const union bpf_attr *attr, 14195 bpfptr_t uattr) 14196 { 14197 const struct btf_type *type, *func_proto, *ret_type; 14198 u32 i, nfuncs, urec_size, min_size; 14199 u32 krec_size = sizeof(struct bpf_func_info); 14200 struct bpf_func_info *krecord; 14201 struct bpf_func_info_aux *info_aux = NULL; 14202 struct bpf_prog *prog; 14203 const struct btf *btf; 14204 bpfptr_t urecord; 14205 u32 prev_offset = 0; 14206 bool scalar_return; 14207 int ret = -ENOMEM; 14208 14209 nfuncs = attr->func_info_cnt; 14210 if (!nfuncs) { 14211 if (check_abnormal_return(env)) 14212 return -EINVAL; 14213 return 0; 14214 } 14215 14216 if (nfuncs != env->subprog_cnt) { 14217 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 14218 return -EINVAL; 14219 } 14220 14221 urec_size = attr->func_info_rec_size; 14222 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 14223 urec_size > MAX_FUNCINFO_REC_SIZE || 14224 urec_size % sizeof(u32)) { 14225 verbose(env, "invalid func info rec size %u\n", urec_size); 14226 return -EINVAL; 14227 } 14228 14229 prog = env->prog; 14230 btf = prog->aux->btf; 14231 14232 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 14233 min_size = min_t(u32, krec_size, urec_size); 14234 14235 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 14236 if (!krecord) 14237 return -ENOMEM; 14238 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 14239 if (!info_aux) 14240 goto err_free; 14241 14242 for (i = 0; i < nfuncs; i++) { 14243 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 14244 if (ret) { 14245 if (ret == -E2BIG) { 14246 verbose(env, "nonzero tailing record in func info"); 14247 /* set the size kernel expects so loader can zero 14248 * out the rest of the record. 14249 */ 14250 if (copy_to_bpfptr_offset(uattr, 14251 offsetof(union bpf_attr, func_info_rec_size), 14252 &min_size, sizeof(min_size))) 14253 ret = -EFAULT; 14254 } 14255 goto err_free; 14256 } 14257 14258 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 14259 ret = -EFAULT; 14260 goto err_free; 14261 } 14262 14263 /* check insn_off */ 14264 ret = -EINVAL; 14265 if (i == 0) { 14266 if (krecord[i].insn_off) { 14267 verbose(env, 14268 "nonzero insn_off %u for the first func info record", 14269 krecord[i].insn_off); 14270 goto err_free; 14271 } 14272 } else if (krecord[i].insn_off <= prev_offset) { 14273 verbose(env, 14274 "same or smaller insn offset (%u) than previous func info record (%u)", 14275 krecord[i].insn_off, prev_offset); 14276 goto err_free; 14277 } 14278 14279 if (env->subprog_info[i].start != krecord[i].insn_off) { 14280 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 14281 goto err_free; 14282 } 14283 14284 /* check type_id */ 14285 type = btf_type_by_id(btf, krecord[i].type_id); 14286 if (!type || !btf_type_is_func(type)) { 14287 verbose(env, "invalid type id %d in func info", 14288 krecord[i].type_id); 14289 goto err_free; 14290 } 14291 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 14292 14293 func_proto = btf_type_by_id(btf, type->type); 14294 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 14295 /* btf_func_check() already verified it during BTF load */ 14296 goto err_free; 14297 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 14298 scalar_return = 14299 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); 14300 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 14301 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 14302 goto err_free; 14303 } 14304 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 14305 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 14306 goto err_free; 14307 } 14308 14309 prev_offset = krecord[i].insn_off; 14310 bpfptr_add(&urecord, urec_size); 14311 } 14312 14313 prog->aux->func_info = krecord; 14314 prog->aux->func_info_cnt = nfuncs; 14315 prog->aux->func_info_aux = info_aux; 14316 return 0; 14317 14318 err_free: 14319 kvfree(krecord); 14320 kfree(info_aux); 14321 return ret; 14322 } 14323 14324 static void adjust_btf_func(struct bpf_verifier_env *env) 14325 { 14326 struct bpf_prog_aux *aux = env->prog->aux; 14327 int i; 14328 14329 if (!aux->func_info) 14330 return; 14331 14332 for (i = 0; i < env->subprog_cnt; i++) 14333 aux->func_info[i].insn_off = env->subprog_info[i].start; 14334 } 14335 14336 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 14337 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 14338 14339 static int check_btf_line(struct bpf_verifier_env *env, 14340 const union bpf_attr *attr, 14341 bpfptr_t uattr) 14342 { 14343 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 14344 struct bpf_subprog_info *sub; 14345 struct bpf_line_info *linfo; 14346 struct bpf_prog *prog; 14347 const struct btf *btf; 14348 bpfptr_t ulinfo; 14349 int err; 14350 14351 nr_linfo = attr->line_info_cnt; 14352 if (!nr_linfo) 14353 return 0; 14354 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 14355 return -EINVAL; 14356 14357 rec_size = attr->line_info_rec_size; 14358 if (rec_size < MIN_BPF_LINEINFO_SIZE || 14359 rec_size > MAX_LINEINFO_REC_SIZE || 14360 rec_size & (sizeof(u32) - 1)) 14361 return -EINVAL; 14362 14363 /* Need to zero it in case the userspace may 14364 * pass in a smaller bpf_line_info object. 14365 */ 14366 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 14367 GFP_KERNEL | __GFP_NOWARN); 14368 if (!linfo) 14369 return -ENOMEM; 14370 14371 prog = env->prog; 14372 btf = prog->aux->btf; 14373 14374 s = 0; 14375 sub = env->subprog_info; 14376 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 14377 expected_size = sizeof(struct bpf_line_info); 14378 ncopy = min_t(u32, expected_size, rec_size); 14379 for (i = 0; i < nr_linfo; i++) { 14380 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 14381 if (err) { 14382 if (err == -E2BIG) { 14383 verbose(env, "nonzero tailing record in line_info"); 14384 if (copy_to_bpfptr_offset(uattr, 14385 offsetof(union bpf_attr, line_info_rec_size), 14386 &expected_size, sizeof(expected_size))) 14387 err = -EFAULT; 14388 } 14389 goto err_free; 14390 } 14391 14392 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 14393 err = -EFAULT; 14394 goto err_free; 14395 } 14396 14397 /* 14398 * Check insn_off to ensure 14399 * 1) strictly increasing AND 14400 * 2) bounded by prog->len 14401 * 14402 * The linfo[0].insn_off == 0 check logically falls into 14403 * the later "missing bpf_line_info for func..." case 14404 * because the first linfo[0].insn_off must be the 14405 * first sub also and the first sub must have 14406 * subprog_info[0].start == 0. 14407 */ 14408 if ((i && linfo[i].insn_off <= prev_offset) || 14409 linfo[i].insn_off >= prog->len) { 14410 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 14411 i, linfo[i].insn_off, prev_offset, 14412 prog->len); 14413 err = -EINVAL; 14414 goto err_free; 14415 } 14416 14417 if (!prog->insnsi[linfo[i].insn_off].code) { 14418 verbose(env, 14419 "Invalid insn code at line_info[%u].insn_off\n", 14420 i); 14421 err = -EINVAL; 14422 goto err_free; 14423 } 14424 14425 if (!btf_name_by_offset(btf, linfo[i].line_off) || 14426 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 14427 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 14428 err = -EINVAL; 14429 goto err_free; 14430 } 14431 14432 if (s != env->subprog_cnt) { 14433 if (linfo[i].insn_off == sub[s].start) { 14434 sub[s].linfo_idx = i; 14435 s++; 14436 } else if (sub[s].start < linfo[i].insn_off) { 14437 verbose(env, "missing bpf_line_info for func#%u\n", s); 14438 err = -EINVAL; 14439 goto err_free; 14440 } 14441 } 14442 14443 prev_offset = linfo[i].insn_off; 14444 bpfptr_add(&ulinfo, rec_size); 14445 } 14446 14447 if (s != env->subprog_cnt) { 14448 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 14449 env->subprog_cnt - s, s); 14450 err = -EINVAL; 14451 goto err_free; 14452 } 14453 14454 prog->aux->linfo = linfo; 14455 prog->aux->nr_linfo = nr_linfo; 14456 14457 return 0; 14458 14459 err_free: 14460 kvfree(linfo); 14461 return err; 14462 } 14463 14464 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 14465 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 14466 14467 static int check_core_relo(struct bpf_verifier_env *env, 14468 const union bpf_attr *attr, 14469 bpfptr_t uattr) 14470 { 14471 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 14472 struct bpf_core_relo core_relo = {}; 14473 struct bpf_prog *prog = env->prog; 14474 const struct btf *btf = prog->aux->btf; 14475 struct bpf_core_ctx ctx = { 14476 .log = &env->log, 14477 .btf = btf, 14478 }; 14479 bpfptr_t u_core_relo; 14480 int err; 14481 14482 nr_core_relo = attr->core_relo_cnt; 14483 if (!nr_core_relo) 14484 return 0; 14485 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 14486 return -EINVAL; 14487 14488 rec_size = attr->core_relo_rec_size; 14489 if (rec_size < MIN_CORE_RELO_SIZE || 14490 rec_size > MAX_CORE_RELO_SIZE || 14491 rec_size % sizeof(u32)) 14492 return -EINVAL; 14493 14494 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 14495 expected_size = sizeof(struct bpf_core_relo); 14496 ncopy = min_t(u32, expected_size, rec_size); 14497 14498 /* Unlike func_info and line_info, copy and apply each CO-RE 14499 * relocation record one at a time. 14500 */ 14501 for (i = 0; i < nr_core_relo; i++) { 14502 /* future proofing when sizeof(bpf_core_relo) changes */ 14503 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 14504 if (err) { 14505 if (err == -E2BIG) { 14506 verbose(env, "nonzero tailing record in core_relo"); 14507 if (copy_to_bpfptr_offset(uattr, 14508 offsetof(union bpf_attr, core_relo_rec_size), 14509 &expected_size, sizeof(expected_size))) 14510 err = -EFAULT; 14511 } 14512 break; 14513 } 14514 14515 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 14516 err = -EFAULT; 14517 break; 14518 } 14519 14520 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 14521 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 14522 i, core_relo.insn_off, prog->len); 14523 err = -EINVAL; 14524 break; 14525 } 14526 14527 err = bpf_core_apply(&ctx, &core_relo, i, 14528 &prog->insnsi[core_relo.insn_off / 8]); 14529 if (err) 14530 break; 14531 bpfptr_add(&u_core_relo, rec_size); 14532 } 14533 return err; 14534 } 14535 14536 static int check_btf_info(struct bpf_verifier_env *env, 14537 const union bpf_attr *attr, 14538 bpfptr_t uattr) 14539 { 14540 struct btf *btf; 14541 int err; 14542 14543 if (!attr->func_info_cnt && !attr->line_info_cnt) { 14544 if (check_abnormal_return(env)) 14545 return -EINVAL; 14546 return 0; 14547 } 14548 14549 btf = btf_get_by_fd(attr->prog_btf_fd); 14550 if (IS_ERR(btf)) 14551 return PTR_ERR(btf); 14552 if (btf_is_kernel(btf)) { 14553 btf_put(btf); 14554 return -EACCES; 14555 } 14556 env->prog->aux->btf = btf; 14557 14558 err = check_btf_func(env, attr, uattr); 14559 if (err) 14560 return err; 14561 14562 err = check_btf_line(env, attr, uattr); 14563 if (err) 14564 return err; 14565 14566 err = check_core_relo(env, attr, uattr); 14567 if (err) 14568 return err; 14569 14570 return 0; 14571 } 14572 14573 /* check %cur's range satisfies %old's */ 14574 static bool range_within(struct bpf_reg_state *old, 14575 struct bpf_reg_state *cur) 14576 { 14577 return old->umin_value <= cur->umin_value && 14578 old->umax_value >= cur->umax_value && 14579 old->smin_value <= cur->smin_value && 14580 old->smax_value >= cur->smax_value && 14581 old->u32_min_value <= cur->u32_min_value && 14582 old->u32_max_value >= cur->u32_max_value && 14583 old->s32_min_value <= cur->s32_min_value && 14584 old->s32_max_value >= cur->s32_max_value; 14585 } 14586 14587 /* If in the old state two registers had the same id, then they need to have 14588 * the same id in the new state as well. But that id could be different from 14589 * the old state, so we need to track the mapping from old to new ids. 14590 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 14591 * regs with old id 5 must also have new id 9 for the new state to be safe. But 14592 * regs with a different old id could still have new id 9, we don't care about 14593 * that. 14594 * So we look through our idmap to see if this old id has been seen before. If 14595 * so, we require the new id to match; otherwise, we add the id pair to the map. 14596 */ 14597 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 14598 { 14599 unsigned int i; 14600 14601 /* either both IDs should be set or both should be zero */ 14602 if (!!old_id != !!cur_id) 14603 return false; 14604 14605 if (old_id == 0) /* cur_id == 0 as well */ 14606 return true; 14607 14608 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 14609 if (!idmap[i].old) { 14610 /* Reached an empty slot; haven't seen this id before */ 14611 idmap[i].old = old_id; 14612 idmap[i].cur = cur_id; 14613 return true; 14614 } 14615 if (idmap[i].old == old_id) 14616 return idmap[i].cur == cur_id; 14617 } 14618 /* We ran out of idmap slots, which should be impossible */ 14619 WARN_ON_ONCE(1); 14620 return false; 14621 } 14622 14623 static void clean_func_state(struct bpf_verifier_env *env, 14624 struct bpf_func_state *st) 14625 { 14626 enum bpf_reg_liveness live; 14627 int i, j; 14628 14629 for (i = 0; i < BPF_REG_FP; i++) { 14630 live = st->regs[i].live; 14631 /* liveness must not touch this register anymore */ 14632 st->regs[i].live |= REG_LIVE_DONE; 14633 if (!(live & REG_LIVE_READ)) 14634 /* since the register is unused, clear its state 14635 * to make further comparison simpler 14636 */ 14637 __mark_reg_not_init(env, &st->regs[i]); 14638 } 14639 14640 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 14641 live = st->stack[i].spilled_ptr.live; 14642 /* liveness must not touch this stack slot anymore */ 14643 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 14644 if (!(live & REG_LIVE_READ)) { 14645 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 14646 for (j = 0; j < BPF_REG_SIZE; j++) 14647 st->stack[i].slot_type[j] = STACK_INVALID; 14648 } 14649 } 14650 } 14651 14652 static void clean_verifier_state(struct bpf_verifier_env *env, 14653 struct bpf_verifier_state *st) 14654 { 14655 int i; 14656 14657 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 14658 /* all regs in this state in all frames were already marked */ 14659 return; 14660 14661 for (i = 0; i <= st->curframe; i++) 14662 clean_func_state(env, st->frame[i]); 14663 } 14664 14665 /* the parentage chains form a tree. 14666 * the verifier states are added to state lists at given insn and 14667 * pushed into state stack for future exploration. 14668 * when the verifier reaches bpf_exit insn some of the verifer states 14669 * stored in the state lists have their final liveness state already, 14670 * but a lot of states will get revised from liveness point of view when 14671 * the verifier explores other branches. 14672 * Example: 14673 * 1: r0 = 1 14674 * 2: if r1 == 100 goto pc+1 14675 * 3: r0 = 2 14676 * 4: exit 14677 * when the verifier reaches exit insn the register r0 in the state list of 14678 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 14679 * of insn 2 and goes exploring further. At the insn 4 it will walk the 14680 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 14681 * 14682 * Since the verifier pushes the branch states as it sees them while exploring 14683 * the program the condition of walking the branch instruction for the second 14684 * time means that all states below this branch were already explored and 14685 * their final liveness marks are already propagated. 14686 * Hence when the verifier completes the search of state list in is_state_visited() 14687 * we can call this clean_live_states() function to mark all liveness states 14688 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 14689 * will not be used. 14690 * This function also clears the registers and stack for states that !READ 14691 * to simplify state merging. 14692 * 14693 * Important note here that walking the same branch instruction in the callee 14694 * doesn't meant that the states are DONE. The verifier has to compare 14695 * the callsites 14696 */ 14697 static void clean_live_states(struct bpf_verifier_env *env, int insn, 14698 struct bpf_verifier_state *cur) 14699 { 14700 struct bpf_verifier_state_list *sl; 14701 int i; 14702 14703 sl = *explored_state(env, insn); 14704 while (sl) { 14705 if (sl->state.branches) 14706 goto next; 14707 if (sl->state.insn_idx != insn || 14708 sl->state.curframe != cur->curframe) 14709 goto next; 14710 for (i = 0; i <= cur->curframe; i++) 14711 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 14712 goto next; 14713 clean_verifier_state(env, &sl->state); 14714 next: 14715 sl = sl->next; 14716 } 14717 } 14718 14719 static bool regs_exact(const struct bpf_reg_state *rold, 14720 const struct bpf_reg_state *rcur, 14721 struct bpf_id_pair *idmap) 14722 { 14723 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 14724 check_ids(rold->id, rcur->id, idmap) && 14725 check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); 14726 } 14727 14728 /* Returns true if (rold safe implies rcur safe) */ 14729 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 14730 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 14731 { 14732 if (!(rold->live & REG_LIVE_READ)) 14733 /* explored state didn't use this */ 14734 return true; 14735 if (rold->type == NOT_INIT) 14736 /* explored state can't have used this */ 14737 return true; 14738 if (rcur->type == NOT_INIT) 14739 return false; 14740 14741 /* Enforce that register types have to match exactly, including their 14742 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general 14743 * rule. 14744 * 14745 * One can make a point that using a pointer register as unbounded 14746 * SCALAR would be technically acceptable, but this could lead to 14747 * pointer leaks because scalars are allowed to leak while pointers 14748 * are not. We could make this safe in special cases if root is 14749 * calling us, but it's probably not worth the hassle. 14750 * 14751 * Also, register types that are *not* MAYBE_NULL could technically be 14752 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE 14753 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point 14754 * to the same map). 14755 * However, if the old MAYBE_NULL register then got NULL checked, 14756 * doing so could have affected others with the same id, and we can't 14757 * check for that because we lost the id when we converted to 14758 * a non-MAYBE_NULL variant. 14759 * So, as a general rule we don't allow mixing MAYBE_NULL and 14760 * non-MAYBE_NULL registers as well. 14761 */ 14762 if (rold->type != rcur->type) 14763 return false; 14764 14765 switch (base_type(rold->type)) { 14766 case SCALAR_VALUE: 14767 if (regs_exact(rold, rcur, idmap)) 14768 return true; 14769 if (env->explore_alu_limits) 14770 return false; 14771 if (!rold->precise) 14772 return true; 14773 /* new val must satisfy old val knowledge */ 14774 return range_within(rold, rcur) && 14775 tnum_in(rold->var_off, rcur->var_off); 14776 case PTR_TO_MAP_KEY: 14777 case PTR_TO_MAP_VALUE: 14778 case PTR_TO_MEM: 14779 case PTR_TO_BUF: 14780 case PTR_TO_TP_BUFFER: 14781 /* If the new min/max/var_off satisfy the old ones and 14782 * everything else matches, we are OK. 14783 */ 14784 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 && 14785 range_within(rold, rcur) && 14786 tnum_in(rold->var_off, rcur->var_off) && 14787 check_ids(rold->id, rcur->id, idmap) && 14788 check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); 14789 case PTR_TO_PACKET_META: 14790 case PTR_TO_PACKET: 14791 /* We must have at least as much range as the old ptr 14792 * did, so that any accesses which were safe before are 14793 * still safe. This is true even if old range < old off, 14794 * since someone could have accessed through (ptr - k), or 14795 * even done ptr -= k in a register, to get a safe access. 14796 */ 14797 if (rold->range > rcur->range) 14798 return false; 14799 /* If the offsets don't match, we can't trust our alignment; 14800 * nor can we be sure that we won't fall out of range. 14801 */ 14802 if (rold->off != rcur->off) 14803 return false; 14804 /* id relations must be preserved */ 14805 if (!check_ids(rold->id, rcur->id, idmap)) 14806 return false; 14807 /* new val must satisfy old val knowledge */ 14808 return range_within(rold, rcur) && 14809 tnum_in(rold->var_off, rcur->var_off); 14810 case PTR_TO_STACK: 14811 /* two stack pointers are equal only if they're pointing to 14812 * the same stack frame, since fp-8 in foo != fp-8 in bar 14813 */ 14814 return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; 14815 default: 14816 return regs_exact(rold, rcur, idmap); 14817 } 14818 } 14819 14820 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 14821 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 14822 { 14823 int i, spi; 14824 14825 /* walk slots of the explored stack and ignore any additional 14826 * slots in the current stack, since explored(safe) state 14827 * didn't use them 14828 */ 14829 for (i = 0; i < old->allocated_stack; i++) { 14830 struct bpf_reg_state *old_reg, *cur_reg; 14831 14832 spi = i / BPF_REG_SIZE; 14833 14834 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 14835 i += BPF_REG_SIZE - 1; 14836 /* explored state didn't use this */ 14837 continue; 14838 } 14839 14840 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 14841 continue; 14842 14843 if (env->allow_uninit_stack && 14844 old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) 14845 continue; 14846 14847 /* explored stack has more populated slots than current stack 14848 * and these slots were used 14849 */ 14850 if (i >= cur->allocated_stack) 14851 return false; 14852 14853 /* if old state was safe with misc data in the stack 14854 * it will be safe with zero-initialized stack. 14855 * The opposite is not true 14856 */ 14857 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 14858 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 14859 continue; 14860 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 14861 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 14862 /* Ex: old explored (safe) state has STACK_SPILL in 14863 * this stack slot, but current has STACK_MISC -> 14864 * this verifier states are not equivalent, 14865 * return false to continue verification of this path 14866 */ 14867 return false; 14868 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 14869 continue; 14870 /* Both old and cur are having same slot_type */ 14871 switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { 14872 case STACK_SPILL: 14873 /* when explored and current stack slot are both storing 14874 * spilled registers, check that stored pointers types 14875 * are the same as well. 14876 * Ex: explored safe path could have stored 14877 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 14878 * but current path has stored: 14879 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 14880 * such verifier states are not equivalent. 14881 * return false to continue verification of this path 14882 */ 14883 if (!regsafe(env, &old->stack[spi].spilled_ptr, 14884 &cur->stack[spi].spilled_ptr, idmap)) 14885 return false; 14886 break; 14887 case STACK_DYNPTR: 14888 old_reg = &old->stack[spi].spilled_ptr; 14889 cur_reg = &cur->stack[spi].spilled_ptr; 14890 if (old_reg->dynptr.type != cur_reg->dynptr.type || 14891 old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || 14892 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 14893 return false; 14894 break; 14895 case STACK_ITER: 14896 old_reg = &old->stack[spi].spilled_ptr; 14897 cur_reg = &cur->stack[spi].spilled_ptr; 14898 /* iter.depth is not compared between states as it 14899 * doesn't matter for correctness and would otherwise 14900 * prevent convergence; we maintain it only to prevent 14901 * infinite loop check triggering, see 14902 * iter_active_depths_differ() 14903 */ 14904 if (old_reg->iter.btf != cur_reg->iter.btf || 14905 old_reg->iter.btf_id != cur_reg->iter.btf_id || 14906 old_reg->iter.state != cur_reg->iter.state || 14907 /* ignore {old_reg,cur_reg}->iter.depth, see above */ 14908 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 14909 return false; 14910 break; 14911 case STACK_MISC: 14912 case STACK_ZERO: 14913 case STACK_INVALID: 14914 continue; 14915 /* Ensure that new unhandled slot types return false by default */ 14916 default: 14917 return false; 14918 } 14919 } 14920 return true; 14921 } 14922 14923 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, 14924 struct bpf_id_pair *idmap) 14925 { 14926 int i; 14927 14928 if (old->acquired_refs != cur->acquired_refs) 14929 return false; 14930 14931 for (i = 0; i < old->acquired_refs; i++) { 14932 if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap)) 14933 return false; 14934 } 14935 14936 return true; 14937 } 14938 14939 /* compare two verifier states 14940 * 14941 * all states stored in state_list are known to be valid, since 14942 * verifier reached 'bpf_exit' instruction through them 14943 * 14944 * this function is called when verifier exploring different branches of 14945 * execution popped from the state stack. If it sees an old state that has 14946 * more strict register state and more strict stack state then this execution 14947 * branch doesn't need to be explored further, since verifier already 14948 * concluded that more strict state leads to valid finish. 14949 * 14950 * Therefore two states are equivalent if register state is more conservative 14951 * and explored stack state is more conservative than the current one. 14952 * Example: 14953 * explored current 14954 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 14955 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 14956 * 14957 * In other words if current stack state (one being explored) has more 14958 * valid slots than old one that already passed validation, it means 14959 * the verifier can stop exploring and conclude that current state is valid too 14960 * 14961 * Similarly with registers. If explored state has register type as invalid 14962 * whereas register type in current state is meaningful, it means that 14963 * the current state will reach 'bpf_exit' instruction safely 14964 */ 14965 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 14966 struct bpf_func_state *cur) 14967 { 14968 int i; 14969 14970 for (i = 0; i < MAX_BPF_REG; i++) 14971 if (!regsafe(env, &old->regs[i], &cur->regs[i], 14972 env->idmap_scratch)) 14973 return false; 14974 14975 if (!stacksafe(env, old, cur, env->idmap_scratch)) 14976 return false; 14977 14978 if (!refsafe(old, cur, env->idmap_scratch)) 14979 return false; 14980 14981 return true; 14982 } 14983 14984 static bool states_equal(struct bpf_verifier_env *env, 14985 struct bpf_verifier_state *old, 14986 struct bpf_verifier_state *cur) 14987 { 14988 int i; 14989 14990 if (old->curframe != cur->curframe) 14991 return false; 14992 14993 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 14994 14995 /* Verification state from speculative execution simulation 14996 * must never prune a non-speculative execution one. 14997 */ 14998 if (old->speculative && !cur->speculative) 14999 return false; 15000 15001 if (old->active_lock.ptr != cur->active_lock.ptr) 15002 return false; 15003 15004 /* Old and cur active_lock's have to be either both present 15005 * or both absent. 15006 */ 15007 if (!!old->active_lock.id != !!cur->active_lock.id) 15008 return false; 15009 15010 if (old->active_lock.id && 15011 !check_ids(old->active_lock.id, cur->active_lock.id, env->idmap_scratch)) 15012 return false; 15013 15014 if (old->active_rcu_lock != cur->active_rcu_lock) 15015 return false; 15016 15017 /* for states to be equal callsites have to be the same 15018 * and all frame states need to be equivalent 15019 */ 15020 for (i = 0; i <= old->curframe; i++) { 15021 if (old->frame[i]->callsite != cur->frame[i]->callsite) 15022 return false; 15023 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 15024 return false; 15025 } 15026 return true; 15027 } 15028 15029 /* Return 0 if no propagation happened. Return negative error code if error 15030 * happened. Otherwise, return the propagated bit. 15031 */ 15032 static int propagate_liveness_reg(struct bpf_verifier_env *env, 15033 struct bpf_reg_state *reg, 15034 struct bpf_reg_state *parent_reg) 15035 { 15036 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 15037 u8 flag = reg->live & REG_LIVE_READ; 15038 int err; 15039 15040 /* When comes here, read flags of PARENT_REG or REG could be any of 15041 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 15042 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 15043 */ 15044 if (parent_flag == REG_LIVE_READ64 || 15045 /* Or if there is no read flag from REG. */ 15046 !flag || 15047 /* Or if the read flag from REG is the same as PARENT_REG. */ 15048 parent_flag == flag) 15049 return 0; 15050 15051 err = mark_reg_read(env, reg, parent_reg, flag); 15052 if (err) 15053 return err; 15054 15055 return flag; 15056 } 15057 15058 /* A write screens off any subsequent reads; but write marks come from the 15059 * straight-line code between a state and its parent. When we arrive at an 15060 * equivalent state (jump target or such) we didn't arrive by the straight-line 15061 * code, so read marks in the state must propagate to the parent regardless 15062 * of the state's write marks. That's what 'parent == state->parent' comparison 15063 * in mark_reg_read() is for. 15064 */ 15065 static int propagate_liveness(struct bpf_verifier_env *env, 15066 const struct bpf_verifier_state *vstate, 15067 struct bpf_verifier_state *vparent) 15068 { 15069 struct bpf_reg_state *state_reg, *parent_reg; 15070 struct bpf_func_state *state, *parent; 15071 int i, frame, err = 0; 15072 15073 if (vparent->curframe != vstate->curframe) { 15074 WARN(1, "propagate_live: parent frame %d current frame %d\n", 15075 vparent->curframe, vstate->curframe); 15076 return -EFAULT; 15077 } 15078 /* Propagate read liveness of registers... */ 15079 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 15080 for (frame = 0; frame <= vstate->curframe; frame++) { 15081 parent = vparent->frame[frame]; 15082 state = vstate->frame[frame]; 15083 parent_reg = parent->regs; 15084 state_reg = state->regs; 15085 /* We don't need to worry about FP liveness, it's read-only */ 15086 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 15087 err = propagate_liveness_reg(env, &state_reg[i], 15088 &parent_reg[i]); 15089 if (err < 0) 15090 return err; 15091 if (err == REG_LIVE_READ64) 15092 mark_insn_zext(env, &parent_reg[i]); 15093 } 15094 15095 /* Propagate stack slots. */ 15096 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 15097 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 15098 parent_reg = &parent->stack[i].spilled_ptr; 15099 state_reg = &state->stack[i].spilled_ptr; 15100 err = propagate_liveness_reg(env, state_reg, 15101 parent_reg); 15102 if (err < 0) 15103 return err; 15104 } 15105 } 15106 return 0; 15107 } 15108 15109 /* find precise scalars in the previous equivalent state and 15110 * propagate them into the current state 15111 */ 15112 static int propagate_precision(struct bpf_verifier_env *env, 15113 const struct bpf_verifier_state *old) 15114 { 15115 struct bpf_reg_state *state_reg; 15116 struct bpf_func_state *state; 15117 int i, err = 0, fr; 15118 15119 for (fr = old->curframe; fr >= 0; fr--) { 15120 state = old->frame[fr]; 15121 state_reg = state->regs; 15122 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 15123 if (state_reg->type != SCALAR_VALUE || 15124 !state_reg->precise || 15125 !(state_reg->live & REG_LIVE_READ)) 15126 continue; 15127 if (env->log.level & BPF_LOG_LEVEL2) 15128 verbose(env, "frame %d: propagating r%d\n", fr, i); 15129 err = mark_chain_precision_frame(env, fr, i); 15130 if (err < 0) 15131 return err; 15132 } 15133 15134 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 15135 if (!is_spilled_reg(&state->stack[i])) 15136 continue; 15137 state_reg = &state->stack[i].spilled_ptr; 15138 if (state_reg->type != SCALAR_VALUE || 15139 !state_reg->precise || 15140 !(state_reg->live & REG_LIVE_READ)) 15141 continue; 15142 if (env->log.level & BPF_LOG_LEVEL2) 15143 verbose(env, "frame %d: propagating fp%d\n", 15144 fr, (-i - 1) * BPF_REG_SIZE); 15145 err = mark_chain_precision_stack_frame(env, fr, i); 15146 if (err < 0) 15147 return err; 15148 } 15149 } 15150 return 0; 15151 } 15152 15153 static bool states_maybe_looping(struct bpf_verifier_state *old, 15154 struct bpf_verifier_state *cur) 15155 { 15156 struct bpf_func_state *fold, *fcur; 15157 int i, fr = cur->curframe; 15158 15159 if (old->curframe != fr) 15160 return false; 15161 15162 fold = old->frame[fr]; 15163 fcur = cur->frame[fr]; 15164 for (i = 0; i < MAX_BPF_REG; i++) 15165 if (memcmp(&fold->regs[i], &fcur->regs[i], 15166 offsetof(struct bpf_reg_state, parent))) 15167 return false; 15168 return true; 15169 } 15170 15171 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx) 15172 { 15173 return env->insn_aux_data[insn_idx].is_iter_next; 15174 } 15175 15176 /* is_state_visited() handles iter_next() (see process_iter_next_call() for 15177 * terminology) calls specially: as opposed to bounded BPF loops, it *expects* 15178 * states to match, which otherwise would look like an infinite loop. So while 15179 * iter_next() calls are taken care of, we still need to be careful and 15180 * prevent erroneous and too eager declaration of "ininite loop", when 15181 * iterators are involved. 15182 * 15183 * Here's a situation in pseudo-BPF assembly form: 15184 * 15185 * 0: again: ; set up iter_next() call args 15186 * 1: r1 = &it ; <CHECKPOINT HERE> 15187 * 2: call bpf_iter_num_next ; this is iter_next() call 15188 * 3: if r0 == 0 goto done 15189 * 4: ... something useful here ... 15190 * 5: goto again ; another iteration 15191 * 6: done: 15192 * 7: r1 = &it 15193 * 8: call bpf_iter_num_destroy ; clean up iter state 15194 * 9: exit 15195 * 15196 * This is a typical loop. Let's assume that we have a prune point at 1:, 15197 * before we get to `call bpf_iter_num_next` (e.g., because of that `goto 15198 * again`, assuming other heuristics don't get in a way). 15199 * 15200 * When we first time come to 1:, let's say we have some state X. We proceed 15201 * to 2:, fork states, enqueue ACTIVE, validate NULL case successfully, exit. 15202 * Now we come back to validate that forked ACTIVE state. We proceed through 15203 * 3-5, come to goto, jump to 1:. Let's assume our state didn't change, so we 15204 * are converging. But the problem is that we don't know that yet, as this 15205 * convergence has to happen at iter_next() call site only. So if nothing is 15206 * done, at 1: verifier will use bounded loop logic and declare infinite 15207 * looping (and would be *technically* correct, if not for iterator's 15208 * "eventual sticky NULL" contract, see process_iter_next_call()). But we 15209 * don't want that. So what we do in process_iter_next_call() when we go on 15210 * another ACTIVE iteration, we bump slot->iter.depth, to mark that it's 15211 * a different iteration. So when we suspect an infinite loop, we additionally 15212 * check if any of the *ACTIVE* iterator states depths differ. If yes, we 15213 * pretend we are not looping and wait for next iter_next() call. 15214 * 15215 * This only applies to ACTIVE state. In DRAINED state we don't expect to 15216 * loop, because that would actually mean infinite loop, as DRAINED state is 15217 * "sticky", and so we'll keep returning into the same instruction with the 15218 * same state (at least in one of possible code paths). 15219 * 15220 * This approach allows to keep infinite loop heuristic even in the face of 15221 * active iterator. E.g., C snippet below is and will be detected as 15222 * inifintely looping: 15223 * 15224 * struct bpf_iter_num it; 15225 * int *p, x; 15226 * 15227 * bpf_iter_num_new(&it, 0, 10); 15228 * while ((p = bpf_iter_num_next(&t))) { 15229 * x = p; 15230 * while (x--) {} // <<-- infinite loop here 15231 * } 15232 * 15233 */ 15234 static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf_verifier_state *cur) 15235 { 15236 struct bpf_reg_state *slot, *cur_slot; 15237 struct bpf_func_state *state; 15238 int i, fr; 15239 15240 for (fr = old->curframe; fr >= 0; fr--) { 15241 state = old->frame[fr]; 15242 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 15243 if (state->stack[i].slot_type[0] != STACK_ITER) 15244 continue; 15245 15246 slot = &state->stack[i].spilled_ptr; 15247 if (slot->iter.state != BPF_ITER_STATE_ACTIVE) 15248 continue; 15249 15250 cur_slot = &cur->frame[fr]->stack[i].spilled_ptr; 15251 if (cur_slot->iter.depth != slot->iter.depth) 15252 return true; 15253 } 15254 } 15255 return false; 15256 } 15257 15258 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 15259 { 15260 struct bpf_verifier_state_list *new_sl; 15261 struct bpf_verifier_state_list *sl, **pprev; 15262 struct bpf_verifier_state *cur = env->cur_state, *new; 15263 int i, j, err, states_cnt = 0; 15264 bool force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx); 15265 bool add_new_state = force_new_state; 15266 15267 /* bpf progs typically have pruning point every 4 instructions 15268 * http://vger.kernel.org/bpfconf2019.html#session-1 15269 * Do not add new state for future pruning if the verifier hasn't seen 15270 * at least 2 jumps and at least 8 instructions. 15271 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 15272 * In tests that amounts to up to 50% reduction into total verifier 15273 * memory consumption and 20% verifier time speedup. 15274 */ 15275 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 15276 env->insn_processed - env->prev_insn_processed >= 8) 15277 add_new_state = true; 15278 15279 pprev = explored_state(env, insn_idx); 15280 sl = *pprev; 15281 15282 clean_live_states(env, insn_idx, cur); 15283 15284 while (sl) { 15285 states_cnt++; 15286 if (sl->state.insn_idx != insn_idx) 15287 goto next; 15288 15289 if (sl->state.branches) { 15290 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 15291 15292 if (frame->in_async_callback_fn && 15293 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 15294 /* Different async_entry_cnt means that the verifier is 15295 * processing another entry into async callback. 15296 * Seeing the same state is not an indication of infinite 15297 * loop or infinite recursion. 15298 * But finding the same state doesn't mean that it's safe 15299 * to stop processing the current state. The previous state 15300 * hasn't yet reached bpf_exit, since state.branches > 0. 15301 * Checking in_async_callback_fn alone is not enough either. 15302 * Since the verifier still needs to catch infinite loops 15303 * inside async callbacks. 15304 */ 15305 goto skip_inf_loop_check; 15306 } 15307 /* BPF open-coded iterators loop detection is special. 15308 * states_maybe_looping() logic is too simplistic in detecting 15309 * states that *might* be equivalent, because it doesn't know 15310 * about ID remapping, so don't even perform it. 15311 * See process_iter_next_call() and iter_active_depths_differ() 15312 * for overview of the logic. When current and one of parent 15313 * states are detected as equivalent, it's a good thing: we prove 15314 * convergence and can stop simulating further iterations. 15315 * It's safe to assume that iterator loop will finish, taking into 15316 * account iter_next() contract of eventually returning 15317 * sticky NULL result. 15318 */ 15319 if (is_iter_next_insn(env, insn_idx)) { 15320 if (states_equal(env, &sl->state, cur)) { 15321 struct bpf_func_state *cur_frame; 15322 struct bpf_reg_state *iter_state, *iter_reg; 15323 int spi; 15324 15325 cur_frame = cur->frame[cur->curframe]; 15326 /* btf_check_iter_kfuncs() enforces that 15327 * iter state pointer is always the first arg 15328 */ 15329 iter_reg = &cur_frame->regs[BPF_REG_1]; 15330 /* current state is valid due to states_equal(), 15331 * so we can assume valid iter and reg state, 15332 * no need for extra (re-)validations 15333 */ 15334 spi = __get_spi(iter_reg->off + iter_reg->var_off.value); 15335 iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr; 15336 if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) 15337 goto hit; 15338 } 15339 goto skip_inf_loop_check; 15340 } 15341 /* attempt to detect infinite loop to avoid unnecessary doomed work */ 15342 if (states_maybe_looping(&sl->state, cur) && 15343 states_equal(env, &sl->state, cur) && 15344 !iter_active_depths_differ(&sl->state, cur)) { 15345 verbose_linfo(env, insn_idx, "; "); 15346 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 15347 return -EINVAL; 15348 } 15349 /* if the verifier is processing a loop, avoid adding new state 15350 * too often, since different loop iterations have distinct 15351 * states and may not help future pruning. 15352 * This threshold shouldn't be too low to make sure that 15353 * a loop with large bound will be rejected quickly. 15354 * The most abusive loop will be: 15355 * r1 += 1 15356 * if r1 < 1000000 goto pc-2 15357 * 1M insn_procssed limit / 100 == 10k peak states. 15358 * This threshold shouldn't be too high either, since states 15359 * at the end of the loop are likely to be useful in pruning. 15360 */ 15361 skip_inf_loop_check: 15362 if (!force_new_state && 15363 env->jmps_processed - env->prev_jmps_processed < 20 && 15364 env->insn_processed - env->prev_insn_processed < 100) 15365 add_new_state = false; 15366 goto miss; 15367 } 15368 if (states_equal(env, &sl->state, cur)) { 15369 hit: 15370 sl->hit_cnt++; 15371 /* reached equivalent register/stack state, 15372 * prune the search. 15373 * Registers read by the continuation are read by us. 15374 * If we have any write marks in env->cur_state, they 15375 * will prevent corresponding reads in the continuation 15376 * from reaching our parent (an explored_state). Our 15377 * own state will get the read marks recorded, but 15378 * they'll be immediately forgotten as we're pruning 15379 * this state and will pop a new one. 15380 */ 15381 err = propagate_liveness(env, &sl->state, cur); 15382 15383 /* if previous state reached the exit with precision and 15384 * current state is equivalent to it (except precsion marks) 15385 * the precision needs to be propagated back in 15386 * the current state. 15387 */ 15388 err = err ? : push_jmp_history(env, cur); 15389 err = err ? : propagate_precision(env, &sl->state); 15390 if (err) 15391 return err; 15392 return 1; 15393 } 15394 miss: 15395 /* when new state is not going to be added do not increase miss count. 15396 * Otherwise several loop iterations will remove the state 15397 * recorded earlier. The goal of these heuristics is to have 15398 * states from some iterations of the loop (some in the beginning 15399 * and some at the end) to help pruning. 15400 */ 15401 if (add_new_state) 15402 sl->miss_cnt++; 15403 /* heuristic to determine whether this state is beneficial 15404 * to keep checking from state equivalence point of view. 15405 * Higher numbers increase max_states_per_insn and verification time, 15406 * but do not meaningfully decrease insn_processed. 15407 */ 15408 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 15409 /* the state is unlikely to be useful. Remove it to 15410 * speed up verification 15411 */ 15412 *pprev = sl->next; 15413 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 15414 u32 br = sl->state.branches; 15415 15416 WARN_ONCE(br, 15417 "BUG live_done but branches_to_explore %d\n", 15418 br); 15419 free_verifier_state(&sl->state, false); 15420 kfree(sl); 15421 env->peak_states--; 15422 } else { 15423 /* cannot free this state, since parentage chain may 15424 * walk it later. Add it for free_list instead to 15425 * be freed at the end of verification 15426 */ 15427 sl->next = env->free_list; 15428 env->free_list = sl; 15429 } 15430 sl = *pprev; 15431 continue; 15432 } 15433 next: 15434 pprev = &sl->next; 15435 sl = *pprev; 15436 } 15437 15438 if (env->max_states_per_insn < states_cnt) 15439 env->max_states_per_insn = states_cnt; 15440 15441 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 15442 return 0; 15443 15444 if (!add_new_state) 15445 return 0; 15446 15447 /* There were no equivalent states, remember the current one. 15448 * Technically the current state is not proven to be safe yet, 15449 * but it will either reach outer most bpf_exit (which means it's safe) 15450 * or it will be rejected. When there are no loops the verifier won't be 15451 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 15452 * again on the way to bpf_exit. 15453 * When looping the sl->state.branches will be > 0 and this state 15454 * will not be considered for equivalence until branches == 0. 15455 */ 15456 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 15457 if (!new_sl) 15458 return -ENOMEM; 15459 env->total_states++; 15460 env->peak_states++; 15461 env->prev_jmps_processed = env->jmps_processed; 15462 env->prev_insn_processed = env->insn_processed; 15463 15464 /* forget precise markings we inherited, see __mark_chain_precision */ 15465 if (env->bpf_capable) 15466 mark_all_scalars_imprecise(env, cur); 15467 15468 /* add new state to the head of linked list */ 15469 new = &new_sl->state; 15470 err = copy_verifier_state(new, cur); 15471 if (err) { 15472 free_verifier_state(new, false); 15473 kfree(new_sl); 15474 return err; 15475 } 15476 new->insn_idx = insn_idx; 15477 WARN_ONCE(new->branches != 1, 15478 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 15479 15480 cur->parent = new; 15481 cur->first_insn_idx = insn_idx; 15482 clear_jmp_history(cur); 15483 new_sl->next = *explored_state(env, insn_idx); 15484 *explored_state(env, insn_idx) = new_sl; 15485 /* connect new state to parentage chain. Current frame needs all 15486 * registers connected. Only r6 - r9 of the callers are alive (pushed 15487 * to the stack implicitly by JITs) so in callers' frames connect just 15488 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 15489 * the state of the call instruction (with WRITTEN set), and r0 comes 15490 * from callee with its full parentage chain, anyway. 15491 */ 15492 /* clear write marks in current state: the writes we did are not writes 15493 * our child did, so they don't screen off its reads from us. 15494 * (There are no read marks in current state, because reads always mark 15495 * their parent and current state never has children yet. Only 15496 * explored_states can get read marks.) 15497 */ 15498 for (j = 0; j <= cur->curframe; j++) { 15499 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 15500 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 15501 for (i = 0; i < BPF_REG_FP; i++) 15502 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 15503 } 15504 15505 /* all stack frames are accessible from callee, clear them all */ 15506 for (j = 0; j <= cur->curframe; j++) { 15507 struct bpf_func_state *frame = cur->frame[j]; 15508 struct bpf_func_state *newframe = new->frame[j]; 15509 15510 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 15511 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 15512 frame->stack[i].spilled_ptr.parent = 15513 &newframe->stack[i].spilled_ptr; 15514 } 15515 } 15516 return 0; 15517 } 15518 15519 /* Return true if it's OK to have the same insn return a different type. */ 15520 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 15521 { 15522 switch (base_type(type)) { 15523 case PTR_TO_CTX: 15524 case PTR_TO_SOCKET: 15525 case PTR_TO_SOCK_COMMON: 15526 case PTR_TO_TCP_SOCK: 15527 case PTR_TO_XDP_SOCK: 15528 case PTR_TO_BTF_ID: 15529 return false; 15530 default: 15531 return true; 15532 } 15533 } 15534 15535 /* If an instruction was previously used with particular pointer types, then we 15536 * need to be careful to avoid cases such as the below, where it may be ok 15537 * for one branch accessing the pointer, but not ok for the other branch: 15538 * 15539 * R1 = sock_ptr 15540 * goto X; 15541 * ... 15542 * R1 = some_other_valid_ptr; 15543 * goto X; 15544 * ... 15545 * R2 = *(u32 *)(R1 + 0); 15546 */ 15547 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 15548 { 15549 return src != prev && (!reg_type_mismatch_ok(src) || 15550 !reg_type_mismatch_ok(prev)); 15551 } 15552 15553 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, 15554 bool allow_trust_missmatch) 15555 { 15556 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; 15557 15558 if (*prev_type == NOT_INIT) { 15559 /* Saw a valid insn 15560 * dst_reg = *(u32 *)(src_reg + off) 15561 * save type to validate intersecting paths 15562 */ 15563 *prev_type = type; 15564 } else if (reg_type_mismatch(type, *prev_type)) { 15565 /* Abuser program is trying to use the same insn 15566 * dst_reg = *(u32*) (src_reg + off) 15567 * with different pointer types: 15568 * src_reg == ctx in one branch and 15569 * src_reg == stack|map in some other branch. 15570 * Reject it. 15571 */ 15572 if (allow_trust_missmatch && 15573 base_type(type) == PTR_TO_BTF_ID && 15574 base_type(*prev_type) == PTR_TO_BTF_ID) { 15575 /* 15576 * Have to support a use case when one path through 15577 * the program yields TRUSTED pointer while another 15578 * is UNTRUSTED. Fallback to UNTRUSTED to generate 15579 * BPF_PROBE_MEM. 15580 */ 15581 *prev_type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 15582 } else { 15583 verbose(env, "same insn cannot be used with different pointers\n"); 15584 return -EINVAL; 15585 } 15586 } 15587 15588 return 0; 15589 } 15590 15591 static int do_check(struct bpf_verifier_env *env) 15592 { 15593 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 15594 struct bpf_verifier_state *state = env->cur_state; 15595 struct bpf_insn *insns = env->prog->insnsi; 15596 struct bpf_reg_state *regs; 15597 int insn_cnt = env->prog->len; 15598 bool do_print_state = false; 15599 int prev_insn_idx = -1; 15600 15601 for (;;) { 15602 struct bpf_insn *insn; 15603 u8 class; 15604 int err; 15605 15606 env->prev_insn_idx = prev_insn_idx; 15607 if (env->insn_idx >= insn_cnt) { 15608 verbose(env, "invalid insn idx %d insn_cnt %d\n", 15609 env->insn_idx, insn_cnt); 15610 return -EFAULT; 15611 } 15612 15613 insn = &insns[env->insn_idx]; 15614 class = BPF_CLASS(insn->code); 15615 15616 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 15617 verbose(env, 15618 "BPF program is too large. Processed %d insn\n", 15619 env->insn_processed); 15620 return -E2BIG; 15621 } 15622 15623 state->last_insn_idx = env->prev_insn_idx; 15624 15625 if (is_prune_point(env, env->insn_idx)) { 15626 err = is_state_visited(env, env->insn_idx); 15627 if (err < 0) 15628 return err; 15629 if (err == 1) { 15630 /* found equivalent state, can prune the search */ 15631 if (env->log.level & BPF_LOG_LEVEL) { 15632 if (do_print_state) 15633 verbose(env, "\nfrom %d to %d%s: safe\n", 15634 env->prev_insn_idx, env->insn_idx, 15635 env->cur_state->speculative ? 15636 " (speculative execution)" : ""); 15637 else 15638 verbose(env, "%d: safe\n", env->insn_idx); 15639 } 15640 goto process_bpf_exit; 15641 } 15642 } 15643 15644 if (is_jmp_point(env, env->insn_idx)) { 15645 err = push_jmp_history(env, state); 15646 if (err) 15647 return err; 15648 } 15649 15650 if (signal_pending(current)) 15651 return -EAGAIN; 15652 15653 if (need_resched()) 15654 cond_resched(); 15655 15656 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 15657 verbose(env, "\nfrom %d to %d%s:", 15658 env->prev_insn_idx, env->insn_idx, 15659 env->cur_state->speculative ? 15660 " (speculative execution)" : ""); 15661 print_verifier_state(env, state->frame[state->curframe], true); 15662 do_print_state = false; 15663 } 15664 15665 if (env->log.level & BPF_LOG_LEVEL) { 15666 const struct bpf_insn_cbs cbs = { 15667 .cb_call = disasm_kfunc_name, 15668 .cb_print = verbose, 15669 .private_data = env, 15670 }; 15671 15672 if (verifier_state_scratched(env)) 15673 print_insn_state(env, state->frame[state->curframe]); 15674 15675 verbose_linfo(env, env->insn_idx, "; "); 15676 env->prev_log_pos = env->log.end_pos; 15677 verbose(env, "%d: ", env->insn_idx); 15678 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 15679 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; 15680 env->prev_log_pos = env->log.end_pos; 15681 } 15682 15683 if (bpf_prog_is_offloaded(env->prog->aux)) { 15684 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 15685 env->prev_insn_idx); 15686 if (err) 15687 return err; 15688 } 15689 15690 regs = cur_regs(env); 15691 sanitize_mark_insn_seen(env); 15692 prev_insn_idx = env->insn_idx; 15693 15694 if (class == BPF_ALU || class == BPF_ALU64) { 15695 err = check_alu_op(env, insn); 15696 if (err) 15697 return err; 15698 15699 } else if (class == BPF_LDX) { 15700 enum bpf_reg_type src_reg_type; 15701 15702 /* check for reserved fields is already done */ 15703 15704 /* check src operand */ 15705 err = check_reg_arg(env, insn->src_reg, SRC_OP); 15706 if (err) 15707 return err; 15708 15709 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 15710 if (err) 15711 return err; 15712 15713 src_reg_type = regs[insn->src_reg].type; 15714 15715 /* check that memory (src_reg + off) is readable, 15716 * the state of dst_reg will be updated by this func 15717 */ 15718 err = check_mem_access(env, env->insn_idx, insn->src_reg, 15719 insn->off, BPF_SIZE(insn->code), 15720 BPF_READ, insn->dst_reg, false); 15721 if (err) 15722 return err; 15723 15724 err = save_aux_ptr_type(env, src_reg_type, true); 15725 if (err) 15726 return err; 15727 } else if (class == BPF_STX) { 15728 enum bpf_reg_type dst_reg_type; 15729 15730 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 15731 err = check_atomic(env, env->insn_idx, insn); 15732 if (err) 15733 return err; 15734 env->insn_idx++; 15735 continue; 15736 } 15737 15738 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 15739 verbose(env, "BPF_STX uses reserved fields\n"); 15740 return -EINVAL; 15741 } 15742 15743 /* check src1 operand */ 15744 err = check_reg_arg(env, insn->src_reg, SRC_OP); 15745 if (err) 15746 return err; 15747 /* check src2 operand */ 15748 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 15749 if (err) 15750 return err; 15751 15752 dst_reg_type = regs[insn->dst_reg].type; 15753 15754 /* check that memory (dst_reg + off) is writeable */ 15755 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 15756 insn->off, BPF_SIZE(insn->code), 15757 BPF_WRITE, insn->src_reg, false); 15758 if (err) 15759 return err; 15760 15761 err = save_aux_ptr_type(env, dst_reg_type, false); 15762 if (err) 15763 return err; 15764 } else if (class == BPF_ST) { 15765 enum bpf_reg_type dst_reg_type; 15766 15767 if (BPF_MODE(insn->code) != BPF_MEM || 15768 insn->src_reg != BPF_REG_0) { 15769 verbose(env, "BPF_ST uses reserved fields\n"); 15770 return -EINVAL; 15771 } 15772 /* check src operand */ 15773 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 15774 if (err) 15775 return err; 15776 15777 dst_reg_type = regs[insn->dst_reg].type; 15778 15779 /* check that memory (dst_reg + off) is writeable */ 15780 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 15781 insn->off, BPF_SIZE(insn->code), 15782 BPF_WRITE, -1, false); 15783 if (err) 15784 return err; 15785 15786 err = save_aux_ptr_type(env, dst_reg_type, false); 15787 if (err) 15788 return err; 15789 } else if (class == BPF_JMP || class == BPF_JMP32) { 15790 u8 opcode = BPF_OP(insn->code); 15791 15792 env->jmps_processed++; 15793 if (opcode == BPF_CALL) { 15794 if (BPF_SRC(insn->code) != BPF_K || 15795 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 15796 && insn->off != 0) || 15797 (insn->src_reg != BPF_REG_0 && 15798 insn->src_reg != BPF_PSEUDO_CALL && 15799 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 15800 insn->dst_reg != BPF_REG_0 || 15801 class == BPF_JMP32) { 15802 verbose(env, "BPF_CALL uses reserved fields\n"); 15803 return -EINVAL; 15804 } 15805 15806 if (env->cur_state->active_lock.ptr) { 15807 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 15808 (insn->src_reg == BPF_PSEUDO_CALL) || 15809 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 15810 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) { 15811 verbose(env, "function calls are not allowed while holding a lock\n"); 15812 return -EINVAL; 15813 } 15814 } 15815 if (insn->src_reg == BPF_PSEUDO_CALL) 15816 err = check_func_call(env, insn, &env->insn_idx); 15817 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 15818 err = check_kfunc_call(env, insn, &env->insn_idx); 15819 else 15820 err = check_helper_call(env, insn, &env->insn_idx); 15821 if (err) 15822 return err; 15823 15824 mark_reg_scratched(env, BPF_REG_0); 15825 } else if (opcode == BPF_JA) { 15826 if (BPF_SRC(insn->code) != BPF_K || 15827 insn->imm != 0 || 15828 insn->src_reg != BPF_REG_0 || 15829 insn->dst_reg != BPF_REG_0 || 15830 class == BPF_JMP32) { 15831 verbose(env, "BPF_JA uses reserved fields\n"); 15832 return -EINVAL; 15833 } 15834 15835 env->insn_idx += insn->off + 1; 15836 continue; 15837 15838 } else if (opcode == BPF_EXIT) { 15839 if (BPF_SRC(insn->code) != BPF_K || 15840 insn->imm != 0 || 15841 insn->src_reg != BPF_REG_0 || 15842 insn->dst_reg != BPF_REG_0 || 15843 class == BPF_JMP32) { 15844 verbose(env, "BPF_EXIT uses reserved fields\n"); 15845 return -EINVAL; 15846 } 15847 15848 if (env->cur_state->active_lock.ptr && 15849 !in_rbtree_lock_required_cb(env)) { 15850 verbose(env, "bpf_spin_unlock is missing\n"); 15851 return -EINVAL; 15852 } 15853 15854 if (env->cur_state->active_rcu_lock) { 15855 verbose(env, "bpf_rcu_read_unlock is missing\n"); 15856 return -EINVAL; 15857 } 15858 15859 /* We must do check_reference_leak here before 15860 * prepare_func_exit to handle the case when 15861 * state->curframe > 0, it may be a callback 15862 * function, for which reference_state must 15863 * match caller reference state when it exits. 15864 */ 15865 err = check_reference_leak(env); 15866 if (err) 15867 return err; 15868 15869 if (state->curframe) { 15870 /* exit from nested function */ 15871 err = prepare_func_exit(env, &env->insn_idx); 15872 if (err) 15873 return err; 15874 do_print_state = true; 15875 continue; 15876 } 15877 15878 err = check_return_code(env); 15879 if (err) 15880 return err; 15881 process_bpf_exit: 15882 mark_verifier_state_scratched(env); 15883 update_branch_counts(env, env->cur_state); 15884 err = pop_stack(env, &prev_insn_idx, 15885 &env->insn_idx, pop_log); 15886 if (err < 0) { 15887 if (err != -ENOENT) 15888 return err; 15889 break; 15890 } else { 15891 do_print_state = true; 15892 continue; 15893 } 15894 } else { 15895 err = check_cond_jmp_op(env, insn, &env->insn_idx); 15896 if (err) 15897 return err; 15898 } 15899 } else if (class == BPF_LD) { 15900 u8 mode = BPF_MODE(insn->code); 15901 15902 if (mode == BPF_ABS || mode == BPF_IND) { 15903 err = check_ld_abs(env, insn); 15904 if (err) 15905 return err; 15906 15907 } else if (mode == BPF_IMM) { 15908 err = check_ld_imm(env, insn); 15909 if (err) 15910 return err; 15911 15912 env->insn_idx++; 15913 sanitize_mark_insn_seen(env); 15914 } else { 15915 verbose(env, "invalid BPF_LD mode\n"); 15916 return -EINVAL; 15917 } 15918 } else { 15919 verbose(env, "unknown insn class %d\n", class); 15920 return -EINVAL; 15921 } 15922 15923 env->insn_idx++; 15924 } 15925 15926 return 0; 15927 } 15928 15929 static int find_btf_percpu_datasec(struct btf *btf) 15930 { 15931 const struct btf_type *t; 15932 const char *tname; 15933 int i, n; 15934 15935 /* 15936 * Both vmlinux and module each have their own ".data..percpu" 15937 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 15938 * types to look at only module's own BTF types. 15939 */ 15940 n = btf_nr_types(btf); 15941 if (btf_is_module(btf)) 15942 i = btf_nr_types(btf_vmlinux); 15943 else 15944 i = 1; 15945 15946 for(; i < n; i++) { 15947 t = btf_type_by_id(btf, i); 15948 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 15949 continue; 15950 15951 tname = btf_name_by_offset(btf, t->name_off); 15952 if (!strcmp(tname, ".data..percpu")) 15953 return i; 15954 } 15955 15956 return -ENOENT; 15957 } 15958 15959 /* replace pseudo btf_id with kernel symbol address */ 15960 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 15961 struct bpf_insn *insn, 15962 struct bpf_insn_aux_data *aux) 15963 { 15964 const struct btf_var_secinfo *vsi; 15965 const struct btf_type *datasec; 15966 struct btf_mod_pair *btf_mod; 15967 const struct btf_type *t; 15968 const char *sym_name; 15969 bool percpu = false; 15970 u32 type, id = insn->imm; 15971 struct btf *btf; 15972 s32 datasec_id; 15973 u64 addr; 15974 int i, btf_fd, err; 15975 15976 btf_fd = insn[1].imm; 15977 if (btf_fd) { 15978 btf = btf_get_by_fd(btf_fd); 15979 if (IS_ERR(btf)) { 15980 verbose(env, "invalid module BTF object FD specified.\n"); 15981 return -EINVAL; 15982 } 15983 } else { 15984 if (!btf_vmlinux) { 15985 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 15986 return -EINVAL; 15987 } 15988 btf = btf_vmlinux; 15989 btf_get(btf); 15990 } 15991 15992 t = btf_type_by_id(btf, id); 15993 if (!t) { 15994 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 15995 err = -ENOENT; 15996 goto err_put; 15997 } 15998 15999 if (!btf_type_is_var(t) && !btf_type_is_func(t)) { 16000 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); 16001 err = -EINVAL; 16002 goto err_put; 16003 } 16004 16005 sym_name = btf_name_by_offset(btf, t->name_off); 16006 addr = kallsyms_lookup_name(sym_name); 16007 if (!addr) { 16008 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 16009 sym_name); 16010 err = -ENOENT; 16011 goto err_put; 16012 } 16013 insn[0].imm = (u32)addr; 16014 insn[1].imm = addr >> 32; 16015 16016 if (btf_type_is_func(t)) { 16017 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 16018 aux->btf_var.mem_size = 0; 16019 goto check_btf; 16020 } 16021 16022 datasec_id = find_btf_percpu_datasec(btf); 16023 if (datasec_id > 0) { 16024 datasec = btf_type_by_id(btf, datasec_id); 16025 for_each_vsi(i, datasec, vsi) { 16026 if (vsi->type == id) { 16027 percpu = true; 16028 break; 16029 } 16030 } 16031 } 16032 16033 type = t->type; 16034 t = btf_type_skip_modifiers(btf, type, NULL); 16035 if (percpu) { 16036 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 16037 aux->btf_var.btf = btf; 16038 aux->btf_var.btf_id = type; 16039 } else if (!btf_type_is_struct(t)) { 16040 const struct btf_type *ret; 16041 const char *tname; 16042 u32 tsize; 16043 16044 /* resolve the type size of ksym. */ 16045 ret = btf_resolve_size(btf, t, &tsize); 16046 if (IS_ERR(ret)) { 16047 tname = btf_name_by_offset(btf, t->name_off); 16048 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 16049 tname, PTR_ERR(ret)); 16050 err = -EINVAL; 16051 goto err_put; 16052 } 16053 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 16054 aux->btf_var.mem_size = tsize; 16055 } else { 16056 aux->btf_var.reg_type = PTR_TO_BTF_ID; 16057 aux->btf_var.btf = btf; 16058 aux->btf_var.btf_id = type; 16059 } 16060 check_btf: 16061 /* check whether we recorded this BTF (and maybe module) already */ 16062 for (i = 0; i < env->used_btf_cnt; i++) { 16063 if (env->used_btfs[i].btf == btf) { 16064 btf_put(btf); 16065 return 0; 16066 } 16067 } 16068 16069 if (env->used_btf_cnt >= MAX_USED_BTFS) { 16070 err = -E2BIG; 16071 goto err_put; 16072 } 16073 16074 btf_mod = &env->used_btfs[env->used_btf_cnt]; 16075 btf_mod->btf = btf; 16076 btf_mod->module = NULL; 16077 16078 /* if we reference variables from kernel module, bump its refcount */ 16079 if (btf_is_module(btf)) { 16080 btf_mod->module = btf_try_get_module(btf); 16081 if (!btf_mod->module) { 16082 err = -ENXIO; 16083 goto err_put; 16084 } 16085 } 16086 16087 env->used_btf_cnt++; 16088 16089 return 0; 16090 err_put: 16091 btf_put(btf); 16092 return err; 16093 } 16094 16095 static bool is_tracing_prog_type(enum bpf_prog_type type) 16096 { 16097 switch (type) { 16098 case BPF_PROG_TYPE_KPROBE: 16099 case BPF_PROG_TYPE_TRACEPOINT: 16100 case BPF_PROG_TYPE_PERF_EVENT: 16101 case BPF_PROG_TYPE_RAW_TRACEPOINT: 16102 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 16103 return true; 16104 default: 16105 return false; 16106 } 16107 } 16108 16109 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 16110 struct bpf_map *map, 16111 struct bpf_prog *prog) 16112 16113 { 16114 enum bpf_prog_type prog_type = resolve_prog_type(prog); 16115 16116 if (btf_record_has_field(map->record, BPF_LIST_HEAD) || 16117 btf_record_has_field(map->record, BPF_RB_ROOT)) { 16118 if (is_tracing_prog_type(prog_type)) { 16119 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); 16120 return -EINVAL; 16121 } 16122 } 16123 16124 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 16125 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 16126 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 16127 return -EINVAL; 16128 } 16129 16130 if (is_tracing_prog_type(prog_type)) { 16131 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 16132 return -EINVAL; 16133 } 16134 16135 if (prog->aux->sleepable) { 16136 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 16137 return -EINVAL; 16138 } 16139 } 16140 16141 if (btf_record_has_field(map->record, BPF_TIMER)) { 16142 if (is_tracing_prog_type(prog_type)) { 16143 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 16144 return -EINVAL; 16145 } 16146 } 16147 16148 if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && 16149 !bpf_offload_prog_map_match(prog, map)) { 16150 verbose(env, "offload device mismatch between prog and map\n"); 16151 return -EINVAL; 16152 } 16153 16154 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 16155 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 16156 return -EINVAL; 16157 } 16158 16159 if (prog->aux->sleepable) 16160 switch (map->map_type) { 16161 case BPF_MAP_TYPE_HASH: 16162 case BPF_MAP_TYPE_LRU_HASH: 16163 case BPF_MAP_TYPE_ARRAY: 16164 case BPF_MAP_TYPE_PERCPU_HASH: 16165 case BPF_MAP_TYPE_PERCPU_ARRAY: 16166 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 16167 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 16168 case BPF_MAP_TYPE_HASH_OF_MAPS: 16169 case BPF_MAP_TYPE_RINGBUF: 16170 case BPF_MAP_TYPE_USER_RINGBUF: 16171 case BPF_MAP_TYPE_INODE_STORAGE: 16172 case BPF_MAP_TYPE_SK_STORAGE: 16173 case BPF_MAP_TYPE_TASK_STORAGE: 16174 case BPF_MAP_TYPE_CGRP_STORAGE: 16175 break; 16176 default: 16177 verbose(env, 16178 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 16179 return -EINVAL; 16180 } 16181 16182 return 0; 16183 } 16184 16185 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 16186 { 16187 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 16188 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 16189 } 16190 16191 /* find and rewrite pseudo imm in ld_imm64 instructions: 16192 * 16193 * 1. if it accesses map FD, replace it with actual map pointer. 16194 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 16195 * 16196 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 16197 */ 16198 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 16199 { 16200 struct bpf_insn *insn = env->prog->insnsi; 16201 int insn_cnt = env->prog->len; 16202 int i, j, err; 16203 16204 err = bpf_prog_calc_tag(env->prog); 16205 if (err) 16206 return err; 16207 16208 for (i = 0; i < insn_cnt; i++, insn++) { 16209 if (BPF_CLASS(insn->code) == BPF_LDX && 16210 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 16211 verbose(env, "BPF_LDX uses reserved fields\n"); 16212 return -EINVAL; 16213 } 16214 16215 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 16216 struct bpf_insn_aux_data *aux; 16217 struct bpf_map *map; 16218 struct fd f; 16219 u64 addr; 16220 u32 fd; 16221 16222 if (i == insn_cnt - 1 || insn[1].code != 0 || 16223 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 16224 insn[1].off != 0) { 16225 verbose(env, "invalid bpf_ld_imm64 insn\n"); 16226 return -EINVAL; 16227 } 16228 16229 if (insn[0].src_reg == 0) 16230 /* valid generic load 64-bit imm */ 16231 goto next_insn; 16232 16233 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 16234 aux = &env->insn_aux_data[i]; 16235 err = check_pseudo_btf_id(env, insn, aux); 16236 if (err) 16237 return err; 16238 goto next_insn; 16239 } 16240 16241 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 16242 aux = &env->insn_aux_data[i]; 16243 aux->ptr_type = PTR_TO_FUNC; 16244 goto next_insn; 16245 } 16246 16247 /* In final convert_pseudo_ld_imm64() step, this is 16248 * converted into regular 64-bit imm load insn. 16249 */ 16250 switch (insn[0].src_reg) { 16251 case BPF_PSEUDO_MAP_VALUE: 16252 case BPF_PSEUDO_MAP_IDX_VALUE: 16253 break; 16254 case BPF_PSEUDO_MAP_FD: 16255 case BPF_PSEUDO_MAP_IDX: 16256 if (insn[1].imm == 0) 16257 break; 16258 fallthrough; 16259 default: 16260 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 16261 return -EINVAL; 16262 } 16263 16264 switch (insn[0].src_reg) { 16265 case BPF_PSEUDO_MAP_IDX_VALUE: 16266 case BPF_PSEUDO_MAP_IDX: 16267 if (bpfptr_is_null(env->fd_array)) { 16268 verbose(env, "fd_idx without fd_array is invalid\n"); 16269 return -EPROTO; 16270 } 16271 if (copy_from_bpfptr_offset(&fd, env->fd_array, 16272 insn[0].imm * sizeof(fd), 16273 sizeof(fd))) 16274 return -EFAULT; 16275 break; 16276 default: 16277 fd = insn[0].imm; 16278 break; 16279 } 16280 16281 f = fdget(fd); 16282 map = __bpf_map_get(f); 16283 if (IS_ERR(map)) { 16284 verbose(env, "fd %d is not pointing to valid bpf_map\n", 16285 insn[0].imm); 16286 return PTR_ERR(map); 16287 } 16288 16289 err = check_map_prog_compatibility(env, map, env->prog); 16290 if (err) { 16291 fdput(f); 16292 return err; 16293 } 16294 16295 aux = &env->insn_aux_data[i]; 16296 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 16297 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 16298 addr = (unsigned long)map; 16299 } else { 16300 u32 off = insn[1].imm; 16301 16302 if (off >= BPF_MAX_VAR_OFF) { 16303 verbose(env, "direct value offset of %u is not allowed\n", off); 16304 fdput(f); 16305 return -EINVAL; 16306 } 16307 16308 if (!map->ops->map_direct_value_addr) { 16309 verbose(env, "no direct value access support for this map type\n"); 16310 fdput(f); 16311 return -EINVAL; 16312 } 16313 16314 err = map->ops->map_direct_value_addr(map, &addr, off); 16315 if (err) { 16316 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 16317 map->value_size, off); 16318 fdput(f); 16319 return err; 16320 } 16321 16322 aux->map_off = off; 16323 addr += off; 16324 } 16325 16326 insn[0].imm = (u32)addr; 16327 insn[1].imm = addr >> 32; 16328 16329 /* check whether we recorded this map already */ 16330 for (j = 0; j < env->used_map_cnt; j++) { 16331 if (env->used_maps[j] == map) { 16332 aux->map_index = j; 16333 fdput(f); 16334 goto next_insn; 16335 } 16336 } 16337 16338 if (env->used_map_cnt >= MAX_USED_MAPS) { 16339 fdput(f); 16340 return -E2BIG; 16341 } 16342 16343 /* hold the map. If the program is rejected by verifier, 16344 * the map will be released by release_maps() or it 16345 * will be used by the valid program until it's unloaded 16346 * and all maps are released in free_used_maps() 16347 */ 16348 bpf_map_inc(map); 16349 16350 aux->map_index = env->used_map_cnt; 16351 env->used_maps[env->used_map_cnt++] = map; 16352 16353 if (bpf_map_is_cgroup_storage(map) && 16354 bpf_cgroup_storage_assign(env->prog->aux, map)) { 16355 verbose(env, "only one cgroup storage of each type is allowed\n"); 16356 fdput(f); 16357 return -EBUSY; 16358 } 16359 16360 fdput(f); 16361 next_insn: 16362 insn++; 16363 i++; 16364 continue; 16365 } 16366 16367 /* Basic sanity check before we invest more work here. */ 16368 if (!bpf_opcode_in_insntable(insn->code)) { 16369 verbose(env, "unknown opcode %02x\n", insn->code); 16370 return -EINVAL; 16371 } 16372 } 16373 16374 /* now all pseudo BPF_LD_IMM64 instructions load valid 16375 * 'struct bpf_map *' into a register instead of user map_fd. 16376 * These pointers will be used later by verifier to validate map access. 16377 */ 16378 return 0; 16379 } 16380 16381 /* drop refcnt of maps used by the rejected program */ 16382 static void release_maps(struct bpf_verifier_env *env) 16383 { 16384 __bpf_free_used_maps(env->prog->aux, env->used_maps, 16385 env->used_map_cnt); 16386 } 16387 16388 /* drop refcnt of maps used by the rejected program */ 16389 static void release_btfs(struct bpf_verifier_env *env) 16390 { 16391 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 16392 env->used_btf_cnt); 16393 } 16394 16395 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 16396 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 16397 { 16398 struct bpf_insn *insn = env->prog->insnsi; 16399 int insn_cnt = env->prog->len; 16400 int i; 16401 16402 for (i = 0; i < insn_cnt; i++, insn++) { 16403 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 16404 continue; 16405 if (insn->src_reg == BPF_PSEUDO_FUNC) 16406 continue; 16407 insn->src_reg = 0; 16408 } 16409 } 16410 16411 /* single env->prog->insni[off] instruction was replaced with the range 16412 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 16413 * [0, off) and [off, end) to new locations, so the patched range stays zero 16414 */ 16415 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 16416 struct bpf_insn_aux_data *new_data, 16417 struct bpf_prog *new_prog, u32 off, u32 cnt) 16418 { 16419 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 16420 struct bpf_insn *insn = new_prog->insnsi; 16421 u32 old_seen = old_data[off].seen; 16422 u32 prog_len; 16423 int i; 16424 16425 /* aux info at OFF always needs adjustment, no matter fast path 16426 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 16427 * original insn at old prog. 16428 */ 16429 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 16430 16431 if (cnt == 1) 16432 return; 16433 prog_len = new_prog->len; 16434 16435 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 16436 memcpy(new_data + off + cnt - 1, old_data + off, 16437 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 16438 for (i = off; i < off + cnt - 1; i++) { 16439 /* Expand insni[off]'s seen count to the patched range. */ 16440 new_data[i].seen = old_seen; 16441 new_data[i].zext_dst = insn_has_def32(env, insn + i); 16442 } 16443 env->insn_aux_data = new_data; 16444 vfree(old_data); 16445 } 16446 16447 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 16448 { 16449 int i; 16450 16451 if (len == 1) 16452 return; 16453 /* NOTE: fake 'exit' subprog should be updated as well. */ 16454 for (i = 0; i <= env->subprog_cnt; i++) { 16455 if (env->subprog_info[i].start <= off) 16456 continue; 16457 env->subprog_info[i].start += len - 1; 16458 } 16459 } 16460 16461 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 16462 { 16463 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 16464 int i, sz = prog->aux->size_poke_tab; 16465 struct bpf_jit_poke_descriptor *desc; 16466 16467 for (i = 0; i < sz; i++) { 16468 desc = &tab[i]; 16469 if (desc->insn_idx <= off) 16470 continue; 16471 desc->insn_idx += len - 1; 16472 } 16473 } 16474 16475 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 16476 const struct bpf_insn *patch, u32 len) 16477 { 16478 struct bpf_prog *new_prog; 16479 struct bpf_insn_aux_data *new_data = NULL; 16480 16481 if (len > 1) { 16482 new_data = vzalloc(array_size(env->prog->len + len - 1, 16483 sizeof(struct bpf_insn_aux_data))); 16484 if (!new_data) 16485 return NULL; 16486 } 16487 16488 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 16489 if (IS_ERR(new_prog)) { 16490 if (PTR_ERR(new_prog) == -ERANGE) 16491 verbose(env, 16492 "insn %d cannot be patched due to 16-bit range\n", 16493 env->insn_aux_data[off].orig_idx); 16494 vfree(new_data); 16495 return NULL; 16496 } 16497 adjust_insn_aux_data(env, new_data, new_prog, off, len); 16498 adjust_subprog_starts(env, off, len); 16499 adjust_poke_descs(new_prog, off, len); 16500 return new_prog; 16501 } 16502 16503 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 16504 u32 off, u32 cnt) 16505 { 16506 int i, j; 16507 16508 /* find first prog starting at or after off (first to remove) */ 16509 for (i = 0; i < env->subprog_cnt; i++) 16510 if (env->subprog_info[i].start >= off) 16511 break; 16512 /* find first prog starting at or after off + cnt (first to stay) */ 16513 for (j = i; j < env->subprog_cnt; j++) 16514 if (env->subprog_info[j].start >= off + cnt) 16515 break; 16516 /* if j doesn't start exactly at off + cnt, we are just removing 16517 * the front of previous prog 16518 */ 16519 if (env->subprog_info[j].start != off + cnt) 16520 j--; 16521 16522 if (j > i) { 16523 struct bpf_prog_aux *aux = env->prog->aux; 16524 int move; 16525 16526 /* move fake 'exit' subprog as well */ 16527 move = env->subprog_cnt + 1 - j; 16528 16529 memmove(env->subprog_info + i, 16530 env->subprog_info + j, 16531 sizeof(*env->subprog_info) * move); 16532 env->subprog_cnt -= j - i; 16533 16534 /* remove func_info */ 16535 if (aux->func_info) { 16536 move = aux->func_info_cnt - j; 16537 16538 memmove(aux->func_info + i, 16539 aux->func_info + j, 16540 sizeof(*aux->func_info) * move); 16541 aux->func_info_cnt -= j - i; 16542 /* func_info->insn_off is set after all code rewrites, 16543 * in adjust_btf_func() - no need to adjust 16544 */ 16545 } 16546 } else { 16547 /* convert i from "first prog to remove" to "first to adjust" */ 16548 if (env->subprog_info[i].start == off) 16549 i++; 16550 } 16551 16552 /* update fake 'exit' subprog as well */ 16553 for (; i <= env->subprog_cnt; i++) 16554 env->subprog_info[i].start -= cnt; 16555 16556 return 0; 16557 } 16558 16559 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 16560 u32 cnt) 16561 { 16562 struct bpf_prog *prog = env->prog; 16563 u32 i, l_off, l_cnt, nr_linfo; 16564 struct bpf_line_info *linfo; 16565 16566 nr_linfo = prog->aux->nr_linfo; 16567 if (!nr_linfo) 16568 return 0; 16569 16570 linfo = prog->aux->linfo; 16571 16572 /* find first line info to remove, count lines to be removed */ 16573 for (i = 0; i < nr_linfo; i++) 16574 if (linfo[i].insn_off >= off) 16575 break; 16576 16577 l_off = i; 16578 l_cnt = 0; 16579 for (; i < nr_linfo; i++) 16580 if (linfo[i].insn_off < off + cnt) 16581 l_cnt++; 16582 else 16583 break; 16584 16585 /* First live insn doesn't match first live linfo, it needs to "inherit" 16586 * last removed linfo. prog is already modified, so prog->len == off 16587 * means no live instructions after (tail of the program was removed). 16588 */ 16589 if (prog->len != off && l_cnt && 16590 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 16591 l_cnt--; 16592 linfo[--i].insn_off = off + cnt; 16593 } 16594 16595 /* remove the line info which refer to the removed instructions */ 16596 if (l_cnt) { 16597 memmove(linfo + l_off, linfo + i, 16598 sizeof(*linfo) * (nr_linfo - i)); 16599 16600 prog->aux->nr_linfo -= l_cnt; 16601 nr_linfo = prog->aux->nr_linfo; 16602 } 16603 16604 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 16605 for (i = l_off; i < nr_linfo; i++) 16606 linfo[i].insn_off -= cnt; 16607 16608 /* fix up all subprogs (incl. 'exit') which start >= off */ 16609 for (i = 0; i <= env->subprog_cnt; i++) 16610 if (env->subprog_info[i].linfo_idx > l_off) { 16611 /* program may have started in the removed region but 16612 * may not be fully removed 16613 */ 16614 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 16615 env->subprog_info[i].linfo_idx -= l_cnt; 16616 else 16617 env->subprog_info[i].linfo_idx = l_off; 16618 } 16619 16620 return 0; 16621 } 16622 16623 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 16624 { 16625 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 16626 unsigned int orig_prog_len = env->prog->len; 16627 int err; 16628 16629 if (bpf_prog_is_offloaded(env->prog->aux)) 16630 bpf_prog_offload_remove_insns(env, off, cnt); 16631 16632 err = bpf_remove_insns(env->prog, off, cnt); 16633 if (err) 16634 return err; 16635 16636 err = adjust_subprog_starts_after_remove(env, off, cnt); 16637 if (err) 16638 return err; 16639 16640 err = bpf_adj_linfo_after_remove(env, off, cnt); 16641 if (err) 16642 return err; 16643 16644 memmove(aux_data + off, aux_data + off + cnt, 16645 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 16646 16647 return 0; 16648 } 16649 16650 /* The verifier does more data flow analysis than llvm and will not 16651 * explore branches that are dead at run time. Malicious programs can 16652 * have dead code too. Therefore replace all dead at-run-time code 16653 * with 'ja -1'. 16654 * 16655 * Just nops are not optimal, e.g. if they would sit at the end of the 16656 * program and through another bug we would manage to jump there, then 16657 * we'd execute beyond program memory otherwise. Returning exception 16658 * code also wouldn't work since we can have subprogs where the dead 16659 * code could be located. 16660 */ 16661 static void sanitize_dead_code(struct bpf_verifier_env *env) 16662 { 16663 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 16664 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 16665 struct bpf_insn *insn = env->prog->insnsi; 16666 const int insn_cnt = env->prog->len; 16667 int i; 16668 16669 for (i = 0; i < insn_cnt; i++) { 16670 if (aux_data[i].seen) 16671 continue; 16672 memcpy(insn + i, &trap, sizeof(trap)); 16673 aux_data[i].zext_dst = false; 16674 } 16675 } 16676 16677 static bool insn_is_cond_jump(u8 code) 16678 { 16679 u8 op; 16680 16681 if (BPF_CLASS(code) == BPF_JMP32) 16682 return true; 16683 16684 if (BPF_CLASS(code) != BPF_JMP) 16685 return false; 16686 16687 op = BPF_OP(code); 16688 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 16689 } 16690 16691 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 16692 { 16693 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 16694 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 16695 struct bpf_insn *insn = env->prog->insnsi; 16696 const int insn_cnt = env->prog->len; 16697 int i; 16698 16699 for (i = 0; i < insn_cnt; i++, insn++) { 16700 if (!insn_is_cond_jump(insn->code)) 16701 continue; 16702 16703 if (!aux_data[i + 1].seen) 16704 ja.off = insn->off; 16705 else if (!aux_data[i + 1 + insn->off].seen) 16706 ja.off = 0; 16707 else 16708 continue; 16709 16710 if (bpf_prog_is_offloaded(env->prog->aux)) 16711 bpf_prog_offload_replace_insn(env, i, &ja); 16712 16713 memcpy(insn, &ja, sizeof(ja)); 16714 } 16715 } 16716 16717 static int opt_remove_dead_code(struct bpf_verifier_env *env) 16718 { 16719 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 16720 int insn_cnt = env->prog->len; 16721 int i, err; 16722 16723 for (i = 0; i < insn_cnt; i++) { 16724 int j; 16725 16726 j = 0; 16727 while (i + j < insn_cnt && !aux_data[i + j].seen) 16728 j++; 16729 if (!j) 16730 continue; 16731 16732 err = verifier_remove_insns(env, i, j); 16733 if (err) 16734 return err; 16735 insn_cnt = env->prog->len; 16736 } 16737 16738 return 0; 16739 } 16740 16741 static int opt_remove_nops(struct bpf_verifier_env *env) 16742 { 16743 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 16744 struct bpf_insn *insn = env->prog->insnsi; 16745 int insn_cnt = env->prog->len; 16746 int i, err; 16747 16748 for (i = 0; i < insn_cnt; i++) { 16749 if (memcmp(&insn[i], &ja, sizeof(ja))) 16750 continue; 16751 16752 err = verifier_remove_insns(env, i, 1); 16753 if (err) 16754 return err; 16755 insn_cnt--; 16756 i--; 16757 } 16758 16759 return 0; 16760 } 16761 16762 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 16763 const union bpf_attr *attr) 16764 { 16765 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 16766 struct bpf_insn_aux_data *aux = env->insn_aux_data; 16767 int i, patch_len, delta = 0, len = env->prog->len; 16768 struct bpf_insn *insns = env->prog->insnsi; 16769 struct bpf_prog *new_prog; 16770 bool rnd_hi32; 16771 16772 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 16773 zext_patch[1] = BPF_ZEXT_REG(0); 16774 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 16775 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 16776 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 16777 for (i = 0; i < len; i++) { 16778 int adj_idx = i + delta; 16779 struct bpf_insn insn; 16780 int load_reg; 16781 16782 insn = insns[adj_idx]; 16783 load_reg = insn_def_regno(&insn); 16784 if (!aux[adj_idx].zext_dst) { 16785 u8 code, class; 16786 u32 imm_rnd; 16787 16788 if (!rnd_hi32) 16789 continue; 16790 16791 code = insn.code; 16792 class = BPF_CLASS(code); 16793 if (load_reg == -1) 16794 continue; 16795 16796 /* NOTE: arg "reg" (the fourth one) is only used for 16797 * BPF_STX + SRC_OP, so it is safe to pass NULL 16798 * here. 16799 */ 16800 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 16801 if (class == BPF_LD && 16802 BPF_MODE(code) == BPF_IMM) 16803 i++; 16804 continue; 16805 } 16806 16807 /* ctx load could be transformed into wider load. */ 16808 if (class == BPF_LDX && 16809 aux[adj_idx].ptr_type == PTR_TO_CTX) 16810 continue; 16811 16812 imm_rnd = get_random_u32(); 16813 rnd_hi32_patch[0] = insn; 16814 rnd_hi32_patch[1].imm = imm_rnd; 16815 rnd_hi32_patch[3].dst_reg = load_reg; 16816 patch = rnd_hi32_patch; 16817 patch_len = 4; 16818 goto apply_patch_buffer; 16819 } 16820 16821 /* Add in an zero-extend instruction if a) the JIT has requested 16822 * it or b) it's a CMPXCHG. 16823 * 16824 * The latter is because: BPF_CMPXCHG always loads a value into 16825 * R0, therefore always zero-extends. However some archs' 16826 * equivalent instruction only does this load when the 16827 * comparison is successful. This detail of CMPXCHG is 16828 * orthogonal to the general zero-extension behaviour of the 16829 * CPU, so it's treated independently of bpf_jit_needs_zext. 16830 */ 16831 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 16832 continue; 16833 16834 /* Zero-extension is done by the caller. */ 16835 if (bpf_pseudo_kfunc_call(&insn)) 16836 continue; 16837 16838 if (WARN_ON(load_reg == -1)) { 16839 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 16840 return -EFAULT; 16841 } 16842 16843 zext_patch[0] = insn; 16844 zext_patch[1].dst_reg = load_reg; 16845 zext_patch[1].src_reg = load_reg; 16846 patch = zext_patch; 16847 patch_len = 2; 16848 apply_patch_buffer: 16849 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 16850 if (!new_prog) 16851 return -ENOMEM; 16852 env->prog = new_prog; 16853 insns = new_prog->insnsi; 16854 aux = env->insn_aux_data; 16855 delta += patch_len - 1; 16856 } 16857 16858 return 0; 16859 } 16860 16861 /* convert load instructions that access fields of a context type into a 16862 * sequence of instructions that access fields of the underlying structure: 16863 * struct __sk_buff -> struct sk_buff 16864 * struct bpf_sock_ops -> struct sock 16865 */ 16866 static int convert_ctx_accesses(struct bpf_verifier_env *env) 16867 { 16868 const struct bpf_verifier_ops *ops = env->ops; 16869 int i, cnt, size, ctx_field_size, delta = 0; 16870 const int insn_cnt = env->prog->len; 16871 struct bpf_insn insn_buf[16], *insn; 16872 u32 target_size, size_default, off; 16873 struct bpf_prog *new_prog; 16874 enum bpf_access_type type; 16875 bool is_narrower_load; 16876 16877 if (ops->gen_prologue || env->seen_direct_write) { 16878 if (!ops->gen_prologue) { 16879 verbose(env, "bpf verifier is misconfigured\n"); 16880 return -EINVAL; 16881 } 16882 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 16883 env->prog); 16884 if (cnt >= ARRAY_SIZE(insn_buf)) { 16885 verbose(env, "bpf verifier is misconfigured\n"); 16886 return -EINVAL; 16887 } else if (cnt) { 16888 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 16889 if (!new_prog) 16890 return -ENOMEM; 16891 16892 env->prog = new_prog; 16893 delta += cnt - 1; 16894 } 16895 } 16896 16897 if (bpf_prog_is_offloaded(env->prog->aux)) 16898 return 0; 16899 16900 insn = env->prog->insnsi + delta; 16901 16902 for (i = 0; i < insn_cnt; i++, insn++) { 16903 bpf_convert_ctx_access_t convert_ctx_access; 16904 16905 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 16906 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 16907 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 16908 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 16909 type = BPF_READ; 16910 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 16911 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 16912 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 16913 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 16914 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 16915 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 16916 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 16917 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 16918 type = BPF_WRITE; 16919 } else { 16920 continue; 16921 } 16922 16923 if (type == BPF_WRITE && 16924 env->insn_aux_data[i + delta].sanitize_stack_spill) { 16925 struct bpf_insn patch[] = { 16926 *insn, 16927 BPF_ST_NOSPEC(), 16928 }; 16929 16930 cnt = ARRAY_SIZE(patch); 16931 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 16932 if (!new_prog) 16933 return -ENOMEM; 16934 16935 delta += cnt - 1; 16936 env->prog = new_prog; 16937 insn = new_prog->insnsi + i + delta; 16938 continue; 16939 } 16940 16941 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 16942 case PTR_TO_CTX: 16943 if (!ops->convert_ctx_access) 16944 continue; 16945 convert_ctx_access = ops->convert_ctx_access; 16946 break; 16947 case PTR_TO_SOCKET: 16948 case PTR_TO_SOCK_COMMON: 16949 convert_ctx_access = bpf_sock_convert_ctx_access; 16950 break; 16951 case PTR_TO_TCP_SOCK: 16952 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 16953 break; 16954 case PTR_TO_XDP_SOCK: 16955 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 16956 break; 16957 case PTR_TO_BTF_ID: 16958 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 16959 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 16960 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot 16961 * be said once it is marked PTR_UNTRUSTED, hence we must handle 16962 * any faults for loads into such types. BPF_WRITE is disallowed 16963 * for this case. 16964 */ 16965 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 16966 if (type == BPF_READ) { 16967 insn->code = BPF_LDX | BPF_PROBE_MEM | 16968 BPF_SIZE((insn)->code); 16969 env->prog->aux->num_exentries++; 16970 } 16971 continue; 16972 default: 16973 continue; 16974 } 16975 16976 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 16977 size = BPF_LDST_BYTES(insn); 16978 16979 /* If the read access is a narrower load of the field, 16980 * convert to a 4/8-byte load, to minimum program type specific 16981 * convert_ctx_access changes. If conversion is successful, 16982 * we will apply proper mask to the result. 16983 */ 16984 is_narrower_load = size < ctx_field_size; 16985 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 16986 off = insn->off; 16987 if (is_narrower_load) { 16988 u8 size_code; 16989 16990 if (type == BPF_WRITE) { 16991 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 16992 return -EINVAL; 16993 } 16994 16995 size_code = BPF_H; 16996 if (ctx_field_size == 4) 16997 size_code = BPF_W; 16998 else if (ctx_field_size == 8) 16999 size_code = BPF_DW; 17000 17001 insn->off = off & ~(size_default - 1); 17002 insn->code = BPF_LDX | BPF_MEM | size_code; 17003 } 17004 17005 target_size = 0; 17006 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 17007 &target_size); 17008 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 17009 (ctx_field_size && !target_size)) { 17010 verbose(env, "bpf verifier is misconfigured\n"); 17011 return -EINVAL; 17012 } 17013 17014 if (is_narrower_load && size < target_size) { 17015 u8 shift = bpf_ctx_narrow_access_offset( 17016 off, size, size_default) * 8; 17017 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 17018 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 17019 return -EINVAL; 17020 } 17021 if (ctx_field_size <= 4) { 17022 if (shift) 17023 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 17024 insn->dst_reg, 17025 shift); 17026 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 17027 (1 << size * 8) - 1); 17028 } else { 17029 if (shift) 17030 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 17031 insn->dst_reg, 17032 shift); 17033 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 17034 (1ULL << size * 8) - 1); 17035 } 17036 } 17037 17038 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17039 if (!new_prog) 17040 return -ENOMEM; 17041 17042 delta += cnt - 1; 17043 17044 /* keep walking new program and skip insns we just inserted */ 17045 env->prog = new_prog; 17046 insn = new_prog->insnsi + i + delta; 17047 } 17048 17049 return 0; 17050 } 17051 17052 static int jit_subprogs(struct bpf_verifier_env *env) 17053 { 17054 struct bpf_prog *prog = env->prog, **func, *tmp; 17055 int i, j, subprog_start, subprog_end = 0, len, subprog; 17056 struct bpf_map *map_ptr; 17057 struct bpf_insn *insn; 17058 void *old_bpf_func; 17059 int err, num_exentries; 17060 17061 if (env->subprog_cnt <= 1) 17062 return 0; 17063 17064 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 17065 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 17066 continue; 17067 17068 /* Upon error here we cannot fall back to interpreter but 17069 * need a hard reject of the program. Thus -EFAULT is 17070 * propagated in any case. 17071 */ 17072 subprog = find_subprog(env, i + insn->imm + 1); 17073 if (subprog < 0) { 17074 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 17075 i + insn->imm + 1); 17076 return -EFAULT; 17077 } 17078 /* temporarily remember subprog id inside insn instead of 17079 * aux_data, since next loop will split up all insns into funcs 17080 */ 17081 insn->off = subprog; 17082 /* remember original imm in case JIT fails and fallback 17083 * to interpreter will be needed 17084 */ 17085 env->insn_aux_data[i].call_imm = insn->imm; 17086 /* point imm to __bpf_call_base+1 from JITs point of view */ 17087 insn->imm = 1; 17088 if (bpf_pseudo_func(insn)) 17089 /* jit (e.g. x86_64) may emit fewer instructions 17090 * if it learns a u32 imm is the same as a u64 imm. 17091 * Force a non zero here. 17092 */ 17093 insn[1].imm = 1; 17094 } 17095 17096 err = bpf_prog_alloc_jited_linfo(prog); 17097 if (err) 17098 goto out_undo_insn; 17099 17100 err = -ENOMEM; 17101 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 17102 if (!func) 17103 goto out_undo_insn; 17104 17105 for (i = 0; i < env->subprog_cnt; i++) { 17106 subprog_start = subprog_end; 17107 subprog_end = env->subprog_info[i + 1].start; 17108 17109 len = subprog_end - subprog_start; 17110 /* bpf_prog_run() doesn't call subprogs directly, 17111 * hence main prog stats include the runtime of subprogs. 17112 * subprogs don't have IDs and not reachable via prog_get_next_id 17113 * func[i]->stats will never be accessed and stays NULL 17114 */ 17115 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 17116 if (!func[i]) 17117 goto out_free; 17118 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 17119 len * sizeof(struct bpf_insn)); 17120 func[i]->type = prog->type; 17121 func[i]->len = len; 17122 if (bpf_prog_calc_tag(func[i])) 17123 goto out_free; 17124 func[i]->is_func = 1; 17125 func[i]->aux->func_idx = i; 17126 /* Below members will be freed only at prog->aux */ 17127 func[i]->aux->btf = prog->aux->btf; 17128 func[i]->aux->func_info = prog->aux->func_info; 17129 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 17130 func[i]->aux->poke_tab = prog->aux->poke_tab; 17131 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 17132 17133 for (j = 0; j < prog->aux->size_poke_tab; j++) { 17134 struct bpf_jit_poke_descriptor *poke; 17135 17136 poke = &prog->aux->poke_tab[j]; 17137 if (poke->insn_idx < subprog_end && 17138 poke->insn_idx >= subprog_start) 17139 poke->aux = func[i]->aux; 17140 } 17141 17142 func[i]->aux->name[0] = 'F'; 17143 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 17144 func[i]->jit_requested = 1; 17145 func[i]->blinding_requested = prog->blinding_requested; 17146 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 17147 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 17148 func[i]->aux->linfo = prog->aux->linfo; 17149 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 17150 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 17151 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 17152 num_exentries = 0; 17153 insn = func[i]->insnsi; 17154 for (j = 0; j < func[i]->len; j++, insn++) { 17155 if (BPF_CLASS(insn->code) == BPF_LDX && 17156 BPF_MODE(insn->code) == BPF_PROBE_MEM) 17157 num_exentries++; 17158 } 17159 func[i]->aux->num_exentries = num_exentries; 17160 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 17161 func[i] = bpf_int_jit_compile(func[i]); 17162 if (!func[i]->jited) { 17163 err = -ENOTSUPP; 17164 goto out_free; 17165 } 17166 cond_resched(); 17167 } 17168 17169 /* at this point all bpf functions were successfully JITed 17170 * now populate all bpf_calls with correct addresses and 17171 * run last pass of JIT 17172 */ 17173 for (i = 0; i < env->subprog_cnt; i++) { 17174 insn = func[i]->insnsi; 17175 for (j = 0; j < func[i]->len; j++, insn++) { 17176 if (bpf_pseudo_func(insn)) { 17177 subprog = insn->off; 17178 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 17179 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 17180 continue; 17181 } 17182 if (!bpf_pseudo_call(insn)) 17183 continue; 17184 subprog = insn->off; 17185 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 17186 } 17187 17188 /* we use the aux data to keep a list of the start addresses 17189 * of the JITed images for each function in the program 17190 * 17191 * for some architectures, such as powerpc64, the imm field 17192 * might not be large enough to hold the offset of the start 17193 * address of the callee's JITed image from __bpf_call_base 17194 * 17195 * in such cases, we can lookup the start address of a callee 17196 * by using its subprog id, available from the off field of 17197 * the call instruction, as an index for this list 17198 */ 17199 func[i]->aux->func = func; 17200 func[i]->aux->func_cnt = env->subprog_cnt; 17201 } 17202 for (i = 0; i < env->subprog_cnt; i++) { 17203 old_bpf_func = func[i]->bpf_func; 17204 tmp = bpf_int_jit_compile(func[i]); 17205 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 17206 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 17207 err = -ENOTSUPP; 17208 goto out_free; 17209 } 17210 cond_resched(); 17211 } 17212 17213 /* finally lock prog and jit images for all functions and 17214 * populate kallsysm 17215 */ 17216 for (i = 0; i < env->subprog_cnt; i++) { 17217 bpf_prog_lock_ro(func[i]); 17218 bpf_prog_kallsyms_add(func[i]); 17219 } 17220 17221 /* Last step: make now unused interpreter insns from main 17222 * prog consistent for later dump requests, so they can 17223 * later look the same as if they were interpreted only. 17224 */ 17225 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 17226 if (bpf_pseudo_func(insn)) { 17227 insn[0].imm = env->insn_aux_data[i].call_imm; 17228 insn[1].imm = insn->off; 17229 insn->off = 0; 17230 continue; 17231 } 17232 if (!bpf_pseudo_call(insn)) 17233 continue; 17234 insn->off = env->insn_aux_data[i].call_imm; 17235 subprog = find_subprog(env, i + insn->off + 1); 17236 insn->imm = subprog; 17237 } 17238 17239 prog->jited = 1; 17240 prog->bpf_func = func[0]->bpf_func; 17241 prog->jited_len = func[0]->jited_len; 17242 prog->aux->func = func; 17243 prog->aux->func_cnt = env->subprog_cnt; 17244 bpf_prog_jit_attempt_done(prog); 17245 return 0; 17246 out_free: 17247 /* We failed JIT'ing, so at this point we need to unregister poke 17248 * descriptors from subprogs, so that kernel is not attempting to 17249 * patch it anymore as we're freeing the subprog JIT memory. 17250 */ 17251 for (i = 0; i < prog->aux->size_poke_tab; i++) { 17252 map_ptr = prog->aux->poke_tab[i].tail_call.map; 17253 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 17254 } 17255 /* At this point we're guaranteed that poke descriptors are not 17256 * live anymore. We can just unlink its descriptor table as it's 17257 * released with the main prog. 17258 */ 17259 for (i = 0; i < env->subprog_cnt; i++) { 17260 if (!func[i]) 17261 continue; 17262 func[i]->aux->poke_tab = NULL; 17263 bpf_jit_free(func[i]); 17264 } 17265 kfree(func); 17266 out_undo_insn: 17267 /* cleanup main prog to be interpreted */ 17268 prog->jit_requested = 0; 17269 prog->blinding_requested = 0; 17270 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 17271 if (!bpf_pseudo_call(insn)) 17272 continue; 17273 insn->off = 0; 17274 insn->imm = env->insn_aux_data[i].call_imm; 17275 } 17276 bpf_prog_jit_attempt_done(prog); 17277 return err; 17278 } 17279 17280 static int fixup_call_args(struct bpf_verifier_env *env) 17281 { 17282 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 17283 struct bpf_prog *prog = env->prog; 17284 struct bpf_insn *insn = prog->insnsi; 17285 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 17286 int i, depth; 17287 #endif 17288 int err = 0; 17289 17290 if (env->prog->jit_requested && 17291 !bpf_prog_is_offloaded(env->prog->aux)) { 17292 err = jit_subprogs(env); 17293 if (err == 0) 17294 return 0; 17295 if (err == -EFAULT) 17296 return err; 17297 } 17298 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 17299 if (has_kfunc_call) { 17300 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 17301 return -EINVAL; 17302 } 17303 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 17304 /* When JIT fails the progs with bpf2bpf calls and tail_calls 17305 * have to be rejected, since interpreter doesn't support them yet. 17306 */ 17307 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 17308 return -EINVAL; 17309 } 17310 for (i = 0; i < prog->len; i++, insn++) { 17311 if (bpf_pseudo_func(insn)) { 17312 /* When JIT fails the progs with callback calls 17313 * have to be rejected, since interpreter doesn't support them yet. 17314 */ 17315 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 17316 return -EINVAL; 17317 } 17318 17319 if (!bpf_pseudo_call(insn)) 17320 continue; 17321 depth = get_callee_stack_depth(env, insn, i); 17322 if (depth < 0) 17323 return depth; 17324 bpf_patch_call_args(insn, depth); 17325 } 17326 err = 0; 17327 #endif 17328 return err; 17329 } 17330 17331 /* replace a generic kfunc with a specialized version if necessary */ 17332 static void specialize_kfunc(struct bpf_verifier_env *env, 17333 u32 func_id, u16 offset, unsigned long *addr) 17334 { 17335 struct bpf_prog *prog = env->prog; 17336 bool seen_direct_write; 17337 void *xdp_kfunc; 17338 bool is_rdonly; 17339 17340 if (bpf_dev_bound_kfunc_id(func_id)) { 17341 xdp_kfunc = bpf_dev_bound_resolve_kfunc(prog, func_id); 17342 if (xdp_kfunc) { 17343 *addr = (unsigned long)xdp_kfunc; 17344 return; 17345 } 17346 /* fallback to default kfunc when not supported by netdev */ 17347 } 17348 17349 if (offset) 17350 return; 17351 17352 if (func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { 17353 seen_direct_write = env->seen_direct_write; 17354 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); 17355 17356 if (is_rdonly) 17357 *addr = (unsigned long)bpf_dynptr_from_skb_rdonly; 17358 17359 /* restore env->seen_direct_write to its original value, since 17360 * may_access_direct_pkt_data mutates it 17361 */ 17362 env->seen_direct_write = seen_direct_write; 17363 } 17364 } 17365 17366 static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux, 17367 u16 struct_meta_reg, 17368 u16 node_offset_reg, 17369 struct bpf_insn *insn, 17370 struct bpf_insn *insn_buf, 17371 int *cnt) 17372 { 17373 struct btf_struct_meta *kptr_struct_meta = insn_aux->kptr_struct_meta; 17374 struct bpf_insn addr[2] = { BPF_LD_IMM64(struct_meta_reg, (long)kptr_struct_meta) }; 17375 17376 insn_buf[0] = addr[0]; 17377 insn_buf[1] = addr[1]; 17378 insn_buf[2] = BPF_MOV64_IMM(node_offset_reg, insn_aux->insert_off); 17379 insn_buf[3] = *insn; 17380 *cnt = 4; 17381 } 17382 17383 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 17384 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 17385 { 17386 const struct bpf_kfunc_desc *desc; 17387 17388 if (!insn->imm) { 17389 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 17390 return -EINVAL; 17391 } 17392 17393 *cnt = 0; 17394 17395 /* insn->imm has the btf func_id. Replace it with an offset relative to 17396 * __bpf_call_base, unless the JIT needs to call functions that are 17397 * further than 32 bits away (bpf_jit_supports_far_kfunc_call()). 17398 */ 17399 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 17400 if (!desc) { 17401 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 17402 insn->imm); 17403 return -EFAULT; 17404 } 17405 17406 if (!bpf_jit_supports_far_kfunc_call()) 17407 insn->imm = BPF_CALL_IMM(desc->addr); 17408 if (insn->off) 17409 return 0; 17410 if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 17411 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 17412 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 17413 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 17414 17415 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 17416 insn_buf[1] = addr[0]; 17417 insn_buf[2] = addr[1]; 17418 insn_buf[3] = *insn; 17419 *cnt = 4; 17420 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl] || 17421 desc->func_id == special_kfunc_list[KF_bpf_refcount_acquire_impl]) { 17422 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 17423 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 17424 17425 insn_buf[0] = addr[0]; 17426 insn_buf[1] = addr[1]; 17427 insn_buf[2] = *insn; 17428 *cnt = 3; 17429 } else if (desc->func_id == special_kfunc_list[KF_bpf_list_push_back_impl] || 17430 desc->func_id == special_kfunc_list[KF_bpf_list_push_front_impl] || 17431 desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 17432 int struct_meta_reg = BPF_REG_3; 17433 int node_offset_reg = BPF_REG_4; 17434 17435 /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */ 17436 if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { 17437 struct_meta_reg = BPF_REG_4; 17438 node_offset_reg = BPF_REG_5; 17439 } 17440 17441 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, 17442 node_offset_reg, insn, insn_buf, cnt); 17443 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 17444 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 17445 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 17446 *cnt = 1; 17447 } 17448 return 0; 17449 } 17450 17451 /* Do various post-verification rewrites in a single program pass. 17452 * These rewrites simplify JIT and interpreter implementations. 17453 */ 17454 static int do_misc_fixups(struct bpf_verifier_env *env) 17455 { 17456 struct bpf_prog *prog = env->prog; 17457 enum bpf_attach_type eatype = prog->expected_attach_type; 17458 enum bpf_prog_type prog_type = resolve_prog_type(prog); 17459 struct bpf_insn *insn = prog->insnsi; 17460 const struct bpf_func_proto *fn; 17461 const int insn_cnt = prog->len; 17462 const struct bpf_map_ops *ops; 17463 struct bpf_insn_aux_data *aux; 17464 struct bpf_insn insn_buf[16]; 17465 struct bpf_prog *new_prog; 17466 struct bpf_map *map_ptr; 17467 int i, ret, cnt, delta = 0; 17468 17469 for (i = 0; i < insn_cnt; i++, insn++) { 17470 /* Make divide-by-zero exceptions impossible. */ 17471 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 17472 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 17473 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 17474 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 17475 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 17476 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 17477 struct bpf_insn *patchlet; 17478 struct bpf_insn chk_and_div[] = { 17479 /* [R,W]x div 0 -> 0 */ 17480 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 17481 BPF_JNE | BPF_K, insn->src_reg, 17482 0, 2, 0), 17483 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 17484 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 17485 *insn, 17486 }; 17487 struct bpf_insn chk_and_mod[] = { 17488 /* [R,W]x mod 0 -> [R,W]x */ 17489 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 17490 BPF_JEQ | BPF_K, insn->src_reg, 17491 0, 1 + (is64 ? 0 : 1), 0), 17492 *insn, 17493 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 17494 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 17495 }; 17496 17497 patchlet = isdiv ? chk_and_div : chk_and_mod; 17498 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 17499 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 17500 17501 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 17502 if (!new_prog) 17503 return -ENOMEM; 17504 17505 delta += cnt - 1; 17506 env->prog = prog = new_prog; 17507 insn = new_prog->insnsi + i + delta; 17508 continue; 17509 } 17510 17511 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 17512 if (BPF_CLASS(insn->code) == BPF_LD && 17513 (BPF_MODE(insn->code) == BPF_ABS || 17514 BPF_MODE(insn->code) == BPF_IND)) { 17515 cnt = env->ops->gen_ld_abs(insn, insn_buf); 17516 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 17517 verbose(env, "bpf verifier is misconfigured\n"); 17518 return -EINVAL; 17519 } 17520 17521 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17522 if (!new_prog) 17523 return -ENOMEM; 17524 17525 delta += cnt - 1; 17526 env->prog = prog = new_prog; 17527 insn = new_prog->insnsi + i + delta; 17528 continue; 17529 } 17530 17531 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 17532 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 17533 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 17534 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 17535 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 17536 struct bpf_insn *patch = &insn_buf[0]; 17537 bool issrc, isneg, isimm; 17538 u32 off_reg; 17539 17540 aux = &env->insn_aux_data[i + delta]; 17541 if (!aux->alu_state || 17542 aux->alu_state == BPF_ALU_NON_POINTER) 17543 continue; 17544 17545 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 17546 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 17547 BPF_ALU_SANITIZE_SRC; 17548 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 17549 17550 off_reg = issrc ? insn->src_reg : insn->dst_reg; 17551 if (isimm) { 17552 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 17553 } else { 17554 if (isneg) 17555 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 17556 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 17557 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 17558 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 17559 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 17560 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 17561 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 17562 } 17563 if (!issrc) 17564 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 17565 insn->src_reg = BPF_REG_AX; 17566 if (isneg) 17567 insn->code = insn->code == code_add ? 17568 code_sub : code_add; 17569 *patch++ = *insn; 17570 if (issrc && isneg && !isimm) 17571 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 17572 cnt = patch - insn_buf; 17573 17574 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17575 if (!new_prog) 17576 return -ENOMEM; 17577 17578 delta += cnt - 1; 17579 env->prog = prog = new_prog; 17580 insn = new_prog->insnsi + i + delta; 17581 continue; 17582 } 17583 17584 if (insn->code != (BPF_JMP | BPF_CALL)) 17585 continue; 17586 if (insn->src_reg == BPF_PSEUDO_CALL) 17587 continue; 17588 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 17589 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 17590 if (ret) 17591 return ret; 17592 if (cnt == 0) 17593 continue; 17594 17595 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17596 if (!new_prog) 17597 return -ENOMEM; 17598 17599 delta += cnt - 1; 17600 env->prog = prog = new_prog; 17601 insn = new_prog->insnsi + i + delta; 17602 continue; 17603 } 17604 17605 if (insn->imm == BPF_FUNC_get_route_realm) 17606 prog->dst_needed = 1; 17607 if (insn->imm == BPF_FUNC_get_prandom_u32) 17608 bpf_user_rnd_init_once(); 17609 if (insn->imm == BPF_FUNC_override_return) 17610 prog->kprobe_override = 1; 17611 if (insn->imm == BPF_FUNC_tail_call) { 17612 /* If we tail call into other programs, we 17613 * cannot make any assumptions since they can 17614 * be replaced dynamically during runtime in 17615 * the program array. 17616 */ 17617 prog->cb_access = 1; 17618 if (!allow_tail_call_in_subprogs(env)) 17619 prog->aux->stack_depth = MAX_BPF_STACK; 17620 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 17621 17622 /* mark bpf_tail_call as different opcode to avoid 17623 * conditional branch in the interpreter for every normal 17624 * call and to prevent accidental JITing by JIT compiler 17625 * that doesn't support bpf_tail_call yet 17626 */ 17627 insn->imm = 0; 17628 insn->code = BPF_JMP | BPF_TAIL_CALL; 17629 17630 aux = &env->insn_aux_data[i + delta]; 17631 if (env->bpf_capable && !prog->blinding_requested && 17632 prog->jit_requested && 17633 !bpf_map_key_poisoned(aux) && 17634 !bpf_map_ptr_poisoned(aux) && 17635 !bpf_map_ptr_unpriv(aux)) { 17636 struct bpf_jit_poke_descriptor desc = { 17637 .reason = BPF_POKE_REASON_TAIL_CALL, 17638 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 17639 .tail_call.key = bpf_map_key_immediate(aux), 17640 .insn_idx = i + delta, 17641 }; 17642 17643 ret = bpf_jit_add_poke_descriptor(prog, &desc); 17644 if (ret < 0) { 17645 verbose(env, "adding tail call poke descriptor failed\n"); 17646 return ret; 17647 } 17648 17649 insn->imm = ret + 1; 17650 continue; 17651 } 17652 17653 if (!bpf_map_ptr_unpriv(aux)) 17654 continue; 17655 17656 /* instead of changing every JIT dealing with tail_call 17657 * emit two extra insns: 17658 * if (index >= max_entries) goto out; 17659 * index &= array->index_mask; 17660 * to avoid out-of-bounds cpu speculation 17661 */ 17662 if (bpf_map_ptr_poisoned(aux)) { 17663 verbose(env, "tail_call abusing map_ptr\n"); 17664 return -EINVAL; 17665 } 17666 17667 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 17668 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 17669 map_ptr->max_entries, 2); 17670 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 17671 container_of(map_ptr, 17672 struct bpf_array, 17673 map)->index_mask); 17674 insn_buf[2] = *insn; 17675 cnt = 3; 17676 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17677 if (!new_prog) 17678 return -ENOMEM; 17679 17680 delta += cnt - 1; 17681 env->prog = prog = new_prog; 17682 insn = new_prog->insnsi + i + delta; 17683 continue; 17684 } 17685 17686 if (insn->imm == BPF_FUNC_timer_set_callback) { 17687 /* The verifier will process callback_fn as many times as necessary 17688 * with different maps and the register states prepared by 17689 * set_timer_callback_state will be accurate. 17690 * 17691 * The following use case is valid: 17692 * map1 is shared by prog1, prog2, prog3. 17693 * prog1 calls bpf_timer_init for some map1 elements 17694 * prog2 calls bpf_timer_set_callback for some map1 elements. 17695 * Those that were not bpf_timer_init-ed will return -EINVAL. 17696 * prog3 calls bpf_timer_start for some map1 elements. 17697 * Those that were not both bpf_timer_init-ed and 17698 * bpf_timer_set_callback-ed will return -EINVAL. 17699 */ 17700 struct bpf_insn ld_addrs[2] = { 17701 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 17702 }; 17703 17704 insn_buf[0] = ld_addrs[0]; 17705 insn_buf[1] = ld_addrs[1]; 17706 insn_buf[2] = *insn; 17707 cnt = 3; 17708 17709 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17710 if (!new_prog) 17711 return -ENOMEM; 17712 17713 delta += cnt - 1; 17714 env->prog = prog = new_prog; 17715 insn = new_prog->insnsi + i + delta; 17716 goto patch_call_imm; 17717 } 17718 17719 if (is_storage_get_function(insn->imm)) { 17720 if (!env->prog->aux->sleepable || 17721 env->insn_aux_data[i + delta].storage_get_func_atomic) 17722 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 17723 else 17724 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 17725 insn_buf[1] = *insn; 17726 cnt = 2; 17727 17728 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17729 if (!new_prog) 17730 return -ENOMEM; 17731 17732 delta += cnt - 1; 17733 env->prog = prog = new_prog; 17734 insn = new_prog->insnsi + i + delta; 17735 goto patch_call_imm; 17736 } 17737 17738 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 17739 * and other inlining handlers are currently limited to 64 bit 17740 * only. 17741 */ 17742 if (prog->jit_requested && BITS_PER_LONG == 64 && 17743 (insn->imm == BPF_FUNC_map_lookup_elem || 17744 insn->imm == BPF_FUNC_map_update_elem || 17745 insn->imm == BPF_FUNC_map_delete_elem || 17746 insn->imm == BPF_FUNC_map_push_elem || 17747 insn->imm == BPF_FUNC_map_pop_elem || 17748 insn->imm == BPF_FUNC_map_peek_elem || 17749 insn->imm == BPF_FUNC_redirect_map || 17750 insn->imm == BPF_FUNC_for_each_map_elem || 17751 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 17752 aux = &env->insn_aux_data[i + delta]; 17753 if (bpf_map_ptr_poisoned(aux)) 17754 goto patch_call_imm; 17755 17756 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 17757 ops = map_ptr->ops; 17758 if (insn->imm == BPF_FUNC_map_lookup_elem && 17759 ops->map_gen_lookup) { 17760 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 17761 if (cnt == -EOPNOTSUPP) 17762 goto patch_map_ops_generic; 17763 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 17764 verbose(env, "bpf verifier is misconfigured\n"); 17765 return -EINVAL; 17766 } 17767 17768 new_prog = bpf_patch_insn_data(env, i + delta, 17769 insn_buf, cnt); 17770 if (!new_prog) 17771 return -ENOMEM; 17772 17773 delta += cnt - 1; 17774 env->prog = prog = new_prog; 17775 insn = new_prog->insnsi + i + delta; 17776 continue; 17777 } 17778 17779 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 17780 (void *(*)(struct bpf_map *map, void *key))NULL)); 17781 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 17782 (long (*)(struct bpf_map *map, void *key))NULL)); 17783 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 17784 (long (*)(struct bpf_map *map, void *key, void *value, 17785 u64 flags))NULL)); 17786 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 17787 (long (*)(struct bpf_map *map, void *value, 17788 u64 flags))NULL)); 17789 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 17790 (long (*)(struct bpf_map *map, void *value))NULL)); 17791 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 17792 (long (*)(struct bpf_map *map, void *value))NULL)); 17793 BUILD_BUG_ON(!__same_type(ops->map_redirect, 17794 (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 17795 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 17796 (long (*)(struct bpf_map *map, 17797 bpf_callback_t callback_fn, 17798 void *callback_ctx, 17799 u64 flags))NULL)); 17800 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 17801 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 17802 17803 patch_map_ops_generic: 17804 switch (insn->imm) { 17805 case BPF_FUNC_map_lookup_elem: 17806 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 17807 continue; 17808 case BPF_FUNC_map_update_elem: 17809 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 17810 continue; 17811 case BPF_FUNC_map_delete_elem: 17812 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 17813 continue; 17814 case BPF_FUNC_map_push_elem: 17815 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 17816 continue; 17817 case BPF_FUNC_map_pop_elem: 17818 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 17819 continue; 17820 case BPF_FUNC_map_peek_elem: 17821 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 17822 continue; 17823 case BPF_FUNC_redirect_map: 17824 insn->imm = BPF_CALL_IMM(ops->map_redirect); 17825 continue; 17826 case BPF_FUNC_for_each_map_elem: 17827 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 17828 continue; 17829 case BPF_FUNC_map_lookup_percpu_elem: 17830 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 17831 continue; 17832 } 17833 17834 goto patch_call_imm; 17835 } 17836 17837 /* Implement bpf_jiffies64 inline. */ 17838 if (prog->jit_requested && BITS_PER_LONG == 64 && 17839 insn->imm == BPF_FUNC_jiffies64) { 17840 struct bpf_insn ld_jiffies_addr[2] = { 17841 BPF_LD_IMM64(BPF_REG_0, 17842 (unsigned long)&jiffies), 17843 }; 17844 17845 insn_buf[0] = ld_jiffies_addr[0]; 17846 insn_buf[1] = ld_jiffies_addr[1]; 17847 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 17848 BPF_REG_0, 0); 17849 cnt = 3; 17850 17851 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 17852 cnt); 17853 if (!new_prog) 17854 return -ENOMEM; 17855 17856 delta += cnt - 1; 17857 env->prog = prog = new_prog; 17858 insn = new_prog->insnsi + i + delta; 17859 continue; 17860 } 17861 17862 /* Implement bpf_get_func_arg inline. */ 17863 if (prog_type == BPF_PROG_TYPE_TRACING && 17864 insn->imm == BPF_FUNC_get_func_arg) { 17865 /* Load nr_args from ctx - 8 */ 17866 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 17867 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 17868 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 17869 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 17870 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 17871 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 17872 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 17873 insn_buf[7] = BPF_JMP_A(1); 17874 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 17875 cnt = 9; 17876 17877 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17878 if (!new_prog) 17879 return -ENOMEM; 17880 17881 delta += cnt - 1; 17882 env->prog = prog = new_prog; 17883 insn = new_prog->insnsi + i + delta; 17884 continue; 17885 } 17886 17887 /* Implement bpf_get_func_ret inline. */ 17888 if (prog_type == BPF_PROG_TYPE_TRACING && 17889 insn->imm == BPF_FUNC_get_func_ret) { 17890 if (eatype == BPF_TRACE_FEXIT || 17891 eatype == BPF_MODIFY_RETURN) { 17892 /* Load nr_args from ctx - 8 */ 17893 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 17894 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 17895 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 17896 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 17897 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 17898 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 17899 cnt = 6; 17900 } else { 17901 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 17902 cnt = 1; 17903 } 17904 17905 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17906 if (!new_prog) 17907 return -ENOMEM; 17908 17909 delta += cnt - 1; 17910 env->prog = prog = new_prog; 17911 insn = new_prog->insnsi + i + delta; 17912 continue; 17913 } 17914 17915 /* Implement get_func_arg_cnt inline. */ 17916 if (prog_type == BPF_PROG_TYPE_TRACING && 17917 insn->imm == BPF_FUNC_get_func_arg_cnt) { 17918 /* Load nr_args from ctx - 8 */ 17919 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 17920 17921 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 17922 if (!new_prog) 17923 return -ENOMEM; 17924 17925 env->prog = prog = new_prog; 17926 insn = new_prog->insnsi + i + delta; 17927 continue; 17928 } 17929 17930 /* Implement bpf_get_func_ip inline. */ 17931 if (prog_type == BPF_PROG_TYPE_TRACING && 17932 insn->imm == BPF_FUNC_get_func_ip) { 17933 /* Load IP address from ctx - 16 */ 17934 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 17935 17936 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 17937 if (!new_prog) 17938 return -ENOMEM; 17939 17940 env->prog = prog = new_prog; 17941 insn = new_prog->insnsi + i + delta; 17942 continue; 17943 } 17944 17945 patch_call_imm: 17946 fn = env->ops->get_func_proto(insn->imm, env->prog); 17947 /* all functions that have prototype and verifier allowed 17948 * programs to call them, must be real in-kernel functions 17949 */ 17950 if (!fn->func) { 17951 verbose(env, 17952 "kernel subsystem misconfigured func %s#%d\n", 17953 func_id_name(insn->imm), insn->imm); 17954 return -EFAULT; 17955 } 17956 insn->imm = fn->func - __bpf_call_base; 17957 } 17958 17959 /* Since poke tab is now finalized, publish aux to tracker. */ 17960 for (i = 0; i < prog->aux->size_poke_tab; i++) { 17961 map_ptr = prog->aux->poke_tab[i].tail_call.map; 17962 if (!map_ptr->ops->map_poke_track || 17963 !map_ptr->ops->map_poke_untrack || 17964 !map_ptr->ops->map_poke_run) { 17965 verbose(env, "bpf verifier is misconfigured\n"); 17966 return -EINVAL; 17967 } 17968 17969 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 17970 if (ret < 0) { 17971 verbose(env, "tracking tail call prog failed\n"); 17972 return ret; 17973 } 17974 } 17975 17976 sort_kfunc_descs_by_imm_off(env->prog); 17977 17978 return 0; 17979 } 17980 17981 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 17982 int position, 17983 s32 stack_base, 17984 u32 callback_subprogno, 17985 u32 *cnt) 17986 { 17987 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 17988 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 17989 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 17990 int reg_loop_max = BPF_REG_6; 17991 int reg_loop_cnt = BPF_REG_7; 17992 int reg_loop_ctx = BPF_REG_8; 17993 17994 struct bpf_prog *new_prog; 17995 u32 callback_start; 17996 u32 call_insn_offset; 17997 s32 callback_offset; 17998 17999 /* This represents an inlined version of bpf_iter.c:bpf_loop, 18000 * be careful to modify this code in sync. 18001 */ 18002 struct bpf_insn insn_buf[] = { 18003 /* Return error and jump to the end of the patch if 18004 * expected number of iterations is too big. 18005 */ 18006 BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), 18007 BPF_MOV32_IMM(BPF_REG_0, -E2BIG), 18008 BPF_JMP_IMM(BPF_JA, 0, 0, 16), 18009 /* spill R6, R7, R8 to use these as loop vars */ 18010 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), 18011 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), 18012 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), 18013 /* initialize loop vars */ 18014 BPF_MOV64_REG(reg_loop_max, BPF_REG_1), 18015 BPF_MOV32_IMM(reg_loop_cnt, 0), 18016 BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), 18017 /* loop header, 18018 * if reg_loop_cnt >= reg_loop_max skip the loop body 18019 */ 18020 BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), 18021 /* callback call, 18022 * correct callback offset would be set after patching 18023 */ 18024 BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), 18025 BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), 18026 BPF_CALL_REL(0), 18027 /* increment loop counter */ 18028 BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), 18029 /* jump to loop header if callback returned 0 */ 18030 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), 18031 /* return value of bpf_loop, 18032 * set R0 to the number of iterations 18033 */ 18034 BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), 18035 /* restore original values of R6, R7, R8 */ 18036 BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), 18037 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), 18038 BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), 18039 }; 18040 18041 *cnt = ARRAY_SIZE(insn_buf); 18042 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); 18043 if (!new_prog) 18044 return new_prog; 18045 18046 /* callback start is known only after patching */ 18047 callback_start = env->subprog_info[callback_subprogno].start; 18048 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 18049 call_insn_offset = position + 12; 18050 callback_offset = callback_start - call_insn_offset - 1; 18051 new_prog->insnsi[call_insn_offset].imm = callback_offset; 18052 18053 return new_prog; 18054 } 18055 18056 static bool is_bpf_loop_call(struct bpf_insn *insn) 18057 { 18058 return insn->code == (BPF_JMP | BPF_CALL) && 18059 insn->src_reg == 0 && 18060 insn->imm == BPF_FUNC_loop; 18061 } 18062 18063 /* For all sub-programs in the program (including main) check 18064 * insn_aux_data to see if there are bpf_loop calls that require 18065 * inlining. If such calls are found the calls are replaced with a 18066 * sequence of instructions produced by `inline_bpf_loop` function and 18067 * subprog stack_depth is increased by the size of 3 registers. 18068 * This stack space is used to spill values of the R6, R7, R8. These 18069 * registers are used to store the loop bound, counter and context 18070 * variables. 18071 */ 18072 static int optimize_bpf_loop(struct bpf_verifier_env *env) 18073 { 18074 struct bpf_subprog_info *subprogs = env->subprog_info; 18075 int i, cur_subprog = 0, cnt, delta = 0; 18076 struct bpf_insn *insn = env->prog->insnsi; 18077 int insn_cnt = env->prog->len; 18078 u16 stack_depth = subprogs[cur_subprog].stack_depth; 18079 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 18080 u16 stack_depth_extra = 0; 18081 18082 for (i = 0; i < insn_cnt; i++, insn++) { 18083 struct bpf_loop_inline_state *inline_state = 18084 &env->insn_aux_data[i + delta].loop_inline_state; 18085 18086 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 18087 struct bpf_prog *new_prog; 18088 18089 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 18090 new_prog = inline_bpf_loop(env, 18091 i + delta, 18092 -(stack_depth + stack_depth_extra), 18093 inline_state->callback_subprogno, 18094 &cnt); 18095 if (!new_prog) 18096 return -ENOMEM; 18097 18098 delta += cnt - 1; 18099 env->prog = new_prog; 18100 insn = new_prog->insnsi + i + delta; 18101 } 18102 18103 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 18104 subprogs[cur_subprog].stack_depth += stack_depth_extra; 18105 cur_subprog++; 18106 stack_depth = subprogs[cur_subprog].stack_depth; 18107 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 18108 stack_depth_extra = 0; 18109 } 18110 } 18111 18112 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 18113 18114 return 0; 18115 } 18116 18117 static void free_states(struct bpf_verifier_env *env) 18118 { 18119 struct bpf_verifier_state_list *sl, *sln; 18120 int i; 18121 18122 sl = env->free_list; 18123 while (sl) { 18124 sln = sl->next; 18125 free_verifier_state(&sl->state, false); 18126 kfree(sl); 18127 sl = sln; 18128 } 18129 env->free_list = NULL; 18130 18131 if (!env->explored_states) 18132 return; 18133 18134 for (i = 0; i < state_htab_size(env); i++) { 18135 sl = env->explored_states[i]; 18136 18137 while (sl) { 18138 sln = sl->next; 18139 free_verifier_state(&sl->state, false); 18140 kfree(sl); 18141 sl = sln; 18142 } 18143 env->explored_states[i] = NULL; 18144 } 18145 } 18146 18147 static int do_check_common(struct bpf_verifier_env *env, int subprog) 18148 { 18149 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 18150 struct bpf_verifier_state *state; 18151 struct bpf_reg_state *regs; 18152 int ret, i; 18153 18154 env->prev_linfo = NULL; 18155 env->pass_cnt++; 18156 18157 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 18158 if (!state) 18159 return -ENOMEM; 18160 state->curframe = 0; 18161 state->speculative = false; 18162 state->branches = 1; 18163 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 18164 if (!state->frame[0]) { 18165 kfree(state); 18166 return -ENOMEM; 18167 } 18168 env->cur_state = state; 18169 init_func_state(env, state->frame[0], 18170 BPF_MAIN_FUNC /* callsite */, 18171 0 /* frameno */, 18172 subprog); 18173 state->first_insn_idx = env->subprog_info[subprog].start; 18174 state->last_insn_idx = -1; 18175 18176 regs = state->frame[state->curframe]->regs; 18177 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 18178 ret = btf_prepare_func_args(env, subprog, regs); 18179 if (ret) 18180 goto out; 18181 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 18182 if (regs[i].type == PTR_TO_CTX) 18183 mark_reg_known_zero(env, regs, i); 18184 else if (regs[i].type == SCALAR_VALUE) 18185 mark_reg_unknown(env, regs, i); 18186 else if (base_type(regs[i].type) == PTR_TO_MEM) { 18187 const u32 mem_size = regs[i].mem_size; 18188 18189 mark_reg_known_zero(env, regs, i); 18190 regs[i].mem_size = mem_size; 18191 regs[i].id = ++env->id_gen; 18192 } 18193 } 18194 } else { 18195 /* 1st arg to a function */ 18196 regs[BPF_REG_1].type = PTR_TO_CTX; 18197 mark_reg_known_zero(env, regs, BPF_REG_1); 18198 ret = btf_check_subprog_arg_match(env, subprog, regs); 18199 if (ret == -EFAULT) 18200 /* unlikely verifier bug. abort. 18201 * ret == 0 and ret < 0 are sadly acceptable for 18202 * main() function due to backward compatibility. 18203 * Like socket filter program may be written as: 18204 * int bpf_prog(struct pt_regs *ctx) 18205 * and never dereference that ctx in the program. 18206 * 'struct pt_regs' is a type mismatch for socket 18207 * filter that should be using 'struct __sk_buff'. 18208 */ 18209 goto out; 18210 } 18211 18212 ret = do_check(env); 18213 out: 18214 /* check for NULL is necessary, since cur_state can be freed inside 18215 * do_check() under memory pressure. 18216 */ 18217 if (env->cur_state) { 18218 free_verifier_state(env->cur_state, true); 18219 env->cur_state = NULL; 18220 } 18221 while (!pop_stack(env, NULL, NULL, false)); 18222 if (!ret && pop_log) 18223 bpf_vlog_reset(&env->log, 0); 18224 free_states(env); 18225 return ret; 18226 } 18227 18228 /* Verify all global functions in a BPF program one by one based on their BTF. 18229 * All global functions must pass verification. Otherwise the whole program is rejected. 18230 * Consider: 18231 * int bar(int); 18232 * int foo(int f) 18233 * { 18234 * return bar(f); 18235 * } 18236 * int bar(int b) 18237 * { 18238 * ... 18239 * } 18240 * foo() will be verified first for R1=any_scalar_value. During verification it 18241 * will be assumed that bar() already verified successfully and call to bar() 18242 * from foo() will be checked for type match only. Later bar() will be verified 18243 * independently to check that it's safe for R1=any_scalar_value. 18244 */ 18245 static int do_check_subprogs(struct bpf_verifier_env *env) 18246 { 18247 struct bpf_prog_aux *aux = env->prog->aux; 18248 int i, ret; 18249 18250 if (!aux->func_info) 18251 return 0; 18252 18253 for (i = 1; i < env->subprog_cnt; i++) { 18254 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 18255 continue; 18256 env->insn_idx = env->subprog_info[i].start; 18257 WARN_ON_ONCE(env->insn_idx == 0); 18258 ret = do_check_common(env, i); 18259 if (ret) { 18260 return ret; 18261 } else if (env->log.level & BPF_LOG_LEVEL) { 18262 verbose(env, 18263 "Func#%d is safe for any args that match its prototype\n", 18264 i); 18265 } 18266 } 18267 return 0; 18268 } 18269 18270 static int do_check_main(struct bpf_verifier_env *env) 18271 { 18272 int ret; 18273 18274 env->insn_idx = 0; 18275 ret = do_check_common(env, 0); 18276 if (!ret) 18277 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 18278 return ret; 18279 } 18280 18281 18282 static void print_verification_stats(struct bpf_verifier_env *env) 18283 { 18284 int i; 18285 18286 if (env->log.level & BPF_LOG_STATS) { 18287 verbose(env, "verification time %lld usec\n", 18288 div_u64(env->verification_time, 1000)); 18289 verbose(env, "stack depth "); 18290 for (i = 0; i < env->subprog_cnt; i++) { 18291 u32 depth = env->subprog_info[i].stack_depth; 18292 18293 verbose(env, "%d", depth); 18294 if (i + 1 < env->subprog_cnt) 18295 verbose(env, "+"); 18296 } 18297 verbose(env, "\n"); 18298 } 18299 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 18300 "total_states %d peak_states %d mark_read %d\n", 18301 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 18302 env->max_states_per_insn, env->total_states, 18303 env->peak_states, env->longest_mark_read_walk); 18304 } 18305 18306 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 18307 { 18308 const struct btf_type *t, *func_proto; 18309 const struct bpf_struct_ops *st_ops; 18310 const struct btf_member *member; 18311 struct bpf_prog *prog = env->prog; 18312 u32 btf_id, member_idx; 18313 const char *mname; 18314 18315 if (!prog->gpl_compatible) { 18316 verbose(env, "struct ops programs must have a GPL compatible license\n"); 18317 return -EINVAL; 18318 } 18319 18320 btf_id = prog->aux->attach_btf_id; 18321 st_ops = bpf_struct_ops_find(btf_id); 18322 if (!st_ops) { 18323 verbose(env, "attach_btf_id %u is not a supported struct\n", 18324 btf_id); 18325 return -ENOTSUPP; 18326 } 18327 18328 t = st_ops->type; 18329 member_idx = prog->expected_attach_type; 18330 if (member_idx >= btf_type_vlen(t)) { 18331 verbose(env, "attach to invalid member idx %u of struct %s\n", 18332 member_idx, st_ops->name); 18333 return -EINVAL; 18334 } 18335 18336 member = &btf_type_member(t)[member_idx]; 18337 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 18338 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 18339 NULL); 18340 if (!func_proto) { 18341 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 18342 mname, member_idx, st_ops->name); 18343 return -EINVAL; 18344 } 18345 18346 if (st_ops->check_member) { 18347 int err = st_ops->check_member(t, member, prog); 18348 18349 if (err) { 18350 verbose(env, "attach to unsupported member %s of struct %s\n", 18351 mname, st_ops->name); 18352 return err; 18353 } 18354 } 18355 18356 prog->aux->attach_func_proto = func_proto; 18357 prog->aux->attach_func_name = mname; 18358 env->ops = st_ops->verifier_ops; 18359 18360 return 0; 18361 } 18362 #define SECURITY_PREFIX "security_" 18363 18364 static int check_attach_modify_return(unsigned long addr, const char *func_name) 18365 { 18366 if (within_error_injection_list(addr) || 18367 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 18368 return 0; 18369 18370 return -EINVAL; 18371 } 18372 18373 /* list of non-sleepable functions that are otherwise on 18374 * ALLOW_ERROR_INJECTION list 18375 */ 18376 BTF_SET_START(btf_non_sleepable_error_inject) 18377 /* Three functions below can be called from sleepable and non-sleepable context. 18378 * Assume non-sleepable from bpf safety point of view. 18379 */ 18380 BTF_ID(func, __filemap_add_folio) 18381 BTF_ID(func, should_fail_alloc_page) 18382 BTF_ID(func, should_failslab) 18383 BTF_SET_END(btf_non_sleepable_error_inject) 18384 18385 static int check_non_sleepable_error_inject(u32 btf_id) 18386 { 18387 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 18388 } 18389 18390 int bpf_check_attach_target(struct bpf_verifier_log *log, 18391 const struct bpf_prog *prog, 18392 const struct bpf_prog *tgt_prog, 18393 u32 btf_id, 18394 struct bpf_attach_target_info *tgt_info) 18395 { 18396 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 18397 const char prefix[] = "btf_trace_"; 18398 int ret = 0, subprog = -1, i; 18399 const struct btf_type *t; 18400 bool conservative = true; 18401 const char *tname; 18402 struct btf *btf; 18403 long addr = 0; 18404 struct module *mod = NULL; 18405 18406 if (!btf_id) { 18407 bpf_log(log, "Tracing programs must provide btf_id\n"); 18408 return -EINVAL; 18409 } 18410 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 18411 if (!btf) { 18412 bpf_log(log, 18413 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 18414 return -EINVAL; 18415 } 18416 t = btf_type_by_id(btf, btf_id); 18417 if (!t) { 18418 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 18419 return -EINVAL; 18420 } 18421 tname = btf_name_by_offset(btf, t->name_off); 18422 if (!tname) { 18423 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 18424 return -EINVAL; 18425 } 18426 if (tgt_prog) { 18427 struct bpf_prog_aux *aux = tgt_prog->aux; 18428 18429 if (bpf_prog_is_dev_bound(prog->aux) && 18430 !bpf_prog_dev_bound_match(prog, tgt_prog)) { 18431 bpf_log(log, "Target program bound device mismatch"); 18432 return -EINVAL; 18433 } 18434 18435 for (i = 0; i < aux->func_info_cnt; i++) 18436 if (aux->func_info[i].type_id == btf_id) { 18437 subprog = i; 18438 break; 18439 } 18440 if (subprog == -1) { 18441 bpf_log(log, "Subprog %s doesn't exist\n", tname); 18442 return -EINVAL; 18443 } 18444 conservative = aux->func_info_aux[subprog].unreliable; 18445 if (prog_extension) { 18446 if (conservative) { 18447 bpf_log(log, 18448 "Cannot replace static functions\n"); 18449 return -EINVAL; 18450 } 18451 if (!prog->jit_requested) { 18452 bpf_log(log, 18453 "Extension programs should be JITed\n"); 18454 return -EINVAL; 18455 } 18456 } 18457 if (!tgt_prog->jited) { 18458 bpf_log(log, "Can attach to only JITed progs\n"); 18459 return -EINVAL; 18460 } 18461 if (tgt_prog->type == prog->type) { 18462 /* Cannot fentry/fexit another fentry/fexit program. 18463 * Cannot attach program extension to another extension. 18464 * It's ok to attach fentry/fexit to extension program. 18465 */ 18466 bpf_log(log, "Cannot recursively attach\n"); 18467 return -EINVAL; 18468 } 18469 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 18470 prog_extension && 18471 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 18472 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 18473 /* Program extensions can extend all program types 18474 * except fentry/fexit. The reason is the following. 18475 * The fentry/fexit programs are used for performance 18476 * analysis, stats and can be attached to any program 18477 * type except themselves. When extension program is 18478 * replacing XDP function it is necessary to allow 18479 * performance analysis of all functions. Both original 18480 * XDP program and its program extension. Hence 18481 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 18482 * allowed. If extending of fentry/fexit was allowed it 18483 * would be possible to create long call chain 18484 * fentry->extension->fentry->extension beyond 18485 * reasonable stack size. Hence extending fentry is not 18486 * allowed. 18487 */ 18488 bpf_log(log, "Cannot extend fentry/fexit\n"); 18489 return -EINVAL; 18490 } 18491 } else { 18492 if (prog_extension) { 18493 bpf_log(log, "Cannot replace kernel functions\n"); 18494 return -EINVAL; 18495 } 18496 } 18497 18498 switch (prog->expected_attach_type) { 18499 case BPF_TRACE_RAW_TP: 18500 if (tgt_prog) { 18501 bpf_log(log, 18502 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 18503 return -EINVAL; 18504 } 18505 if (!btf_type_is_typedef(t)) { 18506 bpf_log(log, "attach_btf_id %u is not a typedef\n", 18507 btf_id); 18508 return -EINVAL; 18509 } 18510 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 18511 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 18512 btf_id, tname); 18513 return -EINVAL; 18514 } 18515 tname += sizeof(prefix) - 1; 18516 t = btf_type_by_id(btf, t->type); 18517 if (!btf_type_is_ptr(t)) 18518 /* should never happen in valid vmlinux build */ 18519 return -EINVAL; 18520 t = btf_type_by_id(btf, t->type); 18521 if (!btf_type_is_func_proto(t)) 18522 /* should never happen in valid vmlinux build */ 18523 return -EINVAL; 18524 18525 break; 18526 case BPF_TRACE_ITER: 18527 if (!btf_type_is_func(t)) { 18528 bpf_log(log, "attach_btf_id %u is not a function\n", 18529 btf_id); 18530 return -EINVAL; 18531 } 18532 t = btf_type_by_id(btf, t->type); 18533 if (!btf_type_is_func_proto(t)) 18534 return -EINVAL; 18535 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 18536 if (ret) 18537 return ret; 18538 break; 18539 default: 18540 if (!prog_extension) 18541 return -EINVAL; 18542 fallthrough; 18543 case BPF_MODIFY_RETURN: 18544 case BPF_LSM_MAC: 18545 case BPF_LSM_CGROUP: 18546 case BPF_TRACE_FENTRY: 18547 case BPF_TRACE_FEXIT: 18548 if (!btf_type_is_func(t)) { 18549 bpf_log(log, "attach_btf_id %u is not a function\n", 18550 btf_id); 18551 return -EINVAL; 18552 } 18553 if (prog_extension && 18554 btf_check_type_match(log, prog, btf, t)) 18555 return -EINVAL; 18556 t = btf_type_by_id(btf, t->type); 18557 if (!btf_type_is_func_proto(t)) 18558 return -EINVAL; 18559 18560 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 18561 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 18562 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 18563 return -EINVAL; 18564 18565 if (tgt_prog && conservative) 18566 t = NULL; 18567 18568 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 18569 if (ret < 0) 18570 return ret; 18571 18572 if (tgt_prog) { 18573 if (subprog == 0) 18574 addr = (long) tgt_prog->bpf_func; 18575 else 18576 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 18577 } else { 18578 if (btf_is_module(btf)) { 18579 mod = btf_try_get_module(btf); 18580 if (mod) 18581 addr = find_kallsyms_symbol_value(mod, tname); 18582 else 18583 addr = 0; 18584 } else { 18585 addr = kallsyms_lookup_name(tname); 18586 } 18587 if (!addr) { 18588 module_put(mod); 18589 bpf_log(log, 18590 "The address of function %s cannot be found\n", 18591 tname); 18592 return -ENOENT; 18593 } 18594 } 18595 18596 if (prog->aux->sleepable) { 18597 ret = -EINVAL; 18598 switch (prog->type) { 18599 case BPF_PROG_TYPE_TRACING: 18600 18601 /* fentry/fexit/fmod_ret progs can be sleepable if they are 18602 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 18603 */ 18604 if (!check_non_sleepable_error_inject(btf_id) && 18605 within_error_injection_list(addr)) 18606 ret = 0; 18607 /* fentry/fexit/fmod_ret progs can also be sleepable if they are 18608 * in the fmodret id set with the KF_SLEEPABLE flag. 18609 */ 18610 else { 18611 u32 *flags = btf_kfunc_is_modify_return(btf, btf_id); 18612 18613 if (flags && (*flags & KF_SLEEPABLE)) 18614 ret = 0; 18615 } 18616 break; 18617 case BPF_PROG_TYPE_LSM: 18618 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 18619 * Only some of them are sleepable. 18620 */ 18621 if (bpf_lsm_is_sleepable_hook(btf_id)) 18622 ret = 0; 18623 break; 18624 default: 18625 break; 18626 } 18627 if (ret) { 18628 module_put(mod); 18629 bpf_log(log, "%s is not sleepable\n", tname); 18630 return ret; 18631 } 18632 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 18633 if (tgt_prog) { 18634 module_put(mod); 18635 bpf_log(log, "can't modify return codes of BPF programs\n"); 18636 return -EINVAL; 18637 } 18638 ret = -EINVAL; 18639 if (btf_kfunc_is_modify_return(btf, btf_id) || 18640 !check_attach_modify_return(addr, tname)) 18641 ret = 0; 18642 if (ret) { 18643 module_put(mod); 18644 bpf_log(log, "%s() is not modifiable\n", tname); 18645 return ret; 18646 } 18647 } 18648 18649 break; 18650 } 18651 tgt_info->tgt_addr = addr; 18652 tgt_info->tgt_name = tname; 18653 tgt_info->tgt_type = t; 18654 tgt_info->tgt_mod = mod; 18655 return 0; 18656 } 18657 18658 BTF_SET_START(btf_id_deny) 18659 BTF_ID_UNUSED 18660 #ifdef CONFIG_SMP 18661 BTF_ID(func, migrate_disable) 18662 BTF_ID(func, migrate_enable) 18663 #endif 18664 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 18665 BTF_ID(func, rcu_read_unlock_strict) 18666 #endif 18667 #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE) 18668 BTF_ID(func, preempt_count_add) 18669 BTF_ID(func, preempt_count_sub) 18670 #endif 18671 BTF_SET_END(btf_id_deny) 18672 18673 static bool can_be_sleepable(struct bpf_prog *prog) 18674 { 18675 if (prog->type == BPF_PROG_TYPE_TRACING) { 18676 switch (prog->expected_attach_type) { 18677 case BPF_TRACE_FENTRY: 18678 case BPF_TRACE_FEXIT: 18679 case BPF_MODIFY_RETURN: 18680 case BPF_TRACE_ITER: 18681 return true; 18682 default: 18683 return false; 18684 } 18685 } 18686 return prog->type == BPF_PROG_TYPE_LSM || 18687 prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || 18688 prog->type == BPF_PROG_TYPE_STRUCT_OPS; 18689 } 18690 18691 static int check_attach_btf_id(struct bpf_verifier_env *env) 18692 { 18693 struct bpf_prog *prog = env->prog; 18694 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 18695 struct bpf_attach_target_info tgt_info = {}; 18696 u32 btf_id = prog->aux->attach_btf_id; 18697 struct bpf_trampoline *tr; 18698 int ret; 18699 u64 key; 18700 18701 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 18702 if (prog->aux->sleepable) 18703 /* attach_btf_id checked to be zero already */ 18704 return 0; 18705 verbose(env, "Syscall programs can only be sleepable\n"); 18706 return -EINVAL; 18707 } 18708 18709 if (prog->aux->sleepable && !can_be_sleepable(prog)) { 18710 verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); 18711 return -EINVAL; 18712 } 18713 18714 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 18715 return check_struct_ops_btf_id(env); 18716 18717 if (prog->type != BPF_PROG_TYPE_TRACING && 18718 prog->type != BPF_PROG_TYPE_LSM && 18719 prog->type != BPF_PROG_TYPE_EXT) 18720 return 0; 18721 18722 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 18723 if (ret) 18724 return ret; 18725 18726 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 18727 /* to make freplace equivalent to their targets, they need to 18728 * inherit env->ops and expected_attach_type for the rest of the 18729 * verification 18730 */ 18731 env->ops = bpf_verifier_ops[tgt_prog->type]; 18732 prog->expected_attach_type = tgt_prog->expected_attach_type; 18733 } 18734 18735 /* store info about the attachment target that will be used later */ 18736 prog->aux->attach_func_proto = tgt_info.tgt_type; 18737 prog->aux->attach_func_name = tgt_info.tgt_name; 18738 prog->aux->mod = tgt_info.tgt_mod; 18739 18740 if (tgt_prog) { 18741 prog->aux->saved_dst_prog_type = tgt_prog->type; 18742 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 18743 } 18744 18745 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 18746 prog->aux->attach_btf_trace = true; 18747 return 0; 18748 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 18749 if (!bpf_iter_prog_supported(prog)) 18750 return -EINVAL; 18751 return 0; 18752 } 18753 18754 if (prog->type == BPF_PROG_TYPE_LSM) { 18755 ret = bpf_lsm_verify_prog(&env->log, prog); 18756 if (ret < 0) 18757 return ret; 18758 } else if (prog->type == BPF_PROG_TYPE_TRACING && 18759 btf_id_set_contains(&btf_id_deny, btf_id)) { 18760 return -EINVAL; 18761 } 18762 18763 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 18764 tr = bpf_trampoline_get(key, &tgt_info); 18765 if (!tr) 18766 return -ENOMEM; 18767 18768 prog->aux->dst_trampoline = tr; 18769 return 0; 18770 } 18771 18772 struct btf *bpf_get_btf_vmlinux(void) 18773 { 18774 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 18775 mutex_lock(&bpf_verifier_lock); 18776 if (!btf_vmlinux) 18777 btf_vmlinux = btf_parse_vmlinux(); 18778 mutex_unlock(&bpf_verifier_lock); 18779 } 18780 return btf_vmlinux; 18781 } 18782 18783 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size) 18784 { 18785 u64 start_time = ktime_get_ns(); 18786 struct bpf_verifier_env *env; 18787 int i, len, ret = -EINVAL, err; 18788 u32 log_true_size; 18789 bool is_priv; 18790 18791 /* no program is valid */ 18792 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 18793 return -EINVAL; 18794 18795 /* 'struct bpf_verifier_env' can be global, but since it's not small, 18796 * allocate/free it every time bpf_check() is called 18797 */ 18798 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 18799 if (!env) 18800 return -ENOMEM; 18801 18802 len = (*prog)->len; 18803 env->insn_aux_data = 18804 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 18805 ret = -ENOMEM; 18806 if (!env->insn_aux_data) 18807 goto err_free_env; 18808 for (i = 0; i < len; i++) 18809 env->insn_aux_data[i].orig_idx = i; 18810 env->prog = *prog; 18811 env->ops = bpf_verifier_ops[env->prog->type]; 18812 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 18813 is_priv = bpf_capable(); 18814 18815 bpf_get_btf_vmlinux(); 18816 18817 /* grab the mutex to protect few globals used by verifier */ 18818 if (!is_priv) 18819 mutex_lock(&bpf_verifier_lock); 18820 18821 /* user could have requested verbose verifier output 18822 * and supplied buffer to store the verification trace 18823 */ 18824 ret = bpf_vlog_init(&env->log, attr->log_level, 18825 (char __user *) (unsigned long) attr->log_buf, 18826 attr->log_size); 18827 if (ret) 18828 goto err_unlock; 18829 18830 mark_verifier_state_clean(env); 18831 18832 if (IS_ERR(btf_vmlinux)) { 18833 /* Either gcc or pahole or kernel are broken. */ 18834 verbose(env, "in-kernel BTF is malformed\n"); 18835 ret = PTR_ERR(btf_vmlinux); 18836 goto skip_full_check; 18837 } 18838 18839 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 18840 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 18841 env->strict_alignment = true; 18842 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 18843 env->strict_alignment = false; 18844 18845 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 18846 env->allow_uninit_stack = bpf_allow_uninit_stack(); 18847 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 18848 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 18849 env->bpf_capable = bpf_capable(); 18850 18851 if (is_priv) 18852 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 18853 18854 env->explored_states = kvcalloc(state_htab_size(env), 18855 sizeof(struct bpf_verifier_state_list *), 18856 GFP_USER); 18857 ret = -ENOMEM; 18858 if (!env->explored_states) 18859 goto skip_full_check; 18860 18861 ret = add_subprog_and_kfunc(env); 18862 if (ret < 0) 18863 goto skip_full_check; 18864 18865 ret = check_subprogs(env); 18866 if (ret < 0) 18867 goto skip_full_check; 18868 18869 ret = check_btf_info(env, attr, uattr); 18870 if (ret < 0) 18871 goto skip_full_check; 18872 18873 ret = check_attach_btf_id(env); 18874 if (ret) 18875 goto skip_full_check; 18876 18877 ret = resolve_pseudo_ldimm64(env); 18878 if (ret < 0) 18879 goto skip_full_check; 18880 18881 if (bpf_prog_is_offloaded(env->prog->aux)) { 18882 ret = bpf_prog_offload_verifier_prep(env->prog); 18883 if (ret) 18884 goto skip_full_check; 18885 } 18886 18887 ret = check_cfg(env); 18888 if (ret < 0) 18889 goto skip_full_check; 18890 18891 ret = do_check_subprogs(env); 18892 ret = ret ?: do_check_main(env); 18893 18894 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) 18895 ret = bpf_prog_offload_finalize(env); 18896 18897 skip_full_check: 18898 kvfree(env->explored_states); 18899 18900 if (ret == 0) 18901 ret = check_max_stack_depth(env); 18902 18903 /* instruction rewrites happen after this point */ 18904 if (ret == 0) 18905 ret = optimize_bpf_loop(env); 18906 18907 if (is_priv) { 18908 if (ret == 0) 18909 opt_hard_wire_dead_code_branches(env); 18910 if (ret == 0) 18911 ret = opt_remove_dead_code(env); 18912 if (ret == 0) 18913 ret = opt_remove_nops(env); 18914 } else { 18915 if (ret == 0) 18916 sanitize_dead_code(env); 18917 } 18918 18919 if (ret == 0) 18920 /* program is valid, convert *(u32*)(ctx + off) accesses */ 18921 ret = convert_ctx_accesses(env); 18922 18923 if (ret == 0) 18924 ret = do_misc_fixups(env); 18925 18926 /* do 32-bit optimization after insn patching has done so those patched 18927 * insns could be handled correctly. 18928 */ 18929 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { 18930 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 18931 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 18932 : false; 18933 } 18934 18935 if (ret == 0) 18936 ret = fixup_call_args(env); 18937 18938 env->verification_time = ktime_get_ns() - start_time; 18939 print_verification_stats(env); 18940 env->prog->aux->verified_insns = env->insn_processed; 18941 18942 /* preserve original error even if log finalization is successful */ 18943 err = bpf_vlog_finalize(&env->log, &log_true_size); 18944 if (err) 18945 ret = err; 18946 18947 if (uattr_size >= offsetofend(union bpf_attr, log_true_size) && 18948 copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size), 18949 &log_true_size, sizeof(log_true_size))) { 18950 ret = -EFAULT; 18951 goto err_release_maps; 18952 } 18953 18954 if (ret) 18955 goto err_release_maps; 18956 18957 if (env->used_map_cnt) { 18958 /* if program passed verifier, update used_maps in bpf_prog_info */ 18959 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 18960 sizeof(env->used_maps[0]), 18961 GFP_KERNEL); 18962 18963 if (!env->prog->aux->used_maps) { 18964 ret = -ENOMEM; 18965 goto err_release_maps; 18966 } 18967 18968 memcpy(env->prog->aux->used_maps, env->used_maps, 18969 sizeof(env->used_maps[0]) * env->used_map_cnt); 18970 env->prog->aux->used_map_cnt = env->used_map_cnt; 18971 } 18972 if (env->used_btf_cnt) { 18973 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 18974 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 18975 sizeof(env->used_btfs[0]), 18976 GFP_KERNEL); 18977 if (!env->prog->aux->used_btfs) { 18978 ret = -ENOMEM; 18979 goto err_release_maps; 18980 } 18981 18982 memcpy(env->prog->aux->used_btfs, env->used_btfs, 18983 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 18984 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 18985 } 18986 if (env->used_map_cnt || env->used_btf_cnt) { 18987 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 18988 * bpf_ld_imm64 instructions 18989 */ 18990 convert_pseudo_ld_imm64(env); 18991 } 18992 18993 adjust_btf_func(env); 18994 18995 err_release_maps: 18996 if (!env->prog->aux->used_maps) 18997 /* if we didn't copy map pointers into bpf_prog_info, release 18998 * them now. Otherwise free_used_maps() will release them. 18999 */ 19000 release_maps(env); 19001 if (!env->prog->aux->used_btfs) 19002 release_btfs(env); 19003 19004 /* extension progs temporarily inherit the attach_type of their targets 19005 for verification purposes, so set it back to zero before returning 19006 */ 19007 if (env->prog->type == BPF_PROG_TYPE_EXT) 19008 env->prog->expected_attach_type = 0; 19009 19010 *prog = env->prog; 19011 err_unlock: 19012 if (!is_priv) 19013 mutex_unlock(&bpf_verifier_lock); 19014 vfree(env->insn_aux_data); 19015 err_free_env: 19016 kfree(env); 19017 return ret; 19018 } 19019