1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 #include <linux/poison.h> 27 28 #include "disasm.h" 29 30 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 31 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 32 [_id] = & _name ## _verifier_ops, 33 #define BPF_MAP_TYPE(_id, _ops) 34 #define BPF_LINK_TYPE(_id, _name) 35 #include <linux/bpf_types.h> 36 #undef BPF_PROG_TYPE 37 #undef BPF_MAP_TYPE 38 #undef BPF_LINK_TYPE 39 }; 40 41 /* bpf_check() is a static code analyzer that walks eBPF program 42 * instruction by instruction and updates register/stack state. 43 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 44 * 45 * The first pass is depth-first-search to check that the program is a DAG. 46 * It rejects the following programs: 47 * - larger than BPF_MAXINSNS insns 48 * - if loop is present (detected via back-edge) 49 * - unreachable insns exist (shouldn't be a forest. program = one function) 50 * - out of bounds or malformed jumps 51 * The second pass is all possible path descent from the 1st insn. 52 * Since it's analyzing all paths through the program, the length of the 53 * analysis is limited to 64k insn, which may be hit even if total number of 54 * insn is less then 4K, but there are too many branches that change stack/regs. 55 * Number of 'branches to be analyzed' is limited to 1k 56 * 57 * On entry to each instruction, each register has a type, and the instruction 58 * changes the types of the registers depending on instruction semantics. 59 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 60 * copied to R1. 61 * 62 * All registers are 64-bit. 63 * R0 - return register 64 * R1-R5 argument passing registers 65 * R6-R9 callee saved registers 66 * R10 - frame pointer read-only 67 * 68 * At the start of BPF program the register R1 contains a pointer to bpf_context 69 * and has type PTR_TO_CTX. 70 * 71 * Verifier tracks arithmetic operations on pointers in case: 72 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 73 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 74 * 1st insn copies R10 (which has FRAME_PTR) type into R1 75 * and 2nd arithmetic instruction is pattern matched to recognize 76 * that it wants to construct a pointer to some element within stack. 77 * So after 2nd insn, the register R1 has type PTR_TO_STACK 78 * (and -20 constant is saved for further stack bounds checking). 79 * Meaning that this reg is a pointer to stack plus known immediate constant. 80 * 81 * Most of the time the registers have SCALAR_VALUE type, which 82 * means the register has some value, but it's not a valid pointer. 83 * (like pointer plus pointer becomes SCALAR_VALUE type) 84 * 85 * When verifier sees load or store instructions the type of base register 86 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 87 * four pointer types recognized by check_mem_access() function. 88 * 89 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 90 * and the range of [ptr, ptr + map's value_size) is accessible. 91 * 92 * registers used to pass values to function calls are checked against 93 * function argument constraints. 94 * 95 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 96 * It means that the register type passed to this function must be 97 * PTR_TO_STACK and it will be used inside the function as 98 * 'pointer to map element key' 99 * 100 * For example the argument constraints for bpf_map_lookup_elem(): 101 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 102 * .arg1_type = ARG_CONST_MAP_PTR, 103 * .arg2_type = ARG_PTR_TO_MAP_KEY, 104 * 105 * ret_type says that this function returns 'pointer to map elem value or null' 106 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 107 * 2nd argument should be a pointer to stack, which will be used inside 108 * the helper function as a pointer to map element key. 109 * 110 * On the kernel side the helper function looks like: 111 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 112 * { 113 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 114 * void *key = (void *) (unsigned long) r2; 115 * void *value; 116 * 117 * here kernel can access 'key' and 'map' pointers safely, knowing that 118 * [key, key + map->key_size) bytes are valid and were initialized on 119 * the stack of eBPF program. 120 * } 121 * 122 * Corresponding eBPF program may look like: 123 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 124 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 125 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 126 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 127 * here verifier looks at prototype of map_lookup_elem() and sees: 128 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 129 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 130 * 131 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 132 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 133 * and were initialized prior to this call. 134 * If it's ok, then verifier allows this BPF_CALL insn and looks at 135 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 136 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 137 * returns either pointer to map value or NULL. 138 * 139 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 140 * insn, the register holding that pointer in the true branch changes state to 141 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 142 * branch. See check_cond_jmp_op(). 143 * 144 * After the call R0 is set to return type of the function and registers R1-R5 145 * are set to NOT_INIT to indicate that they are no longer readable. 146 * 147 * The following reference types represent a potential reference to a kernel 148 * resource which, after first being allocated, must be checked and freed by 149 * the BPF program: 150 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 151 * 152 * When the verifier sees a helper call return a reference type, it allocates a 153 * pointer id for the reference and stores it in the current function state. 154 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 155 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 156 * passes through a NULL-check conditional. For the branch wherein the state is 157 * changed to CONST_IMM, the verifier releases the reference. 158 * 159 * For each helper function that allocates a reference, such as 160 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 161 * bpf_sk_release(). When a reference type passes into the release function, 162 * the verifier also releases the reference. If any unchecked or unreleased 163 * reference remains at the end of the program, the verifier rejects it. 164 */ 165 166 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 167 struct bpf_verifier_stack_elem { 168 /* verifer state is 'st' 169 * before processing instruction 'insn_idx' 170 * and after processing instruction 'prev_insn_idx' 171 */ 172 struct bpf_verifier_state st; 173 int insn_idx; 174 int prev_insn_idx; 175 struct bpf_verifier_stack_elem *next; 176 /* length of verifier log at the time this state was pushed on stack */ 177 u32 log_pos; 178 }; 179 180 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 181 #define BPF_COMPLEXITY_LIMIT_STATES 64 182 183 #define BPF_MAP_KEY_POISON (1ULL << 63) 184 #define BPF_MAP_KEY_SEEN (1ULL << 62) 185 186 #define BPF_MAP_PTR_UNPRIV 1UL 187 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 188 POISON_POINTER_DELTA)) 189 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 190 191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 193 static void invalidate_non_owning_refs(struct bpf_verifier_env *env); 194 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); 195 static int ref_set_non_owning(struct bpf_verifier_env *env, 196 struct bpf_reg_state *reg); 197 198 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 199 { 200 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 201 } 202 203 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 204 { 205 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 206 } 207 208 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 209 const struct bpf_map *map, bool unpriv) 210 { 211 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 212 unpriv |= bpf_map_ptr_unpriv(aux); 213 aux->map_ptr_state = (unsigned long)map | 214 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 215 } 216 217 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 218 { 219 return aux->map_key_state & BPF_MAP_KEY_POISON; 220 } 221 222 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 223 { 224 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 225 } 226 227 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 228 { 229 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 230 } 231 232 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 233 { 234 bool poisoned = bpf_map_key_poisoned(aux); 235 236 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 237 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 238 } 239 240 static bool bpf_pseudo_call(const struct bpf_insn *insn) 241 { 242 return insn->code == (BPF_JMP | BPF_CALL) && 243 insn->src_reg == BPF_PSEUDO_CALL; 244 } 245 246 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 247 { 248 return insn->code == (BPF_JMP | BPF_CALL) && 249 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 250 } 251 252 struct bpf_call_arg_meta { 253 struct bpf_map *map_ptr; 254 bool raw_mode; 255 bool pkt_access; 256 u8 release_regno; 257 int regno; 258 int access_size; 259 int mem_size; 260 u64 msize_max_value; 261 int ref_obj_id; 262 int dynptr_id; 263 int map_uid; 264 int func_id; 265 struct btf *btf; 266 u32 btf_id; 267 struct btf *ret_btf; 268 u32 ret_btf_id; 269 u32 subprogno; 270 struct btf_field *kptr_field; 271 }; 272 273 struct btf *btf_vmlinux; 274 275 static DEFINE_MUTEX(bpf_verifier_lock); 276 277 static const struct bpf_line_info * 278 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 279 { 280 const struct bpf_line_info *linfo; 281 const struct bpf_prog *prog; 282 u32 i, nr_linfo; 283 284 prog = env->prog; 285 nr_linfo = prog->aux->nr_linfo; 286 287 if (!nr_linfo || insn_off >= prog->len) 288 return NULL; 289 290 linfo = prog->aux->linfo; 291 for (i = 1; i < nr_linfo; i++) 292 if (insn_off < linfo[i].insn_off) 293 break; 294 295 return &linfo[i - 1]; 296 } 297 298 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 299 va_list args) 300 { 301 unsigned int n; 302 303 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 304 305 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 306 "verifier log line truncated - local buffer too short\n"); 307 308 if (log->level == BPF_LOG_KERNEL) { 309 bool newline = n > 0 && log->kbuf[n - 1] == '\n'; 310 311 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n"); 312 return; 313 } 314 315 n = min(log->len_total - log->len_used - 1, n); 316 log->kbuf[n] = '\0'; 317 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 318 log->len_used += n; 319 else 320 log->ubuf = NULL; 321 } 322 323 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 324 { 325 char zero = 0; 326 327 if (!bpf_verifier_log_needed(log)) 328 return; 329 330 log->len_used = new_pos; 331 if (put_user(zero, log->ubuf + new_pos)) 332 log->ubuf = NULL; 333 } 334 335 /* log_level controls verbosity level of eBPF verifier. 336 * bpf_verifier_log_write() is used to dump the verification trace to the log, 337 * so the user can figure out what's wrong with the program 338 */ 339 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 340 const char *fmt, ...) 341 { 342 va_list args; 343 344 if (!bpf_verifier_log_needed(&env->log)) 345 return; 346 347 va_start(args, fmt); 348 bpf_verifier_vlog(&env->log, fmt, args); 349 va_end(args); 350 } 351 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 352 353 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 354 { 355 struct bpf_verifier_env *env = private_data; 356 va_list args; 357 358 if (!bpf_verifier_log_needed(&env->log)) 359 return; 360 361 va_start(args, fmt); 362 bpf_verifier_vlog(&env->log, fmt, args); 363 va_end(args); 364 } 365 366 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 367 const char *fmt, ...) 368 { 369 va_list args; 370 371 if (!bpf_verifier_log_needed(log)) 372 return; 373 374 va_start(args, fmt); 375 bpf_verifier_vlog(log, fmt, args); 376 va_end(args); 377 } 378 EXPORT_SYMBOL_GPL(bpf_log); 379 380 static const char *ltrim(const char *s) 381 { 382 while (isspace(*s)) 383 s++; 384 385 return s; 386 } 387 388 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 389 u32 insn_off, 390 const char *prefix_fmt, ...) 391 { 392 const struct bpf_line_info *linfo; 393 394 if (!bpf_verifier_log_needed(&env->log)) 395 return; 396 397 linfo = find_linfo(env, insn_off); 398 if (!linfo || linfo == env->prev_linfo) 399 return; 400 401 if (prefix_fmt) { 402 va_list args; 403 404 va_start(args, prefix_fmt); 405 bpf_verifier_vlog(&env->log, prefix_fmt, args); 406 va_end(args); 407 } 408 409 verbose(env, "%s\n", 410 ltrim(btf_name_by_offset(env->prog->aux->btf, 411 linfo->line_off))); 412 413 env->prev_linfo = linfo; 414 } 415 416 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 417 struct bpf_reg_state *reg, 418 struct tnum *range, const char *ctx, 419 const char *reg_name) 420 { 421 char tn_buf[48]; 422 423 verbose(env, "At %s the register %s ", ctx, reg_name); 424 if (!tnum_is_unknown(reg->var_off)) { 425 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 426 verbose(env, "has value %s", tn_buf); 427 } else { 428 verbose(env, "has unknown scalar value"); 429 } 430 tnum_strn(tn_buf, sizeof(tn_buf), *range); 431 verbose(env, " should have been in %s\n", tn_buf); 432 } 433 434 static bool type_is_pkt_pointer(enum bpf_reg_type type) 435 { 436 type = base_type(type); 437 return type == PTR_TO_PACKET || 438 type == PTR_TO_PACKET_META; 439 } 440 441 static bool type_is_sk_pointer(enum bpf_reg_type type) 442 { 443 return type == PTR_TO_SOCKET || 444 type == PTR_TO_SOCK_COMMON || 445 type == PTR_TO_TCP_SOCK || 446 type == PTR_TO_XDP_SOCK; 447 } 448 449 static bool reg_type_not_null(enum bpf_reg_type type) 450 { 451 return type == PTR_TO_SOCKET || 452 type == PTR_TO_TCP_SOCK || 453 type == PTR_TO_MAP_VALUE || 454 type == PTR_TO_MAP_KEY || 455 type == PTR_TO_SOCK_COMMON; 456 } 457 458 static bool type_is_ptr_alloc_obj(u32 type) 459 { 460 return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; 461 } 462 463 static bool type_is_non_owning_ref(u32 type) 464 { 465 return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; 466 } 467 468 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 469 { 470 struct btf_record *rec = NULL; 471 struct btf_struct_meta *meta; 472 473 if (reg->type == PTR_TO_MAP_VALUE) { 474 rec = reg->map_ptr->record; 475 } else if (type_is_ptr_alloc_obj(reg->type)) { 476 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 477 if (meta) 478 rec = meta->record; 479 } 480 return rec; 481 } 482 483 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 484 { 485 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); 486 } 487 488 static bool type_is_rdonly_mem(u32 type) 489 { 490 return type & MEM_RDONLY; 491 } 492 493 static bool type_may_be_null(u32 type) 494 { 495 return type & PTR_MAYBE_NULL; 496 } 497 498 static bool is_acquire_function(enum bpf_func_id func_id, 499 const struct bpf_map *map) 500 { 501 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 502 503 if (func_id == BPF_FUNC_sk_lookup_tcp || 504 func_id == BPF_FUNC_sk_lookup_udp || 505 func_id == BPF_FUNC_skc_lookup_tcp || 506 func_id == BPF_FUNC_ringbuf_reserve || 507 func_id == BPF_FUNC_kptr_xchg) 508 return true; 509 510 if (func_id == BPF_FUNC_map_lookup_elem && 511 (map_type == BPF_MAP_TYPE_SOCKMAP || 512 map_type == BPF_MAP_TYPE_SOCKHASH)) 513 return true; 514 515 return false; 516 } 517 518 static bool is_ptr_cast_function(enum bpf_func_id func_id) 519 { 520 return func_id == BPF_FUNC_tcp_sock || 521 func_id == BPF_FUNC_sk_fullsock || 522 func_id == BPF_FUNC_skc_to_tcp_sock || 523 func_id == BPF_FUNC_skc_to_tcp6_sock || 524 func_id == BPF_FUNC_skc_to_udp6_sock || 525 func_id == BPF_FUNC_skc_to_mptcp_sock || 526 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 527 func_id == BPF_FUNC_skc_to_tcp_request_sock; 528 } 529 530 static bool is_dynptr_ref_function(enum bpf_func_id func_id) 531 { 532 return func_id == BPF_FUNC_dynptr_data; 533 } 534 535 static bool is_callback_calling_function(enum bpf_func_id func_id) 536 { 537 return func_id == BPF_FUNC_for_each_map_elem || 538 func_id == BPF_FUNC_timer_set_callback || 539 func_id == BPF_FUNC_find_vma || 540 func_id == BPF_FUNC_loop || 541 func_id == BPF_FUNC_user_ringbuf_drain; 542 } 543 544 static bool is_storage_get_function(enum bpf_func_id func_id) 545 { 546 return func_id == BPF_FUNC_sk_storage_get || 547 func_id == BPF_FUNC_inode_storage_get || 548 func_id == BPF_FUNC_task_storage_get || 549 func_id == BPF_FUNC_cgrp_storage_get; 550 } 551 552 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, 553 const struct bpf_map *map) 554 { 555 int ref_obj_uses = 0; 556 557 if (is_ptr_cast_function(func_id)) 558 ref_obj_uses++; 559 if (is_acquire_function(func_id, map)) 560 ref_obj_uses++; 561 if (is_dynptr_ref_function(func_id)) 562 ref_obj_uses++; 563 564 return ref_obj_uses > 1; 565 } 566 567 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 568 { 569 return BPF_CLASS(insn->code) == BPF_STX && 570 BPF_MODE(insn->code) == BPF_ATOMIC && 571 insn->imm == BPF_CMPXCHG; 572 } 573 574 /* string representation of 'enum bpf_reg_type' 575 * 576 * Note that reg_type_str() can not appear more than once in a single verbose() 577 * statement. 578 */ 579 static const char *reg_type_str(struct bpf_verifier_env *env, 580 enum bpf_reg_type type) 581 { 582 char postfix[16] = {0}, prefix[64] = {0}; 583 static const char * const str[] = { 584 [NOT_INIT] = "?", 585 [SCALAR_VALUE] = "scalar", 586 [PTR_TO_CTX] = "ctx", 587 [CONST_PTR_TO_MAP] = "map_ptr", 588 [PTR_TO_MAP_VALUE] = "map_value", 589 [PTR_TO_STACK] = "fp", 590 [PTR_TO_PACKET] = "pkt", 591 [PTR_TO_PACKET_META] = "pkt_meta", 592 [PTR_TO_PACKET_END] = "pkt_end", 593 [PTR_TO_FLOW_KEYS] = "flow_keys", 594 [PTR_TO_SOCKET] = "sock", 595 [PTR_TO_SOCK_COMMON] = "sock_common", 596 [PTR_TO_TCP_SOCK] = "tcp_sock", 597 [PTR_TO_TP_BUFFER] = "tp_buffer", 598 [PTR_TO_XDP_SOCK] = "xdp_sock", 599 [PTR_TO_BTF_ID] = "ptr_", 600 [PTR_TO_MEM] = "mem", 601 [PTR_TO_BUF] = "buf", 602 [PTR_TO_FUNC] = "func", 603 [PTR_TO_MAP_KEY] = "map_key", 604 [CONST_PTR_TO_DYNPTR] = "dynptr_ptr", 605 }; 606 607 if (type & PTR_MAYBE_NULL) { 608 if (base_type(type) == PTR_TO_BTF_ID) 609 strncpy(postfix, "or_null_", 16); 610 else 611 strncpy(postfix, "_or_null", 16); 612 } 613 614 snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", 615 type & MEM_RDONLY ? "rdonly_" : "", 616 type & MEM_RINGBUF ? "ringbuf_" : "", 617 type & MEM_USER ? "user_" : "", 618 type & MEM_PERCPU ? "percpu_" : "", 619 type & MEM_RCU ? "rcu_" : "", 620 type & PTR_UNTRUSTED ? "untrusted_" : "", 621 type & PTR_TRUSTED ? "trusted_" : "" 622 ); 623 624 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", 625 prefix, str[base_type(type)], postfix); 626 return env->type_str_buf; 627 } 628 629 static char slot_type_char[] = { 630 [STACK_INVALID] = '?', 631 [STACK_SPILL] = 'r', 632 [STACK_MISC] = 'm', 633 [STACK_ZERO] = '0', 634 [STACK_DYNPTR] = 'd', 635 }; 636 637 static void print_liveness(struct bpf_verifier_env *env, 638 enum bpf_reg_liveness live) 639 { 640 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 641 verbose(env, "_"); 642 if (live & REG_LIVE_READ) 643 verbose(env, "r"); 644 if (live & REG_LIVE_WRITTEN) 645 verbose(env, "w"); 646 if (live & REG_LIVE_DONE) 647 verbose(env, "D"); 648 } 649 650 static int __get_spi(s32 off) 651 { 652 return (-off - 1) / BPF_REG_SIZE; 653 } 654 655 static struct bpf_func_state *func(struct bpf_verifier_env *env, 656 const struct bpf_reg_state *reg) 657 { 658 struct bpf_verifier_state *cur = env->cur_state; 659 660 return cur->frame[reg->frameno]; 661 } 662 663 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 664 { 665 int allocated_slots = state->allocated_stack / BPF_REG_SIZE; 666 667 /* We need to check that slots between [spi - nr_slots + 1, spi] are 668 * within [0, allocated_stack). 669 * 670 * Please note that the spi grows downwards. For example, a dynptr 671 * takes the size of two stack slots; the first slot will be at 672 * spi and the second slot will be at spi - 1. 673 */ 674 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 675 } 676 677 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 678 { 679 int off, spi; 680 681 if (!tnum_is_const(reg->var_off)) { 682 verbose(env, "dynptr has to be at a constant offset\n"); 683 return -EINVAL; 684 } 685 686 off = reg->off + reg->var_off.value; 687 if (off % BPF_REG_SIZE) { 688 verbose(env, "cannot pass in dynptr at an offset=%d\n", off); 689 return -EINVAL; 690 } 691 692 spi = __get_spi(off); 693 if (spi < 1) { 694 verbose(env, "cannot pass in dynptr at an offset=%d\n", off); 695 return -EINVAL; 696 } 697 698 if (!is_spi_bounds_valid(func(env, reg), spi, BPF_DYNPTR_NR_SLOTS)) 699 return -ERANGE; 700 return spi; 701 } 702 703 static const char *kernel_type_name(const struct btf* btf, u32 id) 704 { 705 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 706 } 707 708 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) 709 { 710 env->scratched_regs |= 1U << regno; 711 } 712 713 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) 714 { 715 env->scratched_stack_slots |= 1ULL << spi; 716 } 717 718 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) 719 { 720 return (env->scratched_regs >> regno) & 1; 721 } 722 723 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) 724 { 725 return (env->scratched_stack_slots >> regno) & 1; 726 } 727 728 static bool verifier_state_scratched(const struct bpf_verifier_env *env) 729 { 730 return env->scratched_regs || env->scratched_stack_slots; 731 } 732 733 static void mark_verifier_state_clean(struct bpf_verifier_env *env) 734 { 735 env->scratched_regs = 0U; 736 env->scratched_stack_slots = 0ULL; 737 } 738 739 /* Used for printing the entire verifier state. */ 740 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) 741 { 742 env->scratched_regs = ~0U; 743 env->scratched_stack_slots = ~0ULL; 744 } 745 746 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 747 { 748 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 749 case DYNPTR_TYPE_LOCAL: 750 return BPF_DYNPTR_TYPE_LOCAL; 751 case DYNPTR_TYPE_RINGBUF: 752 return BPF_DYNPTR_TYPE_RINGBUF; 753 case DYNPTR_TYPE_SKB: 754 return BPF_DYNPTR_TYPE_SKB; 755 case DYNPTR_TYPE_XDP: 756 return BPF_DYNPTR_TYPE_XDP; 757 default: 758 return BPF_DYNPTR_TYPE_INVALID; 759 } 760 } 761 762 static enum bpf_type_flag get_dynptr_type_flag(enum bpf_dynptr_type type) 763 { 764 switch (type) { 765 case BPF_DYNPTR_TYPE_LOCAL: 766 return DYNPTR_TYPE_LOCAL; 767 case BPF_DYNPTR_TYPE_RINGBUF: 768 return DYNPTR_TYPE_RINGBUF; 769 case BPF_DYNPTR_TYPE_SKB: 770 return DYNPTR_TYPE_SKB; 771 case BPF_DYNPTR_TYPE_XDP: 772 return DYNPTR_TYPE_XDP; 773 default: 774 return 0; 775 } 776 } 777 778 static bool dynptr_type_refcounted(enum bpf_dynptr_type type) 779 { 780 return type == BPF_DYNPTR_TYPE_RINGBUF; 781 } 782 783 static void __mark_dynptr_reg(struct bpf_reg_state *reg, 784 enum bpf_dynptr_type type, 785 bool first_slot, int dynptr_id); 786 787 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 788 struct bpf_reg_state *reg); 789 790 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, 791 struct bpf_reg_state *sreg1, 792 struct bpf_reg_state *sreg2, 793 enum bpf_dynptr_type type) 794 { 795 int id = ++env->id_gen; 796 797 __mark_dynptr_reg(sreg1, type, true, id); 798 __mark_dynptr_reg(sreg2, type, false, id); 799 } 800 801 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, 802 struct bpf_reg_state *reg, 803 enum bpf_dynptr_type type) 804 { 805 __mark_dynptr_reg(reg, type, true, ++env->id_gen); 806 } 807 808 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 809 struct bpf_func_state *state, int spi); 810 811 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 812 enum bpf_arg_type arg_type, int insn_idx) 813 { 814 struct bpf_func_state *state = func(env, reg); 815 enum bpf_dynptr_type type; 816 int spi, i, id, err; 817 818 spi = dynptr_get_spi(env, reg); 819 if (spi < 0) 820 return spi; 821 822 /* We cannot assume both spi and spi - 1 belong to the same dynptr, 823 * hence we need to call destroy_if_dynptr_stack_slot twice for both, 824 * to ensure that for the following example: 825 * [d1][d1][d2][d2] 826 * spi 3 2 1 0 827 * So marking spi = 2 should lead to destruction of both d1 and d2. In 828 * case they do belong to same dynptr, second call won't see slot_type 829 * as STACK_DYNPTR and will simply skip destruction. 830 */ 831 err = destroy_if_dynptr_stack_slot(env, state, spi); 832 if (err) 833 return err; 834 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); 835 if (err) 836 return err; 837 838 for (i = 0; i < BPF_REG_SIZE; i++) { 839 state->stack[spi].slot_type[i] = STACK_DYNPTR; 840 state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; 841 } 842 843 type = arg_to_dynptr_type(arg_type); 844 if (type == BPF_DYNPTR_TYPE_INVALID) 845 return -EINVAL; 846 847 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, 848 &state->stack[spi - 1].spilled_ptr, type); 849 850 if (dynptr_type_refcounted(type)) { 851 /* The id is used to track proper releasing */ 852 id = acquire_reference_state(env, insn_idx); 853 if (id < 0) 854 return id; 855 856 state->stack[spi].spilled_ptr.ref_obj_id = id; 857 state->stack[spi - 1].spilled_ptr.ref_obj_id = id; 858 } 859 860 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 861 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 862 863 return 0; 864 } 865 866 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 867 { 868 struct bpf_func_state *state = func(env, reg); 869 int spi, i; 870 871 spi = dynptr_get_spi(env, reg); 872 if (spi < 0) 873 return spi; 874 875 for (i = 0; i < BPF_REG_SIZE; i++) { 876 state->stack[spi].slot_type[i] = STACK_INVALID; 877 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 878 } 879 880 /* Invalidate any slices associated with this dynptr */ 881 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) 882 WARN_ON_ONCE(release_reference(env, state->stack[spi].spilled_ptr.ref_obj_id)); 883 884 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 885 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 886 887 /* Why do we need to set REG_LIVE_WRITTEN for STACK_INVALID slot? 888 * 889 * While we don't allow reading STACK_INVALID, it is still possible to 890 * do <8 byte writes marking some but not all slots as STACK_MISC. Then, 891 * helpers or insns can do partial read of that part without failing, 892 * but check_stack_range_initialized, check_stack_read_var_off, and 893 * check_stack_read_fixed_off will do mark_reg_read for all 8-bytes of 894 * the slot conservatively. Hence we need to prevent those liveness 895 * marking walks. 896 * 897 * This was not a problem before because STACK_INVALID is only set by 898 * default (where the default reg state has its reg->parent as NULL), or 899 * in clean_live_states after REG_LIVE_DONE (at which point 900 * mark_reg_read won't walk reg->parent chain), but not randomly during 901 * verifier state exploration (like we did above). Hence, for our case 902 * parentage chain will still be live (i.e. reg->parent may be 903 * non-NULL), while earlier reg->parent was NULL, so we need 904 * REG_LIVE_WRITTEN to screen off read marker propagation when it is 905 * done later on reads or by mark_dynptr_read as well to unnecessary 906 * mark registers in verifier state. 907 */ 908 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 909 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 910 911 return 0; 912 } 913 914 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 915 struct bpf_reg_state *reg); 916 917 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) 918 { 919 if (!env->allow_ptr_leaks) 920 __mark_reg_not_init(env, reg); 921 else 922 __mark_reg_unknown(env, reg); 923 } 924 925 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, 926 struct bpf_func_state *state, int spi) 927 { 928 struct bpf_func_state *fstate; 929 struct bpf_reg_state *dreg; 930 int i, dynptr_id; 931 932 /* We always ensure that STACK_DYNPTR is never set partially, 933 * hence just checking for slot_type[0] is enough. This is 934 * different for STACK_SPILL, where it may be only set for 935 * 1 byte, so code has to use is_spilled_reg. 936 */ 937 if (state->stack[spi].slot_type[0] != STACK_DYNPTR) 938 return 0; 939 940 /* Reposition spi to first slot */ 941 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 942 spi = spi + 1; 943 944 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 945 verbose(env, "cannot overwrite referenced dynptr\n"); 946 return -EINVAL; 947 } 948 949 mark_stack_slot_scratched(env, spi); 950 mark_stack_slot_scratched(env, spi - 1); 951 952 /* Writing partially to one dynptr stack slot destroys both. */ 953 for (i = 0; i < BPF_REG_SIZE; i++) { 954 state->stack[spi].slot_type[i] = STACK_INVALID; 955 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 956 } 957 958 dynptr_id = state->stack[spi].spilled_ptr.id; 959 /* Invalidate any slices associated with this dynptr */ 960 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ 961 /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */ 962 if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM) 963 continue; 964 if (dreg->dynptr_id == dynptr_id) 965 mark_reg_invalid(env, dreg); 966 })); 967 968 /* Do not release reference state, we are destroying dynptr on stack, 969 * not using some helper to release it. Just reset register. 970 */ 971 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); 972 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); 973 974 /* Same reason as unmark_stack_slots_dynptr above */ 975 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 976 state->stack[spi - 1].spilled_ptr.live |= REG_LIVE_WRITTEN; 977 978 return 0; 979 } 980 981 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 982 { 983 int spi; 984 985 if (reg->type == CONST_PTR_TO_DYNPTR) 986 return false; 987 988 spi = dynptr_get_spi(env, reg); 989 990 /* -ERANGE (i.e. spi not falling into allocated stack slots) isn't an 991 * error because this just means the stack state hasn't been updated yet. 992 * We will do check_mem_access to check and update stack bounds later. 993 */ 994 if (spi < 0 && spi != -ERANGE) 995 return false; 996 997 /* We don't need to check if the stack slots are marked by previous 998 * dynptr initializations because we allow overwriting existing unreferenced 999 * STACK_DYNPTR slots, see mark_stack_slots_dynptr which calls 1000 * destroy_if_dynptr_stack_slot to ensure dynptr objects at the slots we are 1001 * touching are completely destructed before we reinitialize them for a new 1002 * one. For referenced ones, destroy_if_dynptr_stack_slot returns an error early 1003 * instead of delaying it until the end where the user will get "Unreleased 1004 * reference" error. 1005 */ 1006 return true; 1007 } 1008 1009 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 1010 { 1011 struct bpf_func_state *state = func(env, reg); 1012 int i, spi; 1013 1014 /* This already represents first slot of initialized bpf_dynptr. 1015 * 1016 * CONST_PTR_TO_DYNPTR already has fixed and var_off as 0 due to 1017 * check_func_arg_reg_off's logic, so we don't need to check its 1018 * offset and alignment. 1019 */ 1020 if (reg->type == CONST_PTR_TO_DYNPTR) 1021 return true; 1022 1023 spi = dynptr_get_spi(env, reg); 1024 if (spi < 0) 1025 return false; 1026 if (!state->stack[spi].spilled_ptr.dynptr.first_slot) 1027 return false; 1028 1029 for (i = 0; i < BPF_REG_SIZE; i++) { 1030 if (state->stack[spi].slot_type[i] != STACK_DYNPTR || 1031 state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) 1032 return false; 1033 } 1034 1035 return true; 1036 } 1037 1038 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 1039 enum bpf_arg_type arg_type) 1040 { 1041 struct bpf_func_state *state = func(env, reg); 1042 enum bpf_dynptr_type dynptr_type; 1043 int spi; 1044 1045 /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ 1046 if (arg_type == ARG_PTR_TO_DYNPTR) 1047 return true; 1048 1049 dynptr_type = arg_to_dynptr_type(arg_type); 1050 if (reg->type == CONST_PTR_TO_DYNPTR) { 1051 return reg->dynptr.type == dynptr_type; 1052 } else { 1053 spi = dynptr_get_spi(env, reg); 1054 if (spi < 0) 1055 return false; 1056 return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; 1057 } 1058 } 1059 1060 /* The reg state of a pointer or a bounded scalar was saved when 1061 * it was spilled to the stack. 1062 */ 1063 static bool is_spilled_reg(const struct bpf_stack_state *stack) 1064 { 1065 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 1066 } 1067 1068 static void scrub_spilled_slot(u8 *stype) 1069 { 1070 if (*stype != STACK_INVALID) 1071 *stype = STACK_MISC; 1072 } 1073 1074 static void print_verifier_state(struct bpf_verifier_env *env, 1075 const struct bpf_func_state *state, 1076 bool print_all) 1077 { 1078 const struct bpf_reg_state *reg; 1079 enum bpf_reg_type t; 1080 int i; 1081 1082 if (state->frameno) 1083 verbose(env, " frame%d:", state->frameno); 1084 for (i = 0; i < MAX_BPF_REG; i++) { 1085 reg = &state->regs[i]; 1086 t = reg->type; 1087 if (t == NOT_INIT) 1088 continue; 1089 if (!print_all && !reg_scratched(env, i)) 1090 continue; 1091 verbose(env, " R%d", i); 1092 print_liveness(env, reg->live); 1093 verbose(env, "="); 1094 if (t == SCALAR_VALUE && reg->precise) 1095 verbose(env, "P"); 1096 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 1097 tnum_is_const(reg->var_off)) { 1098 /* reg->off should be 0 for SCALAR_VALUE */ 1099 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 1100 verbose(env, "%lld", reg->var_off.value + reg->off); 1101 } else { 1102 const char *sep = ""; 1103 1104 verbose(env, "%s", reg_type_str(env, t)); 1105 if (base_type(t) == PTR_TO_BTF_ID) 1106 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 1107 verbose(env, "("); 1108 /* 1109 * _a stands for append, was shortened to avoid multiline statements below. 1110 * This macro is used to output a comma separated list of attributes. 1111 */ 1112 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) 1113 1114 if (reg->id) 1115 verbose_a("id=%d", reg->id); 1116 if (reg->ref_obj_id) 1117 verbose_a("ref_obj_id=%d", reg->ref_obj_id); 1118 if (type_is_non_owning_ref(reg->type)) 1119 verbose_a("%s", "non_own_ref"); 1120 if (t != SCALAR_VALUE) 1121 verbose_a("off=%d", reg->off); 1122 if (type_is_pkt_pointer(t)) 1123 verbose_a("r=%d", reg->range); 1124 else if (base_type(t) == CONST_PTR_TO_MAP || 1125 base_type(t) == PTR_TO_MAP_KEY || 1126 base_type(t) == PTR_TO_MAP_VALUE) 1127 verbose_a("ks=%d,vs=%d", 1128 reg->map_ptr->key_size, 1129 reg->map_ptr->value_size); 1130 if (tnum_is_const(reg->var_off)) { 1131 /* Typically an immediate SCALAR_VALUE, but 1132 * could be a pointer whose offset is too big 1133 * for reg->off 1134 */ 1135 verbose_a("imm=%llx", reg->var_off.value); 1136 } else { 1137 if (reg->smin_value != reg->umin_value && 1138 reg->smin_value != S64_MIN) 1139 verbose_a("smin=%lld", (long long)reg->smin_value); 1140 if (reg->smax_value != reg->umax_value && 1141 reg->smax_value != S64_MAX) 1142 verbose_a("smax=%lld", (long long)reg->smax_value); 1143 if (reg->umin_value != 0) 1144 verbose_a("umin=%llu", (unsigned long long)reg->umin_value); 1145 if (reg->umax_value != U64_MAX) 1146 verbose_a("umax=%llu", (unsigned long long)reg->umax_value); 1147 if (!tnum_is_unknown(reg->var_off)) { 1148 char tn_buf[48]; 1149 1150 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1151 verbose_a("var_off=%s", tn_buf); 1152 } 1153 if (reg->s32_min_value != reg->smin_value && 1154 reg->s32_min_value != S32_MIN) 1155 verbose_a("s32_min=%d", (int)(reg->s32_min_value)); 1156 if (reg->s32_max_value != reg->smax_value && 1157 reg->s32_max_value != S32_MAX) 1158 verbose_a("s32_max=%d", (int)(reg->s32_max_value)); 1159 if (reg->u32_min_value != reg->umin_value && 1160 reg->u32_min_value != U32_MIN) 1161 verbose_a("u32_min=%d", (int)(reg->u32_min_value)); 1162 if (reg->u32_max_value != reg->umax_value && 1163 reg->u32_max_value != U32_MAX) 1164 verbose_a("u32_max=%d", (int)(reg->u32_max_value)); 1165 } 1166 #undef verbose_a 1167 1168 verbose(env, ")"); 1169 } 1170 } 1171 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 1172 char types_buf[BPF_REG_SIZE + 1]; 1173 bool valid = false; 1174 int j; 1175 1176 for (j = 0; j < BPF_REG_SIZE; j++) { 1177 if (state->stack[i].slot_type[j] != STACK_INVALID) 1178 valid = true; 1179 types_buf[j] = slot_type_char[ 1180 state->stack[i].slot_type[j]]; 1181 } 1182 types_buf[BPF_REG_SIZE] = 0; 1183 if (!valid) 1184 continue; 1185 if (!print_all && !stack_slot_scratched(env, i)) 1186 continue; 1187 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 1188 print_liveness(env, state->stack[i].spilled_ptr.live); 1189 if (is_spilled_reg(&state->stack[i])) { 1190 reg = &state->stack[i].spilled_ptr; 1191 t = reg->type; 1192 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 1193 if (t == SCALAR_VALUE && reg->precise) 1194 verbose(env, "P"); 1195 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 1196 verbose(env, "%lld", reg->var_off.value + reg->off); 1197 } else { 1198 verbose(env, "=%s", types_buf); 1199 } 1200 } 1201 if (state->acquired_refs && state->refs[0].id) { 1202 verbose(env, " refs=%d", state->refs[0].id); 1203 for (i = 1; i < state->acquired_refs; i++) 1204 if (state->refs[i].id) 1205 verbose(env, ",%d", state->refs[i].id); 1206 } 1207 if (state->in_callback_fn) 1208 verbose(env, " cb"); 1209 if (state->in_async_callback_fn) 1210 verbose(env, " async_cb"); 1211 verbose(env, "\n"); 1212 mark_verifier_state_clean(env); 1213 } 1214 1215 static inline u32 vlog_alignment(u32 pos) 1216 { 1217 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), 1218 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 1219 } 1220 1221 static void print_insn_state(struct bpf_verifier_env *env, 1222 const struct bpf_func_state *state) 1223 { 1224 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { 1225 /* remove new line character */ 1226 bpf_vlog_reset(&env->log, env->prev_log_len - 1); 1227 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); 1228 } else { 1229 verbose(env, "%d:", env->insn_idx); 1230 } 1231 print_verifier_state(env, state, false); 1232 } 1233 1234 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1235 * small to hold src. This is different from krealloc since we don't want to preserve 1236 * the contents of dst. 1237 * 1238 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1239 * not be allocated. 1240 */ 1241 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1242 { 1243 size_t alloc_bytes; 1244 void *orig = dst; 1245 size_t bytes; 1246 1247 if (ZERO_OR_NULL_PTR(src)) 1248 goto out; 1249 1250 if (unlikely(check_mul_overflow(n, size, &bytes))) 1251 return NULL; 1252 1253 alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes)); 1254 dst = krealloc(orig, alloc_bytes, flags); 1255 if (!dst) { 1256 kfree(orig); 1257 return NULL; 1258 } 1259 1260 memcpy(dst, src, bytes); 1261 out: 1262 return dst ? dst : ZERO_SIZE_PTR; 1263 } 1264 1265 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1266 * small to hold new_n items. new items are zeroed out if the array grows. 1267 * 1268 * Contrary to krealloc_array, does not free arr if new_n is zero. 1269 */ 1270 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1271 { 1272 size_t alloc_size; 1273 void *new_arr; 1274 1275 if (!new_n || old_n == new_n) 1276 goto out; 1277 1278 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1279 new_arr = krealloc(arr, alloc_size, GFP_KERNEL); 1280 if (!new_arr) { 1281 kfree(arr); 1282 return NULL; 1283 } 1284 arr = new_arr; 1285 1286 if (new_n > old_n) 1287 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1288 1289 out: 1290 return arr ? arr : ZERO_SIZE_PTR; 1291 } 1292 1293 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1294 { 1295 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1296 sizeof(struct bpf_reference_state), GFP_KERNEL); 1297 if (!dst->refs) 1298 return -ENOMEM; 1299 1300 dst->acquired_refs = src->acquired_refs; 1301 return 0; 1302 } 1303 1304 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1305 { 1306 size_t n = src->allocated_stack / BPF_REG_SIZE; 1307 1308 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1309 GFP_KERNEL); 1310 if (!dst->stack) 1311 return -ENOMEM; 1312 1313 dst->allocated_stack = src->allocated_stack; 1314 return 0; 1315 } 1316 1317 static int resize_reference_state(struct bpf_func_state *state, size_t n) 1318 { 1319 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1320 sizeof(struct bpf_reference_state)); 1321 if (!state->refs) 1322 return -ENOMEM; 1323 1324 state->acquired_refs = n; 1325 return 0; 1326 } 1327 1328 static int grow_stack_state(struct bpf_func_state *state, int size) 1329 { 1330 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 1331 1332 if (old_n >= n) 1333 return 0; 1334 1335 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1336 if (!state->stack) 1337 return -ENOMEM; 1338 1339 state->allocated_stack = size; 1340 return 0; 1341 } 1342 1343 /* Acquire a pointer id from the env and update the state->refs to include 1344 * this new pointer reference. 1345 * On success, returns a valid pointer id to associate with the register 1346 * On failure, returns a negative errno. 1347 */ 1348 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1349 { 1350 struct bpf_func_state *state = cur_func(env); 1351 int new_ofs = state->acquired_refs; 1352 int id, err; 1353 1354 err = resize_reference_state(state, state->acquired_refs + 1); 1355 if (err) 1356 return err; 1357 id = ++env->id_gen; 1358 state->refs[new_ofs].id = id; 1359 state->refs[new_ofs].insn_idx = insn_idx; 1360 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; 1361 1362 return id; 1363 } 1364 1365 /* release function corresponding to acquire_reference_state(). Idempotent. */ 1366 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1367 { 1368 int i, last_idx; 1369 1370 last_idx = state->acquired_refs - 1; 1371 for (i = 0; i < state->acquired_refs; i++) { 1372 if (state->refs[i].id == ptr_id) { 1373 /* Cannot release caller references in callbacks */ 1374 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 1375 return -EINVAL; 1376 if (last_idx && i != last_idx) 1377 memcpy(&state->refs[i], &state->refs[last_idx], 1378 sizeof(*state->refs)); 1379 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1380 state->acquired_refs--; 1381 return 0; 1382 } 1383 } 1384 return -EINVAL; 1385 } 1386 1387 static void free_func_state(struct bpf_func_state *state) 1388 { 1389 if (!state) 1390 return; 1391 kfree(state->refs); 1392 kfree(state->stack); 1393 kfree(state); 1394 } 1395 1396 static void clear_jmp_history(struct bpf_verifier_state *state) 1397 { 1398 kfree(state->jmp_history); 1399 state->jmp_history = NULL; 1400 state->jmp_history_cnt = 0; 1401 } 1402 1403 static void free_verifier_state(struct bpf_verifier_state *state, 1404 bool free_self) 1405 { 1406 int i; 1407 1408 for (i = 0; i <= state->curframe; i++) { 1409 free_func_state(state->frame[i]); 1410 state->frame[i] = NULL; 1411 } 1412 clear_jmp_history(state); 1413 if (free_self) 1414 kfree(state); 1415 } 1416 1417 /* copy verifier state from src to dst growing dst stack space 1418 * when necessary to accommodate larger src stack 1419 */ 1420 static int copy_func_state(struct bpf_func_state *dst, 1421 const struct bpf_func_state *src) 1422 { 1423 int err; 1424 1425 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1426 err = copy_reference_state(dst, src); 1427 if (err) 1428 return err; 1429 return copy_stack_state(dst, src); 1430 } 1431 1432 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1433 const struct bpf_verifier_state *src) 1434 { 1435 struct bpf_func_state *dst; 1436 int i, err; 1437 1438 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1439 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 1440 GFP_USER); 1441 if (!dst_state->jmp_history) 1442 return -ENOMEM; 1443 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1444 1445 /* if dst has more stack frames then src frame, free them */ 1446 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1447 free_func_state(dst_state->frame[i]); 1448 dst_state->frame[i] = NULL; 1449 } 1450 dst_state->speculative = src->speculative; 1451 dst_state->active_rcu_lock = src->active_rcu_lock; 1452 dst_state->curframe = src->curframe; 1453 dst_state->active_lock.ptr = src->active_lock.ptr; 1454 dst_state->active_lock.id = src->active_lock.id; 1455 dst_state->branches = src->branches; 1456 dst_state->parent = src->parent; 1457 dst_state->first_insn_idx = src->first_insn_idx; 1458 dst_state->last_insn_idx = src->last_insn_idx; 1459 for (i = 0; i <= src->curframe; i++) { 1460 dst = dst_state->frame[i]; 1461 if (!dst) { 1462 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1463 if (!dst) 1464 return -ENOMEM; 1465 dst_state->frame[i] = dst; 1466 } 1467 err = copy_func_state(dst, src->frame[i]); 1468 if (err) 1469 return err; 1470 } 1471 return 0; 1472 } 1473 1474 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1475 { 1476 while (st) { 1477 u32 br = --st->branches; 1478 1479 /* WARN_ON(br > 1) technically makes sense here, 1480 * but see comment in push_stack(), hence: 1481 */ 1482 WARN_ONCE((int)br < 0, 1483 "BUG update_branch_counts:branches_to_explore=%d\n", 1484 br); 1485 if (br) 1486 break; 1487 st = st->parent; 1488 } 1489 } 1490 1491 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1492 int *insn_idx, bool pop_log) 1493 { 1494 struct bpf_verifier_state *cur = env->cur_state; 1495 struct bpf_verifier_stack_elem *elem, *head = env->head; 1496 int err; 1497 1498 if (env->head == NULL) 1499 return -ENOENT; 1500 1501 if (cur) { 1502 err = copy_verifier_state(cur, &head->st); 1503 if (err) 1504 return err; 1505 } 1506 if (pop_log) 1507 bpf_vlog_reset(&env->log, head->log_pos); 1508 if (insn_idx) 1509 *insn_idx = head->insn_idx; 1510 if (prev_insn_idx) 1511 *prev_insn_idx = head->prev_insn_idx; 1512 elem = head->next; 1513 free_verifier_state(&head->st, false); 1514 kfree(head); 1515 env->head = elem; 1516 env->stack_size--; 1517 return 0; 1518 } 1519 1520 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1521 int insn_idx, int prev_insn_idx, 1522 bool speculative) 1523 { 1524 struct bpf_verifier_state *cur = env->cur_state; 1525 struct bpf_verifier_stack_elem *elem; 1526 int err; 1527 1528 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1529 if (!elem) 1530 goto err; 1531 1532 elem->insn_idx = insn_idx; 1533 elem->prev_insn_idx = prev_insn_idx; 1534 elem->next = env->head; 1535 elem->log_pos = env->log.len_used; 1536 env->head = elem; 1537 env->stack_size++; 1538 err = copy_verifier_state(&elem->st, cur); 1539 if (err) 1540 goto err; 1541 elem->st.speculative |= speculative; 1542 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1543 verbose(env, "The sequence of %d jumps is too complex.\n", 1544 env->stack_size); 1545 goto err; 1546 } 1547 if (elem->st.parent) { 1548 ++elem->st.parent->branches; 1549 /* WARN_ON(branches > 2) technically makes sense here, 1550 * but 1551 * 1. speculative states will bump 'branches' for non-branch 1552 * instructions 1553 * 2. is_state_visited() heuristics may decide not to create 1554 * a new state for a sequence of branches and all such current 1555 * and cloned states will be pointing to a single parent state 1556 * which might have large 'branches' count. 1557 */ 1558 } 1559 return &elem->st; 1560 err: 1561 free_verifier_state(env->cur_state, true); 1562 env->cur_state = NULL; 1563 /* pop all elements and return */ 1564 while (!pop_stack(env, NULL, NULL, false)); 1565 return NULL; 1566 } 1567 1568 #define CALLER_SAVED_REGS 6 1569 static const int caller_saved[CALLER_SAVED_REGS] = { 1570 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1571 }; 1572 1573 /* This helper doesn't clear reg->id */ 1574 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1575 { 1576 reg->var_off = tnum_const(imm); 1577 reg->smin_value = (s64)imm; 1578 reg->smax_value = (s64)imm; 1579 reg->umin_value = imm; 1580 reg->umax_value = imm; 1581 1582 reg->s32_min_value = (s32)imm; 1583 reg->s32_max_value = (s32)imm; 1584 reg->u32_min_value = (u32)imm; 1585 reg->u32_max_value = (u32)imm; 1586 } 1587 1588 /* Mark the unknown part of a register (variable offset or scalar value) as 1589 * known to have the value @imm. 1590 */ 1591 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1592 { 1593 /* Clear off and union(map_ptr, range) */ 1594 memset(((u8 *)reg) + sizeof(reg->type), 0, 1595 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1596 reg->id = 0; 1597 reg->ref_obj_id = 0; 1598 ___mark_reg_known(reg, imm); 1599 } 1600 1601 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1602 { 1603 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1604 reg->s32_min_value = (s32)imm; 1605 reg->s32_max_value = (s32)imm; 1606 reg->u32_min_value = (u32)imm; 1607 reg->u32_max_value = (u32)imm; 1608 } 1609 1610 /* Mark the 'variable offset' part of a register as zero. This should be 1611 * used only on registers holding a pointer type. 1612 */ 1613 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1614 { 1615 __mark_reg_known(reg, 0); 1616 } 1617 1618 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1619 { 1620 __mark_reg_known(reg, 0); 1621 reg->type = SCALAR_VALUE; 1622 } 1623 1624 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1625 struct bpf_reg_state *regs, u32 regno) 1626 { 1627 if (WARN_ON(regno >= MAX_BPF_REG)) { 1628 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1629 /* Something bad happened, let's kill all regs */ 1630 for (regno = 0; regno < MAX_BPF_REG; regno++) 1631 __mark_reg_not_init(env, regs + regno); 1632 return; 1633 } 1634 __mark_reg_known_zero(regs + regno); 1635 } 1636 1637 static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, 1638 bool first_slot, int dynptr_id) 1639 { 1640 /* reg->type has no meaning for STACK_DYNPTR, but when we set reg for 1641 * callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply 1642 * set it unconditionally as it is ignored for STACK_DYNPTR anyway. 1643 */ 1644 __mark_reg_known_zero(reg); 1645 reg->type = CONST_PTR_TO_DYNPTR; 1646 /* Give each dynptr a unique id to uniquely associate slices to it. */ 1647 reg->id = dynptr_id; 1648 reg->dynptr.type = type; 1649 reg->dynptr.first_slot = first_slot; 1650 } 1651 1652 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1653 { 1654 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1655 const struct bpf_map *map = reg->map_ptr; 1656 1657 if (map->inner_map_meta) { 1658 reg->type = CONST_PTR_TO_MAP; 1659 reg->map_ptr = map->inner_map_meta; 1660 /* transfer reg's id which is unique for every map_lookup_elem 1661 * as UID of the inner map. 1662 */ 1663 if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER)) 1664 reg->map_uid = reg->id; 1665 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1666 reg->type = PTR_TO_XDP_SOCK; 1667 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1668 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1669 reg->type = PTR_TO_SOCKET; 1670 } else { 1671 reg->type = PTR_TO_MAP_VALUE; 1672 } 1673 return; 1674 } 1675 1676 reg->type &= ~PTR_MAYBE_NULL; 1677 } 1678 1679 static void mark_reg_graph_node(struct bpf_reg_state *regs, u32 regno, 1680 struct btf_field_graph_root *ds_head) 1681 { 1682 __mark_reg_known_zero(®s[regno]); 1683 regs[regno].type = PTR_TO_BTF_ID | MEM_ALLOC; 1684 regs[regno].btf = ds_head->btf; 1685 regs[regno].btf_id = ds_head->value_btf_id; 1686 regs[regno].off = ds_head->node_offset; 1687 } 1688 1689 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1690 { 1691 return type_is_pkt_pointer(reg->type); 1692 } 1693 1694 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1695 { 1696 return reg_is_pkt_pointer(reg) || 1697 reg->type == PTR_TO_PACKET_END; 1698 } 1699 1700 static bool reg_is_dynptr_slice_pkt(const struct bpf_reg_state *reg) 1701 { 1702 return base_type(reg->type) == PTR_TO_MEM && 1703 (reg->type & DYNPTR_TYPE_SKB || reg->type & DYNPTR_TYPE_XDP); 1704 } 1705 1706 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1707 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1708 enum bpf_reg_type which) 1709 { 1710 /* The register can already have a range from prior markings. 1711 * This is fine as long as it hasn't been advanced from its 1712 * origin. 1713 */ 1714 return reg->type == which && 1715 reg->id == 0 && 1716 reg->off == 0 && 1717 tnum_equals_const(reg->var_off, 0); 1718 } 1719 1720 /* Reset the min/max bounds of a register */ 1721 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1722 { 1723 reg->smin_value = S64_MIN; 1724 reg->smax_value = S64_MAX; 1725 reg->umin_value = 0; 1726 reg->umax_value = U64_MAX; 1727 1728 reg->s32_min_value = S32_MIN; 1729 reg->s32_max_value = S32_MAX; 1730 reg->u32_min_value = 0; 1731 reg->u32_max_value = U32_MAX; 1732 } 1733 1734 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1735 { 1736 reg->smin_value = S64_MIN; 1737 reg->smax_value = S64_MAX; 1738 reg->umin_value = 0; 1739 reg->umax_value = U64_MAX; 1740 } 1741 1742 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1743 { 1744 reg->s32_min_value = S32_MIN; 1745 reg->s32_max_value = S32_MAX; 1746 reg->u32_min_value = 0; 1747 reg->u32_max_value = U32_MAX; 1748 } 1749 1750 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1751 { 1752 struct tnum var32_off = tnum_subreg(reg->var_off); 1753 1754 /* min signed is max(sign bit) | min(other bits) */ 1755 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1756 var32_off.value | (var32_off.mask & S32_MIN)); 1757 /* max signed is min(sign bit) | max(other bits) */ 1758 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1759 var32_off.value | (var32_off.mask & S32_MAX)); 1760 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1761 reg->u32_max_value = min(reg->u32_max_value, 1762 (u32)(var32_off.value | var32_off.mask)); 1763 } 1764 1765 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1766 { 1767 /* min signed is max(sign bit) | min(other bits) */ 1768 reg->smin_value = max_t(s64, reg->smin_value, 1769 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1770 /* max signed is min(sign bit) | max(other bits) */ 1771 reg->smax_value = min_t(s64, reg->smax_value, 1772 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1773 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1774 reg->umax_value = min(reg->umax_value, 1775 reg->var_off.value | reg->var_off.mask); 1776 } 1777 1778 static void __update_reg_bounds(struct bpf_reg_state *reg) 1779 { 1780 __update_reg32_bounds(reg); 1781 __update_reg64_bounds(reg); 1782 } 1783 1784 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1785 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1786 { 1787 /* Learn sign from signed bounds. 1788 * If we cannot cross the sign boundary, then signed and unsigned bounds 1789 * are the same, so combine. This works even in the negative case, e.g. 1790 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1791 */ 1792 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1793 reg->s32_min_value = reg->u32_min_value = 1794 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1795 reg->s32_max_value = reg->u32_max_value = 1796 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1797 return; 1798 } 1799 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1800 * boundary, so we must be careful. 1801 */ 1802 if ((s32)reg->u32_max_value >= 0) { 1803 /* Positive. We can't learn anything from the smin, but smax 1804 * is positive, hence safe. 1805 */ 1806 reg->s32_min_value = reg->u32_min_value; 1807 reg->s32_max_value = reg->u32_max_value = 1808 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1809 } else if ((s32)reg->u32_min_value < 0) { 1810 /* Negative. We can't learn anything from the smax, but smin 1811 * is negative, hence safe. 1812 */ 1813 reg->s32_min_value = reg->u32_min_value = 1814 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1815 reg->s32_max_value = reg->u32_max_value; 1816 } 1817 } 1818 1819 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1820 { 1821 /* Learn sign from signed bounds. 1822 * If we cannot cross the sign boundary, then signed and unsigned bounds 1823 * are the same, so combine. This works even in the negative case, e.g. 1824 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1825 */ 1826 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1827 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1828 reg->umin_value); 1829 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1830 reg->umax_value); 1831 return; 1832 } 1833 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1834 * boundary, so we must be careful. 1835 */ 1836 if ((s64)reg->umax_value >= 0) { 1837 /* Positive. We can't learn anything from the smin, but smax 1838 * is positive, hence safe. 1839 */ 1840 reg->smin_value = reg->umin_value; 1841 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1842 reg->umax_value); 1843 } else if ((s64)reg->umin_value < 0) { 1844 /* Negative. We can't learn anything from the smax, but smin 1845 * is negative, hence safe. 1846 */ 1847 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1848 reg->umin_value); 1849 reg->smax_value = reg->umax_value; 1850 } 1851 } 1852 1853 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1854 { 1855 __reg32_deduce_bounds(reg); 1856 __reg64_deduce_bounds(reg); 1857 } 1858 1859 /* Attempts to improve var_off based on unsigned min/max information */ 1860 static void __reg_bound_offset(struct bpf_reg_state *reg) 1861 { 1862 struct tnum var64_off = tnum_intersect(reg->var_off, 1863 tnum_range(reg->umin_value, 1864 reg->umax_value)); 1865 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1866 tnum_range(reg->u32_min_value, 1867 reg->u32_max_value)); 1868 1869 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1870 } 1871 1872 static void reg_bounds_sync(struct bpf_reg_state *reg) 1873 { 1874 /* We might have learned new bounds from the var_off. */ 1875 __update_reg_bounds(reg); 1876 /* We might have learned something about the sign bit. */ 1877 __reg_deduce_bounds(reg); 1878 /* We might have learned some bits from the bounds. */ 1879 __reg_bound_offset(reg); 1880 /* Intersecting with the old var_off might have improved our bounds 1881 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1882 * then new var_off is (0; 0x7f...fc) which improves our umax. 1883 */ 1884 __update_reg_bounds(reg); 1885 } 1886 1887 static bool __reg32_bound_s64(s32 a) 1888 { 1889 return a >= 0 && a <= S32_MAX; 1890 } 1891 1892 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1893 { 1894 reg->umin_value = reg->u32_min_value; 1895 reg->umax_value = reg->u32_max_value; 1896 1897 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 1898 * be positive otherwise set to worse case bounds and refine later 1899 * from tnum. 1900 */ 1901 if (__reg32_bound_s64(reg->s32_min_value) && 1902 __reg32_bound_s64(reg->s32_max_value)) { 1903 reg->smin_value = reg->s32_min_value; 1904 reg->smax_value = reg->s32_max_value; 1905 } else { 1906 reg->smin_value = 0; 1907 reg->smax_value = U32_MAX; 1908 } 1909 } 1910 1911 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1912 { 1913 /* special case when 64-bit register has upper 32-bit register 1914 * zeroed. Typically happens after zext or <<32, >>32 sequence 1915 * allowing us to use 32-bit bounds directly, 1916 */ 1917 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1918 __reg_assign_32_into_64(reg); 1919 } else { 1920 /* Otherwise the best we can do is push lower 32bit known and 1921 * unknown bits into register (var_off set from jmp logic) 1922 * then learn as much as possible from the 64-bit tnum 1923 * known and unknown bits. The previous smin/smax bounds are 1924 * invalid here because of jmp32 compare so mark them unknown 1925 * so they do not impact tnum bounds calculation. 1926 */ 1927 __mark_reg64_unbounded(reg); 1928 } 1929 reg_bounds_sync(reg); 1930 } 1931 1932 static bool __reg64_bound_s32(s64 a) 1933 { 1934 return a >= S32_MIN && a <= S32_MAX; 1935 } 1936 1937 static bool __reg64_bound_u32(u64 a) 1938 { 1939 return a >= U32_MIN && a <= U32_MAX; 1940 } 1941 1942 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1943 { 1944 __mark_reg32_unbounded(reg); 1945 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1946 reg->s32_min_value = (s32)reg->smin_value; 1947 reg->s32_max_value = (s32)reg->smax_value; 1948 } 1949 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1950 reg->u32_min_value = (u32)reg->umin_value; 1951 reg->u32_max_value = (u32)reg->umax_value; 1952 } 1953 reg_bounds_sync(reg); 1954 } 1955 1956 /* Mark a register as having a completely unknown (scalar) value. */ 1957 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1958 struct bpf_reg_state *reg) 1959 { 1960 /* 1961 * Clear type, off, and union(map_ptr, range) and 1962 * padding between 'type' and union 1963 */ 1964 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1965 reg->type = SCALAR_VALUE; 1966 reg->id = 0; 1967 reg->ref_obj_id = 0; 1968 reg->var_off = tnum_unknown; 1969 reg->frameno = 0; 1970 reg->precise = !env->bpf_capable; 1971 __mark_reg_unbounded(reg); 1972 } 1973 1974 static void mark_reg_unknown(struct bpf_verifier_env *env, 1975 struct bpf_reg_state *regs, u32 regno) 1976 { 1977 if (WARN_ON(regno >= MAX_BPF_REG)) { 1978 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1979 /* Something bad happened, let's kill all regs except FP */ 1980 for (regno = 0; regno < BPF_REG_FP; regno++) 1981 __mark_reg_not_init(env, regs + regno); 1982 return; 1983 } 1984 __mark_reg_unknown(env, regs + regno); 1985 } 1986 1987 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1988 struct bpf_reg_state *reg) 1989 { 1990 __mark_reg_unknown(env, reg); 1991 reg->type = NOT_INIT; 1992 } 1993 1994 static void mark_reg_not_init(struct bpf_verifier_env *env, 1995 struct bpf_reg_state *regs, u32 regno) 1996 { 1997 if (WARN_ON(regno >= MAX_BPF_REG)) { 1998 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1999 /* Something bad happened, let's kill all regs except FP */ 2000 for (regno = 0; regno < BPF_REG_FP; regno++) 2001 __mark_reg_not_init(env, regs + regno); 2002 return; 2003 } 2004 __mark_reg_not_init(env, regs + regno); 2005 } 2006 2007 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 2008 struct bpf_reg_state *regs, u32 regno, 2009 enum bpf_reg_type reg_type, 2010 struct btf *btf, u32 btf_id, 2011 enum bpf_type_flag flag) 2012 { 2013 if (reg_type == SCALAR_VALUE) { 2014 mark_reg_unknown(env, regs, regno); 2015 return; 2016 } 2017 mark_reg_known_zero(env, regs, regno); 2018 regs[regno].type = PTR_TO_BTF_ID | flag; 2019 regs[regno].btf = btf; 2020 regs[regno].btf_id = btf_id; 2021 } 2022 2023 #define DEF_NOT_SUBREG (0) 2024 static void init_reg_state(struct bpf_verifier_env *env, 2025 struct bpf_func_state *state) 2026 { 2027 struct bpf_reg_state *regs = state->regs; 2028 int i; 2029 2030 for (i = 0; i < MAX_BPF_REG; i++) { 2031 mark_reg_not_init(env, regs, i); 2032 regs[i].live = REG_LIVE_NONE; 2033 regs[i].parent = NULL; 2034 regs[i].subreg_def = DEF_NOT_SUBREG; 2035 } 2036 2037 /* frame pointer */ 2038 regs[BPF_REG_FP].type = PTR_TO_STACK; 2039 mark_reg_known_zero(env, regs, BPF_REG_FP); 2040 regs[BPF_REG_FP].frameno = state->frameno; 2041 } 2042 2043 #define BPF_MAIN_FUNC (-1) 2044 static void init_func_state(struct bpf_verifier_env *env, 2045 struct bpf_func_state *state, 2046 int callsite, int frameno, int subprogno) 2047 { 2048 state->callsite = callsite; 2049 state->frameno = frameno; 2050 state->subprogno = subprogno; 2051 state->callback_ret_range = tnum_range(0, 0); 2052 init_reg_state(env, state); 2053 mark_verifier_state_scratched(env); 2054 } 2055 2056 /* Similar to push_stack(), but for async callbacks */ 2057 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 2058 int insn_idx, int prev_insn_idx, 2059 int subprog) 2060 { 2061 struct bpf_verifier_stack_elem *elem; 2062 struct bpf_func_state *frame; 2063 2064 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 2065 if (!elem) 2066 goto err; 2067 2068 elem->insn_idx = insn_idx; 2069 elem->prev_insn_idx = prev_insn_idx; 2070 elem->next = env->head; 2071 elem->log_pos = env->log.len_used; 2072 env->head = elem; 2073 env->stack_size++; 2074 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 2075 verbose(env, 2076 "The sequence of %d jumps is too complex for async cb.\n", 2077 env->stack_size); 2078 goto err; 2079 } 2080 /* Unlike push_stack() do not copy_verifier_state(). 2081 * The caller state doesn't matter. 2082 * This is async callback. It starts in a fresh stack. 2083 * Initialize it similar to do_check_common(). 2084 */ 2085 elem->st.branches = 1; 2086 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 2087 if (!frame) 2088 goto err; 2089 init_func_state(env, frame, 2090 BPF_MAIN_FUNC /* callsite */, 2091 0 /* frameno within this callchain */, 2092 subprog /* subprog number within this prog */); 2093 elem->st.frame[0] = frame; 2094 return &elem->st; 2095 err: 2096 free_verifier_state(env->cur_state, true); 2097 env->cur_state = NULL; 2098 /* pop all elements and return */ 2099 while (!pop_stack(env, NULL, NULL, false)); 2100 return NULL; 2101 } 2102 2103 2104 enum reg_arg_type { 2105 SRC_OP, /* register is used as source operand */ 2106 DST_OP, /* register is used as destination operand */ 2107 DST_OP_NO_MARK /* same as above, check only, don't mark */ 2108 }; 2109 2110 static int cmp_subprogs(const void *a, const void *b) 2111 { 2112 return ((struct bpf_subprog_info *)a)->start - 2113 ((struct bpf_subprog_info *)b)->start; 2114 } 2115 2116 static int find_subprog(struct bpf_verifier_env *env, int off) 2117 { 2118 struct bpf_subprog_info *p; 2119 2120 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 2121 sizeof(env->subprog_info[0]), cmp_subprogs); 2122 if (!p) 2123 return -ENOENT; 2124 return p - env->subprog_info; 2125 2126 } 2127 2128 static int add_subprog(struct bpf_verifier_env *env, int off) 2129 { 2130 int insn_cnt = env->prog->len; 2131 int ret; 2132 2133 if (off >= insn_cnt || off < 0) { 2134 verbose(env, "call to invalid destination\n"); 2135 return -EINVAL; 2136 } 2137 ret = find_subprog(env, off); 2138 if (ret >= 0) 2139 return ret; 2140 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 2141 verbose(env, "too many subprograms\n"); 2142 return -E2BIG; 2143 } 2144 /* determine subprog starts. The end is one before the next starts */ 2145 env->subprog_info[env->subprog_cnt++].start = off; 2146 sort(env->subprog_info, env->subprog_cnt, 2147 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 2148 return env->subprog_cnt - 1; 2149 } 2150 2151 #define MAX_KFUNC_DESCS 256 2152 #define MAX_KFUNC_BTFS 256 2153 2154 struct bpf_kfunc_desc { 2155 struct btf_func_model func_model; 2156 u32 func_id; 2157 s32 imm; 2158 u16 offset; 2159 }; 2160 2161 struct bpf_kfunc_btf { 2162 struct btf *btf; 2163 struct module *module; 2164 u16 offset; 2165 }; 2166 2167 struct bpf_kfunc_desc_tab { 2168 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 2169 u32 nr_descs; 2170 }; 2171 2172 struct bpf_kfunc_btf_tab { 2173 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 2174 u32 nr_descs; 2175 }; 2176 2177 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 2178 { 2179 const struct bpf_kfunc_desc *d0 = a; 2180 const struct bpf_kfunc_desc *d1 = b; 2181 2182 /* func_id is not greater than BTF_MAX_TYPE */ 2183 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 2184 } 2185 2186 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 2187 { 2188 const struct bpf_kfunc_btf *d0 = a; 2189 const struct bpf_kfunc_btf *d1 = b; 2190 2191 return d0->offset - d1->offset; 2192 } 2193 2194 static const struct bpf_kfunc_desc * 2195 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 2196 { 2197 struct bpf_kfunc_desc desc = { 2198 .func_id = func_id, 2199 .offset = offset, 2200 }; 2201 struct bpf_kfunc_desc_tab *tab; 2202 2203 tab = prog->aux->kfunc_tab; 2204 return bsearch(&desc, tab->descs, tab->nr_descs, 2205 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 2206 } 2207 2208 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 2209 s16 offset) 2210 { 2211 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 2212 struct bpf_kfunc_btf_tab *tab; 2213 struct bpf_kfunc_btf *b; 2214 struct module *mod; 2215 struct btf *btf; 2216 int btf_fd; 2217 2218 tab = env->prog->aux->kfunc_btf_tab; 2219 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 2220 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 2221 if (!b) { 2222 if (tab->nr_descs == MAX_KFUNC_BTFS) { 2223 verbose(env, "too many different module BTFs\n"); 2224 return ERR_PTR(-E2BIG); 2225 } 2226 2227 if (bpfptr_is_null(env->fd_array)) { 2228 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 2229 return ERR_PTR(-EPROTO); 2230 } 2231 2232 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 2233 offset * sizeof(btf_fd), 2234 sizeof(btf_fd))) 2235 return ERR_PTR(-EFAULT); 2236 2237 btf = btf_get_by_fd(btf_fd); 2238 if (IS_ERR(btf)) { 2239 verbose(env, "invalid module BTF fd specified\n"); 2240 return btf; 2241 } 2242 2243 if (!btf_is_module(btf)) { 2244 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 2245 btf_put(btf); 2246 return ERR_PTR(-EINVAL); 2247 } 2248 2249 mod = btf_try_get_module(btf); 2250 if (!mod) { 2251 btf_put(btf); 2252 return ERR_PTR(-ENXIO); 2253 } 2254 2255 b = &tab->descs[tab->nr_descs++]; 2256 b->btf = btf; 2257 b->module = mod; 2258 b->offset = offset; 2259 2260 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2261 kfunc_btf_cmp_by_off, NULL); 2262 } 2263 return b->btf; 2264 } 2265 2266 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2267 { 2268 if (!tab) 2269 return; 2270 2271 while (tab->nr_descs--) { 2272 module_put(tab->descs[tab->nr_descs].module); 2273 btf_put(tab->descs[tab->nr_descs].btf); 2274 } 2275 kfree(tab); 2276 } 2277 2278 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2279 { 2280 if (offset) { 2281 if (offset < 0) { 2282 /* In the future, this can be allowed to increase limit 2283 * of fd index into fd_array, interpreted as u16. 2284 */ 2285 verbose(env, "negative offset disallowed for kernel module function call\n"); 2286 return ERR_PTR(-EINVAL); 2287 } 2288 2289 return __find_kfunc_desc_btf(env, offset); 2290 } 2291 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2292 } 2293 2294 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 2295 { 2296 const struct btf_type *func, *func_proto; 2297 struct bpf_kfunc_btf_tab *btf_tab; 2298 struct bpf_kfunc_desc_tab *tab; 2299 struct bpf_prog_aux *prog_aux; 2300 struct bpf_kfunc_desc *desc; 2301 const char *func_name; 2302 struct btf *desc_btf; 2303 unsigned long call_imm; 2304 unsigned long addr; 2305 int err; 2306 2307 prog_aux = env->prog->aux; 2308 tab = prog_aux->kfunc_tab; 2309 btf_tab = prog_aux->kfunc_btf_tab; 2310 if (!tab) { 2311 if (!btf_vmlinux) { 2312 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 2313 return -ENOTSUPP; 2314 } 2315 2316 if (!env->prog->jit_requested) { 2317 verbose(env, "JIT is required for calling kernel function\n"); 2318 return -ENOTSUPP; 2319 } 2320 2321 if (!bpf_jit_supports_kfunc_call()) { 2322 verbose(env, "JIT does not support calling kernel function\n"); 2323 return -ENOTSUPP; 2324 } 2325 2326 if (!env->prog->gpl_compatible) { 2327 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 2328 return -EINVAL; 2329 } 2330 2331 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 2332 if (!tab) 2333 return -ENOMEM; 2334 prog_aux->kfunc_tab = tab; 2335 } 2336 2337 /* func_id == 0 is always invalid, but instead of returning an error, be 2338 * conservative and wait until the code elimination pass before returning 2339 * error, so that invalid calls that get pruned out can be in BPF programs 2340 * loaded from userspace. It is also required that offset be untouched 2341 * for such calls. 2342 */ 2343 if (!func_id && !offset) 2344 return 0; 2345 2346 if (!btf_tab && offset) { 2347 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 2348 if (!btf_tab) 2349 return -ENOMEM; 2350 prog_aux->kfunc_btf_tab = btf_tab; 2351 } 2352 2353 desc_btf = find_kfunc_desc_btf(env, offset); 2354 if (IS_ERR(desc_btf)) { 2355 verbose(env, "failed to find BTF for kernel function\n"); 2356 return PTR_ERR(desc_btf); 2357 } 2358 2359 if (find_kfunc_desc(env->prog, func_id, offset)) 2360 return 0; 2361 2362 if (tab->nr_descs == MAX_KFUNC_DESCS) { 2363 verbose(env, "too many different kernel function calls\n"); 2364 return -E2BIG; 2365 } 2366 2367 func = btf_type_by_id(desc_btf, func_id); 2368 if (!func || !btf_type_is_func(func)) { 2369 verbose(env, "kernel btf_id %u is not a function\n", 2370 func_id); 2371 return -EINVAL; 2372 } 2373 func_proto = btf_type_by_id(desc_btf, func->type); 2374 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 2375 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 2376 func_id); 2377 return -EINVAL; 2378 } 2379 2380 func_name = btf_name_by_offset(desc_btf, func->name_off); 2381 addr = kallsyms_lookup_name(func_name); 2382 if (!addr) { 2383 verbose(env, "cannot find address for kernel function %s\n", 2384 func_name); 2385 return -EINVAL; 2386 } 2387 2388 call_imm = BPF_CALL_IMM(addr); 2389 /* Check whether or not the relative offset overflows desc->imm */ 2390 if ((unsigned long)(s32)call_imm != call_imm) { 2391 verbose(env, "address of kernel function %s is out of range\n", 2392 func_name); 2393 return -EINVAL; 2394 } 2395 2396 if (bpf_dev_bound_kfunc_id(func_id)) { 2397 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); 2398 if (err) 2399 return err; 2400 } 2401 2402 desc = &tab->descs[tab->nr_descs++]; 2403 desc->func_id = func_id; 2404 desc->imm = call_imm; 2405 desc->offset = offset; 2406 err = btf_distill_func_proto(&env->log, desc_btf, 2407 func_proto, func_name, 2408 &desc->func_model); 2409 if (!err) 2410 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2411 kfunc_desc_cmp_by_id_off, NULL); 2412 return err; 2413 } 2414 2415 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 2416 { 2417 const struct bpf_kfunc_desc *d0 = a; 2418 const struct bpf_kfunc_desc *d1 = b; 2419 2420 if (d0->imm > d1->imm) 2421 return 1; 2422 else if (d0->imm < d1->imm) 2423 return -1; 2424 return 0; 2425 } 2426 2427 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 2428 { 2429 struct bpf_kfunc_desc_tab *tab; 2430 2431 tab = prog->aux->kfunc_tab; 2432 if (!tab) 2433 return; 2434 2435 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2436 kfunc_desc_cmp_by_imm, NULL); 2437 } 2438 2439 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2440 { 2441 return !!prog->aux->kfunc_tab; 2442 } 2443 2444 const struct btf_func_model * 2445 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2446 const struct bpf_insn *insn) 2447 { 2448 const struct bpf_kfunc_desc desc = { 2449 .imm = insn->imm, 2450 }; 2451 const struct bpf_kfunc_desc *res; 2452 struct bpf_kfunc_desc_tab *tab; 2453 2454 tab = prog->aux->kfunc_tab; 2455 res = bsearch(&desc, tab->descs, tab->nr_descs, 2456 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 2457 2458 return res ? &res->func_model : NULL; 2459 } 2460 2461 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2462 { 2463 struct bpf_subprog_info *subprog = env->subprog_info; 2464 struct bpf_insn *insn = env->prog->insnsi; 2465 int i, ret, insn_cnt = env->prog->len; 2466 2467 /* Add entry function. */ 2468 ret = add_subprog(env, 0); 2469 if (ret) 2470 return ret; 2471 2472 for (i = 0; i < insn_cnt; i++, insn++) { 2473 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2474 !bpf_pseudo_kfunc_call(insn)) 2475 continue; 2476 2477 if (!env->bpf_capable) { 2478 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2479 return -EPERM; 2480 } 2481 2482 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2483 ret = add_subprog(env, i + insn->imm + 1); 2484 else 2485 ret = add_kfunc_call(env, insn->imm, insn->off); 2486 2487 if (ret < 0) 2488 return ret; 2489 } 2490 2491 /* Add a fake 'exit' subprog which could simplify subprog iteration 2492 * logic. 'subprog_cnt' should not be increased. 2493 */ 2494 subprog[env->subprog_cnt].start = insn_cnt; 2495 2496 if (env->log.level & BPF_LOG_LEVEL2) 2497 for (i = 0; i < env->subprog_cnt; i++) 2498 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2499 2500 return 0; 2501 } 2502 2503 static int check_subprogs(struct bpf_verifier_env *env) 2504 { 2505 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2506 struct bpf_subprog_info *subprog = env->subprog_info; 2507 struct bpf_insn *insn = env->prog->insnsi; 2508 int insn_cnt = env->prog->len; 2509 2510 /* now check that all jumps are within the same subprog */ 2511 subprog_start = subprog[cur_subprog].start; 2512 subprog_end = subprog[cur_subprog + 1].start; 2513 for (i = 0; i < insn_cnt; i++) { 2514 u8 code = insn[i].code; 2515 2516 if (code == (BPF_JMP | BPF_CALL) && 2517 insn[i].src_reg == 0 && 2518 insn[i].imm == BPF_FUNC_tail_call) 2519 subprog[cur_subprog].has_tail_call = true; 2520 if (BPF_CLASS(code) == BPF_LD && 2521 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2522 subprog[cur_subprog].has_ld_abs = true; 2523 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2524 goto next; 2525 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2526 goto next; 2527 off = i + insn[i].off + 1; 2528 if (off < subprog_start || off >= subprog_end) { 2529 verbose(env, "jump out of range from insn %d to %d\n", i, off); 2530 return -EINVAL; 2531 } 2532 next: 2533 if (i == subprog_end - 1) { 2534 /* to avoid fall-through from one subprog into another 2535 * the last insn of the subprog should be either exit 2536 * or unconditional jump back 2537 */ 2538 if (code != (BPF_JMP | BPF_EXIT) && 2539 code != (BPF_JMP | BPF_JA)) { 2540 verbose(env, "last insn is not an exit or jmp\n"); 2541 return -EINVAL; 2542 } 2543 subprog_start = subprog_end; 2544 cur_subprog++; 2545 if (cur_subprog < env->subprog_cnt) 2546 subprog_end = subprog[cur_subprog + 1].start; 2547 } 2548 } 2549 return 0; 2550 } 2551 2552 /* Parentage chain of this register (or stack slot) should take care of all 2553 * issues like callee-saved registers, stack slot allocation time, etc. 2554 */ 2555 static int mark_reg_read(struct bpf_verifier_env *env, 2556 const struct bpf_reg_state *state, 2557 struct bpf_reg_state *parent, u8 flag) 2558 { 2559 bool writes = parent == state->parent; /* Observe write marks */ 2560 int cnt = 0; 2561 2562 while (parent) { 2563 /* if read wasn't screened by an earlier write ... */ 2564 if (writes && state->live & REG_LIVE_WRITTEN) 2565 break; 2566 if (parent->live & REG_LIVE_DONE) { 2567 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 2568 reg_type_str(env, parent->type), 2569 parent->var_off.value, parent->off); 2570 return -EFAULT; 2571 } 2572 /* The first condition is more likely to be true than the 2573 * second, checked it first. 2574 */ 2575 if ((parent->live & REG_LIVE_READ) == flag || 2576 parent->live & REG_LIVE_READ64) 2577 /* The parentage chain never changes and 2578 * this parent was already marked as LIVE_READ. 2579 * There is no need to keep walking the chain again and 2580 * keep re-marking all parents as LIVE_READ. 2581 * This case happens when the same register is read 2582 * multiple times without writes into it in-between. 2583 * Also, if parent has the stronger REG_LIVE_READ64 set, 2584 * then no need to set the weak REG_LIVE_READ32. 2585 */ 2586 break; 2587 /* ... then we depend on parent's value */ 2588 parent->live |= flag; 2589 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 2590 if (flag == REG_LIVE_READ64) 2591 parent->live &= ~REG_LIVE_READ32; 2592 state = parent; 2593 parent = state->parent; 2594 writes = true; 2595 cnt++; 2596 } 2597 2598 if (env->longest_mark_read_walk < cnt) 2599 env->longest_mark_read_walk = cnt; 2600 return 0; 2601 } 2602 2603 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 2604 { 2605 struct bpf_func_state *state = func(env, reg); 2606 int spi, ret; 2607 2608 /* For CONST_PTR_TO_DYNPTR, it must have already been done by 2609 * check_reg_arg in check_helper_call and mark_btf_func_reg_size in 2610 * check_kfunc_call. 2611 */ 2612 if (reg->type == CONST_PTR_TO_DYNPTR) 2613 return 0; 2614 spi = dynptr_get_spi(env, reg); 2615 if (spi < 0) 2616 return spi; 2617 /* Caller ensures dynptr is valid and initialized, which means spi is in 2618 * bounds and spi is the first dynptr slot. Simply mark stack slot as 2619 * read. 2620 */ 2621 ret = mark_reg_read(env, &state->stack[spi].spilled_ptr, 2622 state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64); 2623 if (ret) 2624 return ret; 2625 return mark_reg_read(env, &state->stack[spi - 1].spilled_ptr, 2626 state->stack[spi - 1].spilled_ptr.parent, REG_LIVE_READ64); 2627 } 2628 2629 /* This function is supposed to be used by the following 32-bit optimization 2630 * code only. It returns TRUE if the source or destination register operates 2631 * on 64-bit, otherwise return FALSE. 2632 */ 2633 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 2634 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 2635 { 2636 u8 code, class, op; 2637 2638 code = insn->code; 2639 class = BPF_CLASS(code); 2640 op = BPF_OP(code); 2641 if (class == BPF_JMP) { 2642 /* BPF_EXIT for "main" will reach here. Return TRUE 2643 * conservatively. 2644 */ 2645 if (op == BPF_EXIT) 2646 return true; 2647 if (op == BPF_CALL) { 2648 /* BPF to BPF call will reach here because of marking 2649 * caller saved clobber with DST_OP_NO_MARK for which we 2650 * don't care the register def because they are anyway 2651 * marked as NOT_INIT already. 2652 */ 2653 if (insn->src_reg == BPF_PSEUDO_CALL) 2654 return false; 2655 /* Helper call will reach here because of arg type 2656 * check, conservatively return TRUE. 2657 */ 2658 if (t == SRC_OP) 2659 return true; 2660 2661 return false; 2662 } 2663 } 2664 2665 if (class == BPF_ALU64 || class == BPF_JMP || 2666 /* BPF_END always use BPF_ALU class. */ 2667 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 2668 return true; 2669 2670 if (class == BPF_ALU || class == BPF_JMP32) 2671 return false; 2672 2673 if (class == BPF_LDX) { 2674 if (t != SRC_OP) 2675 return BPF_SIZE(code) == BPF_DW; 2676 /* LDX source must be ptr. */ 2677 return true; 2678 } 2679 2680 if (class == BPF_STX) { 2681 /* BPF_STX (including atomic variants) has multiple source 2682 * operands, one of which is a ptr. Check whether the caller is 2683 * asking about it. 2684 */ 2685 if (t == SRC_OP && reg->type != SCALAR_VALUE) 2686 return true; 2687 return BPF_SIZE(code) == BPF_DW; 2688 } 2689 2690 if (class == BPF_LD) { 2691 u8 mode = BPF_MODE(code); 2692 2693 /* LD_IMM64 */ 2694 if (mode == BPF_IMM) 2695 return true; 2696 2697 /* Both LD_IND and LD_ABS return 32-bit data. */ 2698 if (t != SRC_OP) 2699 return false; 2700 2701 /* Implicit ctx ptr. */ 2702 if (regno == BPF_REG_6) 2703 return true; 2704 2705 /* Explicit source could be any width. */ 2706 return true; 2707 } 2708 2709 if (class == BPF_ST) 2710 /* The only source register for BPF_ST is a ptr. */ 2711 return true; 2712 2713 /* Conservatively return true at default. */ 2714 return true; 2715 } 2716 2717 /* Return the regno defined by the insn, or -1. */ 2718 static int insn_def_regno(const struct bpf_insn *insn) 2719 { 2720 switch (BPF_CLASS(insn->code)) { 2721 case BPF_JMP: 2722 case BPF_JMP32: 2723 case BPF_ST: 2724 return -1; 2725 case BPF_STX: 2726 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2727 (insn->imm & BPF_FETCH)) { 2728 if (insn->imm == BPF_CMPXCHG) 2729 return BPF_REG_0; 2730 else 2731 return insn->src_reg; 2732 } else { 2733 return -1; 2734 } 2735 default: 2736 return insn->dst_reg; 2737 } 2738 } 2739 2740 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 2741 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 2742 { 2743 int dst_reg = insn_def_regno(insn); 2744 2745 if (dst_reg == -1) 2746 return false; 2747 2748 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2749 } 2750 2751 static void mark_insn_zext(struct bpf_verifier_env *env, 2752 struct bpf_reg_state *reg) 2753 { 2754 s32 def_idx = reg->subreg_def; 2755 2756 if (def_idx == DEF_NOT_SUBREG) 2757 return; 2758 2759 env->insn_aux_data[def_idx - 1].zext_dst = true; 2760 /* The dst will be zero extended, so won't be sub-register anymore. */ 2761 reg->subreg_def = DEF_NOT_SUBREG; 2762 } 2763 2764 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2765 enum reg_arg_type t) 2766 { 2767 struct bpf_verifier_state *vstate = env->cur_state; 2768 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2769 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2770 struct bpf_reg_state *reg, *regs = state->regs; 2771 bool rw64; 2772 2773 if (regno >= MAX_BPF_REG) { 2774 verbose(env, "R%d is invalid\n", regno); 2775 return -EINVAL; 2776 } 2777 2778 mark_reg_scratched(env, regno); 2779 2780 reg = ®s[regno]; 2781 rw64 = is_reg64(env, insn, regno, reg, t); 2782 if (t == SRC_OP) { 2783 /* check whether register used as source operand can be read */ 2784 if (reg->type == NOT_INIT) { 2785 verbose(env, "R%d !read_ok\n", regno); 2786 return -EACCES; 2787 } 2788 /* We don't need to worry about FP liveness because it's read-only */ 2789 if (regno == BPF_REG_FP) 2790 return 0; 2791 2792 if (rw64) 2793 mark_insn_zext(env, reg); 2794 2795 return mark_reg_read(env, reg, reg->parent, 2796 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2797 } else { 2798 /* check whether register used as dest operand can be written to */ 2799 if (regno == BPF_REG_FP) { 2800 verbose(env, "frame pointer is read only\n"); 2801 return -EACCES; 2802 } 2803 reg->live |= REG_LIVE_WRITTEN; 2804 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2805 if (t == DST_OP) 2806 mark_reg_unknown(env, regs, regno); 2807 } 2808 return 0; 2809 } 2810 2811 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) 2812 { 2813 env->insn_aux_data[idx].jmp_point = true; 2814 } 2815 2816 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) 2817 { 2818 return env->insn_aux_data[insn_idx].jmp_point; 2819 } 2820 2821 /* for any branch, call, exit record the history of jmps in the given state */ 2822 static int push_jmp_history(struct bpf_verifier_env *env, 2823 struct bpf_verifier_state *cur) 2824 { 2825 u32 cnt = cur->jmp_history_cnt; 2826 struct bpf_idx_pair *p; 2827 size_t alloc_size; 2828 2829 if (!is_jmp_point(env, env->insn_idx)) 2830 return 0; 2831 2832 cnt++; 2833 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); 2834 p = krealloc(cur->jmp_history, alloc_size, GFP_USER); 2835 if (!p) 2836 return -ENOMEM; 2837 p[cnt - 1].idx = env->insn_idx; 2838 p[cnt - 1].prev_idx = env->prev_insn_idx; 2839 cur->jmp_history = p; 2840 cur->jmp_history_cnt = cnt; 2841 return 0; 2842 } 2843 2844 /* Backtrack one insn at a time. If idx is not at the top of recorded 2845 * history then previous instruction came from straight line execution. 2846 */ 2847 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2848 u32 *history) 2849 { 2850 u32 cnt = *history; 2851 2852 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2853 i = st->jmp_history[cnt - 1].prev_idx; 2854 (*history)--; 2855 } else { 2856 i--; 2857 } 2858 return i; 2859 } 2860 2861 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2862 { 2863 const struct btf_type *func; 2864 struct btf *desc_btf; 2865 2866 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2867 return NULL; 2868 2869 desc_btf = find_kfunc_desc_btf(data, insn->off); 2870 if (IS_ERR(desc_btf)) 2871 return "<error>"; 2872 2873 func = btf_type_by_id(desc_btf, insn->imm); 2874 return btf_name_by_offset(desc_btf, func->name_off); 2875 } 2876 2877 /* For given verifier state backtrack_insn() is called from the last insn to 2878 * the first insn. Its purpose is to compute a bitmask of registers and 2879 * stack slots that needs precision in the parent verifier state. 2880 */ 2881 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2882 u32 *reg_mask, u64 *stack_mask) 2883 { 2884 const struct bpf_insn_cbs cbs = { 2885 .cb_call = disasm_kfunc_name, 2886 .cb_print = verbose, 2887 .private_data = env, 2888 }; 2889 struct bpf_insn *insn = env->prog->insnsi + idx; 2890 u8 class = BPF_CLASS(insn->code); 2891 u8 opcode = BPF_OP(insn->code); 2892 u8 mode = BPF_MODE(insn->code); 2893 u32 dreg = 1u << insn->dst_reg; 2894 u32 sreg = 1u << insn->src_reg; 2895 u32 spi; 2896 2897 if (insn->code == 0) 2898 return 0; 2899 if (env->log.level & BPF_LOG_LEVEL2) { 2900 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2901 verbose(env, "%d: ", idx); 2902 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2903 } 2904 2905 if (class == BPF_ALU || class == BPF_ALU64) { 2906 if (!(*reg_mask & dreg)) 2907 return 0; 2908 if (opcode == BPF_MOV) { 2909 if (BPF_SRC(insn->code) == BPF_X) { 2910 /* dreg = sreg 2911 * dreg needs precision after this insn 2912 * sreg needs precision before this insn 2913 */ 2914 *reg_mask &= ~dreg; 2915 *reg_mask |= sreg; 2916 } else { 2917 /* dreg = K 2918 * dreg needs precision after this insn. 2919 * Corresponding register is already marked 2920 * as precise=true in this verifier state. 2921 * No further markings in parent are necessary 2922 */ 2923 *reg_mask &= ~dreg; 2924 } 2925 } else { 2926 if (BPF_SRC(insn->code) == BPF_X) { 2927 /* dreg += sreg 2928 * both dreg and sreg need precision 2929 * before this insn 2930 */ 2931 *reg_mask |= sreg; 2932 } /* else dreg += K 2933 * dreg still needs precision before this insn 2934 */ 2935 } 2936 } else if (class == BPF_LDX) { 2937 if (!(*reg_mask & dreg)) 2938 return 0; 2939 *reg_mask &= ~dreg; 2940 2941 /* scalars can only be spilled into stack w/o losing precision. 2942 * Load from any other memory can be zero extended. 2943 * The desire to keep that precision is already indicated 2944 * by 'precise' mark in corresponding register of this state. 2945 * No further tracking necessary. 2946 */ 2947 if (insn->src_reg != BPF_REG_FP) 2948 return 0; 2949 2950 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2951 * that [fp - off] slot contains scalar that needs to be 2952 * tracked with precision 2953 */ 2954 spi = (-insn->off - 1) / BPF_REG_SIZE; 2955 if (spi >= 64) { 2956 verbose(env, "BUG spi %d\n", spi); 2957 WARN_ONCE(1, "verifier backtracking bug"); 2958 return -EFAULT; 2959 } 2960 *stack_mask |= 1ull << spi; 2961 } else if (class == BPF_STX || class == BPF_ST) { 2962 if (*reg_mask & dreg) 2963 /* stx & st shouldn't be using _scalar_ dst_reg 2964 * to access memory. It means backtracking 2965 * encountered a case of pointer subtraction. 2966 */ 2967 return -ENOTSUPP; 2968 /* scalars can only be spilled into stack */ 2969 if (insn->dst_reg != BPF_REG_FP) 2970 return 0; 2971 spi = (-insn->off - 1) / BPF_REG_SIZE; 2972 if (spi >= 64) { 2973 verbose(env, "BUG spi %d\n", spi); 2974 WARN_ONCE(1, "verifier backtracking bug"); 2975 return -EFAULT; 2976 } 2977 if (!(*stack_mask & (1ull << spi))) 2978 return 0; 2979 *stack_mask &= ~(1ull << spi); 2980 if (class == BPF_STX) 2981 *reg_mask |= sreg; 2982 } else if (class == BPF_JMP || class == BPF_JMP32) { 2983 if (opcode == BPF_CALL) { 2984 if (insn->src_reg == BPF_PSEUDO_CALL) 2985 return -ENOTSUPP; 2986 /* BPF helpers that invoke callback subprogs are 2987 * equivalent to BPF_PSEUDO_CALL above 2988 */ 2989 if (insn->src_reg == 0 && is_callback_calling_function(insn->imm)) 2990 return -ENOTSUPP; 2991 /* kfunc with imm==0 is invalid and fixup_kfunc_call will 2992 * catch this error later. Make backtracking conservative 2993 * with ENOTSUPP. 2994 */ 2995 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0) 2996 return -ENOTSUPP; 2997 /* regular helper call sets R0 */ 2998 *reg_mask &= ~1; 2999 if (*reg_mask & 0x3f) { 3000 /* if backtracing was looking for registers R1-R5 3001 * they should have been found already. 3002 */ 3003 verbose(env, "BUG regs %x\n", *reg_mask); 3004 WARN_ONCE(1, "verifier backtracking bug"); 3005 return -EFAULT; 3006 } 3007 } else if (opcode == BPF_EXIT) { 3008 return -ENOTSUPP; 3009 } 3010 } else if (class == BPF_LD) { 3011 if (!(*reg_mask & dreg)) 3012 return 0; 3013 *reg_mask &= ~dreg; 3014 /* It's ld_imm64 or ld_abs or ld_ind. 3015 * For ld_imm64 no further tracking of precision 3016 * into parent is necessary 3017 */ 3018 if (mode == BPF_IND || mode == BPF_ABS) 3019 /* to be analyzed */ 3020 return -ENOTSUPP; 3021 } 3022 return 0; 3023 } 3024 3025 /* the scalar precision tracking algorithm: 3026 * . at the start all registers have precise=false. 3027 * . scalar ranges are tracked as normal through alu and jmp insns. 3028 * . once precise value of the scalar register is used in: 3029 * . ptr + scalar alu 3030 * . if (scalar cond K|scalar) 3031 * . helper_call(.., scalar, ...) where ARG_CONST is expected 3032 * backtrack through the verifier states and mark all registers and 3033 * stack slots with spilled constants that these scalar regisers 3034 * should be precise. 3035 * . during state pruning two registers (or spilled stack slots) 3036 * are equivalent if both are not precise. 3037 * 3038 * Note the verifier cannot simply walk register parentage chain, 3039 * since many different registers and stack slots could have been 3040 * used to compute single precise scalar. 3041 * 3042 * The approach of starting with precise=true for all registers and then 3043 * backtrack to mark a register as not precise when the verifier detects 3044 * that program doesn't care about specific value (e.g., when helper 3045 * takes register as ARG_ANYTHING parameter) is not safe. 3046 * 3047 * It's ok to walk single parentage chain of the verifier states. 3048 * It's possible that this backtracking will go all the way till 1st insn. 3049 * All other branches will be explored for needing precision later. 3050 * 3051 * The backtracking needs to deal with cases like: 3052 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 3053 * r9 -= r8 3054 * r5 = r9 3055 * if r5 > 0x79f goto pc+7 3056 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 3057 * r5 += 1 3058 * ... 3059 * call bpf_perf_event_output#25 3060 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 3061 * 3062 * and this case: 3063 * r6 = 1 3064 * call foo // uses callee's r6 inside to compute r0 3065 * r0 += r6 3066 * if r0 == 0 goto 3067 * 3068 * to track above reg_mask/stack_mask needs to be independent for each frame. 3069 * 3070 * Also if parent's curframe > frame where backtracking started, 3071 * the verifier need to mark registers in both frames, otherwise callees 3072 * may incorrectly prune callers. This is similar to 3073 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 3074 * 3075 * For now backtracking falls back into conservative marking. 3076 */ 3077 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 3078 struct bpf_verifier_state *st) 3079 { 3080 struct bpf_func_state *func; 3081 struct bpf_reg_state *reg; 3082 int i, j; 3083 3084 /* big hammer: mark all scalars precise in this path. 3085 * pop_stack may still get !precise scalars. 3086 * We also skip current state and go straight to first parent state, 3087 * because precision markings in current non-checkpointed state are 3088 * not needed. See why in the comment in __mark_chain_precision below. 3089 */ 3090 for (st = st->parent; st; st = st->parent) { 3091 for (i = 0; i <= st->curframe; i++) { 3092 func = st->frame[i]; 3093 for (j = 0; j < BPF_REG_FP; j++) { 3094 reg = &func->regs[j]; 3095 if (reg->type != SCALAR_VALUE) 3096 continue; 3097 reg->precise = true; 3098 } 3099 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3100 if (!is_spilled_reg(&func->stack[j])) 3101 continue; 3102 reg = &func->stack[j].spilled_ptr; 3103 if (reg->type != SCALAR_VALUE) 3104 continue; 3105 reg->precise = true; 3106 } 3107 } 3108 } 3109 } 3110 3111 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 3112 { 3113 struct bpf_func_state *func; 3114 struct bpf_reg_state *reg; 3115 int i, j; 3116 3117 for (i = 0; i <= st->curframe; i++) { 3118 func = st->frame[i]; 3119 for (j = 0; j < BPF_REG_FP; j++) { 3120 reg = &func->regs[j]; 3121 if (reg->type != SCALAR_VALUE) 3122 continue; 3123 reg->precise = false; 3124 } 3125 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 3126 if (!is_spilled_reg(&func->stack[j])) 3127 continue; 3128 reg = &func->stack[j].spilled_ptr; 3129 if (reg->type != SCALAR_VALUE) 3130 continue; 3131 reg->precise = false; 3132 } 3133 } 3134 } 3135 3136 /* 3137 * __mark_chain_precision() backtracks BPF program instruction sequence and 3138 * chain of verifier states making sure that register *regno* (if regno >= 0) 3139 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked 3140 * SCALARS, as well as any other registers and slots that contribute to 3141 * a tracked state of given registers/stack slots, depending on specific BPF 3142 * assembly instructions (see backtrack_insns() for exact instruction handling 3143 * logic). This backtracking relies on recorded jmp_history and is able to 3144 * traverse entire chain of parent states. This process ends only when all the 3145 * necessary registers/slots and their transitive dependencies are marked as 3146 * precise. 3147 * 3148 * One important and subtle aspect is that precise marks *do not matter* in 3149 * the currently verified state (current state). It is important to understand 3150 * why this is the case. 3151 * 3152 * First, note that current state is the state that is not yet "checkpointed", 3153 * i.e., it is not yet put into env->explored_states, and it has no children 3154 * states as well. It's ephemeral, and can end up either a) being discarded if 3155 * compatible explored state is found at some point or BPF_EXIT instruction is 3156 * reached or b) checkpointed and put into env->explored_states, branching out 3157 * into one or more children states. 3158 * 3159 * In the former case, precise markings in current state are completely 3160 * ignored by state comparison code (see regsafe() for details). Only 3161 * checkpointed ("old") state precise markings are important, and if old 3162 * state's register/slot is precise, regsafe() assumes current state's 3163 * register/slot as precise and checks value ranges exactly and precisely. If 3164 * states turn out to be compatible, current state's necessary precise 3165 * markings and any required parent states' precise markings are enforced 3166 * after the fact with propagate_precision() logic, after the fact. But it's 3167 * important to realize that in this case, even after marking current state 3168 * registers/slots as precise, we immediately discard current state. So what 3169 * actually matters is any of the precise markings propagated into current 3170 * state's parent states, which are always checkpointed (due to b) case above). 3171 * As such, for scenario a) it doesn't matter if current state has precise 3172 * markings set or not. 3173 * 3174 * Now, for the scenario b), checkpointing and forking into child(ren) 3175 * state(s). Note that before current state gets to checkpointing step, any 3176 * processed instruction always assumes precise SCALAR register/slot 3177 * knowledge: if precise value or range is useful to prune jump branch, BPF 3178 * verifier takes this opportunity enthusiastically. Similarly, when 3179 * register's value is used to calculate offset or memory address, exact 3180 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to 3181 * what we mentioned above about state comparison ignoring precise markings 3182 * during state comparison, BPF verifier ignores and also assumes precise 3183 * markings *at will* during instruction verification process. But as verifier 3184 * assumes precision, it also propagates any precision dependencies across 3185 * parent states, which are not yet finalized, so can be further restricted 3186 * based on new knowledge gained from restrictions enforced by their children 3187 * states. This is so that once those parent states are finalized, i.e., when 3188 * they have no more active children state, state comparison logic in 3189 * is_state_visited() would enforce strict and precise SCALAR ranges, if 3190 * required for correctness. 3191 * 3192 * To build a bit more intuition, note also that once a state is checkpointed, 3193 * the path we took to get to that state is not important. This is crucial 3194 * property for state pruning. When state is checkpointed and finalized at 3195 * some instruction index, it can be correctly and safely used to "short 3196 * circuit" any *compatible* state that reaches exactly the same instruction 3197 * index. I.e., if we jumped to that instruction from a completely different 3198 * code path than original finalized state was derived from, it doesn't 3199 * matter, current state can be discarded because from that instruction 3200 * forward having a compatible state will ensure we will safely reach the 3201 * exit. States describe preconditions for further exploration, but completely 3202 * forget the history of how we got here. 3203 * 3204 * This also means that even if we needed precise SCALAR range to get to 3205 * finalized state, but from that point forward *that same* SCALAR register is 3206 * never used in a precise context (i.e., it's precise value is not needed for 3207 * correctness), it's correct and safe to mark such register as "imprecise" 3208 * (i.e., precise marking set to false). This is what we rely on when we do 3209 * not set precise marking in current state. If no child state requires 3210 * precision for any given SCALAR register, it's safe to dictate that it can 3211 * be imprecise. If any child state does require this register to be precise, 3212 * we'll mark it precise later retroactively during precise markings 3213 * propagation from child state to parent states. 3214 * 3215 * Skipping precise marking setting in current state is a mild version of 3216 * relying on the above observation. But we can utilize this property even 3217 * more aggressively by proactively forgetting any precise marking in the 3218 * current state (which we inherited from the parent state), right before we 3219 * checkpoint it and branch off into new child state. This is done by 3220 * mark_all_scalars_imprecise() to hopefully get more permissive and generic 3221 * finalized states which help in short circuiting more future states. 3222 */ 3223 static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno, 3224 int spi) 3225 { 3226 struct bpf_verifier_state *st = env->cur_state; 3227 int first_idx = st->first_insn_idx; 3228 int last_idx = env->insn_idx; 3229 struct bpf_func_state *func; 3230 struct bpf_reg_state *reg; 3231 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 3232 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 3233 bool skip_first = true; 3234 bool new_marks = false; 3235 int i, err; 3236 3237 if (!env->bpf_capable) 3238 return 0; 3239 3240 /* Do sanity checks against current state of register and/or stack 3241 * slot, but don't set precise flag in current state, as precision 3242 * tracking in the current state is unnecessary. 3243 */ 3244 func = st->frame[frame]; 3245 if (regno >= 0) { 3246 reg = &func->regs[regno]; 3247 if (reg->type != SCALAR_VALUE) { 3248 WARN_ONCE(1, "backtracing misuse"); 3249 return -EFAULT; 3250 } 3251 new_marks = true; 3252 } 3253 3254 while (spi >= 0) { 3255 if (!is_spilled_reg(&func->stack[spi])) { 3256 stack_mask = 0; 3257 break; 3258 } 3259 reg = &func->stack[spi].spilled_ptr; 3260 if (reg->type != SCALAR_VALUE) { 3261 stack_mask = 0; 3262 break; 3263 } 3264 new_marks = true; 3265 break; 3266 } 3267 3268 if (!new_marks) 3269 return 0; 3270 if (!reg_mask && !stack_mask) 3271 return 0; 3272 3273 for (;;) { 3274 DECLARE_BITMAP(mask, 64); 3275 u32 history = st->jmp_history_cnt; 3276 3277 if (env->log.level & BPF_LOG_LEVEL2) 3278 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 3279 3280 if (last_idx < 0) { 3281 /* we are at the entry into subprog, which 3282 * is expected for global funcs, but only if 3283 * requested precise registers are R1-R5 3284 * (which are global func's input arguments) 3285 */ 3286 if (st->curframe == 0 && 3287 st->frame[0]->subprogno > 0 && 3288 st->frame[0]->callsite == BPF_MAIN_FUNC && 3289 stack_mask == 0 && (reg_mask & ~0x3e) == 0) { 3290 bitmap_from_u64(mask, reg_mask); 3291 for_each_set_bit(i, mask, 32) { 3292 reg = &st->frame[0]->regs[i]; 3293 if (reg->type != SCALAR_VALUE) { 3294 reg_mask &= ~(1u << i); 3295 continue; 3296 } 3297 reg->precise = true; 3298 } 3299 return 0; 3300 } 3301 3302 verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n", 3303 st->frame[0]->subprogno, reg_mask, stack_mask); 3304 WARN_ONCE(1, "verifier backtracking bug"); 3305 return -EFAULT; 3306 } 3307 3308 for (i = last_idx;;) { 3309 if (skip_first) { 3310 err = 0; 3311 skip_first = false; 3312 } else { 3313 err = backtrack_insn(env, i, ®_mask, &stack_mask); 3314 } 3315 if (err == -ENOTSUPP) { 3316 mark_all_scalars_precise(env, st); 3317 return 0; 3318 } else if (err) { 3319 return err; 3320 } 3321 if (!reg_mask && !stack_mask) 3322 /* Found assignment(s) into tracked register in this state. 3323 * Since this state is already marked, just return. 3324 * Nothing to be tracked further in the parent state. 3325 */ 3326 return 0; 3327 if (i == first_idx) 3328 break; 3329 i = get_prev_insn_idx(st, i, &history); 3330 if (i >= env->prog->len) { 3331 /* This can happen if backtracking reached insn 0 3332 * and there are still reg_mask or stack_mask 3333 * to backtrack. 3334 * It means the backtracking missed the spot where 3335 * particular register was initialized with a constant. 3336 */ 3337 verbose(env, "BUG backtracking idx %d\n", i); 3338 WARN_ONCE(1, "verifier backtracking bug"); 3339 return -EFAULT; 3340 } 3341 } 3342 st = st->parent; 3343 if (!st) 3344 break; 3345 3346 new_marks = false; 3347 func = st->frame[frame]; 3348 bitmap_from_u64(mask, reg_mask); 3349 for_each_set_bit(i, mask, 32) { 3350 reg = &func->regs[i]; 3351 if (reg->type != SCALAR_VALUE) { 3352 reg_mask &= ~(1u << i); 3353 continue; 3354 } 3355 if (!reg->precise) 3356 new_marks = true; 3357 reg->precise = true; 3358 } 3359 3360 bitmap_from_u64(mask, stack_mask); 3361 for_each_set_bit(i, mask, 64) { 3362 if (i >= func->allocated_stack / BPF_REG_SIZE) { 3363 /* the sequence of instructions: 3364 * 2: (bf) r3 = r10 3365 * 3: (7b) *(u64 *)(r3 -8) = r0 3366 * 4: (79) r4 = *(u64 *)(r10 -8) 3367 * doesn't contain jmps. It's backtracked 3368 * as a single block. 3369 * During backtracking insn 3 is not recognized as 3370 * stack access, so at the end of backtracking 3371 * stack slot fp-8 is still marked in stack_mask. 3372 * However the parent state may not have accessed 3373 * fp-8 and it's "unallocated" stack space. 3374 * In such case fallback to conservative. 3375 */ 3376 mark_all_scalars_precise(env, st); 3377 return 0; 3378 } 3379 3380 if (!is_spilled_reg(&func->stack[i])) { 3381 stack_mask &= ~(1ull << i); 3382 continue; 3383 } 3384 reg = &func->stack[i].spilled_ptr; 3385 if (reg->type != SCALAR_VALUE) { 3386 stack_mask &= ~(1ull << i); 3387 continue; 3388 } 3389 if (!reg->precise) 3390 new_marks = true; 3391 reg->precise = true; 3392 } 3393 if (env->log.level & BPF_LOG_LEVEL2) { 3394 verbose(env, "parent %s regs=%x stack=%llx marks:", 3395 new_marks ? "didn't have" : "already had", 3396 reg_mask, stack_mask); 3397 print_verifier_state(env, func, true); 3398 } 3399 3400 if (!reg_mask && !stack_mask) 3401 break; 3402 if (!new_marks) 3403 break; 3404 3405 last_idx = st->last_insn_idx; 3406 first_idx = st->first_insn_idx; 3407 } 3408 return 0; 3409 } 3410 3411 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 3412 { 3413 return __mark_chain_precision(env, env->cur_state->curframe, regno, -1); 3414 } 3415 3416 static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno) 3417 { 3418 return __mark_chain_precision(env, frame, regno, -1); 3419 } 3420 3421 static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi) 3422 { 3423 return __mark_chain_precision(env, frame, -1, spi); 3424 } 3425 3426 static bool is_spillable_regtype(enum bpf_reg_type type) 3427 { 3428 switch (base_type(type)) { 3429 case PTR_TO_MAP_VALUE: 3430 case PTR_TO_STACK: 3431 case PTR_TO_CTX: 3432 case PTR_TO_PACKET: 3433 case PTR_TO_PACKET_META: 3434 case PTR_TO_PACKET_END: 3435 case PTR_TO_FLOW_KEYS: 3436 case CONST_PTR_TO_MAP: 3437 case PTR_TO_SOCKET: 3438 case PTR_TO_SOCK_COMMON: 3439 case PTR_TO_TCP_SOCK: 3440 case PTR_TO_XDP_SOCK: 3441 case PTR_TO_BTF_ID: 3442 case PTR_TO_BUF: 3443 case PTR_TO_MEM: 3444 case PTR_TO_FUNC: 3445 case PTR_TO_MAP_KEY: 3446 return true; 3447 default: 3448 return false; 3449 } 3450 } 3451 3452 /* Does this register contain a constant zero? */ 3453 static bool register_is_null(struct bpf_reg_state *reg) 3454 { 3455 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 3456 } 3457 3458 static bool register_is_const(struct bpf_reg_state *reg) 3459 { 3460 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 3461 } 3462 3463 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 3464 { 3465 return tnum_is_unknown(reg->var_off) && 3466 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 3467 reg->umin_value == 0 && reg->umax_value == U64_MAX && 3468 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 3469 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 3470 } 3471 3472 static bool register_is_bounded(struct bpf_reg_state *reg) 3473 { 3474 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 3475 } 3476 3477 static bool __is_pointer_value(bool allow_ptr_leaks, 3478 const struct bpf_reg_state *reg) 3479 { 3480 if (allow_ptr_leaks) 3481 return false; 3482 3483 return reg->type != SCALAR_VALUE; 3484 } 3485 3486 /* Copy src state preserving dst->parent and dst->live fields */ 3487 static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src) 3488 { 3489 struct bpf_reg_state *parent = dst->parent; 3490 enum bpf_reg_liveness live = dst->live; 3491 3492 *dst = *src; 3493 dst->parent = parent; 3494 dst->live = live; 3495 } 3496 3497 static void save_register_state(struct bpf_func_state *state, 3498 int spi, struct bpf_reg_state *reg, 3499 int size) 3500 { 3501 int i; 3502 3503 copy_register_state(&state->stack[spi].spilled_ptr, reg); 3504 if (size == BPF_REG_SIZE) 3505 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3506 3507 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 3508 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 3509 3510 /* size < 8 bytes spill */ 3511 for (; i; i--) 3512 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); 3513 } 3514 3515 static bool is_bpf_st_mem(struct bpf_insn *insn) 3516 { 3517 return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM; 3518 } 3519 3520 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 3521 * stack boundary and alignment are checked in check_mem_access() 3522 */ 3523 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 3524 /* stack frame we're writing to */ 3525 struct bpf_func_state *state, 3526 int off, int size, int value_regno, 3527 int insn_idx) 3528 { 3529 struct bpf_func_state *cur; /* state of the current function */ 3530 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 3531 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3532 struct bpf_reg_state *reg = NULL; 3533 u32 dst_reg = insn->dst_reg; 3534 3535 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 3536 if (err) 3537 return err; 3538 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 3539 * so it's aligned access and [off, off + size) are within stack limits 3540 */ 3541 if (!env->allow_ptr_leaks && 3542 state->stack[spi].slot_type[0] == STACK_SPILL && 3543 size != BPF_REG_SIZE) { 3544 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 3545 return -EACCES; 3546 } 3547 3548 cur = env->cur_state->frame[env->cur_state->curframe]; 3549 if (value_regno >= 0) 3550 reg = &cur->regs[value_regno]; 3551 if (!env->bypass_spec_v4) { 3552 bool sanitize = reg && is_spillable_regtype(reg->type); 3553 3554 for (i = 0; i < size; i++) { 3555 u8 type = state->stack[spi].slot_type[i]; 3556 3557 if (type != STACK_MISC && type != STACK_ZERO) { 3558 sanitize = true; 3559 break; 3560 } 3561 } 3562 3563 if (sanitize) 3564 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 3565 } 3566 3567 err = destroy_if_dynptr_stack_slot(env, state, spi); 3568 if (err) 3569 return err; 3570 3571 mark_stack_slot_scratched(env, spi); 3572 if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && 3573 !register_is_null(reg) && env->bpf_capable) { 3574 if (dst_reg != BPF_REG_FP) { 3575 /* The backtracking logic can only recognize explicit 3576 * stack slot address like [fp - 8]. Other spill of 3577 * scalar via different register has to be conservative. 3578 * Backtrack from here and mark all registers as precise 3579 * that contributed into 'reg' being a constant. 3580 */ 3581 err = mark_chain_precision(env, value_regno); 3582 if (err) 3583 return err; 3584 } 3585 save_register_state(state, spi, reg, size); 3586 } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && 3587 insn->imm != 0 && env->bpf_capable) { 3588 struct bpf_reg_state fake_reg = {}; 3589 3590 __mark_reg_known(&fake_reg, (u32)insn->imm); 3591 fake_reg.type = SCALAR_VALUE; 3592 save_register_state(state, spi, &fake_reg, size); 3593 } else if (reg && is_spillable_regtype(reg->type)) { 3594 /* register containing pointer is being spilled into stack */ 3595 if (size != BPF_REG_SIZE) { 3596 verbose_linfo(env, insn_idx, "; "); 3597 verbose(env, "invalid size of register spill\n"); 3598 return -EACCES; 3599 } 3600 if (state != cur && reg->type == PTR_TO_STACK) { 3601 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 3602 return -EINVAL; 3603 } 3604 save_register_state(state, spi, reg, size); 3605 } else { 3606 u8 type = STACK_MISC; 3607 3608 /* regular write of data into stack destroys any spilled ptr */ 3609 state->stack[spi].spilled_ptr.type = NOT_INIT; 3610 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 3611 if (is_spilled_reg(&state->stack[spi])) 3612 for (i = 0; i < BPF_REG_SIZE; i++) 3613 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 3614 3615 /* only mark the slot as written if all 8 bytes were written 3616 * otherwise read propagation may incorrectly stop too soon 3617 * when stack slots are partially written. 3618 * This heuristic means that read propagation will be 3619 * conservative, since it will add reg_live_read marks 3620 * to stack slots all the way to first state when programs 3621 * writes+reads less than 8 bytes 3622 */ 3623 if (size == BPF_REG_SIZE) 3624 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3625 3626 /* when we zero initialize stack slots mark them as such */ 3627 if ((reg && register_is_null(reg)) || 3628 (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) { 3629 /* backtracking doesn't work for STACK_ZERO yet. */ 3630 err = mark_chain_precision(env, value_regno); 3631 if (err) 3632 return err; 3633 type = STACK_ZERO; 3634 } 3635 3636 /* Mark slots affected by this stack write. */ 3637 for (i = 0; i < size; i++) 3638 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 3639 type; 3640 } 3641 return 0; 3642 } 3643 3644 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 3645 * known to contain a variable offset. 3646 * This function checks whether the write is permitted and conservatively 3647 * tracks the effects of the write, considering that each stack slot in the 3648 * dynamic range is potentially written to. 3649 * 3650 * 'off' includes 'regno->off'. 3651 * 'value_regno' can be -1, meaning that an unknown value is being written to 3652 * the stack. 3653 * 3654 * Spilled pointers in range are not marked as written because we don't know 3655 * what's going to be actually written. This means that read propagation for 3656 * future reads cannot be terminated by this write. 3657 * 3658 * For privileged programs, uninitialized stack slots are considered 3659 * initialized by this write (even though we don't know exactly what offsets 3660 * are going to be written to). The idea is that we don't want the verifier to 3661 * reject future reads that access slots written to through variable offsets. 3662 */ 3663 static int check_stack_write_var_off(struct bpf_verifier_env *env, 3664 /* func where register points to */ 3665 struct bpf_func_state *state, 3666 int ptr_regno, int off, int size, 3667 int value_regno, int insn_idx) 3668 { 3669 struct bpf_func_state *cur; /* state of the current function */ 3670 int min_off, max_off; 3671 int i, err; 3672 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 3673 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3674 bool writing_zero = false; 3675 /* set if the fact that we're writing a zero is used to let any 3676 * stack slots remain STACK_ZERO 3677 */ 3678 bool zero_used = false; 3679 3680 cur = env->cur_state->frame[env->cur_state->curframe]; 3681 ptr_reg = &cur->regs[ptr_regno]; 3682 min_off = ptr_reg->smin_value + off; 3683 max_off = ptr_reg->smax_value + off + size; 3684 if (value_regno >= 0) 3685 value_reg = &cur->regs[value_regno]; 3686 if ((value_reg && register_is_null(value_reg)) || 3687 (!value_reg && is_bpf_st_mem(insn) && insn->imm == 0)) 3688 writing_zero = true; 3689 3690 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 3691 if (err) 3692 return err; 3693 3694 for (i = min_off; i < max_off; i++) { 3695 int spi; 3696 3697 spi = __get_spi(i); 3698 err = destroy_if_dynptr_stack_slot(env, state, spi); 3699 if (err) 3700 return err; 3701 } 3702 3703 /* Variable offset writes destroy any spilled pointers in range. */ 3704 for (i = min_off; i < max_off; i++) { 3705 u8 new_type, *stype; 3706 int slot, spi; 3707 3708 slot = -i - 1; 3709 spi = slot / BPF_REG_SIZE; 3710 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3711 mark_stack_slot_scratched(env, spi); 3712 3713 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 3714 /* Reject the write if range we may write to has not 3715 * been initialized beforehand. If we didn't reject 3716 * here, the ptr status would be erased below (even 3717 * though not all slots are actually overwritten), 3718 * possibly opening the door to leaks. 3719 * 3720 * We do however catch STACK_INVALID case below, and 3721 * only allow reading possibly uninitialized memory 3722 * later for CAP_PERFMON, as the write may not happen to 3723 * that slot. 3724 */ 3725 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 3726 insn_idx, i); 3727 return -EINVAL; 3728 } 3729 3730 /* Erase all spilled pointers. */ 3731 state->stack[spi].spilled_ptr.type = NOT_INIT; 3732 3733 /* Update the slot type. */ 3734 new_type = STACK_MISC; 3735 if (writing_zero && *stype == STACK_ZERO) { 3736 new_type = STACK_ZERO; 3737 zero_used = true; 3738 } 3739 /* If the slot is STACK_INVALID, we check whether it's OK to 3740 * pretend that it will be initialized by this write. The slot 3741 * might not actually be written to, and so if we mark it as 3742 * initialized future reads might leak uninitialized memory. 3743 * For privileged programs, we will accept such reads to slots 3744 * that may or may not be written because, if we're reject 3745 * them, the error would be too confusing. 3746 */ 3747 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 3748 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 3749 insn_idx, i); 3750 return -EINVAL; 3751 } 3752 *stype = new_type; 3753 } 3754 if (zero_used) { 3755 /* backtracking doesn't work for STACK_ZERO yet. */ 3756 err = mark_chain_precision(env, value_regno); 3757 if (err) 3758 return err; 3759 } 3760 return 0; 3761 } 3762 3763 /* When register 'dst_regno' is assigned some values from stack[min_off, 3764 * max_off), we set the register's type according to the types of the 3765 * respective stack slots. If all the stack values are known to be zeros, then 3766 * so is the destination reg. Otherwise, the register is considered to be 3767 * SCALAR. This function does not deal with register filling; the caller must 3768 * ensure that all spilled registers in the stack range have been marked as 3769 * read. 3770 */ 3771 static void mark_reg_stack_read(struct bpf_verifier_env *env, 3772 /* func where src register points to */ 3773 struct bpf_func_state *ptr_state, 3774 int min_off, int max_off, int dst_regno) 3775 { 3776 struct bpf_verifier_state *vstate = env->cur_state; 3777 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3778 int i, slot, spi; 3779 u8 *stype; 3780 int zeros = 0; 3781 3782 for (i = min_off; i < max_off; i++) { 3783 slot = -i - 1; 3784 spi = slot / BPF_REG_SIZE; 3785 stype = ptr_state->stack[spi].slot_type; 3786 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 3787 break; 3788 zeros++; 3789 } 3790 if (zeros == max_off - min_off) { 3791 /* any access_size read into register is zero extended, 3792 * so the whole register == const_zero 3793 */ 3794 __mark_reg_const_zero(&state->regs[dst_regno]); 3795 /* backtracking doesn't support STACK_ZERO yet, 3796 * so mark it precise here, so that later 3797 * backtracking can stop here. 3798 * Backtracking may not need this if this register 3799 * doesn't participate in pointer adjustment. 3800 * Forward propagation of precise flag is not 3801 * necessary either. This mark is only to stop 3802 * backtracking. Any register that contributed 3803 * to const 0 was marked precise before spill. 3804 */ 3805 state->regs[dst_regno].precise = true; 3806 } else { 3807 /* have read misc data from the stack */ 3808 mark_reg_unknown(env, state->regs, dst_regno); 3809 } 3810 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3811 } 3812 3813 /* Read the stack at 'off' and put the results into the register indicated by 3814 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 3815 * spilled reg. 3816 * 3817 * 'dst_regno' can be -1, meaning that the read value is not going to a 3818 * register. 3819 * 3820 * The access is assumed to be within the current stack bounds. 3821 */ 3822 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 3823 /* func where src register points to */ 3824 struct bpf_func_state *reg_state, 3825 int off, int size, int dst_regno) 3826 { 3827 struct bpf_verifier_state *vstate = env->cur_state; 3828 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3829 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 3830 struct bpf_reg_state *reg; 3831 u8 *stype, type; 3832 3833 stype = reg_state->stack[spi].slot_type; 3834 reg = ®_state->stack[spi].spilled_ptr; 3835 3836 if (is_spilled_reg(®_state->stack[spi])) { 3837 u8 spill_size = 1; 3838 3839 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 3840 spill_size++; 3841 3842 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 3843 if (reg->type != SCALAR_VALUE) { 3844 verbose_linfo(env, env->insn_idx, "; "); 3845 verbose(env, "invalid size of register fill\n"); 3846 return -EACCES; 3847 } 3848 3849 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3850 if (dst_regno < 0) 3851 return 0; 3852 3853 if (!(off % BPF_REG_SIZE) && size == spill_size) { 3854 /* The earlier check_reg_arg() has decided the 3855 * subreg_def for this insn. Save it first. 3856 */ 3857 s32 subreg_def = state->regs[dst_regno].subreg_def; 3858 3859 copy_register_state(&state->regs[dst_regno], reg); 3860 state->regs[dst_regno].subreg_def = subreg_def; 3861 } else { 3862 for (i = 0; i < size; i++) { 3863 type = stype[(slot - i) % BPF_REG_SIZE]; 3864 if (type == STACK_SPILL) 3865 continue; 3866 if (type == STACK_MISC) 3867 continue; 3868 if (type == STACK_INVALID && env->allow_uninit_stack) 3869 continue; 3870 verbose(env, "invalid read from stack off %d+%d size %d\n", 3871 off, i, size); 3872 return -EACCES; 3873 } 3874 mark_reg_unknown(env, state->regs, dst_regno); 3875 } 3876 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3877 return 0; 3878 } 3879 3880 if (dst_regno >= 0) { 3881 /* restore register state from stack */ 3882 copy_register_state(&state->regs[dst_regno], reg); 3883 /* mark reg as written since spilled pointer state likely 3884 * has its liveness marks cleared by is_state_visited() 3885 * which resets stack/reg liveness for state transitions 3886 */ 3887 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3888 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 3889 /* If dst_regno==-1, the caller is asking us whether 3890 * it is acceptable to use this value as a SCALAR_VALUE 3891 * (e.g. for XADD). 3892 * We must not allow unprivileged callers to do that 3893 * with spilled pointers. 3894 */ 3895 verbose(env, "leaking pointer from stack off %d\n", 3896 off); 3897 return -EACCES; 3898 } 3899 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3900 } else { 3901 for (i = 0; i < size; i++) { 3902 type = stype[(slot - i) % BPF_REG_SIZE]; 3903 if (type == STACK_MISC) 3904 continue; 3905 if (type == STACK_ZERO) 3906 continue; 3907 if (type == STACK_INVALID && env->allow_uninit_stack) 3908 continue; 3909 verbose(env, "invalid read from stack off %d+%d size %d\n", 3910 off, i, size); 3911 return -EACCES; 3912 } 3913 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3914 if (dst_regno >= 0) 3915 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 3916 } 3917 return 0; 3918 } 3919 3920 enum bpf_access_src { 3921 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 3922 ACCESS_HELPER = 2, /* the access is performed by a helper */ 3923 }; 3924 3925 static int check_stack_range_initialized(struct bpf_verifier_env *env, 3926 int regno, int off, int access_size, 3927 bool zero_size_allowed, 3928 enum bpf_access_src type, 3929 struct bpf_call_arg_meta *meta); 3930 3931 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 3932 { 3933 return cur_regs(env) + regno; 3934 } 3935 3936 /* Read the stack at 'ptr_regno + off' and put the result into the register 3937 * 'dst_regno'. 3938 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 3939 * but not its variable offset. 3940 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 3941 * 3942 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 3943 * filling registers (i.e. reads of spilled register cannot be detected when 3944 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 3945 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 3946 * offset; for a fixed offset check_stack_read_fixed_off should be used 3947 * instead. 3948 */ 3949 static int check_stack_read_var_off(struct bpf_verifier_env *env, 3950 int ptr_regno, int off, int size, int dst_regno) 3951 { 3952 /* The state of the source register. */ 3953 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3954 struct bpf_func_state *ptr_state = func(env, reg); 3955 int err; 3956 int min_off, max_off; 3957 3958 /* Note that we pass a NULL meta, so raw access will not be permitted. 3959 */ 3960 err = check_stack_range_initialized(env, ptr_regno, off, size, 3961 false, ACCESS_DIRECT, NULL); 3962 if (err) 3963 return err; 3964 3965 min_off = reg->smin_value + off; 3966 max_off = reg->smax_value + off; 3967 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3968 return 0; 3969 } 3970 3971 /* check_stack_read dispatches to check_stack_read_fixed_off or 3972 * check_stack_read_var_off. 3973 * 3974 * The caller must ensure that the offset falls within the allocated stack 3975 * bounds. 3976 * 3977 * 'dst_regno' is a register which will receive the value from the stack. It 3978 * can be -1, meaning that the read value is not going to a register. 3979 */ 3980 static int check_stack_read(struct bpf_verifier_env *env, 3981 int ptr_regno, int off, int size, 3982 int dst_regno) 3983 { 3984 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3985 struct bpf_func_state *state = func(env, reg); 3986 int err; 3987 /* Some accesses are only permitted with a static offset. */ 3988 bool var_off = !tnum_is_const(reg->var_off); 3989 3990 /* The offset is required to be static when reads don't go to a 3991 * register, in order to not leak pointers (see 3992 * check_stack_read_fixed_off). 3993 */ 3994 if (dst_regno < 0 && var_off) { 3995 char tn_buf[48]; 3996 3997 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3998 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3999 tn_buf, off, size); 4000 return -EACCES; 4001 } 4002 /* Variable offset is prohibited for unprivileged mode for simplicity 4003 * since it requires corresponding support in Spectre masking for stack 4004 * ALU. See also retrieve_ptr_limit(). 4005 */ 4006 if (!env->bypass_spec_v1 && var_off) { 4007 char tn_buf[48]; 4008 4009 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4010 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 4011 ptr_regno, tn_buf); 4012 return -EACCES; 4013 } 4014 4015 if (!var_off) { 4016 off += reg->var_off.value; 4017 err = check_stack_read_fixed_off(env, state, off, size, 4018 dst_regno); 4019 } else { 4020 /* Variable offset stack reads need more conservative handling 4021 * than fixed offset ones. Note that dst_regno >= 0 on this 4022 * branch. 4023 */ 4024 err = check_stack_read_var_off(env, ptr_regno, off, size, 4025 dst_regno); 4026 } 4027 return err; 4028 } 4029 4030 4031 /* check_stack_write dispatches to check_stack_write_fixed_off or 4032 * check_stack_write_var_off. 4033 * 4034 * 'ptr_regno' is the register used as a pointer into the stack. 4035 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 4036 * 'value_regno' is the register whose value we're writing to the stack. It can 4037 * be -1, meaning that we're not writing from a register. 4038 * 4039 * The caller must ensure that the offset falls within the maximum stack size. 4040 */ 4041 static int check_stack_write(struct bpf_verifier_env *env, 4042 int ptr_regno, int off, int size, 4043 int value_regno, int insn_idx) 4044 { 4045 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 4046 struct bpf_func_state *state = func(env, reg); 4047 int err; 4048 4049 if (tnum_is_const(reg->var_off)) { 4050 off += reg->var_off.value; 4051 err = check_stack_write_fixed_off(env, state, off, size, 4052 value_regno, insn_idx); 4053 } else { 4054 /* Variable offset stack reads need more conservative handling 4055 * than fixed offset ones. 4056 */ 4057 err = check_stack_write_var_off(env, state, 4058 ptr_regno, off, size, 4059 value_regno, insn_idx); 4060 } 4061 return err; 4062 } 4063 4064 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 4065 int off, int size, enum bpf_access_type type) 4066 { 4067 struct bpf_reg_state *regs = cur_regs(env); 4068 struct bpf_map *map = regs[regno].map_ptr; 4069 u32 cap = bpf_map_flags_to_cap(map); 4070 4071 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 4072 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 4073 map->value_size, off, size); 4074 return -EACCES; 4075 } 4076 4077 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 4078 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 4079 map->value_size, off, size); 4080 return -EACCES; 4081 } 4082 4083 return 0; 4084 } 4085 4086 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 4087 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 4088 int off, int size, u32 mem_size, 4089 bool zero_size_allowed) 4090 { 4091 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 4092 struct bpf_reg_state *reg; 4093 4094 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 4095 return 0; 4096 4097 reg = &cur_regs(env)[regno]; 4098 switch (reg->type) { 4099 case PTR_TO_MAP_KEY: 4100 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 4101 mem_size, off, size); 4102 break; 4103 case PTR_TO_MAP_VALUE: 4104 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 4105 mem_size, off, size); 4106 break; 4107 case PTR_TO_PACKET: 4108 case PTR_TO_PACKET_META: 4109 case PTR_TO_PACKET_END: 4110 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 4111 off, size, regno, reg->id, off, mem_size); 4112 break; 4113 case PTR_TO_MEM: 4114 default: 4115 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 4116 mem_size, off, size); 4117 } 4118 4119 return -EACCES; 4120 } 4121 4122 /* check read/write into a memory region with possible variable offset */ 4123 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 4124 int off, int size, u32 mem_size, 4125 bool zero_size_allowed) 4126 { 4127 struct bpf_verifier_state *vstate = env->cur_state; 4128 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4129 struct bpf_reg_state *reg = &state->regs[regno]; 4130 int err; 4131 4132 /* We may have adjusted the register pointing to memory region, so we 4133 * need to try adding each of min_value and max_value to off 4134 * to make sure our theoretical access will be safe. 4135 * 4136 * The minimum value is only important with signed 4137 * comparisons where we can't assume the floor of a 4138 * value is 0. If we are using signed variables for our 4139 * index'es we need to make sure that whatever we use 4140 * will have a set floor within our range. 4141 */ 4142 if (reg->smin_value < 0 && 4143 (reg->smin_value == S64_MIN || 4144 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 4145 reg->smin_value + off < 0)) { 4146 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4147 regno); 4148 return -EACCES; 4149 } 4150 err = __check_mem_access(env, regno, reg->smin_value + off, size, 4151 mem_size, zero_size_allowed); 4152 if (err) { 4153 verbose(env, "R%d min value is outside of the allowed memory range\n", 4154 regno); 4155 return err; 4156 } 4157 4158 /* If we haven't set a max value then we need to bail since we can't be 4159 * sure we won't do bad things. 4160 * If reg->umax_value + off could overflow, treat that as unbounded too. 4161 */ 4162 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 4163 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 4164 regno); 4165 return -EACCES; 4166 } 4167 err = __check_mem_access(env, regno, reg->umax_value + off, size, 4168 mem_size, zero_size_allowed); 4169 if (err) { 4170 verbose(env, "R%d max value is outside of the allowed memory range\n", 4171 regno); 4172 return err; 4173 } 4174 4175 return 0; 4176 } 4177 4178 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 4179 const struct bpf_reg_state *reg, int regno, 4180 bool fixed_off_ok) 4181 { 4182 /* Access to this pointer-typed register or passing it to a helper 4183 * is only allowed in its original, unmodified form. 4184 */ 4185 4186 if (reg->off < 0) { 4187 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 4188 reg_type_str(env, reg->type), regno, reg->off); 4189 return -EACCES; 4190 } 4191 4192 if (!fixed_off_ok && reg->off) { 4193 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 4194 reg_type_str(env, reg->type), regno, reg->off); 4195 return -EACCES; 4196 } 4197 4198 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4199 char tn_buf[48]; 4200 4201 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4202 verbose(env, "variable %s access var_off=%s disallowed\n", 4203 reg_type_str(env, reg->type), tn_buf); 4204 return -EACCES; 4205 } 4206 4207 return 0; 4208 } 4209 4210 int check_ptr_off_reg(struct bpf_verifier_env *env, 4211 const struct bpf_reg_state *reg, int regno) 4212 { 4213 return __check_ptr_off_reg(env, reg, regno, false); 4214 } 4215 4216 static int map_kptr_match_type(struct bpf_verifier_env *env, 4217 struct btf_field *kptr_field, 4218 struct bpf_reg_state *reg, u32 regno) 4219 { 4220 const char *targ_name = kernel_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 4221 int perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED; 4222 const char *reg_name = ""; 4223 4224 /* Only unreferenced case accepts untrusted pointers */ 4225 if (kptr_field->type == BPF_KPTR_UNREF) 4226 perm_flags |= PTR_UNTRUSTED; 4227 4228 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 4229 goto bad_type; 4230 4231 if (!btf_is_kernel(reg->btf)) { 4232 verbose(env, "R%d must point to kernel BTF\n", regno); 4233 return -EINVAL; 4234 } 4235 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 4236 reg_name = kernel_type_name(reg->btf, reg->btf_id); 4237 4238 /* For ref_ptr case, release function check should ensure we get one 4239 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 4240 * normal store of unreferenced kptr, we must ensure var_off is zero. 4241 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 4242 * reg->off and reg->ref_obj_id are not needed here. 4243 */ 4244 if (__check_ptr_off_reg(env, reg, regno, true)) 4245 return -EACCES; 4246 4247 /* A full type match is needed, as BTF can be vmlinux or module BTF, and 4248 * we also need to take into account the reg->off. 4249 * 4250 * We want to support cases like: 4251 * 4252 * struct foo { 4253 * struct bar br; 4254 * struct baz bz; 4255 * }; 4256 * 4257 * struct foo *v; 4258 * v = func(); // PTR_TO_BTF_ID 4259 * val->foo = v; // reg->off is zero, btf and btf_id match type 4260 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 4261 * // first member type of struct after comparison fails 4262 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 4263 * // to match type 4264 * 4265 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 4266 * is zero. We must also ensure that btf_struct_ids_match does not walk 4267 * the struct to match type against first member of struct, i.e. reject 4268 * second case from above. Hence, when type is BPF_KPTR_REF, we set 4269 * strict mode to true for type match. 4270 */ 4271 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 4272 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 4273 kptr_field->type == BPF_KPTR_REF)) 4274 goto bad_type; 4275 return 0; 4276 bad_type: 4277 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 4278 reg_type_str(env, reg->type), reg_name); 4279 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 4280 if (kptr_field->type == BPF_KPTR_UNREF) 4281 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 4282 targ_name); 4283 else 4284 verbose(env, "\n"); 4285 return -EINVAL; 4286 } 4287 4288 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 4289 int value_regno, int insn_idx, 4290 struct btf_field *kptr_field) 4291 { 4292 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 4293 int class = BPF_CLASS(insn->code); 4294 struct bpf_reg_state *val_reg; 4295 4296 /* Things we already checked for in check_map_access and caller: 4297 * - Reject cases where variable offset may touch kptr 4298 * - size of access (must be BPF_DW) 4299 * - tnum_is_const(reg->var_off) 4300 * - kptr_field->offset == off + reg->var_off.value 4301 */ 4302 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 4303 if (BPF_MODE(insn->code) != BPF_MEM) { 4304 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 4305 return -EACCES; 4306 } 4307 4308 /* We only allow loading referenced kptr, since it will be marked as 4309 * untrusted, similar to unreferenced kptr. 4310 */ 4311 if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 4312 verbose(env, "store to referenced kptr disallowed\n"); 4313 return -EACCES; 4314 } 4315 4316 if (class == BPF_LDX) { 4317 val_reg = reg_state(env, value_regno); 4318 /* We can simply mark the value_regno receiving the pointer 4319 * value from map as PTR_TO_BTF_ID, with the correct type. 4320 */ 4321 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 4322 kptr_field->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 4323 /* For mark_ptr_or_null_reg */ 4324 val_reg->id = ++env->id_gen; 4325 } else if (class == BPF_STX) { 4326 val_reg = reg_state(env, value_regno); 4327 if (!register_is_null(val_reg) && 4328 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 4329 return -EACCES; 4330 } else if (class == BPF_ST) { 4331 if (insn->imm) { 4332 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 4333 kptr_field->offset); 4334 return -EACCES; 4335 } 4336 } else { 4337 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 4338 return -EACCES; 4339 } 4340 return 0; 4341 } 4342 4343 /* check read/write into a map element with possible variable offset */ 4344 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 4345 int off, int size, bool zero_size_allowed, 4346 enum bpf_access_src src) 4347 { 4348 struct bpf_verifier_state *vstate = env->cur_state; 4349 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4350 struct bpf_reg_state *reg = &state->regs[regno]; 4351 struct bpf_map *map = reg->map_ptr; 4352 struct btf_record *rec; 4353 int err, i; 4354 4355 err = check_mem_region_access(env, regno, off, size, map->value_size, 4356 zero_size_allowed); 4357 if (err) 4358 return err; 4359 4360 if (IS_ERR_OR_NULL(map->record)) 4361 return 0; 4362 rec = map->record; 4363 for (i = 0; i < rec->cnt; i++) { 4364 struct btf_field *field = &rec->fields[i]; 4365 u32 p = field->offset; 4366 4367 /* If any part of a field can be touched by load/store, reject 4368 * this program. To check that [x1, x2) overlaps with [y1, y2), 4369 * it is sufficient to check x1 < y2 && y1 < x2. 4370 */ 4371 if (reg->smin_value + off < p + btf_field_type_size(field->type) && 4372 p < reg->umax_value + off + size) { 4373 switch (field->type) { 4374 case BPF_KPTR_UNREF: 4375 case BPF_KPTR_REF: 4376 if (src != ACCESS_DIRECT) { 4377 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 4378 return -EACCES; 4379 } 4380 if (!tnum_is_const(reg->var_off)) { 4381 verbose(env, "kptr access cannot have variable offset\n"); 4382 return -EACCES; 4383 } 4384 if (p != off + reg->var_off.value) { 4385 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 4386 p, off + reg->var_off.value); 4387 return -EACCES; 4388 } 4389 if (size != bpf_size_to_bytes(BPF_DW)) { 4390 verbose(env, "kptr access size must be BPF_DW\n"); 4391 return -EACCES; 4392 } 4393 break; 4394 default: 4395 verbose(env, "%s cannot be accessed directly by load/store\n", 4396 btf_field_type_name(field->type)); 4397 return -EACCES; 4398 } 4399 } 4400 } 4401 return 0; 4402 } 4403 4404 #define MAX_PACKET_OFF 0xffff 4405 4406 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 4407 const struct bpf_call_arg_meta *meta, 4408 enum bpf_access_type t) 4409 { 4410 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 4411 4412 switch (prog_type) { 4413 /* Program types only with direct read access go here! */ 4414 case BPF_PROG_TYPE_LWT_IN: 4415 case BPF_PROG_TYPE_LWT_OUT: 4416 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 4417 case BPF_PROG_TYPE_SK_REUSEPORT: 4418 case BPF_PROG_TYPE_FLOW_DISSECTOR: 4419 case BPF_PROG_TYPE_CGROUP_SKB: 4420 if (t == BPF_WRITE) 4421 return false; 4422 fallthrough; 4423 4424 /* Program types with direct read + write access go here! */ 4425 case BPF_PROG_TYPE_SCHED_CLS: 4426 case BPF_PROG_TYPE_SCHED_ACT: 4427 case BPF_PROG_TYPE_XDP: 4428 case BPF_PROG_TYPE_LWT_XMIT: 4429 case BPF_PROG_TYPE_SK_SKB: 4430 case BPF_PROG_TYPE_SK_MSG: 4431 if (meta) 4432 return meta->pkt_access; 4433 4434 env->seen_direct_write = true; 4435 return true; 4436 4437 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4438 if (t == BPF_WRITE) 4439 env->seen_direct_write = true; 4440 4441 return true; 4442 4443 default: 4444 return false; 4445 } 4446 } 4447 4448 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 4449 int size, bool zero_size_allowed) 4450 { 4451 struct bpf_reg_state *regs = cur_regs(env); 4452 struct bpf_reg_state *reg = ®s[regno]; 4453 int err; 4454 4455 /* We may have added a variable offset to the packet pointer; but any 4456 * reg->range we have comes after that. We are only checking the fixed 4457 * offset. 4458 */ 4459 4460 /* We don't allow negative numbers, because we aren't tracking enough 4461 * detail to prove they're safe. 4462 */ 4463 if (reg->smin_value < 0) { 4464 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4465 regno); 4466 return -EACCES; 4467 } 4468 4469 err = reg->range < 0 ? -EINVAL : 4470 __check_mem_access(env, regno, off, size, reg->range, 4471 zero_size_allowed); 4472 if (err) { 4473 verbose(env, "R%d offset is outside of the packet\n", regno); 4474 return err; 4475 } 4476 4477 /* __check_mem_access has made sure "off + size - 1" is within u16. 4478 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 4479 * otherwise find_good_pkt_pointers would have refused to set range info 4480 * that __check_mem_access would have rejected this pkt access. 4481 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 4482 */ 4483 env->prog->aux->max_pkt_offset = 4484 max_t(u32, env->prog->aux->max_pkt_offset, 4485 off + reg->umax_value + size - 1); 4486 4487 return err; 4488 } 4489 4490 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 4491 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 4492 enum bpf_access_type t, enum bpf_reg_type *reg_type, 4493 struct btf **btf, u32 *btf_id) 4494 { 4495 struct bpf_insn_access_aux info = { 4496 .reg_type = *reg_type, 4497 .log = &env->log, 4498 }; 4499 4500 if (env->ops->is_valid_access && 4501 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 4502 /* A non zero info.ctx_field_size indicates that this field is a 4503 * candidate for later verifier transformation to load the whole 4504 * field and then apply a mask when accessed with a narrower 4505 * access than actual ctx access size. A zero info.ctx_field_size 4506 * will only allow for whole field access and rejects any other 4507 * type of narrower access. 4508 */ 4509 *reg_type = info.reg_type; 4510 4511 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 4512 *btf = info.btf; 4513 *btf_id = info.btf_id; 4514 } else { 4515 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 4516 } 4517 /* remember the offset of last byte accessed in ctx */ 4518 if (env->prog->aux->max_ctx_offset < off + size) 4519 env->prog->aux->max_ctx_offset = off + size; 4520 return 0; 4521 } 4522 4523 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 4524 return -EACCES; 4525 } 4526 4527 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 4528 int size) 4529 { 4530 if (size < 0 || off < 0 || 4531 (u64)off + size > sizeof(struct bpf_flow_keys)) { 4532 verbose(env, "invalid access to flow keys off=%d size=%d\n", 4533 off, size); 4534 return -EACCES; 4535 } 4536 return 0; 4537 } 4538 4539 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 4540 u32 regno, int off, int size, 4541 enum bpf_access_type t) 4542 { 4543 struct bpf_reg_state *regs = cur_regs(env); 4544 struct bpf_reg_state *reg = ®s[regno]; 4545 struct bpf_insn_access_aux info = {}; 4546 bool valid; 4547 4548 if (reg->smin_value < 0) { 4549 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4550 regno); 4551 return -EACCES; 4552 } 4553 4554 switch (reg->type) { 4555 case PTR_TO_SOCK_COMMON: 4556 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 4557 break; 4558 case PTR_TO_SOCKET: 4559 valid = bpf_sock_is_valid_access(off, size, t, &info); 4560 break; 4561 case PTR_TO_TCP_SOCK: 4562 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 4563 break; 4564 case PTR_TO_XDP_SOCK: 4565 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 4566 break; 4567 default: 4568 valid = false; 4569 } 4570 4571 4572 if (valid) { 4573 env->insn_aux_data[insn_idx].ctx_field_size = 4574 info.ctx_field_size; 4575 return 0; 4576 } 4577 4578 verbose(env, "R%d invalid %s access off=%d size=%d\n", 4579 regno, reg_type_str(env, reg->type), off, size); 4580 4581 return -EACCES; 4582 } 4583 4584 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 4585 { 4586 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 4587 } 4588 4589 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 4590 { 4591 const struct bpf_reg_state *reg = reg_state(env, regno); 4592 4593 return reg->type == PTR_TO_CTX; 4594 } 4595 4596 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 4597 { 4598 const struct bpf_reg_state *reg = reg_state(env, regno); 4599 4600 return type_is_sk_pointer(reg->type); 4601 } 4602 4603 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 4604 { 4605 const struct bpf_reg_state *reg = reg_state(env, regno); 4606 4607 return type_is_pkt_pointer(reg->type); 4608 } 4609 4610 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 4611 { 4612 const struct bpf_reg_state *reg = reg_state(env, regno); 4613 4614 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 4615 return reg->type == PTR_TO_FLOW_KEYS; 4616 } 4617 4618 static bool is_trusted_reg(const struct bpf_reg_state *reg) 4619 { 4620 /* A referenced register is always trusted. */ 4621 if (reg->ref_obj_id) 4622 return true; 4623 4624 /* If a register is not referenced, it is trusted if it has the 4625 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 4626 * other type modifiers may be safe, but we elect to take an opt-in 4627 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 4628 * not. 4629 * 4630 * Eventually, we should make PTR_TRUSTED the single source of truth 4631 * for whether a register is trusted. 4632 */ 4633 return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && 4634 !bpf_type_has_unsafe_modifiers(reg->type); 4635 } 4636 4637 static bool is_rcu_reg(const struct bpf_reg_state *reg) 4638 { 4639 return reg->type & MEM_RCU; 4640 } 4641 4642 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 4643 const struct bpf_reg_state *reg, 4644 int off, int size, bool strict) 4645 { 4646 struct tnum reg_off; 4647 int ip_align; 4648 4649 /* Byte size accesses are always allowed. */ 4650 if (!strict || size == 1) 4651 return 0; 4652 4653 /* For platforms that do not have a Kconfig enabling 4654 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 4655 * NET_IP_ALIGN is universally set to '2'. And on platforms 4656 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 4657 * to this code only in strict mode where we want to emulate 4658 * the NET_IP_ALIGN==2 checking. Therefore use an 4659 * unconditional IP align value of '2'. 4660 */ 4661 ip_align = 2; 4662 4663 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 4664 if (!tnum_is_aligned(reg_off, size)) { 4665 char tn_buf[48]; 4666 4667 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4668 verbose(env, 4669 "misaligned packet access off %d+%s+%d+%d size %d\n", 4670 ip_align, tn_buf, reg->off, off, size); 4671 return -EACCES; 4672 } 4673 4674 return 0; 4675 } 4676 4677 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 4678 const struct bpf_reg_state *reg, 4679 const char *pointer_desc, 4680 int off, int size, bool strict) 4681 { 4682 struct tnum reg_off; 4683 4684 /* Byte size accesses are always allowed. */ 4685 if (!strict || size == 1) 4686 return 0; 4687 4688 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 4689 if (!tnum_is_aligned(reg_off, size)) { 4690 char tn_buf[48]; 4691 4692 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4693 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 4694 pointer_desc, tn_buf, reg->off, off, size); 4695 return -EACCES; 4696 } 4697 4698 return 0; 4699 } 4700 4701 static int check_ptr_alignment(struct bpf_verifier_env *env, 4702 const struct bpf_reg_state *reg, int off, 4703 int size, bool strict_alignment_once) 4704 { 4705 bool strict = env->strict_alignment || strict_alignment_once; 4706 const char *pointer_desc = ""; 4707 4708 switch (reg->type) { 4709 case PTR_TO_PACKET: 4710 case PTR_TO_PACKET_META: 4711 /* Special case, because of NET_IP_ALIGN. Given metadata sits 4712 * right in front, treat it the very same way. 4713 */ 4714 return check_pkt_ptr_alignment(env, reg, off, size, strict); 4715 case PTR_TO_FLOW_KEYS: 4716 pointer_desc = "flow keys "; 4717 break; 4718 case PTR_TO_MAP_KEY: 4719 pointer_desc = "key "; 4720 break; 4721 case PTR_TO_MAP_VALUE: 4722 pointer_desc = "value "; 4723 break; 4724 case PTR_TO_CTX: 4725 pointer_desc = "context "; 4726 break; 4727 case PTR_TO_STACK: 4728 pointer_desc = "stack "; 4729 /* The stack spill tracking logic in check_stack_write_fixed_off() 4730 * and check_stack_read_fixed_off() relies on stack accesses being 4731 * aligned. 4732 */ 4733 strict = true; 4734 break; 4735 case PTR_TO_SOCKET: 4736 pointer_desc = "sock "; 4737 break; 4738 case PTR_TO_SOCK_COMMON: 4739 pointer_desc = "sock_common "; 4740 break; 4741 case PTR_TO_TCP_SOCK: 4742 pointer_desc = "tcp_sock "; 4743 break; 4744 case PTR_TO_XDP_SOCK: 4745 pointer_desc = "xdp_sock "; 4746 break; 4747 default: 4748 break; 4749 } 4750 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 4751 strict); 4752 } 4753 4754 static int update_stack_depth(struct bpf_verifier_env *env, 4755 const struct bpf_func_state *func, 4756 int off) 4757 { 4758 u16 stack = env->subprog_info[func->subprogno].stack_depth; 4759 4760 if (stack >= -off) 4761 return 0; 4762 4763 /* update known max for given subprogram */ 4764 env->subprog_info[func->subprogno].stack_depth = -off; 4765 return 0; 4766 } 4767 4768 /* starting from main bpf function walk all instructions of the function 4769 * and recursively walk all callees that given function can call. 4770 * Ignore jump and exit insns. 4771 * Since recursion is prevented by check_cfg() this algorithm 4772 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 4773 */ 4774 static int check_max_stack_depth(struct bpf_verifier_env *env) 4775 { 4776 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 4777 struct bpf_subprog_info *subprog = env->subprog_info; 4778 struct bpf_insn *insn = env->prog->insnsi; 4779 bool tail_call_reachable = false; 4780 int ret_insn[MAX_CALL_FRAMES]; 4781 int ret_prog[MAX_CALL_FRAMES]; 4782 int j; 4783 4784 process_func: 4785 /* protect against potential stack overflow that might happen when 4786 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 4787 * depth for such case down to 256 so that the worst case scenario 4788 * would result in 8k stack size (32 which is tailcall limit * 256 = 4789 * 8k). 4790 * 4791 * To get the idea what might happen, see an example: 4792 * func1 -> sub rsp, 128 4793 * subfunc1 -> sub rsp, 256 4794 * tailcall1 -> add rsp, 256 4795 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 4796 * subfunc2 -> sub rsp, 64 4797 * subfunc22 -> sub rsp, 128 4798 * tailcall2 -> add rsp, 128 4799 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 4800 * 4801 * tailcall will unwind the current stack frame but it will not get rid 4802 * of caller's stack as shown on the example above. 4803 */ 4804 if (idx && subprog[idx].has_tail_call && depth >= 256) { 4805 verbose(env, 4806 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 4807 depth); 4808 return -EACCES; 4809 } 4810 /* round up to 32-bytes, since this is granularity 4811 * of interpreter stack size 4812 */ 4813 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4814 if (depth > MAX_BPF_STACK) { 4815 verbose(env, "combined stack size of %d calls is %d. Too large\n", 4816 frame + 1, depth); 4817 return -EACCES; 4818 } 4819 continue_func: 4820 subprog_end = subprog[idx + 1].start; 4821 for (; i < subprog_end; i++) { 4822 int next_insn; 4823 4824 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 4825 continue; 4826 /* remember insn and function to return to */ 4827 ret_insn[frame] = i + 1; 4828 ret_prog[frame] = idx; 4829 4830 /* find the callee */ 4831 next_insn = i + insn[i].imm + 1; 4832 idx = find_subprog(env, next_insn); 4833 if (idx < 0) { 4834 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4835 next_insn); 4836 return -EFAULT; 4837 } 4838 if (subprog[idx].is_async_cb) { 4839 if (subprog[idx].has_tail_call) { 4840 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 4841 return -EFAULT; 4842 } 4843 /* async callbacks don't increase bpf prog stack size */ 4844 continue; 4845 } 4846 i = next_insn; 4847 4848 if (subprog[idx].has_tail_call) 4849 tail_call_reachable = true; 4850 4851 frame++; 4852 if (frame >= MAX_CALL_FRAMES) { 4853 verbose(env, "the call stack of %d frames is too deep !\n", 4854 frame); 4855 return -E2BIG; 4856 } 4857 goto process_func; 4858 } 4859 /* if tail call got detected across bpf2bpf calls then mark each of the 4860 * currently present subprog frames as tail call reachable subprogs; 4861 * this info will be utilized by JIT so that we will be preserving the 4862 * tail call counter throughout bpf2bpf calls combined with tailcalls 4863 */ 4864 if (tail_call_reachable) 4865 for (j = 0; j < frame; j++) 4866 subprog[ret_prog[j]].tail_call_reachable = true; 4867 if (subprog[0].tail_call_reachable) 4868 env->prog->aux->tail_call_reachable = true; 4869 4870 /* end of for() loop means the last insn of the 'subprog' 4871 * was reached. Doesn't matter whether it was JA or EXIT 4872 */ 4873 if (frame == 0) 4874 return 0; 4875 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4876 frame--; 4877 i = ret_insn[frame]; 4878 idx = ret_prog[frame]; 4879 goto continue_func; 4880 } 4881 4882 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 4883 static int get_callee_stack_depth(struct bpf_verifier_env *env, 4884 const struct bpf_insn *insn, int idx) 4885 { 4886 int start = idx + insn->imm + 1, subprog; 4887 4888 subprog = find_subprog(env, start); 4889 if (subprog < 0) { 4890 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4891 start); 4892 return -EFAULT; 4893 } 4894 return env->subprog_info[subprog].stack_depth; 4895 } 4896 #endif 4897 4898 static int __check_buffer_access(struct bpf_verifier_env *env, 4899 const char *buf_info, 4900 const struct bpf_reg_state *reg, 4901 int regno, int off, int size) 4902 { 4903 if (off < 0) { 4904 verbose(env, 4905 "R%d invalid %s buffer access: off=%d, size=%d\n", 4906 regno, buf_info, off, size); 4907 return -EACCES; 4908 } 4909 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4910 char tn_buf[48]; 4911 4912 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4913 verbose(env, 4914 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 4915 regno, off, tn_buf); 4916 return -EACCES; 4917 } 4918 4919 return 0; 4920 } 4921 4922 static int check_tp_buffer_access(struct bpf_verifier_env *env, 4923 const struct bpf_reg_state *reg, 4924 int regno, int off, int size) 4925 { 4926 int err; 4927 4928 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 4929 if (err) 4930 return err; 4931 4932 if (off + size > env->prog->aux->max_tp_access) 4933 env->prog->aux->max_tp_access = off + size; 4934 4935 return 0; 4936 } 4937 4938 static int check_buffer_access(struct bpf_verifier_env *env, 4939 const struct bpf_reg_state *reg, 4940 int regno, int off, int size, 4941 bool zero_size_allowed, 4942 u32 *max_access) 4943 { 4944 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 4945 int err; 4946 4947 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 4948 if (err) 4949 return err; 4950 4951 if (off + size > *max_access) 4952 *max_access = off + size; 4953 4954 return 0; 4955 } 4956 4957 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 4958 static void zext_32_to_64(struct bpf_reg_state *reg) 4959 { 4960 reg->var_off = tnum_subreg(reg->var_off); 4961 __reg_assign_32_into_64(reg); 4962 } 4963 4964 /* truncate register to smaller size (in bytes) 4965 * must be called with size < BPF_REG_SIZE 4966 */ 4967 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 4968 { 4969 u64 mask; 4970 4971 /* clear high bits in bit representation */ 4972 reg->var_off = tnum_cast(reg->var_off, size); 4973 4974 /* fix arithmetic bounds */ 4975 mask = ((u64)1 << (size * 8)) - 1; 4976 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 4977 reg->umin_value &= mask; 4978 reg->umax_value &= mask; 4979 } else { 4980 reg->umin_value = 0; 4981 reg->umax_value = mask; 4982 } 4983 reg->smin_value = reg->umin_value; 4984 reg->smax_value = reg->umax_value; 4985 4986 /* If size is smaller than 32bit register the 32bit register 4987 * values are also truncated so we push 64-bit bounds into 4988 * 32-bit bounds. Above were truncated < 32-bits already. 4989 */ 4990 if (size >= 4) 4991 return; 4992 __reg_combine_64_into_32(reg); 4993 } 4994 4995 static bool bpf_map_is_rdonly(const struct bpf_map *map) 4996 { 4997 /* A map is considered read-only if the following condition are true: 4998 * 4999 * 1) BPF program side cannot change any of the map content. The 5000 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 5001 * and was set at map creation time. 5002 * 2) The map value(s) have been initialized from user space by a 5003 * loader and then "frozen", such that no new map update/delete 5004 * operations from syscall side are possible for the rest of 5005 * the map's lifetime from that point onwards. 5006 * 3) Any parallel/pending map update/delete operations from syscall 5007 * side have been completed. Only after that point, it's safe to 5008 * assume that map value(s) are immutable. 5009 */ 5010 return (map->map_flags & BPF_F_RDONLY_PROG) && 5011 READ_ONCE(map->frozen) && 5012 !bpf_map_write_active(map); 5013 } 5014 5015 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 5016 { 5017 void *ptr; 5018 u64 addr; 5019 int err; 5020 5021 err = map->ops->map_direct_value_addr(map, &addr, off); 5022 if (err) 5023 return err; 5024 ptr = (void *)(long)addr + off; 5025 5026 switch (size) { 5027 case sizeof(u8): 5028 *val = (u64)*(u8 *)ptr; 5029 break; 5030 case sizeof(u16): 5031 *val = (u64)*(u16 *)ptr; 5032 break; 5033 case sizeof(u32): 5034 *val = (u64)*(u32 *)ptr; 5035 break; 5036 case sizeof(u64): 5037 *val = *(u64 *)ptr; 5038 break; 5039 default: 5040 return -EINVAL; 5041 } 5042 return 0; 5043 } 5044 5045 #define BTF_TYPE_SAFE_NESTED(__type) __PASTE(__type, __safe_fields) 5046 5047 BTF_TYPE_SAFE_NESTED(struct task_struct) { 5048 const cpumask_t *cpus_ptr; 5049 struct css_set __rcu *cgroups; 5050 }; 5051 5052 BTF_TYPE_SAFE_NESTED(struct css_set) { 5053 struct cgroup *dfl_cgrp; 5054 }; 5055 5056 static bool nested_ptr_is_trusted(struct bpf_verifier_env *env, 5057 struct bpf_reg_state *reg, 5058 int off) 5059 { 5060 /* If its parent is not trusted, it can't regain its trusted status. */ 5061 if (!is_trusted_reg(reg)) 5062 return false; 5063 5064 BTF_TYPE_EMIT(BTF_TYPE_SAFE_NESTED(struct task_struct)); 5065 BTF_TYPE_EMIT(BTF_TYPE_SAFE_NESTED(struct css_set)); 5066 5067 return btf_nested_type_is_trusted(&env->log, reg, off); 5068 } 5069 5070 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 5071 struct bpf_reg_state *regs, 5072 int regno, int off, int size, 5073 enum bpf_access_type atype, 5074 int value_regno) 5075 { 5076 struct bpf_reg_state *reg = regs + regno; 5077 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 5078 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 5079 enum bpf_type_flag flag = 0; 5080 u32 btf_id; 5081 int ret; 5082 5083 if (!env->allow_ptr_leaks) { 5084 verbose(env, 5085 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 5086 tname); 5087 return -EPERM; 5088 } 5089 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 5090 verbose(env, 5091 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 5092 tname); 5093 return -EINVAL; 5094 } 5095 if (off < 0) { 5096 verbose(env, 5097 "R%d is ptr_%s invalid negative access: off=%d\n", 5098 regno, tname, off); 5099 return -EACCES; 5100 } 5101 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 5102 char tn_buf[48]; 5103 5104 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5105 verbose(env, 5106 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 5107 regno, tname, off, tn_buf); 5108 return -EACCES; 5109 } 5110 5111 if (reg->type & MEM_USER) { 5112 verbose(env, 5113 "R%d is ptr_%s access user memory: off=%d\n", 5114 regno, tname, off); 5115 return -EACCES; 5116 } 5117 5118 if (reg->type & MEM_PERCPU) { 5119 verbose(env, 5120 "R%d is ptr_%s access percpu memory: off=%d\n", 5121 regno, tname, off); 5122 return -EACCES; 5123 } 5124 5125 if (env->ops->btf_struct_access && !type_is_alloc(reg->type)) { 5126 if (!btf_is_kernel(reg->btf)) { 5127 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); 5128 return -EFAULT; 5129 } 5130 ret = env->ops->btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 5131 } else { 5132 /* Writes are permitted with default btf_struct_access for 5133 * program allocated objects (which always have ref_obj_id > 0), 5134 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. 5135 */ 5136 if (atype != BPF_READ && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 5137 verbose(env, "only read is supported\n"); 5138 return -EACCES; 5139 } 5140 5141 if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) && 5142 !reg->ref_obj_id) { 5143 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 5144 return -EFAULT; 5145 } 5146 5147 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 5148 } 5149 5150 if (ret < 0) 5151 return ret; 5152 5153 /* If this is an untrusted pointer, all pointers formed by walking it 5154 * also inherit the untrusted flag. 5155 */ 5156 if (type_flag(reg->type) & PTR_UNTRUSTED) 5157 flag |= PTR_UNTRUSTED; 5158 5159 /* By default any pointer obtained from walking a trusted pointer is no 5160 * longer trusted, unless the field being accessed has explicitly been 5161 * marked as inheriting its parent's state of trust. 5162 * 5163 * An RCU-protected pointer can also be deemed trusted if we are in an 5164 * RCU read region. This case is handled below. 5165 */ 5166 if (nested_ptr_is_trusted(env, reg, off)) 5167 flag |= PTR_TRUSTED; 5168 else 5169 flag &= ~PTR_TRUSTED; 5170 5171 if (flag & MEM_RCU) { 5172 /* Mark value register as MEM_RCU only if it is protected by 5173 * bpf_rcu_read_lock() and the ptr reg is rcu or trusted. MEM_RCU 5174 * itself can already indicate trustedness inside the rcu 5175 * read lock region. Also mark rcu pointer as PTR_MAYBE_NULL since 5176 * it could be null in some cases. 5177 */ 5178 if (!env->cur_state->active_rcu_lock || 5179 !(is_trusted_reg(reg) || is_rcu_reg(reg))) 5180 flag &= ~MEM_RCU; 5181 else 5182 flag |= PTR_MAYBE_NULL; 5183 } else if (reg->type & MEM_RCU) { 5184 /* ptr (reg) is marked as MEM_RCU, but the struct field is not tagged 5185 * with __rcu. Mark the flag as PTR_UNTRUSTED conservatively. 5186 */ 5187 flag |= PTR_UNTRUSTED; 5188 } 5189 5190 if (atype == BPF_READ && value_regno >= 0) 5191 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 5192 5193 return 0; 5194 } 5195 5196 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 5197 struct bpf_reg_state *regs, 5198 int regno, int off, int size, 5199 enum bpf_access_type atype, 5200 int value_regno) 5201 { 5202 struct bpf_reg_state *reg = regs + regno; 5203 struct bpf_map *map = reg->map_ptr; 5204 struct bpf_reg_state map_reg; 5205 enum bpf_type_flag flag = 0; 5206 const struct btf_type *t; 5207 const char *tname; 5208 u32 btf_id; 5209 int ret; 5210 5211 if (!btf_vmlinux) { 5212 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 5213 return -ENOTSUPP; 5214 } 5215 5216 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 5217 verbose(env, "map_ptr access not supported for map type %d\n", 5218 map->map_type); 5219 return -ENOTSUPP; 5220 } 5221 5222 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 5223 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 5224 5225 if (!env->allow_ptr_leaks) { 5226 verbose(env, 5227 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 5228 tname); 5229 return -EPERM; 5230 } 5231 5232 if (off < 0) { 5233 verbose(env, "R%d is %s invalid negative access: off=%d\n", 5234 regno, tname, off); 5235 return -EACCES; 5236 } 5237 5238 if (atype != BPF_READ) { 5239 verbose(env, "only read from %s is supported\n", tname); 5240 return -EACCES; 5241 } 5242 5243 /* Simulate access to a PTR_TO_BTF_ID */ 5244 memset(&map_reg, 0, sizeof(map_reg)); 5245 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); 5246 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag); 5247 if (ret < 0) 5248 return ret; 5249 5250 if (value_regno >= 0) 5251 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 5252 5253 return 0; 5254 } 5255 5256 /* Check that the stack access at the given offset is within bounds. The 5257 * maximum valid offset is -1. 5258 * 5259 * The minimum valid offset is -MAX_BPF_STACK for writes, and 5260 * -state->allocated_stack for reads. 5261 */ 5262 static int check_stack_slot_within_bounds(int off, 5263 struct bpf_func_state *state, 5264 enum bpf_access_type t) 5265 { 5266 int min_valid_off; 5267 5268 if (t == BPF_WRITE) 5269 min_valid_off = -MAX_BPF_STACK; 5270 else 5271 min_valid_off = -state->allocated_stack; 5272 5273 if (off < min_valid_off || off > -1) 5274 return -EACCES; 5275 return 0; 5276 } 5277 5278 /* Check that the stack access at 'regno + off' falls within the maximum stack 5279 * bounds. 5280 * 5281 * 'off' includes `regno->offset`, but not its dynamic part (if any). 5282 */ 5283 static int check_stack_access_within_bounds( 5284 struct bpf_verifier_env *env, 5285 int regno, int off, int access_size, 5286 enum bpf_access_src src, enum bpf_access_type type) 5287 { 5288 struct bpf_reg_state *regs = cur_regs(env); 5289 struct bpf_reg_state *reg = regs + regno; 5290 struct bpf_func_state *state = func(env, reg); 5291 int min_off, max_off; 5292 int err; 5293 char *err_extra; 5294 5295 if (src == ACCESS_HELPER) 5296 /* We don't know if helpers are reading or writing (or both). */ 5297 err_extra = " indirect access to"; 5298 else if (type == BPF_READ) 5299 err_extra = " read from"; 5300 else 5301 err_extra = " write to"; 5302 5303 if (tnum_is_const(reg->var_off)) { 5304 min_off = reg->var_off.value + off; 5305 if (access_size > 0) 5306 max_off = min_off + access_size - 1; 5307 else 5308 max_off = min_off; 5309 } else { 5310 if (reg->smax_value >= BPF_MAX_VAR_OFF || 5311 reg->smin_value <= -BPF_MAX_VAR_OFF) { 5312 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 5313 err_extra, regno); 5314 return -EACCES; 5315 } 5316 min_off = reg->smin_value + off; 5317 if (access_size > 0) 5318 max_off = reg->smax_value + off + access_size - 1; 5319 else 5320 max_off = min_off; 5321 } 5322 5323 err = check_stack_slot_within_bounds(min_off, state, type); 5324 if (!err) 5325 err = check_stack_slot_within_bounds(max_off, state, type); 5326 5327 if (err) { 5328 if (tnum_is_const(reg->var_off)) { 5329 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 5330 err_extra, regno, off, access_size); 5331 } else { 5332 char tn_buf[48]; 5333 5334 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5335 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 5336 err_extra, regno, tn_buf, access_size); 5337 } 5338 } 5339 return err; 5340 } 5341 5342 /* check whether memory at (regno + off) is accessible for t = (read | write) 5343 * if t==write, value_regno is a register which value is stored into memory 5344 * if t==read, value_regno is a register which will receive the value from memory 5345 * if t==write && value_regno==-1, some unknown value is stored into memory 5346 * if t==read && value_regno==-1, don't care what we read from memory 5347 */ 5348 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 5349 int off, int bpf_size, enum bpf_access_type t, 5350 int value_regno, bool strict_alignment_once) 5351 { 5352 struct bpf_reg_state *regs = cur_regs(env); 5353 struct bpf_reg_state *reg = regs + regno; 5354 struct bpf_func_state *state; 5355 int size, err = 0; 5356 5357 size = bpf_size_to_bytes(bpf_size); 5358 if (size < 0) 5359 return size; 5360 5361 /* alignment checks will add in reg->off themselves */ 5362 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 5363 if (err) 5364 return err; 5365 5366 /* for access checks, reg->off is just part of off */ 5367 off += reg->off; 5368 5369 if (reg->type == PTR_TO_MAP_KEY) { 5370 if (t == BPF_WRITE) { 5371 verbose(env, "write to change key R%d not allowed\n", regno); 5372 return -EACCES; 5373 } 5374 5375 err = check_mem_region_access(env, regno, off, size, 5376 reg->map_ptr->key_size, false); 5377 if (err) 5378 return err; 5379 if (value_regno >= 0) 5380 mark_reg_unknown(env, regs, value_regno); 5381 } else if (reg->type == PTR_TO_MAP_VALUE) { 5382 struct btf_field *kptr_field = NULL; 5383 5384 if (t == BPF_WRITE && value_regno >= 0 && 5385 is_pointer_value(env, value_regno)) { 5386 verbose(env, "R%d leaks addr into map\n", value_regno); 5387 return -EACCES; 5388 } 5389 err = check_map_access_type(env, regno, off, size, t); 5390 if (err) 5391 return err; 5392 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 5393 if (err) 5394 return err; 5395 if (tnum_is_const(reg->var_off)) 5396 kptr_field = btf_record_find(reg->map_ptr->record, 5397 off + reg->var_off.value, BPF_KPTR); 5398 if (kptr_field) { 5399 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 5400 } else if (t == BPF_READ && value_regno >= 0) { 5401 struct bpf_map *map = reg->map_ptr; 5402 5403 /* if map is read-only, track its contents as scalars */ 5404 if (tnum_is_const(reg->var_off) && 5405 bpf_map_is_rdonly(map) && 5406 map->ops->map_direct_value_addr) { 5407 int map_off = off + reg->var_off.value; 5408 u64 val = 0; 5409 5410 err = bpf_map_direct_read(map, map_off, size, 5411 &val); 5412 if (err) 5413 return err; 5414 5415 regs[value_regno].type = SCALAR_VALUE; 5416 __mark_reg_known(®s[value_regno], val); 5417 } else { 5418 mark_reg_unknown(env, regs, value_regno); 5419 } 5420 } 5421 } else if (base_type(reg->type) == PTR_TO_MEM) { 5422 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5423 5424 if (type_may_be_null(reg->type)) { 5425 verbose(env, "R%d invalid mem access '%s'\n", regno, 5426 reg_type_str(env, reg->type)); 5427 return -EACCES; 5428 } 5429 5430 if (t == BPF_WRITE && rdonly_mem) { 5431 verbose(env, "R%d cannot write into %s\n", 5432 regno, reg_type_str(env, reg->type)); 5433 return -EACCES; 5434 } 5435 5436 if (t == BPF_WRITE && value_regno >= 0 && 5437 is_pointer_value(env, value_regno)) { 5438 verbose(env, "R%d leaks addr into mem\n", value_regno); 5439 return -EACCES; 5440 } 5441 5442 err = check_mem_region_access(env, regno, off, size, 5443 reg->mem_size, false); 5444 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 5445 mark_reg_unknown(env, regs, value_regno); 5446 } else if (reg->type == PTR_TO_CTX) { 5447 enum bpf_reg_type reg_type = SCALAR_VALUE; 5448 struct btf *btf = NULL; 5449 u32 btf_id = 0; 5450 5451 if (t == BPF_WRITE && value_regno >= 0 && 5452 is_pointer_value(env, value_regno)) { 5453 verbose(env, "R%d leaks addr into ctx\n", value_regno); 5454 return -EACCES; 5455 } 5456 5457 err = check_ptr_off_reg(env, reg, regno); 5458 if (err < 0) 5459 return err; 5460 5461 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 5462 &btf_id); 5463 if (err) 5464 verbose_linfo(env, insn_idx, "; "); 5465 if (!err && t == BPF_READ && value_regno >= 0) { 5466 /* ctx access returns either a scalar, or a 5467 * PTR_TO_PACKET[_META,_END]. In the latter 5468 * case, we know the offset is zero. 5469 */ 5470 if (reg_type == SCALAR_VALUE) { 5471 mark_reg_unknown(env, regs, value_regno); 5472 } else { 5473 mark_reg_known_zero(env, regs, 5474 value_regno); 5475 if (type_may_be_null(reg_type)) 5476 regs[value_regno].id = ++env->id_gen; 5477 /* A load of ctx field could have different 5478 * actual load size with the one encoded in the 5479 * insn. When the dst is PTR, it is for sure not 5480 * a sub-register. 5481 */ 5482 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 5483 if (base_type(reg_type) == PTR_TO_BTF_ID) { 5484 regs[value_regno].btf = btf; 5485 regs[value_regno].btf_id = btf_id; 5486 } 5487 } 5488 regs[value_regno].type = reg_type; 5489 } 5490 5491 } else if (reg->type == PTR_TO_STACK) { 5492 /* Basic bounds checks. */ 5493 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 5494 if (err) 5495 return err; 5496 5497 state = func(env, reg); 5498 err = update_stack_depth(env, state, off); 5499 if (err) 5500 return err; 5501 5502 if (t == BPF_READ) 5503 err = check_stack_read(env, regno, off, size, 5504 value_regno); 5505 else 5506 err = check_stack_write(env, regno, off, size, 5507 value_regno, insn_idx); 5508 } else if (reg_is_pkt_pointer(reg)) { 5509 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 5510 verbose(env, "cannot write into packet\n"); 5511 return -EACCES; 5512 } 5513 if (t == BPF_WRITE && value_regno >= 0 && 5514 is_pointer_value(env, value_regno)) { 5515 verbose(env, "R%d leaks addr into packet\n", 5516 value_regno); 5517 return -EACCES; 5518 } 5519 err = check_packet_access(env, regno, off, size, false); 5520 if (!err && t == BPF_READ && value_regno >= 0) 5521 mark_reg_unknown(env, regs, value_regno); 5522 } else if (reg->type == PTR_TO_FLOW_KEYS) { 5523 if (t == BPF_WRITE && value_regno >= 0 && 5524 is_pointer_value(env, value_regno)) { 5525 verbose(env, "R%d leaks addr into flow keys\n", 5526 value_regno); 5527 return -EACCES; 5528 } 5529 5530 err = check_flow_keys_access(env, off, size); 5531 if (!err && t == BPF_READ && value_regno >= 0) 5532 mark_reg_unknown(env, regs, value_regno); 5533 } else if (type_is_sk_pointer(reg->type)) { 5534 if (t == BPF_WRITE) { 5535 verbose(env, "R%d cannot write into %s\n", 5536 regno, reg_type_str(env, reg->type)); 5537 return -EACCES; 5538 } 5539 err = check_sock_access(env, insn_idx, regno, off, size, t); 5540 if (!err && value_regno >= 0) 5541 mark_reg_unknown(env, regs, value_regno); 5542 } else if (reg->type == PTR_TO_TP_BUFFER) { 5543 err = check_tp_buffer_access(env, reg, regno, off, size); 5544 if (!err && t == BPF_READ && value_regno >= 0) 5545 mark_reg_unknown(env, regs, value_regno); 5546 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 5547 !type_may_be_null(reg->type)) { 5548 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 5549 value_regno); 5550 } else if (reg->type == CONST_PTR_TO_MAP) { 5551 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 5552 value_regno); 5553 } else if (base_type(reg->type) == PTR_TO_BUF) { 5554 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5555 u32 *max_access; 5556 5557 if (rdonly_mem) { 5558 if (t == BPF_WRITE) { 5559 verbose(env, "R%d cannot write into %s\n", 5560 regno, reg_type_str(env, reg->type)); 5561 return -EACCES; 5562 } 5563 max_access = &env->prog->aux->max_rdonly_access; 5564 } else { 5565 max_access = &env->prog->aux->max_rdwr_access; 5566 } 5567 5568 err = check_buffer_access(env, reg, regno, off, size, false, 5569 max_access); 5570 5571 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 5572 mark_reg_unknown(env, regs, value_regno); 5573 } else { 5574 verbose(env, "R%d invalid mem access '%s'\n", regno, 5575 reg_type_str(env, reg->type)); 5576 return -EACCES; 5577 } 5578 5579 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 5580 regs[value_regno].type == SCALAR_VALUE) { 5581 /* b/h/w load zero-extends, mark upper bits as known 0 */ 5582 coerce_reg_to_size(®s[value_regno], size); 5583 } 5584 return err; 5585 } 5586 5587 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 5588 { 5589 int load_reg; 5590 int err; 5591 5592 switch (insn->imm) { 5593 case BPF_ADD: 5594 case BPF_ADD | BPF_FETCH: 5595 case BPF_AND: 5596 case BPF_AND | BPF_FETCH: 5597 case BPF_OR: 5598 case BPF_OR | BPF_FETCH: 5599 case BPF_XOR: 5600 case BPF_XOR | BPF_FETCH: 5601 case BPF_XCHG: 5602 case BPF_CMPXCHG: 5603 break; 5604 default: 5605 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 5606 return -EINVAL; 5607 } 5608 5609 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 5610 verbose(env, "invalid atomic operand size\n"); 5611 return -EINVAL; 5612 } 5613 5614 /* check src1 operand */ 5615 err = check_reg_arg(env, insn->src_reg, SRC_OP); 5616 if (err) 5617 return err; 5618 5619 /* check src2 operand */ 5620 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 5621 if (err) 5622 return err; 5623 5624 if (insn->imm == BPF_CMPXCHG) { 5625 /* Check comparison of R0 with memory location */ 5626 const u32 aux_reg = BPF_REG_0; 5627 5628 err = check_reg_arg(env, aux_reg, SRC_OP); 5629 if (err) 5630 return err; 5631 5632 if (is_pointer_value(env, aux_reg)) { 5633 verbose(env, "R%d leaks addr into mem\n", aux_reg); 5634 return -EACCES; 5635 } 5636 } 5637 5638 if (is_pointer_value(env, insn->src_reg)) { 5639 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 5640 return -EACCES; 5641 } 5642 5643 if (is_ctx_reg(env, insn->dst_reg) || 5644 is_pkt_reg(env, insn->dst_reg) || 5645 is_flow_key_reg(env, insn->dst_reg) || 5646 is_sk_reg(env, insn->dst_reg)) { 5647 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 5648 insn->dst_reg, 5649 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 5650 return -EACCES; 5651 } 5652 5653 if (insn->imm & BPF_FETCH) { 5654 if (insn->imm == BPF_CMPXCHG) 5655 load_reg = BPF_REG_0; 5656 else 5657 load_reg = insn->src_reg; 5658 5659 /* check and record load of old value */ 5660 err = check_reg_arg(env, load_reg, DST_OP); 5661 if (err) 5662 return err; 5663 } else { 5664 /* This instruction accesses a memory location but doesn't 5665 * actually load it into a register. 5666 */ 5667 load_reg = -1; 5668 } 5669 5670 /* Check whether we can read the memory, with second call for fetch 5671 * case to simulate the register fill. 5672 */ 5673 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5674 BPF_SIZE(insn->code), BPF_READ, -1, true); 5675 if (!err && load_reg >= 0) 5676 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5677 BPF_SIZE(insn->code), BPF_READ, load_reg, 5678 true); 5679 if (err) 5680 return err; 5681 5682 /* Check whether we can write into the same memory. */ 5683 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5684 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 5685 if (err) 5686 return err; 5687 5688 return 0; 5689 } 5690 5691 /* When register 'regno' is used to read the stack (either directly or through 5692 * a helper function) make sure that it's within stack boundary and, depending 5693 * on the access type, that all elements of the stack are initialized. 5694 * 5695 * 'off' includes 'regno->off', but not its dynamic part (if any). 5696 * 5697 * All registers that have been spilled on the stack in the slots within the 5698 * read offsets are marked as read. 5699 */ 5700 static int check_stack_range_initialized( 5701 struct bpf_verifier_env *env, int regno, int off, 5702 int access_size, bool zero_size_allowed, 5703 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 5704 { 5705 struct bpf_reg_state *reg = reg_state(env, regno); 5706 struct bpf_func_state *state = func(env, reg); 5707 int err, min_off, max_off, i, j, slot, spi; 5708 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 5709 enum bpf_access_type bounds_check_type; 5710 /* Some accesses can write anything into the stack, others are 5711 * read-only. 5712 */ 5713 bool clobber = false; 5714 5715 if (access_size == 0 && !zero_size_allowed) { 5716 verbose(env, "invalid zero-sized read\n"); 5717 return -EACCES; 5718 } 5719 5720 if (type == ACCESS_HELPER) { 5721 /* The bounds checks for writes are more permissive than for 5722 * reads. However, if raw_mode is not set, we'll do extra 5723 * checks below. 5724 */ 5725 bounds_check_type = BPF_WRITE; 5726 clobber = true; 5727 } else { 5728 bounds_check_type = BPF_READ; 5729 } 5730 err = check_stack_access_within_bounds(env, regno, off, access_size, 5731 type, bounds_check_type); 5732 if (err) 5733 return err; 5734 5735 5736 if (tnum_is_const(reg->var_off)) { 5737 min_off = max_off = reg->var_off.value + off; 5738 } else { 5739 /* Variable offset is prohibited for unprivileged mode for 5740 * simplicity since it requires corresponding support in 5741 * Spectre masking for stack ALU. 5742 * See also retrieve_ptr_limit(). 5743 */ 5744 if (!env->bypass_spec_v1) { 5745 char tn_buf[48]; 5746 5747 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5748 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 5749 regno, err_extra, tn_buf); 5750 return -EACCES; 5751 } 5752 /* Only initialized buffer on stack is allowed to be accessed 5753 * with variable offset. With uninitialized buffer it's hard to 5754 * guarantee that whole memory is marked as initialized on 5755 * helper return since specific bounds are unknown what may 5756 * cause uninitialized stack leaking. 5757 */ 5758 if (meta && meta->raw_mode) 5759 meta = NULL; 5760 5761 min_off = reg->smin_value + off; 5762 max_off = reg->smax_value + off; 5763 } 5764 5765 if (meta && meta->raw_mode) { 5766 /* Ensure we won't be overwriting dynptrs when simulating byte 5767 * by byte access in check_helper_call using meta.access_size. 5768 * This would be a problem if we have a helper in the future 5769 * which takes: 5770 * 5771 * helper(uninit_mem, len, dynptr) 5772 * 5773 * Now, uninint_mem may overlap with dynptr pointer. Hence, it 5774 * may end up writing to dynptr itself when touching memory from 5775 * arg 1. This can be relaxed on a case by case basis for known 5776 * safe cases, but reject due to the possibilitiy of aliasing by 5777 * default. 5778 */ 5779 for (i = min_off; i < max_off + access_size; i++) { 5780 int stack_off = -i - 1; 5781 5782 spi = __get_spi(i); 5783 /* raw_mode may write past allocated_stack */ 5784 if (state->allocated_stack <= stack_off) 5785 continue; 5786 if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { 5787 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); 5788 return -EACCES; 5789 } 5790 } 5791 meta->access_size = access_size; 5792 meta->regno = regno; 5793 return 0; 5794 } 5795 5796 for (i = min_off; i < max_off + access_size; i++) { 5797 u8 *stype; 5798 5799 slot = -i - 1; 5800 spi = slot / BPF_REG_SIZE; 5801 if (state->allocated_stack <= slot) 5802 goto err; 5803 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 5804 if (*stype == STACK_MISC) 5805 goto mark; 5806 if ((*stype == STACK_ZERO) || 5807 (*stype == STACK_INVALID && env->allow_uninit_stack)) { 5808 if (clobber) { 5809 /* helper can write anything into the stack */ 5810 *stype = STACK_MISC; 5811 } 5812 goto mark; 5813 } 5814 5815 if (is_spilled_reg(&state->stack[spi]) && 5816 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 5817 env->allow_ptr_leaks)) { 5818 if (clobber) { 5819 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 5820 for (j = 0; j < BPF_REG_SIZE; j++) 5821 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 5822 } 5823 goto mark; 5824 } 5825 5826 err: 5827 if (tnum_is_const(reg->var_off)) { 5828 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 5829 err_extra, regno, min_off, i - min_off, access_size); 5830 } else { 5831 char tn_buf[48]; 5832 5833 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5834 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 5835 err_extra, regno, tn_buf, i - min_off, access_size); 5836 } 5837 return -EACCES; 5838 mark: 5839 /* reading any byte out of 8-byte 'spill_slot' will cause 5840 * the whole slot to be marked as 'read' 5841 */ 5842 mark_reg_read(env, &state->stack[spi].spilled_ptr, 5843 state->stack[spi].spilled_ptr.parent, 5844 REG_LIVE_READ64); 5845 /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not 5846 * be sure that whether stack slot is written to or not. Hence, 5847 * we must still conservatively propagate reads upwards even if 5848 * helper may write to the entire memory range. 5849 */ 5850 } 5851 return update_stack_depth(env, state, min_off); 5852 } 5853 5854 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 5855 int access_size, bool zero_size_allowed, 5856 struct bpf_call_arg_meta *meta) 5857 { 5858 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5859 u32 *max_access; 5860 5861 switch (base_type(reg->type)) { 5862 case PTR_TO_PACKET: 5863 case PTR_TO_PACKET_META: 5864 return check_packet_access(env, regno, reg->off, access_size, 5865 zero_size_allowed); 5866 case PTR_TO_MAP_KEY: 5867 if (meta && meta->raw_mode) { 5868 verbose(env, "R%d cannot write into %s\n", regno, 5869 reg_type_str(env, reg->type)); 5870 return -EACCES; 5871 } 5872 return check_mem_region_access(env, regno, reg->off, access_size, 5873 reg->map_ptr->key_size, false); 5874 case PTR_TO_MAP_VALUE: 5875 if (check_map_access_type(env, regno, reg->off, access_size, 5876 meta && meta->raw_mode ? BPF_WRITE : 5877 BPF_READ)) 5878 return -EACCES; 5879 return check_map_access(env, regno, reg->off, access_size, 5880 zero_size_allowed, ACCESS_HELPER); 5881 case PTR_TO_MEM: 5882 if (type_is_rdonly_mem(reg->type)) { 5883 if (meta && meta->raw_mode) { 5884 verbose(env, "R%d cannot write into %s\n", regno, 5885 reg_type_str(env, reg->type)); 5886 return -EACCES; 5887 } 5888 } 5889 return check_mem_region_access(env, regno, reg->off, 5890 access_size, reg->mem_size, 5891 zero_size_allowed); 5892 case PTR_TO_BUF: 5893 if (type_is_rdonly_mem(reg->type)) { 5894 if (meta && meta->raw_mode) { 5895 verbose(env, "R%d cannot write into %s\n", regno, 5896 reg_type_str(env, reg->type)); 5897 return -EACCES; 5898 } 5899 5900 max_access = &env->prog->aux->max_rdonly_access; 5901 } else { 5902 max_access = &env->prog->aux->max_rdwr_access; 5903 } 5904 return check_buffer_access(env, reg, regno, reg->off, 5905 access_size, zero_size_allowed, 5906 max_access); 5907 case PTR_TO_STACK: 5908 return check_stack_range_initialized( 5909 env, 5910 regno, reg->off, access_size, 5911 zero_size_allowed, ACCESS_HELPER, meta); 5912 case PTR_TO_CTX: 5913 /* in case the function doesn't know how to access the context, 5914 * (because we are in a program of type SYSCALL for example), we 5915 * can not statically check its size. 5916 * Dynamically check it now. 5917 */ 5918 if (!env->ops->convert_ctx_access) { 5919 enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; 5920 int offset = access_size - 1; 5921 5922 /* Allow zero-byte read from PTR_TO_CTX */ 5923 if (access_size == 0) 5924 return zero_size_allowed ? 0 : -EACCES; 5925 5926 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, 5927 atype, -1, false); 5928 } 5929 5930 fallthrough; 5931 default: /* scalar_value or invalid ptr */ 5932 /* Allow zero-byte read from NULL, regardless of pointer type */ 5933 if (zero_size_allowed && access_size == 0 && 5934 register_is_null(reg)) 5935 return 0; 5936 5937 verbose(env, "R%d type=%s ", regno, 5938 reg_type_str(env, reg->type)); 5939 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 5940 return -EACCES; 5941 } 5942 } 5943 5944 static int check_mem_size_reg(struct bpf_verifier_env *env, 5945 struct bpf_reg_state *reg, u32 regno, 5946 bool zero_size_allowed, 5947 struct bpf_call_arg_meta *meta) 5948 { 5949 int err; 5950 5951 /* This is used to refine r0 return value bounds for helpers 5952 * that enforce this value as an upper bound on return values. 5953 * See do_refine_retval_range() for helpers that can refine 5954 * the return value. C type of helper is u32 so we pull register 5955 * bound from umax_value however, if negative verifier errors 5956 * out. Only upper bounds can be learned because retval is an 5957 * int type and negative retvals are allowed. 5958 */ 5959 meta->msize_max_value = reg->umax_value; 5960 5961 /* The register is SCALAR_VALUE; the access check 5962 * happens using its boundaries. 5963 */ 5964 if (!tnum_is_const(reg->var_off)) 5965 /* For unprivileged variable accesses, disable raw 5966 * mode so that the program is required to 5967 * initialize all the memory that the helper could 5968 * just partially fill up. 5969 */ 5970 meta = NULL; 5971 5972 if (reg->smin_value < 0) { 5973 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5974 regno); 5975 return -EACCES; 5976 } 5977 5978 if (reg->umin_value == 0) { 5979 err = check_helper_mem_access(env, regno - 1, 0, 5980 zero_size_allowed, 5981 meta); 5982 if (err) 5983 return err; 5984 } 5985 5986 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5987 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5988 regno); 5989 return -EACCES; 5990 } 5991 err = check_helper_mem_access(env, regno - 1, 5992 reg->umax_value, 5993 zero_size_allowed, meta); 5994 if (!err) 5995 err = mark_chain_precision(env, regno); 5996 return err; 5997 } 5998 5999 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 6000 u32 regno, u32 mem_size) 6001 { 6002 bool may_be_null = type_may_be_null(reg->type); 6003 struct bpf_reg_state saved_reg; 6004 struct bpf_call_arg_meta meta; 6005 int err; 6006 6007 if (register_is_null(reg)) 6008 return 0; 6009 6010 memset(&meta, 0, sizeof(meta)); 6011 /* Assuming that the register contains a value check if the memory 6012 * access is safe. Temporarily save and restore the register's state as 6013 * the conversion shouldn't be visible to a caller. 6014 */ 6015 if (may_be_null) { 6016 saved_reg = *reg; 6017 mark_ptr_not_null_reg(reg); 6018 } 6019 6020 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 6021 /* Check access for BPF_WRITE */ 6022 meta.raw_mode = true; 6023 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 6024 6025 if (may_be_null) 6026 *reg = saved_reg; 6027 6028 return err; 6029 } 6030 6031 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 6032 u32 regno) 6033 { 6034 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 6035 bool may_be_null = type_may_be_null(mem_reg->type); 6036 struct bpf_reg_state saved_reg; 6037 struct bpf_call_arg_meta meta; 6038 int err; 6039 6040 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 6041 6042 memset(&meta, 0, sizeof(meta)); 6043 6044 if (may_be_null) { 6045 saved_reg = *mem_reg; 6046 mark_ptr_not_null_reg(mem_reg); 6047 } 6048 6049 err = check_mem_size_reg(env, reg, regno, true, &meta); 6050 /* Check access for BPF_WRITE */ 6051 meta.raw_mode = true; 6052 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 6053 6054 if (may_be_null) 6055 *mem_reg = saved_reg; 6056 return err; 6057 } 6058 6059 /* Implementation details: 6060 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 6061 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 6062 * Two bpf_map_lookups (even with the same key) will have different reg->id. 6063 * Two separate bpf_obj_new will also have different reg->id. 6064 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 6065 * clears reg->id after value_or_null->value transition, since the verifier only 6066 * cares about the range of access to valid map value pointer and doesn't care 6067 * about actual address of the map element. 6068 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 6069 * reg->id > 0 after value_or_null->value transition. By doing so 6070 * two bpf_map_lookups will be considered two different pointers that 6071 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 6072 * returned from bpf_obj_new. 6073 * The verifier allows taking only one bpf_spin_lock at a time to avoid 6074 * dead-locks. 6075 * Since only one bpf_spin_lock is allowed the checks are simpler than 6076 * reg_is_refcounted() logic. The verifier needs to remember only 6077 * one spin_lock instead of array of acquired_refs. 6078 * cur_state->active_lock remembers which map value element or allocated 6079 * object got locked and clears it after bpf_spin_unlock. 6080 */ 6081 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 6082 bool is_lock) 6083 { 6084 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6085 struct bpf_verifier_state *cur = env->cur_state; 6086 bool is_const = tnum_is_const(reg->var_off); 6087 u64 val = reg->var_off.value; 6088 struct bpf_map *map = NULL; 6089 struct btf *btf = NULL; 6090 struct btf_record *rec; 6091 6092 if (!is_const) { 6093 verbose(env, 6094 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 6095 regno); 6096 return -EINVAL; 6097 } 6098 if (reg->type == PTR_TO_MAP_VALUE) { 6099 map = reg->map_ptr; 6100 if (!map->btf) { 6101 verbose(env, 6102 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 6103 map->name); 6104 return -EINVAL; 6105 } 6106 } else { 6107 btf = reg->btf; 6108 } 6109 6110 rec = reg_btf_record(reg); 6111 if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) { 6112 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", 6113 map ? map->name : "kptr"); 6114 return -EINVAL; 6115 } 6116 if (rec->spin_lock_off != val + reg->off) { 6117 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", 6118 val + reg->off, rec->spin_lock_off); 6119 return -EINVAL; 6120 } 6121 if (is_lock) { 6122 if (cur->active_lock.ptr) { 6123 verbose(env, 6124 "Locking two bpf_spin_locks are not allowed\n"); 6125 return -EINVAL; 6126 } 6127 if (map) 6128 cur->active_lock.ptr = map; 6129 else 6130 cur->active_lock.ptr = btf; 6131 cur->active_lock.id = reg->id; 6132 } else { 6133 void *ptr; 6134 6135 if (map) 6136 ptr = map; 6137 else 6138 ptr = btf; 6139 6140 if (!cur->active_lock.ptr) { 6141 verbose(env, "bpf_spin_unlock without taking a lock\n"); 6142 return -EINVAL; 6143 } 6144 if (cur->active_lock.ptr != ptr || 6145 cur->active_lock.id != reg->id) { 6146 verbose(env, "bpf_spin_unlock of different lock\n"); 6147 return -EINVAL; 6148 } 6149 6150 invalidate_non_owning_refs(env); 6151 6152 cur->active_lock.ptr = NULL; 6153 cur->active_lock.id = 0; 6154 } 6155 return 0; 6156 } 6157 6158 static int process_timer_func(struct bpf_verifier_env *env, int regno, 6159 struct bpf_call_arg_meta *meta) 6160 { 6161 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6162 bool is_const = tnum_is_const(reg->var_off); 6163 struct bpf_map *map = reg->map_ptr; 6164 u64 val = reg->var_off.value; 6165 6166 if (!is_const) { 6167 verbose(env, 6168 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 6169 regno); 6170 return -EINVAL; 6171 } 6172 if (!map->btf) { 6173 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 6174 map->name); 6175 return -EINVAL; 6176 } 6177 if (!btf_record_has_field(map->record, BPF_TIMER)) { 6178 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); 6179 return -EINVAL; 6180 } 6181 if (map->record->timer_off != val + reg->off) { 6182 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 6183 val + reg->off, map->record->timer_off); 6184 return -EINVAL; 6185 } 6186 if (meta->map_ptr) { 6187 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 6188 return -EFAULT; 6189 } 6190 meta->map_uid = reg->map_uid; 6191 meta->map_ptr = map; 6192 return 0; 6193 } 6194 6195 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 6196 struct bpf_call_arg_meta *meta) 6197 { 6198 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6199 struct bpf_map *map_ptr = reg->map_ptr; 6200 struct btf_field *kptr_field; 6201 u32 kptr_off; 6202 6203 if (!tnum_is_const(reg->var_off)) { 6204 verbose(env, 6205 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 6206 regno); 6207 return -EINVAL; 6208 } 6209 if (!map_ptr->btf) { 6210 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 6211 map_ptr->name); 6212 return -EINVAL; 6213 } 6214 if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 6215 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 6216 return -EINVAL; 6217 } 6218 6219 meta->map_ptr = map_ptr; 6220 kptr_off = reg->off + reg->var_off.value; 6221 kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 6222 if (!kptr_field) { 6223 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 6224 return -EACCES; 6225 } 6226 if (kptr_field->type != BPF_KPTR_REF) { 6227 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 6228 return -EACCES; 6229 } 6230 meta->kptr_field = kptr_field; 6231 return 0; 6232 } 6233 6234 /* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK 6235 * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR. 6236 * 6237 * In both cases we deal with the first 8 bytes, but need to mark the next 8 6238 * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of 6239 * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object. 6240 * 6241 * Mutability of bpf_dynptr is at two levels, one is at the level of struct 6242 * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct 6243 * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can 6244 * mutate the view of the dynptr and also possibly destroy it. In the latter 6245 * case, it cannot mutate the bpf_dynptr itself but it can still mutate the 6246 * memory that dynptr points to. 6247 * 6248 * The verifier will keep track both levels of mutation (bpf_dynptr's in 6249 * reg->type and the memory's in reg->dynptr.type), but there is no support for 6250 * readonly dynptr view yet, hence only the first case is tracked and checked. 6251 * 6252 * This is consistent with how C applies the const modifier to a struct object, 6253 * where the pointer itself inside bpf_dynptr becomes const but not what it 6254 * points to. 6255 * 6256 * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument 6257 * type, and declare it as 'const struct bpf_dynptr *' in their prototype. 6258 */ 6259 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, 6260 enum bpf_arg_type arg_type) 6261 { 6262 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6263 int err; 6264 6265 /* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an 6266 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*): 6267 */ 6268 if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) { 6269 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); 6270 return -EFAULT; 6271 } 6272 6273 /* MEM_UNINIT - Points to memory that is an appropriate candidate for 6274 * constructing a mutable bpf_dynptr object. 6275 * 6276 * Currently, this is only possible with PTR_TO_STACK 6277 * pointing to a region of at least 16 bytes which doesn't 6278 * contain an existing bpf_dynptr. 6279 * 6280 * MEM_RDONLY - Points to a initialized bpf_dynptr that will not be 6281 * mutated or destroyed. However, the memory it points to 6282 * may be mutated. 6283 * 6284 * None - Points to a initialized dynptr that can be mutated and 6285 * destroyed, including mutation of the memory it points 6286 * to. 6287 */ 6288 if (arg_type & MEM_UNINIT) { 6289 int i; 6290 6291 if (!is_dynptr_reg_valid_uninit(env, reg)) { 6292 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 6293 return -EINVAL; 6294 } 6295 6296 /* we write BPF_DW bits (8 bytes) at a time */ 6297 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 6298 err = check_mem_access(env, insn_idx, regno, 6299 i, BPF_DW, BPF_WRITE, -1, false); 6300 if (err) 6301 return err; 6302 } 6303 6304 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx); 6305 } else /* MEM_RDONLY and None case from above */ { 6306 /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ 6307 if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) { 6308 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); 6309 return -EINVAL; 6310 } 6311 6312 if (!is_dynptr_reg_valid_init(env, reg)) { 6313 verbose(env, 6314 "Expected an initialized dynptr as arg #%d\n", 6315 regno); 6316 return -EINVAL; 6317 } 6318 6319 /* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */ 6320 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { 6321 const char *err_extra = ""; 6322 6323 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 6324 case DYNPTR_TYPE_LOCAL: 6325 err_extra = "local"; 6326 break; 6327 case DYNPTR_TYPE_RINGBUF: 6328 err_extra = "ringbuf"; 6329 break; 6330 case DYNPTR_TYPE_SKB: 6331 err_extra = "skb "; 6332 break; 6333 case DYNPTR_TYPE_XDP: 6334 err_extra = "xdp "; 6335 break; 6336 default: 6337 err_extra = "<unknown>"; 6338 break; 6339 } 6340 verbose(env, 6341 "Expected a dynptr of type %s as arg #%d\n", 6342 err_extra, regno); 6343 return -EINVAL; 6344 } 6345 6346 err = mark_dynptr_read(env, reg); 6347 } 6348 return err; 6349 } 6350 6351 static bool arg_type_is_mem_size(enum bpf_arg_type type) 6352 { 6353 return type == ARG_CONST_SIZE || 6354 type == ARG_CONST_SIZE_OR_ZERO; 6355 } 6356 6357 static bool arg_type_is_release(enum bpf_arg_type type) 6358 { 6359 return type & OBJ_RELEASE; 6360 } 6361 6362 static bool arg_type_is_dynptr(enum bpf_arg_type type) 6363 { 6364 return base_type(type) == ARG_PTR_TO_DYNPTR; 6365 } 6366 6367 static int int_ptr_type_to_size(enum bpf_arg_type type) 6368 { 6369 if (type == ARG_PTR_TO_INT) 6370 return sizeof(u32); 6371 else if (type == ARG_PTR_TO_LONG) 6372 return sizeof(u64); 6373 6374 return -EINVAL; 6375 } 6376 6377 static int resolve_map_arg_type(struct bpf_verifier_env *env, 6378 const struct bpf_call_arg_meta *meta, 6379 enum bpf_arg_type *arg_type) 6380 { 6381 if (!meta->map_ptr) { 6382 /* kernel subsystem misconfigured verifier */ 6383 verbose(env, "invalid map_ptr to access map->type\n"); 6384 return -EACCES; 6385 } 6386 6387 switch (meta->map_ptr->map_type) { 6388 case BPF_MAP_TYPE_SOCKMAP: 6389 case BPF_MAP_TYPE_SOCKHASH: 6390 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 6391 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 6392 } else { 6393 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 6394 return -EINVAL; 6395 } 6396 break; 6397 case BPF_MAP_TYPE_BLOOM_FILTER: 6398 if (meta->func_id == BPF_FUNC_map_peek_elem) 6399 *arg_type = ARG_PTR_TO_MAP_VALUE; 6400 break; 6401 default: 6402 break; 6403 } 6404 return 0; 6405 } 6406 6407 struct bpf_reg_types { 6408 const enum bpf_reg_type types[10]; 6409 u32 *btf_id; 6410 }; 6411 6412 static const struct bpf_reg_types sock_types = { 6413 .types = { 6414 PTR_TO_SOCK_COMMON, 6415 PTR_TO_SOCKET, 6416 PTR_TO_TCP_SOCK, 6417 PTR_TO_XDP_SOCK, 6418 }, 6419 }; 6420 6421 #ifdef CONFIG_NET 6422 static const struct bpf_reg_types btf_id_sock_common_types = { 6423 .types = { 6424 PTR_TO_SOCK_COMMON, 6425 PTR_TO_SOCKET, 6426 PTR_TO_TCP_SOCK, 6427 PTR_TO_XDP_SOCK, 6428 PTR_TO_BTF_ID, 6429 PTR_TO_BTF_ID | PTR_TRUSTED, 6430 }, 6431 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 6432 }; 6433 #endif 6434 6435 static const struct bpf_reg_types mem_types = { 6436 .types = { 6437 PTR_TO_STACK, 6438 PTR_TO_PACKET, 6439 PTR_TO_PACKET_META, 6440 PTR_TO_MAP_KEY, 6441 PTR_TO_MAP_VALUE, 6442 PTR_TO_MEM, 6443 PTR_TO_MEM | MEM_RINGBUF, 6444 PTR_TO_BUF, 6445 }, 6446 }; 6447 6448 static const struct bpf_reg_types int_ptr_types = { 6449 .types = { 6450 PTR_TO_STACK, 6451 PTR_TO_PACKET, 6452 PTR_TO_PACKET_META, 6453 PTR_TO_MAP_KEY, 6454 PTR_TO_MAP_VALUE, 6455 }, 6456 }; 6457 6458 static const struct bpf_reg_types spin_lock_types = { 6459 .types = { 6460 PTR_TO_MAP_VALUE, 6461 PTR_TO_BTF_ID | MEM_ALLOC, 6462 } 6463 }; 6464 6465 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 6466 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 6467 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 6468 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 6469 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 6470 static const struct bpf_reg_types btf_ptr_types = { 6471 .types = { 6472 PTR_TO_BTF_ID, 6473 PTR_TO_BTF_ID | PTR_TRUSTED, 6474 PTR_TO_BTF_ID | MEM_RCU, 6475 }, 6476 }; 6477 static const struct bpf_reg_types percpu_btf_ptr_types = { 6478 .types = { 6479 PTR_TO_BTF_ID | MEM_PERCPU, 6480 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 6481 } 6482 }; 6483 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 6484 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 6485 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 6486 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 6487 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 6488 static const struct bpf_reg_types dynptr_types = { 6489 .types = { 6490 PTR_TO_STACK, 6491 CONST_PTR_TO_DYNPTR, 6492 } 6493 }; 6494 6495 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 6496 [ARG_PTR_TO_MAP_KEY] = &mem_types, 6497 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 6498 [ARG_CONST_SIZE] = &scalar_types, 6499 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 6500 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 6501 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 6502 [ARG_PTR_TO_CTX] = &context_types, 6503 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 6504 #ifdef CONFIG_NET 6505 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 6506 #endif 6507 [ARG_PTR_TO_SOCKET] = &fullsock_types, 6508 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 6509 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 6510 [ARG_PTR_TO_MEM] = &mem_types, 6511 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 6512 [ARG_PTR_TO_INT] = &int_ptr_types, 6513 [ARG_PTR_TO_LONG] = &int_ptr_types, 6514 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 6515 [ARG_PTR_TO_FUNC] = &func_ptr_types, 6516 [ARG_PTR_TO_STACK] = &stack_ptr_types, 6517 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 6518 [ARG_PTR_TO_TIMER] = &timer_types, 6519 [ARG_PTR_TO_KPTR] = &kptr_types, 6520 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 6521 }; 6522 6523 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 6524 enum bpf_arg_type arg_type, 6525 const u32 *arg_btf_id, 6526 struct bpf_call_arg_meta *meta) 6527 { 6528 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6529 enum bpf_reg_type expected, type = reg->type; 6530 const struct bpf_reg_types *compatible; 6531 int i, j; 6532 6533 compatible = compatible_reg_types[base_type(arg_type)]; 6534 if (!compatible) { 6535 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 6536 return -EFAULT; 6537 } 6538 6539 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 6540 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 6541 * 6542 * Same for MAYBE_NULL: 6543 * 6544 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 6545 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 6546 * 6547 * Therefore we fold these flags depending on the arg_type before comparison. 6548 */ 6549 if (arg_type & MEM_RDONLY) 6550 type &= ~MEM_RDONLY; 6551 if (arg_type & PTR_MAYBE_NULL) 6552 type &= ~PTR_MAYBE_NULL; 6553 6554 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 6555 expected = compatible->types[i]; 6556 if (expected == NOT_INIT) 6557 break; 6558 6559 if (type == expected) 6560 goto found; 6561 } 6562 6563 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 6564 for (j = 0; j + 1 < i; j++) 6565 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 6566 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 6567 return -EACCES; 6568 6569 found: 6570 if (base_type(reg->type) != PTR_TO_BTF_ID) 6571 return 0; 6572 6573 switch ((int)reg->type) { 6574 case PTR_TO_BTF_ID: 6575 case PTR_TO_BTF_ID | PTR_TRUSTED: 6576 case PTR_TO_BTF_ID | MEM_RCU: 6577 { 6578 /* For bpf_sk_release, it needs to match against first member 6579 * 'struct sock_common', hence make an exception for it. This 6580 * allows bpf_sk_release to work for multiple socket types. 6581 */ 6582 bool strict_type_match = arg_type_is_release(arg_type) && 6583 meta->func_id != BPF_FUNC_sk_release; 6584 6585 if (!arg_btf_id) { 6586 if (!compatible->btf_id) { 6587 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 6588 return -EFAULT; 6589 } 6590 arg_btf_id = compatible->btf_id; 6591 } 6592 6593 if (meta->func_id == BPF_FUNC_kptr_xchg) { 6594 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 6595 return -EACCES; 6596 } else { 6597 if (arg_btf_id == BPF_PTR_POISON) { 6598 verbose(env, "verifier internal error:"); 6599 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", 6600 regno); 6601 return -EACCES; 6602 } 6603 6604 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 6605 btf_vmlinux, *arg_btf_id, 6606 strict_type_match)) { 6607 verbose(env, "R%d is of type %s but %s is expected\n", 6608 regno, kernel_type_name(reg->btf, reg->btf_id), 6609 kernel_type_name(btf_vmlinux, *arg_btf_id)); 6610 return -EACCES; 6611 } 6612 } 6613 break; 6614 } 6615 case PTR_TO_BTF_ID | MEM_ALLOC: 6616 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock) { 6617 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); 6618 return -EFAULT; 6619 } 6620 /* Handled by helper specific checks */ 6621 break; 6622 case PTR_TO_BTF_ID | MEM_PERCPU: 6623 case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED: 6624 /* Handled by helper specific checks */ 6625 break; 6626 default: 6627 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); 6628 return -EFAULT; 6629 } 6630 return 0; 6631 } 6632 6633 static struct btf_field * 6634 reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) 6635 { 6636 struct btf_field *field; 6637 struct btf_record *rec; 6638 6639 rec = reg_btf_record(reg); 6640 if (!rec) 6641 return NULL; 6642 6643 field = btf_record_find(rec, off, fields); 6644 if (!field) 6645 return NULL; 6646 6647 return field; 6648 } 6649 6650 int check_func_arg_reg_off(struct bpf_verifier_env *env, 6651 const struct bpf_reg_state *reg, int regno, 6652 enum bpf_arg_type arg_type) 6653 { 6654 u32 type = reg->type; 6655 6656 /* When referenced register is passed to release function, its fixed 6657 * offset must be 0. 6658 * 6659 * We will check arg_type_is_release reg has ref_obj_id when storing 6660 * meta->release_regno. 6661 */ 6662 if (arg_type_is_release(arg_type)) { 6663 /* ARG_PTR_TO_DYNPTR with OBJ_RELEASE is a bit special, as it 6664 * may not directly point to the object being released, but to 6665 * dynptr pointing to such object, which might be at some offset 6666 * on the stack. In that case, we simply to fallback to the 6667 * default handling. 6668 */ 6669 if (arg_type_is_dynptr(arg_type) && type == PTR_TO_STACK) 6670 return 0; 6671 6672 if ((type_is_ptr_alloc_obj(type) || type_is_non_owning_ref(type)) && reg->off) { 6673 if (reg_find_field_offset(reg, reg->off, BPF_GRAPH_NODE_OR_ROOT)) 6674 return __check_ptr_off_reg(env, reg, regno, true); 6675 6676 verbose(env, "R%d must have zero offset when passed to release func\n", 6677 regno); 6678 verbose(env, "No graph node or root found at R%d type:%s off:%d\n", regno, 6679 kernel_type_name(reg->btf, reg->btf_id), reg->off); 6680 return -EINVAL; 6681 } 6682 6683 /* Doing check_ptr_off_reg check for the offset will catch this 6684 * because fixed_off_ok is false, but checking here allows us 6685 * to give the user a better error message. 6686 */ 6687 if (reg->off) { 6688 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", 6689 regno); 6690 return -EINVAL; 6691 } 6692 return __check_ptr_off_reg(env, reg, regno, false); 6693 } 6694 6695 switch (type) { 6696 /* Pointer types where both fixed and variable offset is explicitly allowed: */ 6697 case PTR_TO_STACK: 6698 case PTR_TO_PACKET: 6699 case PTR_TO_PACKET_META: 6700 case PTR_TO_MAP_KEY: 6701 case PTR_TO_MAP_VALUE: 6702 case PTR_TO_MEM: 6703 case PTR_TO_MEM | MEM_RDONLY: 6704 case PTR_TO_MEM | MEM_RINGBUF: 6705 case PTR_TO_BUF: 6706 case PTR_TO_BUF | MEM_RDONLY: 6707 case SCALAR_VALUE: 6708 return 0; 6709 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 6710 * fixed offset. 6711 */ 6712 case PTR_TO_BTF_ID: 6713 case PTR_TO_BTF_ID | MEM_ALLOC: 6714 case PTR_TO_BTF_ID | PTR_TRUSTED: 6715 case PTR_TO_BTF_ID | MEM_RCU: 6716 case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF: 6717 /* When referenced PTR_TO_BTF_ID is passed to release function, 6718 * its fixed offset must be 0. In the other cases, fixed offset 6719 * can be non-zero. This was already checked above. So pass 6720 * fixed_off_ok as true to allow fixed offset for all other 6721 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we 6722 * still need to do checks instead of returning. 6723 */ 6724 return __check_ptr_off_reg(env, reg, regno, true); 6725 default: 6726 return __check_ptr_off_reg(env, reg, regno, false); 6727 } 6728 } 6729 6730 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, 6731 const struct bpf_func_proto *fn, 6732 struct bpf_reg_state *regs) 6733 { 6734 struct bpf_reg_state *state = NULL; 6735 int i; 6736 6737 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) 6738 if (arg_type_is_dynptr(fn->arg_type[i])) { 6739 if (state) { 6740 verbose(env, "verifier internal error: multiple dynptr args\n"); 6741 return NULL; 6742 } 6743 state = ®s[BPF_REG_1 + i]; 6744 } 6745 6746 if (!state) 6747 verbose(env, "verifier internal error: no dynptr arg found\n"); 6748 6749 return state; 6750 } 6751 6752 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 6753 { 6754 struct bpf_func_state *state = func(env, reg); 6755 int spi; 6756 6757 if (reg->type == CONST_PTR_TO_DYNPTR) 6758 return reg->id; 6759 spi = dynptr_get_spi(env, reg); 6760 if (spi < 0) 6761 return spi; 6762 return state->stack[spi].spilled_ptr.id; 6763 } 6764 6765 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 6766 { 6767 struct bpf_func_state *state = func(env, reg); 6768 int spi; 6769 6770 if (reg->type == CONST_PTR_TO_DYNPTR) 6771 return reg->ref_obj_id; 6772 spi = dynptr_get_spi(env, reg); 6773 if (spi < 0) 6774 return spi; 6775 return state->stack[spi].spilled_ptr.ref_obj_id; 6776 } 6777 6778 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, 6779 struct bpf_reg_state *reg) 6780 { 6781 struct bpf_func_state *state = func(env, reg); 6782 int spi; 6783 6784 if (reg->type == CONST_PTR_TO_DYNPTR) 6785 return reg->dynptr.type; 6786 6787 spi = __get_spi(reg->off); 6788 if (spi < 0) { 6789 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); 6790 return BPF_DYNPTR_TYPE_INVALID; 6791 } 6792 6793 return state->stack[spi].spilled_ptr.dynptr.type; 6794 } 6795 6796 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 6797 struct bpf_call_arg_meta *meta, 6798 const struct bpf_func_proto *fn, 6799 int insn_idx) 6800 { 6801 u32 regno = BPF_REG_1 + arg; 6802 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6803 enum bpf_arg_type arg_type = fn->arg_type[arg]; 6804 enum bpf_reg_type type = reg->type; 6805 u32 *arg_btf_id = NULL; 6806 int err = 0; 6807 6808 if (arg_type == ARG_DONTCARE) 6809 return 0; 6810 6811 err = check_reg_arg(env, regno, SRC_OP); 6812 if (err) 6813 return err; 6814 6815 if (arg_type == ARG_ANYTHING) { 6816 if (is_pointer_value(env, regno)) { 6817 verbose(env, "R%d leaks addr into helper function\n", 6818 regno); 6819 return -EACCES; 6820 } 6821 return 0; 6822 } 6823 6824 if (type_is_pkt_pointer(type) && 6825 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 6826 verbose(env, "helper access to the packet is not allowed\n"); 6827 return -EACCES; 6828 } 6829 6830 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { 6831 err = resolve_map_arg_type(env, meta, &arg_type); 6832 if (err) 6833 return err; 6834 } 6835 6836 if (register_is_null(reg) && type_may_be_null(arg_type)) 6837 /* A NULL register has a SCALAR_VALUE type, so skip 6838 * type checking. 6839 */ 6840 goto skip_type_check; 6841 6842 /* arg_btf_id and arg_size are in a union. */ 6843 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || 6844 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 6845 arg_btf_id = fn->arg_btf_id[arg]; 6846 6847 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 6848 if (err) 6849 return err; 6850 6851 err = check_func_arg_reg_off(env, reg, regno, arg_type); 6852 if (err) 6853 return err; 6854 6855 skip_type_check: 6856 if (arg_type_is_release(arg_type)) { 6857 if (arg_type_is_dynptr(arg_type)) { 6858 struct bpf_func_state *state = func(env, reg); 6859 int spi; 6860 6861 /* Only dynptr created on stack can be released, thus 6862 * the get_spi and stack state checks for spilled_ptr 6863 * should only be done before process_dynptr_func for 6864 * PTR_TO_STACK. 6865 */ 6866 if (reg->type == PTR_TO_STACK) { 6867 spi = dynptr_get_spi(env, reg); 6868 if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) { 6869 verbose(env, "arg %d is an unacquired reference\n", regno); 6870 return -EINVAL; 6871 } 6872 } else { 6873 verbose(env, "cannot release unowned const bpf_dynptr\n"); 6874 return -EINVAL; 6875 } 6876 } else if (!reg->ref_obj_id && !register_is_null(reg)) { 6877 verbose(env, "R%d must be referenced when passed to release function\n", 6878 regno); 6879 return -EINVAL; 6880 } 6881 if (meta->release_regno) { 6882 verbose(env, "verifier internal error: more than one release argument\n"); 6883 return -EFAULT; 6884 } 6885 meta->release_regno = regno; 6886 } 6887 6888 if (reg->ref_obj_id) { 6889 if (meta->ref_obj_id) { 6890 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 6891 regno, reg->ref_obj_id, 6892 meta->ref_obj_id); 6893 return -EFAULT; 6894 } 6895 meta->ref_obj_id = reg->ref_obj_id; 6896 } 6897 6898 switch (base_type(arg_type)) { 6899 case ARG_CONST_MAP_PTR: 6900 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 6901 if (meta->map_ptr) { 6902 /* Use map_uid (which is unique id of inner map) to reject: 6903 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 6904 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 6905 * if (inner_map1 && inner_map2) { 6906 * timer = bpf_map_lookup_elem(inner_map1); 6907 * if (timer) 6908 * // mismatch would have been allowed 6909 * bpf_timer_init(timer, inner_map2); 6910 * } 6911 * 6912 * Comparing map_ptr is enough to distinguish normal and outer maps. 6913 */ 6914 if (meta->map_ptr != reg->map_ptr || 6915 meta->map_uid != reg->map_uid) { 6916 verbose(env, 6917 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 6918 meta->map_uid, reg->map_uid); 6919 return -EINVAL; 6920 } 6921 } 6922 meta->map_ptr = reg->map_ptr; 6923 meta->map_uid = reg->map_uid; 6924 break; 6925 case ARG_PTR_TO_MAP_KEY: 6926 /* bpf_map_xxx(..., map_ptr, ..., key) call: 6927 * check that [key, key + map->key_size) are within 6928 * stack limits and initialized 6929 */ 6930 if (!meta->map_ptr) { 6931 /* in function declaration map_ptr must come before 6932 * map_key, so that it's verified and known before 6933 * we have to check map_key here. Otherwise it means 6934 * that kernel subsystem misconfigured verifier 6935 */ 6936 verbose(env, "invalid map_ptr to access map->key\n"); 6937 return -EACCES; 6938 } 6939 err = check_helper_mem_access(env, regno, 6940 meta->map_ptr->key_size, false, 6941 NULL); 6942 break; 6943 case ARG_PTR_TO_MAP_VALUE: 6944 if (type_may_be_null(arg_type) && register_is_null(reg)) 6945 return 0; 6946 6947 /* bpf_map_xxx(..., map_ptr, ..., value) call: 6948 * check [value, value + map->value_size) validity 6949 */ 6950 if (!meta->map_ptr) { 6951 /* kernel subsystem misconfigured verifier */ 6952 verbose(env, "invalid map_ptr to access map->value\n"); 6953 return -EACCES; 6954 } 6955 meta->raw_mode = arg_type & MEM_UNINIT; 6956 err = check_helper_mem_access(env, regno, 6957 meta->map_ptr->value_size, false, 6958 meta); 6959 break; 6960 case ARG_PTR_TO_PERCPU_BTF_ID: 6961 if (!reg->btf_id) { 6962 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 6963 return -EACCES; 6964 } 6965 meta->ret_btf = reg->btf; 6966 meta->ret_btf_id = reg->btf_id; 6967 break; 6968 case ARG_PTR_TO_SPIN_LOCK: 6969 if (in_rbtree_lock_required_cb(env)) { 6970 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); 6971 return -EACCES; 6972 } 6973 if (meta->func_id == BPF_FUNC_spin_lock) { 6974 err = process_spin_lock(env, regno, true); 6975 if (err) 6976 return err; 6977 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 6978 err = process_spin_lock(env, regno, false); 6979 if (err) 6980 return err; 6981 } else { 6982 verbose(env, "verifier internal error\n"); 6983 return -EFAULT; 6984 } 6985 break; 6986 case ARG_PTR_TO_TIMER: 6987 err = process_timer_func(env, regno, meta); 6988 if (err) 6989 return err; 6990 break; 6991 case ARG_PTR_TO_FUNC: 6992 meta->subprogno = reg->subprogno; 6993 break; 6994 case ARG_PTR_TO_MEM: 6995 /* The access to this pointer is only checked when we hit the 6996 * next is_mem_size argument below. 6997 */ 6998 meta->raw_mode = arg_type & MEM_UNINIT; 6999 if (arg_type & MEM_FIXED_SIZE) { 7000 err = check_helper_mem_access(env, regno, 7001 fn->arg_size[arg], false, 7002 meta); 7003 } 7004 break; 7005 case ARG_CONST_SIZE: 7006 err = check_mem_size_reg(env, reg, regno, false, meta); 7007 break; 7008 case ARG_CONST_SIZE_OR_ZERO: 7009 err = check_mem_size_reg(env, reg, regno, true, meta); 7010 break; 7011 case ARG_PTR_TO_DYNPTR: 7012 err = process_dynptr_func(env, regno, insn_idx, arg_type); 7013 if (err) 7014 return err; 7015 break; 7016 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 7017 if (!tnum_is_const(reg->var_off)) { 7018 verbose(env, "R%d is not a known constant'\n", 7019 regno); 7020 return -EACCES; 7021 } 7022 meta->mem_size = reg->var_off.value; 7023 err = mark_chain_precision(env, regno); 7024 if (err) 7025 return err; 7026 break; 7027 case ARG_PTR_TO_INT: 7028 case ARG_PTR_TO_LONG: 7029 { 7030 int size = int_ptr_type_to_size(arg_type); 7031 7032 err = check_helper_mem_access(env, regno, size, false, meta); 7033 if (err) 7034 return err; 7035 err = check_ptr_alignment(env, reg, 0, size, true); 7036 break; 7037 } 7038 case ARG_PTR_TO_CONST_STR: 7039 { 7040 struct bpf_map *map = reg->map_ptr; 7041 int map_off; 7042 u64 map_addr; 7043 char *str_ptr; 7044 7045 if (!bpf_map_is_rdonly(map)) { 7046 verbose(env, "R%d does not point to a readonly map'\n", regno); 7047 return -EACCES; 7048 } 7049 7050 if (!tnum_is_const(reg->var_off)) { 7051 verbose(env, "R%d is not a constant address'\n", regno); 7052 return -EACCES; 7053 } 7054 7055 if (!map->ops->map_direct_value_addr) { 7056 verbose(env, "no direct value access support for this map type\n"); 7057 return -EACCES; 7058 } 7059 7060 err = check_map_access(env, regno, reg->off, 7061 map->value_size - reg->off, false, 7062 ACCESS_HELPER); 7063 if (err) 7064 return err; 7065 7066 map_off = reg->off + reg->var_off.value; 7067 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 7068 if (err) { 7069 verbose(env, "direct value access on string failed\n"); 7070 return err; 7071 } 7072 7073 str_ptr = (char *)(long)(map_addr); 7074 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 7075 verbose(env, "string is not zero-terminated\n"); 7076 return -EINVAL; 7077 } 7078 break; 7079 } 7080 case ARG_PTR_TO_KPTR: 7081 err = process_kptr_func(env, regno, meta); 7082 if (err) 7083 return err; 7084 break; 7085 } 7086 7087 return err; 7088 } 7089 7090 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 7091 { 7092 enum bpf_attach_type eatype = env->prog->expected_attach_type; 7093 enum bpf_prog_type type = resolve_prog_type(env->prog); 7094 7095 if (func_id != BPF_FUNC_map_update_elem) 7096 return false; 7097 7098 /* It's not possible to get access to a locked struct sock in these 7099 * contexts, so updating is safe. 7100 */ 7101 switch (type) { 7102 case BPF_PROG_TYPE_TRACING: 7103 if (eatype == BPF_TRACE_ITER) 7104 return true; 7105 break; 7106 case BPF_PROG_TYPE_SOCKET_FILTER: 7107 case BPF_PROG_TYPE_SCHED_CLS: 7108 case BPF_PROG_TYPE_SCHED_ACT: 7109 case BPF_PROG_TYPE_XDP: 7110 case BPF_PROG_TYPE_SK_REUSEPORT: 7111 case BPF_PROG_TYPE_FLOW_DISSECTOR: 7112 case BPF_PROG_TYPE_SK_LOOKUP: 7113 return true; 7114 default: 7115 break; 7116 } 7117 7118 verbose(env, "cannot update sockmap in this context\n"); 7119 return false; 7120 } 7121 7122 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 7123 { 7124 return env->prog->jit_requested && 7125 bpf_jit_supports_subprog_tailcalls(); 7126 } 7127 7128 static int check_map_func_compatibility(struct bpf_verifier_env *env, 7129 struct bpf_map *map, int func_id) 7130 { 7131 if (!map) 7132 return 0; 7133 7134 /* We need a two way check, first is from map perspective ... */ 7135 switch (map->map_type) { 7136 case BPF_MAP_TYPE_PROG_ARRAY: 7137 if (func_id != BPF_FUNC_tail_call) 7138 goto error; 7139 break; 7140 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 7141 if (func_id != BPF_FUNC_perf_event_read && 7142 func_id != BPF_FUNC_perf_event_output && 7143 func_id != BPF_FUNC_skb_output && 7144 func_id != BPF_FUNC_perf_event_read_value && 7145 func_id != BPF_FUNC_xdp_output) 7146 goto error; 7147 break; 7148 case BPF_MAP_TYPE_RINGBUF: 7149 if (func_id != BPF_FUNC_ringbuf_output && 7150 func_id != BPF_FUNC_ringbuf_reserve && 7151 func_id != BPF_FUNC_ringbuf_query && 7152 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 7153 func_id != BPF_FUNC_ringbuf_submit_dynptr && 7154 func_id != BPF_FUNC_ringbuf_discard_dynptr) 7155 goto error; 7156 break; 7157 case BPF_MAP_TYPE_USER_RINGBUF: 7158 if (func_id != BPF_FUNC_user_ringbuf_drain) 7159 goto error; 7160 break; 7161 case BPF_MAP_TYPE_STACK_TRACE: 7162 if (func_id != BPF_FUNC_get_stackid) 7163 goto error; 7164 break; 7165 case BPF_MAP_TYPE_CGROUP_ARRAY: 7166 if (func_id != BPF_FUNC_skb_under_cgroup && 7167 func_id != BPF_FUNC_current_task_under_cgroup) 7168 goto error; 7169 break; 7170 case BPF_MAP_TYPE_CGROUP_STORAGE: 7171 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 7172 if (func_id != BPF_FUNC_get_local_storage) 7173 goto error; 7174 break; 7175 case BPF_MAP_TYPE_DEVMAP: 7176 case BPF_MAP_TYPE_DEVMAP_HASH: 7177 if (func_id != BPF_FUNC_redirect_map && 7178 func_id != BPF_FUNC_map_lookup_elem) 7179 goto error; 7180 break; 7181 /* Restrict bpf side of cpumap and xskmap, open when use-cases 7182 * appear. 7183 */ 7184 case BPF_MAP_TYPE_CPUMAP: 7185 if (func_id != BPF_FUNC_redirect_map) 7186 goto error; 7187 break; 7188 case BPF_MAP_TYPE_XSKMAP: 7189 if (func_id != BPF_FUNC_redirect_map && 7190 func_id != BPF_FUNC_map_lookup_elem) 7191 goto error; 7192 break; 7193 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 7194 case BPF_MAP_TYPE_HASH_OF_MAPS: 7195 if (func_id != BPF_FUNC_map_lookup_elem) 7196 goto error; 7197 break; 7198 case BPF_MAP_TYPE_SOCKMAP: 7199 if (func_id != BPF_FUNC_sk_redirect_map && 7200 func_id != BPF_FUNC_sock_map_update && 7201 func_id != BPF_FUNC_map_delete_elem && 7202 func_id != BPF_FUNC_msg_redirect_map && 7203 func_id != BPF_FUNC_sk_select_reuseport && 7204 func_id != BPF_FUNC_map_lookup_elem && 7205 !may_update_sockmap(env, func_id)) 7206 goto error; 7207 break; 7208 case BPF_MAP_TYPE_SOCKHASH: 7209 if (func_id != BPF_FUNC_sk_redirect_hash && 7210 func_id != BPF_FUNC_sock_hash_update && 7211 func_id != BPF_FUNC_map_delete_elem && 7212 func_id != BPF_FUNC_msg_redirect_hash && 7213 func_id != BPF_FUNC_sk_select_reuseport && 7214 func_id != BPF_FUNC_map_lookup_elem && 7215 !may_update_sockmap(env, func_id)) 7216 goto error; 7217 break; 7218 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 7219 if (func_id != BPF_FUNC_sk_select_reuseport) 7220 goto error; 7221 break; 7222 case BPF_MAP_TYPE_QUEUE: 7223 case BPF_MAP_TYPE_STACK: 7224 if (func_id != BPF_FUNC_map_peek_elem && 7225 func_id != BPF_FUNC_map_pop_elem && 7226 func_id != BPF_FUNC_map_push_elem) 7227 goto error; 7228 break; 7229 case BPF_MAP_TYPE_SK_STORAGE: 7230 if (func_id != BPF_FUNC_sk_storage_get && 7231 func_id != BPF_FUNC_sk_storage_delete && 7232 func_id != BPF_FUNC_kptr_xchg) 7233 goto error; 7234 break; 7235 case BPF_MAP_TYPE_INODE_STORAGE: 7236 if (func_id != BPF_FUNC_inode_storage_get && 7237 func_id != BPF_FUNC_inode_storage_delete && 7238 func_id != BPF_FUNC_kptr_xchg) 7239 goto error; 7240 break; 7241 case BPF_MAP_TYPE_TASK_STORAGE: 7242 if (func_id != BPF_FUNC_task_storage_get && 7243 func_id != BPF_FUNC_task_storage_delete && 7244 func_id != BPF_FUNC_kptr_xchg) 7245 goto error; 7246 break; 7247 case BPF_MAP_TYPE_CGRP_STORAGE: 7248 if (func_id != BPF_FUNC_cgrp_storage_get && 7249 func_id != BPF_FUNC_cgrp_storage_delete && 7250 func_id != BPF_FUNC_kptr_xchg) 7251 goto error; 7252 break; 7253 case BPF_MAP_TYPE_BLOOM_FILTER: 7254 if (func_id != BPF_FUNC_map_peek_elem && 7255 func_id != BPF_FUNC_map_push_elem) 7256 goto error; 7257 break; 7258 default: 7259 break; 7260 } 7261 7262 /* ... and second from the function itself. */ 7263 switch (func_id) { 7264 case BPF_FUNC_tail_call: 7265 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 7266 goto error; 7267 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 7268 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 7269 return -EINVAL; 7270 } 7271 break; 7272 case BPF_FUNC_perf_event_read: 7273 case BPF_FUNC_perf_event_output: 7274 case BPF_FUNC_perf_event_read_value: 7275 case BPF_FUNC_skb_output: 7276 case BPF_FUNC_xdp_output: 7277 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 7278 goto error; 7279 break; 7280 case BPF_FUNC_ringbuf_output: 7281 case BPF_FUNC_ringbuf_reserve: 7282 case BPF_FUNC_ringbuf_query: 7283 case BPF_FUNC_ringbuf_reserve_dynptr: 7284 case BPF_FUNC_ringbuf_submit_dynptr: 7285 case BPF_FUNC_ringbuf_discard_dynptr: 7286 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 7287 goto error; 7288 break; 7289 case BPF_FUNC_user_ringbuf_drain: 7290 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 7291 goto error; 7292 break; 7293 case BPF_FUNC_get_stackid: 7294 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 7295 goto error; 7296 break; 7297 case BPF_FUNC_current_task_under_cgroup: 7298 case BPF_FUNC_skb_under_cgroup: 7299 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 7300 goto error; 7301 break; 7302 case BPF_FUNC_redirect_map: 7303 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 7304 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 7305 map->map_type != BPF_MAP_TYPE_CPUMAP && 7306 map->map_type != BPF_MAP_TYPE_XSKMAP) 7307 goto error; 7308 break; 7309 case BPF_FUNC_sk_redirect_map: 7310 case BPF_FUNC_msg_redirect_map: 7311 case BPF_FUNC_sock_map_update: 7312 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 7313 goto error; 7314 break; 7315 case BPF_FUNC_sk_redirect_hash: 7316 case BPF_FUNC_msg_redirect_hash: 7317 case BPF_FUNC_sock_hash_update: 7318 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 7319 goto error; 7320 break; 7321 case BPF_FUNC_get_local_storage: 7322 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 7323 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 7324 goto error; 7325 break; 7326 case BPF_FUNC_sk_select_reuseport: 7327 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 7328 map->map_type != BPF_MAP_TYPE_SOCKMAP && 7329 map->map_type != BPF_MAP_TYPE_SOCKHASH) 7330 goto error; 7331 break; 7332 case BPF_FUNC_map_pop_elem: 7333 if (map->map_type != BPF_MAP_TYPE_QUEUE && 7334 map->map_type != BPF_MAP_TYPE_STACK) 7335 goto error; 7336 break; 7337 case BPF_FUNC_map_peek_elem: 7338 case BPF_FUNC_map_push_elem: 7339 if (map->map_type != BPF_MAP_TYPE_QUEUE && 7340 map->map_type != BPF_MAP_TYPE_STACK && 7341 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 7342 goto error; 7343 break; 7344 case BPF_FUNC_map_lookup_percpu_elem: 7345 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 7346 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 7347 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 7348 goto error; 7349 break; 7350 case BPF_FUNC_sk_storage_get: 7351 case BPF_FUNC_sk_storage_delete: 7352 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 7353 goto error; 7354 break; 7355 case BPF_FUNC_inode_storage_get: 7356 case BPF_FUNC_inode_storage_delete: 7357 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 7358 goto error; 7359 break; 7360 case BPF_FUNC_task_storage_get: 7361 case BPF_FUNC_task_storage_delete: 7362 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 7363 goto error; 7364 break; 7365 case BPF_FUNC_cgrp_storage_get: 7366 case BPF_FUNC_cgrp_storage_delete: 7367 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 7368 goto error; 7369 break; 7370 default: 7371 break; 7372 } 7373 7374 return 0; 7375 error: 7376 verbose(env, "cannot pass map_type %d into func %s#%d\n", 7377 map->map_type, func_id_name(func_id), func_id); 7378 return -EINVAL; 7379 } 7380 7381 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 7382 { 7383 int count = 0; 7384 7385 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 7386 count++; 7387 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 7388 count++; 7389 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 7390 count++; 7391 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 7392 count++; 7393 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 7394 count++; 7395 7396 /* We only support one arg being in raw mode at the moment, 7397 * which is sufficient for the helper functions we have 7398 * right now. 7399 */ 7400 return count <= 1; 7401 } 7402 7403 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 7404 { 7405 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 7406 bool has_size = fn->arg_size[arg] != 0; 7407 bool is_next_size = false; 7408 7409 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 7410 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 7411 7412 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 7413 return is_next_size; 7414 7415 return has_size == is_next_size || is_next_size == is_fixed; 7416 } 7417 7418 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 7419 { 7420 /* bpf_xxx(..., buf, len) call will access 'len' 7421 * bytes from memory 'buf'. Both arg types need 7422 * to be paired, so make sure there's no buggy 7423 * helper function specification. 7424 */ 7425 if (arg_type_is_mem_size(fn->arg1_type) || 7426 check_args_pair_invalid(fn, 0) || 7427 check_args_pair_invalid(fn, 1) || 7428 check_args_pair_invalid(fn, 2) || 7429 check_args_pair_invalid(fn, 3) || 7430 check_args_pair_invalid(fn, 4)) 7431 return false; 7432 7433 return true; 7434 } 7435 7436 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 7437 { 7438 int i; 7439 7440 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 7441 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 7442 return !!fn->arg_btf_id[i]; 7443 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 7444 return fn->arg_btf_id[i] == BPF_PTR_POISON; 7445 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 7446 /* arg_btf_id and arg_size are in a union. */ 7447 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 7448 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 7449 return false; 7450 } 7451 7452 return true; 7453 } 7454 7455 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 7456 { 7457 return check_raw_mode_ok(fn) && 7458 check_arg_pair_ok(fn) && 7459 check_btf_id_ok(fn) ? 0 : -EINVAL; 7460 } 7461 7462 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 7463 * are now invalid, so turn them into unknown SCALAR_VALUE. 7464 * 7465 * This also applies to dynptr slices belonging to skb and xdp dynptrs, 7466 * since these slices point to packet data. 7467 */ 7468 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 7469 { 7470 struct bpf_func_state *state; 7471 struct bpf_reg_state *reg; 7472 7473 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 7474 if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) 7475 mark_reg_invalid(env, reg); 7476 })); 7477 } 7478 7479 enum { 7480 AT_PKT_END = -1, 7481 BEYOND_PKT_END = -2, 7482 }; 7483 7484 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 7485 { 7486 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7487 struct bpf_reg_state *reg = &state->regs[regn]; 7488 7489 if (reg->type != PTR_TO_PACKET) 7490 /* PTR_TO_PACKET_META is not supported yet */ 7491 return; 7492 7493 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 7494 * How far beyond pkt_end it goes is unknown. 7495 * if (!range_open) it's the case of pkt >= pkt_end 7496 * if (range_open) it's the case of pkt > pkt_end 7497 * hence this pointer is at least 1 byte bigger than pkt_end 7498 */ 7499 if (range_open) 7500 reg->range = BEYOND_PKT_END; 7501 else 7502 reg->range = AT_PKT_END; 7503 } 7504 7505 /* The pointer with the specified id has released its reference to kernel 7506 * resources. Identify all copies of the same pointer and clear the reference. 7507 */ 7508 static int release_reference(struct bpf_verifier_env *env, 7509 int ref_obj_id) 7510 { 7511 struct bpf_func_state *state; 7512 struct bpf_reg_state *reg; 7513 int err; 7514 7515 err = release_reference_state(cur_func(env), ref_obj_id); 7516 if (err) 7517 return err; 7518 7519 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 7520 if (reg->ref_obj_id == ref_obj_id) 7521 mark_reg_invalid(env, reg); 7522 })); 7523 7524 return 0; 7525 } 7526 7527 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) 7528 { 7529 struct bpf_func_state *unused; 7530 struct bpf_reg_state *reg; 7531 7532 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 7533 if (type_is_non_owning_ref(reg->type)) 7534 mark_reg_invalid(env, reg); 7535 })); 7536 } 7537 7538 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 7539 struct bpf_reg_state *regs) 7540 { 7541 int i; 7542 7543 /* after the call registers r0 - r5 were scratched */ 7544 for (i = 0; i < CALLER_SAVED_REGS; i++) { 7545 mark_reg_not_init(env, regs, caller_saved[i]); 7546 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 7547 } 7548 } 7549 7550 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 7551 struct bpf_func_state *caller, 7552 struct bpf_func_state *callee, 7553 int insn_idx); 7554 7555 static int set_callee_state(struct bpf_verifier_env *env, 7556 struct bpf_func_state *caller, 7557 struct bpf_func_state *callee, int insn_idx); 7558 7559 static bool is_callback_calling_kfunc(u32 btf_id); 7560 7561 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7562 int *insn_idx, int subprog, 7563 set_callee_state_fn set_callee_state_cb) 7564 { 7565 struct bpf_verifier_state *state = env->cur_state; 7566 struct bpf_func_info_aux *func_info_aux; 7567 struct bpf_func_state *caller, *callee; 7568 int err; 7569 bool is_global = false; 7570 7571 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 7572 verbose(env, "the call stack of %d frames is too deep\n", 7573 state->curframe + 2); 7574 return -E2BIG; 7575 } 7576 7577 caller = state->frame[state->curframe]; 7578 if (state->frame[state->curframe + 1]) { 7579 verbose(env, "verifier bug. Frame %d already allocated\n", 7580 state->curframe + 1); 7581 return -EFAULT; 7582 } 7583 7584 func_info_aux = env->prog->aux->func_info_aux; 7585 if (func_info_aux) 7586 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 7587 err = btf_check_subprog_call(env, subprog, caller->regs); 7588 if (err == -EFAULT) 7589 return err; 7590 if (is_global) { 7591 if (err) { 7592 verbose(env, "Caller passes invalid args into func#%d\n", 7593 subprog); 7594 return err; 7595 } else { 7596 if (env->log.level & BPF_LOG_LEVEL) 7597 verbose(env, 7598 "Func#%d is global and valid. Skipping.\n", 7599 subprog); 7600 clear_caller_saved_regs(env, caller->regs); 7601 7602 /* All global functions return a 64-bit SCALAR_VALUE */ 7603 mark_reg_unknown(env, caller->regs, BPF_REG_0); 7604 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7605 7606 /* continue with next insn after call */ 7607 return 0; 7608 } 7609 } 7610 7611 /* set_callee_state is used for direct subprog calls, but we are 7612 * interested in validating only BPF helpers that can call subprogs as 7613 * callbacks 7614 */ 7615 if (set_callee_state_cb != set_callee_state) { 7616 if (bpf_pseudo_kfunc_call(insn) && 7617 !is_callback_calling_kfunc(insn->imm)) { 7618 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", 7619 func_id_name(insn->imm), insn->imm); 7620 return -EFAULT; 7621 } else if (!bpf_pseudo_kfunc_call(insn) && 7622 !is_callback_calling_function(insn->imm)) { /* helper */ 7623 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", 7624 func_id_name(insn->imm), insn->imm); 7625 return -EFAULT; 7626 } 7627 } 7628 7629 if (insn->code == (BPF_JMP | BPF_CALL) && 7630 insn->src_reg == 0 && 7631 insn->imm == BPF_FUNC_timer_set_callback) { 7632 struct bpf_verifier_state *async_cb; 7633 7634 /* there is no real recursion here. timer callbacks are async */ 7635 env->subprog_info[subprog].is_async_cb = true; 7636 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 7637 *insn_idx, subprog); 7638 if (!async_cb) 7639 return -EFAULT; 7640 callee = async_cb->frame[0]; 7641 callee->async_entry_cnt = caller->async_entry_cnt + 1; 7642 7643 /* Convert bpf_timer_set_callback() args into timer callback args */ 7644 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7645 if (err) 7646 return err; 7647 7648 clear_caller_saved_regs(env, caller->regs); 7649 mark_reg_unknown(env, caller->regs, BPF_REG_0); 7650 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7651 /* continue with next insn after call */ 7652 return 0; 7653 } 7654 7655 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 7656 if (!callee) 7657 return -ENOMEM; 7658 state->frame[state->curframe + 1] = callee; 7659 7660 /* callee cannot access r0, r6 - r9 for reading and has to write 7661 * into its own stack before reading from it. 7662 * callee can read/write into caller's stack 7663 */ 7664 init_func_state(env, callee, 7665 /* remember the callsite, it will be used by bpf_exit */ 7666 *insn_idx /* callsite */, 7667 state->curframe + 1 /* frameno within this callchain */, 7668 subprog /* subprog number within this prog */); 7669 7670 /* Transfer references to the callee */ 7671 err = copy_reference_state(callee, caller); 7672 if (err) 7673 goto err_out; 7674 7675 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7676 if (err) 7677 goto err_out; 7678 7679 clear_caller_saved_regs(env, caller->regs); 7680 7681 /* only increment it after check_reg_arg() finished */ 7682 state->curframe++; 7683 7684 /* and go analyze first insn of the callee */ 7685 *insn_idx = env->subprog_info[subprog].start - 1; 7686 7687 if (env->log.level & BPF_LOG_LEVEL) { 7688 verbose(env, "caller:\n"); 7689 print_verifier_state(env, caller, true); 7690 verbose(env, "callee:\n"); 7691 print_verifier_state(env, callee, true); 7692 } 7693 return 0; 7694 7695 err_out: 7696 free_func_state(callee); 7697 state->frame[state->curframe + 1] = NULL; 7698 return err; 7699 } 7700 7701 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 7702 struct bpf_func_state *caller, 7703 struct bpf_func_state *callee) 7704 { 7705 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 7706 * void *callback_ctx, u64 flags); 7707 * callback_fn(struct bpf_map *map, void *key, void *value, 7708 * void *callback_ctx); 7709 */ 7710 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7711 7712 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7713 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7714 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7715 7716 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7717 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7718 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7719 7720 /* pointer to stack or null */ 7721 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 7722 7723 /* unused */ 7724 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7725 return 0; 7726 } 7727 7728 static int set_callee_state(struct bpf_verifier_env *env, 7729 struct bpf_func_state *caller, 7730 struct bpf_func_state *callee, int insn_idx) 7731 { 7732 int i; 7733 7734 /* copy r1 - r5 args that callee can access. The copy includes parent 7735 * pointers, which connects us up to the liveness chain 7736 */ 7737 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 7738 callee->regs[i] = caller->regs[i]; 7739 return 0; 7740 } 7741 7742 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7743 int *insn_idx) 7744 { 7745 int subprog, target_insn; 7746 7747 target_insn = *insn_idx + insn->imm + 1; 7748 subprog = find_subprog(env, target_insn); 7749 if (subprog < 0) { 7750 verbose(env, "verifier bug. No program starts at insn %d\n", 7751 target_insn); 7752 return -EFAULT; 7753 } 7754 7755 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 7756 } 7757 7758 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 7759 struct bpf_func_state *caller, 7760 struct bpf_func_state *callee, 7761 int insn_idx) 7762 { 7763 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 7764 struct bpf_map *map; 7765 int err; 7766 7767 if (bpf_map_ptr_poisoned(insn_aux)) { 7768 verbose(env, "tail_call abusing map_ptr\n"); 7769 return -EINVAL; 7770 } 7771 7772 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 7773 if (!map->ops->map_set_for_each_callback_args || 7774 !map->ops->map_for_each_callback) { 7775 verbose(env, "callback function not allowed for map\n"); 7776 return -ENOTSUPP; 7777 } 7778 7779 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 7780 if (err) 7781 return err; 7782 7783 callee->in_callback_fn = true; 7784 callee->callback_ret_range = tnum_range(0, 1); 7785 return 0; 7786 } 7787 7788 static int set_loop_callback_state(struct bpf_verifier_env *env, 7789 struct bpf_func_state *caller, 7790 struct bpf_func_state *callee, 7791 int insn_idx) 7792 { 7793 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 7794 * u64 flags); 7795 * callback_fn(u32 index, void *callback_ctx); 7796 */ 7797 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 7798 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7799 7800 /* unused */ 7801 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7802 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7803 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7804 7805 callee->in_callback_fn = true; 7806 callee->callback_ret_range = tnum_range(0, 1); 7807 return 0; 7808 } 7809 7810 static int set_timer_callback_state(struct bpf_verifier_env *env, 7811 struct bpf_func_state *caller, 7812 struct bpf_func_state *callee, 7813 int insn_idx) 7814 { 7815 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 7816 7817 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 7818 * callback_fn(struct bpf_map *map, void *key, void *value); 7819 */ 7820 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 7821 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 7822 callee->regs[BPF_REG_1].map_ptr = map_ptr; 7823 7824 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7825 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7826 callee->regs[BPF_REG_2].map_ptr = map_ptr; 7827 7828 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7829 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7830 callee->regs[BPF_REG_3].map_ptr = map_ptr; 7831 7832 /* unused */ 7833 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7834 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7835 callee->in_async_callback_fn = true; 7836 callee->callback_ret_range = tnum_range(0, 1); 7837 return 0; 7838 } 7839 7840 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 7841 struct bpf_func_state *caller, 7842 struct bpf_func_state *callee, 7843 int insn_idx) 7844 { 7845 /* bpf_find_vma(struct task_struct *task, u64 addr, 7846 * void *callback_fn, void *callback_ctx, u64 flags) 7847 * (callback_fn)(struct task_struct *task, 7848 * struct vm_area_struct *vma, void *callback_ctx); 7849 */ 7850 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7851 7852 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 7853 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7854 callee->regs[BPF_REG_2].btf = btf_vmlinux; 7855 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], 7856 7857 /* pointer to stack or null */ 7858 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 7859 7860 /* unused */ 7861 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7862 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7863 callee->in_callback_fn = true; 7864 callee->callback_ret_range = tnum_range(0, 1); 7865 return 0; 7866 } 7867 7868 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 7869 struct bpf_func_state *caller, 7870 struct bpf_func_state *callee, 7871 int insn_idx) 7872 { 7873 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 7874 * callback_ctx, u64 flags); 7875 * callback_fn(const struct bpf_dynptr_t* dynptr, void *callback_ctx); 7876 */ 7877 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 7878 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); 7879 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7880 7881 /* unused */ 7882 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7883 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7884 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7885 7886 callee->in_callback_fn = true; 7887 callee->callback_ret_range = tnum_range(0, 1); 7888 return 0; 7889 } 7890 7891 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, 7892 struct bpf_func_state *caller, 7893 struct bpf_func_state *callee, 7894 int insn_idx) 7895 { 7896 /* void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node, 7897 * bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)); 7898 * 7899 * 'struct bpf_rb_node *node' arg to bpf_rbtree_add is the same PTR_TO_BTF_ID w/ offset 7900 * that 'less' callback args will be receiving. However, 'node' arg was release_reference'd 7901 * by this point, so look at 'root' 7902 */ 7903 struct btf_field *field; 7904 7905 field = reg_find_field_offset(&caller->regs[BPF_REG_1], caller->regs[BPF_REG_1].off, 7906 BPF_RB_ROOT); 7907 if (!field || !field->graph_root.value_btf_id) 7908 return -EFAULT; 7909 7910 mark_reg_graph_node(callee->regs, BPF_REG_1, &field->graph_root); 7911 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); 7912 mark_reg_graph_node(callee->regs, BPF_REG_2, &field->graph_root); 7913 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); 7914 7915 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7916 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7917 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7918 callee->in_callback_fn = true; 7919 callee->callback_ret_range = tnum_range(0, 1); 7920 return 0; 7921 } 7922 7923 static bool is_rbtree_lock_required_kfunc(u32 btf_id); 7924 7925 /* Are we currently verifying the callback for a rbtree helper that must 7926 * be called with lock held? If so, no need to complain about unreleased 7927 * lock 7928 */ 7929 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) 7930 { 7931 struct bpf_verifier_state *state = env->cur_state; 7932 struct bpf_insn *insn = env->prog->insnsi; 7933 struct bpf_func_state *callee; 7934 int kfunc_btf_id; 7935 7936 if (!state->curframe) 7937 return false; 7938 7939 callee = state->frame[state->curframe]; 7940 7941 if (!callee->in_callback_fn) 7942 return false; 7943 7944 kfunc_btf_id = insn[callee->callsite].imm; 7945 return is_rbtree_lock_required_kfunc(kfunc_btf_id); 7946 } 7947 7948 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 7949 { 7950 struct bpf_verifier_state *state = env->cur_state; 7951 struct bpf_func_state *caller, *callee; 7952 struct bpf_reg_state *r0; 7953 int err; 7954 7955 callee = state->frame[state->curframe]; 7956 r0 = &callee->regs[BPF_REG_0]; 7957 if (r0->type == PTR_TO_STACK) { 7958 /* technically it's ok to return caller's stack pointer 7959 * (or caller's caller's pointer) back to the caller, 7960 * since these pointers are valid. Only current stack 7961 * pointer will be invalid as soon as function exits, 7962 * but let's be conservative 7963 */ 7964 verbose(env, "cannot return stack pointer to the caller\n"); 7965 return -EINVAL; 7966 } 7967 7968 caller = state->frame[state->curframe - 1]; 7969 if (callee->in_callback_fn) { 7970 /* enforce R0 return value range [0, 1]. */ 7971 struct tnum range = callee->callback_ret_range; 7972 7973 if (r0->type != SCALAR_VALUE) { 7974 verbose(env, "R0 not a scalar value\n"); 7975 return -EACCES; 7976 } 7977 if (!tnum_in(range, r0->var_off)) { 7978 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 7979 return -EINVAL; 7980 } 7981 } else { 7982 /* return to the caller whatever r0 had in the callee */ 7983 caller->regs[BPF_REG_0] = *r0; 7984 } 7985 7986 /* callback_fn frame should have released its own additions to parent's 7987 * reference state at this point, or check_reference_leak would 7988 * complain, hence it must be the same as the caller. There is no need 7989 * to copy it back. 7990 */ 7991 if (!callee->in_callback_fn) { 7992 /* Transfer references to the caller */ 7993 err = copy_reference_state(caller, callee); 7994 if (err) 7995 return err; 7996 } 7997 7998 *insn_idx = callee->callsite + 1; 7999 if (env->log.level & BPF_LOG_LEVEL) { 8000 verbose(env, "returning from callee:\n"); 8001 print_verifier_state(env, callee, true); 8002 verbose(env, "to caller at %d:\n", *insn_idx); 8003 print_verifier_state(env, caller, true); 8004 } 8005 /* clear everything in the callee */ 8006 free_func_state(callee); 8007 state->frame[state->curframe--] = NULL; 8008 return 0; 8009 } 8010 8011 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 8012 int func_id, 8013 struct bpf_call_arg_meta *meta) 8014 { 8015 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 8016 8017 if (ret_type != RET_INTEGER || 8018 (func_id != BPF_FUNC_get_stack && 8019 func_id != BPF_FUNC_get_task_stack && 8020 func_id != BPF_FUNC_probe_read_str && 8021 func_id != BPF_FUNC_probe_read_kernel_str && 8022 func_id != BPF_FUNC_probe_read_user_str)) 8023 return; 8024 8025 ret_reg->smax_value = meta->msize_max_value; 8026 ret_reg->s32_max_value = meta->msize_max_value; 8027 ret_reg->smin_value = -MAX_ERRNO; 8028 ret_reg->s32_min_value = -MAX_ERRNO; 8029 reg_bounds_sync(ret_reg); 8030 } 8031 8032 static int 8033 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 8034 int func_id, int insn_idx) 8035 { 8036 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 8037 struct bpf_map *map = meta->map_ptr; 8038 8039 if (func_id != BPF_FUNC_tail_call && 8040 func_id != BPF_FUNC_map_lookup_elem && 8041 func_id != BPF_FUNC_map_update_elem && 8042 func_id != BPF_FUNC_map_delete_elem && 8043 func_id != BPF_FUNC_map_push_elem && 8044 func_id != BPF_FUNC_map_pop_elem && 8045 func_id != BPF_FUNC_map_peek_elem && 8046 func_id != BPF_FUNC_for_each_map_elem && 8047 func_id != BPF_FUNC_redirect_map && 8048 func_id != BPF_FUNC_map_lookup_percpu_elem) 8049 return 0; 8050 8051 if (map == NULL) { 8052 verbose(env, "kernel subsystem misconfigured verifier\n"); 8053 return -EINVAL; 8054 } 8055 8056 /* In case of read-only, some additional restrictions 8057 * need to be applied in order to prevent altering the 8058 * state of the map from program side. 8059 */ 8060 if ((map->map_flags & BPF_F_RDONLY_PROG) && 8061 (func_id == BPF_FUNC_map_delete_elem || 8062 func_id == BPF_FUNC_map_update_elem || 8063 func_id == BPF_FUNC_map_push_elem || 8064 func_id == BPF_FUNC_map_pop_elem)) { 8065 verbose(env, "write into map forbidden\n"); 8066 return -EACCES; 8067 } 8068 8069 if (!BPF_MAP_PTR(aux->map_ptr_state)) 8070 bpf_map_ptr_store(aux, meta->map_ptr, 8071 !meta->map_ptr->bypass_spec_v1); 8072 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 8073 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 8074 !meta->map_ptr->bypass_spec_v1); 8075 return 0; 8076 } 8077 8078 static int 8079 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 8080 int func_id, int insn_idx) 8081 { 8082 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 8083 struct bpf_reg_state *regs = cur_regs(env), *reg; 8084 struct bpf_map *map = meta->map_ptr; 8085 u64 val, max; 8086 int err; 8087 8088 if (func_id != BPF_FUNC_tail_call) 8089 return 0; 8090 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 8091 verbose(env, "kernel subsystem misconfigured verifier\n"); 8092 return -EINVAL; 8093 } 8094 8095 reg = ®s[BPF_REG_3]; 8096 val = reg->var_off.value; 8097 max = map->max_entries; 8098 8099 if (!(register_is_const(reg) && val < max)) { 8100 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 8101 return 0; 8102 } 8103 8104 err = mark_chain_precision(env, BPF_REG_3); 8105 if (err) 8106 return err; 8107 if (bpf_map_key_unseen(aux)) 8108 bpf_map_key_store(aux, val); 8109 else if (!bpf_map_key_poisoned(aux) && 8110 bpf_map_key_immediate(aux) != val) 8111 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 8112 return 0; 8113 } 8114 8115 static int check_reference_leak(struct bpf_verifier_env *env) 8116 { 8117 struct bpf_func_state *state = cur_func(env); 8118 bool refs_lingering = false; 8119 int i; 8120 8121 if (state->frameno && !state->in_callback_fn) 8122 return 0; 8123 8124 for (i = 0; i < state->acquired_refs; i++) { 8125 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 8126 continue; 8127 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 8128 state->refs[i].id, state->refs[i].insn_idx); 8129 refs_lingering = true; 8130 } 8131 return refs_lingering ? -EINVAL : 0; 8132 } 8133 8134 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 8135 struct bpf_reg_state *regs) 8136 { 8137 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 8138 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 8139 struct bpf_map *fmt_map = fmt_reg->map_ptr; 8140 struct bpf_bprintf_data data = {}; 8141 int err, fmt_map_off, num_args; 8142 u64 fmt_addr; 8143 char *fmt; 8144 8145 /* data must be an array of u64 */ 8146 if (data_len_reg->var_off.value % 8) 8147 return -EINVAL; 8148 num_args = data_len_reg->var_off.value / 8; 8149 8150 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 8151 * and map_direct_value_addr is set. 8152 */ 8153 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 8154 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 8155 fmt_map_off); 8156 if (err) { 8157 verbose(env, "verifier bug\n"); 8158 return -EFAULT; 8159 } 8160 fmt = (char *)(long)fmt_addr + fmt_map_off; 8161 8162 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 8163 * can focus on validating the format specifiers. 8164 */ 8165 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, num_args, &data); 8166 if (err < 0) 8167 verbose(env, "Invalid format string\n"); 8168 8169 return err; 8170 } 8171 8172 static int check_get_func_ip(struct bpf_verifier_env *env) 8173 { 8174 enum bpf_prog_type type = resolve_prog_type(env->prog); 8175 int func_id = BPF_FUNC_get_func_ip; 8176 8177 if (type == BPF_PROG_TYPE_TRACING) { 8178 if (!bpf_prog_has_trampoline(env->prog)) { 8179 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 8180 func_id_name(func_id), func_id); 8181 return -ENOTSUPP; 8182 } 8183 return 0; 8184 } else if (type == BPF_PROG_TYPE_KPROBE) { 8185 return 0; 8186 } 8187 8188 verbose(env, "func %s#%d not supported for program type %d\n", 8189 func_id_name(func_id), func_id, type); 8190 return -ENOTSUPP; 8191 } 8192 8193 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 8194 { 8195 return &env->insn_aux_data[env->insn_idx]; 8196 } 8197 8198 static bool loop_flag_is_zero(struct bpf_verifier_env *env) 8199 { 8200 struct bpf_reg_state *regs = cur_regs(env); 8201 struct bpf_reg_state *reg = ®s[BPF_REG_4]; 8202 bool reg_is_null = register_is_null(reg); 8203 8204 if (reg_is_null) 8205 mark_chain_precision(env, BPF_REG_4); 8206 8207 return reg_is_null; 8208 } 8209 8210 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 8211 { 8212 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 8213 8214 if (!state->initialized) { 8215 state->initialized = 1; 8216 state->fit_for_inline = loop_flag_is_zero(env); 8217 state->callback_subprogno = subprogno; 8218 return; 8219 } 8220 8221 if (!state->fit_for_inline) 8222 return; 8223 8224 state->fit_for_inline = (loop_flag_is_zero(env) && 8225 state->callback_subprogno == subprogno); 8226 } 8227 8228 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8229 int *insn_idx_p) 8230 { 8231 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 8232 const struct bpf_func_proto *fn = NULL; 8233 enum bpf_return_type ret_type; 8234 enum bpf_type_flag ret_flag; 8235 struct bpf_reg_state *regs; 8236 struct bpf_call_arg_meta meta; 8237 int insn_idx = *insn_idx_p; 8238 bool changes_data; 8239 int i, err, func_id; 8240 8241 /* find function prototype */ 8242 func_id = insn->imm; 8243 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 8244 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 8245 func_id); 8246 return -EINVAL; 8247 } 8248 8249 if (env->ops->get_func_proto) 8250 fn = env->ops->get_func_proto(func_id, env->prog); 8251 if (!fn) { 8252 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 8253 func_id); 8254 return -EINVAL; 8255 } 8256 8257 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 8258 if (!env->prog->gpl_compatible && fn->gpl_only) { 8259 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 8260 return -EINVAL; 8261 } 8262 8263 if (fn->allowed && !fn->allowed(env->prog)) { 8264 verbose(env, "helper call is not allowed in probe\n"); 8265 return -EINVAL; 8266 } 8267 8268 if (!env->prog->aux->sleepable && fn->might_sleep) { 8269 verbose(env, "helper call might sleep in a non-sleepable prog\n"); 8270 return -EINVAL; 8271 } 8272 8273 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 8274 changes_data = bpf_helper_changes_pkt_data(fn->func); 8275 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 8276 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 8277 func_id_name(func_id), func_id); 8278 return -EINVAL; 8279 } 8280 8281 memset(&meta, 0, sizeof(meta)); 8282 meta.pkt_access = fn->pkt_access; 8283 8284 err = check_func_proto(fn, func_id); 8285 if (err) { 8286 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 8287 func_id_name(func_id), func_id); 8288 return err; 8289 } 8290 8291 if (env->cur_state->active_rcu_lock) { 8292 if (fn->might_sleep) { 8293 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", 8294 func_id_name(func_id), func_id); 8295 return -EINVAL; 8296 } 8297 8298 if (env->prog->aux->sleepable && is_storage_get_function(func_id)) 8299 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 8300 } 8301 8302 meta.func_id = func_id; 8303 /* check args */ 8304 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 8305 err = check_func_arg(env, i, &meta, fn, insn_idx); 8306 if (err) 8307 return err; 8308 } 8309 8310 err = record_func_map(env, &meta, func_id, insn_idx); 8311 if (err) 8312 return err; 8313 8314 err = record_func_key(env, &meta, func_id, insn_idx); 8315 if (err) 8316 return err; 8317 8318 /* Mark slots with STACK_MISC in case of raw mode, stack offset 8319 * is inferred from register state. 8320 */ 8321 for (i = 0; i < meta.access_size; i++) { 8322 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 8323 BPF_WRITE, -1, false); 8324 if (err) 8325 return err; 8326 } 8327 8328 regs = cur_regs(env); 8329 8330 if (meta.release_regno) { 8331 err = -EINVAL; 8332 /* This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot 8333 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr 8334 * is safe to do directly. 8335 */ 8336 if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) { 8337 if (regs[meta.release_regno].type == CONST_PTR_TO_DYNPTR) { 8338 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); 8339 return -EFAULT; 8340 } 8341 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); 8342 } else if (meta.ref_obj_id) { 8343 err = release_reference(env, meta.ref_obj_id); 8344 } else if (register_is_null(®s[meta.release_regno])) { 8345 /* meta.ref_obj_id can only be 0 if register that is meant to be 8346 * released is NULL, which must be > R0. 8347 */ 8348 err = 0; 8349 } 8350 if (err) { 8351 verbose(env, "func %s#%d reference has not been acquired before\n", 8352 func_id_name(func_id), func_id); 8353 return err; 8354 } 8355 } 8356 8357 switch (func_id) { 8358 case BPF_FUNC_tail_call: 8359 err = check_reference_leak(env); 8360 if (err) { 8361 verbose(env, "tail_call would lead to reference leak\n"); 8362 return err; 8363 } 8364 break; 8365 case BPF_FUNC_get_local_storage: 8366 /* check that flags argument in get_local_storage(map, flags) is 0, 8367 * this is required because get_local_storage() can't return an error. 8368 */ 8369 if (!register_is_null(®s[BPF_REG_2])) { 8370 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 8371 return -EINVAL; 8372 } 8373 break; 8374 case BPF_FUNC_for_each_map_elem: 8375 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8376 set_map_elem_callback_state); 8377 break; 8378 case BPF_FUNC_timer_set_callback: 8379 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8380 set_timer_callback_state); 8381 break; 8382 case BPF_FUNC_find_vma: 8383 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8384 set_find_vma_callback_state); 8385 break; 8386 case BPF_FUNC_snprintf: 8387 err = check_bpf_snprintf_call(env, regs); 8388 break; 8389 case BPF_FUNC_loop: 8390 update_loop_inline_state(env, meta.subprogno); 8391 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8392 set_loop_callback_state); 8393 break; 8394 case BPF_FUNC_dynptr_from_mem: 8395 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 8396 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 8397 reg_type_str(env, regs[BPF_REG_1].type)); 8398 return -EACCES; 8399 } 8400 break; 8401 case BPF_FUNC_set_retval: 8402 if (prog_type == BPF_PROG_TYPE_LSM && 8403 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 8404 if (!env->prog->aux->attach_func_proto->type) { 8405 /* Make sure programs that attach to void 8406 * hooks don't try to modify return value. 8407 */ 8408 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 8409 return -EINVAL; 8410 } 8411 } 8412 break; 8413 case BPF_FUNC_dynptr_data: 8414 { 8415 struct bpf_reg_state *reg; 8416 int id, ref_obj_id; 8417 8418 reg = get_dynptr_arg_reg(env, fn, regs); 8419 if (!reg) 8420 return -EFAULT; 8421 8422 8423 if (meta.dynptr_id) { 8424 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); 8425 return -EFAULT; 8426 } 8427 if (meta.ref_obj_id) { 8428 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); 8429 return -EFAULT; 8430 } 8431 8432 id = dynptr_id(env, reg); 8433 if (id < 0) { 8434 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 8435 return id; 8436 } 8437 8438 ref_obj_id = dynptr_ref_obj_id(env, reg); 8439 if (ref_obj_id < 0) { 8440 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); 8441 return ref_obj_id; 8442 } 8443 8444 meta.dynptr_id = id; 8445 meta.ref_obj_id = ref_obj_id; 8446 8447 break; 8448 } 8449 case BPF_FUNC_dynptr_write: 8450 { 8451 enum bpf_dynptr_type dynptr_type; 8452 struct bpf_reg_state *reg; 8453 8454 reg = get_dynptr_arg_reg(env, fn, regs); 8455 if (!reg) 8456 return -EFAULT; 8457 8458 dynptr_type = dynptr_get_type(env, reg); 8459 if (dynptr_type == BPF_DYNPTR_TYPE_INVALID) 8460 return -EFAULT; 8461 8462 if (dynptr_type == BPF_DYNPTR_TYPE_SKB) 8463 /* this will trigger clear_all_pkt_pointers(), which will 8464 * invalidate all dynptr slices associated with the skb 8465 */ 8466 changes_data = true; 8467 8468 break; 8469 } 8470 case BPF_FUNC_user_ringbuf_drain: 8471 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 8472 set_user_ringbuf_callback_state); 8473 break; 8474 } 8475 8476 if (err) 8477 return err; 8478 8479 /* reset caller saved regs */ 8480 for (i = 0; i < CALLER_SAVED_REGS; i++) { 8481 mark_reg_not_init(env, regs, caller_saved[i]); 8482 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 8483 } 8484 8485 /* helper call returns 64-bit value. */ 8486 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 8487 8488 /* update return register (already marked as written above) */ 8489 ret_type = fn->ret_type; 8490 ret_flag = type_flag(ret_type); 8491 8492 switch (base_type(ret_type)) { 8493 case RET_INTEGER: 8494 /* sets type to SCALAR_VALUE */ 8495 mark_reg_unknown(env, regs, BPF_REG_0); 8496 break; 8497 case RET_VOID: 8498 regs[BPF_REG_0].type = NOT_INIT; 8499 break; 8500 case RET_PTR_TO_MAP_VALUE: 8501 /* There is no offset yet applied, variable or fixed */ 8502 mark_reg_known_zero(env, regs, BPF_REG_0); 8503 /* remember map_ptr, so that check_map_access() 8504 * can check 'value_size' boundary of memory access 8505 * to map element returned from bpf_map_lookup_elem() 8506 */ 8507 if (meta.map_ptr == NULL) { 8508 verbose(env, 8509 "kernel subsystem misconfigured verifier\n"); 8510 return -EINVAL; 8511 } 8512 regs[BPF_REG_0].map_ptr = meta.map_ptr; 8513 regs[BPF_REG_0].map_uid = meta.map_uid; 8514 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 8515 if (!type_may_be_null(ret_type) && 8516 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 8517 regs[BPF_REG_0].id = ++env->id_gen; 8518 } 8519 break; 8520 case RET_PTR_TO_SOCKET: 8521 mark_reg_known_zero(env, regs, BPF_REG_0); 8522 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 8523 break; 8524 case RET_PTR_TO_SOCK_COMMON: 8525 mark_reg_known_zero(env, regs, BPF_REG_0); 8526 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 8527 break; 8528 case RET_PTR_TO_TCP_SOCK: 8529 mark_reg_known_zero(env, regs, BPF_REG_0); 8530 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 8531 break; 8532 case RET_PTR_TO_MEM: 8533 mark_reg_known_zero(env, regs, BPF_REG_0); 8534 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 8535 regs[BPF_REG_0].mem_size = meta.mem_size; 8536 break; 8537 case RET_PTR_TO_MEM_OR_BTF_ID: 8538 { 8539 const struct btf_type *t; 8540 8541 mark_reg_known_zero(env, regs, BPF_REG_0); 8542 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 8543 if (!btf_type_is_struct(t)) { 8544 u32 tsize; 8545 const struct btf_type *ret; 8546 const char *tname; 8547 8548 /* resolve the type size of ksym. */ 8549 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 8550 if (IS_ERR(ret)) { 8551 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 8552 verbose(env, "unable to resolve the size of type '%s': %ld\n", 8553 tname, PTR_ERR(ret)); 8554 return -EINVAL; 8555 } 8556 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 8557 regs[BPF_REG_0].mem_size = tsize; 8558 } else { 8559 /* MEM_RDONLY may be carried from ret_flag, but it 8560 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 8561 * it will confuse the check of PTR_TO_BTF_ID in 8562 * check_mem_access(). 8563 */ 8564 ret_flag &= ~MEM_RDONLY; 8565 8566 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 8567 regs[BPF_REG_0].btf = meta.ret_btf; 8568 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 8569 } 8570 break; 8571 } 8572 case RET_PTR_TO_BTF_ID: 8573 { 8574 struct btf *ret_btf; 8575 int ret_btf_id; 8576 8577 mark_reg_known_zero(env, regs, BPF_REG_0); 8578 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 8579 if (func_id == BPF_FUNC_kptr_xchg) { 8580 ret_btf = meta.kptr_field->kptr.btf; 8581 ret_btf_id = meta.kptr_field->kptr.btf_id; 8582 } else { 8583 if (fn->ret_btf_id == BPF_PTR_POISON) { 8584 verbose(env, "verifier internal error:"); 8585 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", 8586 func_id_name(func_id)); 8587 return -EINVAL; 8588 } 8589 ret_btf = btf_vmlinux; 8590 ret_btf_id = *fn->ret_btf_id; 8591 } 8592 if (ret_btf_id == 0) { 8593 verbose(env, "invalid return type %u of func %s#%d\n", 8594 base_type(ret_type), func_id_name(func_id), 8595 func_id); 8596 return -EINVAL; 8597 } 8598 regs[BPF_REG_0].btf = ret_btf; 8599 regs[BPF_REG_0].btf_id = ret_btf_id; 8600 break; 8601 } 8602 default: 8603 verbose(env, "unknown return type %u of func %s#%d\n", 8604 base_type(ret_type), func_id_name(func_id), func_id); 8605 return -EINVAL; 8606 } 8607 8608 if (type_may_be_null(regs[BPF_REG_0].type)) 8609 regs[BPF_REG_0].id = ++env->id_gen; 8610 8611 if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { 8612 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", 8613 func_id_name(func_id), func_id); 8614 return -EFAULT; 8615 } 8616 8617 if (is_dynptr_ref_function(func_id)) 8618 regs[BPF_REG_0].dynptr_id = meta.dynptr_id; 8619 8620 if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { 8621 /* For release_reference() */ 8622 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 8623 } else if (is_acquire_function(func_id, meta.map_ptr)) { 8624 int id = acquire_reference_state(env, insn_idx); 8625 8626 if (id < 0) 8627 return id; 8628 /* For mark_ptr_or_null_reg() */ 8629 regs[BPF_REG_0].id = id; 8630 /* For release_reference() */ 8631 regs[BPF_REG_0].ref_obj_id = id; 8632 } 8633 8634 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 8635 8636 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 8637 if (err) 8638 return err; 8639 8640 if ((func_id == BPF_FUNC_get_stack || 8641 func_id == BPF_FUNC_get_task_stack) && 8642 !env->prog->has_callchain_buf) { 8643 const char *err_str; 8644 8645 #ifdef CONFIG_PERF_EVENTS 8646 err = get_callchain_buffers(sysctl_perf_event_max_stack); 8647 err_str = "cannot get callchain buffer for func %s#%d\n"; 8648 #else 8649 err = -ENOTSUPP; 8650 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 8651 #endif 8652 if (err) { 8653 verbose(env, err_str, func_id_name(func_id), func_id); 8654 return err; 8655 } 8656 8657 env->prog->has_callchain_buf = true; 8658 } 8659 8660 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 8661 env->prog->call_get_stack = true; 8662 8663 if (func_id == BPF_FUNC_get_func_ip) { 8664 if (check_get_func_ip(env)) 8665 return -ENOTSUPP; 8666 env->prog->call_get_func_ip = true; 8667 } 8668 8669 if (changes_data) 8670 clear_all_pkt_pointers(env); 8671 return 0; 8672 } 8673 8674 /* mark_btf_func_reg_size() is used when the reg size is determined by 8675 * the BTF func_proto's return value size and argument. 8676 */ 8677 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 8678 size_t reg_size) 8679 { 8680 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 8681 8682 if (regno == BPF_REG_0) { 8683 /* Function return value */ 8684 reg->live |= REG_LIVE_WRITTEN; 8685 reg->subreg_def = reg_size == sizeof(u64) ? 8686 DEF_NOT_SUBREG : env->insn_idx + 1; 8687 } else { 8688 /* Function argument */ 8689 if (reg_size == sizeof(u64)) { 8690 mark_insn_zext(env, reg); 8691 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 8692 } else { 8693 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 8694 } 8695 } 8696 } 8697 8698 struct bpf_kfunc_call_arg_meta { 8699 /* In parameters */ 8700 struct btf *btf; 8701 u32 func_id; 8702 u32 kfunc_flags; 8703 const struct btf_type *func_proto; 8704 const char *func_name; 8705 /* Out parameters */ 8706 u32 ref_obj_id; 8707 u8 release_regno; 8708 bool r0_rdonly; 8709 u32 ret_btf_id; 8710 u64 r0_size; 8711 u32 subprogno; 8712 struct { 8713 u64 value; 8714 bool found; 8715 } arg_constant; 8716 struct { 8717 struct btf *btf; 8718 u32 btf_id; 8719 } arg_obj_drop; 8720 struct { 8721 struct btf_field *field; 8722 } arg_list_head; 8723 struct { 8724 struct btf_field *field; 8725 } arg_rbtree_root; 8726 struct { 8727 enum bpf_dynptr_type type; 8728 u32 id; 8729 } initialized_dynptr; 8730 u64 mem_size; 8731 }; 8732 8733 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) 8734 { 8735 return meta->kfunc_flags & KF_ACQUIRE; 8736 } 8737 8738 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 8739 { 8740 return meta->kfunc_flags & KF_RET_NULL; 8741 } 8742 8743 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) 8744 { 8745 return meta->kfunc_flags & KF_RELEASE; 8746 } 8747 8748 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta) 8749 { 8750 return meta->kfunc_flags & KF_TRUSTED_ARGS; 8751 } 8752 8753 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) 8754 { 8755 return meta->kfunc_flags & KF_SLEEPABLE; 8756 } 8757 8758 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) 8759 { 8760 return meta->kfunc_flags & KF_DESTRUCTIVE; 8761 } 8762 8763 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) 8764 { 8765 return meta->kfunc_flags & KF_RCU; 8766 } 8767 8768 static bool is_kfunc_arg_kptr_get(struct bpf_kfunc_call_arg_meta *meta, int arg) 8769 { 8770 return arg == 0 && (meta->kfunc_flags & KF_KPTR_GET); 8771 } 8772 8773 static bool __kfunc_param_match_suffix(const struct btf *btf, 8774 const struct btf_param *arg, 8775 const char *suffix) 8776 { 8777 int suffix_len = strlen(suffix), len; 8778 const char *param_name; 8779 8780 /* In the future, this can be ported to use BTF tagging */ 8781 param_name = btf_name_by_offset(btf, arg->name_off); 8782 if (str_is_empty(param_name)) 8783 return false; 8784 len = strlen(param_name); 8785 if (len < suffix_len) 8786 return false; 8787 param_name += len - suffix_len; 8788 return !strncmp(param_name, suffix, suffix_len); 8789 } 8790 8791 static bool is_kfunc_arg_mem_size(const struct btf *btf, 8792 const struct btf_param *arg, 8793 const struct bpf_reg_state *reg) 8794 { 8795 const struct btf_type *t; 8796 8797 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8798 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 8799 return false; 8800 8801 return __kfunc_param_match_suffix(btf, arg, "__sz"); 8802 } 8803 8804 static bool is_kfunc_arg_const_mem_size(const struct btf *btf, 8805 const struct btf_param *arg, 8806 const struct bpf_reg_state *reg) 8807 { 8808 const struct btf_type *t; 8809 8810 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8811 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 8812 return false; 8813 8814 return __kfunc_param_match_suffix(btf, arg, "__szk"); 8815 } 8816 8817 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 8818 { 8819 return __kfunc_param_match_suffix(btf, arg, "__k"); 8820 } 8821 8822 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 8823 { 8824 return __kfunc_param_match_suffix(btf, arg, "__ign"); 8825 } 8826 8827 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 8828 { 8829 return __kfunc_param_match_suffix(btf, arg, "__alloc"); 8830 } 8831 8832 static bool is_kfunc_arg_uninit(const struct btf *btf, const struct btf_param *arg) 8833 { 8834 return __kfunc_param_match_suffix(btf, arg, "__uninit"); 8835 } 8836 8837 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 8838 const struct btf_param *arg, 8839 const char *name) 8840 { 8841 int len, target_len = strlen(name); 8842 const char *param_name; 8843 8844 param_name = btf_name_by_offset(btf, arg->name_off); 8845 if (str_is_empty(param_name)) 8846 return false; 8847 len = strlen(param_name); 8848 if (len != target_len) 8849 return false; 8850 if (strcmp(param_name, name)) 8851 return false; 8852 8853 return true; 8854 } 8855 8856 enum { 8857 KF_ARG_DYNPTR_ID, 8858 KF_ARG_LIST_HEAD_ID, 8859 KF_ARG_LIST_NODE_ID, 8860 KF_ARG_RB_ROOT_ID, 8861 KF_ARG_RB_NODE_ID, 8862 }; 8863 8864 BTF_ID_LIST(kf_arg_btf_ids) 8865 BTF_ID(struct, bpf_dynptr_kern) 8866 BTF_ID(struct, bpf_list_head) 8867 BTF_ID(struct, bpf_list_node) 8868 BTF_ID(struct, bpf_rb_root) 8869 BTF_ID(struct, bpf_rb_node) 8870 8871 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 8872 const struct btf_param *arg, int type) 8873 { 8874 const struct btf_type *t; 8875 u32 res_id; 8876 8877 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8878 if (!t) 8879 return false; 8880 if (!btf_type_is_ptr(t)) 8881 return false; 8882 t = btf_type_skip_modifiers(btf, t->type, &res_id); 8883 if (!t) 8884 return false; 8885 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 8886 } 8887 8888 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 8889 { 8890 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 8891 } 8892 8893 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 8894 { 8895 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 8896 } 8897 8898 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 8899 { 8900 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 8901 } 8902 8903 static bool is_kfunc_arg_rbtree_root(const struct btf *btf, const struct btf_param *arg) 8904 { 8905 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_ROOT_ID); 8906 } 8907 8908 static bool is_kfunc_arg_rbtree_node(const struct btf *btf, const struct btf_param *arg) 8909 { 8910 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RB_NODE_ID); 8911 } 8912 8913 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, 8914 const struct btf_param *arg) 8915 { 8916 const struct btf_type *t; 8917 8918 t = btf_type_resolve_func_ptr(btf, arg->type, NULL); 8919 if (!t) 8920 return false; 8921 8922 return true; 8923 } 8924 8925 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 8926 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 8927 const struct btf *btf, 8928 const struct btf_type *t, int rec) 8929 { 8930 const struct btf_type *member_type; 8931 const struct btf_member *member; 8932 u32 i; 8933 8934 if (!btf_type_is_struct(t)) 8935 return false; 8936 8937 for_each_member(i, t, member) { 8938 const struct btf_array *array; 8939 8940 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 8941 if (btf_type_is_struct(member_type)) { 8942 if (rec >= 3) { 8943 verbose(env, "max struct nesting depth exceeded\n"); 8944 return false; 8945 } 8946 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) 8947 return false; 8948 continue; 8949 } 8950 if (btf_type_is_array(member_type)) { 8951 array = btf_array(member_type); 8952 if (!array->nelems) 8953 return false; 8954 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 8955 if (!btf_type_is_scalar(member_type)) 8956 return false; 8957 continue; 8958 } 8959 if (!btf_type_is_scalar(member_type)) 8960 return false; 8961 } 8962 return true; 8963 } 8964 8965 8966 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 8967 #ifdef CONFIG_NET 8968 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 8969 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 8970 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 8971 #endif 8972 }; 8973 8974 enum kfunc_ptr_arg_type { 8975 KF_ARG_PTR_TO_CTX, 8976 KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ 8977 KF_ARG_PTR_TO_KPTR, /* PTR_TO_KPTR but type specific */ 8978 KF_ARG_PTR_TO_DYNPTR, 8979 KF_ARG_PTR_TO_LIST_HEAD, 8980 KF_ARG_PTR_TO_LIST_NODE, 8981 KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ 8982 KF_ARG_PTR_TO_MEM, 8983 KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ 8984 KF_ARG_PTR_TO_CALLBACK, 8985 KF_ARG_PTR_TO_RB_ROOT, 8986 KF_ARG_PTR_TO_RB_NODE, 8987 }; 8988 8989 enum special_kfunc_type { 8990 KF_bpf_obj_new_impl, 8991 KF_bpf_obj_drop_impl, 8992 KF_bpf_list_push_front, 8993 KF_bpf_list_push_back, 8994 KF_bpf_list_pop_front, 8995 KF_bpf_list_pop_back, 8996 KF_bpf_cast_to_kern_ctx, 8997 KF_bpf_rdonly_cast, 8998 KF_bpf_rcu_read_lock, 8999 KF_bpf_rcu_read_unlock, 9000 KF_bpf_rbtree_remove, 9001 KF_bpf_rbtree_add, 9002 KF_bpf_rbtree_first, 9003 KF_bpf_dynptr_from_skb, 9004 KF_bpf_dynptr_from_xdp, 9005 KF_bpf_dynptr_slice, 9006 KF_bpf_dynptr_slice_rdwr, 9007 }; 9008 9009 BTF_SET_START(special_kfunc_set) 9010 BTF_ID(func, bpf_obj_new_impl) 9011 BTF_ID(func, bpf_obj_drop_impl) 9012 BTF_ID(func, bpf_list_push_front) 9013 BTF_ID(func, bpf_list_push_back) 9014 BTF_ID(func, bpf_list_pop_front) 9015 BTF_ID(func, bpf_list_pop_back) 9016 BTF_ID(func, bpf_cast_to_kern_ctx) 9017 BTF_ID(func, bpf_rdonly_cast) 9018 BTF_ID(func, bpf_rbtree_remove) 9019 BTF_ID(func, bpf_rbtree_add) 9020 BTF_ID(func, bpf_rbtree_first) 9021 BTF_ID(func, bpf_dynptr_from_skb) 9022 BTF_ID(func, bpf_dynptr_from_xdp) 9023 BTF_ID(func, bpf_dynptr_slice) 9024 BTF_ID(func, bpf_dynptr_slice_rdwr) 9025 BTF_SET_END(special_kfunc_set) 9026 9027 BTF_ID_LIST(special_kfunc_list) 9028 BTF_ID(func, bpf_obj_new_impl) 9029 BTF_ID(func, bpf_obj_drop_impl) 9030 BTF_ID(func, bpf_list_push_front) 9031 BTF_ID(func, bpf_list_push_back) 9032 BTF_ID(func, bpf_list_pop_front) 9033 BTF_ID(func, bpf_list_pop_back) 9034 BTF_ID(func, bpf_cast_to_kern_ctx) 9035 BTF_ID(func, bpf_rdonly_cast) 9036 BTF_ID(func, bpf_rcu_read_lock) 9037 BTF_ID(func, bpf_rcu_read_unlock) 9038 BTF_ID(func, bpf_rbtree_remove) 9039 BTF_ID(func, bpf_rbtree_add) 9040 BTF_ID(func, bpf_rbtree_first) 9041 BTF_ID(func, bpf_dynptr_from_skb) 9042 BTF_ID(func, bpf_dynptr_from_xdp) 9043 BTF_ID(func, bpf_dynptr_slice) 9044 BTF_ID(func, bpf_dynptr_slice_rdwr) 9045 9046 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) 9047 { 9048 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; 9049 } 9050 9051 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) 9052 { 9053 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; 9054 } 9055 9056 static enum kfunc_ptr_arg_type 9057 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, 9058 struct bpf_kfunc_call_arg_meta *meta, 9059 const struct btf_type *t, const struct btf_type *ref_t, 9060 const char *ref_tname, const struct btf_param *args, 9061 int argno, int nargs) 9062 { 9063 u32 regno = argno + 1; 9064 struct bpf_reg_state *regs = cur_regs(env); 9065 struct bpf_reg_state *reg = ®s[regno]; 9066 bool arg_mem_size = false; 9067 9068 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) 9069 return KF_ARG_PTR_TO_CTX; 9070 9071 /* In this function, we verify the kfunc's BTF as per the argument type, 9072 * leaving the rest of the verification with respect to the register 9073 * type to our caller. When a set of conditions hold in the BTF type of 9074 * arguments, we resolve it to a known kfunc_ptr_arg_type. 9075 */ 9076 if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) 9077 return KF_ARG_PTR_TO_CTX; 9078 9079 if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) 9080 return KF_ARG_PTR_TO_ALLOC_BTF_ID; 9081 9082 if (is_kfunc_arg_kptr_get(meta, argno)) { 9083 if (!btf_type_is_ptr(ref_t)) { 9084 verbose(env, "arg#0 BTF type must be a double pointer for kptr_get kfunc\n"); 9085 return -EINVAL; 9086 } 9087 ref_t = btf_type_by_id(meta->btf, ref_t->type); 9088 ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); 9089 if (!btf_type_is_struct(ref_t)) { 9090 verbose(env, "kernel function %s args#0 pointer type %s %s is not supported\n", 9091 meta->func_name, btf_type_str(ref_t), ref_tname); 9092 return -EINVAL; 9093 } 9094 return KF_ARG_PTR_TO_KPTR; 9095 } 9096 9097 if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) 9098 return KF_ARG_PTR_TO_DYNPTR; 9099 9100 if (is_kfunc_arg_list_head(meta->btf, &args[argno])) 9101 return KF_ARG_PTR_TO_LIST_HEAD; 9102 9103 if (is_kfunc_arg_list_node(meta->btf, &args[argno])) 9104 return KF_ARG_PTR_TO_LIST_NODE; 9105 9106 if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno])) 9107 return KF_ARG_PTR_TO_RB_ROOT; 9108 9109 if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno])) 9110 return KF_ARG_PTR_TO_RB_NODE; 9111 9112 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 9113 if (!btf_type_is_struct(ref_t)) { 9114 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", 9115 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 9116 return -EINVAL; 9117 } 9118 return KF_ARG_PTR_TO_BTF_ID; 9119 } 9120 9121 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) 9122 return KF_ARG_PTR_TO_CALLBACK; 9123 9124 9125 if (argno + 1 < nargs && 9126 (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]) || 9127 is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]))) 9128 arg_mem_size = true; 9129 9130 /* This is the catch all argument type of register types supported by 9131 * check_helper_mem_access. However, we only allow when argument type is 9132 * pointer to scalar, or struct composed (recursively) of scalars. When 9133 * arg_mem_size is true, the pointer can be void *. 9134 */ 9135 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && 9136 (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { 9137 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", 9138 argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); 9139 return -EINVAL; 9140 } 9141 return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; 9142 } 9143 9144 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, 9145 struct bpf_reg_state *reg, 9146 const struct btf_type *ref_t, 9147 const char *ref_tname, u32 ref_id, 9148 struct bpf_kfunc_call_arg_meta *meta, 9149 int argno) 9150 { 9151 const struct btf_type *reg_ref_t; 9152 bool strict_type_match = false; 9153 const struct btf *reg_btf; 9154 const char *reg_ref_tname; 9155 u32 reg_ref_id; 9156 9157 if (base_type(reg->type) == PTR_TO_BTF_ID) { 9158 reg_btf = reg->btf; 9159 reg_ref_id = reg->btf_id; 9160 } else { 9161 reg_btf = btf_vmlinux; 9162 reg_ref_id = *reg2btf_ids[base_type(reg->type)]; 9163 } 9164 9165 /* Enforce strict type matching for calls to kfuncs that are acquiring 9166 * or releasing a reference, or are no-cast aliases. We do _not_ 9167 * enforce strict matching for plain KF_TRUSTED_ARGS kfuncs by default, 9168 * as we want to enable BPF programs to pass types that are bitwise 9169 * equivalent without forcing them to explicitly cast with something 9170 * like bpf_cast_to_kern_ctx(). 9171 * 9172 * For example, say we had a type like the following: 9173 * 9174 * struct bpf_cpumask { 9175 * cpumask_t cpumask; 9176 * refcount_t usage; 9177 * }; 9178 * 9179 * Note that as specified in <linux/cpumask.h>, cpumask_t is typedef'ed 9180 * to a struct cpumask, so it would be safe to pass a struct 9181 * bpf_cpumask * to a kfunc expecting a struct cpumask *. 9182 * 9183 * The philosophy here is similar to how we allow scalars of different 9184 * types to be passed to kfuncs as long as the size is the same. The 9185 * only difference here is that we're simply allowing 9186 * btf_struct_ids_match() to walk the struct at the 0th offset, and 9187 * resolve types. 9188 */ 9189 if (is_kfunc_acquire(meta) || 9190 (is_kfunc_release(meta) && reg->ref_obj_id) || 9191 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) 9192 strict_type_match = true; 9193 9194 WARN_ON_ONCE(is_kfunc_trusted_args(meta) && reg->off); 9195 9196 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); 9197 reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); 9198 if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match)) { 9199 verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", 9200 meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, 9201 btf_type_str(reg_ref_t), reg_ref_tname); 9202 return -EINVAL; 9203 } 9204 return 0; 9205 } 9206 9207 static int process_kf_arg_ptr_to_kptr(struct bpf_verifier_env *env, 9208 struct bpf_reg_state *reg, 9209 const struct btf_type *ref_t, 9210 const char *ref_tname, 9211 struct bpf_kfunc_call_arg_meta *meta, 9212 int argno) 9213 { 9214 struct btf_field *kptr_field; 9215 9216 /* check_func_arg_reg_off allows var_off for 9217 * PTR_TO_MAP_VALUE, but we need fixed offset to find 9218 * off_desc. 9219 */ 9220 if (!tnum_is_const(reg->var_off)) { 9221 verbose(env, "arg#0 must have constant offset\n"); 9222 return -EINVAL; 9223 } 9224 9225 kptr_field = btf_record_find(reg->map_ptr->record, reg->off + reg->var_off.value, BPF_KPTR); 9226 if (!kptr_field || kptr_field->type != BPF_KPTR_REF) { 9227 verbose(env, "arg#0 no referenced kptr at map value offset=%llu\n", 9228 reg->off + reg->var_off.value); 9229 return -EINVAL; 9230 } 9231 9232 if (!btf_struct_ids_match(&env->log, meta->btf, ref_t->type, 0, kptr_field->kptr.btf, 9233 kptr_field->kptr.btf_id, true)) { 9234 verbose(env, "kernel function %s args#%d expected pointer to %s %s\n", 9235 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 9236 return -EINVAL; 9237 } 9238 return 0; 9239 } 9240 9241 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 9242 { 9243 struct bpf_verifier_state *state = env->cur_state; 9244 9245 if (!state->active_lock.ptr) { 9246 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); 9247 return -EFAULT; 9248 } 9249 9250 if (type_flag(reg->type) & NON_OWN_REF) { 9251 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); 9252 return -EFAULT; 9253 } 9254 9255 reg->type |= NON_OWN_REF; 9256 return 0; 9257 } 9258 9259 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) 9260 { 9261 struct bpf_func_state *state, *unused; 9262 struct bpf_reg_state *reg; 9263 int i; 9264 9265 state = cur_func(env); 9266 9267 if (!ref_obj_id) { 9268 verbose(env, "verifier internal error: ref_obj_id is zero for " 9269 "owning -> non-owning conversion\n"); 9270 return -EFAULT; 9271 } 9272 9273 for (i = 0; i < state->acquired_refs; i++) { 9274 if (state->refs[i].id != ref_obj_id) 9275 continue; 9276 9277 /* Clear ref_obj_id here so release_reference doesn't clobber 9278 * the whole reg 9279 */ 9280 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ 9281 if (reg->ref_obj_id == ref_obj_id) { 9282 reg->ref_obj_id = 0; 9283 ref_set_non_owning(env, reg); 9284 } 9285 })); 9286 return 0; 9287 } 9288 9289 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); 9290 return -EFAULT; 9291 } 9292 9293 /* Implementation details: 9294 * 9295 * Each register points to some region of memory, which we define as an 9296 * allocation. Each allocation may embed a bpf_spin_lock which protects any 9297 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 9298 * allocation. The lock and the data it protects are colocated in the same 9299 * memory region. 9300 * 9301 * Hence, everytime a register holds a pointer value pointing to such 9302 * allocation, the verifier preserves a unique reg->id for it. 9303 * 9304 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 9305 * bpf_spin_lock is called. 9306 * 9307 * To enable this, lock state in the verifier captures two values: 9308 * active_lock.ptr = Register's type specific pointer 9309 * active_lock.id = A unique ID for each register pointer value 9310 * 9311 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 9312 * supported register types. 9313 * 9314 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 9315 * allocated objects is the reg->btf pointer. 9316 * 9317 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 9318 * can establish the provenance of the map value statically for each distinct 9319 * lookup into such maps. They always contain a single map value hence unique 9320 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 9321 * 9322 * So, in case of global variables, they use array maps with max_entries = 1, 9323 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 9324 * into the same map value as max_entries is 1, as described above). 9325 * 9326 * In case of inner map lookups, the inner map pointer has same map_ptr as the 9327 * outer map pointer (in verifier context), but each lookup into an inner map 9328 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 9329 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 9330 * will get different reg->id assigned to each lookup, hence different 9331 * active_lock.id. 9332 * 9333 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 9334 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 9335 * returned from bpf_obj_new. Each allocation receives a new reg->id. 9336 */ 9337 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 9338 { 9339 void *ptr; 9340 u32 id; 9341 9342 switch ((int)reg->type) { 9343 case PTR_TO_MAP_VALUE: 9344 ptr = reg->map_ptr; 9345 break; 9346 case PTR_TO_BTF_ID | MEM_ALLOC: 9347 ptr = reg->btf; 9348 break; 9349 default: 9350 verbose(env, "verifier internal error: unknown reg type for lock check\n"); 9351 return -EFAULT; 9352 } 9353 id = reg->id; 9354 9355 if (!env->cur_state->active_lock.ptr) 9356 return -EINVAL; 9357 if (env->cur_state->active_lock.ptr != ptr || 9358 env->cur_state->active_lock.id != id) { 9359 verbose(env, "held lock and object are not in the same allocation\n"); 9360 return -EINVAL; 9361 } 9362 return 0; 9363 } 9364 9365 static bool is_bpf_list_api_kfunc(u32 btf_id) 9366 { 9367 return btf_id == special_kfunc_list[KF_bpf_list_push_front] || 9368 btf_id == special_kfunc_list[KF_bpf_list_push_back] || 9369 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 9370 btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 9371 } 9372 9373 static bool is_bpf_rbtree_api_kfunc(u32 btf_id) 9374 { 9375 return btf_id == special_kfunc_list[KF_bpf_rbtree_add] || 9376 btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 9377 btf_id == special_kfunc_list[KF_bpf_rbtree_first]; 9378 } 9379 9380 static bool is_bpf_graph_api_kfunc(u32 btf_id) 9381 { 9382 return is_bpf_list_api_kfunc(btf_id) || is_bpf_rbtree_api_kfunc(btf_id); 9383 } 9384 9385 static bool is_callback_calling_kfunc(u32 btf_id) 9386 { 9387 return btf_id == special_kfunc_list[KF_bpf_rbtree_add]; 9388 } 9389 9390 static bool is_rbtree_lock_required_kfunc(u32 btf_id) 9391 { 9392 return is_bpf_rbtree_api_kfunc(btf_id); 9393 } 9394 9395 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, 9396 enum btf_field_type head_field_type, 9397 u32 kfunc_btf_id) 9398 { 9399 bool ret; 9400 9401 switch (head_field_type) { 9402 case BPF_LIST_HEAD: 9403 ret = is_bpf_list_api_kfunc(kfunc_btf_id); 9404 break; 9405 case BPF_RB_ROOT: 9406 ret = is_bpf_rbtree_api_kfunc(kfunc_btf_id); 9407 break; 9408 default: 9409 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", 9410 btf_field_type_name(head_field_type)); 9411 return false; 9412 } 9413 9414 if (!ret) 9415 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", 9416 btf_field_type_name(head_field_type)); 9417 return ret; 9418 } 9419 9420 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, 9421 enum btf_field_type node_field_type, 9422 u32 kfunc_btf_id) 9423 { 9424 bool ret; 9425 9426 switch (node_field_type) { 9427 case BPF_LIST_NODE: 9428 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_front] || 9429 kfunc_btf_id == special_kfunc_list[KF_bpf_list_push_back]); 9430 break; 9431 case BPF_RB_NODE: 9432 ret = (kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_remove] || 9433 kfunc_btf_id == special_kfunc_list[KF_bpf_rbtree_add]); 9434 break; 9435 default: 9436 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", 9437 btf_field_type_name(node_field_type)); 9438 return false; 9439 } 9440 9441 if (!ret) 9442 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", 9443 btf_field_type_name(node_field_type)); 9444 return ret; 9445 } 9446 9447 static int 9448 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, 9449 struct bpf_reg_state *reg, u32 regno, 9450 struct bpf_kfunc_call_arg_meta *meta, 9451 enum btf_field_type head_field_type, 9452 struct btf_field **head_field) 9453 { 9454 const char *head_type_name; 9455 struct btf_field *field; 9456 struct btf_record *rec; 9457 u32 head_off; 9458 9459 if (meta->btf != btf_vmlinux) { 9460 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 9461 return -EFAULT; 9462 } 9463 9464 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) 9465 return -EFAULT; 9466 9467 head_type_name = btf_field_type_name(head_field_type); 9468 if (!tnum_is_const(reg->var_off)) { 9469 verbose(env, 9470 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 9471 regno, head_type_name); 9472 return -EINVAL; 9473 } 9474 9475 rec = reg_btf_record(reg); 9476 head_off = reg->off + reg->var_off.value; 9477 field = btf_record_find(rec, head_off, head_field_type); 9478 if (!field) { 9479 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); 9480 return -EINVAL; 9481 } 9482 9483 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 9484 if (check_reg_allocation_locked(env, reg)) { 9485 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", 9486 rec->spin_lock_off, head_type_name); 9487 return -EINVAL; 9488 } 9489 9490 if (*head_field) { 9491 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); 9492 return -EFAULT; 9493 } 9494 *head_field = field; 9495 return 0; 9496 } 9497 9498 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 9499 struct bpf_reg_state *reg, u32 regno, 9500 struct bpf_kfunc_call_arg_meta *meta) 9501 { 9502 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, 9503 &meta->arg_list_head.field); 9504 } 9505 9506 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, 9507 struct bpf_reg_state *reg, u32 regno, 9508 struct bpf_kfunc_call_arg_meta *meta) 9509 { 9510 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, 9511 &meta->arg_rbtree_root.field); 9512 } 9513 9514 static int 9515 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, 9516 struct bpf_reg_state *reg, u32 regno, 9517 struct bpf_kfunc_call_arg_meta *meta, 9518 enum btf_field_type head_field_type, 9519 enum btf_field_type node_field_type, 9520 struct btf_field **node_field) 9521 { 9522 const char *node_type_name; 9523 const struct btf_type *et, *t; 9524 struct btf_field *field; 9525 u32 node_off; 9526 9527 if (meta->btf != btf_vmlinux) { 9528 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); 9529 return -EFAULT; 9530 } 9531 9532 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) 9533 return -EFAULT; 9534 9535 node_type_name = btf_field_type_name(node_field_type); 9536 if (!tnum_is_const(reg->var_off)) { 9537 verbose(env, 9538 "R%d doesn't have constant offset. %s has to be at the constant offset\n", 9539 regno, node_type_name); 9540 return -EINVAL; 9541 } 9542 9543 node_off = reg->off + reg->var_off.value; 9544 field = reg_find_field_offset(reg, node_off, node_field_type); 9545 if (!field || field->offset != node_off) { 9546 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); 9547 return -EINVAL; 9548 } 9549 9550 field = *node_field; 9551 9552 et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); 9553 t = btf_type_by_id(reg->btf, reg->btf_id); 9554 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, 9555 field->graph_root.value_btf_id, true)) { 9556 verbose(env, "operation on %s expects arg#1 %s at offset=%d " 9557 "in struct %s, but arg is at offset=%d in struct %s\n", 9558 btf_field_type_name(head_field_type), 9559 btf_field_type_name(node_field_type), 9560 field->graph_root.node_offset, 9561 btf_name_by_offset(field->graph_root.btf, et->name_off), 9562 node_off, btf_name_by_offset(reg->btf, t->name_off)); 9563 return -EINVAL; 9564 } 9565 9566 if (node_off != field->graph_root.node_offset) { 9567 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", 9568 node_off, btf_field_type_name(node_field_type), 9569 field->graph_root.node_offset, 9570 btf_name_by_offset(field->graph_root.btf, et->name_off)); 9571 return -EINVAL; 9572 } 9573 9574 return 0; 9575 } 9576 9577 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 9578 struct bpf_reg_state *reg, u32 regno, 9579 struct bpf_kfunc_call_arg_meta *meta) 9580 { 9581 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 9582 BPF_LIST_HEAD, BPF_LIST_NODE, 9583 &meta->arg_list_head.field); 9584 } 9585 9586 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, 9587 struct bpf_reg_state *reg, u32 regno, 9588 struct bpf_kfunc_call_arg_meta *meta) 9589 { 9590 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, 9591 BPF_RB_ROOT, BPF_RB_NODE, 9592 &meta->arg_rbtree_root.field); 9593 } 9594 9595 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, 9596 int insn_idx) 9597 { 9598 const char *func_name = meta->func_name, *ref_tname; 9599 const struct btf *btf = meta->btf; 9600 const struct btf_param *args; 9601 u32 i, nargs; 9602 int ret; 9603 9604 args = (const struct btf_param *)(meta->func_proto + 1); 9605 nargs = btf_type_vlen(meta->func_proto); 9606 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 9607 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, 9608 MAX_BPF_FUNC_REG_ARGS); 9609 return -EINVAL; 9610 } 9611 9612 /* Check that BTF function arguments match actual types that the 9613 * verifier sees. 9614 */ 9615 for (i = 0; i < nargs; i++) { 9616 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; 9617 const struct btf_type *t, *ref_t, *resolve_ret; 9618 enum bpf_arg_type arg_type = ARG_DONTCARE; 9619 u32 regno = i + 1, ref_id, type_size; 9620 bool is_ret_buf_sz = false; 9621 int kf_arg_type; 9622 9623 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 9624 9625 if (is_kfunc_arg_ignore(btf, &args[i])) 9626 continue; 9627 9628 if (btf_type_is_scalar(t)) { 9629 if (reg->type != SCALAR_VALUE) { 9630 verbose(env, "R%d is not a scalar\n", regno); 9631 return -EINVAL; 9632 } 9633 9634 if (is_kfunc_arg_constant(meta->btf, &args[i])) { 9635 if (meta->arg_constant.found) { 9636 verbose(env, "verifier internal error: only one constant argument permitted\n"); 9637 return -EFAULT; 9638 } 9639 if (!tnum_is_const(reg->var_off)) { 9640 verbose(env, "R%d must be a known constant\n", regno); 9641 return -EINVAL; 9642 } 9643 ret = mark_chain_precision(env, regno); 9644 if (ret < 0) 9645 return ret; 9646 meta->arg_constant.found = true; 9647 meta->arg_constant.value = reg->var_off.value; 9648 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { 9649 meta->r0_rdonly = true; 9650 is_ret_buf_sz = true; 9651 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { 9652 is_ret_buf_sz = true; 9653 } 9654 9655 if (is_ret_buf_sz) { 9656 if (meta->r0_size) { 9657 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); 9658 return -EINVAL; 9659 } 9660 9661 if (!tnum_is_const(reg->var_off)) { 9662 verbose(env, "R%d is not a const\n", regno); 9663 return -EINVAL; 9664 } 9665 9666 meta->r0_size = reg->var_off.value; 9667 ret = mark_chain_precision(env, regno); 9668 if (ret) 9669 return ret; 9670 } 9671 continue; 9672 } 9673 9674 if (!btf_type_is_ptr(t)) { 9675 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); 9676 return -EINVAL; 9677 } 9678 9679 if (is_kfunc_trusted_args(meta) && 9680 (register_is_null(reg) || type_may_be_null(reg->type))) { 9681 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); 9682 return -EACCES; 9683 } 9684 9685 if (reg->ref_obj_id) { 9686 if (is_kfunc_release(meta) && meta->ref_obj_id) { 9687 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 9688 regno, reg->ref_obj_id, 9689 meta->ref_obj_id); 9690 return -EFAULT; 9691 } 9692 meta->ref_obj_id = reg->ref_obj_id; 9693 if (is_kfunc_release(meta)) 9694 meta->release_regno = regno; 9695 } 9696 9697 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 9698 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 9699 9700 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); 9701 if (kf_arg_type < 0) 9702 return kf_arg_type; 9703 9704 switch (kf_arg_type) { 9705 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 9706 case KF_ARG_PTR_TO_BTF_ID: 9707 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 9708 break; 9709 9710 if (!is_trusted_reg(reg)) { 9711 if (!is_kfunc_rcu(meta)) { 9712 verbose(env, "R%d must be referenced or trusted\n", regno); 9713 return -EINVAL; 9714 } 9715 if (!is_rcu_reg(reg)) { 9716 verbose(env, "R%d must be a rcu pointer\n", regno); 9717 return -EINVAL; 9718 } 9719 } 9720 9721 fallthrough; 9722 case KF_ARG_PTR_TO_CTX: 9723 /* Trusted arguments have the same offset checks as release arguments */ 9724 arg_type |= OBJ_RELEASE; 9725 break; 9726 case KF_ARG_PTR_TO_KPTR: 9727 case KF_ARG_PTR_TO_DYNPTR: 9728 case KF_ARG_PTR_TO_LIST_HEAD: 9729 case KF_ARG_PTR_TO_LIST_NODE: 9730 case KF_ARG_PTR_TO_RB_ROOT: 9731 case KF_ARG_PTR_TO_RB_NODE: 9732 case KF_ARG_PTR_TO_MEM: 9733 case KF_ARG_PTR_TO_MEM_SIZE: 9734 case KF_ARG_PTR_TO_CALLBACK: 9735 /* Trusted by default */ 9736 break; 9737 default: 9738 WARN_ON_ONCE(1); 9739 return -EFAULT; 9740 } 9741 9742 if (is_kfunc_release(meta) && reg->ref_obj_id) 9743 arg_type |= OBJ_RELEASE; 9744 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 9745 if (ret < 0) 9746 return ret; 9747 9748 switch (kf_arg_type) { 9749 case KF_ARG_PTR_TO_CTX: 9750 if (reg->type != PTR_TO_CTX) { 9751 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); 9752 return -EINVAL; 9753 } 9754 9755 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 9756 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 9757 if (ret < 0) 9758 return -EINVAL; 9759 meta->ret_btf_id = ret; 9760 } 9761 break; 9762 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 9763 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 9764 verbose(env, "arg#%d expected pointer to allocated object\n", i); 9765 return -EINVAL; 9766 } 9767 if (!reg->ref_obj_id) { 9768 verbose(env, "allocated object must be referenced\n"); 9769 return -EINVAL; 9770 } 9771 if (meta->btf == btf_vmlinux && 9772 meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 9773 meta->arg_obj_drop.btf = reg->btf; 9774 meta->arg_obj_drop.btf_id = reg->btf_id; 9775 } 9776 break; 9777 case KF_ARG_PTR_TO_KPTR: 9778 if (reg->type != PTR_TO_MAP_VALUE) { 9779 verbose(env, "arg#0 expected pointer to map value\n"); 9780 return -EINVAL; 9781 } 9782 ret = process_kf_arg_ptr_to_kptr(env, reg, ref_t, ref_tname, meta, i); 9783 if (ret < 0) 9784 return ret; 9785 break; 9786 case KF_ARG_PTR_TO_DYNPTR: 9787 { 9788 enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR; 9789 9790 if (reg->type != PTR_TO_STACK && 9791 reg->type != CONST_PTR_TO_DYNPTR) { 9792 verbose(env, "arg#%d expected pointer to stack or dynptr_ptr\n", i); 9793 return -EINVAL; 9794 } 9795 9796 if (reg->type == CONST_PTR_TO_DYNPTR) 9797 dynptr_arg_type |= MEM_RDONLY; 9798 9799 if (is_kfunc_arg_uninit(btf, &args[i])) 9800 dynptr_arg_type |= MEM_UNINIT; 9801 9802 if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) 9803 dynptr_arg_type |= DYNPTR_TYPE_SKB; 9804 else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) 9805 dynptr_arg_type |= DYNPTR_TYPE_XDP; 9806 9807 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type); 9808 if (ret < 0) 9809 return ret; 9810 9811 if (!(dynptr_arg_type & MEM_UNINIT)) { 9812 int id = dynptr_id(env, reg); 9813 9814 if (id < 0) { 9815 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); 9816 return id; 9817 } 9818 meta->initialized_dynptr.id = id; 9819 meta->initialized_dynptr.type = dynptr_get_type(env, reg); 9820 } 9821 9822 break; 9823 } 9824 case KF_ARG_PTR_TO_LIST_HEAD: 9825 if (reg->type != PTR_TO_MAP_VALUE && 9826 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 9827 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 9828 return -EINVAL; 9829 } 9830 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 9831 verbose(env, "allocated object must be referenced\n"); 9832 return -EINVAL; 9833 } 9834 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); 9835 if (ret < 0) 9836 return ret; 9837 break; 9838 case KF_ARG_PTR_TO_RB_ROOT: 9839 if (reg->type != PTR_TO_MAP_VALUE && 9840 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 9841 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 9842 return -EINVAL; 9843 } 9844 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 9845 verbose(env, "allocated object must be referenced\n"); 9846 return -EINVAL; 9847 } 9848 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); 9849 if (ret < 0) 9850 return ret; 9851 break; 9852 case KF_ARG_PTR_TO_LIST_NODE: 9853 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 9854 verbose(env, "arg#%d expected pointer to allocated object\n", i); 9855 return -EINVAL; 9856 } 9857 if (!reg->ref_obj_id) { 9858 verbose(env, "allocated object must be referenced\n"); 9859 return -EINVAL; 9860 } 9861 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); 9862 if (ret < 0) 9863 return ret; 9864 break; 9865 case KF_ARG_PTR_TO_RB_NODE: 9866 if (meta->func_id == special_kfunc_list[KF_bpf_rbtree_remove]) { 9867 if (!type_is_non_owning_ref(reg->type) || reg->ref_obj_id) { 9868 verbose(env, "rbtree_remove node input must be non-owning ref\n"); 9869 return -EINVAL; 9870 } 9871 if (in_rbtree_lock_required_cb(env)) { 9872 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); 9873 return -EINVAL; 9874 } 9875 } else { 9876 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 9877 verbose(env, "arg#%d expected pointer to allocated object\n", i); 9878 return -EINVAL; 9879 } 9880 if (!reg->ref_obj_id) { 9881 verbose(env, "allocated object must be referenced\n"); 9882 return -EINVAL; 9883 } 9884 } 9885 9886 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); 9887 if (ret < 0) 9888 return ret; 9889 break; 9890 case KF_ARG_PTR_TO_BTF_ID: 9891 /* Only base_type is checked, further checks are done here */ 9892 if ((base_type(reg->type) != PTR_TO_BTF_ID || 9893 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && 9894 !reg2btf_ids[base_type(reg->type)]) { 9895 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); 9896 verbose(env, "expected %s or socket\n", 9897 reg_type_str(env, base_type(reg->type) | 9898 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 9899 return -EINVAL; 9900 } 9901 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 9902 if (ret < 0) 9903 return ret; 9904 break; 9905 case KF_ARG_PTR_TO_MEM: 9906 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 9907 if (IS_ERR(resolve_ret)) { 9908 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 9909 i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); 9910 return -EINVAL; 9911 } 9912 ret = check_mem_reg(env, reg, regno, type_size); 9913 if (ret < 0) 9914 return ret; 9915 break; 9916 case KF_ARG_PTR_TO_MEM_SIZE: 9917 { 9918 struct bpf_reg_state *size_reg = ®s[regno + 1]; 9919 const struct btf_param *size_arg = &args[i + 1]; 9920 9921 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); 9922 if (ret < 0) { 9923 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); 9924 return ret; 9925 } 9926 9927 if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { 9928 if (meta->arg_constant.found) { 9929 verbose(env, "verifier internal error: only one constant argument permitted\n"); 9930 return -EFAULT; 9931 } 9932 if (!tnum_is_const(size_reg->var_off)) { 9933 verbose(env, "R%d must be a known constant\n", regno + 1); 9934 return -EINVAL; 9935 } 9936 meta->arg_constant.found = true; 9937 meta->arg_constant.value = size_reg->var_off.value; 9938 } 9939 9940 /* Skip next '__sz' or '__szk' argument */ 9941 i++; 9942 break; 9943 } 9944 case KF_ARG_PTR_TO_CALLBACK: 9945 meta->subprogno = reg->subprogno; 9946 break; 9947 } 9948 } 9949 9950 if (is_kfunc_release(meta) && !meta->release_regno) { 9951 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", 9952 func_name); 9953 return -EINVAL; 9954 } 9955 9956 return 0; 9957 } 9958 9959 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 9960 int *insn_idx_p) 9961 { 9962 const struct btf_type *t, *func, *func_proto, *ptr_type; 9963 u32 i, nargs, func_id, ptr_type_id, release_ref_obj_id; 9964 struct bpf_reg_state *regs = cur_regs(env); 9965 const char *func_name, *ptr_type_name; 9966 bool sleepable, rcu_lock, rcu_unlock; 9967 struct bpf_kfunc_call_arg_meta meta; 9968 int err, insn_idx = *insn_idx_p; 9969 const struct btf_param *args; 9970 const struct btf_type *ret_t; 9971 struct btf *desc_btf; 9972 u32 *kfunc_flags; 9973 9974 /* skip for now, but return error when we find this in fixup_kfunc_call */ 9975 if (!insn->imm) 9976 return 0; 9977 9978 desc_btf = find_kfunc_desc_btf(env, insn->off); 9979 if (IS_ERR(desc_btf)) 9980 return PTR_ERR(desc_btf); 9981 9982 func_id = insn->imm; 9983 func = btf_type_by_id(desc_btf, func_id); 9984 func_name = btf_name_by_offset(desc_btf, func->name_off); 9985 func_proto = btf_type_by_id(desc_btf, func->type); 9986 9987 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); 9988 if (!kfunc_flags) { 9989 verbose(env, "calling kernel function %s is not allowed\n", 9990 func_name); 9991 return -EACCES; 9992 } 9993 9994 /* Prepare kfunc call metadata */ 9995 memset(&meta, 0, sizeof(meta)); 9996 meta.btf = desc_btf; 9997 meta.func_id = func_id; 9998 meta.kfunc_flags = *kfunc_flags; 9999 meta.func_proto = func_proto; 10000 meta.func_name = func_name; 10001 10002 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 10003 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 10004 return -EACCES; 10005 } 10006 10007 sleepable = is_kfunc_sleepable(&meta); 10008 if (sleepable && !env->prog->aux->sleepable) { 10009 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); 10010 return -EACCES; 10011 } 10012 10013 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 10014 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 10015 if ((rcu_lock || rcu_unlock) && !env->rcu_tag_supported) { 10016 verbose(env, "no vmlinux btf rcu tag support for kfunc %s\n", func_name); 10017 return -EACCES; 10018 } 10019 10020 if (env->cur_state->active_rcu_lock) { 10021 struct bpf_func_state *state; 10022 struct bpf_reg_state *reg; 10023 10024 if (rcu_lock) { 10025 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); 10026 return -EINVAL; 10027 } else if (rcu_unlock) { 10028 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 10029 if (reg->type & MEM_RCU) { 10030 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); 10031 reg->type |= PTR_UNTRUSTED; 10032 } 10033 })); 10034 env->cur_state->active_rcu_lock = false; 10035 } else if (sleepable) { 10036 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); 10037 return -EACCES; 10038 } 10039 } else if (rcu_lock) { 10040 env->cur_state->active_rcu_lock = true; 10041 } else if (rcu_unlock) { 10042 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 10043 return -EINVAL; 10044 } 10045 10046 /* Check the arguments */ 10047 err = check_kfunc_args(env, &meta, insn_idx); 10048 if (err < 0) 10049 return err; 10050 /* In case of release function, we get register number of refcounted 10051 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 10052 */ 10053 if (meta.release_regno) { 10054 err = release_reference(env, regs[meta.release_regno].ref_obj_id); 10055 if (err) { 10056 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 10057 func_name, func_id); 10058 return err; 10059 } 10060 } 10061 10062 if (meta.func_id == special_kfunc_list[KF_bpf_list_push_front] || 10063 meta.func_id == special_kfunc_list[KF_bpf_list_push_back] || 10064 meta.func_id == special_kfunc_list[KF_bpf_rbtree_add]) { 10065 release_ref_obj_id = regs[BPF_REG_2].ref_obj_id; 10066 err = ref_convert_owning_non_owning(env, release_ref_obj_id); 10067 if (err) { 10068 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", 10069 func_name, func_id); 10070 return err; 10071 } 10072 10073 err = release_reference(env, release_ref_obj_id); 10074 if (err) { 10075 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 10076 func_name, func_id); 10077 return err; 10078 } 10079 } 10080 10081 if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add]) { 10082 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 10083 set_rbtree_add_callback_state); 10084 if (err) { 10085 verbose(env, "kfunc %s#%d failed callback verification\n", 10086 func_name, func_id); 10087 return err; 10088 } 10089 } 10090 10091 for (i = 0; i < CALLER_SAVED_REGS; i++) 10092 mark_reg_not_init(env, regs, caller_saved[i]); 10093 10094 /* Check return type */ 10095 t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL); 10096 10097 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 10098 /* Only exception is bpf_obj_new_impl */ 10099 if (meta.btf != btf_vmlinux || meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl]) { 10100 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 10101 return -EINVAL; 10102 } 10103 } 10104 10105 if (btf_type_is_scalar(t)) { 10106 mark_reg_unknown(env, regs, BPF_REG_0); 10107 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 10108 } else if (btf_type_is_ptr(t)) { 10109 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 10110 10111 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 10112 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 10113 struct btf *ret_btf; 10114 u32 ret_btf_id; 10115 10116 if (unlikely(!bpf_global_ma_set)) 10117 return -ENOMEM; 10118 10119 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { 10120 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 10121 return -EINVAL; 10122 } 10123 10124 ret_btf = env->prog->aux->btf; 10125 ret_btf_id = meta.arg_constant.value; 10126 10127 /* This may be NULL due to user not supplying a BTF */ 10128 if (!ret_btf) { 10129 verbose(env, "bpf_obj_new requires prog BTF\n"); 10130 return -EINVAL; 10131 } 10132 10133 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 10134 if (!ret_t || !__btf_type_is_struct(ret_t)) { 10135 verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); 10136 return -EINVAL; 10137 } 10138 10139 mark_reg_known_zero(env, regs, BPF_REG_0); 10140 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 10141 regs[BPF_REG_0].btf = ret_btf; 10142 regs[BPF_REG_0].btf_id = ret_btf_id; 10143 10144 env->insn_aux_data[insn_idx].obj_new_size = ret_t->size; 10145 env->insn_aux_data[insn_idx].kptr_struct_meta = 10146 btf_find_struct_meta(ret_btf, ret_btf_id); 10147 } else if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 10148 env->insn_aux_data[insn_idx].kptr_struct_meta = 10149 btf_find_struct_meta(meta.arg_obj_drop.btf, 10150 meta.arg_obj_drop.btf_id); 10151 } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 10152 meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 10153 struct btf_field *field = meta.arg_list_head.field; 10154 10155 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 10156 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove] || 10157 meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 10158 struct btf_field *field = meta.arg_rbtree_root.field; 10159 10160 mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root); 10161 } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 10162 mark_reg_known_zero(env, regs, BPF_REG_0); 10163 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 10164 regs[BPF_REG_0].btf = desc_btf; 10165 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 10166 } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 10167 ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value); 10168 if (!ret_t || !btf_type_is_struct(ret_t)) { 10169 verbose(env, 10170 "kfunc bpf_rdonly_cast type ID argument must be of a struct\n"); 10171 return -EINVAL; 10172 } 10173 10174 mark_reg_known_zero(env, regs, BPF_REG_0); 10175 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 10176 regs[BPF_REG_0].btf = desc_btf; 10177 regs[BPF_REG_0].btf_id = meta.arg_constant.value; 10178 } else if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice] || 10179 meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) { 10180 enum bpf_type_flag type_flag = get_dynptr_type_flag(meta.initialized_dynptr.type); 10181 10182 mark_reg_known_zero(env, regs, BPF_REG_0); 10183 10184 if (!meta.arg_constant.found) { 10185 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); 10186 return -EFAULT; 10187 } 10188 10189 regs[BPF_REG_0].mem_size = meta.arg_constant.value; 10190 10191 /* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */ 10192 regs[BPF_REG_0].type = PTR_TO_MEM | type_flag; 10193 10194 if (meta.func_id == special_kfunc_list[KF_bpf_dynptr_slice]) { 10195 regs[BPF_REG_0].type |= MEM_RDONLY; 10196 } else { 10197 /* this will set env->seen_direct_write to true */ 10198 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { 10199 verbose(env, "the prog does not allow writes to packet data\n"); 10200 return -EINVAL; 10201 } 10202 } 10203 10204 if (!meta.initialized_dynptr.id) { 10205 verbose(env, "verifier internal error: no dynptr id\n"); 10206 return -EFAULT; 10207 } 10208 regs[BPF_REG_0].dynptr_id = meta.initialized_dynptr.id; 10209 10210 /* we don't need to set BPF_REG_0's ref obj id 10211 * because packet slices are not refcounted (see 10212 * dynptr_type_refcounted) 10213 */ 10214 } else { 10215 verbose(env, "kernel function %s unhandled dynamic return type\n", 10216 meta.func_name); 10217 return -EFAULT; 10218 } 10219 } else if (!__btf_type_is_struct(ptr_type)) { 10220 if (!meta.r0_size) { 10221 ptr_type_name = btf_name_by_offset(desc_btf, 10222 ptr_type->name_off); 10223 verbose(env, 10224 "kernel function %s returns pointer type %s %s is not supported\n", 10225 func_name, 10226 btf_type_str(ptr_type), 10227 ptr_type_name); 10228 return -EINVAL; 10229 } 10230 10231 mark_reg_known_zero(env, regs, BPF_REG_0); 10232 regs[BPF_REG_0].type = PTR_TO_MEM; 10233 regs[BPF_REG_0].mem_size = meta.r0_size; 10234 10235 if (meta.r0_rdonly) 10236 regs[BPF_REG_0].type |= MEM_RDONLY; 10237 10238 /* Ensures we don't access the memory after a release_reference() */ 10239 if (meta.ref_obj_id) 10240 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 10241 } else { 10242 mark_reg_known_zero(env, regs, BPF_REG_0); 10243 regs[BPF_REG_0].btf = desc_btf; 10244 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 10245 regs[BPF_REG_0].btf_id = ptr_type_id; 10246 } 10247 10248 if (is_kfunc_ret_null(&meta)) { 10249 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 10250 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 10251 regs[BPF_REG_0].id = ++env->id_gen; 10252 } 10253 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 10254 if (is_kfunc_acquire(&meta)) { 10255 int id = acquire_reference_state(env, insn_idx); 10256 10257 if (id < 0) 10258 return id; 10259 if (is_kfunc_ret_null(&meta)) 10260 regs[BPF_REG_0].id = id; 10261 regs[BPF_REG_0].ref_obj_id = id; 10262 } else if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_first]) { 10263 ref_set_non_owning(env, ®s[BPF_REG_0]); 10264 } 10265 10266 if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_remove]) 10267 invalidate_non_owning_refs(env); 10268 10269 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 10270 regs[BPF_REG_0].id = ++env->id_gen; 10271 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 10272 10273 nargs = btf_type_vlen(func_proto); 10274 args = (const struct btf_param *)(func_proto + 1); 10275 for (i = 0; i < nargs; i++) { 10276 u32 regno = i + 1; 10277 10278 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 10279 if (btf_type_is_ptr(t)) 10280 mark_btf_func_reg_size(env, regno, sizeof(void *)); 10281 else 10282 /* scalar. ensured by btf_check_kfunc_arg_match() */ 10283 mark_btf_func_reg_size(env, regno, t->size); 10284 } 10285 10286 return 0; 10287 } 10288 10289 static bool signed_add_overflows(s64 a, s64 b) 10290 { 10291 /* Do the add in u64, where overflow is well-defined */ 10292 s64 res = (s64)((u64)a + (u64)b); 10293 10294 if (b < 0) 10295 return res > a; 10296 return res < a; 10297 } 10298 10299 static bool signed_add32_overflows(s32 a, s32 b) 10300 { 10301 /* Do the add in u32, where overflow is well-defined */ 10302 s32 res = (s32)((u32)a + (u32)b); 10303 10304 if (b < 0) 10305 return res > a; 10306 return res < a; 10307 } 10308 10309 static bool signed_sub_overflows(s64 a, s64 b) 10310 { 10311 /* Do the sub in u64, where overflow is well-defined */ 10312 s64 res = (s64)((u64)a - (u64)b); 10313 10314 if (b < 0) 10315 return res < a; 10316 return res > a; 10317 } 10318 10319 static bool signed_sub32_overflows(s32 a, s32 b) 10320 { 10321 /* Do the sub in u32, where overflow is well-defined */ 10322 s32 res = (s32)((u32)a - (u32)b); 10323 10324 if (b < 0) 10325 return res < a; 10326 return res > a; 10327 } 10328 10329 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 10330 const struct bpf_reg_state *reg, 10331 enum bpf_reg_type type) 10332 { 10333 bool known = tnum_is_const(reg->var_off); 10334 s64 val = reg->var_off.value; 10335 s64 smin = reg->smin_value; 10336 10337 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 10338 verbose(env, "math between %s pointer and %lld is not allowed\n", 10339 reg_type_str(env, type), val); 10340 return false; 10341 } 10342 10343 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 10344 verbose(env, "%s pointer offset %d is not allowed\n", 10345 reg_type_str(env, type), reg->off); 10346 return false; 10347 } 10348 10349 if (smin == S64_MIN) { 10350 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 10351 reg_type_str(env, type)); 10352 return false; 10353 } 10354 10355 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 10356 verbose(env, "value %lld makes %s pointer be out of bounds\n", 10357 smin, reg_type_str(env, type)); 10358 return false; 10359 } 10360 10361 return true; 10362 } 10363 10364 enum { 10365 REASON_BOUNDS = -1, 10366 REASON_TYPE = -2, 10367 REASON_PATHS = -3, 10368 REASON_LIMIT = -4, 10369 REASON_STACK = -5, 10370 }; 10371 10372 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 10373 u32 *alu_limit, bool mask_to_left) 10374 { 10375 u32 max = 0, ptr_limit = 0; 10376 10377 switch (ptr_reg->type) { 10378 case PTR_TO_STACK: 10379 /* Offset 0 is out-of-bounds, but acceptable start for the 10380 * left direction, see BPF_REG_FP. Also, unknown scalar 10381 * offset where we would need to deal with min/max bounds is 10382 * currently prohibited for unprivileged. 10383 */ 10384 max = MAX_BPF_STACK + mask_to_left; 10385 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 10386 break; 10387 case PTR_TO_MAP_VALUE: 10388 max = ptr_reg->map_ptr->value_size; 10389 ptr_limit = (mask_to_left ? 10390 ptr_reg->smin_value : 10391 ptr_reg->umax_value) + ptr_reg->off; 10392 break; 10393 default: 10394 return REASON_TYPE; 10395 } 10396 10397 if (ptr_limit >= max) 10398 return REASON_LIMIT; 10399 *alu_limit = ptr_limit; 10400 return 0; 10401 } 10402 10403 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 10404 const struct bpf_insn *insn) 10405 { 10406 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 10407 } 10408 10409 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 10410 u32 alu_state, u32 alu_limit) 10411 { 10412 /* If we arrived here from different branches with different 10413 * state or limits to sanitize, then this won't work. 10414 */ 10415 if (aux->alu_state && 10416 (aux->alu_state != alu_state || 10417 aux->alu_limit != alu_limit)) 10418 return REASON_PATHS; 10419 10420 /* Corresponding fixup done in do_misc_fixups(). */ 10421 aux->alu_state = alu_state; 10422 aux->alu_limit = alu_limit; 10423 return 0; 10424 } 10425 10426 static int sanitize_val_alu(struct bpf_verifier_env *env, 10427 struct bpf_insn *insn) 10428 { 10429 struct bpf_insn_aux_data *aux = cur_aux(env); 10430 10431 if (can_skip_alu_sanitation(env, insn)) 10432 return 0; 10433 10434 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 10435 } 10436 10437 static bool sanitize_needed(u8 opcode) 10438 { 10439 return opcode == BPF_ADD || opcode == BPF_SUB; 10440 } 10441 10442 struct bpf_sanitize_info { 10443 struct bpf_insn_aux_data aux; 10444 bool mask_to_left; 10445 }; 10446 10447 static struct bpf_verifier_state * 10448 sanitize_speculative_path(struct bpf_verifier_env *env, 10449 const struct bpf_insn *insn, 10450 u32 next_idx, u32 curr_idx) 10451 { 10452 struct bpf_verifier_state *branch; 10453 struct bpf_reg_state *regs; 10454 10455 branch = push_stack(env, next_idx, curr_idx, true); 10456 if (branch && insn) { 10457 regs = branch->frame[branch->curframe]->regs; 10458 if (BPF_SRC(insn->code) == BPF_K) { 10459 mark_reg_unknown(env, regs, insn->dst_reg); 10460 } else if (BPF_SRC(insn->code) == BPF_X) { 10461 mark_reg_unknown(env, regs, insn->dst_reg); 10462 mark_reg_unknown(env, regs, insn->src_reg); 10463 } 10464 } 10465 return branch; 10466 } 10467 10468 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 10469 struct bpf_insn *insn, 10470 const struct bpf_reg_state *ptr_reg, 10471 const struct bpf_reg_state *off_reg, 10472 struct bpf_reg_state *dst_reg, 10473 struct bpf_sanitize_info *info, 10474 const bool commit_window) 10475 { 10476 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 10477 struct bpf_verifier_state *vstate = env->cur_state; 10478 bool off_is_imm = tnum_is_const(off_reg->var_off); 10479 bool off_is_neg = off_reg->smin_value < 0; 10480 bool ptr_is_dst_reg = ptr_reg == dst_reg; 10481 u8 opcode = BPF_OP(insn->code); 10482 u32 alu_state, alu_limit; 10483 struct bpf_reg_state tmp; 10484 bool ret; 10485 int err; 10486 10487 if (can_skip_alu_sanitation(env, insn)) 10488 return 0; 10489 10490 /* We already marked aux for masking from non-speculative 10491 * paths, thus we got here in the first place. We only care 10492 * to explore bad access from here. 10493 */ 10494 if (vstate->speculative) 10495 goto do_sim; 10496 10497 if (!commit_window) { 10498 if (!tnum_is_const(off_reg->var_off) && 10499 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 10500 return REASON_BOUNDS; 10501 10502 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 10503 (opcode == BPF_SUB && !off_is_neg); 10504 } 10505 10506 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 10507 if (err < 0) 10508 return err; 10509 10510 if (commit_window) { 10511 /* In commit phase we narrow the masking window based on 10512 * the observed pointer move after the simulated operation. 10513 */ 10514 alu_state = info->aux.alu_state; 10515 alu_limit = abs(info->aux.alu_limit - alu_limit); 10516 } else { 10517 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 10518 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 10519 alu_state |= ptr_is_dst_reg ? 10520 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 10521 10522 /* Limit pruning on unknown scalars to enable deep search for 10523 * potential masking differences from other program paths. 10524 */ 10525 if (!off_is_imm) 10526 env->explore_alu_limits = true; 10527 } 10528 10529 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 10530 if (err < 0) 10531 return err; 10532 do_sim: 10533 /* If we're in commit phase, we're done here given we already 10534 * pushed the truncated dst_reg into the speculative verification 10535 * stack. 10536 * 10537 * Also, when register is a known constant, we rewrite register-based 10538 * operation to immediate-based, and thus do not need masking (and as 10539 * a consequence, do not need to simulate the zero-truncation either). 10540 */ 10541 if (commit_window || off_is_imm) 10542 return 0; 10543 10544 /* Simulate and find potential out-of-bounds access under 10545 * speculative execution from truncation as a result of 10546 * masking when off was not within expected range. If off 10547 * sits in dst, then we temporarily need to move ptr there 10548 * to simulate dst (== 0) +/-= ptr. Needed, for example, 10549 * for cases where we use K-based arithmetic in one direction 10550 * and truncated reg-based in the other in order to explore 10551 * bad access. 10552 */ 10553 if (!ptr_is_dst_reg) { 10554 tmp = *dst_reg; 10555 copy_register_state(dst_reg, ptr_reg); 10556 } 10557 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 10558 env->insn_idx); 10559 if (!ptr_is_dst_reg && ret) 10560 *dst_reg = tmp; 10561 return !ret ? REASON_STACK : 0; 10562 } 10563 10564 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 10565 { 10566 struct bpf_verifier_state *vstate = env->cur_state; 10567 10568 /* If we simulate paths under speculation, we don't update the 10569 * insn as 'seen' such that when we verify unreachable paths in 10570 * the non-speculative domain, sanitize_dead_code() can still 10571 * rewrite/sanitize them. 10572 */ 10573 if (!vstate->speculative) 10574 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 10575 } 10576 10577 static int sanitize_err(struct bpf_verifier_env *env, 10578 const struct bpf_insn *insn, int reason, 10579 const struct bpf_reg_state *off_reg, 10580 const struct bpf_reg_state *dst_reg) 10581 { 10582 static const char *err = "pointer arithmetic with it prohibited for !root"; 10583 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 10584 u32 dst = insn->dst_reg, src = insn->src_reg; 10585 10586 switch (reason) { 10587 case REASON_BOUNDS: 10588 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 10589 off_reg == dst_reg ? dst : src, err); 10590 break; 10591 case REASON_TYPE: 10592 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 10593 off_reg == dst_reg ? src : dst, err); 10594 break; 10595 case REASON_PATHS: 10596 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 10597 dst, op, err); 10598 break; 10599 case REASON_LIMIT: 10600 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 10601 dst, op, err); 10602 break; 10603 case REASON_STACK: 10604 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 10605 dst, err); 10606 break; 10607 default: 10608 verbose(env, "verifier internal error: unknown reason (%d)\n", 10609 reason); 10610 break; 10611 } 10612 10613 return -EACCES; 10614 } 10615 10616 /* check that stack access falls within stack limits and that 'reg' doesn't 10617 * have a variable offset. 10618 * 10619 * Variable offset is prohibited for unprivileged mode for simplicity since it 10620 * requires corresponding support in Spectre masking for stack ALU. See also 10621 * retrieve_ptr_limit(). 10622 * 10623 * 10624 * 'off' includes 'reg->off'. 10625 */ 10626 static int check_stack_access_for_ptr_arithmetic( 10627 struct bpf_verifier_env *env, 10628 int regno, 10629 const struct bpf_reg_state *reg, 10630 int off) 10631 { 10632 if (!tnum_is_const(reg->var_off)) { 10633 char tn_buf[48]; 10634 10635 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 10636 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 10637 regno, tn_buf, off); 10638 return -EACCES; 10639 } 10640 10641 if (off >= 0 || off < -MAX_BPF_STACK) { 10642 verbose(env, "R%d stack pointer arithmetic goes out of range, " 10643 "prohibited for !root; off=%d\n", regno, off); 10644 return -EACCES; 10645 } 10646 10647 return 0; 10648 } 10649 10650 static int sanitize_check_bounds(struct bpf_verifier_env *env, 10651 const struct bpf_insn *insn, 10652 const struct bpf_reg_state *dst_reg) 10653 { 10654 u32 dst = insn->dst_reg; 10655 10656 /* For unprivileged we require that resulting offset must be in bounds 10657 * in order to be able to sanitize access later on. 10658 */ 10659 if (env->bypass_spec_v1) 10660 return 0; 10661 10662 switch (dst_reg->type) { 10663 case PTR_TO_STACK: 10664 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 10665 dst_reg->off + dst_reg->var_off.value)) 10666 return -EACCES; 10667 break; 10668 case PTR_TO_MAP_VALUE: 10669 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 10670 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 10671 "prohibited for !root\n", dst); 10672 return -EACCES; 10673 } 10674 break; 10675 default: 10676 break; 10677 } 10678 10679 return 0; 10680 } 10681 10682 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 10683 * Caller should also handle BPF_MOV case separately. 10684 * If we return -EACCES, caller may want to try again treating pointer as a 10685 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 10686 */ 10687 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 10688 struct bpf_insn *insn, 10689 const struct bpf_reg_state *ptr_reg, 10690 const struct bpf_reg_state *off_reg) 10691 { 10692 struct bpf_verifier_state *vstate = env->cur_state; 10693 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 10694 struct bpf_reg_state *regs = state->regs, *dst_reg; 10695 bool known = tnum_is_const(off_reg->var_off); 10696 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 10697 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 10698 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 10699 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 10700 struct bpf_sanitize_info info = {}; 10701 u8 opcode = BPF_OP(insn->code); 10702 u32 dst = insn->dst_reg; 10703 int ret; 10704 10705 dst_reg = ®s[dst]; 10706 10707 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 10708 smin_val > smax_val || umin_val > umax_val) { 10709 /* Taint dst register if offset had invalid bounds derived from 10710 * e.g. dead branches. 10711 */ 10712 __mark_reg_unknown(env, dst_reg); 10713 return 0; 10714 } 10715 10716 if (BPF_CLASS(insn->code) != BPF_ALU64) { 10717 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 10718 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 10719 __mark_reg_unknown(env, dst_reg); 10720 return 0; 10721 } 10722 10723 verbose(env, 10724 "R%d 32-bit pointer arithmetic prohibited\n", 10725 dst); 10726 return -EACCES; 10727 } 10728 10729 if (ptr_reg->type & PTR_MAYBE_NULL) { 10730 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 10731 dst, reg_type_str(env, ptr_reg->type)); 10732 return -EACCES; 10733 } 10734 10735 switch (base_type(ptr_reg->type)) { 10736 case CONST_PTR_TO_MAP: 10737 /* smin_val represents the known value */ 10738 if (known && smin_val == 0 && opcode == BPF_ADD) 10739 break; 10740 fallthrough; 10741 case PTR_TO_PACKET_END: 10742 case PTR_TO_SOCKET: 10743 case PTR_TO_SOCK_COMMON: 10744 case PTR_TO_TCP_SOCK: 10745 case PTR_TO_XDP_SOCK: 10746 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 10747 dst, reg_type_str(env, ptr_reg->type)); 10748 return -EACCES; 10749 default: 10750 break; 10751 } 10752 10753 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 10754 * The id may be overwritten later if we create a new variable offset. 10755 */ 10756 dst_reg->type = ptr_reg->type; 10757 dst_reg->id = ptr_reg->id; 10758 10759 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 10760 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 10761 return -EINVAL; 10762 10763 /* pointer types do not carry 32-bit bounds at the moment. */ 10764 __mark_reg32_unbounded(dst_reg); 10765 10766 if (sanitize_needed(opcode)) { 10767 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 10768 &info, false); 10769 if (ret < 0) 10770 return sanitize_err(env, insn, ret, off_reg, dst_reg); 10771 } 10772 10773 switch (opcode) { 10774 case BPF_ADD: 10775 /* We can take a fixed offset as long as it doesn't overflow 10776 * the s32 'off' field 10777 */ 10778 if (known && (ptr_reg->off + smin_val == 10779 (s64)(s32)(ptr_reg->off + smin_val))) { 10780 /* pointer += K. Accumulate it into fixed offset */ 10781 dst_reg->smin_value = smin_ptr; 10782 dst_reg->smax_value = smax_ptr; 10783 dst_reg->umin_value = umin_ptr; 10784 dst_reg->umax_value = umax_ptr; 10785 dst_reg->var_off = ptr_reg->var_off; 10786 dst_reg->off = ptr_reg->off + smin_val; 10787 dst_reg->raw = ptr_reg->raw; 10788 break; 10789 } 10790 /* A new variable offset is created. Note that off_reg->off 10791 * == 0, since it's a scalar. 10792 * dst_reg gets the pointer type and since some positive 10793 * integer value was added to the pointer, give it a new 'id' 10794 * if it's a PTR_TO_PACKET. 10795 * this creates a new 'base' pointer, off_reg (variable) gets 10796 * added into the variable offset, and we copy the fixed offset 10797 * from ptr_reg. 10798 */ 10799 if (signed_add_overflows(smin_ptr, smin_val) || 10800 signed_add_overflows(smax_ptr, smax_val)) { 10801 dst_reg->smin_value = S64_MIN; 10802 dst_reg->smax_value = S64_MAX; 10803 } else { 10804 dst_reg->smin_value = smin_ptr + smin_val; 10805 dst_reg->smax_value = smax_ptr + smax_val; 10806 } 10807 if (umin_ptr + umin_val < umin_ptr || 10808 umax_ptr + umax_val < umax_ptr) { 10809 dst_reg->umin_value = 0; 10810 dst_reg->umax_value = U64_MAX; 10811 } else { 10812 dst_reg->umin_value = umin_ptr + umin_val; 10813 dst_reg->umax_value = umax_ptr + umax_val; 10814 } 10815 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 10816 dst_reg->off = ptr_reg->off; 10817 dst_reg->raw = ptr_reg->raw; 10818 if (reg_is_pkt_pointer(ptr_reg)) { 10819 dst_reg->id = ++env->id_gen; 10820 /* something was added to pkt_ptr, set range to zero */ 10821 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 10822 } 10823 break; 10824 case BPF_SUB: 10825 if (dst_reg == off_reg) { 10826 /* scalar -= pointer. Creates an unknown scalar */ 10827 verbose(env, "R%d tried to subtract pointer from scalar\n", 10828 dst); 10829 return -EACCES; 10830 } 10831 /* We don't allow subtraction from FP, because (according to 10832 * test_verifier.c test "invalid fp arithmetic", JITs might not 10833 * be able to deal with it. 10834 */ 10835 if (ptr_reg->type == PTR_TO_STACK) { 10836 verbose(env, "R%d subtraction from stack pointer prohibited\n", 10837 dst); 10838 return -EACCES; 10839 } 10840 if (known && (ptr_reg->off - smin_val == 10841 (s64)(s32)(ptr_reg->off - smin_val))) { 10842 /* pointer -= K. Subtract it from fixed offset */ 10843 dst_reg->smin_value = smin_ptr; 10844 dst_reg->smax_value = smax_ptr; 10845 dst_reg->umin_value = umin_ptr; 10846 dst_reg->umax_value = umax_ptr; 10847 dst_reg->var_off = ptr_reg->var_off; 10848 dst_reg->id = ptr_reg->id; 10849 dst_reg->off = ptr_reg->off - smin_val; 10850 dst_reg->raw = ptr_reg->raw; 10851 break; 10852 } 10853 /* A new variable offset is created. If the subtrahend is known 10854 * nonnegative, then any reg->range we had before is still good. 10855 */ 10856 if (signed_sub_overflows(smin_ptr, smax_val) || 10857 signed_sub_overflows(smax_ptr, smin_val)) { 10858 /* Overflow possible, we know nothing */ 10859 dst_reg->smin_value = S64_MIN; 10860 dst_reg->smax_value = S64_MAX; 10861 } else { 10862 dst_reg->smin_value = smin_ptr - smax_val; 10863 dst_reg->smax_value = smax_ptr - smin_val; 10864 } 10865 if (umin_ptr < umax_val) { 10866 /* Overflow possible, we know nothing */ 10867 dst_reg->umin_value = 0; 10868 dst_reg->umax_value = U64_MAX; 10869 } else { 10870 /* Cannot overflow (as long as bounds are consistent) */ 10871 dst_reg->umin_value = umin_ptr - umax_val; 10872 dst_reg->umax_value = umax_ptr - umin_val; 10873 } 10874 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 10875 dst_reg->off = ptr_reg->off; 10876 dst_reg->raw = ptr_reg->raw; 10877 if (reg_is_pkt_pointer(ptr_reg)) { 10878 dst_reg->id = ++env->id_gen; 10879 /* something was added to pkt_ptr, set range to zero */ 10880 if (smin_val < 0) 10881 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 10882 } 10883 break; 10884 case BPF_AND: 10885 case BPF_OR: 10886 case BPF_XOR: 10887 /* bitwise ops on pointers are troublesome, prohibit. */ 10888 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 10889 dst, bpf_alu_string[opcode >> 4]); 10890 return -EACCES; 10891 default: 10892 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 10893 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 10894 dst, bpf_alu_string[opcode >> 4]); 10895 return -EACCES; 10896 } 10897 10898 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 10899 return -EINVAL; 10900 reg_bounds_sync(dst_reg); 10901 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 10902 return -EACCES; 10903 if (sanitize_needed(opcode)) { 10904 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 10905 &info, true); 10906 if (ret < 0) 10907 return sanitize_err(env, insn, ret, off_reg, dst_reg); 10908 } 10909 10910 return 0; 10911 } 10912 10913 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 10914 struct bpf_reg_state *src_reg) 10915 { 10916 s32 smin_val = src_reg->s32_min_value; 10917 s32 smax_val = src_reg->s32_max_value; 10918 u32 umin_val = src_reg->u32_min_value; 10919 u32 umax_val = src_reg->u32_max_value; 10920 10921 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 10922 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 10923 dst_reg->s32_min_value = S32_MIN; 10924 dst_reg->s32_max_value = S32_MAX; 10925 } else { 10926 dst_reg->s32_min_value += smin_val; 10927 dst_reg->s32_max_value += smax_val; 10928 } 10929 if (dst_reg->u32_min_value + umin_val < umin_val || 10930 dst_reg->u32_max_value + umax_val < umax_val) { 10931 dst_reg->u32_min_value = 0; 10932 dst_reg->u32_max_value = U32_MAX; 10933 } else { 10934 dst_reg->u32_min_value += umin_val; 10935 dst_reg->u32_max_value += umax_val; 10936 } 10937 } 10938 10939 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 10940 struct bpf_reg_state *src_reg) 10941 { 10942 s64 smin_val = src_reg->smin_value; 10943 s64 smax_val = src_reg->smax_value; 10944 u64 umin_val = src_reg->umin_value; 10945 u64 umax_val = src_reg->umax_value; 10946 10947 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 10948 signed_add_overflows(dst_reg->smax_value, smax_val)) { 10949 dst_reg->smin_value = S64_MIN; 10950 dst_reg->smax_value = S64_MAX; 10951 } else { 10952 dst_reg->smin_value += smin_val; 10953 dst_reg->smax_value += smax_val; 10954 } 10955 if (dst_reg->umin_value + umin_val < umin_val || 10956 dst_reg->umax_value + umax_val < umax_val) { 10957 dst_reg->umin_value = 0; 10958 dst_reg->umax_value = U64_MAX; 10959 } else { 10960 dst_reg->umin_value += umin_val; 10961 dst_reg->umax_value += umax_val; 10962 } 10963 } 10964 10965 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 10966 struct bpf_reg_state *src_reg) 10967 { 10968 s32 smin_val = src_reg->s32_min_value; 10969 s32 smax_val = src_reg->s32_max_value; 10970 u32 umin_val = src_reg->u32_min_value; 10971 u32 umax_val = src_reg->u32_max_value; 10972 10973 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 10974 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 10975 /* Overflow possible, we know nothing */ 10976 dst_reg->s32_min_value = S32_MIN; 10977 dst_reg->s32_max_value = S32_MAX; 10978 } else { 10979 dst_reg->s32_min_value -= smax_val; 10980 dst_reg->s32_max_value -= smin_val; 10981 } 10982 if (dst_reg->u32_min_value < umax_val) { 10983 /* Overflow possible, we know nothing */ 10984 dst_reg->u32_min_value = 0; 10985 dst_reg->u32_max_value = U32_MAX; 10986 } else { 10987 /* Cannot overflow (as long as bounds are consistent) */ 10988 dst_reg->u32_min_value -= umax_val; 10989 dst_reg->u32_max_value -= umin_val; 10990 } 10991 } 10992 10993 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 10994 struct bpf_reg_state *src_reg) 10995 { 10996 s64 smin_val = src_reg->smin_value; 10997 s64 smax_val = src_reg->smax_value; 10998 u64 umin_val = src_reg->umin_value; 10999 u64 umax_val = src_reg->umax_value; 11000 11001 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 11002 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 11003 /* Overflow possible, we know nothing */ 11004 dst_reg->smin_value = S64_MIN; 11005 dst_reg->smax_value = S64_MAX; 11006 } else { 11007 dst_reg->smin_value -= smax_val; 11008 dst_reg->smax_value -= smin_val; 11009 } 11010 if (dst_reg->umin_value < umax_val) { 11011 /* Overflow possible, we know nothing */ 11012 dst_reg->umin_value = 0; 11013 dst_reg->umax_value = U64_MAX; 11014 } else { 11015 /* Cannot overflow (as long as bounds are consistent) */ 11016 dst_reg->umin_value -= umax_val; 11017 dst_reg->umax_value -= umin_val; 11018 } 11019 } 11020 11021 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 11022 struct bpf_reg_state *src_reg) 11023 { 11024 s32 smin_val = src_reg->s32_min_value; 11025 u32 umin_val = src_reg->u32_min_value; 11026 u32 umax_val = src_reg->u32_max_value; 11027 11028 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 11029 /* Ain't nobody got time to multiply that sign */ 11030 __mark_reg32_unbounded(dst_reg); 11031 return; 11032 } 11033 /* Both values are positive, so we can work with unsigned and 11034 * copy the result to signed (unless it exceeds S32_MAX). 11035 */ 11036 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 11037 /* Potential overflow, we know nothing */ 11038 __mark_reg32_unbounded(dst_reg); 11039 return; 11040 } 11041 dst_reg->u32_min_value *= umin_val; 11042 dst_reg->u32_max_value *= umax_val; 11043 if (dst_reg->u32_max_value > S32_MAX) { 11044 /* Overflow possible, we know nothing */ 11045 dst_reg->s32_min_value = S32_MIN; 11046 dst_reg->s32_max_value = S32_MAX; 11047 } else { 11048 dst_reg->s32_min_value = dst_reg->u32_min_value; 11049 dst_reg->s32_max_value = dst_reg->u32_max_value; 11050 } 11051 } 11052 11053 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 11054 struct bpf_reg_state *src_reg) 11055 { 11056 s64 smin_val = src_reg->smin_value; 11057 u64 umin_val = src_reg->umin_value; 11058 u64 umax_val = src_reg->umax_value; 11059 11060 if (smin_val < 0 || dst_reg->smin_value < 0) { 11061 /* Ain't nobody got time to multiply that sign */ 11062 __mark_reg64_unbounded(dst_reg); 11063 return; 11064 } 11065 /* Both values are positive, so we can work with unsigned and 11066 * copy the result to signed (unless it exceeds S64_MAX). 11067 */ 11068 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 11069 /* Potential overflow, we know nothing */ 11070 __mark_reg64_unbounded(dst_reg); 11071 return; 11072 } 11073 dst_reg->umin_value *= umin_val; 11074 dst_reg->umax_value *= umax_val; 11075 if (dst_reg->umax_value > S64_MAX) { 11076 /* Overflow possible, we know nothing */ 11077 dst_reg->smin_value = S64_MIN; 11078 dst_reg->smax_value = S64_MAX; 11079 } else { 11080 dst_reg->smin_value = dst_reg->umin_value; 11081 dst_reg->smax_value = dst_reg->umax_value; 11082 } 11083 } 11084 11085 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 11086 struct bpf_reg_state *src_reg) 11087 { 11088 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11089 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11090 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11091 s32 smin_val = src_reg->s32_min_value; 11092 u32 umax_val = src_reg->u32_max_value; 11093 11094 if (src_known && dst_known) { 11095 __mark_reg32_known(dst_reg, var32_off.value); 11096 return; 11097 } 11098 11099 /* We get our minimum from the var_off, since that's inherently 11100 * bitwise. Our maximum is the minimum of the operands' maxima. 11101 */ 11102 dst_reg->u32_min_value = var32_off.value; 11103 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 11104 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 11105 /* Lose signed bounds when ANDing negative numbers, 11106 * ain't nobody got time for that. 11107 */ 11108 dst_reg->s32_min_value = S32_MIN; 11109 dst_reg->s32_max_value = S32_MAX; 11110 } else { 11111 /* ANDing two positives gives a positive, so safe to 11112 * cast result into s64. 11113 */ 11114 dst_reg->s32_min_value = dst_reg->u32_min_value; 11115 dst_reg->s32_max_value = dst_reg->u32_max_value; 11116 } 11117 } 11118 11119 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 11120 struct bpf_reg_state *src_reg) 11121 { 11122 bool src_known = tnum_is_const(src_reg->var_off); 11123 bool dst_known = tnum_is_const(dst_reg->var_off); 11124 s64 smin_val = src_reg->smin_value; 11125 u64 umax_val = src_reg->umax_value; 11126 11127 if (src_known && dst_known) { 11128 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11129 return; 11130 } 11131 11132 /* We get our minimum from the var_off, since that's inherently 11133 * bitwise. Our maximum is the minimum of the operands' maxima. 11134 */ 11135 dst_reg->umin_value = dst_reg->var_off.value; 11136 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 11137 if (dst_reg->smin_value < 0 || smin_val < 0) { 11138 /* Lose signed bounds when ANDing negative numbers, 11139 * ain't nobody got time for that. 11140 */ 11141 dst_reg->smin_value = S64_MIN; 11142 dst_reg->smax_value = S64_MAX; 11143 } else { 11144 /* ANDing two positives gives a positive, so safe to 11145 * cast result into s64. 11146 */ 11147 dst_reg->smin_value = dst_reg->umin_value; 11148 dst_reg->smax_value = dst_reg->umax_value; 11149 } 11150 /* We may learn something more from the var_off */ 11151 __update_reg_bounds(dst_reg); 11152 } 11153 11154 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 11155 struct bpf_reg_state *src_reg) 11156 { 11157 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11158 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11159 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11160 s32 smin_val = src_reg->s32_min_value; 11161 u32 umin_val = src_reg->u32_min_value; 11162 11163 if (src_known && dst_known) { 11164 __mark_reg32_known(dst_reg, var32_off.value); 11165 return; 11166 } 11167 11168 /* We get our maximum from the var_off, and our minimum is the 11169 * maximum of the operands' minima 11170 */ 11171 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 11172 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 11173 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 11174 /* Lose signed bounds when ORing negative numbers, 11175 * ain't nobody got time for that. 11176 */ 11177 dst_reg->s32_min_value = S32_MIN; 11178 dst_reg->s32_max_value = S32_MAX; 11179 } else { 11180 /* ORing two positives gives a positive, so safe to 11181 * cast result into s64. 11182 */ 11183 dst_reg->s32_min_value = dst_reg->u32_min_value; 11184 dst_reg->s32_max_value = dst_reg->u32_max_value; 11185 } 11186 } 11187 11188 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 11189 struct bpf_reg_state *src_reg) 11190 { 11191 bool src_known = tnum_is_const(src_reg->var_off); 11192 bool dst_known = tnum_is_const(dst_reg->var_off); 11193 s64 smin_val = src_reg->smin_value; 11194 u64 umin_val = src_reg->umin_value; 11195 11196 if (src_known && dst_known) { 11197 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11198 return; 11199 } 11200 11201 /* We get our maximum from the var_off, and our minimum is the 11202 * maximum of the operands' minima 11203 */ 11204 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 11205 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 11206 if (dst_reg->smin_value < 0 || smin_val < 0) { 11207 /* Lose signed bounds when ORing negative numbers, 11208 * ain't nobody got time for that. 11209 */ 11210 dst_reg->smin_value = S64_MIN; 11211 dst_reg->smax_value = S64_MAX; 11212 } else { 11213 /* ORing two positives gives a positive, so safe to 11214 * cast result into s64. 11215 */ 11216 dst_reg->smin_value = dst_reg->umin_value; 11217 dst_reg->smax_value = dst_reg->umax_value; 11218 } 11219 /* We may learn something more from the var_off */ 11220 __update_reg_bounds(dst_reg); 11221 } 11222 11223 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 11224 struct bpf_reg_state *src_reg) 11225 { 11226 bool src_known = tnum_subreg_is_const(src_reg->var_off); 11227 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 11228 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 11229 s32 smin_val = src_reg->s32_min_value; 11230 11231 if (src_known && dst_known) { 11232 __mark_reg32_known(dst_reg, var32_off.value); 11233 return; 11234 } 11235 11236 /* We get both minimum and maximum from the var32_off. */ 11237 dst_reg->u32_min_value = var32_off.value; 11238 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 11239 11240 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 11241 /* XORing two positive sign numbers gives a positive, 11242 * so safe to cast u32 result into s32. 11243 */ 11244 dst_reg->s32_min_value = dst_reg->u32_min_value; 11245 dst_reg->s32_max_value = dst_reg->u32_max_value; 11246 } else { 11247 dst_reg->s32_min_value = S32_MIN; 11248 dst_reg->s32_max_value = S32_MAX; 11249 } 11250 } 11251 11252 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 11253 struct bpf_reg_state *src_reg) 11254 { 11255 bool src_known = tnum_is_const(src_reg->var_off); 11256 bool dst_known = tnum_is_const(dst_reg->var_off); 11257 s64 smin_val = src_reg->smin_value; 11258 11259 if (src_known && dst_known) { 11260 /* dst_reg->var_off.value has been updated earlier */ 11261 __mark_reg_known(dst_reg, dst_reg->var_off.value); 11262 return; 11263 } 11264 11265 /* We get both minimum and maximum from the var_off. */ 11266 dst_reg->umin_value = dst_reg->var_off.value; 11267 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 11268 11269 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 11270 /* XORing two positive sign numbers gives a positive, 11271 * so safe to cast u64 result into s64. 11272 */ 11273 dst_reg->smin_value = dst_reg->umin_value; 11274 dst_reg->smax_value = dst_reg->umax_value; 11275 } else { 11276 dst_reg->smin_value = S64_MIN; 11277 dst_reg->smax_value = S64_MAX; 11278 } 11279 11280 __update_reg_bounds(dst_reg); 11281 } 11282 11283 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 11284 u64 umin_val, u64 umax_val) 11285 { 11286 /* We lose all sign bit information (except what we can pick 11287 * up from var_off) 11288 */ 11289 dst_reg->s32_min_value = S32_MIN; 11290 dst_reg->s32_max_value = S32_MAX; 11291 /* If we might shift our top bit out, then we know nothing */ 11292 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 11293 dst_reg->u32_min_value = 0; 11294 dst_reg->u32_max_value = U32_MAX; 11295 } else { 11296 dst_reg->u32_min_value <<= umin_val; 11297 dst_reg->u32_max_value <<= umax_val; 11298 } 11299 } 11300 11301 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 11302 struct bpf_reg_state *src_reg) 11303 { 11304 u32 umax_val = src_reg->u32_max_value; 11305 u32 umin_val = src_reg->u32_min_value; 11306 /* u32 alu operation will zext upper bits */ 11307 struct tnum subreg = tnum_subreg(dst_reg->var_off); 11308 11309 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 11310 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 11311 /* Not required but being careful mark reg64 bounds as unknown so 11312 * that we are forced to pick them up from tnum and zext later and 11313 * if some path skips this step we are still safe. 11314 */ 11315 __mark_reg64_unbounded(dst_reg); 11316 __update_reg32_bounds(dst_reg); 11317 } 11318 11319 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 11320 u64 umin_val, u64 umax_val) 11321 { 11322 /* Special case <<32 because it is a common compiler pattern to sign 11323 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 11324 * positive we know this shift will also be positive so we can track 11325 * bounds correctly. Otherwise we lose all sign bit information except 11326 * what we can pick up from var_off. Perhaps we can generalize this 11327 * later to shifts of any length. 11328 */ 11329 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 11330 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 11331 else 11332 dst_reg->smax_value = S64_MAX; 11333 11334 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 11335 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 11336 else 11337 dst_reg->smin_value = S64_MIN; 11338 11339 /* If we might shift our top bit out, then we know nothing */ 11340 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 11341 dst_reg->umin_value = 0; 11342 dst_reg->umax_value = U64_MAX; 11343 } else { 11344 dst_reg->umin_value <<= umin_val; 11345 dst_reg->umax_value <<= umax_val; 11346 } 11347 } 11348 11349 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 11350 struct bpf_reg_state *src_reg) 11351 { 11352 u64 umax_val = src_reg->umax_value; 11353 u64 umin_val = src_reg->umin_value; 11354 11355 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 11356 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 11357 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 11358 11359 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 11360 /* We may learn something more from the var_off */ 11361 __update_reg_bounds(dst_reg); 11362 } 11363 11364 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 11365 struct bpf_reg_state *src_reg) 11366 { 11367 struct tnum subreg = tnum_subreg(dst_reg->var_off); 11368 u32 umax_val = src_reg->u32_max_value; 11369 u32 umin_val = src_reg->u32_min_value; 11370 11371 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 11372 * be negative, then either: 11373 * 1) src_reg might be zero, so the sign bit of the result is 11374 * unknown, so we lose our signed bounds 11375 * 2) it's known negative, thus the unsigned bounds capture the 11376 * signed bounds 11377 * 3) the signed bounds cross zero, so they tell us nothing 11378 * about the result 11379 * If the value in dst_reg is known nonnegative, then again the 11380 * unsigned bounds capture the signed bounds. 11381 * Thus, in all cases it suffices to blow away our signed bounds 11382 * and rely on inferring new ones from the unsigned bounds and 11383 * var_off of the result. 11384 */ 11385 dst_reg->s32_min_value = S32_MIN; 11386 dst_reg->s32_max_value = S32_MAX; 11387 11388 dst_reg->var_off = tnum_rshift(subreg, umin_val); 11389 dst_reg->u32_min_value >>= umax_val; 11390 dst_reg->u32_max_value >>= umin_val; 11391 11392 __mark_reg64_unbounded(dst_reg); 11393 __update_reg32_bounds(dst_reg); 11394 } 11395 11396 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 11397 struct bpf_reg_state *src_reg) 11398 { 11399 u64 umax_val = src_reg->umax_value; 11400 u64 umin_val = src_reg->umin_value; 11401 11402 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 11403 * be negative, then either: 11404 * 1) src_reg might be zero, so the sign bit of the result is 11405 * unknown, so we lose our signed bounds 11406 * 2) it's known negative, thus the unsigned bounds capture the 11407 * signed bounds 11408 * 3) the signed bounds cross zero, so they tell us nothing 11409 * about the result 11410 * If the value in dst_reg is known nonnegative, then again the 11411 * unsigned bounds capture the signed bounds. 11412 * Thus, in all cases it suffices to blow away our signed bounds 11413 * and rely on inferring new ones from the unsigned bounds and 11414 * var_off of the result. 11415 */ 11416 dst_reg->smin_value = S64_MIN; 11417 dst_reg->smax_value = S64_MAX; 11418 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 11419 dst_reg->umin_value >>= umax_val; 11420 dst_reg->umax_value >>= umin_val; 11421 11422 /* Its not easy to operate on alu32 bounds here because it depends 11423 * on bits being shifted in. Take easy way out and mark unbounded 11424 * so we can recalculate later from tnum. 11425 */ 11426 __mark_reg32_unbounded(dst_reg); 11427 __update_reg_bounds(dst_reg); 11428 } 11429 11430 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 11431 struct bpf_reg_state *src_reg) 11432 { 11433 u64 umin_val = src_reg->u32_min_value; 11434 11435 /* Upon reaching here, src_known is true and 11436 * umax_val is equal to umin_val. 11437 */ 11438 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 11439 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 11440 11441 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 11442 11443 /* blow away the dst_reg umin_value/umax_value and rely on 11444 * dst_reg var_off to refine the result. 11445 */ 11446 dst_reg->u32_min_value = 0; 11447 dst_reg->u32_max_value = U32_MAX; 11448 11449 __mark_reg64_unbounded(dst_reg); 11450 __update_reg32_bounds(dst_reg); 11451 } 11452 11453 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 11454 struct bpf_reg_state *src_reg) 11455 { 11456 u64 umin_val = src_reg->umin_value; 11457 11458 /* Upon reaching here, src_known is true and umax_val is equal 11459 * to umin_val. 11460 */ 11461 dst_reg->smin_value >>= umin_val; 11462 dst_reg->smax_value >>= umin_val; 11463 11464 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 11465 11466 /* blow away the dst_reg umin_value/umax_value and rely on 11467 * dst_reg var_off to refine the result. 11468 */ 11469 dst_reg->umin_value = 0; 11470 dst_reg->umax_value = U64_MAX; 11471 11472 /* Its not easy to operate on alu32 bounds here because it depends 11473 * on bits being shifted in from upper 32-bits. Take easy way out 11474 * and mark unbounded so we can recalculate later from tnum. 11475 */ 11476 __mark_reg32_unbounded(dst_reg); 11477 __update_reg_bounds(dst_reg); 11478 } 11479 11480 /* WARNING: This function does calculations on 64-bit values, but the actual 11481 * execution may occur on 32-bit values. Therefore, things like bitshifts 11482 * need extra checks in the 32-bit case. 11483 */ 11484 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 11485 struct bpf_insn *insn, 11486 struct bpf_reg_state *dst_reg, 11487 struct bpf_reg_state src_reg) 11488 { 11489 struct bpf_reg_state *regs = cur_regs(env); 11490 u8 opcode = BPF_OP(insn->code); 11491 bool src_known; 11492 s64 smin_val, smax_val; 11493 u64 umin_val, umax_val; 11494 s32 s32_min_val, s32_max_val; 11495 u32 u32_min_val, u32_max_val; 11496 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 11497 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 11498 int ret; 11499 11500 smin_val = src_reg.smin_value; 11501 smax_val = src_reg.smax_value; 11502 umin_val = src_reg.umin_value; 11503 umax_val = src_reg.umax_value; 11504 11505 s32_min_val = src_reg.s32_min_value; 11506 s32_max_val = src_reg.s32_max_value; 11507 u32_min_val = src_reg.u32_min_value; 11508 u32_max_val = src_reg.u32_max_value; 11509 11510 if (alu32) { 11511 src_known = tnum_subreg_is_const(src_reg.var_off); 11512 if ((src_known && 11513 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 11514 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 11515 /* Taint dst register if offset had invalid bounds 11516 * derived from e.g. dead branches. 11517 */ 11518 __mark_reg_unknown(env, dst_reg); 11519 return 0; 11520 } 11521 } else { 11522 src_known = tnum_is_const(src_reg.var_off); 11523 if ((src_known && 11524 (smin_val != smax_val || umin_val != umax_val)) || 11525 smin_val > smax_val || umin_val > umax_val) { 11526 /* Taint dst register if offset had invalid bounds 11527 * derived from e.g. dead branches. 11528 */ 11529 __mark_reg_unknown(env, dst_reg); 11530 return 0; 11531 } 11532 } 11533 11534 if (!src_known && 11535 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 11536 __mark_reg_unknown(env, dst_reg); 11537 return 0; 11538 } 11539 11540 if (sanitize_needed(opcode)) { 11541 ret = sanitize_val_alu(env, insn); 11542 if (ret < 0) 11543 return sanitize_err(env, insn, ret, NULL, NULL); 11544 } 11545 11546 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 11547 * There are two classes of instructions: The first class we track both 11548 * alu32 and alu64 sign/unsigned bounds independently this provides the 11549 * greatest amount of precision when alu operations are mixed with jmp32 11550 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 11551 * and BPF_OR. This is possible because these ops have fairly easy to 11552 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 11553 * See alu32 verifier tests for examples. The second class of 11554 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 11555 * with regards to tracking sign/unsigned bounds because the bits may 11556 * cross subreg boundaries in the alu64 case. When this happens we mark 11557 * the reg unbounded in the subreg bound space and use the resulting 11558 * tnum to calculate an approximation of the sign/unsigned bounds. 11559 */ 11560 switch (opcode) { 11561 case BPF_ADD: 11562 scalar32_min_max_add(dst_reg, &src_reg); 11563 scalar_min_max_add(dst_reg, &src_reg); 11564 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 11565 break; 11566 case BPF_SUB: 11567 scalar32_min_max_sub(dst_reg, &src_reg); 11568 scalar_min_max_sub(dst_reg, &src_reg); 11569 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 11570 break; 11571 case BPF_MUL: 11572 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 11573 scalar32_min_max_mul(dst_reg, &src_reg); 11574 scalar_min_max_mul(dst_reg, &src_reg); 11575 break; 11576 case BPF_AND: 11577 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 11578 scalar32_min_max_and(dst_reg, &src_reg); 11579 scalar_min_max_and(dst_reg, &src_reg); 11580 break; 11581 case BPF_OR: 11582 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 11583 scalar32_min_max_or(dst_reg, &src_reg); 11584 scalar_min_max_or(dst_reg, &src_reg); 11585 break; 11586 case BPF_XOR: 11587 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 11588 scalar32_min_max_xor(dst_reg, &src_reg); 11589 scalar_min_max_xor(dst_reg, &src_reg); 11590 break; 11591 case BPF_LSH: 11592 if (umax_val >= insn_bitness) { 11593 /* Shifts greater than 31 or 63 are undefined. 11594 * This includes shifts by a negative number. 11595 */ 11596 mark_reg_unknown(env, regs, insn->dst_reg); 11597 break; 11598 } 11599 if (alu32) 11600 scalar32_min_max_lsh(dst_reg, &src_reg); 11601 else 11602 scalar_min_max_lsh(dst_reg, &src_reg); 11603 break; 11604 case BPF_RSH: 11605 if (umax_val >= insn_bitness) { 11606 /* Shifts greater than 31 or 63 are undefined. 11607 * This includes shifts by a negative number. 11608 */ 11609 mark_reg_unknown(env, regs, insn->dst_reg); 11610 break; 11611 } 11612 if (alu32) 11613 scalar32_min_max_rsh(dst_reg, &src_reg); 11614 else 11615 scalar_min_max_rsh(dst_reg, &src_reg); 11616 break; 11617 case BPF_ARSH: 11618 if (umax_val >= insn_bitness) { 11619 /* Shifts greater than 31 or 63 are undefined. 11620 * This includes shifts by a negative number. 11621 */ 11622 mark_reg_unknown(env, regs, insn->dst_reg); 11623 break; 11624 } 11625 if (alu32) 11626 scalar32_min_max_arsh(dst_reg, &src_reg); 11627 else 11628 scalar_min_max_arsh(dst_reg, &src_reg); 11629 break; 11630 default: 11631 mark_reg_unknown(env, regs, insn->dst_reg); 11632 break; 11633 } 11634 11635 /* ALU32 ops are zero extended into 64bit register */ 11636 if (alu32) 11637 zext_32_to_64(dst_reg); 11638 reg_bounds_sync(dst_reg); 11639 return 0; 11640 } 11641 11642 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 11643 * and var_off. 11644 */ 11645 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 11646 struct bpf_insn *insn) 11647 { 11648 struct bpf_verifier_state *vstate = env->cur_state; 11649 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 11650 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 11651 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 11652 u8 opcode = BPF_OP(insn->code); 11653 int err; 11654 11655 dst_reg = ®s[insn->dst_reg]; 11656 src_reg = NULL; 11657 if (dst_reg->type != SCALAR_VALUE) 11658 ptr_reg = dst_reg; 11659 else 11660 /* Make sure ID is cleared otherwise dst_reg min/max could be 11661 * incorrectly propagated into other registers by find_equal_scalars() 11662 */ 11663 dst_reg->id = 0; 11664 if (BPF_SRC(insn->code) == BPF_X) { 11665 src_reg = ®s[insn->src_reg]; 11666 if (src_reg->type != SCALAR_VALUE) { 11667 if (dst_reg->type != SCALAR_VALUE) { 11668 /* Combining two pointers by any ALU op yields 11669 * an arbitrary scalar. Disallow all math except 11670 * pointer subtraction 11671 */ 11672 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 11673 mark_reg_unknown(env, regs, insn->dst_reg); 11674 return 0; 11675 } 11676 verbose(env, "R%d pointer %s pointer prohibited\n", 11677 insn->dst_reg, 11678 bpf_alu_string[opcode >> 4]); 11679 return -EACCES; 11680 } else { 11681 /* scalar += pointer 11682 * This is legal, but we have to reverse our 11683 * src/dest handling in computing the range 11684 */ 11685 err = mark_chain_precision(env, insn->dst_reg); 11686 if (err) 11687 return err; 11688 return adjust_ptr_min_max_vals(env, insn, 11689 src_reg, dst_reg); 11690 } 11691 } else if (ptr_reg) { 11692 /* pointer += scalar */ 11693 err = mark_chain_precision(env, insn->src_reg); 11694 if (err) 11695 return err; 11696 return adjust_ptr_min_max_vals(env, insn, 11697 dst_reg, src_reg); 11698 } else if (dst_reg->precise) { 11699 /* if dst_reg is precise, src_reg should be precise as well */ 11700 err = mark_chain_precision(env, insn->src_reg); 11701 if (err) 11702 return err; 11703 } 11704 } else { 11705 /* Pretend the src is a reg with a known value, since we only 11706 * need to be able to read from this state. 11707 */ 11708 off_reg.type = SCALAR_VALUE; 11709 __mark_reg_known(&off_reg, insn->imm); 11710 src_reg = &off_reg; 11711 if (ptr_reg) /* pointer += K */ 11712 return adjust_ptr_min_max_vals(env, insn, 11713 ptr_reg, src_reg); 11714 } 11715 11716 /* Got here implies adding two SCALAR_VALUEs */ 11717 if (WARN_ON_ONCE(ptr_reg)) { 11718 print_verifier_state(env, state, true); 11719 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 11720 return -EINVAL; 11721 } 11722 if (WARN_ON(!src_reg)) { 11723 print_verifier_state(env, state, true); 11724 verbose(env, "verifier internal error: no src_reg\n"); 11725 return -EINVAL; 11726 } 11727 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 11728 } 11729 11730 /* check validity of 32-bit and 64-bit arithmetic operations */ 11731 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 11732 { 11733 struct bpf_reg_state *regs = cur_regs(env); 11734 u8 opcode = BPF_OP(insn->code); 11735 int err; 11736 11737 if (opcode == BPF_END || opcode == BPF_NEG) { 11738 if (opcode == BPF_NEG) { 11739 if (BPF_SRC(insn->code) != BPF_K || 11740 insn->src_reg != BPF_REG_0 || 11741 insn->off != 0 || insn->imm != 0) { 11742 verbose(env, "BPF_NEG uses reserved fields\n"); 11743 return -EINVAL; 11744 } 11745 } else { 11746 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 11747 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 11748 BPF_CLASS(insn->code) == BPF_ALU64) { 11749 verbose(env, "BPF_END uses reserved fields\n"); 11750 return -EINVAL; 11751 } 11752 } 11753 11754 /* check src operand */ 11755 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11756 if (err) 11757 return err; 11758 11759 if (is_pointer_value(env, insn->dst_reg)) { 11760 verbose(env, "R%d pointer arithmetic prohibited\n", 11761 insn->dst_reg); 11762 return -EACCES; 11763 } 11764 11765 /* check dest operand */ 11766 err = check_reg_arg(env, insn->dst_reg, DST_OP); 11767 if (err) 11768 return err; 11769 11770 } else if (opcode == BPF_MOV) { 11771 11772 if (BPF_SRC(insn->code) == BPF_X) { 11773 if (insn->imm != 0 || insn->off != 0) { 11774 verbose(env, "BPF_MOV uses reserved fields\n"); 11775 return -EINVAL; 11776 } 11777 11778 /* check src operand */ 11779 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11780 if (err) 11781 return err; 11782 } else { 11783 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 11784 verbose(env, "BPF_MOV uses reserved fields\n"); 11785 return -EINVAL; 11786 } 11787 } 11788 11789 /* check dest operand, mark as required later */ 11790 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 11791 if (err) 11792 return err; 11793 11794 if (BPF_SRC(insn->code) == BPF_X) { 11795 struct bpf_reg_state *src_reg = regs + insn->src_reg; 11796 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 11797 11798 if (BPF_CLASS(insn->code) == BPF_ALU64) { 11799 /* case: R1 = R2 11800 * copy register state to dest reg 11801 */ 11802 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 11803 /* Assign src and dst registers the same ID 11804 * that will be used by find_equal_scalars() 11805 * to propagate min/max range. 11806 */ 11807 src_reg->id = ++env->id_gen; 11808 copy_register_state(dst_reg, src_reg); 11809 dst_reg->live |= REG_LIVE_WRITTEN; 11810 dst_reg->subreg_def = DEF_NOT_SUBREG; 11811 } else { 11812 /* R1 = (u32) R2 */ 11813 if (is_pointer_value(env, insn->src_reg)) { 11814 verbose(env, 11815 "R%d partial copy of pointer\n", 11816 insn->src_reg); 11817 return -EACCES; 11818 } else if (src_reg->type == SCALAR_VALUE) { 11819 copy_register_state(dst_reg, src_reg); 11820 /* Make sure ID is cleared otherwise 11821 * dst_reg min/max could be incorrectly 11822 * propagated into src_reg by find_equal_scalars() 11823 */ 11824 dst_reg->id = 0; 11825 dst_reg->live |= REG_LIVE_WRITTEN; 11826 dst_reg->subreg_def = env->insn_idx + 1; 11827 } else { 11828 mark_reg_unknown(env, regs, 11829 insn->dst_reg); 11830 } 11831 zext_32_to_64(dst_reg); 11832 reg_bounds_sync(dst_reg); 11833 } 11834 } else { 11835 /* case: R = imm 11836 * remember the value we stored into this reg 11837 */ 11838 /* clear any state __mark_reg_known doesn't set */ 11839 mark_reg_unknown(env, regs, insn->dst_reg); 11840 regs[insn->dst_reg].type = SCALAR_VALUE; 11841 if (BPF_CLASS(insn->code) == BPF_ALU64) { 11842 __mark_reg_known(regs + insn->dst_reg, 11843 insn->imm); 11844 } else { 11845 __mark_reg_known(regs + insn->dst_reg, 11846 (u32)insn->imm); 11847 } 11848 } 11849 11850 } else if (opcode > BPF_END) { 11851 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 11852 return -EINVAL; 11853 11854 } else { /* all other ALU ops: and, sub, xor, add, ... */ 11855 11856 if (BPF_SRC(insn->code) == BPF_X) { 11857 if (insn->imm != 0 || insn->off != 0) { 11858 verbose(env, "BPF_ALU uses reserved fields\n"); 11859 return -EINVAL; 11860 } 11861 /* check src1 operand */ 11862 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11863 if (err) 11864 return err; 11865 } else { 11866 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 11867 verbose(env, "BPF_ALU uses reserved fields\n"); 11868 return -EINVAL; 11869 } 11870 } 11871 11872 /* check src2 operand */ 11873 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11874 if (err) 11875 return err; 11876 11877 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 11878 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 11879 verbose(env, "div by zero\n"); 11880 return -EINVAL; 11881 } 11882 11883 if ((opcode == BPF_LSH || opcode == BPF_RSH || 11884 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 11885 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 11886 11887 if (insn->imm < 0 || insn->imm >= size) { 11888 verbose(env, "invalid shift %d\n", insn->imm); 11889 return -EINVAL; 11890 } 11891 } 11892 11893 /* check dest operand */ 11894 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 11895 if (err) 11896 return err; 11897 11898 return adjust_reg_min_max_vals(env, insn); 11899 } 11900 11901 return 0; 11902 } 11903 11904 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 11905 struct bpf_reg_state *dst_reg, 11906 enum bpf_reg_type type, 11907 bool range_right_open) 11908 { 11909 struct bpf_func_state *state; 11910 struct bpf_reg_state *reg; 11911 int new_range; 11912 11913 if (dst_reg->off < 0 || 11914 (dst_reg->off == 0 && range_right_open)) 11915 /* This doesn't give us any range */ 11916 return; 11917 11918 if (dst_reg->umax_value > MAX_PACKET_OFF || 11919 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 11920 /* Risk of overflow. For instance, ptr + (1<<63) may be less 11921 * than pkt_end, but that's because it's also less than pkt. 11922 */ 11923 return; 11924 11925 new_range = dst_reg->off; 11926 if (range_right_open) 11927 new_range++; 11928 11929 /* Examples for register markings: 11930 * 11931 * pkt_data in dst register: 11932 * 11933 * r2 = r3; 11934 * r2 += 8; 11935 * if (r2 > pkt_end) goto <handle exception> 11936 * <access okay> 11937 * 11938 * r2 = r3; 11939 * r2 += 8; 11940 * if (r2 < pkt_end) goto <access okay> 11941 * <handle exception> 11942 * 11943 * Where: 11944 * r2 == dst_reg, pkt_end == src_reg 11945 * r2=pkt(id=n,off=8,r=0) 11946 * r3=pkt(id=n,off=0,r=0) 11947 * 11948 * pkt_data in src register: 11949 * 11950 * r2 = r3; 11951 * r2 += 8; 11952 * if (pkt_end >= r2) goto <access okay> 11953 * <handle exception> 11954 * 11955 * r2 = r3; 11956 * r2 += 8; 11957 * if (pkt_end <= r2) goto <handle exception> 11958 * <access okay> 11959 * 11960 * Where: 11961 * pkt_end == dst_reg, r2 == src_reg 11962 * r2=pkt(id=n,off=8,r=0) 11963 * r3=pkt(id=n,off=0,r=0) 11964 * 11965 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 11966 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 11967 * and [r3, r3 + 8-1) respectively is safe to access depending on 11968 * the check. 11969 */ 11970 11971 /* If our ids match, then we must have the same max_value. And we 11972 * don't care about the other reg's fixed offset, since if it's too big 11973 * the range won't allow anything. 11974 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 11975 */ 11976 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 11977 if (reg->type == type && reg->id == dst_reg->id) 11978 /* keep the maximum range already checked */ 11979 reg->range = max(reg->range, new_range); 11980 })); 11981 } 11982 11983 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 11984 { 11985 struct tnum subreg = tnum_subreg(reg->var_off); 11986 s32 sval = (s32)val; 11987 11988 switch (opcode) { 11989 case BPF_JEQ: 11990 if (tnum_is_const(subreg)) 11991 return !!tnum_equals_const(subreg, val); 11992 break; 11993 case BPF_JNE: 11994 if (tnum_is_const(subreg)) 11995 return !tnum_equals_const(subreg, val); 11996 break; 11997 case BPF_JSET: 11998 if ((~subreg.mask & subreg.value) & val) 11999 return 1; 12000 if (!((subreg.mask | subreg.value) & val)) 12001 return 0; 12002 break; 12003 case BPF_JGT: 12004 if (reg->u32_min_value > val) 12005 return 1; 12006 else if (reg->u32_max_value <= val) 12007 return 0; 12008 break; 12009 case BPF_JSGT: 12010 if (reg->s32_min_value > sval) 12011 return 1; 12012 else if (reg->s32_max_value <= sval) 12013 return 0; 12014 break; 12015 case BPF_JLT: 12016 if (reg->u32_max_value < val) 12017 return 1; 12018 else if (reg->u32_min_value >= val) 12019 return 0; 12020 break; 12021 case BPF_JSLT: 12022 if (reg->s32_max_value < sval) 12023 return 1; 12024 else if (reg->s32_min_value >= sval) 12025 return 0; 12026 break; 12027 case BPF_JGE: 12028 if (reg->u32_min_value >= val) 12029 return 1; 12030 else if (reg->u32_max_value < val) 12031 return 0; 12032 break; 12033 case BPF_JSGE: 12034 if (reg->s32_min_value >= sval) 12035 return 1; 12036 else if (reg->s32_max_value < sval) 12037 return 0; 12038 break; 12039 case BPF_JLE: 12040 if (reg->u32_max_value <= val) 12041 return 1; 12042 else if (reg->u32_min_value > val) 12043 return 0; 12044 break; 12045 case BPF_JSLE: 12046 if (reg->s32_max_value <= sval) 12047 return 1; 12048 else if (reg->s32_min_value > sval) 12049 return 0; 12050 break; 12051 } 12052 12053 return -1; 12054 } 12055 12056 12057 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 12058 { 12059 s64 sval = (s64)val; 12060 12061 switch (opcode) { 12062 case BPF_JEQ: 12063 if (tnum_is_const(reg->var_off)) 12064 return !!tnum_equals_const(reg->var_off, val); 12065 break; 12066 case BPF_JNE: 12067 if (tnum_is_const(reg->var_off)) 12068 return !tnum_equals_const(reg->var_off, val); 12069 break; 12070 case BPF_JSET: 12071 if ((~reg->var_off.mask & reg->var_off.value) & val) 12072 return 1; 12073 if (!((reg->var_off.mask | reg->var_off.value) & val)) 12074 return 0; 12075 break; 12076 case BPF_JGT: 12077 if (reg->umin_value > val) 12078 return 1; 12079 else if (reg->umax_value <= val) 12080 return 0; 12081 break; 12082 case BPF_JSGT: 12083 if (reg->smin_value > sval) 12084 return 1; 12085 else if (reg->smax_value <= sval) 12086 return 0; 12087 break; 12088 case BPF_JLT: 12089 if (reg->umax_value < val) 12090 return 1; 12091 else if (reg->umin_value >= val) 12092 return 0; 12093 break; 12094 case BPF_JSLT: 12095 if (reg->smax_value < sval) 12096 return 1; 12097 else if (reg->smin_value >= sval) 12098 return 0; 12099 break; 12100 case BPF_JGE: 12101 if (reg->umin_value >= val) 12102 return 1; 12103 else if (reg->umax_value < val) 12104 return 0; 12105 break; 12106 case BPF_JSGE: 12107 if (reg->smin_value >= sval) 12108 return 1; 12109 else if (reg->smax_value < sval) 12110 return 0; 12111 break; 12112 case BPF_JLE: 12113 if (reg->umax_value <= val) 12114 return 1; 12115 else if (reg->umin_value > val) 12116 return 0; 12117 break; 12118 case BPF_JSLE: 12119 if (reg->smax_value <= sval) 12120 return 1; 12121 else if (reg->smin_value > sval) 12122 return 0; 12123 break; 12124 } 12125 12126 return -1; 12127 } 12128 12129 /* compute branch direction of the expression "if (reg opcode val) goto target;" 12130 * and return: 12131 * 1 - branch will be taken and "goto target" will be executed 12132 * 0 - branch will not be taken and fall-through to next insn 12133 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 12134 * range [0,10] 12135 */ 12136 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 12137 bool is_jmp32) 12138 { 12139 if (__is_pointer_value(false, reg)) { 12140 if (!reg_type_not_null(reg->type)) 12141 return -1; 12142 12143 /* If pointer is valid tests against zero will fail so we can 12144 * use this to direct branch taken. 12145 */ 12146 if (val != 0) 12147 return -1; 12148 12149 switch (opcode) { 12150 case BPF_JEQ: 12151 return 0; 12152 case BPF_JNE: 12153 return 1; 12154 default: 12155 return -1; 12156 } 12157 } 12158 12159 if (is_jmp32) 12160 return is_branch32_taken(reg, val, opcode); 12161 return is_branch64_taken(reg, val, opcode); 12162 } 12163 12164 static int flip_opcode(u32 opcode) 12165 { 12166 /* How can we transform "a <op> b" into "b <op> a"? */ 12167 static const u8 opcode_flip[16] = { 12168 /* these stay the same */ 12169 [BPF_JEQ >> 4] = BPF_JEQ, 12170 [BPF_JNE >> 4] = BPF_JNE, 12171 [BPF_JSET >> 4] = BPF_JSET, 12172 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 12173 [BPF_JGE >> 4] = BPF_JLE, 12174 [BPF_JGT >> 4] = BPF_JLT, 12175 [BPF_JLE >> 4] = BPF_JGE, 12176 [BPF_JLT >> 4] = BPF_JGT, 12177 [BPF_JSGE >> 4] = BPF_JSLE, 12178 [BPF_JSGT >> 4] = BPF_JSLT, 12179 [BPF_JSLE >> 4] = BPF_JSGE, 12180 [BPF_JSLT >> 4] = BPF_JSGT 12181 }; 12182 return opcode_flip[opcode >> 4]; 12183 } 12184 12185 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 12186 struct bpf_reg_state *src_reg, 12187 u8 opcode) 12188 { 12189 struct bpf_reg_state *pkt; 12190 12191 if (src_reg->type == PTR_TO_PACKET_END) { 12192 pkt = dst_reg; 12193 } else if (dst_reg->type == PTR_TO_PACKET_END) { 12194 pkt = src_reg; 12195 opcode = flip_opcode(opcode); 12196 } else { 12197 return -1; 12198 } 12199 12200 if (pkt->range >= 0) 12201 return -1; 12202 12203 switch (opcode) { 12204 case BPF_JLE: 12205 /* pkt <= pkt_end */ 12206 fallthrough; 12207 case BPF_JGT: 12208 /* pkt > pkt_end */ 12209 if (pkt->range == BEYOND_PKT_END) 12210 /* pkt has at last one extra byte beyond pkt_end */ 12211 return opcode == BPF_JGT; 12212 break; 12213 case BPF_JLT: 12214 /* pkt < pkt_end */ 12215 fallthrough; 12216 case BPF_JGE: 12217 /* pkt >= pkt_end */ 12218 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 12219 return opcode == BPF_JGE; 12220 break; 12221 } 12222 return -1; 12223 } 12224 12225 /* Adjusts the register min/max values in the case that the dst_reg is the 12226 * variable register that we are working on, and src_reg is a constant or we're 12227 * simply doing a BPF_K check. 12228 * In JEQ/JNE cases we also adjust the var_off values. 12229 */ 12230 static void reg_set_min_max(struct bpf_reg_state *true_reg, 12231 struct bpf_reg_state *false_reg, 12232 u64 val, u32 val32, 12233 u8 opcode, bool is_jmp32) 12234 { 12235 struct tnum false_32off = tnum_subreg(false_reg->var_off); 12236 struct tnum false_64off = false_reg->var_off; 12237 struct tnum true_32off = tnum_subreg(true_reg->var_off); 12238 struct tnum true_64off = true_reg->var_off; 12239 s64 sval = (s64)val; 12240 s32 sval32 = (s32)val32; 12241 12242 /* If the dst_reg is a pointer, we can't learn anything about its 12243 * variable offset from the compare (unless src_reg were a pointer into 12244 * the same object, but we don't bother with that. 12245 * Since false_reg and true_reg have the same type by construction, we 12246 * only need to check one of them for pointerness. 12247 */ 12248 if (__is_pointer_value(false, false_reg)) 12249 return; 12250 12251 switch (opcode) { 12252 /* JEQ/JNE comparison doesn't change the register equivalence. 12253 * 12254 * r1 = r2; 12255 * if (r1 == 42) goto label; 12256 * ... 12257 * label: // here both r1 and r2 are known to be 42. 12258 * 12259 * Hence when marking register as known preserve it's ID. 12260 */ 12261 case BPF_JEQ: 12262 if (is_jmp32) { 12263 __mark_reg32_known(true_reg, val32); 12264 true_32off = tnum_subreg(true_reg->var_off); 12265 } else { 12266 ___mark_reg_known(true_reg, val); 12267 true_64off = true_reg->var_off; 12268 } 12269 break; 12270 case BPF_JNE: 12271 if (is_jmp32) { 12272 __mark_reg32_known(false_reg, val32); 12273 false_32off = tnum_subreg(false_reg->var_off); 12274 } else { 12275 ___mark_reg_known(false_reg, val); 12276 false_64off = false_reg->var_off; 12277 } 12278 break; 12279 case BPF_JSET: 12280 if (is_jmp32) { 12281 false_32off = tnum_and(false_32off, tnum_const(~val32)); 12282 if (is_power_of_2(val32)) 12283 true_32off = tnum_or(true_32off, 12284 tnum_const(val32)); 12285 } else { 12286 false_64off = tnum_and(false_64off, tnum_const(~val)); 12287 if (is_power_of_2(val)) 12288 true_64off = tnum_or(true_64off, 12289 tnum_const(val)); 12290 } 12291 break; 12292 case BPF_JGE: 12293 case BPF_JGT: 12294 { 12295 if (is_jmp32) { 12296 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 12297 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 12298 12299 false_reg->u32_max_value = min(false_reg->u32_max_value, 12300 false_umax); 12301 true_reg->u32_min_value = max(true_reg->u32_min_value, 12302 true_umin); 12303 } else { 12304 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 12305 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 12306 12307 false_reg->umax_value = min(false_reg->umax_value, false_umax); 12308 true_reg->umin_value = max(true_reg->umin_value, true_umin); 12309 } 12310 break; 12311 } 12312 case BPF_JSGE: 12313 case BPF_JSGT: 12314 { 12315 if (is_jmp32) { 12316 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 12317 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 12318 12319 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 12320 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 12321 } else { 12322 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 12323 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 12324 12325 false_reg->smax_value = min(false_reg->smax_value, false_smax); 12326 true_reg->smin_value = max(true_reg->smin_value, true_smin); 12327 } 12328 break; 12329 } 12330 case BPF_JLE: 12331 case BPF_JLT: 12332 { 12333 if (is_jmp32) { 12334 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 12335 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 12336 12337 false_reg->u32_min_value = max(false_reg->u32_min_value, 12338 false_umin); 12339 true_reg->u32_max_value = min(true_reg->u32_max_value, 12340 true_umax); 12341 } else { 12342 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 12343 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 12344 12345 false_reg->umin_value = max(false_reg->umin_value, false_umin); 12346 true_reg->umax_value = min(true_reg->umax_value, true_umax); 12347 } 12348 break; 12349 } 12350 case BPF_JSLE: 12351 case BPF_JSLT: 12352 { 12353 if (is_jmp32) { 12354 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 12355 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 12356 12357 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 12358 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 12359 } else { 12360 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 12361 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 12362 12363 false_reg->smin_value = max(false_reg->smin_value, false_smin); 12364 true_reg->smax_value = min(true_reg->smax_value, true_smax); 12365 } 12366 break; 12367 } 12368 default: 12369 return; 12370 } 12371 12372 if (is_jmp32) { 12373 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 12374 tnum_subreg(false_32off)); 12375 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 12376 tnum_subreg(true_32off)); 12377 __reg_combine_32_into_64(false_reg); 12378 __reg_combine_32_into_64(true_reg); 12379 } else { 12380 false_reg->var_off = false_64off; 12381 true_reg->var_off = true_64off; 12382 __reg_combine_64_into_32(false_reg); 12383 __reg_combine_64_into_32(true_reg); 12384 } 12385 } 12386 12387 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 12388 * the variable reg. 12389 */ 12390 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 12391 struct bpf_reg_state *false_reg, 12392 u64 val, u32 val32, 12393 u8 opcode, bool is_jmp32) 12394 { 12395 opcode = flip_opcode(opcode); 12396 /* This uses zero as "not present in table"; luckily the zero opcode, 12397 * BPF_JA, can't get here. 12398 */ 12399 if (opcode) 12400 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 12401 } 12402 12403 /* Regs are known to be equal, so intersect their min/max/var_off */ 12404 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 12405 struct bpf_reg_state *dst_reg) 12406 { 12407 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 12408 dst_reg->umin_value); 12409 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 12410 dst_reg->umax_value); 12411 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 12412 dst_reg->smin_value); 12413 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 12414 dst_reg->smax_value); 12415 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 12416 dst_reg->var_off); 12417 reg_bounds_sync(src_reg); 12418 reg_bounds_sync(dst_reg); 12419 } 12420 12421 static void reg_combine_min_max(struct bpf_reg_state *true_src, 12422 struct bpf_reg_state *true_dst, 12423 struct bpf_reg_state *false_src, 12424 struct bpf_reg_state *false_dst, 12425 u8 opcode) 12426 { 12427 switch (opcode) { 12428 case BPF_JEQ: 12429 __reg_combine_min_max(true_src, true_dst); 12430 break; 12431 case BPF_JNE: 12432 __reg_combine_min_max(false_src, false_dst); 12433 break; 12434 } 12435 } 12436 12437 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 12438 struct bpf_reg_state *reg, u32 id, 12439 bool is_null) 12440 { 12441 if (type_may_be_null(reg->type) && reg->id == id && 12442 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 12443 /* Old offset (both fixed and variable parts) should have been 12444 * known-zero, because we don't allow pointer arithmetic on 12445 * pointers that might be NULL. If we see this happening, don't 12446 * convert the register. 12447 * 12448 * But in some cases, some helpers that return local kptrs 12449 * advance offset for the returned pointer. In those cases, it 12450 * is fine to expect to see reg->off. 12451 */ 12452 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) 12453 return; 12454 if (!(type_is_ptr_alloc_obj(reg->type) || type_is_non_owning_ref(reg->type)) && 12455 WARN_ON_ONCE(reg->off)) 12456 return; 12457 12458 if (is_null) { 12459 reg->type = SCALAR_VALUE; 12460 /* We don't need id and ref_obj_id from this point 12461 * onwards anymore, thus we should better reset it, 12462 * so that state pruning has chances to take effect. 12463 */ 12464 reg->id = 0; 12465 reg->ref_obj_id = 0; 12466 12467 return; 12468 } 12469 12470 mark_ptr_not_null_reg(reg); 12471 12472 if (!reg_may_point_to_spin_lock(reg)) { 12473 /* For not-NULL ptr, reg->ref_obj_id will be reset 12474 * in release_reference(). 12475 * 12476 * reg->id is still used by spin_lock ptr. Other 12477 * than spin_lock ptr type, reg->id can be reset. 12478 */ 12479 reg->id = 0; 12480 } 12481 } 12482 } 12483 12484 /* The logic is similar to find_good_pkt_pointers(), both could eventually 12485 * be folded together at some point. 12486 */ 12487 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 12488 bool is_null) 12489 { 12490 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 12491 struct bpf_reg_state *regs = state->regs, *reg; 12492 u32 ref_obj_id = regs[regno].ref_obj_id; 12493 u32 id = regs[regno].id; 12494 12495 if (ref_obj_id && ref_obj_id == id && is_null) 12496 /* regs[regno] is in the " == NULL" branch. 12497 * No one could have freed the reference state before 12498 * doing the NULL check. 12499 */ 12500 WARN_ON_ONCE(release_reference_state(state, id)); 12501 12502 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 12503 mark_ptr_or_null_reg(state, reg, id, is_null); 12504 })); 12505 } 12506 12507 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 12508 struct bpf_reg_state *dst_reg, 12509 struct bpf_reg_state *src_reg, 12510 struct bpf_verifier_state *this_branch, 12511 struct bpf_verifier_state *other_branch) 12512 { 12513 if (BPF_SRC(insn->code) != BPF_X) 12514 return false; 12515 12516 /* Pointers are always 64-bit. */ 12517 if (BPF_CLASS(insn->code) == BPF_JMP32) 12518 return false; 12519 12520 switch (BPF_OP(insn->code)) { 12521 case BPF_JGT: 12522 if ((dst_reg->type == PTR_TO_PACKET && 12523 src_reg->type == PTR_TO_PACKET_END) || 12524 (dst_reg->type == PTR_TO_PACKET_META && 12525 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 12526 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 12527 find_good_pkt_pointers(this_branch, dst_reg, 12528 dst_reg->type, false); 12529 mark_pkt_end(other_branch, insn->dst_reg, true); 12530 } else if ((dst_reg->type == PTR_TO_PACKET_END && 12531 src_reg->type == PTR_TO_PACKET) || 12532 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 12533 src_reg->type == PTR_TO_PACKET_META)) { 12534 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 12535 find_good_pkt_pointers(other_branch, src_reg, 12536 src_reg->type, true); 12537 mark_pkt_end(this_branch, insn->src_reg, false); 12538 } else { 12539 return false; 12540 } 12541 break; 12542 case BPF_JLT: 12543 if ((dst_reg->type == PTR_TO_PACKET && 12544 src_reg->type == PTR_TO_PACKET_END) || 12545 (dst_reg->type == PTR_TO_PACKET_META && 12546 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 12547 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 12548 find_good_pkt_pointers(other_branch, dst_reg, 12549 dst_reg->type, true); 12550 mark_pkt_end(this_branch, insn->dst_reg, false); 12551 } else if ((dst_reg->type == PTR_TO_PACKET_END && 12552 src_reg->type == PTR_TO_PACKET) || 12553 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 12554 src_reg->type == PTR_TO_PACKET_META)) { 12555 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 12556 find_good_pkt_pointers(this_branch, src_reg, 12557 src_reg->type, false); 12558 mark_pkt_end(other_branch, insn->src_reg, true); 12559 } else { 12560 return false; 12561 } 12562 break; 12563 case BPF_JGE: 12564 if ((dst_reg->type == PTR_TO_PACKET && 12565 src_reg->type == PTR_TO_PACKET_END) || 12566 (dst_reg->type == PTR_TO_PACKET_META && 12567 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 12568 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 12569 find_good_pkt_pointers(this_branch, dst_reg, 12570 dst_reg->type, true); 12571 mark_pkt_end(other_branch, insn->dst_reg, false); 12572 } else if ((dst_reg->type == PTR_TO_PACKET_END && 12573 src_reg->type == PTR_TO_PACKET) || 12574 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 12575 src_reg->type == PTR_TO_PACKET_META)) { 12576 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 12577 find_good_pkt_pointers(other_branch, src_reg, 12578 src_reg->type, false); 12579 mark_pkt_end(this_branch, insn->src_reg, true); 12580 } else { 12581 return false; 12582 } 12583 break; 12584 case BPF_JLE: 12585 if ((dst_reg->type == PTR_TO_PACKET && 12586 src_reg->type == PTR_TO_PACKET_END) || 12587 (dst_reg->type == PTR_TO_PACKET_META && 12588 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 12589 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 12590 find_good_pkt_pointers(other_branch, dst_reg, 12591 dst_reg->type, false); 12592 mark_pkt_end(this_branch, insn->dst_reg, true); 12593 } else if ((dst_reg->type == PTR_TO_PACKET_END && 12594 src_reg->type == PTR_TO_PACKET) || 12595 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 12596 src_reg->type == PTR_TO_PACKET_META)) { 12597 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 12598 find_good_pkt_pointers(this_branch, src_reg, 12599 src_reg->type, true); 12600 mark_pkt_end(other_branch, insn->src_reg, false); 12601 } else { 12602 return false; 12603 } 12604 break; 12605 default: 12606 return false; 12607 } 12608 12609 return true; 12610 } 12611 12612 static void find_equal_scalars(struct bpf_verifier_state *vstate, 12613 struct bpf_reg_state *known_reg) 12614 { 12615 struct bpf_func_state *state; 12616 struct bpf_reg_state *reg; 12617 12618 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 12619 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 12620 copy_register_state(reg, known_reg); 12621 })); 12622 } 12623 12624 static int check_cond_jmp_op(struct bpf_verifier_env *env, 12625 struct bpf_insn *insn, int *insn_idx) 12626 { 12627 struct bpf_verifier_state *this_branch = env->cur_state; 12628 struct bpf_verifier_state *other_branch; 12629 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 12630 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 12631 struct bpf_reg_state *eq_branch_regs; 12632 u8 opcode = BPF_OP(insn->code); 12633 bool is_jmp32; 12634 int pred = -1; 12635 int err; 12636 12637 /* Only conditional jumps are expected to reach here. */ 12638 if (opcode == BPF_JA || opcode > BPF_JSLE) { 12639 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 12640 return -EINVAL; 12641 } 12642 12643 if (BPF_SRC(insn->code) == BPF_X) { 12644 if (insn->imm != 0) { 12645 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 12646 return -EINVAL; 12647 } 12648 12649 /* check src1 operand */ 12650 err = check_reg_arg(env, insn->src_reg, SRC_OP); 12651 if (err) 12652 return err; 12653 12654 if (is_pointer_value(env, insn->src_reg)) { 12655 verbose(env, "R%d pointer comparison prohibited\n", 12656 insn->src_reg); 12657 return -EACCES; 12658 } 12659 src_reg = ®s[insn->src_reg]; 12660 } else { 12661 if (insn->src_reg != BPF_REG_0) { 12662 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 12663 return -EINVAL; 12664 } 12665 } 12666 12667 /* check src2 operand */ 12668 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 12669 if (err) 12670 return err; 12671 12672 dst_reg = ®s[insn->dst_reg]; 12673 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 12674 12675 if (BPF_SRC(insn->code) == BPF_K) { 12676 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 12677 } else if (src_reg->type == SCALAR_VALUE && 12678 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 12679 pred = is_branch_taken(dst_reg, 12680 tnum_subreg(src_reg->var_off).value, 12681 opcode, 12682 is_jmp32); 12683 } else if (src_reg->type == SCALAR_VALUE && 12684 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 12685 pred = is_branch_taken(dst_reg, 12686 src_reg->var_off.value, 12687 opcode, 12688 is_jmp32); 12689 } else if (reg_is_pkt_pointer_any(dst_reg) && 12690 reg_is_pkt_pointer_any(src_reg) && 12691 !is_jmp32) { 12692 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 12693 } 12694 12695 if (pred >= 0) { 12696 /* If we get here with a dst_reg pointer type it is because 12697 * above is_branch_taken() special cased the 0 comparison. 12698 */ 12699 if (!__is_pointer_value(false, dst_reg)) 12700 err = mark_chain_precision(env, insn->dst_reg); 12701 if (BPF_SRC(insn->code) == BPF_X && !err && 12702 !__is_pointer_value(false, src_reg)) 12703 err = mark_chain_precision(env, insn->src_reg); 12704 if (err) 12705 return err; 12706 } 12707 12708 if (pred == 1) { 12709 /* Only follow the goto, ignore fall-through. If needed, push 12710 * the fall-through branch for simulation under speculative 12711 * execution. 12712 */ 12713 if (!env->bypass_spec_v1 && 12714 !sanitize_speculative_path(env, insn, *insn_idx + 1, 12715 *insn_idx)) 12716 return -EFAULT; 12717 *insn_idx += insn->off; 12718 return 0; 12719 } else if (pred == 0) { 12720 /* Only follow the fall-through branch, since that's where the 12721 * program will go. If needed, push the goto branch for 12722 * simulation under speculative execution. 12723 */ 12724 if (!env->bypass_spec_v1 && 12725 !sanitize_speculative_path(env, insn, 12726 *insn_idx + insn->off + 1, 12727 *insn_idx)) 12728 return -EFAULT; 12729 return 0; 12730 } 12731 12732 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 12733 false); 12734 if (!other_branch) 12735 return -EFAULT; 12736 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 12737 12738 /* detect if we are comparing against a constant value so we can adjust 12739 * our min/max values for our dst register. 12740 * this is only legit if both are scalars (or pointers to the same 12741 * object, I suppose, see the PTR_MAYBE_NULL related if block below), 12742 * because otherwise the different base pointers mean the offsets aren't 12743 * comparable. 12744 */ 12745 if (BPF_SRC(insn->code) == BPF_X) { 12746 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 12747 12748 if (dst_reg->type == SCALAR_VALUE && 12749 src_reg->type == SCALAR_VALUE) { 12750 if (tnum_is_const(src_reg->var_off) || 12751 (is_jmp32 && 12752 tnum_is_const(tnum_subreg(src_reg->var_off)))) 12753 reg_set_min_max(&other_branch_regs[insn->dst_reg], 12754 dst_reg, 12755 src_reg->var_off.value, 12756 tnum_subreg(src_reg->var_off).value, 12757 opcode, is_jmp32); 12758 else if (tnum_is_const(dst_reg->var_off) || 12759 (is_jmp32 && 12760 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 12761 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 12762 src_reg, 12763 dst_reg->var_off.value, 12764 tnum_subreg(dst_reg->var_off).value, 12765 opcode, is_jmp32); 12766 else if (!is_jmp32 && 12767 (opcode == BPF_JEQ || opcode == BPF_JNE)) 12768 /* Comparing for equality, we can combine knowledge */ 12769 reg_combine_min_max(&other_branch_regs[insn->src_reg], 12770 &other_branch_regs[insn->dst_reg], 12771 src_reg, dst_reg, opcode); 12772 if (src_reg->id && 12773 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 12774 find_equal_scalars(this_branch, src_reg); 12775 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 12776 } 12777 12778 } 12779 } else if (dst_reg->type == SCALAR_VALUE) { 12780 reg_set_min_max(&other_branch_regs[insn->dst_reg], 12781 dst_reg, insn->imm, (u32)insn->imm, 12782 opcode, is_jmp32); 12783 } 12784 12785 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 12786 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 12787 find_equal_scalars(this_branch, dst_reg); 12788 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 12789 } 12790 12791 /* if one pointer register is compared to another pointer 12792 * register check if PTR_MAYBE_NULL could be lifted. 12793 * E.g. register A - maybe null 12794 * register B - not null 12795 * for JNE A, B, ... - A is not null in the false branch; 12796 * for JEQ A, B, ... - A is not null in the true branch. 12797 * 12798 * Since PTR_TO_BTF_ID points to a kernel struct that does 12799 * not need to be null checked by the BPF program, i.e., 12800 * could be null even without PTR_MAYBE_NULL marking, so 12801 * only propagate nullness when neither reg is that type. 12802 */ 12803 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 12804 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 12805 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && 12806 base_type(src_reg->type) != PTR_TO_BTF_ID && 12807 base_type(dst_reg->type) != PTR_TO_BTF_ID) { 12808 eq_branch_regs = NULL; 12809 switch (opcode) { 12810 case BPF_JEQ: 12811 eq_branch_regs = other_branch_regs; 12812 break; 12813 case BPF_JNE: 12814 eq_branch_regs = regs; 12815 break; 12816 default: 12817 /* do nothing */ 12818 break; 12819 } 12820 if (eq_branch_regs) { 12821 if (type_may_be_null(src_reg->type)) 12822 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 12823 else 12824 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 12825 } 12826 } 12827 12828 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 12829 * NOTE: these optimizations below are related with pointer comparison 12830 * which will never be JMP32. 12831 */ 12832 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 12833 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 12834 type_may_be_null(dst_reg->type)) { 12835 /* Mark all identical registers in each branch as either 12836 * safe or unknown depending R == 0 or R != 0 conditional. 12837 */ 12838 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 12839 opcode == BPF_JNE); 12840 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 12841 opcode == BPF_JEQ); 12842 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 12843 this_branch, other_branch) && 12844 is_pointer_value(env, insn->dst_reg)) { 12845 verbose(env, "R%d pointer comparison prohibited\n", 12846 insn->dst_reg); 12847 return -EACCES; 12848 } 12849 if (env->log.level & BPF_LOG_LEVEL) 12850 print_insn_state(env, this_branch->frame[this_branch->curframe]); 12851 return 0; 12852 } 12853 12854 /* verify BPF_LD_IMM64 instruction */ 12855 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 12856 { 12857 struct bpf_insn_aux_data *aux = cur_aux(env); 12858 struct bpf_reg_state *regs = cur_regs(env); 12859 struct bpf_reg_state *dst_reg; 12860 struct bpf_map *map; 12861 int err; 12862 12863 if (BPF_SIZE(insn->code) != BPF_DW) { 12864 verbose(env, "invalid BPF_LD_IMM insn\n"); 12865 return -EINVAL; 12866 } 12867 if (insn->off != 0) { 12868 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 12869 return -EINVAL; 12870 } 12871 12872 err = check_reg_arg(env, insn->dst_reg, DST_OP); 12873 if (err) 12874 return err; 12875 12876 dst_reg = ®s[insn->dst_reg]; 12877 if (insn->src_reg == 0) { 12878 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 12879 12880 dst_reg->type = SCALAR_VALUE; 12881 __mark_reg_known(®s[insn->dst_reg], imm); 12882 return 0; 12883 } 12884 12885 /* All special src_reg cases are listed below. From this point onwards 12886 * we either succeed and assign a corresponding dst_reg->type after 12887 * zeroing the offset, or fail and reject the program. 12888 */ 12889 mark_reg_known_zero(env, regs, insn->dst_reg); 12890 12891 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 12892 dst_reg->type = aux->btf_var.reg_type; 12893 switch (base_type(dst_reg->type)) { 12894 case PTR_TO_MEM: 12895 dst_reg->mem_size = aux->btf_var.mem_size; 12896 break; 12897 case PTR_TO_BTF_ID: 12898 dst_reg->btf = aux->btf_var.btf; 12899 dst_reg->btf_id = aux->btf_var.btf_id; 12900 break; 12901 default: 12902 verbose(env, "bpf verifier is misconfigured\n"); 12903 return -EFAULT; 12904 } 12905 return 0; 12906 } 12907 12908 if (insn->src_reg == BPF_PSEUDO_FUNC) { 12909 struct bpf_prog_aux *aux = env->prog->aux; 12910 u32 subprogno = find_subprog(env, 12911 env->insn_idx + insn->imm + 1); 12912 12913 if (!aux->func_info) { 12914 verbose(env, "missing btf func_info\n"); 12915 return -EINVAL; 12916 } 12917 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 12918 verbose(env, "callback function not static\n"); 12919 return -EINVAL; 12920 } 12921 12922 dst_reg->type = PTR_TO_FUNC; 12923 dst_reg->subprogno = subprogno; 12924 return 0; 12925 } 12926 12927 map = env->used_maps[aux->map_index]; 12928 dst_reg->map_ptr = map; 12929 12930 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 12931 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 12932 dst_reg->type = PTR_TO_MAP_VALUE; 12933 dst_reg->off = aux->map_off; 12934 WARN_ON_ONCE(map->max_entries != 1); 12935 /* We want reg->id to be same (0) as map_value is not distinct */ 12936 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 12937 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 12938 dst_reg->type = CONST_PTR_TO_MAP; 12939 } else { 12940 verbose(env, "bpf verifier is misconfigured\n"); 12941 return -EINVAL; 12942 } 12943 12944 return 0; 12945 } 12946 12947 static bool may_access_skb(enum bpf_prog_type type) 12948 { 12949 switch (type) { 12950 case BPF_PROG_TYPE_SOCKET_FILTER: 12951 case BPF_PROG_TYPE_SCHED_CLS: 12952 case BPF_PROG_TYPE_SCHED_ACT: 12953 return true; 12954 default: 12955 return false; 12956 } 12957 } 12958 12959 /* verify safety of LD_ABS|LD_IND instructions: 12960 * - they can only appear in the programs where ctx == skb 12961 * - since they are wrappers of function calls, they scratch R1-R5 registers, 12962 * preserve R6-R9, and store return value into R0 12963 * 12964 * Implicit input: 12965 * ctx == skb == R6 == CTX 12966 * 12967 * Explicit input: 12968 * SRC == any register 12969 * IMM == 32-bit immediate 12970 * 12971 * Output: 12972 * R0 - 8/16/32-bit skb data converted to cpu endianness 12973 */ 12974 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 12975 { 12976 struct bpf_reg_state *regs = cur_regs(env); 12977 static const int ctx_reg = BPF_REG_6; 12978 u8 mode = BPF_MODE(insn->code); 12979 int i, err; 12980 12981 if (!may_access_skb(resolve_prog_type(env->prog))) { 12982 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 12983 return -EINVAL; 12984 } 12985 12986 if (!env->ops->gen_ld_abs) { 12987 verbose(env, "bpf verifier is misconfigured\n"); 12988 return -EINVAL; 12989 } 12990 12991 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 12992 BPF_SIZE(insn->code) == BPF_DW || 12993 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 12994 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 12995 return -EINVAL; 12996 } 12997 12998 /* check whether implicit source operand (register R6) is readable */ 12999 err = check_reg_arg(env, ctx_reg, SRC_OP); 13000 if (err) 13001 return err; 13002 13003 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 13004 * gen_ld_abs() may terminate the program at runtime, leading to 13005 * reference leak. 13006 */ 13007 err = check_reference_leak(env); 13008 if (err) { 13009 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 13010 return err; 13011 } 13012 13013 if (env->cur_state->active_lock.ptr) { 13014 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 13015 return -EINVAL; 13016 } 13017 13018 if (env->cur_state->active_rcu_lock) { 13019 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); 13020 return -EINVAL; 13021 } 13022 13023 if (regs[ctx_reg].type != PTR_TO_CTX) { 13024 verbose(env, 13025 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 13026 return -EINVAL; 13027 } 13028 13029 if (mode == BPF_IND) { 13030 /* check explicit source operand */ 13031 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13032 if (err) 13033 return err; 13034 } 13035 13036 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 13037 if (err < 0) 13038 return err; 13039 13040 /* reset caller saved regs to unreadable */ 13041 for (i = 0; i < CALLER_SAVED_REGS; i++) { 13042 mark_reg_not_init(env, regs, caller_saved[i]); 13043 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 13044 } 13045 13046 /* mark destination R0 register as readable, since it contains 13047 * the value fetched from the packet. 13048 * Already marked as written above. 13049 */ 13050 mark_reg_unknown(env, regs, BPF_REG_0); 13051 /* ld_abs load up to 32-bit skb data. */ 13052 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 13053 return 0; 13054 } 13055 13056 static int check_return_code(struct bpf_verifier_env *env) 13057 { 13058 struct tnum enforce_attach_type_range = tnum_unknown; 13059 const struct bpf_prog *prog = env->prog; 13060 struct bpf_reg_state *reg; 13061 struct tnum range = tnum_range(0, 1); 13062 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 13063 int err; 13064 struct bpf_func_state *frame = env->cur_state->frame[0]; 13065 const bool is_subprog = frame->subprogno; 13066 13067 /* LSM and struct_ops func-ptr's return type could be "void" */ 13068 if (!is_subprog) { 13069 switch (prog_type) { 13070 case BPF_PROG_TYPE_LSM: 13071 if (prog->expected_attach_type == BPF_LSM_CGROUP) 13072 /* See below, can be 0 or 0-1 depending on hook. */ 13073 break; 13074 fallthrough; 13075 case BPF_PROG_TYPE_STRUCT_OPS: 13076 if (!prog->aux->attach_func_proto->type) 13077 return 0; 13078 break; 13079 default: 13080 break; 13081 } 13082 } 13083 13084 /* eBPF calling convention is such that R0 is used 13085 * to return the value from eBPF program. 13086 * Make sure that it's readable at this time 13087 * of bpf_exit, which means that program wrote 13088 * something into it earlier 13089 */ 13090 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 13091 if (err) 13092 return err; 13093 13094 if (is_pointer_value(env, BPF_REG_0)) { 13095 verbose(env, "R0 leaks addr as return value\n"); 13096 return -EACCES; 13097 } 13098 13099 reg = cur_regs(env) + BPF_REG_0; 13100 13101 if (frame->in_async_callback_fn) { 13102 /* enforce return zero from async callbacks like timer */ 13103 if (reg->type != SCALAR_VALUE) { 13104 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 13105 reg_type_str(env, reg->type)); 13106 return -EINVAL; 13107 } 13108 13109 if (!tnum_in(tnum_const(0), reg->var_off)) { 13110 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 13111 return -EINVAL; 13112 } 13113 return 0; 13114 } 13115 13116 if (is_subprog) { 13117 if (reg->type != SCALAR_VALUE) { 13118 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 13119 reg_type_str(env, reg->type)); 13120 return -EINVAL; 13121 } 13122 return 0; 13123 } 13124 13125 switch (prog_type) { 13126 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 13127 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 13128 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 13129 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 13130 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 13131 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 13132 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 13133 range = tnum_range(1, 1); 13134 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 13135 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 13136 range = tnum_range(0, 3); 13137 break; 13138 case BPF_PROG_TYPE_CGROUP_SKB: 13139 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 13140 range = tnum_range(0, 3); 13141 enforce_attach_type_range = tnum_range(2, 3); 13142 } 13143 break; 13144 case BPF_PROG_TYPE_CGROUP_SOCK: 13145 case BPF_PROG_TYPE_SOCK_OPS: 13146 case BPF_PROG_TYPE_CGROUP_DEVICE: 13147 case BPF_PROG_TYPE_CGROUP_SYSCTL: 13148 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 13149 break; 13150 case BPF_PROG_TYPE_RAW_TRACEPOINT: 13151 if (!env->prog->aux->attach_btf_id) 13152 return 0; 13153 range = tnum_const(0); 13154 break; 13155 case BPF_PROG_TYPE_TRACING: 13156 switch (env->prog->expected_attach_type) { 13157 case BPF_TRACE_FENTRY: 13158 case BPF_TRACE_FEXIT: 13159 range = tnum_const(0); 13160 break; 13161 case BPF_TRACE_RAW_TP: 13162 case BPF_MODIFY_RETURN: 13163 return 0; 13164 case BPF_TRACE_ITER: 13165 break; 13166 default: 13167 return -ENOTSUPP; 13168 } 13169 break; 13170 case BPF_PROG_TYPE_SK_LOOKUP: 13171 range = tnum_range(SK_DROP, SK_PASS); 13172 break; 13173 13174 case BPF_PROG_TYPE_LSM: 13175 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 13176 /* Regular BPF_PROG_TYPE_LSM programs can return 13177 * any value. 13178 */ 13179 return 0; 13180 } 13181 if (!env->prog->aux->attach_func_proto->type) { 13182 /* Make sure programs that attach to void 13183 * hooks don't try to modify return value. 13184 */ 13185 range = tnum_range(1, 1); 13186 } 13187 break; 13188 13189 case BPF_PROG_TYPE_EXT: 13190 /* freplace program can return anything as its return value 13191 * depends on the to-be-replaced kernel func or bpf program. 13192 */ 13193 default: 13194 return 0; 13195 } 13196 13197 if (reg->type != SCALAR_VALUE) { 13198 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 13199 reg_type_str(env, reg->type)); 13200 return -EINVAL; 13201 } 13202 13203 if (!tnum_in(range, reg->var_off)) { 13204 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 13205 if (prog->expected_attach_type == BPF_LSM_CGROUP && 13206 prog_type == BPF_PROG_TYPE_LSM && 13207 !prog->aux->attach_func_proto->type) 13208 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 13209 return -EINVAL; 13210 } 13211 13212 if (!tnum_is_unknown(enforce_attach_type_range) && 13213 tnum_in(enforce_attach_type_range, reg->var_off)) 13214 env->prog->enforce_expected_attach_type = 1; 13215 return 0; 13216 } 13217 13218 /* non-recursive DFS pseudo code 13219 * 1 procedure DFS-iterative(G,v): 13220 * 2 label v as discovered 13221 * 3 let S be a stack 13222 * 4 S.push(v) 13223 * 5 while S is not empty 13224 * 6 t <- S.peek() 13225 * 7 if t is what we're looking for: 13226 * 8 return t 13227 * 9 for all edges e in G.adjacentEdges(t) do 13228 * 10 if edge e is already labelled 13229 * 11 continue with the next edge 13230 * 12 w <- G.adjacentVertex(t,e) 13231 * 13 if vertex w is not discovered and not explored 13232 * 14 label e as tree-edge 13233 * 15 label w as discovered 13234 * 16 S.push(w) 13235 * 17 continue at 5 13236 * 18 else if vertex w is discovered 13237 * 19 label e as back-edge 13238 * 20 else 13239 * 21 // vertex w is explored 13240 * 22 label e as forward- or cross-edge 13241 * 23 label t as explored 13242 * 24 S.pop() 13243 * 13244 * convention: 13245 * 0x10 - discovered 13246 * 0x11 - discovered and fall-through edge labelled 13247 * 0x12 - discovered and fall-through and branch edges labelled 13248 * 0x20 - explored 13249 */ 13250 13251 enum { 13252 DISCOVERED = 0x10, 13253 EXPLORED = 0x20, 13254 FALLTHROUGH = 1, 13255 BRANCH = 2, 13256 }; 13257 13258 static u32 state_htab_size(struct bpf_verifier_env *env) 13259 { 13260 return env->prog->len; 13261 } 13262 13263 static struct bpf_verifier_state_list **explored_state( 13264 struct bpf_verifier_env *env, 13265 int idx) 13266 { 13267 struct bpf_verifier_state *cur = env->cur_state; 13268 struct bpf_func_state *state = cur->frame[cur->curframe]; 13269 13270 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 13271 } 13272 13273 static void mark_prune_point(struct bpf_verifier_env *env, int idx) 13274 { 13275 env->insn_aux_data[idx].prune_point = true; 13276 } 13277 13278 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) 13279 { 13280 return env->insn_aux_data[insn_idx].prune_point; 13281 } 13282 13283 enum { 13284 DONE_EXPLORING = 0, 13285 KEEP_EXPLORING = 1, 13286 }; 13287 13288 /* t, w, e - match pseudo-code above: 13289 * t - index of current instruction 13290 * w - next instruction 13291 * e - edge 13292 */ 13293 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 13294 bool loop_ok) 13295 { 13296 int *insn_stack = env->cfg.insn_stack; 13297 int *insn_state = env->cfg.insn_state; 13298 13299 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 13300 return DONE_EXPLORING; 13301 13302 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 13303 return DONE_EXPLORING; 13304 13305 if (w < 0 || w >= env->prog->len) { 13306 verbose_linfo(env, t, "%d: ", t); 13307 verbose(env, "jump out of range from insn %d to %d\n", t, w); 13308 return -EINVAL; 13309 } 13310 13311 if (e == BRANCH) { 13312 /* mark branch target for state pruning */ 13313 mark_prune_point(env, w); 13314 mark_jmp_point(env, w); 13315 } 13316 13317 if (insn_state[w] == 0) { 13318 /* tree-edge */ 13319 insn_state[t] = DISCOVERED | e; 13320 insn_state[w] = DISCOVERED; 13321 if (env->cfg.cur_stack >= env->prog->len) 13322 return -E2BIG; 13323 insn_stack[env->cfg.cur_stack++] = w; 13324 return KEEP_EXPLORING; 13325 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 13326 if (loop_ok && env->bpf_capable) 13327 return DONE_EXPLORING; 13328 verbose_linfo(env, t, "%d: ", t); 13329 verbose_linfo(env, w, "%d: ", w); 13330 verbose(env, "back-edge from insn %d to %d\n", t, w); 13331 return -EINVAL; 13332 } else if (insn_state[w] == EXPLORED) { 13333 /* forward- or cross-edge */ 13334 insn_state[t] = DISCOVERED | e; 13335 } else { 13336 verbose(env, "insn state internal bug\n"); 13337 return -EFAULT; 13338 } 13339 return DONE_EXPLORING; 13340 } 13341 13342 static int visit_func_call_insn(int t, struct bpf_insn *insns, 13343 struct bpf_verifier_env *env, 13344 bool visit_callee) 13345 { 13346 int ret; 13347 13348 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 13349 if (ret) 13350 return ret; 13351 13352 mark_prune_point(env, t + 1); 13353 /* when we exit from subprog, we need to record non-linear history */ 13354 mark_jmp_point(env, t + 1); 13355 13356 if (visit_callee) { 13357 mark_prune_point(env, t); 13358 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 13359 /* It's ok to allow recursion from CFG point of 13360 * view. __check_func_call() will do the actual 13361 * check. 13362 */ 13363 bpf_pseudo_func(insns + t)); 13364 } 13365 return ret; 13366 } 13367 13368 /* Visits the instruction at index t and returns one of the following: 13369 * < 0 - an error occurred 13370 * DONE_EXPLORING - the instruction was fully explored 13371 * KEEP_EXPLORING - there is still work to be done before it is fully explored 13372 */ 13373 static int visit_insn(int t, struct bpf_verifier_env *env) 13374 { 13375 struct bpf_insn *insns = env->prog->insnsi; 13376 int ret; 13377 13378 if (bpf_pseudo_func(insns + t)) 13379 return visit_func_call_insn(t, insns, env, true); 13380 13381 /* All non-branch instructions have a single fall-through edge. */ 13382 if (BPF_CLASS(insns[t].code) != BPF_JMP && 13383 BPF_CLASS(insns[t].code) != BPF_JMP32) 13384 return push_insn(t, t + 1, FALLTHROUGH, env, false); 13385 13386 switch (BPF_OP(insns[t].code)) { 13387 case BPF_EXIT: 13388 return DONE_EXPLORING; 13389 13390 case BPF_CALL: 13391 if (insns[t].imm == BPF_FUNC_timer_set_callback) 13392 /* Mark this call insn as a prune point to trigger 13393 * is_state_visited() check before call itself is 13394 * processed by __check_func_call(). Otherwise new 13395 * async state will be pushed for further exploration. 13396 */ 13397 mark_prune_point(env, t); 13398 return visit_func_call_insn(t, insns, env, 13399 insns[t].src_reg == BPF_PSEUDO_CALL); 13400 13401 case BPF_JA: 13402 if (BPF_SRC(insns[t].code) != BPF_K) 13403 return -EINVAL; 13404 13405 /* unconditional jump with single edge */ 13406 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 13407 true); 13408 if (ret) 13409 return ret; 13410 13411 mark_prune_point(env, t + insns[t].off + 1); 13412 mark_jmp_point(env, t + insns[t].off + 1); 13413 13414 return ret; 13415 13416 default: 13417 /* conditional jump with two edges */ 13418 mark_prune_point(env, t); 13419 13420 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 13421 if (ret) 13422 return ret; 13423 13424 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 13425 } 13426 } 13427 13428 /* non-recursive depth-first-search to detect loops in BPF program 13429 * loop == back-edge in directed graph 13430 */ 13431 static int check_cfg(struct bpf_verifier_env *env) 13432 { 13433 int insn_cnt = env->prog->len; 13434 int *insn_stack, *insn_state; 13435 int ret = 0; 13436 int i; 13437 13438 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 13439 if (!insn_state) 13440 return -ENOMEM; 13441 13442 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 13443 if (!insn_stack) { 13444 kvfree(insn_state); 13445 return -ENOMEM; 13446 } 13447 13448 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 13449 insn_stack[0] = 0; /* 0 is the first instruction */ 13450 env->cfg.cur_stack = 1; 13451 13452 while (env->cfg.cur_stack > 0) { 13453 int t = insn_stack[env->cfg.cur_stack - 1]; 13454 13455 ret = visit_insn(t, env); 13456 switch (ret) { 13457 case DONE_EXPLORING: 13458 insn_state[t] = EXPLORED; 13459 env->cfg.cur_stack--; 13460 break; 13461 case KEEP_EXPLORING: 13462 break; 13463 default: 13464 if (ret > 0) { 13465 verbose(env, "visit_insn internal bug\n"); 13466 ret = -EFAULT; 13467 } 13468 goto err_free; 13469 } 13470 } 13471 13472 if (env->cfg.cur_stack < 0) { 13473 verbose(env, "pop stack internal bug\n"); 13474 ret = -EFAULT; 13475 goto err_free; 13476 } 13477 13478 for (i = 0; i < insn_cnt; i++) { 13479 if (insn_state[i] != EXPLORED) { 13480 verbose(env, "unreachable insn %d\n", i); 13481 ret = -EINVAL; 13482 goto err_free; 13483 } 13484 } 13485 ret = 0; /* cfg looks good */ 13486 13487 err_free: 13488 kvfree(insn_state); 13489 kvfree(insn_stack); 13490 env->cfg.insn_state = env->cfg.insn_stack = NULL; 13491 return ret; 13492 } 13493 13494 static int check_abnormal_return(struct bpf_verifier_env *env) 13495 { 13496 int i; 13497 13498 for (i = 1; i < env->subprog_cnt; i++) { 13499 if (env->subprog_info[i].has_ld_abs) { 13500 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 13501 return -EINVAL; 13502 } 13503 if (env->subprog_info[i].has_tail_call) { 13504 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 13505 return -EINVAL; 13506 } 13507 } 13508 return 0; 13509 } 13510 13511 /* The minimum supported BTF func info size */ 13512 #define MIN_BPF_FUNCINFO_SIZE 8 13513 #define MAX_FUNCINFO_REC_SIZE 252 13514 13515 static int check_btf_func(struct bpf_verifier_env *env, 13516 const union bpf_attr *attr, 13517 bpfptr_t uattr) 13518 { 13519 const struct btf_type *type, *func_proto, *ret_type; 13520 u32 i, nfuncs, urec_size, min_size; 13521 u32 krec_size = sizeof(struct bpf_func_info); 13522 struct bpf_func_info *krecord; 13523 struct bpf_func_info_aux *info_aux = NULL; 13524 struct bpf_prog *prog; 13525 const struct btf *btf; 13526 bpfptr_t urecord; 13527 u32 prev_offset = 0; 13528 bool scalar_return; 13529 int ret = -ENOMEM; 13530 13531 nfuncs = attr->func_info_cnt; 13532 if (!nfuncs) { 13533 if (check_abnormal_return(env)) 13534 return -EINVAL; 13535 return 0; 13536 } 13537 13538 if (nfuncs != env->subprog_cnt) { 13539 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 13540 return -EINVAL; 13541 } 13542 13543 urec_size = attr->func_info_rec_size; 13544 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 13545 urec_size > MAX_FUNCINFO_REC_SIZE || 13546 urec_size % sizeof(u32)) { 13547 verbose(env, "invalid func info rec size %u\n", urec_size); 13548 return -EINVAL; 13549 } 13550 13551 prog = env->prog; 13552 btf = prog->aux->btf; 13553 13554 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 13555 min_size = min_t(u32, krec_size, urec_size); 13556 13557 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 13558 if (!krecord) 13559 return -ENOMEM; 13560 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 13561 if (!info_aux) 13562 goto err_free; 13563 13564 for (i = 0; i < nfuncs; i++) { 13565 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 13566 if (ret) { 13567 if (ret == -E2BIG) { 13568 verbose(env, "nonzero tailing record in func info"); 13569 /* set the size kernel expects so loader can zero 13570 * out the rest of the record. 13571 */ 13572 if (copy_to_bpfptr_offset(uattr, 13573 offsetof(union bpf_attr, func_info_rec_size), 13574 &min_size, sizeof(min_size))) 13575 ret = -EFAULT; 13576 } 13577 goto err_free; 13578 } 13579 13580 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 13581 ret = -EFAULT; 13582 goto err_free; 13583 } 13584 13585 /* check insn_off */ 13586 ret = -EINVAL; 13587 if (i == 0) { 13588 if (krecord[i].insn_off) { 13589 verbose(env, 13590 "nonzero insn_off %u for the first func info record", 13591 krecord[i].insn_off); 13592 goto err_free; 13593 } 13594 } else if (krecord[i].insn_off <= prev_offset) { 13595 verbose(env, 13596 "same or smaller insn offset (%u) than previous func info record (%u)", 13597 krecord[i].insn_off, prev_offset); 13598 goto err_free; 13599 } 13600 13601 if (env->subprog_info[i].start != krecord[i].insn_off) { 13602 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 13603 goto err_free; 13604 } 13605 13606 /* check type_id */ 13607 type = btf_type_by_id(btf, krecord[i].type_id); 13608 if (!type || !btf_type_is_func(type)) { 13609 verbose(env, "invalid type id %d in func info", 13610 krecord[i].type_id); 13611 goto err_free; 13612 } 13613 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 13614 13615 func_proto = btf_type_by_id(btf, type->type); 13616 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 13617 /* btf_func_check() already verified it during BTF load */ 13618 goto err_free; 13619 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 13620 scalar_return = 13621 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); 13622 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 13623 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 13624 goto err_free; 13625 } 13626 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 13627 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 13628 goto err_free; 13629 } 13630 13631 prev_offset = krecord[i].insn_off; 13632 bpfptr_add(&urecord, urec_size); 13633 } 13634 13635 prog->aux->func_info = krecord; 13636 prog->aux->func_info_cnt = nfuncs; 13637 prog->aux->func_info_aux = info_aux; 13638 return 0; 13639 13640 err_free: 13641 kvfree(krecord); 13642 kfree(info_aux); 13643 return ret; 13644 } 13645 13646 static void adjust_btf_func(struct bpf_verifier_env *env) 13647 { 13648 struct bpf_prog_aux *aux = env->prog->aux; 13649 int i; 13650 13651 if (!aux->func_info) 13652 return; 13653 13654 for (i = 0; i < env->subprog_cnt; i++) 13655 aux->func_info[i].insn_off = env->subprog_info[i].start; 13656 } 13657 13658 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 13659 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 13660 13661 static int check_btf_line(struct bpf_verifier_env *env, 13662 const union bpf_attr *attr, 13663 bpfptr_t uattr) 13664 { 13665 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 13666 struct bpf_subprog_info *sub; 13667 struct bpf_line_info *linfo; 13668 struct bpf_prog *prog; 13669 const struct btf *btf; 13670 bpfptr_t ulinfo; 13671 int err; 13672 13673 nr_linfo = attr->line_info_cnt; 13674 if (!nr_linfo) 13675 return 0; 13676 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 13677 return -EINVAL; 13678 13679 rec_size = attr->line_info_rec_size; 13680 if (rec_size < MIN_BPF_LINEINFO_SIZE || 13681 rec_size > MAX_LINEINFO_REC_SIZE || 13682 rec_size & (sizeof(u32) - 1)) 13683 return -EINVAL; 13684 13685 /* Need to zero it in case the userspace may 13686 * pass in a smaller bpf_line_info object. 13687 */ 13688 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 13689 GFP_KERNEL | __GFP_NOWARN); 13690 if (!linfo) 13691 return -ENOMEM; 13692 13693 prog = env->prog; 13694 btf = prog->aux->btf; 13695 13696 s = 0; 13697 sub = env->subprog_info; 13698 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 13699 expected_size = sizeof(struct bpf_line_info); 13700 ncopy = min_t(u32, expected_size, rec_size); 13701 for (i = 0; i < nr_linfo; i++) { 13702 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 13703 if (err) { 13704 if (err == -E2BIG) { 13705 verbose(env, "nonzero tailing record in line_info"); 13706 if (copy_to_bpfptr_offset(uattr, 13707 offsetof(union bpf_attr, line_info_rec_size), 13708 &expected_size, sizeof(expected_size))) 13709 err = -EFAULT; 13710 } 13711 goto err_free; 13712 } 13713 13714 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 13715 err = -EFAULT; 13716 goto err_free; 13717 } 13718 13719 /* 13720 * Check insn_off to ensure 13721 * 1) strictly increasing AND 13722 * 2) bounded by prog->len 13723 * 13724 * The linfo[0].insn_off == 0 check logically falls into 13725 * the later "missing bpf_line_info for func..." case 13726 * because the first linfo[0].insn_off must be the 13727 * first sub also and the first sub must have 13728 * subprog_info[0].start == 0. 13729 */ 13730 if ((i && linfo[i].insn_off <= prev_offset) || 13731 linfo[i].insn_off >= prog->len) { 13732 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 13733 i, linfo[i].insn_off, prev_offset, 13734 prog->len); 13735 err = -EINVAL; 13736 goto err_free; 13737 } 13738 13739 if (!prog->insnsi[linfo[i].insn_off].code) { 13740 verbose(env, 13741 "Invalid insn code at line_info[%u].insn_off\n", 13742 i); 13743 err = -EINVAL; 13744 goto err_free; 13745 } 13746 13747 if (!btf_name_by_offset(btf, linfo[i].line_off) || 13748 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 13749 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 13750 err = -EINVAL; 13751 goto err_free; 13752 } 13753 13754 if (s != env->subprog_cnt) { 13755 if (linfo[i].insn_off == sub[s].start) { 13756 sub[s].linfo_idx = i; 13757 s++; 13758 } else if (sub[s].start < linfo[i].insn_off) { 13759 verbose(env, "missing bpf_line_info for func#%u\n", s); 13760 err = -EINVAL; 13761 goto err_free; 13762 } 13763 } 13764 13765 prev_offset = linfo[i].insn_off; 13766 bpfptr_add(&ulinfo, rec_size); 13767 } 13768 13769 if (s != env->subprog_cnt) { 13770 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 13771 env->subprog_cnt - s, s); 13772 err = -EINVAL; 13773 goto err_free; 13774 } 13775 13776 prog->aux->linfo = linfo; 13777 prog->aux->nr_linfo = nr_linfo; 13778 13779 return 0; 13780 13781 err_free: 13782 kvfree(linfo); 13783 return err; 13784 } 13785 13786 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 13787 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 13788 13789 static int check_core_relo(struct bpf_verifier_env *env, 13790 const union bpf_attr *attr, 13791 bpfptr_t uattr) 13792 { 13793 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 13794 struct bpf_core_relo core_relo = {}; 13795 struct bpf_prog *prog = env->prog; 13796 const struct btf *btf = prog->aux->btf; 13797 struct bpf_core_ctx ctx = { 13798 .log = &env->log, 13799 .btf = btf, 13800 }; 13801 bpfptr_t u_core_relo; 13802 int err; 13803 13804 nr_core_relo = attr->core_relo_cnt; 13805 if (!nr_core_relo) 13806 return 0; 13807 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 13808 return -EINVAL; 13809 13810 rec_size = attr->core_relo_rec_size; 13811 if (rec_size < MIN_CORE_RELO_SIZE || 13812 rec_size > MAX_CORE_RELO_SIZE || 13813 rec_size % sizeof(u32)) 13814 return -EINVAL; 13815 13816 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 13817 expected_size = sizeof(struct bpf_core_relo); 13818 ncopy = min_t(u32, expected_size, rec_size); 13819 13820 /* Unlike func_info and line_info, copy and apply each CO-RE 13821 * relocation record one at a time. 13822 */ 13823 for (i = 0; i < nr_core_relo; i++) { 13824 /* future proofing when sizeof(bpf_core_relo) changes */ 13825 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 13826 if (err) { 13827 if (err == -E2BIG) { 13828 verbose(env, "nonzero tailing record in core_relo"); 13829 if (copy_to_bpfptr_offset(uattr, 13830 offsetof(union bpf_attr, core_relo_rec_size), 13831 &expected_size, sizeof(expected_size))) 13832 err = -EFAULT; 13833 } 13834 break; 13835 } 13836 13837 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 13838 err = -EFAULT; 13839 break; 13840 } 13841 13842 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 13843 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 13844 i, core_relo.insn_off, prog->len); 13845 err = -EINVAL; 13846 break; 13847 } 13848 13849 err = bpf_core_apply(&ctx, &core_relo, i, 13850 &prog->insnsi[core_relo.insn_off / 8]); 13851 if (err) 13852 break; 13853 bpfptr_add(&u_core_relo, rec_size); 13854 } 13855 return err; 13856 } 13857 13858 static int check_btf_info(struct bpf_verifier_env *env, 13859 const union bpf_attr *attr, 13860 bpfptr_t uattr) 13861 { 13862 struct btf *btf; 13863 int err; 13864 13865 if (!attr->func_info_cnt && !attr->line_info_cnt) { 13866 if (check_abnormal_return(env)) 13867 return -EINVAL; 13868 return 0; 13869 } 13870 13871 btf = btf_get_by_fd(attr->prog_btf_fd); 13872 if (IS_ERR(btf)) 13873 return PTR_ERR(btf); 13874 if (btf_is_kernel(btf)) { 13875 btf_put(btf); 13876 return -EACCES; 13877 } 13878 env->prog->aux->btf = btf; 13879 13880 err = check_btf_func(env, attr, uattr); 13881 if (err) 13882 return err; 13883 13884 err = check_btf_line(env, attr, uattr); 13885 if (err) 13886 return err; 13887 13888 err = check_core_relo(env, attr, uattr); 13889 if (err) 13890 return err; 13891 13892 return 0; 13893 } 13894 13895 /* check %cur's range satisfies %old's */ 13896 static bool range_within(struct bpf_reg_state *old, 13897 struct bpf_reg_state *cur) 13898 { 13899 return old->umin_value <= cur->umin_value && 13900 old->umax_value >= cur->umax_value && 13901 old->smin_value <= cur->smin_value && 13902 old->smax_value >= cur->smax_value && 13903 old->u32_min_value <= cur->u32_min_value && 13904 old->u32_max_value >= cur->u32_max_value && 13905 old->s32_min_value <= cur->s32_min_value && 13906 old->s32_max_value >= cur->s32_max_value; 13907 } 13908 13909 /* If in the old state two registers had the same id, then they need to have 13910 * the same id in the new state as well. But that id could be different from 13911 * the old state, so we need to track the mapping from old to new ids. 13912 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 13913 * regs with old id 5 must also have new id 9 for the new state to be safe. But 13914 * regs with a different old id could still have new id 9, we don't care about 13915 * that. 13916 * So we look through our idmap to see if this old id has been seen before. If 13917 * so, we require the new id to match; otherwise, we add the id pair to the map. 13918 */ 13919 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 13920 { 13921 unsigned int i; 13922 13923 /* either both IDs should be set or both should be zero */ 13924 if (!!old_id != !!cur_id) 13925 return false; 13926 13927 if (old_id == 0) /* cur_id == 0 as well */ 13928 return true; 13929 13930 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 13931 if (!idmap[i].old) { 13932 /* Reached an empty slot; haven't seen this id before */ 13933 idmap[i].old = old_id; 13934 idmap[i].cur = cur_id; 13935 return true; 13936 } 13937 if (idmap[i].old == old_id) 13938 return idmap[i].cur == cur_id; 13939 } 13940 /* We ran out of idmap slots, which should be impossible */ 13941 WARN_ON_ONCE(1); 13942 return false; 13943 } 13944 13945 static void clean_func_state(struct bpf_verifier_env *env, 13946 struct bpf_func_state *st) 13947 { 13948 enum bpf_reg_liveness live; 13949 int i, j; 13950 13951 for (i = 0; i < BPF_REG_FP; i++) { 13952 live = st->regs[i].live; 13953 /* liveness must not touch this register anymore */ 13954 st->regs[i].live |= REG_LIVE_DONE; 13955 if (!(live & REG_LIVE_READ)) 13956 /* since the register is unused, clear its state 13957 * to make further comparison simpler 13958 */ 13959 __mark_reg_not_init(env, &st->regs[i]); 13960 } 13961 13962 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 13963 live = st->stack[i].spilled_ptr.live; 13964 /* liveness must not touch this stack slot anymore */ 13965 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 13966 if (!(live & REG_LIVE_READ)) { 13967 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 13968 for (j = 0; j < BPF_REG_SIZE; j++) 13969 st->stack[i].slot_type[j] = STACK_INVALID; 13970 } 13971 } 13972 } 13973 13974 static void clean_verifier_state(struct bpf_verifier_env *env, 13975 struct bpf_verifier_state *st) 13976 { 13977 int i; 13978 13979 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 13980 /* all regs in this state in all frames were already marked */ 13981 return; 13982 13983 for (i = 0; i <= st->curframe; i++) 13984 clean_func_state(env, st->frame[i]); 13985 } 13986 13987 /* the parentage chains form a tree. 13988 * the verifier states are added to state lists at given insn and 13989 * pushed into state stack for future exploration. 13990 * when the verifier reaches bpf_exit insn some of the verifer states 13991 * stored in the state lists have their final liveness state already, 13992 * but a lot of states will get revised from liveness point of view when 13993 * the verifier explores other branches. 13994 * Example: 13995 * 1: r0 = 1 13996 * 2: if r1 == 100 goto pc+1 13997 * 3: r0 = 2 13998 * 4: exit 13999 * when the verifier reaches exit insn the register r0 in the state list of 14000 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 14001 * of insn 2 and goes exploring further. At the insn 4 it will walk the 14002 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 14003 * 14004 * Since the verifier pushes the branch states as it sees them while exploring 14005 * the program the condition of walking the branch instruction for the second 14006 * time means that all states below this branch were already explored and 14007 * their final liveness marks are already propagated. 14008 * Hence when the verifier completes the search of state list in is_state_visited() 14009 * we can call this clean_live_states() function to mark all liveness states 14010 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 14011 * will not be used. 14012 * This function also clears the registers and stack for states that !READ 14013 * to simplify state merging. 14014 * 14015 * Important note here that walking the same branch instruction in the callee 14016 * doesn't meant that the states are DONE. The verifier has to compare 14017 * the callsites 14018 */ 14019 static void clean_live_states(struct bpf_verifier_env *env, int insn, 14020 struct bpf_verifier_state *cur) 14021 { 14022 struct bpf_verifier_state_list *sl; 14023 int i; 14024 14025 sl = *explored_state(env, insn); 14026 while (sl) { 14027 if (sl->state.branches) 14028 goto next; 14029 if (sl->state.insn_idx != insn || 14030 sl->state.curframe != cur->curframe) 14031 goto next; 14032 for (i = 0; i <= cur->curframe; i++) 14033 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 14034 goto next; 14035 clean_verifier_state(env, &sl->state); 14036 next: 14037 sl = sl->next; 14038 } 14039 } 14040 14041 static bool regs_exact(const struct bpf_reg_state *rold, 14042 const struct bpf_reg_state *rcur, 14043 struct bpf_id_pair *idmap) 14044 { 14045 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 14046 check_ids(rold->id, rcur->id, idmap) && 14047 check_ids(rold->ref_obj_id, rcur->ref_obj_id, idmap); 14048 } 14049 14050 /* Returns true if (rold safe implies rcur safe) */ 14051 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 14052 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 14053 { 14054 if (!(rold->live & REG_LIVE_READ)) 14055 /* explored state didn't use this */ 14056 return true; 14057 if (rold->type == NOT_INIT) 14058 /* explored state can't have used this */ 14059 return true; 14060 if (rcur->type == NOT_INIT) 14061 return false; 14062 14063 /* Enforce that register types have to match exactly, including their 14064 * modifiers (like PTR_MAYBE_NULL, MEM_RDONLY, etc), as a general 14065 * rule. 14066 * 14067 * One can make a point that using a pointer register as unbounded 14068 * SCALAR would be technically acceptable, but this could lead to 14069 * pointer leaks because scalars are allowed to leak while pointers 14070 * are not. We could make this safe in special cases if root is 14071 * calling us, but it's probably not worth the hassle. 14072 * 14073 * Also, register types that are *not* MAYBE_NULL could technically be 14074 * safe to use as their MAYBE_NULL variants (e.g., PTR_TO_MAP_VALUE 14075 * is safe to be used as PTR_TO_MAP_VALUE_OR_NULL, provided both point 14076 * to the same map). 14077 * However, if the old MAYBE_NULL register then got NULL checked, 14078 * doing so could have affected others with the same id, and we can't 14079 * check for that because we lost the id when we converted to 14080 * a non-MAYBE_NULL variant. 14081 * So, as a general rule we don't allow mixing MAYBE_NULL and 14082 * non-MAYBE_NULL registers as well. 14083 */ 14084 if (rold->type != rcur->type) 14085 return false; 14086 14087 switch (base_type(rold->type)) { 14088 case SCALAR_VALUE: 14089 if (regs_exact(rold, rcur, idmap)) 14090 return true; 14091 if (env->explore_alu_limits) 14092 return false; 14093 if (!rold->precise) 14094 return true; 14095 /* new val must satisfy old val knowledge */ 14096 return range_within(rold, rcur) && 14097 tnum_in(rold->var_off, rcur->var_off); 14098 case PTR_TO_MAP_KEY: 14099 case PTR_TO_MAP_VALUE: 14100 /* If the new min/max/var_off satisfy the old ones and 14101 * everything else matches, we are OK. 14102 */ 14103 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, var_off)) == 0 && 14104 range_within(rold, rcur) && 14105 tnum_in(rold->var_off, rcur->var_off) && 14106 check_ids(rold->id, rcur->id, idmap); 14107 case PTR_TO_PACKET_META: 14108 case PTR_TO_PACKET: 14109 /* We must have at least as much range as the old ptr 14110 * did, so that any accesses which were safe before are 14111 * still safe. This is true even if old range < old off, 14112 * since someone could have accessed through (ptr - k), or 14113 * even done ptr -= k in a register, to get a safe access. 14114 */ 14115 if (rold->range > rcur->range) 14116 return false; 14117 /* If the offsets don't match, we can't trust our alignment; 14118 * nor can we be sure that we won't fall out of range. 14119 */ 14120 if (rold->off != rcur->off) 14121 return false; 14122 /* id relations must be preserved */ 14123 if (!check_ids(rold->id, rcur->id, idmap)) 14124 return false; 14125 /* new val must satisfy old val knowledge */ 14126 return range_within(rold, rcur) && 14127 tnum_in(rold->var_off, rcur->var_off); 14128 case PTR_TO_STACK: 14129 /* two stack pointers are equal only if they're pointing to 14130 * the same stack frame, since fp-8 in foo != fp-8 in bar 14131 */ 14132 return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; 14133 default: 14134 return regs_exact(rold, rcur, idmap); 14135 } 14136 } 14137 14138 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 14139 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 14140 { 14141 int i, spi; 14142 14143 /* walk slots of the explored stack and ignore any additional 14144 * slots in the current stack, since explored(safe) state 14145 * didn't use them 14146 */ 14147 for (i = 0; i < old->allocated_stack; i++) { 14148 spi = i / BPF_REG_SIZE; 14149 14150 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 14151 i += BPF_REG_SIZE - 1; 14152 /* explored state didn't use this */ 14153 continue; 14154 } 14155 14156 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 14157 continue; 14158 14159 if (env->allow_uninit_stack && 14160 old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC) 14161 continue; 14162 14163 /* explored stack has more populated slots than current stack 14164 * and these slots were used 14165 */ 14166 if (i >= cur->allocated_stack) 14167 return false; 14168 14169 /* if old state was safe with misc data in the stack 14170 * it will be safe with zero-initialized stack. 14171 * The opposite is not true 14172 */ 14173 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 14174 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 14175 continue; 14176 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 14177 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 14178 /* Ex: old explored (safe) state has STACK_SPILL in 14179 * this stack slot, but current has STACK_MISC -> 14180 * this verifier states are not equivalent, 14181 * return false to continue verification of this path 14182 */ 14183 return false; 14184 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 14185 continue; 14186 /* Both old and cur are having same slot_type */ 14187 switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) { 14188 case STACK_SPILL: 14189 /* when explored and current stack slot are both storing 14190 * spilled registers, check that stored pointers types 14191 * are the same as well. 14192 * Ex: explored safe path could have stored 14193 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 14194 * but current path has stored: 14195 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 14196 * such verifier states are not equivalent. 14197 * return false to continue verification of this path 14198 */ 14199 if (!regsafe(env, &old->stack[spi].spilled_ptr, 14200 &cur->stack[spi].spilled_ptr, idmap)) 14201 return false; 14202 break; 14203 case STACK_DYNPTR: 14204 { 14205 const struct bpf_reg_state *old_reg, *cur_reg; 14206 14207 old_reg = &old->stack[spi].spilled_ptr; 14208 cur_reg = &cur->stack[spi].spilled_ptr; 14209 if (old_reg->dynptr.type != cur_reg->dynptr.type || 14210 old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot || 14211 !check_ids(old_reg->ref_obj_id, cur_reg->ref_obj_id, idmap)) 14212 return false; 14213 break; 14214 } 14215 case STACK_MISC: 14216 case STACK_ZERO: 14217 case STACK_INVALID: 14218 continue; 14219 /* Ensure that new unhandled slot types return false by default */ 14220 default: 14221 return false; 14222 } 14223 } 14224 return true; 14225 } 14226 14227 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, 14228 struct bpf_id_pair *idmap) 14229 { 14230 int i; 14231 14232 if (old->acquired_refs != cur->acquired_refs) 14233 return false; 14234 14235 for (i = 0; i < old->acquired_refs; i++) { 14236 if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap)) 14237 return false; 14238 } 14239 14240 return true; 14241 } 14242 14243 /* compare two verifier states 14244 * 14245 * all states stored in state_list are known to be valid, since 14246 * verifier reached 'bpf_exit' instruction through them 14247 * 14248 * this function is called when verifier exploring different branches of 14249 * execution popped from the state stack. If it sees an old state that has 14250 * more strict register state and more strict stack state then this execution 14251 * branch doesn't need to be explored further, since verifier already 14252 * concluded that more strict state leads to valid finish. 14253 * 14254 * Therefore two states are equivalent if register state is more conservative 14255 * and explored stack state is more conservative than the current one. 14256 * Example: 14257 * explored current 14258 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 14259 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 14260 * 14261 * In other words if current stack state (one being explored) has more 14262 * valid slots than old one that already passed validation, it means 14263 * the verifier can stop exploring and conclude that current state is valid too 14264 * 14265 * Similarly with registers. If explored state has register type as invalid 14266 * whereas register type in current state is meaningful, it means that 14267 * the current state will reach 'bpf_exit' instruction safely 14268 */ 14269 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 14270 struct bpf_func_state *cur) 14271 { 14272 int i; 14273 14274 for (i = 0; i < MAX_BPF_REG; i++) 14275 if (!regsafe(env, &old->regs[i], &cur->regs[i], 14276 env->idmap_scratch)) 14277 return false; 14278 14279 if (!stacksafe(env, old, cur, env->idmap_scratch)) 14280 return false; 14281 14282 if (!refsafe(old, cur, env->idmap_scratch)) 14283 return false; 14284 14285 return true; 14286 } 14287 14288 static bool states_equal(struct bpf_verifier_env *env, 14289 struct bpf_verifier_state *old, 14290 struct bpf_verifier_state *cur) 14291 { 14292 int i; 14293 14294 if (old->curframe != cur->curframe) 14295 return false; 14296 14297 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 14298 14299 /* Verification state from speculative execution simulation 14300 * must never prune a non-speculative execution one. 14301 */ 14302 if (old->speculative && !cur->speculative) 14303 return false; 14304 14305 if (old->active_lock.ptr != cur->active_lock.ptr) 14306 return false; 14307 14308 /* Old and cur active_lock's have to be either both present 14309 * or both absent. 14310 */ 14311 if (!!old->active_lock.id != !!cur->active_lock.id) 14312 return false; 14313 14314 if (old->active_lock.id && 14315 !check_ids(old->active_lock.id, cur->active_lock.id, env->idmap_scratch)) 14316 return false; 14317 14318 if (old->active_rcu_lock != cur->active_rcu_lock) 14319 return false; 14320 14321 /* for states to be equal callsites have to be the same 14322 * and all frame states need to be equivalent 14323 */ 14324 for (i = 0; i <= old->curframe; i++) { 14325 if (old->frame[i]->callsite != cur->frame[i]->callsite) 14326 return false; 14327 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 14328 return false; 14329 } 14330 return true; 14331 } 14332 14333 /* Return 0 if no propagation happened. Return negative error code if error 14334 * happened. Otherwise, return the propagated bit. 14335 */ 14336 static int propagate_liveness_reg(struct bpf_verifier_env *env, 14337 struct bpf_reg_state *reg, 14338 struct bpf_reg_state *parent_reg) 14339 { 14340 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 14341 u8 flag = reg->live & REG_LIVE_READ; 14342 int err; 14343 14344 /* When comes here, read flags of PARENT_REG or REG could be any of 14345 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 14346 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 14347 */ 14348 if (parent_flag == REG_LIVE_READ64 || 14349 /* Or if there is no read flag from REG. */ 14350 !flag || 14351 /* Or if the read flag from REG is the same as PARENT_REG. */ 14352 parent_flag == flag) 14353 return 0; 14354 14355 err = mark_reg_read(env, reg, parent_reg, flag); 14356 if (err) 14357 return err; 14358 14359 return flag; 14360 } 14361 14362 /* A write screens off any subsequent reads; but write marks come from the 14363 * straight-line code between a state and its parent. When we arrive at an 14364 * equivalent state (jump target or such) we didn't arrive by the straight-line 14365 * code, so read marks in the state must propagate to the parent regardless 14366 * of the state's write marks. That's what 'parent == state->parent' comparison 14367 * in mark_reg_read() is for. 14368 */ 14369 static int propagate_liveness(struct bpf_verifier_env *env, 14370 const struct bpf_verifier_state *vstate, 14371 struct bpf_verifier_state *vparent) 14372 { 14373 struct bpf_reg_state *state_reg, *parent_reg; 14374 struct bpf_func_state *state, *parent; 14375 int i, frame, err = 0; 14376 14377 if (vparent->curframe != vstate->curframe) { 14378 WARN(1, "propagate_live: parent frame %d current frame %d\n", 14379 vparent->curframe, vstate->curframe); 14380 return -EFAULT; 14381 } 14382 /* Propagate read liveness of registers... */ 14383 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 14384 for (frame = 0; frame <= vstate->curframe; frame++) { 14385 parent = vparent->frame[frame]; 14386 state = vstate->frame[frame]; 14387 parent_reg = parent->regs; 14388 state_reg = state->regs; 14389 /* We don't need to worry about FP liveness, it's read-only */ 14390 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 14391 err = propagate_liveness_reg(env, &state_reg[i], 14392 &parent_reg[i]); 14393 if (err < 0) 14394 return err; 14395 if (err == REG_LIVE_READ64) 14396 mark_insn_zext(env, &parent_reg[i]); 14397 } 14398 14399 /* Propagate stack slots. */ 14400 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 14401 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 14402 parent_reg = &parent->stack[i].spilled_ptr; 14403 state_reg = &state->stack[i].spilled_ptr; 14404 err = propagate_liveness_reg(env, state_reg, 14405 parent_reg); 14406 if (err < 0) 14407 return err; 14408 } 14409 } 14410 return 0; 14411 } 14412 14413 /* find precise scalars in the previous equivalent state and 14414 * propagate them into the current state 14415 */ 14416 static int propagate_precision(struct bpf_verifier_env *env, 14417 const struct bpf_verifier_state *old) 14418 { 14419 struct bpf_reg_state *state_reg; 14420 struct bpf_func_state *state; 14421 int i, err = 0, fr; 14422 14423 for (fr = old->curframe; fr >= 0; fr--) { 14424 state = old->frame[fr]; 14425 state_reg = state->regs; 14426 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 14427 if (state_reg->type != SCALAR_VALUE || 14428 !state_reg->precise) 14429 continue; 14430 if (env->log.level & BPF_LOG_LEVEL2) 14431 verbose(env, "frame %d: propagating r%d\n", i, fr); 14432 err = mark_chain_precision_frame(env, fr, i); 14433 if (err < 0) 14434 return err; 14435 } 14436 14437 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 14438 if (!is_spilled_reg(&state->stack[i])) 14439 continue; 14440 state_reg = &state->stack[i].spilled_ptr; 14441 if (state_reg->type != SCALAR_VALUE || 14442 !state_reg->precise) 14443 continue; 14444 if (env->log.level & BPF_LOG_LEVEL2) 14445 verbose(env, "frame %d: propagating fp%d\n", 14446 (-i - 1) * BPF_REG_SIZE, fr); 14447 err = mark_chain_precision_stack_frame(env, fr, i); 14448 if (err < 0) 14449 return err; 14450 } 14451 } 14452 return 0; 14453 } 14454 14455 static bool states_maybe_looping(struct bpf_verifier_state *old, 14456 struct bpf_verifier_state *cur) 14457 { 14458 struct bpf_func_state *fold, *fcur; 14459 int i, fr = cur->curframe; 14460 14461 if (old->curframe != fr) 14462 return false; 14463 14464 fold = old->frame[fr]; 14465 fcur = cur->frame[fr]; 14466 for (i = 0; i < MAX_BPF_REG; i++) 14467 if (memcmp(&fold->regs[i], &fcur->regs[i], 14468 offsetof(struct bpf_reg_state, parent))) 14469 return false; 14470 return true; 14471 } 14472 14473 14474 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 14475 { 14476 struct bpf_verifier_state_list *new_sl; 14477 struct bpf_verifier_state_list *sl, **pprev; 14478 struct bpf_verifier_state *cur = env->cur_state, *new; 14479 int i, j, err, states_cnt = 0; 14480 bool add_new_state = env->test_state_freq ? true : false; 14481 14482 /* bpf progs typically have pruning point every 4 instructions 14483 * http://vger.kernel.org/bpfconf2019.html#session-1 14484 * Do not add new state for future pruning if the verifier hasn't seen 14485 * at least 2 jumps and at least 8 instructions. 14486 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 14487 * In tests that amounts to up to 50% reduction into total verifier 14488 * memory consumption and 20% verifier time speedup. 14489 */ 14490 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 14491 env->insn_processed - env->prev_insn_processed >= 8) 14492 add_new_state = true; 14493 14494 pprev = explored_state(env, insn_idx); 14495 sl = *pprev; 14496 14497 clean_live_states(env, insn_idx, cur); 14498 14499 while (sl) { 14500 states_cnt++; 14501 if (sl->state.insn_idx != insn_idx) 14502 goto next; 14503 14504 if (sl->state.branches) { 14505 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 14506 14507 if (frame->in_async_callback_fn && 14508 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 14509 /* Different async_entry_cnt means that the verifier is 14510 * processing another entry into async callback. 14511 * Seeing the same state is not an indication of infinite 14512 * loop or infinite recursion. 14513 * But finding the same state doesn't mean that it's safe 14514 * to stop processing the current state. The previous state 14515 * hasn't yet reached bpf_exit, since state.branches > 0. 14516 * Checking in_async_callback_fn alone is not enough either. 14517 * Since the verifier still needs to catch infinite loops 14518 * inside async callbacks. 14519 */ 14520 } else if (states_maybe_looping(&sl->state, cur) && 14521 states_equal(env, &sl->state, cur)) { 14522 verbose_linfo(env, insn_idx, "; "); 14523 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 14524 return -EINVAL; 14525 } 14526 /* if the verifier is processing a loop, avoid adding new state 14527 * too often, since different loop iterations have distinct 14528 * states and may not help future pruning. 14529 * This threshold shouldn't be too low to make sure that 14530 * a loop with large bound will be rejected quickly. 14531 * The most abusive loop will be: 14532 * r1 += 1 14533 * if r1 < 1000000 goto pc-2 14534 * 1M insn_procssed limit / 100 == 10k peak states. 14535 * This threshold shouldn't be too high either, since states 14536 * at the end of the loop are likely to be useful in pruning. 14537 */ 14538 if (env->jmps_processed - env->prev_jmps_processed < 20 && 14539 env->insn_processed - env->prev_insn_processed < 100) 14540 add_new_state = false; 14541 goto miss; 14542 } 14543 if (states_equal(env, &sl->state, cur)) { 14544 sl->hit_cnt++; 14545 /* reached equivalent register/stack state, 14546 * prune the search. 14547 * Registers read by the continuation are read by us. 14548 * If we have any write marks in env->cur_state, they 14549 * will prevent corresponding reads in the continuation 14550 * from reaching our parent (an explored_state). Our 14551 * own state will get the read marks recorded, but 14552 * they'll be immediately forgotten as we're pruning 14553 * this state and will pop a new one. 14554 */ 14555 err = propagate_liveness(env, &sl->state, cur); 14556 14557 /* if previous state reached the exit with precision and 14558 * current state is equivalent to it (except precsion marks) 14559 * the precision needs to be propagated back in 14560 * the current state. 14561 */ 14562 err = err ? : push_jmp_history(env, cur); 14563 err = err ? : propagate_precision(env, &sl->state); 14564 if (err) 14565 return err; 14566 return 1; 14567 } 14568 miss: 14569 /* when new state is not going to be added do not increase miss count. 14570 * Otherwise several loop iterations will remove the state 14571 * recorded earlier. The goal of these heuristics is to have 14572 * states from some iterations of the loop (some in the beginning 14573 * and some at the end) to help pruning. 14574 */ 14575 if (add_new_state) 14576 sl->miss_cnt++; 14577 /* heuristic to determine whether this state is beneficial 14578 * to keep checking from state equivalence point of view. 14579 * Higher numbers increase max_states_per_insn and verification time, 14580 * but do not meaningfully decrease insn_processed. 14581 */ 14582 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 14583 /* the state is unlikely to be useful. Remove it to 14584 * speed up verification 14585 */ 14586 *pprev = sl->next; 14587 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 14588 u32 br = sl->state.branches; 14589 14590 WARN_ONCE(br, 14591 "BUG live_done but branches_to_explore %d\n", 14592 br); 14593 free_verifier_state(&sl->state, false); 14594 kfree(sl); 14595 env->peak_states--; 14596 } else { 14597 /* cannot free this state, since parentage chain may 14598 * walk it later. Add it for free_list instead to 14599 * be freed at the end of verification 14600 */ 14601 sl->next = env->free_list; 14602 env->free_list = sl; 14603 } 14604 sl = *pprev; 14605 continue; 14606 } 14607 next: 14608 pprev = &sl->next; 14609 sl = *pprev; 14610 } 14611 14612 if (env->max_states_per_insn < states_cnt) 14613 env->max_states_per_insn = states_cnt; 14614 14615 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 14616 return 0; 14617 14618 if (!add_new_state) 14619 return 0; 14620 14621 /* There were no equivalent states, remember the current one. 14622 * Technically the current state is not proven to be safe yet, 14623 * but it will either reach outer most bpf_exit (which means it's safe) 14624 * or it will be rejected. When there are no loops the verifier won't be 14625 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 14626 * again on the way to bpf_exit. 14627 * When looping the sl->state.branches will be > 0 and this state 14628 * will not be considered for equivalence until branches == 0. 14629 */ 14630 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 14631 if (!new_sl) 14632 return -ENOMEM; 14633 env->total_states++; 14634 env->peak_states++; 14635 env->prev_jmps_processed = env->jmps_processed; 14636 env->prev_insn_processed = env->insn_processed; 14637 14638 /* forget precise markings we inherited, see __mark_chain_precision */ 14639 if (env->bpf_capable) 14640 mark_all_scalars_imprecise(env, cur); 14641 14642 /* add new state to the head of linked list */ 14643 new = &new_sl->state; 14644 err = copy_verifier_state(new, cur); 14645 if (err) { 14646 free_verifier_state(new, false); 14647 kfree(new_sl); 14648 return err; 14649 } 14650 new->insn_idx = insn_idx; 14651 WARN_ONCE(new->branches != 1, 14652 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 14653 14654 cur->parent = new; 14655 cur->first_insn_idx = insn_idx; 14656 clear_jmp_history(cur); 14657 new_sl->next = *explored_state(env, insn_idx); 14658 *explored_state(env, insn_idx) = new_sl; 14659 /* connect new state to parentage chain. Current frame needs all 14660 * registers connected. Only r6 - r9 of the callers are alive (pushed 14661 * to the stack implicitly by JITs) so in callers' frames connect just 14662 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 14663 * the state of the call instruction (with WRITTEN set), and r0 comes 14664 * from callee with its full parentage chain, anyway. 14665 */ 14666 /* clear write marks in current state: the writes we did are not writes 14667 * our child did, so they don't screen off its reads from us. 14668 * (There are no read marks in current state, because reads always mark 14669 * their parent and current state never has children yet. Only 14670 * explored_states can get read marks.) 14671 */ 14672 for (j = 0; j <= cur->curframe; j++) { 14673 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 14674 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 14675 for (i = 0; i < BPF_REG_FP; i++) 14676 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 14677 } 14678 14679 /* all stack frames are accessible from callee, clear them all */ 14680 for (j = 0; j <= cur->curframe; j++) { 14681 struct bpf_func_state *frame = cur->frame[j]; 14682 struct bpf_func_state *newframe = new->frame[j]; 14683 14684 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 14685 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 14686 frame->stack[i].spilled_ptr.parent = 14687 &newframe->stack[i].spilled_ptr; 14688 } 14689 } 14690 return 0; 14691 } 14692 14693 /* Return true if it's OK to have the same insn return a different type. */ 14694 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 14695 { 14696 switch (base_type(type)) { 14697 case PTR_TO_CTX: 14698 case PTR_TO_SOCKET: 14699 case PTR_TO_SOCK_COMMON: 14700 case PTR_TO_TCP_SOCK: 14701 case PTR_TO_XDP_SOCK: 14702 case PTR_TO_BTF_ID: 14703 return false; 14704 default: 14705 return true; 14706 } 14707 } 14708 14709 /* If an instruction was previously used with particular pointer types, then we 14710 * need to be careful to avoid cases such as the below, where it may be ok 14711 * for one branch accessing the pointer, but not ok for the other branch: 14712 * 14713 * R1 = sock_ptr 14714 * goto X; 14715 * ... 14716 * R1 = some_other_valid_ptr; 14717 * goto X; 14718 * ... 14719 * R2 = *(u32 *)(R1 + 0); 14720 */ 14721 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 14722 { 14723 return src != prev && (!reg_type_mismatch_ok(src) || 14724 !reg_type_mismatch_ok(prev)); 14725 } 14726 14727 static int do_check(struct bpf_verifier_env *env) 14728 { 14729 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 14730 struct bpf_verifier_state *state = env->cur_state; 14731 struct bpf_insn *insns = env->prog->insnsi; 14732 struct bpf_reg_state *regs; 14733 int insn_cnt = env->prog->len; 14734 bool do_print_state = false; 14735 int prev_insn_idx = -1; 14736 14737 for (;;) { 14738 struct bpf_insn *insn; 14739 u8 class; 14740 int err; 14741 14742 env->prev_insn_idx = prev_insn_idx; 14743 if (env->insn_idx >= insn_cnt) { 14744 verbose(env, "invalid insn idx %d insn_cnt %d\n", 14745 env->insn_idx, insn_cnt); 14746 return -EFAULT; 14747 } 14748 14749 insn = &insns[env->insn_idx]; 14750 class = BPF_CLASS(insn->code); 14751 14752 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 14753 verbose(env, 14754 "BPF program is too large. Processed %d insn\n", 14755 env->insn_processed); 14756 return -E2BIG; 14757 } 14758 14759 state->last_insn_idx = env->prev_insn_idx; 14760 14761 if (is_prune_point(env, env->insn_idx)) { 14762 err = is_state_visited(env, env->insn_idx); 14763 if (err < 0) 14764 return err; 14765 if (err == 1) { 14766 /* found equivalent state, can prune the search */ 14767 if (env->log.level & BPF_LOG_LEVEL) { 14768 if (do_print_state) 14769 verbose(env, "\nfrom %d to %d%s: safe\n", 14770 env->prev_insn_idx, env->insn_idx, 14771 env->cur_state->speculative ? 14772 " (speculative execution)" : ""); 14773 else 14774 verbose(env, "%d: safe\n", env->insn_idx); 14775 } 14776 goto process_bpf_exit; 14777 } 14778 } 14779 14780 if (is_jmp_point(env, env->insn_idx)) { 14781 err = push_jmp_history(env, state); 14782 if (err) 14783 return err; 14784 } 14785 14786 if (signal_pending(current)) 14787 return -EAGAIN; 14788 14789 if (need_resched()) 14790 cond_resched(); 14791 14792 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 14793 verbose(env, "\nfrom %d to %d%s:", 14794 env->prev_insn_idx, env->insn_idx, 14795 env->cur_state->speculative ? 14796 " (speculative execution)" : ""); 14797 print_verifier_state(env, state->frame[state->curframe], true); 14798 do_print_state = false; 14799 } 14800 14801 if (env->log.level & BPF_LOG_LEVEL) { 14802 const struct bpf_insn_cbs cbs = { 14803 .cb_call = disasm_kfunc_name, 14804 .cb_print = verbose, 14805 .private_data = env, 14806 }; 14807 14808 if (verifier_state_scratched(env)) 14809 print_insn_state(env, state->frame[state->curframe]); 14810 14811 verbose_linfo(env, env->insn_idx, "; "); 14812 env->prev_log_len = env->log.len_used; 14813 verbose(env, "%d: ", env->insn_idx); 14814 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 14815 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; 14816 env->prev_log_len = env->log.len_used; 14817 } 14818 14819 if (bpf_prog_is_offloaded(env->prog->aux)) { 14820 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 14821 env->prev_insn_idx); 14822 if (err) 14823 return err; 14824 } 14825 14826 regs = cur_regs(env); 14827 sanitize_mark_insn_seen(env); 14828 prev_insn_idx = env->insn_idx; 14829 14830 if (class == BPF_ALU || class == BPF_ALU64) { 14831 err = check_alu_op(env, insn); 14832 if (err) 14833 return err; 14834 14835 } else if (class == BPF_LDX) { 14836 enum bpf_reg_type *prev_src_type, src_reg_type; 14837 14838 /* check for reserved fields is already done */ 14839 14840 /* check src operand */ 14841 err = check_reg_arg(env, insn->src_reg, SRC_OP); 14842 if (err) 14843 return err; 14844 14845 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 14846 if (err) 14847 return err; 14848 14849 src_reg_type = regs[insn->src_reg].type; 14850 14851 /* check that memory (src_reg + off) is readable, 14852 * the state of dst_reg will be updated by this func 14853 */ 14854 err = check_mem_access(env, env->insn_idx, insn->src_reg, 14855 insn->off, BPF_SIZE(insn->code), 14856 BPF_READ, insn->dst_reg, false); 14857 if (err) 14858 return err; 14859 14860 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 14861 14862 if (*prev_src_type == NOT_INIT) { 14863 /* saw a valid insn 14864 * dst_reg = *(u32 *)(src_reg + off) 14865 * save type to validate intersecting paths 14866 */ 14867 *prev_src_type = src_reg_type; 14868 14869 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 14870 /* ABuser program is trying to use the same insn 14871 * dst_reg = *(u32*) (src_reg + off) 14872 * with different pointer types: 14873 * src_reg == ctx in one branch and 14874 * src_reg == stack|map in some other branch. 14875 * Reject it. 14876 */ 14877 verbose(env, "same insn cannot be used with different pointers\n"); 14878 return -EINVAL; 14879 } 14880 14881 } else if (class == BPF_STX) { 14882 enum bpf_reg_type *prev_dst_type, dst_reg_type; 14883 14884 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 14885 err = check_atomic(env, env->insn_idx, insn); 14886 if (err) 14887 return err; 14888 env->insn_idx++; 14889 continue; 14890 } 14891 14892 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 14893 verbose(env, "BPF_STX uses reserved fields\n"); 14894 return -EINVAL; 14895 } 14896 14897 /* check src1 operand */ 14898 err = check_reg_arg(env, insn->src_reg, SRC_OP); 14899 if (err) 14900 return err; 14901 /* check src2 operand */ 14902 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 14903 if (err) 14904 return err; 14905 14906 dst_reg_type = regs[insn->dst_reg].type; 14907 14908 /* check that memory (dst_reg + off) is writeable */ 14909 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 14910 insn->off, BPF_SIZE(insn->code), 14911 BPF_WRITE, insn->src_reg, false); 14912 if (err) 14913 return err; 14914 14915 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 14916 14917 if (*prev_dst_type == NOT_INIT) { 14918 *prev_dst_type = dst_reg_type; 14919 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 14920 verbose(env, "same insn cannot be used with different pointers\n"); 14921 return -EINVAL; 14922 } 14923 14924 } else if (class == BPF_ST) { 14925 if (BPF_MODE(insn->code) != BPF_MEM || 14926 insn->src_reg != BPF_REG_0) { 14927 verbose(env, "BPF_ST uses reserved fields\n"); 14928 return -EINVAL; 14929 } 14930 /* check src operand */ 14931 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 14932 if (err) 14933 return err; 14934 14935 if (is_ctx_reg(env, insn->dst_reg)) { 14936 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 14937 insn->dst_reg, 14938 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 14939 return -EACCES; 14940 } 14941 14942 /* check that memory (dst_reg + off) is writeable */ 14943 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 14944 insn->off, BPF_SIZE(insn->code), 14945 BPF_WRITE, -1, false); 14946 if (err) 14947 return err; 14948 14949 } else if (class == BPF_JMP || class == BPF_JMP32) { 14950 u8 opcode = BPF_OP(insn->code); 14951 14952 env->jmps_processed++; 14953 if (opcode == BPF_CALL) { 14954 if (BPF_SRC(insn->code) != BPF_K || 14955 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 14956 && insn->off != 0) || 14957 (insn->src_reg != BPF_REG_0 && 14958 insn->src_reg != BPF_PSEUDO_CALL && 14959 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 14960 insn->dst_reg != BPF_REG_0 || 14961 class == BPF_JMP32) { 14962 verbose(env, "BPF_CALL uses reserved fields\n"); 14963 return -EINVAL; 14964 } 14965 14966 if (env->cur_state->active_lock.ptr) { 14967 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 14968 (insn->src_reg == BPF_PSEUDO_CALL) || 14969 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 14970 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) { 14971 verbose(env, "function calls are not allowed while holding a lock\n"); 14972 return -EINVAL; 14973 } 14974 } 14975 if (insn->src_reg == BPF_PSEUDO_CALL) 14976 err = check_func_call(env, insn, &env->insn_idx); 14977 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 14978 err = check_kfunc_call(env, insn, &env->insn_idx); 14979 else 14980 err = check_helper_call(env, insn, &env->insn_idx); 14981 if (err) 14982 return err; 14983 } else if (opcode == BPF_JA) { 14984 if (BPF_SRC(insn->code) != BPF_K || 14985 insn->imm != 0 || 14986 insn->src_reg != BPF_REG_0 || 14987 insn->dst_reg != BPF_REG_0 || 14988 class == BPF_JMP32) { 14989 verbose(env, "BPF_JA uses reserved fields\n"); 14990 return -EINVAL; 14991 } 14992 14993 env->insn_idx += insn->off + 1; 14994 continue; 14995 14996 } else if (opcode == BPF_EXIT) { 14997 if (BPF_SRC(insn->code) != BPF_K || 14998 insn->imm != 0 || 14999 insn->src_reg != BPF_REG_0 || 15000 insn->dst_reg != BPF_REG_0 || 15001 class == BPF_JMP32) { 15002 verbose(env, "BPF_EXIT uses reserved fields\n"); 15003 return -EINVAL; 15004 } 15005 15006 if (env->cur_state->active_lock.ptr && 15007 !in_rbtree_lock_required_cb(env)) { 15008 verbose(env, "bpf_spin_unlock is missing\n"); 15009 return -EINVAL; 15010 } 15011 15012 if (env->cur_state->active_rcu_lock) { 15013 verbose(env, "bpf_rcu_read_unlock is missing\n"); 15014 return -EINVAL; 15015 } 15016 15017 /* We must do check_reference_leak here before 15018 * prepare_func_exit to handle the case when 15019 * state->curframe > 0, it may be a callback 15020 * function, for which reference_state must 15021 * match caller reference state when it exits. 15022 */ 15023 err = check_reference_leak(env); 15024 if (err) 15025 return err; 15026 15027 if (state->curframe) { 15028 /* exit from nested function */ 15029 err = prepare_func_exit(env, &env->insn_idx); 15030 if (err) 15031 return err; 15032 do_print_state = true; 15033 continue; 15034 } 15035 15036 err = check_return_code(env); 15037 if (err) 15038 return err; 15039 process_bpf_exit: 15040 mark_verifier_state_scratched(env); 15041 update_branch_counts(env, env->cur_state); 15042 err = pop_stack(env, &prev_insn_idx, 15043 &env->insn_idx, pop_log); 15044 if (err < 0) { 15045 if (err != -ENOENT) 15046 return err; 15047 break; 15048 } else { 15049 do_print_state = true; 15050 continue; 15051 } 15052 } else { 15053 err = check_cond_jmp_op(env, insn, &env->insn_idx); 15054 if (err) 15055 return err; 15056 } 15057 } else if (class == BPF_LD) { 15058 u8 mode = BPF_MODE(insn->code); 15059 15060 if (mode == BPF_ABS || mode == BPF_IND) { 15061 err = check_ld_abs(env, insn); 15062 if (err) 15063 return err; 15064 15065 } else if (mode == BPF_IMM) { 15066 err = check_ld_imm(env, insn); 15067 if (err) 15068 return err; 15069 15070 env->insn_idx++; 15071 sanitize_mark_insn_seen(env); 15072 } else { 15073 verbose(env, "invalid BPF_LD mode\n"); 15074 return -EINVAL; 15075 } 15076 } else { 15077 verbose(env, "unknown insn class %d\n", class); 15078 return -EINVAL; 15079 } 15080 15081 env->insn_idx++; 15082 } 15083 15084 return 0; 15085 } 15086 15087 static int find_btf_percpu_datasec(struct btf *btf) 15088 { 15089 const struct btf_type *t; 15090 const char *tname; 15091 int i, n; 15092 15093 /* 15094 * Both vmlinux and module each have their own ".data..percpu" 15095 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 15096 * types to look at only module's own BTF types. 15097 */ 15098 n = btf_nr_types(btf); 15099 if (btf_is_module(btf)) 15100 i = btf_nr_types(btf_vmlinux); 15101 else 15102 i = 1; 15103 15104 for(; i < n; i++) { 15105 t = btf_type_by_id(btf, i); 15106 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 15107 continue; 15108 15109 tname = btf_name_by_offset(btf, t->name_off); 15110 if (!strcmp(tname, ".data..percpu")) 15111 return i; 15112 } 15113 15114 return -ENOENT; 15115 } 15116 15117 /* replace pseudo btf_id with kernel symbol address */ 15118 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 15119 struct bpf_insn *insn, 15120 struct bpf_insn_aux_data *aux) 15121 { 15122 const struct btf_var_secinfo *vsi; 15123 const struct btf_type *datasec; 15124 struct btf_mod_pair *btf_mod; 15125 const struct btf_type *t; 15126 const char *sym_name; 15127 bool percpu = false; 15128 u32 type, id = insn->imm; 15129 struct btf *btf; 15130 s32 datasec_id; 15131 u64 addr; 15132 int i, btf_fd, err; 15133 15134 btf_fd = insn[1].imm; 15135 if (btf_fd) { 15136 btf = btf_get_by_fd(btf_fd); 15137 if (IS_ERR(btf)) { 15138 verbose(env, "invalid module BTF object FD specified.\n"); 15139 return -EINVAL; 15140 } 15141 } else { 15142 if (!btf_vmlinux) { 15143 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 15144 return -EINVAL; 15145 } 15146 btf = btf_vmlinux; 15147 btf_get(btf); 15148 } 15149 15150 t = btf_type_by_id(btf, id); 15151 if (!t) { 15152 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 15153 err = -ENOENT; 15154 goto err_put; 15155 } 15156 15157 if (!btf_type_is_var(t)) { 15158 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 15159 err = -EINVAL; 15160 goto err_put; 15161 } 15162 15163 sym_name = btf_name_by_offset(btf, t->name_off); 15164 addr = kallsyms_lookup_name(sym_name); 15165 if (!addr) { 15166 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 15167 sym_name); 15168 err = -ENOENT; 15169 goto err_put; 15170 } 15171 15172 datasec_id = find_btf_percpu_datasec(btf); 15173 if (datasec_id > 0) { 15174 datasec = btf_type_by_id(btf, datasec_id); 15175 for_each_vsi(i, datasec, vsi) { 15176 if (vsi->type == id) { 15177 percpu = true; 15178 break; 15179 } 15180 } 15181 } 15182 15183 insn[0].imm = (u32)addr; 15184 insn[1].imm = addr >> 32; 15185 15186 type = t->type; 15187 t = btf_type_skip_modifiers(btf, type, NULL); 15188 if (percpu) { 15189 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 15190 aux->btf_var.btf = btf; 15191 aux->btf_var.btf_id = type; 15192 } else if (!btf_type_is_struct(t)) { 15193 const struct btf_type *ret; 15194 const char *tname; 15195 u32 tsize; 15196 15197 /* resolve the type size of ksym. */ 15198 ret = btf_resolve_size(btf, t, &tsize); 15199 if (IS_ERR(ret)) { 15200 tname = btf_name_by_offset(btf, t->name_off); 15201 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 15202 tname, PTR_ERR(ret)); 15203 err = -EINVAL; 15204 goto err_put; 15205 } 15206 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 15207 aux->btf_var.mem_size = tsize; 15208 } else { 15209 aux->btf_var.reg_type = PTR_TO_BTF_ID; 15210 aux->btf_var.btf = btf; 15211 aux->btf_var.btf_id = type; 15212 } 15213 15214 /* check whether we recorded this BTF (and maybe module) already */ 15215 for (i = 0; i < env->used_btf_cnt; i++) { 15216 if (env->used_btfs[i].btf == btf) { 15217 btf_put(btf); 15218 return 0; 15219 } 15220 } 15221 15222 if (env->used_btf_cnt >= MAX_USED_BTFS) { 15223 err = -E2BIG; 15224 goto err_put; 15225 } 15226 15227 btf_mod = &env->used_btfs[env->used_btf_cnt]; 15228 btf_mod->btf = btf; 15229 btf_mod->module = NULL; 15230 15231 /* if we reference variables from kernel module, bump its refcount */ 15232 if (btf_is_module(btf)) { 15233 btf_mod->module = btf_try_get_module(btf); 15234 if (!btf_mod->module) { 15235 err = -ENXIO; 15236 goto err_put; 15237 } 15238 } 15239 15240 env->used_btf_cnt++; 15241 15242 return 0; 15243 err_put: 15244 btf_put(btf); 15245 return err; 15246 } 15247 15248 static bool is_tracing_prog_type(enum bpf_prog_type type) 15249 { 15250 switch (type) { 15251 case BPF_PROG_TYPE_KPROBE: 15252 case BPF_PROG_TYPE_TRACEPOINT: 15253 case BPF_PROG_TYPE_PERF_EVENT: 15254 case BPF_PROG_TYPE_RAW_TRACEPOINT: 15255 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 15256 return true; 15257 default: 15258 return false; 15259 } 15260 } 15261 15262 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 15263 struct bpf_map *map, 15264 struct bpf_prog *prog) 15265 15266 { 15267 enum bpf_prog_type prog_type = resolve_prog_type(prog); 15268 15269 if (btf_record_has_field(map->record, BPF_LIST_HEAD) || 15270 btf_record_has_field(map->record, BPF_RB_ROOT)) { 15271 if (is_tracing_prog_type(prog_type)) { 15272 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); 15273 return -EINVAL; 15274 } 15275 } 15276 15277 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 15278 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 15279 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 15280 return -EINVAL; 15281 } 15282 15283 if (is_tracing_prog_type(prog_type)) { 15284 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 15285 return -EINVAL; 15286 } 15287 15288 if (prog->aux->sleepable) { 15289 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 15290 return -EINVAL; 15291 } 15292 } 15293 15294 if (btf_record_has_field(map->record, BPF_TIMER)) { 15295 if (is_tracing_prog_type(prog_type)) { 15296 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 15297 return -EINVAL; 15298 } 15299 } 15300 15301 if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && 15302 !bpf_offload_prog_map_match(prog, map)) { 15303 verbose(env, "offload device mismatch between prog and map\n"); 15304 return -EINVAL; 15305 } 15306 15307 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 15308 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 15309 return -EINVAL; 15310 } 15311 15312 if (prog->aux->sleepable) 15313 switch (map->map_type) { 15314 case BPF_MAP_TYPE_HASH: 15315 case BPF_MAP_TYPE_LRU_HASH: 15316 case BPF_MAP_TYPE_ARRAY: 15317 case BPF_MAP_TYPE_PERCPU_HASH: 15318 case BPF_MAP_TYPE_PERCPU_ARRAY: 15319 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 15320 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 15321 case BPF_MAP_TYPE_HASH_OF_MAPS: 15322 case BPF_MAP_TYPE_RINGBUF: 15323 case BPF_MAP_TYPE_USER_RINGBUF: 15324 case BPF_MAP_TYPE_INODE_STORAGE: 15325 case BPF_MAP_TYPE_SK_STORAGE: 15326 case BPF_MAP_TYPE_TASK_STORAGE: 15327 case BPF_MAP_TYPE_CGRP_STORAGE: 15328 break; 15329 default: 15330 verbose(env, 15331 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 15332 return -EINVAL; 15333 } 15334 15335 return 0; 15336 } 15337 15338 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 15339 { 15340 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 15341 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 15342 } 15343 15344 /* find and rewrite pseudo imm in ld_imm64 instructions: 15345 * 15346 * 1. if it accesses map FD, replace it with actual map pointer. 15347 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 15348 * 15349 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 15350 */ 15351 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 15352 { 15353 struct bpf_insn *insn = env->prog->insnsi; 15354 int insn_cnt = env->prog->len; 15355 int i, j, err; 15356 15357 err = bpf_prog_calc_tag(env->prog); 15358 if (err) 15359 return err; 15360 15361 for (i = 0; i < insn_cnt; i++, insn++) { 15362 if (BPF_CLASS(insn->code) == BPF_LDX && 15363 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 15364 verbose(env, "BPF_LDX uses reserved fields\n"); 15365 return -EINVAL; 15366 } 15367 15368 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 15369 struct bpf_insn_aux_data *aux; 15370 struct bpf_map *map; 15371 struct fd f; 15372 u64 addr; 15373 u32 fd; 15374 15375 if (i == insn_cnt - 1 || insn[1].code != 0 || 15376 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 15377 insn[1].off != 0) { 15378 verbose(env, "invalid bpf_ld_imm64 insn\n"); 15379 return -EINVAL; 15380 } 15381 15382 if (insn[0].src_reg == 0) 15383 /* valid generic load 64-bit imm */ 15384 goto next_insn; 15385 15386 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 15387 aux = &env->insn_aux_data[i]; 15388 err = check_pseudo_btf_id(env, insn, aux); 15389 if (err) 15390 return err; 15391 goto next_insn; 15392 } 15393 15394 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 15395 aux = &env->insn_aux_data[i]; 15396 aux->ptr_type = PTR_TO_FUNC; 15397 goto next_insn; 15398 } 15399 15400 /* In final convert_pseudo_ld_imm64() step, this is 15401 * converted into regular 64-bit imm load insn. 15402 */ 15403 switch (insn[0].src_reg) { 15404 case BPF_PSEUDO_MAP_VALUE: 15405 case BPF_PSEUDO_MAP_IDX_VALUE: 15406 break; 15407 case BPF_PSEUDO_MAP_FD: 15408 case BPF_PSEUDO_MAP_IDX: 15409 if (insn[1].imm == 0) 15410 break; 15411 fallthrough; 15412 default: 15413 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 15414 return -EINVAL; 15415 } 15416 15417 switch (insn[0].src_reg) { 15418 case BPF_PSEUDO_MAP_IDX_VALUE: 15419 case BPF_PSEUDO_MAP_IDX: 15420 if (bpfptr_is_null(env->fd_array)) { 15421 verbose(env, "fd_idx without fd_array is invalid\n"); 15422 return -EPROTO; 15423 } 15424 if (copy_from_bpfptr_offset(&fd, env->fd_array, 15425 insn[0].imm * sizeof(fd), 15426 sizeof(fd))) 15427 return -EFAULT; 15428 break; 15429 default: 15430 fd = insn[0].imm; 15431 break; 15432 } 15433 15434 f = fdget(fd); 15435 map = __bpf_map_get(f); 15436 if (IS_ERR(map)) { 15437 verbose(env, "fd %d is not pointing to valid bpf_map\n", 15438 insn[0].imm); 15439 return PTR_ERR(map); 15440 } 15441 15442 err = check_map_prog_compatibility(env, map, env->prog); 15443 if (err) { 15444 fdput(f); 15445 return err; 15446 } 15447 15448 aux = &env->insn_aux_data[i]; 15449 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 15450 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 15451 addr = (unsigned long)map; 15452 } else { 15453 u32 off = insn[1].imm; 15454 15455 if (off >= BPF_MAX_VAR_OFF) { 15456 verbose(env, "direct value offset of %u is not allowed\n", off); 15457 fdput(f); 15458 return -EINVAL; 15459 } 15460 15461 if (!map->ops->map_direct_value_addr) { 15462 verbose(env, "no direct value access support for this map type\n"); 15463 fdput(f); 15464 return -EINVAL; 15465 } 15466 15467 err = map->ops->map_direct_value_addr(map, &addr, off); 15468 if (err) { 15469 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 15470 map->value_size, off); 15471 fdput(f); 15472 return err; 15473 } 15474 15475 aux->map_off = off; 15476 addr += off; 15477 } 15478 15479 insn[0].imm = (u32)addr; 15480 insn[1].imm = addr >> 32; 15481 15482 /* check whether we recorded this map already */ 15483 for (j = 0; j < env->used_map_cnt; j++) { 15484 if (env->used_maps[j] == map) { 15485 aux->map_index = j; 15486 fdput(f); 15487 goto next_insn; 15488 } 15489 } 15490 15491 if (env->used_map_cnt >= MAX_USED_MAPS) { 15492 fdput(f); 15493 return -E2BIG; 15494 } 15495 15496 /* hold the map. If the program is rejected by verifier, 15497 * the map will be released by release_maps() or it 15498 * will be used by the valid program until it's unloaded 15499 * and all maps are released in free_used_maps() 15500 */ 15501 bpf_map_inc(map); 15502 15503 aux->map_index = env->used_map_cnt; 15504 env->used_maps[env->used_map_cnt++] = map; 15505 15506 if (bpf_map_is_cgroup_storage(map) && 15507 bpf_cgroup_storage_assign(env->prog->aux, map)) { 15508 verbose(env, "only one cgroup storage of each type is allowed\n"); 15509 fdput(f); 15510 return -EBUSY; 15511 } 15512 15513 fdput(f); 15514 next_insn: 15515 insn++; 15516 i++; 15517 continue; 15518 } 15519 15520 /* Basic sanity check before we invest more work here. */ 15521 if (!bpf_opcode_in_insntable(insn->code)) { 15522 verbose(env, "unknown opcode %02x\n", insn->code); 15523 return -EINVAL; 15524 } 15525 } 15526 15527 /* now all pseudo BPF_LD_IMM64 instructions load valid 15528 * 'struct bpf_map *' into a register instead of user map_fd. 15529 * These pointers will be used later by verifier to validate map access. 15530 */ 15531 return 0; 15532 } 15533 15534 /* drop refcnt of maps used by the rejected program */ 15535 static void release_maps(struct bpf_verifier_env *env) 15536 { 15537 __bpf_free_used_maps(env->prog->aux, env->used_maps, 15538 env->used_map_cnt); 15539 } 15540 15541 /* drop refcnt of maps used by the rejected program */ 15542 static void release_btfs(struct bpf_verifier_env *env) 15543 { 15544 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 15545 env->used_btf_cnt); 15546 } 15547 15548 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 15549 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 15550 { 15551 struct bpf_insn *insn = env->prog->insnsi; 15552 int insn_cnt = env->prog->len; 15553 int i; 15554 15555 for (i = 0; i < insn_cnt; i++, insn++) { 15556 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 15557 continue; 15558 if (insn->src_reg == BPF_PSEUDO_FUNC) 15559 continue; 15560 insn->src_reg = 0; 15561 } 15562 } 15563 15564 /* single env->prog->insni[off] instruction was replaced with the range 15565 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 15566 * [0, off) and [off, end) to new locations, so the patched range stays zero 15567 */ 15568 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 15569 struct bpf_insn_aux_data *new_data, 15570 struct bpf_prog *new_prog, u32 off, u32 cnt) 15571 { 15572 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 15573 struct bpf_insn *insn = new_prog->insnsi; 15574 u32 old_seen = old_data[off].seen; 15575 u32 prog_len; 15576 int i; 15577 15578 /* aux info at OFF always needs adjustment, no matter fast path 15579 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 15580 * original insn at old prog. 15581 */ 15582 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 15583 15584 if (cnt == 1) 15585 return; 15586 prog_len = new_prog->len; 15587 15588 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 15589 memcpy(new_data + off + cnt - 1, old_data + off, 15590 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 15591 for (i = off; i < off + cnt - 1; i++) { 15592 /* Expand insni[off]'s seen count to the patched range. */ 15593 new_data[i].seen = old_seen; 15594 new_data[i].zext_dst = insn_has_def32(env, insn + i); 15595 } 15596 env->insn_aux_data = new_data; 15597 vfree(old_data); 15598 } 15599 15600 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 15601 { 15602 int i; 15603 15604 if (len == 1) 15605 return; 15606 /* NOTE: fake 'exit' subprog should be updated as well. */ 15607 for (i = 0; i <= env->subprog_cnt; i++) { 15608 if (env->subprog_info[i].start <= off) 15609 continue; 15610 env->subprog_info[i].start += len - 1; 15611 } 15612 } 15613 15614 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 15615 { 15616 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 15617 int i, sz = prog->aux->size_poke_tab; 15618 struct bpf_jit_poke_descriptor *desc; 15619 15620 for (i = 0; i < sz; i++) { 15621 desc = &tab[i]; 15622 if (desc->insn_idx <= off) 15623 continue; 15624 desc->insn_idx += len - 1; 15625 } 15626 } 15627 15628 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 15629 const struct bpf_insn *patch, u32 len) 15630 { 15631 struct bpf_prog *new_prog; 15632 struct bpf_insn_aux_data *new_data = NULL; 15633 15634 if (len > 1) { 15635 new_data = vzalloc(array_size(env->prog->len + len - 1, 15636 sizeof(struct bpf_insn_aux_data))); 15637 if (!new_data) 15638 return NULL; 15639 } 15640 15641 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 15642 if (IS_ERR(new_prog)) { 15643 if (PTR_ERR(new_prog) == -ERANGE) 15644 verbose(env, 15645 "insn %d cannot be patched due to 16-bit range\n", 15646 env->insn_aux_data[off].orig_idx); 15647 vfree(new_data); 15648 return NULL; 15649 } 15650 adjust_insn_aux_data(env, new_data, new_prog, off, len); 15651 adjust_subprog_starts(env, off, len); 15652 adjust_poke_descs(new_prog, off, len); 15653 return new_prog; 15654 } 15655 15656 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 15657 u32 off, u32 cnt) 15658 { 15659 int i, j; 15660 15661 /* find first prog starting at or after off (first to remove) */ 15662 for (i = 0; i < env->subprog_cnt; i++) 15663 if (env->subprog_info[i].start >= off) 15664 break; 15665 /* find first prog starting at or after off + cnt (first to stay) */ 15666 for (j = i; j < env->subprog_cnt; j++) 15667 if (env->subprog_info[j].start >= off + cnt) 15668 break; 15669 /* if j doesn't start exactly at off + cnt, we are just removing 15670 * the front of previous prog 15671 */ 15672 if (env->subprog_info[j].start != off + cnt) 15673 j--; 15674 15675 if (j > i) { 15676 struct bpf_prog_aux *aux = env->prog->aux; 15677 int move; 15678 15679 /* move fake 'exit' subprog as well */ 15680 move = env->subprog_cnt + 1 - j; 15681 15682 memmove(env->subprog_info + i, 15683 env->subprog_info + j, 15684 sizeof(*env->subprog_info) * move); 15685 env->subprog_cnt -= j - i; 15686 15687 /* remove func_info */ 15688 if (aux->func_info) { 15689 move = aux->func_info_cnt - j; 15690 15691 memmove(aux->func_info + i, 15692 aux->func_info + j, 15693 sizeof(*aux->func_info) * move); 15694 aux->func_info_cnt -= j - i; 15695 /* func_info->insn_off is set after all code rewrites, 15696 * in adjust_btf_func() - no need to adjust 15697 */ 15698 } 15699 } else { 15700 /* convert i from "first prog to remove" to "first to adjust" */ 15701 if (env->subprog_info[i].start == off) 15702 i++; 15703 } 15704 15705 /* update fake 'exit' subprog as well */ 15706 for (; i <= env->subprog_cnt; i++) 15707 env->subprog_info[i].start -= cnt; 15708 15709 return 0; 15710 } 15711 15712 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 15713 u32 cnt) 15714 { 15715 struct bpf_prog *prog = env->prog; 15716 u32 i, l_off, l_cnt, nr_linfo; 15717 struct bpf_line_info *linfo; 15718 15719 nr_linfo = prog->aux->nr_linfo; 15720 if (!nr_linfo) 15721 return 0; 15722 15723 linfo = prog->aux->linfo; 15724 15725 /* find first line info to remove, count lines to be removed */ 15726 for (i = 0; i < nr_linfo; i++) 15727 if (linfo[i].insn_off >= off) 15728 break; 15729 15730 l_off = i; 15731 l_cnt = 0; 15732 for (; i < nr_linfo; i++) 15733 if (linfo[i].insn_off < off + cnt) 15734 l_cnt++; 15735 else 15736 break; 15737 15738 /* First live insn doesn't match first live linfo, it needs to "inherit" 15739 * last removed linfo. prog is already modified, so prog->len == off 15740 * means no live instructions after (tail of the program was removed). 15741 */ 15742 if (prog->len != off && l_cnt && 15743 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 15744 l_cnt--; 15745 linfo[--i].insn_off = off + cnt; 15746 } 15747 15748 /* remove the line info which refer to the removed instructions */ 15749 if (l_cnt) { 15750 memmove(linfo + l_off, linfo + i, 15751 sizeof(*linfo) * (nr_linfo - i)); 15752 15753 prog->aux->nr_linfo -= l_cnt; 15754 nr_linfo = prog->aux->nr_linfo; 15755 } 15756 15757 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 15758 for (i = l_off; i < nr_linfo; i++) 15759 linfo[i].insn_off -= cnt; 15760 15761 /* fix up all subprogs (incl. 'exit') which start >= off */ 15762 for (i = 0; i <= env->subprog_cnt; i++) 15763 if (env->subprog_info[i].linfo_idx > l_off) { 15764 /* program may have started in the removed region but 15765 * may not be fully removed 15766 */ 15767 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 15768 env->subprog_info[i].linfo_idx -= l_cnt; 15769 else 15770 env->subprog_info[i].linfo_idx = l_off; 15771 } 15772 15773 return 0; 15774 } 15775 15776 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 15777 { 15778 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 15779 unsigned int orig_prog_len = env->prog->len; 15780 int err; 15781 15782 if (bpf_prog_is_offloaded(env->prog->aux)) 15783 bpf_prog_offload_remove_insns(env, off, cnt); 15784 15785 err = bpf_remove_insns(env->prog, off, cnt); 15786 if (err) 15787 return err; 15788 15789 err = adjust_subprog_starts_after_remove(env, off, cnt); 15790 if (err) 15791 return err; 15792 15793 err = bpf_adj_linfo_after_remove(env, off, cnt); 15794 if (err) 15795 return err; 15796 15797 memmove(aux_data + off, aux_data + off + cnt, 15798 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 15799 15800 return 0; 15801 } 15802 15803 /* The verifier does more data flow analysis than llvm and will not 15804 * explore branches that are dead at run time. Malicious programs can 15805 * have dead code too. Therefore replace all dead at-run-time code 15806 * with 'ja -1'. 15807 * 15808 * Just nops are not optimal, e.g. if they would sit at the end of the 15809 * program and through another bug we would manage to jump there, then 15810 * we'd execute beyond program memory otherwise. Returning exception 15811 * code also wouldn't work since we can have subprogs where the dead 15812 * code could be located. 15813 */ 15814 static void sanitize_dead_code(struct bpf_verifier_env *env) 15815 { 15816 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 15817 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 15818 struct bpf_insn *insn = env->prog->insnsi; 15819 const int insn_cnt = env->prog->len; 15820 int i; 15821 15822 for (i = 0; i < insn_cnt; i++) { 15823 if (aux_data[i].seen) 15824 continue; 15825 memcpy(insn + i, &trap, sizeof(trap)); 15826 aux_data[i].zext_dst = false; 15827 } 15828 } 15829 15830 static bool insn_is_cond_jump(u8 code) 15831 { 15832 u8 op; 15833 15834 if (BPF_CLASS(code) == BPF_JMP32) 15835 return true; 15836 15837 if (BPF_CLASS(code) != BPF_JMP) 15838 return false; 15839 15840 op = BPF_OP(code); 15841 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 15842 } 15843 15844 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 15845 { 15846 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 15847 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 15848 struct bpf_insn *insn = env->prog->insnsi; 15849 const int insn_cnt = env->prog->len; 15850 int i; 15851 15852 for (i = 0; i < insn_cnt; i++, insn++) { 15853 if (!insn_is_cond_jump(insn->code)) 15854 continue; 15855 15856 if (!aux_data[i + 1].seen) 15857 ja.off = insn->off; 15858 else if (!aux_data[i + 1 + insn->off].seen) 15859 ja.off = 0; 15860 else 15861 continue; 15862 15863 if (bpf_prog_is_offloaded(env->prog->aux)) 15864 bpf_prog_offload_replace_insn(env, i, &ja); 15865 15866 memcpy(insn, &ja, sizeof(ja)); 15867 } 15868 } 15869 15870 static int opt_remove_dead_code(struct bpf_verifier_env *env) 15871 { 15872 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 15873 int insn_cnt = env->prog->len; 15874 int i, err; 15875 15876 for (i = 0; i < insn_cnt; i++) { 15877 int j; 15878 15879 j = 0; 15880 while (i + j < insn_cnt && !aux_data[i + j].seen) 15881 j++; 15882 if (!j) 15883 continue; 15884 15885 err = verifier_remove_insns(env, i, j); 15886 if (err) 15887 return err; 15888 insn_cnt = env->prog->len; 15889 } 15890 15891 return 0; 15892 } 15893 15894 static int opt_remove_nops(struct bpf_verifier_env *env) 15895 { 15896 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 15897 struct bpf_insn *insn = env->prog->insnsi; 15898 int insn_cnt = env->prog->len; 15899 int i, err; 15900 15901 for (i = 0; i < insn_cnt; i++) { 15902 if (memcmp(&insn[i], &ja, sizeof(ja))) 15903 continue; 15904 15905 err = verifier_remove_insns(env, i, 1); 15906 if (err) 15907 return err; 15908 insn_cnt--; 15909 i--; 15910 } 15911 15912 return 0; 15913 } 15914 15915 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 15916 const union bpf_attr *attr) 15917 { 15918 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 15919 struct bpf_insn_aux_data *aux = env->insn_aux_data; 15920 int i, patch_len, delta = 0, len = env->prog->len; 15921 struct bpf_insn *insns = env->prog->insnsi; 15922 struct bpf_prog *new_prog; 15923 bool rnd_hi32; 15924 15925 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 15926 zext_patch[1] = BPF_ZEXT_REG(0); 15927 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 15928 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 15929 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 15930 for (i = 0; i < len; i++) { 15931 int adj_idx = i + delta; 15932 struct bpf_insn insn; 15933 int load_reg; 15934 15935 insn = insns[adj_idx]; 15936 load_reg = insn_def_regno(&insn); 15937 if (!aux[adj_idx].zext_dst) { 15938 u8 code, class; 15939 u32 imm_rnd; 15940 15941 if (!rnd_hi32) 15942 continue; 15943 15944 code = insn.code; 15945 class = BPF_CLASS(code); 15946 if (load_reg == -1) 15947 continue; 15948 15949 /* NOTE: arg "reg" (the fourth one) is only used for 15950 * BPF_STX + SRC_OP, so it is safe to pass NULL 15951 * here. 15952 */ 15953 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 15954 if (class == BPF_LD && 15955 BPF_MODE(code) == BPF_IMM) 15956 i++; 15957 continue; 15958 } 15959 15960 /* ctx load could be transformed into wider load. */ 15961 if (class == BPF_LDX && 15962 aux[adj_idx].ptr_type == PTR_TO_CTX) 15963 continue; 15964 15965 imm_rnd = get_random_u32(); 15966 rnd_hi32_patch[0] = insn; 15967 rnd_hi32_patch[1].imm = imm_rnd; 15968 rnd_hi32_patch[3].dst_reg = load_reg; 15969 patch = rnd_hi32_patch; 15970 patch_len = 4; 15971 goto apply_patch_buffer; 15972 } 15973 15974 /* Add in an zero-extend instruction if a) the JIT has requested 15975 * it or b) it's a CMPXCHG. 15976 * 15977 * The latter is because: BPF_CMPXCHG always loads a value into 15978 * R0, therefore always zero-extends. However some archs' 15979 * equivalent instruction only does this load when the 15980 * comparison is successful. This detail of CMPXCHG is 15981 * orthogonal to the general zero-extension behaviour of the 15982 * CPU, so it's treated independently of bpf_jit_needs_zext. 15983 */ 15984 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 15985 continue; 15986 15987 /* Zero-extension is done by the caller. */ 15988 if (bpf_pseudo_kfunc_call(&insn)) 15989 continue; 15990 15991 if (WARN_ON(load_reg == -1)) { 15992 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 15993 return -EFAULT; 15994 } 15995 15996 zext_patch[0] = insn; 15997 zext_patch[1].dst_reg = load_reg; 15998 zext_patch[1].src_reg = load_reg; 15999 patch = zext_patch; 16000 patch_len = 2; 16001 apply_patch_buffer: 16002 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 16003 if (!new_prog) 16004 return -ENOMEM; 16005 env->prog = new_prog; 16006 insns = new_prog->insnsi; 16007 aux = env->insn_aux_data; 16008 delta += patch_len - 1; 16009 } 16010 16011 return 0; 16012 } 16013 16014 /* convert load instructions that access fields of a context type into a 16015 * sequence of instructions that access fields of the underlying structure: 16016 * struct __sk_buff -> struct sk_buff 16017 * struct bpf_sock_ops -> struct sock 16018 */ 16019 static int convert_ctx_accesses(struct bpf_verifier_env *env) 16020 { 16021 const struct bpf_verifier_ops *ops = env->ops; 16022 int i, cnt, size, ctx_field_size, delta = 0; 16023 const int insn_cnt = env->prog->len; 16024 struct bpf_insn insn_buf[16], *insn; 16025 u32 target_size, size_default, off; 16026 struct bpf_prog *new_prog; 16027 enum bpf_access_type type; 16028 bool is_narrower_load; 16029 16030 if (ops->gen_prologue || env->seen_direct_write) { 16031 if (!ops->gen_prologue) { 16032 verbose(env, "bpf verifier is misconfigured\n"); 16033 return -EINVAL; 16034 } 16035 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 16036 env->prog); 16037 if (cnt >= ARRAY_SIZE(insn_buf)) { 16038 verbose(env, "bpf verifier is misconfigured\n"); 16039 return -EINVAL; 16040 } else if (cnt) { 16041 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 16042 if (!new_prog) 16043 return -ENOMEM; 16044 16045 env->prog = new_prog; 16046 delta += cnt - 1; 16047 } 16048 } 16049 16050 if (bpf_prog_is_offloaded(env->prog->aux)) 16051 return 0; 16052 16053 insn = env->prog->insnsi + delta; 16054 16055 for (i = 0; i < insn_cnt; i++, insn++) { 16056 bpf_convert_ctx_access_t convert_ctx_access; 16057 bool ctx_access; 16058 16059 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 16060 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 16061 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 16062 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 16063 type = BPF_READ; 16064 ctx_access = true; 16065 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 16066 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 16067 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 16068 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 16069 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 16070 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 16071 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 16072 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 16073 type = BPF_WRITE; 16074 ctx_access = BPF_CLASS(insn->code) == BPF_STX; 16075 } else { 16076 continue; 16077 } 16078 16079 if (type == BPF_WRITE && 16080 env->insn_aux_data[i + delta].sanitize_stack_spill) { 16081 struct bpf_insn patch[] = { 16082 *insn, 16083 BPF_ST_NOSPEC(), 16084 }; 16085 16086 cnt = ARRAY_SIZE(patch); 16087 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 16088 if (!new_prog) 16089 return -ENOMEM; 16090 16091 delta += cnt - 1; 16092 env->prog = new_prog; 16093 insn = new_prog->insnsi + i + delta; 16094 continue; 16095 } 16096 16097 if (!ctx_access) 16098 continue; 16099 16100 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 16101 case PTR_TO_CTX: 16102 if (!ops->convert_ctx_access) 16103 continue; 16104 convert_ctx_access = ops->convert_ctx_access; 16105 break; 16106 case PTR_TO_SOCKET: 16107 case PTR_TO_SOCK_COMMON: 16108 convert_ctx_access = bpf_sock_convert_ctx_access; 16109 break; 16110 case PTR_TO_TCP_SOCK: 16111 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 16112 break; 16113 case PTR_TO_XDP_SOCK: 16114 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 16115 break; 16116 case PTR_TO_BTF_ID: 16117 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 16118 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 16119 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot 16120 * be said once it is marked PTR_UNTRUSTED, hence we must handle 16121 * any faults for loads into such types. BPF_WRITE is disallowed 16122 * for this case. 16123 */ 16124 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 16125 if (type == BPF_READ) { 16126 insn->code = BPF_LDX | BPF_PROBE_MEM | 16127 BPF_SIZE((insn)->code); 16128 env->prog->aux->num_exentries++; 16129 } 16130 continue; 16131 default: 16132 continue; 16133 } 16134 16135 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 16136 size = BPF_LDST_BYTES(insn); 16137 16138 /* If the read access is a narrower load of the field, 16139 * convert to a 4/8-byte load, to minimum program type specific 16140 * convert_ctx_access changes. If conversion is successful, 16141 * we will apply proper mask to the result. 16142 */ 16143 is_narrower_load = size < ctx_field_size; 16144 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 16145 off = insn->off; 16146 if (is_narrower_load) { 16147 u8 size_code; 16148 16149 if (type == BPF_WRITE) { 16150 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 16151 return -EINVAL; 16152 } 16153 16154 size_code = BPF_H; 16155 if (ctx_field_size == 4) 16156 size_code = BPF_W; 16157 else if (ctx_field_size == 8) 16158 size_code = BPF_DW; 16159 16160 insn->off = off & ~(size_default - 1); 16161 insn->code = BPF_LDX | BPF_MEM | size_code; 16162 } 16163 16164 target_size = 0; 16165 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 16166 &target_size); 16167 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 16168 (ctx_field_size && !target_size)) { 16169 verbose(env, "bpf verifier is misconfigured\n"); 16170 return -EINVAL; 16171 } 16172 16173 if (is_narrower_load && size < target_size) { 16174 u8 shift = bpf_ctx_narrow_access_offset( 16175 off, size, size_default) * 8; 16176 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 16177 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 16178 return -EINVAL; 16179 } 16180 if (ctx_field_size <= 4) { 16181 if (shift) 16182 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 16183 insn->dst_reg, 16184 shift); 16185 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 16186 (1 << size * 8) - 1); 16187 } else { 16188 if (shift) 16189 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 16190 insn->dst_reg, 16191 shift); 16192 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 16193 (1ULL << size * 8) - 1); 16194 } 16195 } 16196 16197 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16198 if (!new_prog) 16199 return -ENOMEM; 16200 16201 delta += cnt - 1; 16202 16203 /* keep walking new program and skip insns we just inserted */ 16204 env->prog = new_prog; 16205 insn = new_prog->insnsi + i + delta; 16206 } 16207 16208 return 0; 16209 } 16210 16211 static int jit_subprogs(struct bpf_verifier_env *env) 16212 { 16213 struct bpf_prog *prog = env->prog, **func, *tmp; 16214 int i, j, subprog_start, subprog_end = 0, len, subprog; 16215 struct bpf_map *map_ptr; 16216 struct bpf_insn *insn; 16217 void *old_bpf_func; 16218 int err, num_exentries; 16219 16220 if (env->subprog_cnt <= 1) 16221 return 0; 16222 16223 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 16224 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 16225 continue; 16226 16227 /* Upon error here we cannot fall back to interpreter but 16228 * need a hard reject of the program. Thus -EFAULT is 16229 * propagated in any case. 16230 */ 16231 subprog = find_subprog(env, i + insn->imm + 1); 16232 if (subprog < 0) { 16233 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 16234 i + insn->imm + 1); 16235 return -EFAULT; 16236 } 16237 /* temporarily remember subprog id inside insn instead of 16238 * aux_data, since next loop will split up all insns into funcs 16239 */ 16240 insn->off = subprog; 16241 /* remember original imm in case JIT fails and fallback 16242 * to interpreter will be needed 16243 */ 16244 env->insn_aux_data[i].call_imm = insn->imm; 16245 /* point imm to __bpf_call_base+1 from JITs point of view */ 16246 insn->imm = 1; 16247 if (bpf_pseudo_func(insn)) 16248 /* jit (e.g. x86_64) may emit fewer instructions 16249 * if it learns a u32 imm is the same as a u64 imm. 16250 * Force a non zero here. 16251 */ 16252 insn[1].imm = 1; 16253 } 16254 16255 err = bpf_prog_alloc_jited_linfo(prog); 16256 if (err) 16257 goto out_undo_insn; 16258 16259 err = -ENOMEM; 16260 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 16261 if (!func) 16262 goto out_undo_insn; 16263 16264 for (i = 0; i < env->subprog_cnt; i++) { 16265 subprog_start = subprog_end; 16266 subprog_end = env->subprog_info[i + 1].start; 16267 16268 len = subprog_end - subprog_start; 16269 /* bpf_prog_run() doesn't call subprogs directly, 16270 * hence main prog stats include the runtime of subprogs. 16271 * subprogs don't have IDs and not reachable via prog_get_next_id 16272 * func[i]->stats will never be accessed and stays NULL 16273 */ 16274 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 16275 if (!func[i]) 16276 goto out_free; 16277 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 16278 len * sizeof(struct bpf_insn)); 16279 func[i]->type = prog->type; 16280 func[i]->len = len; 16281 if (bpf_prog_calc_tag(func[i])) 16282 goto out_free; 16283 func[i]->is_func = 1; 16284 func[i]->aux->func_idx = i; 16285 /* Below members will be freed only at prog->aux */ 16286 func[i]->aux->btf = prog->aux->btf; 16287 func[i]->aux->func_info = prog->aux->func_info; 16288 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 16289 func[i]->aux->poke_tab = prog->aux->poke_tab; 16290 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 16291 16292 for (j = 0; j < prog->aux->size_poke_tab; j++) { 16293 struct bpf_jit_poke_descriptor *poke; 16294 16295 poke = &prog->aux->poke_tab[j]; 16296 if (poke->insn_idx < subprog_end && 16297 poke->insn_idx >= subprog_start) 16298 poke->aux = func[i]->aux; 16299 } 16300 16301 func[i]->aux->name[0] = 'F'; 16302 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 16303 func[i]->jit_requested = 1; 16304 func[i]->blinding_requested = prog->blinding_requested; 16305 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 16306 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 16307 func[i]->aux->linfo = prog->aux->linfo; 16308 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 16309 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 16310 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 16311 num_exentries = 0; 16312 insn = func[i]->insnsi; 16313 for (j = 0; j < func[i]->len; j++, insn++) { 16314 if (BPF_CLASS(insn->code) == BPF_LDX && 16315 BPF_MODE(insn->code) == BPF_PROBE_MEM) 16316 num_exentries++; 16317 } 16318 func[i]->aux->num_exentries = num_exentries; 16319 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 16320 func[i] = bpf_int_jit_compile(func[i]); 16321 if (!func[i]->jited) { 16322 err = -ENOTSUPP; 16323 goto out_free; 16324 } 16325 cond_resched(); 16326 } 16327 16328 /* at this point all bpf functions were successfully JITed 16329 * now populate all bpf_calls with correct addresses and 16330 * run last pass of JIT 16331 */ 16332 for (i = 0; i < env->subprog_cnt; i++) { 16333 insn = func[i]->insnsi; 16334 for (j = 0; j < func[i]->len; j++, insn++) { 16335 if (bpf_pseudo_func(insn)) { 16336 subprog = insn->off; 16337 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 16338 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 16339 continue; 16340 } 16341 if (!bpf_pseudo_call(insn)) 16342 continue; 16343 subprog = insn->off; 16344 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 16345 } 16346 16347 /* we use the aux data to keep a list of the start addresses 16348 * of the JITed images for each function in the program 16349 * 16350 * for some architectures, such as powerpc64, the imm field 16351 * might not be large enough to hold the offset of the start 16352 * address of the callee's JITed image from __bpf_call_base 16353 * 16354 * in such cases, we can lookup the start address of a callee 16355 * by using its subprog id, available from the off field of 16356 * the call instruction, as an index for this list 16357 */ 16358 func[i]->aux->func = func; 16359 func[i]->aux->func_cnt = env->subprog_cnt; 16360 } 16361 for (i = 0; i < env->subprog_cnt; i++) { 16362 old_bpf_func = func[i]->bpf_func; 16363 tmp = bpf_int_jit_compile(func[i]); 16364 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 16365 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 16366 err = -ENOTSUPP; 16367 goto out_free; 16368 } 16369 cond_resched(); 16370 } 16371 16372 /* finally lock prog and jit images for all functions and 16373 * populate kallsysm 16374 */ 16375 for (i = 0; i < env->subprog_cnt; i++) { 16376 bpf_prog_lock_ro(func[i]); 16377 bpf_prog_kallsyms_add(func[i]); 16378 } 16379 16380 /* Last step: make now unused interpreter insns from main 16381 * prog consistent for later dump requests, so they can 16382 * later look the same as if they were interpreted only. 16383 */ 16384 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 16385 if (bpf_pseudo_func(insn)) { 16386 insn[0].imm = env->insn_aux_data[i].call_imm; 16387 insn[1].imm = insn->off; 16388 insn->off = 0; 16389 continue; 16390 } 16391 if (!bpf_pseudo_call(insn)) 16392 continue; 16393 insn->off = env->insn_aux_data[i].call_imm; 16394 subprog = find_subprog(env, i + insn->off + 1); 16395 insn->imm = subprog; 16396 } 16397 16398 prog->jited = 1; 16399 prog->bpf_func = func[0]->bpf_func; 16400 prog->jited_len = func[0]->jited_len; 16401 prog->aux->func = func; 16402 prog->aux->func_cnt = env->subprog_cnt; 16403 bpf_prog_jit_attempt_done(prog); 16404 return 0; 16405 out_free: 16406 /* We failed JIT'ing, so at this point we need to unregister poke 16407 * descriptors from subprogs, so that kernel is not attempting to 16408 * patch it anymore as we're freeing the subprog JIT memory. 16409 */ 16410 for (i = 0; i < prog->aux->size_poke_tab; i++) { 16411 map_ptr = prog->aux->poke_tab[i].tail_call.map; 16412 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 16413 } 16414 /* At this point we're guaranteed that poke descriptors are not 16415 * live anymore. We can just unlink its descriptor table as it's 16416 * released with the main prog. 16417 */ 16418 for (i = 0; i < env->subprog_cnt; i++) { 16419 if (!func[i]) 16420 continue; 16421 func[i]->aux->poke_tab = NULL; 16422 bpf_jit_free(func[i]); 16423 } 16424 kfree(func); 16425 out_undo_insn: 16426 /* cleanup main prog to be interpreted */ 16427 prog->jit_requested = 0; 16428 prog->blinding_requested = 0; 16429 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 16430 if (!bpf_pseudo_call(insn)) 16431 continue; 16432 insn->off = 0; 16433 insn->imm = env->insn_aux_data[i].call_imm; 16434 } 16435 bpf_prog_jit_attempt_done(prog); 16436 return err; 16437 } 16438 16439 static int fixup_call_args(struct bpf_verifier_env *env) 16440 { 16441 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 16442 struct bpf_prog *prog = env->prog; 16443 struct bpf_insn *insn = prog->insnsi; 16444 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 16445 int i, depth; 16446 #endif 16447 int err = 0; 16448 16449 if (env->prog->jit_requested && 16450 !bpf_prog_is_offloaded(env->prog->aux)) { 16451 err = jit_subprogs(env); 16452 if (err == 0) 16453 return 0; 16454 if (err == -EFAULT) 16455 return err; 16456 } 16457 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 16458 if (has_kfunc_call) { 16459 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 16460 return -EINVAL; 16461 } 16462 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 16463 /* When JIT fails the progs with bpf2bpf calls and tail_calls 16464 * have to be rejected, since interpreter doesn't support them yet. 16465 */ 16466 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 16467 return -EINVAL; 16468 } 16469 for (i = 0; i < prog->len; i++, insn++) { 16470 if (bpf_pseudo_func(insn)) { 16471 /* When JIT fails the progs with callback calls 16472 * have to be rejected, since interpreter doesn't support them yet. 16473 */ 16474 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 16475 return -EINVAL; 16476 } 16477 16478 if (!bpf_pseudo_call(insn)) 16479 continue; 16480 depth = get_callee_stack_depth(env, insn, i); 16481 if (depth < 0) 16482 return depth; 16483 bpf_patch_call_args(insn, depth); 16484 } 16485 err = 0; 16486 #endif 16487 return err; 16488 } 16489 16490 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 16491 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 16492 { 16493 const struct bpf_kfunc_desc *desc; 16494 void *xdp_kfunc; 16495 16496 if (!insn->imm) { 16497 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 16498 return -EINVAL; 16499 } 16500 16501 *cnt = 0; 16502 16503 if (bpf_dev_bound_kfunc_id(insn->imm)) { 16504 xdp_kfunc = bpf_dev_bound_resolve_kfunc(env->prog, insn->imm); 16505 if (xdp_kfunc) { 16506 insn->imm = BPF_CALL_IMM(xdp_kfunc); 16507 return 0; 16508 } 16509 16510 /* fallback to default kfunc when not supported by netdev */ 16511 } 16512 16513 /* insn->imm has the btf func_id. Replace it with 16514 * an address (relative to __bpf_call_base). 16515 */ 16516 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 16517 if (!desc) { 16518 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 16519 insn->imm); 16520 return -EFAULT; 16521 } 16522 16523 insn->imm = desc->imm; 16524 if (insn->off) 16525 return 0; 16526 if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 16527 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 16528 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 16529 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 16530 16531 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 16532 insn_buf[1] = addr[0]; 16533 insn_buf[2] = addr[1]; 16534 insn_buf[3] = *insn; 16535 *cnt = 4; 16536 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 16537 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 16538 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 16539 16540 insn_buf[0] = addr[0]; 16541 insn_buf[1] = addr[1]; 16542 insn_buf[2] = *insn; 16543 *cnt = 3; 16544 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 16545 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 16546 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 16547 *cnt = 1; 16548 } else if (desc->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) { 16549 bool seen_direct_write = env->seen_direct_write; 16550 bool is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); 16551 16552 if (is_rdonly) 16553 insn->imm = BPF_CALL_IMM(bpf_dynptr_from_skb_rdonly); 16554 16555 /* restore env->seen_direct_write to its original value, since 16556 * may_access_direct_pkt_data mutates it 16557 */ 16558 env->seen_direct_write = seen_direct_write; 16559 } 16560 return 0; 16561 } 16562 16563 /* Do various post-verification rewrites in a single program pass. 16564 * These rewrites simplify JIT and interpreter implementations. 16565 */ 16566 static int do_misc_fixups(struct bpf_verifier_env *env) 16567 { 16568 struct bpf_prog *prog = env->prog; 16569 enum bpf_attach_type eatype = prog->expected_attach_type; 16570 enum bpf_prog_type prog_type = resolve_prog_type(prog); 16571 struct bpf_insn *insn = prog->insnsi; 16572 const struct bpf_func_proto *fn; 16573 const int insn_cnt = prog->len; 16574 const struct bpf_map_ops *ops; 16575 struct bpf_insn_aux_data *aux; 16576 struct bpf_insn insn_buf[16]; 16577 struct bpf_prog *new_prog; 16578 struct bpf_map *map_ptr; 16579 int i, ret, cnt, delta = 0; 16580 16581 for (i = 0; i < insn_cnt; i++, insn++) { 16582 /* Make divide-by-zero exceptions impossible. */ 16583 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 16584 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 16585 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 16586 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 16587 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 16588 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 16589 struct bpf_insn *patchlet; 16590 struct bpf_insn chk_and_div[] = { 16591 /* [R,W]x div 0 -> 0 */ 16592 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 16593 BPF_JNE | BPF_K, insn->src_reg, 16594 0, 2, 0), 16595 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 16596 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 16597 *insn, 16598 }; 16599 struct bpf_insn chk_and_mod[] = { 16600 /* [R,W]x mod 0 -> [R,W]x */ 16601 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 16602 BPF_JEQ | BPF_K, insn->src_reg, 16603 0, 1 + (is64 ? 0 : 1), 0), 16604 *insn, 16605 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 16606 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 16607 }; 16608 16609 patchlet = isdiv ? chk_and_div : chk_and_mod; 16610 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 16611 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 16612 16613 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 16614 if (!new_prog) 16615 return -ENOMEM; 16616 16617 delta += cnt - 1; 16618 env->prog = prog = new_prog; 16619 insn = new_prog->insnsi + i + delta; 16620 continue; 16621 } 16622 16623 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 16624 if (BPF_CLASS(insn->code) == BPF_LD && 16625 (BPF_MODE(insn->code) == BPF_ABS || 16626 BPF_MODE(insn->code) == BPF_IND)) { 16627 cnt = env->ops->gen_ld_abs(insn, insn_buf); 16628 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 16629 verbose(env, "bpf verifier is misconfigured\n"); 16630 return -EINVAL; 16631 } 16632 16633 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16634 if (!new_prog) 16635 return -ENOMEM; 16636 16637 delta += cnt - 1; 16638 env->prog = prog = new_prog; 16639 insn = new_prog->insnsi + i + delta; 16640 continue; 16641 } 16642 16643 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 16644 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 16645 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 16646 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 16647 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 16648 struct bpf_insn *patch = &insn_buf[0]; 16649 bool issrc, isneg, isimm; 16650 u32 off_reg; 16651 16652 aux = &env->insn_aux_data[i + delta]; 16653 if (!aux->alu_state || 16654 aux->alu_state == BPF_ALU_NON_POINTER) 16655 continue; 16656 16657 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 16658 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 16659 BPF_ALU_SANITIZE_SRC; 16660 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 16661 16662 off_reg = issrc ? insn->src_reg : insn->dst_reg; 16663 if (isimm) { 16664 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 16665 } else { 16666 if (isneg) 16667 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 16668 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 16669 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 16670 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 16671 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 16672 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 16673 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 16674 } 16675 if (!issrc) 16676 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 16677 insn->src_reg = BPF_REG_AX; 16678 if (isneg) 16679 insn->code = insn->code == code_add ? 16680 code_sub : code_add; 16681 *patch++ = *insn; 16682 if (issrc && isneg && !isimm) 16683 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 16684 cnt = patch - insn_buf; 16685 16686 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16687 if (!new_prog) 16688 return -ENOMEM; 16689 16690 delta += cnt - 1; 16691 env->prog = prog = new_prog; 16692 insn = new_prog->insnsi + i + delta; 16693 continue; 16694 } 16695 16696 if (insn->code != (BPF_JMP | BPF_CALL)) 16697 continue; 16698 if (insn->src_reg == BPF_PSEUDO_CALL) 16699 continue; 16700 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 16701 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 16702 if (ret) 16703 return ret; 16704 if (cnt == 0) 16705 continue; 16706 16707 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16708 if (!new_prog) 16709 return -ENOMEM; 16710 16711 delta += cnt - 1; 16712 env->prog = prog = new_prog; 16713 insn = new_prog->insnsi + i + delta; 16714 continue; 16715 } 16716 16717 if (insn->imm == BPF_FUNC_get_route_realm) 16718 prog->dst_needed = 1; 16719 if (insn->imm == BPF_FUNC_get_prandom_u32) 16720 bpf_user_rnd_init_once(); 16721 if (insn->imm == BPF_FUNC_override_return) 16722 prog->kprobe_override = 1; 16723 if (insn->imm == BPF_FUNC_tail_call) { 16724 /* If we tail call into other programs, we 16725 * cannot make any assumptions since they can 16726 * be replaced dynamically during runtime in 16727 * the program array. 16728 */ 16729 prog->cb_access = 1; 16730 if (!allow_tail_call_in_subprogs(env)) 16731 prog->aux->stack_depth = MAX_BPF_STACK; 16732 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 16733 16734 /* mark bpf_tail_call as different opcode to avoid 16735 * conditional branch in the interpreter for every normal 16736 * call and to prevent accidental JITing by JIT compiler 16737 * that doesn't support bpf_tail_call yet 16738 */ 16739 insn->imm = 0; 16740 insn->code = BPF_JMP | BPF_TAIL_CALL; 16741 16742 aux = &env->insn_aux_data[i + delta]; 16743 if (env->bpf_capable && !prog->blinding_requested && 16744 prog->jit_requested && 16745 !bpf_map_key_poisoned(aux) && 16746 !bpf_map_ptr_poisoned(aux) && 16747 !bpf_map_ptr_unpriv(aux)) { 16748 struct bpf_jit_poke_descriptor desc = { 16749 .reason = BPF_POKE_REASON_TAIL_CALL, 16750 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 16751 .tail_call.key = bpf_map_key_immediate(aux), 16752 .insn_idx = i + delta, 16753 }; 16754 16755 ret = bpf_jit_add_poke_descriptor(prog, &desc); 16756 if (ret < 0) { 16757 verbose(env, "adding tail call poke descriptor failed\n"); 16758 return ret; 16759 } 16760 16761 insn->imm = ret + 1; 16762 continue; 16763 } 16764 16765 if (!bpf_map_ptr_unpriv(aux)) 16766 continue; 16767 16768 /* instead of changing every JIT dealing with tail_call 16769 * emit two extra insns: 16770 * if (index >= max_entries) goto out; 16771 * index &= array->index_mask; 16772 * to avoid out-of-bounds cpu speculation 16773 */ 16774 if (bpf_map_ptr_poisoned(aux)) { 16775 verbose(env, "tail_call abusing map_ptr\n"); 16776 return -EINVAL; 16777 } 16778 16779 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 16780 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 16781 map_ptr->max_entries, 2); 16782 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 16783 container_of(map_ptr, 16784 struct bpf_array, 16785 map)->index_mask); 16786 insn_buf[2] = *insn; 16787 cnt = 3; 16788 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16789 if (!new_prog) 16790 return -ENOMEM; 16791 16792 delta += cnt - 1; 16793 env->prog = prog = new_prog; 16794 insn = new_prog->insnsi + i + delta; 16795 continue; 16796 } 16797 16798 if (insn->imm == BPF_FUNC_timer_set_callback) { 16799 /* The verifier will process callback_fn as many times as necessary 16800 * with different maps and the register states prepared by 16801 * set_timer_callback_state will be accurate. 16802 * 16803 * The following use case is valid: 16804 * map1 is shared by prog1, prog2, prog3. 16805 * prog1 calls bpf_timer_init for some map1 elements 16806 * prog2 calls bpf_timer_set_callback for some map1 elements. 16807 * Those that were not bpf_timer_init-ed will return -EINVAL. 16808 * prog3 calls bpf_timer_start for some map1 elements. 16809 * Those that were not both bpf_timer_init-ed and 16810 * bpf_timer_set_callback-ed will return -EINVAL. 16811 */ 16812 struct bpf_insn ld_addrs[2] = { 16813 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 16814 }; 16815 16816 insn_buf[0] = ld_addrs[0]; 16817 insn_buf[1] = ld_addrs[1]; 16818 insn_buf[2] = *insn; 16819 cnt = 3; 16820 16821 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16822 if (!new_prog) 16823 return -ENOMEM; 16824 16825 delta += cnt - 1; 16826 env->prog = prog = new_prog; 16827 insn = new_prog->insnsi + i + delta; 16828 goto patch_call_imm; 16829 } 16830 16831 if (is_storage_get_function(insn->imm)) { 16832 if (!env->prog->aux->sleepable || 16833 env->insn_aux_data[i + delta].storage_get_func_atomic) 16834 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 16835 else 16836 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 16837 insn_buf[1] = *insn; 16838 cnt = 2; 16839 16840 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16841 if (!new_prog) 16842 return -ENOMEM; 16843 16844 delta += cnt - 1; 16845 env->prog = prog = new_prog; 16846 insn = new_prog->insnsi + i + delta; 16847 goto patch_call_imm; 16848 } 16849 16850 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 16851 * and other inlining handlers are currently limited to 64 bit 16852 * only. 16853 */ 16854 if (prog->jit_requested && BITS_PER_LONG == 64 && 16855 (insn->imm == BPF_FUNC_map_lookup_elem || 16856 insn->imm == BPF_FUNC_map_update_elem || 16857 insn->imm == BPF_FUNC_map_delete_elem || 16858 insn->imm == BPF_FUNC_map_push_elem || 16859 insn->imm == BPF_FUNC_map_pop_elem || 16860 insn->imm == BPF_FUNC_map_peek_elem || 16861 insn->imm == BPF_FUNC_redirect_map || 16862 insn->imm == BPF_FUNC_for_each_map_elem || 16863 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 16864 aux = &env->insn_aux_data[i + delta]; 16865 if (bpf_map_ptr_poisoned(aux)) 16866 goto patch_call_imm; 16867 16868 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 16869 ops = map_ptr->ops; 16870 if (insn->imm == BPF_FUNC_map_lookup_elem && 16871 ops->map_gen_lookup) { 16872 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 16873 if (cnt == -EOPNOTSUPP) 16874 goto patch_map_ops_generic; 16875 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 16876 verbose(env, "bpf verifier is misconfigured\n"); 16877 return -EINVAL; 16878 } 16879 16880 new_prog = bpf_patch_insn_data(env, i + delta, 16881 insn_buf, cnt); 16882 if (!new_prog) 16883 return -ENOMEM; 16884 16885 delta += cnt - 1; 16886 env->prog = prog = new_prog; 16887 insn = new_prog->insnsi + i + delta; 16888 continue; 16889 } 16890 16891 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 16892 (void *(*)(struct bpf_map *map, void *key))NULL)); 16893 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 16894 (int (*)(struct bpf_map *map, void *key))NULL)); 16895 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 16896 (int (*)(struct bpf_map *map, void *key, void *value, 16897 u64 flags))NULL)); 16898 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 16899 (int (*)(struct bpf_map *map, void *value, 16900 u64 flags))NULL)); 16901 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 16902 (int (*)(struct bpf_map *map, void *value))NULL)); 16903 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 16904 (int (*)(struct bpf_map *map, void *value))NULL)); 16905 BUILD_BUG_ON(!__same_type(ops->map_redirect, 16906 (int (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 16907 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 16908 (int (*)(struct bpf_map *map, 16909 bpf_callback_t callback_fn, 16910 void *callback_ctx, 16911 u64 flags))NULL)); 16912 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 16913 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 16914 16915 patch_map_ops_generic: 16916 switch (insn->imm) { 16917 case BPF_FUNC_map_lookup_elem: 16918 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 16919 continue; 16920 case BPF_FUNC_map_update_elem: 16921 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 16922 continue; 16923 case BPF_FUNC_map_delete_elem: 16924 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 16925 continue; 16926 case BPF_FUNC_map_push_elem: 16927 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 16928 continue; 16929 case BPF_FUNC_map_pop_elem: 16930 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 16931 continue; 16932 case BPF_FUNC_map_peek_elem: 16933 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 16934 continue; 16935 case BPF_FUNC_redirect_map: 16936 insn->imm = BPF_CALL_IMM(ops->map_redirect); 16937 continue; 16938 case BPF_FUNC_for_each_map_elem: 16939 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 16940 continue; 16941 case BPF_FUNC_map_lookup_percpu_elem: 16942 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 16943 continue; 16944 } 16945 16946 goto patch_call_imm; 16947 } 16948 16949 /* Implement bpf_jiffies64 inline. */ 16950 if (prog->jit_requested && BITS_PER_LONG == 64 && 16951 insn->imm == BPF_FUNC_jiffies64) { 16952 struct bpf_insn ld_jiffies_addr[2] = { 16953 BPF_LD_IMM64(BPF_REG_0, 16954 (unsigned long)&jiffies), 16955 }; 16956 16957 insn_buf[0] = ld_jiffies_addr[0]; 16958 insn_buf[1] = ld_jiffies_addr[1]; 16959 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 16960 BPF_REG_0, 0); 16961 cnt = 3; 16962 16963 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 16964 cnt); 16965 if (!new_prog) 16966 return -ENOMEM; 16967 16968 delta += cnt - 1; 16969 env->prog = prog = new_prog; 16970 insn = new_prog->insnsi + i + delta; 16971 continue; 16972 } 16973 16974 /* Implement bpf_get_func_arg inline. */ 16975 if (prog_type == BPF_PROG_TYPE_TRACING && 16976 insn->imm == BPF_FUNC_get_func_arg) { 16977 /* Load nr_args from ctx - 8 */ 16978 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 16979 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 16980 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 16981 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 16982 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 16983 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 16984 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 16985 insn_buf[7] = BPF_JMP_A(1); 16986 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 16987 cnt = 9; 16988 16989 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 16990 if (!new_prog) 16991 return -ENOMEM; 16992 16993 delta += cnt - 1; 16994 env->prog = prog = new_prog; 16995 insn = new_prog->insnsi + i + delta; 16996 continue; 16997 } 16998 16999 /* Implement bpf_get_func_ret inline. */ 17000 if (prog_type == BPF_PROG_TYPE_TRACING && 17001 insn->imm == BPF_FUNC_get_func_ret) { 17002 if (eatype == BPF_TRACE_FEXIT || 17003 eatype == BPF_MODIFY_RETURN) { 17004 /* Load nr_args from ctx - 8 */ 17005 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 17006 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 17007 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 17008 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 17009 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 17010 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 17011 cnt = 6; 17012 } else { 17013 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 17014 cnt = 1; 17015 } 17016 17017 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 17018 if (!new_prog) 17019 return -ENOMEM; 17020 17021 delta += cnt - 1; 17022 env->prog = prog = new_prog; 17023 insn = new_prog->insnsi + i + delta; 17024 continue; 17025 } 17026 17027 /* Implement get_func_arg_cnt inline. */ 17028 if (prog_type == BPF_PROG_TYPE_TRACING && 17029 insn->imm == BPF_FUNC_get_func_arg_cnt) { 17030 /* Load nr_args from ctx - 8 */ 17031 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 17032 17033 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 17034 if (!new_prog) 17035 return -ENOMEM; 17036 17037 env->prog = prog = new_prog; 17038 insn = new_prog->insnsi + i + delta; 17039 continue; 17040 } 17041 17042 /* Implement bpf_get_func_ip inline. */ 17043 if (prog_type == BPF_PROG_TYPE_TRACING && 17044 insn->imm == BPF_FUNC_get_func_ip) { 17045 /* Load IP address from ctx - 16 */ 17046 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 17047 17048 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 17049 if (!new_prog) 17050 return -ENOMEM; 17051 17052 env->prog = prog = new_prog; 17053 insn = new_prog->insnsi + i + delta; 17054 continue; 17055 } 17056 17057 patch_call_imm: 17058 fn = env->ops->get_func_proto(insn->imm, env->prog); 17059 /* all functions that have prototype and verifier allowed 17060 * programs to call them, must be real in-kernel functions 17061 */ 17062 if (!fn->func) { 17063 verbose(env, 17064 "kernel subsystem misconfigured func %s#%d\n", 17065 func_id_name(insn->imm), insn->imm); 17066 return -EFAULT; 17067 } 17068 insn->imm = fn->func - __bpf_call_base; 17069 } 17070 17071 /* Since poke tab is now finalized, publish aux to tracker. */ 17072 for (i = 0; i < prog->aux->size_poke_tab; i++) { 17073 map_ptr = prog->aux->poke_tab[i].tail_call.map; 17074 if (!map_ptr->ops->map_poke_track || 17075 !map_ptr->ops->map_poke_untrack || 17076 !map_ptr->ops->map_poke_run) { 17077 verbose(env, "bpf verifier is misconfigured\n"); 17078 return -EINVAL; 17079 } 17080 17081 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 17082 if (ret < 0) { 17083 verbose(env, "tracking tail call prog failed\n"); 17084 return ret; 17085 } 17086 } 17087 17088 sort_kfunc_descs_by_imm(env->prog); 17089 17090 return 0; 17091 } 17092 17093 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 17094 int position, 17095 s32 stack_base, 17096 u32 callback_subprogno, 17097 u32 *cnt) 17098 { 17099 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 17100 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 17101 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 17102 int reg_loop_max = BPF_REG_6; 17103 int reg_loop_cnt = BPF_REG_7; 17104 int reg_loop_ctx = BPF_REG_8; 17105 17106 struct bpf_prog *new_prog; 17107 u32 callback_start; 17108 u32 call_insn_offset; 17109 s32 callback_offset; 17110 17111 /* This represents an inlined version of bpf_iter.c:bpf_loop, 17112 * be careful to modify this code in sync. 17113 */ 17114 struct bpf_insn insn_buf[] = { 17115 /* Return error and jump to the end of the patch if 17116 * expected number of iterations is too big. 17117 */ 17118 BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), 17119 BPF_MOV32_IMM(BPF_REG_0, -E2BIG), 17120 BPF_JMP_IMM(BPF_JA, 0, 0, 16), 17121 /* spill R6, R7, R8 to use these as loop vars */ 17122 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), 17123 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), 17124 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), 17125 /* initialize loop vars */ 17126 BPF_MOV64_REG(reg_loop_max, BPF_REG_1), 17127 BPF_MOV32_IMM(reg_loop_cnt, 0), 17128 BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), 17129 /* loop header, 17130 * if reg_loop_cnt >= reg_loop_max skip the loop body 17131 */ 17132 BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), 17133 /* callback call, 17134 * correct callback offset would be set after patching 17135 */ 17136 BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), 17137 BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), 17138 BPF_CALL_REL(0), 17139 /* increment loop counter */ 17140 BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), 17141 /* jump to loop header if callback returned 0 */ 17142 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), 17143 /* return value of bpf_loop, 17144 * set R0 to the number of iterations 17145 */ 17146 BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), 17147 /* restore original values of R6, R7, R8 */ 17148 BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), 17149 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), 17150 BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), 17151 }; 17152 17153 *cnt = ARRAY_SIZE(insn_buf); 17154 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); 17155 if (!new_prog) 17156 return new_prog; 17157 17158 /* callback start is known only after patching */ 17159 callback_start = env->subprog_info[callback_subprogno].start; 17160 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 17161 call_insn_offset = position + 12; 17162 callback_offset = callback_start - call_insn_offset - 1; 17163 new_prog->insnsi[call_insn_offset].imm = callback_offset; 17164 17165 return new_prog; 17166 } 17167 17168 static bool is_bpf_loop_call(struct bpf_insn *insn) 17169 { 17170 return insn->code == (BPF_JMP | BPF_CALL) && 17171 insn->src_reg == 0 && 17172 insn->imm == BPF_FUNC_loop; 17173 } 17174 17175 /* For all sub-programs in the program (including main) check 17176 * insn_aux_data to see if there are bpf_loop calls that require 17177 * inlining. If such calls are found the calls are replaced with a 17178 * sequence of instructions produced by `inline_bpf_loop` function and 17179 * subprog stack_depth is increased by the size of 3 registers. 17180 * This stack space is used to spill values of the R6, R7, R8. These 17181 * registers are used to store the loop bound, counter and context 17182 * variables. 17183 */ 17184 static int optimize_bpf_loop(struct bpf_verifier_env *env) 17185 { 17186 struct bpf_subprog_info *subprogs = env->subprog_info; 17187 int i, cur_subprog = 0, cnt, delta = 0; 17188 struct bpf_insn *insn = env->prog->insnsi; 17189 int insn_cnt = env->prog->len; 17190 u16 stack_depth = subprogs[cur_subprog].stack_depth; 17191 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 17192 u16 stack_depth_extra = 0; 17193 17194 for (i = 0; i < insn_cnt; i++, insn++) { 17195 struct bpf_loop_inline_state *inline_state = 17196 &env->insn_aux_data[i + delta].loop_inline_state; 17197 17198 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 17199 struct bpf_prog *new_prog; 17200 17201 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 17202 new_prog = inline_bpf_loop(env, 17203 i + delta, 17204 -(stack_depth + stack_depth_extra), 17205 inline_state->callback_subprogno, 17206 &cnt); 17207 if (!new_prog) 17208 return -ENOMEM; 17209 17210 delta += cnt - 1; 17211 env->prog = new_prog; 17212 insn = new_prog->insnsi + i + delta; 17213 } 17214 17215 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 17216 subprogs[cur_subprog].stack_depth += stack_depth_extra; 17217 cur_subprog++; 17218 stack_depth = subprogs[cur_subprog].stack_depth; 17219 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 17220 stack_depth_extra = 0; 17221 } 17222 } 17223 17224 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 17225 17226 return 0; 17227 } 17228 17229 static void free_states(struct bpf_verifier_env *env) 17230 { 17231 struct bpf_verifier_state_list *sl, *sln; 17232 int i; 17233 17234 sl = env->free_list; 17235 while (sl) { 17236 sln = sl->next; 17237 free_verifier_state(&sl->state, false); 17238 kfree(sl); 17239 sl = sln; 17240 } 17241 env->free_list = NULL; 17242 17243 if (!env->explored_states) 17244 return; 17245 17246 for (i = 0; i < state_htab_size(env); i++) { 17247 sl = env->explored_states[i]; 17248 17249 while (sl) { 17250 sln = sl->next; 17251 free_verifier_state(&sl->state, false); 17252 kfree(sl); 17253 sl = sln; 17254 } 17255 env->explored_states[i] = NULL; 17256 } 17257 } 17258 17259 static int do_check_common(struct bpf_verifier_env *env, int subprog) 17260 { 17261 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 17262 struct bpf_verifier_state *state; 17263 struct bpf_reg_state *regs; 17264 int ret, i; 17265 17266 env->prev_linfo = NULL; 17267 env->pass_cnt++; 17268 17269 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 17270 if (!state) 17271 return -ENOMEM; 17272 state->curframe = 0; 17273 state->speculative = false; 17274 state->branches = 1; 17275 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 17276 if (!state->frame[0]) { 17277 kfree(state); 17278 return -ENOMEM; 17279 } 17280 env->cur_state = state; 17281 init_func_state(env, state->frame[0], 17282 BPF_MAIN_FUNC /* callsite */, 17283 0 /* frameno */, 17284 subprog); 17285 state->first_insn_idx = env->subprog_info[subprog].start; 17286 state->last_insn_idx = -1; 17287 17288 regs = state->frame[state->curframe]->regs; 17289 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 17290 ret = btf_prepare_func_args(env, subprog, regs); 17291 if (ret) 17292 goto out; 17293 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 17294 if (regs[i].type == PTR_TO_CTX) 17295 mark_reg_known_zero(env, regs, i); 17296 else if (regs[i].type == SCALAR_VALUE) 17297 mark_reg_unknown(env, regs, i); 17298 else if (base_type(regs[i].type) == PTR_TO_MEM) { 17299 const u32 mem_size = regs[i].mem_size; 17300 17301 mark_reg_known_zero(env, regs, i); 17302 regs[i].mem_size = mem_size; 17303 regs[i].id = ++env->id_gen; 17304 } 17305 } 17306 } else { 17307 /* 1st arg to a function */ 17308 regs[BPF_REG_1].type = PTR_TO_CTX; 17309 mark_reg_known_zero(env, regs, BPF_REG_1); 17310 ret = btf_check_subprog_arg_match(env, subprog, regs); 17311 if (ret == -EFAULT) 17312 /* unlikely verifier bug. abort. 17313 * ret == 0 and ret < 0 are sadly acceptable for 17314 * main() function due to backward compatibility. 17315 * Like socket filter program may be written as: 17316 * int bpf_prog(struct pt_regs *ctx) 17317 * and never dereference that ctx in the program. 17318 * 'struct pt_regs' is a type mismatch for socket 17319 * filter that should be using 'struct __sk_buff'. 17320 */ 17321 goto out; 17322 } 17323 17324 ret = do_check(env); 17325 out: 17326 /* check for NULL is necessary, since cur_state can be freed inside 17327 * do_check() under memory pressure. 17328 */ 17329 if (env->cur_state) { 17330 free_verifier_state(env->cur_state, true); 17331 env->cur_state = NULL; 17332 } 17333 while (!pop_stack(env, NULL, NULL, false)); 17334 if (!ret && pop_log) 17335 bpf_vlog_reset(&env->log, 0); 17336 free_states(env); 17337 return ret; 17338 } 17339 17340 /* Verify all global functions in a BPF program one by one based on their BTF. 17341 * All global functions must pass verification. Otherwise the whole program is rejected. 17342 * Consider: 17343 * int bar(int); 17344 * int foo(int f) 17345 * { 17346 * return bar(f); 17347 * } 17348 * int bar(int b) 17349 * { 17350 * ... 17351 * } 17352 * foo() will be verified first for R1=any_scalar_value. During verification it 17353 * will be assumed that bar() already verified successfully and call to bar() 17354 * from foo() will be checked for type match only. Later bar() will be verified 17355 * independently to check that it's safe for R1=any_scalar_value. 17356 */ 17357 static int do_check_subprogs(struct bpf_verifier_env *env) 17358 { 17359 struct bpf_prog_aux *aux = env->prog->aux; 17360 int i, ret; 17361 17362 if (!aux->func_info) 17363 return 0; 17364 17365 for (i = 1; i < env->subprog_cnt; i++) { 17366 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 17367 continue; 17368 env->insn_idx = env->subprog_info[i].start; 17369 WARN_ON_ONCE(env->insn_idx == 0); 17370 ret = do_check_common(env, i); 17371 if (ret) { 17372 return ret; 17373 } else if (env->log.level & BPF_LOG_LEVEL) { 17374 verbose(env, 17375 "Func#%d is safe for any args that match its prototype\n", 17376 i); 17377 } 17378 } 17379 return 0; 17380 } 17381 17382 static int do_check_main(struct bpf_verifier_env *env) 17383 { 17384 int ret; 17385 17386 env->insn_idx = 0; 17387 ret = do_check_common(env, 0); 17388 if (!ret) 17389 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 17390 return ret; 17391 } 17392 17393 17394 static void print_verification_stats(struct bpf_verifier_env *env) 17395 { 17396 int i; 17397 17398 if (env->log.level & BPF_LOG_STATS) { 17399 verbose(env, "verification time %lld usec\n", 17400 div_u64(env->verification_time, 1000)); 17401 verbose(env, "stack depth "); 17402 for (i = 0; i < env->subprog_cnt; i++) { 17403 u32 depth = env->subprog_info[i].stack_depth; 17404 17405 verbose(env, "%d", depth); 17406 if (i + 1 < env->subprog_cnt) 17407 verbose(env, "+"); 17408 } 17409 verbose(env, "\n"); 17410 } 17411 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 17412 "total_states %d peak_states %d mark_read %d\n", 17413 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 17414 env->max_states_per_insn, env->total_states, 17415 env->peak_states, env->longest_mark_read_walk); 17416 } 17417 17418 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 17419 { 17420 const struct btf_type *t, *func_proto; 17421 const struct bpf_struct_ops *st_ops; 17422 const struct btf_member *member; 17423 struct bpf_prog *prog = env->prog; 17424 u32 btf_id, member_idx; 17425 const char *mname; 17426 17427 if (!prog->gpl_compatible) { 17428 verbose(env, "struct ops programs must have a GPL compatible license\n"); 17429 return -EINVAL; 17430 } 17431 17432 btf_id = prog->aux->attach_btf_id; 17433 st_ops = bpf_struct_ops_find(btf_id); 17434 if (!st_ops) { 17435 verbose(env, "attach_btf_id %u is not a supported struct\n", 17436 btf_id); 17437 return -ENOTSUPP; 17438 } 17439 17440 t = st_ops->type; 17441 member_idx = prog->expected_attach_type; 17442 if (member_idx >= btf_type_vlen(t)) { 17443 verbose(env, "attach to invalid member idx %u of struct %s\n", 17444 member_idx, st_ops->name); 17445 return -EINVAL; 17446 } 17447 17448 member = &btf_type_member(t)[member_idx]; 17449 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 17450 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 17451 NULL); 17452 if (!func_proto) { 17453 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 17454 mname, member_idx, st_ops->name); 17455 return -EINVAL; 17456 } 17457 17458 if (st_ops->check_member) { 17459 int err = st_ops->check_member(t, member, prog); 17460 17461 if (err) { 17462 verbose(env, "attach to unsupported member %s of struct %s\n", 17463 mname, st_ops->name); 17464 return err; 17465 } 17466 } 17467 17468 prog->aux->attach_func_proto = func_proto; 17469 prog->aux->attach_func_name = mname; 17470 env->ops = st_ops->verifier_ops; 17471 17472 return 0; 17473 } 17474 #define SECURITY_PREFIX "security_" 17475 17476 static int check_attach_modify_return(unsigned long addr, const char *func_name) 17477 { 17478 if (within_error_injection_list(addr) || 17479 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 17480 return 0; 17481 17482 return -EINVAL; 17483 } 17484 17485 /* list of non-sleepable functions that are otherwise on 17486 * ALLOW_ERROR_INJECTION list 17487 */ 17488 BTF_SET_START(btf_non_sleepable_error_inject) 17489 /* Three functions below can be called from sleepable and non-sleepable context. 17490 * Assume non-sleepable from bpf safety point of view. 17491 */ 17492 BTF_ID(func, __filemap_add_folio) 17493 BTF_ID(func, should_fail_alloc_page) 17494 BTF_ID(func, should_failslab) 17495 BTF_SET_END(btf_non_sleepable_error_inject) 17496 17497 static int check_non_sleepable_error_inject(u32 btf_id) 17498 { 17499 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 17500 } 17501 17502 int bpf_check_attach_target(struct bpf_verifier_log *log, 17503 const struct bpf_prog *prog, 17504 const struct bpf_prog *tgt_prog, 17505 u32 btf_id, 17506 struct bpf_attach_target_info *tgt_info) 17507 { 17508 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 17509 const char prefix[] = "btf_trace_"; 17510 int ret = 0, subprog = -1, i; 17511 const struct btf_type *t; 17512 bool conservative = true; 17513 const char *tname; 17514 struct btf *btf; 17515 long addr = 0; 17516 17517 if (!btf_id) { 17518 bpf_log(log, "Tracing programs must provide btf_id\n"); 17519 return -EINVAL; 17520 } 17521 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 17522 if (!btf) { 17523 bpf_log(log, 17524 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 17525 return -EINVAL; 17526 } 17527 t = btf_type_by_id(btf, btf_id); 17528 if (!t) { 17529 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 17530 return -EINVAL; 17531 } 17532 tname = btf_name_by_offset(btf, t->name_off); 17533 if (!tname) { 17534 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 17535 return -EINVAL; 17536 } 17537 if (tgt_prog) { 17538 struct bpf_prog_aux *aux = tgt_prog->aux; 17539 17540 if (bpf_prog_is_dev_bound(prog->aux) && 17541 !bpf_prog_dev_bound_match(prog, tgt_prog)) { 17542 bpf_log(log, "Target program bound device mismatch"); 17543 return -EINVAL; 17544 } 17545 17546 for (i = 0; i < aux->func_info_cnt; i++) 17547 if (aux->func_info[i].type_id == btf_id) { 17548 subprog = i; 17549 break; 17550 } 17551 if (subprog == -1) { 17552 bpf_log(log, "Subprog %s doesn't exist\n", tname); 17553 return -EINVAL; 17554 } 17555 conservative = aux->func_info_aux[subprog].unreliable; 17556 if (prog_extension) { 17557 if (conservative) { 17558 bpf_log(log, 17559 "Cannot replace static functions\n"); 17560 return -EINVAL; 17561 } 17562 if (!prog->jit_requested) { 17563 bpf_log(log, 17564 "Extension programs should be JITed\n"); 17565 return -EINVAL; 17566 } 17567 } 17568 if (!tgt_prog->jited) { 17569 bpf_log(log, "Can attach to only JITed progs\n"); 17570 return -EINVAL; 17571 } 17572 if (tgt_prog->type == prog->type) { 17573 /* Cannot fentry/fexit another fentry/fexit program. 17574 * Cannot attach program extension to another extension. 17575 * It's ok to attach fentry/fexit to extension program. 17576 */ 17577 bpf_log(log, "Cannot recursively attach\n"); 17578 return -EINVAL; 17579 } 17580 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 17581 prog_extension && 17582 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 17583 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 17584 /* Program extensions can extend all program types 17585 * except fentry/fexit. The reason is the following. 17586 * The fentry/fexit programs are used for performance 17587 * analysis, stats and can be attached to any program 17588 * type except themselves. When extension program is 17589 * replacing XDP function it is necessary to allow 17590 * performance analysis of all functions. Both original 17591 * XDP program and its program extension. Hence 17592 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 17593 * allowed. If extending of fentry/fexit was allowed it 17594 * would be possible to create long call chain 17595 * fentry->extension->fentry->extension beyond 17596 * reasonable stack size. Hence extending fentry is not 17597 * allowed. 17598 */ 17599 bpf_log(log, "Cannot extend fentry/fexit\n"); 17600 return -EINVAL; 17601 } 17602 } else { 17603 if (prog_extension) { 17604 bpf_log(log, "Cannot replace kernel functions\n"); 17605 return -EINVAL; 17606 } 17607 } 17608 17609 switch (prog->expected_attach_type) { 17610 case BPF_TRACE_RAW_TP: 17611 if (tgt_prog) { 17612 bpf_log(log, 17613 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 17614 return -EINVAL; 17615 } 17616 if (!btf_type_is_typedef(t)) { 17617 bpf_log(log, "attach_btf_id %u is not a typedef\n", 17618 btf_id); 17619 return -EINVAL; 17620 } 17621 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 17622 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 17623 btf_id, tname); 17624 return -EINVAL; 17625 } 17626 tname += sizeof(prefix) - 1; 17627 t = btf_type_by_id(btf, t->type); 17628 if (!btf_type_is_ptr(t)) 17629 /* should never happen in valid vmlinux build */ 17630 return -EINVAL; 17631 t = btf_type_by_id(btf, t->type); 17632 if (!btf_type_is_func_proto(t)) 17633 /* should never happen in valid vmlinux build */ 17634 return -EINVAL; 17635 17636 break; 17637 case BPF_TRACE_ITER: 17638 if (!btf_type_is_func(t)) { 17639 bpf_log(log, "attach_btf_id %u is not a function\n", 17640 btf_id); 17641 return -EINVAL; 17642 } 17643 t = btf_type_by_id(btf, t->type); 17644 if (!btf_type_is_func_proto(t)) 17645 return -EINVAL; 17646 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 17647 if (ret) 17648 return ret; 17649 break; 17650 default: 17651 if (!prog_extension) 17652 return -EINVAL; 17653 fallthrough; 17654 case BPF_MODIFY_RETURN: 17655 case BPF_LSM_MAC: 17656 case BPF_LSM_CGROUP: 17657 case BPF_TRACE_FENTRY: 17658 case BPF_TRACE_FEXIT: 17659 if (!btf_type_is_func(t)) { 17660 bpf_log(log, "attach_btf_id %u is not a function\n", 17661 btf_id); 17662 return -EINVAL; 17663 } 17664 if (prog_extension && 17665 btf_check_type_match(log, prog, btf, t)) 17666 return -EINVAL; 17667 t = btf_type_by_id(btf, t->type); 17668 if (!btf_type_is_func_proto(t)) 17669 return -EINVAL; 17670 17671 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 17672 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 17673 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 17674 return -EINVAL; 17675 17676 if (tgt_prog && conservative) 17677 t = NULL; 17678 17679 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 17680 if (ret < 0) 17681 return ret; 17682 17683 if (tgt_prog) { 17684 if (subprog == 0) 17685 addr = (long) tgt_prog->bpf_func; 17686 else 17687 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 17688 } else { 17689 addr = kallsyms_lookup_name(tname); 17690 if (!addr) { 17691 bpf_log(log, 17692 "The address of function %s cannot be found\n", 17693 tname); 17694 return -ENOENT; 17695 } 17696 } 17697 17698 if (prog->aux->sleepable) { 17699 ret = -EINVAL; 17700 switch (prog->type) { 17701 case BPF_PROG_TYPE_TRACING: 17702 17703 /* fentry/fexit/fmod_ret progs can be sleepable if they are 17704 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 17705 */ 17706 if (!check_non_sleepable_error_inject(btf_id) && 17707 within_error_injection_list(addr)) 17708 ret = 0; 17709 /* fentry/fexit/fmod_ret progs can also be sleepable if they are 17710 * in the fmodret id set with the KF_SLEEPABLE flag. 17711 */ 17712 else { 17713 u32 *flags = btf_kfunc_is_modify_return(btf, btf_id); 17714 17715 if (flags && (*flags & KF_SLEEPABLE)) 17716 ret = 0; 17717 } 17718 break; 17719 case BPF_PROG_TYPE_LSM: 17720 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 17721 * Only some of them are sleepable. 17722 */ 17723 if (bpf_lsm_is_sleepable_hook(btf_id)) 17724 ret = 0; 17725 break; 17726 default: 17727 break; 17728 } 17729 if (ret) { 17730 bpf_log(log, "%s is not sleepable\n", tname); 17731 return ret; 17732 } 17733 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 17734 if (tgt_prog) { 17735 bpf_log(log, "can't modify return codes of BPF programs\n"); 17736 return -EINVAL; 17737 } 17738 ret = -EINVAL; 17739 if (btf_kfunc_is_modify_return(btf, btf_id) || 17740 !check_attach_modify_return(addr, tname)) 17741 ret = 0; 17742 if (ret) { 17743 bpf_log(log, "%s() is not modifiable\n", tname); 17744 return ret; 17745 } 17746 } 17747 17748 break; 17749 } 17750 tgt_info->tgt_addr = addr; 17751 tgt_info->tgt_name = tname; 17752 tgt_info->tgt_type = t; 17753 return 0; 17754 } 17755 17756 BTF_SET_START(btf_id_deny) 17757 BTF_ID_UNUSED 17758 #ifdef CONFIG_SMP 17759 BTF_ID(func, migrate_disable) 17760 BTF_ID(func, migrate_enable) 17761 #endif 17762 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 17763 BTF_ID(func, rcu_read_unlock_strict) 17764 #endif 17765 BTF_SET_END(btf_id_deny) 17766 17767 static bool can_be_sleepable(struct bpf_prog *prog) 17768 { 17769 if (prog->type == BPF_PROG_TYPE_TRACING) { 17770 switch (prog->expected_attach_type) { 17771 case BPF_TRACE_FENTRY: 17772 case BPF_TRACE_FEXIT: 17773 case BPF_MODIFY_RETURN: 17774 case BPF_TRACE_ITER: 17775 return true; 17776 default: 17777 return false; 17778 } 17779 } 17780 return prog->type == BPF_PROG_TYPE_LSM || 17781 prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ || 17782 prog->type == BPF_PROG_TYPE_STRUCT_OPS; 17783 } 17784 17785 static int check_attach_btf_id(struct bpf_verifier_env *env) 17786 { 17787 struct bpf_prog *prog = env->prog; 17788 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 17789 struct bpf_attach_target_info tgt_info = {}; 17790 u32 btf_id = prog->aux->attach_btf_id; 17791 struct bpf_trampoline *tr; 17792 int ret; 17793 u64 key; 17794 17795 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 17796 if (prog->aux->sleepable) 17797 /* attach_btf_id checked to be zero already */ 17798 return 0; 17799 verbose(env, "Syscall programs can only be sleepable\n"); 17800 return -EINVAL; 17801 } 17802 17803 if (prog->aux->sleepable && !can_be_sleepable(prog)) { 17804 verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n"); 17805 return -EINVAL; 17806 } 17807 17808 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 17809 return check_struct_ops_btf_id(env); 17810 17811 if (prog->type != BPF_PROG_TYPE_TRACING && 17812 prog->type != BPF_PROG_TYPE_LSM && 17813 prog->type != BPF_PROG_TYPE_EXT) 17814 return 0; 17815 17816 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 17817 if (ret) 17818 return ret; 17819 17820 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 17821 /* to make freplace equivalent to their targets, they need to 17822 * inherit env->ops and expected_attach_type for the rest of the 17823 * verification 17824 */ 17825 env->ops = bpf_verifier_ops[tgt_prog->type]; 17826 prog->expected_attach_type = tgt_prog->expected_attach_type; 17827 } 17828 17829 /* store info about the attachment target that will be used later */ 17830 prog->aux->attach_func_proto = tgt_info.tgt_type; 17831 prog->aux->attach_func_name = tgt_info.tgt_name; 17832 17833 if (tgt_prog) { 17834 prog->aux->saved_dst_prog_type = tgt_prog->type; 17835 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 17836 } 17837 17838 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 17839 prog->aux->attach_btf_trace = true; 17840 return 0; 17841 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 17842 if (!bpf_iter_prog_supported(prog)) 17843 return -EINVAL; 17844 return 0; 17845 } 17846 17847 if (prog->type == BPF_PROG_TYPE_LSM) { 17848 ret = bpf_lsm_verify_prog(&env->log, prog); 17849 if (ret < 0) 17850 return ret; 17851 } else if (prog->type == BPF_PROG_TYPE_TRACING && 17852 btf_id_set_contains(&btf_id_deny, btf_id)) { 17853 return -EINVAL; 17854 } 17855 17856 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 17857 tr = bpf_trampoline_get(key, &tgt_info); 17858 if (!tr) 17859 return -ENOMEM; 17860 17861 prog->aux->dst_trampoline = tr; 17862 return 0; 17863 } 17864 17865 struct btf *bpf_get_btf_vmlinux(void) 17866 { 17867 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 17868 mutex_lock(&bpf_verifier_lock); 17869 if (!btf_vmlinux) 17870 btf_vmlinux = btf_parse_vmlinux(); 17871 mutex_unlock(&bpf_verifier_lock); 17872 } 17873 return btf_vmlinux; 17874 } 17875 17876 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) 17877 { 17878 u64 start_time = ktime_get_ns(); 17879 struct bpf_verifier_env *env; 17880 struct bpf_verifier_log *log; 17881 int i, len, ret = -EINVAL; 17882 bool is_priv; 17883 17884 /* no program is valid */ 17885 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 17886 return -EINVAL; 17887 17888 /* 'struct bpf_verifier_env' can be global, but since it's not small, 17889 * allocate/free it every time bpf_check() is called 17890 */ 17891 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 17892 if (!env) 17893 return -ENOMEM; 17894 log = &env->log; 17895 17896 len = (*prog)->len; 17897 env->insn_aux_data = 17898 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 17899 ret = -ENOMEM; 17900 if (!env->insn_aux_data) 17901 goto err_free_env; 17902 for (i = 0; i < len; i++) 17903 env->insn_aux_data[i].orig_idx = i; 17904 env->prog = *prog; 17905 env->ops = bpf_verifier_ops[env->prog->type]; 17906 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 17907 is_priv = bpf_capable(); 17908 17909 bpf_get_btf_vmlinux(); 17910 17911 /* grab the mutex to protect few globals used by verifier */ 17912 if (!is_priv) 17913 mutex_lock(&bpf_verifier_lock); 17914 17915 if (attr->log_level || attr->log_buf || attr->log_size) { 17916 /* user requested verbose verifier output 17917 * and supplied buffer to store the verification trace 17918 */ 17919 log->level = attr->log_level; 17920 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 17921 log->len_total = attr->log_size; 17922 17923 /* log attributes have to be sane */ 17924 if (!bpf_verifier_log_attr_valid(log)) { 17925 ret = -EINVAL; 17926 goto err_unlock; 17927 } 17928 } 17929 17930 mark_verifier_state_clean(env); 17931 17932 if (IS_ERR(btf_vmlinux)) { 17933 /* Either gcc or pahole or kernel are broken. */ 17934 verbose(env, "in-kernel BTF is malformed\n"); 17935 ret = PTR_ERR(btf_vmlinux); 17936 goto skip_full_check; 17937 } 17938 17939 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 17940 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 17941 env->strict_alignment = true; 17942 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 17943 env->strict_alignment = false; 17944 17945 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 17946 env->allow_uninit_stack = bpf_allow_uninit_stack(); 17947 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 17948 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 17949 env->bpf_capable = bpf_capable(); 17950 env->rcu_tag_supported = btf_vmlinux && 17951 btf_find_by_name_kind(btf_vmlinux, "rcu", BTF_KIND_TYPE_TAG) > 0; 17952 17953 if (is_priv) 17954 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 17955 17956 env->explored_states = kvcalloc(state_htab_size(env), 17957 sizeof(struct bpf_verifier_state_list *), 17958 GFP_USER); 17959 ret = -ENOMEM; 17960 if (!env->explored_states) 17961 goto skip_full_check; 17962 17963 ret = add_subprog_and_kfunc(env); 17964 if (ret < 0) 17965 goto skip_full_check; 17966 17967 ret = check_subprogs(env); 17968 if (ret < 0) 17969 goto skip_full_check; 17970 17971 ret = check_btf_info(env, attr, uattr); 17972 if (ret < 0) 17973 goto skip_full_check; 17974 17975 ret = check_attach_btf_id(env); 17976 if (ret) 17977 goto skip_full_check; 17978 17979 ret = resolve_pseudo_ldimm64(env); 17980 if (ret < 0) 17981 goto skip_full_check; 17982 17983 if (bpf_prog_is_offloaded(env->prog->aux)) { 17984 ret = bpf_prog_offload_verifier_prep(env->prog); 17985 if (ret) 17986 goto skip_full_check; 17987 } 17988 17989 ret = check_cfg(env); 17990 if (ret < 0) 17991 goto skip_full_check; 17992 17993 ret = do_check_subprogs(env); 17994 ret = ret ?: do_check_main(env); 17995 17996 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) 17997 ret = bpf_prog_offload_finalize(env); 17998 17999 skip_full_check: 18000 kvfree(env->explored_states); 18001 18002 if (ret == 0) 18003 ret = check_max_stack_depth(env); 18004 18005 /* instruction rewrites happen after this point */ 18006 if (ret == 0) 18007 ret = optimize_bpf_loop(env); 18008 18009 if (is_priv) { 18010 if (ret == 0) 18011 opt_hard_wire_dead_code_branches(env); 18012 if (ret == 0) 18013 ret = opt_remove_dead_code(env); 18014 if (ret == 0) 18015 ret = opt_remove_nops(env); 18016 } else { 18017 if (ret == 0) 18018 sanitize_dead_code(env); 18019 } 18020 18021 if (ret == 0) 18022 /* program is valid, convert *(u32*)(ctx + off) accesses */ 18023 ret = convert_ctx_accesses(env); 18024 18025 if (ret == 0) 18026 ret = do_misc_fixups(env); 18027 18028 /* do 32-bit optimization after insn patching has done so those patched 18029 * insns could be handled correctly. 18030 */ 18031 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { 18032 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 18033 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 18034 : false; 18035 } 18036 18037 if (ret == 0) 18038 ret = fixup_call_args(env); 18039 18040 env->verification_time = ktime_get_ns() - start_time; 18041 print_verification_stats(env); 18042 env->prog->aux->verified_insns = env->insn_processed; 18043 18044 if (log->level && bpf_verifier_log_full(log)) 18045 ret = -ENOSPC; 18046 if (log->level && !log->ubuf) { 18047 ret = -EFAULT; 18048 goto err_release_maps; 18049 } 18050 18051 if (ret) 18052 goto err_release_maps; 18053 18054 if (env->used_map_cnt) { 18055 /* if program passed verifier, update used_maps in bpf_prog_info */ 18056 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 18057 sizeof(env->used_maps[0]), 18058 GFP_KERNEL); 18059 18060 if (!env->prog->aux->used_maps) { 18061 ret = -ENOMEM; 18062 goto err_release_maps; 18063 } 18064 18065 memcpy(env->prog->aux->used_maps, env->used_maps, 18066 sizeof(env->used_maps[0]) * env->used_map_cnt); 18067 env->prog->aux->used_map_cnt = env->used_map_cnt; 18068 } 18069 if (env->used_btf_cnt) { 18070 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 18071 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 18072 sizeof(env->used_btfs[0]), 18073 GFP_KERNEL); 18074 if (!env->prog->aux->used_btfs) { 18075 ret = -ENOMEM; 18076 goto err_release_maps; 18077 } 18078 18079 memcpy(env->prog->aux->used_btfs, env->used_btfs, 18080 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 18081 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 18082 } 18083 if (env->used_map_cnt || env->used_btf_cnt) { 18084 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 18085 * bpf_ld_imm64 instructions 18086 */ 18087 convert_pseudo_ld_imm64(env); 18088 } 18089 18090 adjust_btf_func(env); 18091 18092 err_release_maps: 18093 if (!env->prog->aux->used_maps) 18094 /* if we didn't copy map pointers into bpf_prog_info, release 18095 * them now. Otherwise free_used_maps() will release them. 18096 */ 18097 release_maps(env); 18098 if (!env->prog->aux->used_btfs) 18099 release_btfs(env); 18100 18101 /* extension progs temporarily inherit the attach_type of their targets 18102 for verification purposes, so set it back to zero before returning 18103 */ 18104 if (env->prog->type == BPF_PROG_TYPE_EXT) 18105 env->prog->expected_attach_type = 0; 18106 18107 *prog = env->prog; 18108 err_unlock: 18109 if (!is_priv) 18110 mutex_unlock(&bpf_verifier_lock); 18111 vfree(env->insn_aux_data); 18112 err_free_env: 18113 kfree(env); 18114 return ret; 18115 } 18116