1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 #include <linux/poison.h> 27 28 #include "disasm.h" 29 30 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 31 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 32 [_id] = & _name ## _verifier_ops, 33 #define BPF_MAP_TYPE(_id, _ops) 34 #define BPF_LINK_TYPE(_id, _name) 35 #include <linux/bpf_types.h> 36 #undef BPF_PROG_TYPE 37 #undef BPF_MAP_TYPE 38 #undef BPF_LINK_TYPE 39 }; 40 41 /* bpf_check() is a static code analyzer that walks eBPF program 42 * instruction by instruction and updates register/stack state. 43 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 44 * 45 * The first pass is depth-first-search to check that the program is a DAG. 46 * It rejects the following programs: 47 * - larger than BPF_MAXINSNS insns 48 * - if loop is present (detected via back-edge) 49 * - unreachable insns exist (shouldn't be a forest. program = one function) 50 * - out of bounds or malformed jumps 51 * The second pass is all possible path descent from the 1st insn. 52 * Since it's analyzing all paths through the program, the length of the 53 * analysis is limited to 64k insn, which may be hit even if total number of 54 * insn is less then 4K, but there are too many branches that change stack/regs. 55 * Number of 'branches to be analyzed' is limited to 1k 56 * 57 * On entry to each instruction, each register has a type, and the instruction 58 * changes the types of the registers depending on instruction semantics. 59 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 60 * copied to R1. 61 * 62 * All registers are 64-bit. 63 * R0 - return register 64 * R1-R5 argument passing registers 65 * R6-R9 callee saved registers 66 * R10 - frame pointer read-only 67 * 68 * At the start of BPF program the register R1 contains a pointer to bpf_context 69 * and has type PTR_TO_CTX. 70 * 71 * Verifier tracks arithmetic operations on pointers in case: 72 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 73 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 74 * 1st insn copies R10 (which has FRAME_PTR) type into R1 75 * and 2nd arithmetic instruction is pattern matched to recognize 76 * that it wants to construct a pointer to some element within stack. 77 * So after 2nd insn, the register R1 has type PTR_TO_STACK 78 * (and -20 constant is saved for further stack bounds checking). 79 * Meaning that this reg is a pointer to stack plus known immediate constant. 80 * 81 * Most of the time the registers have SCALAR_VALUE type, which 82 * means the register has some value, but it's not a valid pointer. 83 * (like pointer plus pointer becomes SCALAR_VALUE type) 84 * 85 * When verifier sees load or store instructions the type of base register 86 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 87 * four pointer types recognized by check_mem_access() function. 88 * 89 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 90 * and the range of [ptr, ptr + map's value_size) is accessible. 91 * 92 * registers used to pass values to function calls are checked against 93 * function argument constraints. 94 * 95 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 96 * It means that the register type passed to this function must be 97 * PTR_TO_STACK and it will be used inside the function as 98 * 'pointer to map element key' 99 * 100 * For example the argument constraints for bpf_map_lookup_elem(): 101 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 102 * .arg1_type = ARG_CONST_MAP_PTR, 103 * .arg2_type = ARG_PTR_TO_MAP_KEY, 104 * 105 * ret_type says that this function returns 'pointer to map elem value or null' 106 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 107 * 2nd argument should be a pointer to stack, which will be used inside 108 * the helper function as a pointer to map element key. 109 * 110 * On the kernel side the helper function looks like: 111 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 112 * { 113 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 114 * void *key = (void *) (unsigned long) r2; 115 * void *value; 116 * 117 * here kernel can access 'key' and 'map' pointers safely, knowing that 118 * [key, key + map->key_size) bytes are valid and were initialized on 119 * the stack of eBPF program. 120 * } 121 * 122 * Corresponding eBPF program may look like: 123 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 124 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 125 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 126 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 127 * here verifier looks at prototype of map_lookup_elem() and sees: 128 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 129 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 130 * 131 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 132 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 133 * and were initialized prior to this call. 134 * If it's ok, then verifier allows this BPF_CALL insn and looks at 135 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 136 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 137 * returns either pointer to map value or NULL. 138 * 139 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 140 * insn, the register holding that pointer in the true branch changes state to 141 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 142 * branch. See check_cond_jmp_op(). 143 * 144 * After the call R0 is set to return type of the function and registers R1-R5 145 * are set to NOT_INIT to indicate that they are no longer readable. 146 * 147 * The following reference types represent a potential reference to a kernel 148 * resource which, after first being allocated, must be checked and freed by 149 * the BPF program: 150 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 151 * 152 * When the verifier sees a helper call return a reference type, it allocates a 153 * pointer id for the reference and stores it in the current function state. 154 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 155 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 156 * passes through a NULL-check conditional. For the branch wherein the state is 157 * changed to CONST_IMM, the verifier releases the reference. 158 * 159 * For each helper function that allocates a reference, such as 160 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 161 * bpf_sk_release(). When a reference type passes into the release function, 162 * the verifier also releases the reference. If any unchecked or unreleased 163 * reference remains at the end of the program, the verifier rejects it. 164 */ 165 166 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 167 struct bpf_verifier_stack_elem { 168 /* verifer state is 'st' 169 * before processing instruction 'insn_idx' 170 * and after processing instruction 'prev_insn_idx' 171 */ 172 struct bpf_verifier_state st; 173 int insn_idx; 174 int prev_insn_idx; 175 struct bpf_verifier_stack_elem *next; 176 /* length of verifier log at the time this state was pushed on stack */ 177 u32 log_pos; 178 }; 179 180 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 181 #define BPF_COMPLEXITY_LIMIT_STATES 64 182 183 #define BPF_MAP_KEY_POISON (1ULL << 63) 184 #define BPF_MAP_KEY_SEEN (1ULL << 62) 185 186 #define BPF_MAP_PTR_UNPRIV 1UL 187 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 188 POISON_POINTER_DELTA)) 189 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 190 191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 193 194 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 195 { 196 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 197 } 198 199 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 200 { 201 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 202 } 203 204 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 205 const struct bpf_map *map, bool unpriv) 206 { 207 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 208 unpriv |= bpf_map_ptr_unpriv(aux); 209 aux->map_ptr_state = (unsigned long)map | 210 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 211 } 212 213 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 214 { 215 return aux->map_key_state & BPF_MAP_KEY_POISON; 216 } 217 218 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 219 { 220 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 221 } 222 223 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 224 { 225 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 226 } 227 228 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 229 { 230 bool poisoned = bpf_map_key_poisoned(aux); 231 232 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 233 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 234 } 235 236 static bool bpf_pseudo_call(const struct bpf_insn *insn) 237 { 238 return insn->code == (BPF_JMP | BPF_CALL) && 239 insn->src_reg == BPF_PSEUDO_CALL; 240 } 241 242 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 243 { 244 return insn->code == (BPF_JMP | BPF_CALL) && 245 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 246 } 247 248 struct bpf_call_arg_meta { 249 struct bpf_map *map_ptr; 250 bool raw_mode; 251 bool pkt_access; 252 u8 release_regno; 253 int regno; 254 int access_size; 255 int mem_size; 256 u64 msize_max_value; 257 int ref_obj_id; 258 int map_uid; 259 int func_id; 260 struct btf *btf; 261 u32 btf_id; 262 struct btf *ret_btf; 263 u32 ret_btf_id; 264 u32 subprogno; 265 struct btf_field *kptr_field; 266 u8 uninit_dynptr_regno; 267 }; 268 269 struct btf *btf_vmlinux; 270 271 static DEFINE_MUTEX(bpf_verifier_lock); 272 273 static const struct bpf_line_info * 274 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 275 { 276 const struct bpf_line_info *linfo; 277 const struct bpf_prog *prog; 278 u32 i, nr_linfo; 279 280 prog = env->prog; 281 nr_linfo = prog->aux->nr_linfo; 282 283 if (!nr_linfo || insn_off >= prog->len) 284 return NULL; 285 286 linfo = prog->aux->linfo; 287 for (i = 1; i < nr_linfo; i++) 288 if (insn_off < linfo[i].insn_off) 289 break; 290 291 return &linfo[i - 1]; 292 } 293 294 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 295 va_list args) 296 { 297 unsigned int n; 298 299 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 300 301 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 302 "verifier log line truncated - local buffer too short\n"); 303 304 if (log->level == BPF_LOG_KERNEL) { 305 bool newline = n > 0 && log->kbuf[n - 1] == '\n'; 306 307 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n"); 308 return; 309 } 310 311 n = min(log->len_total - log->len_used - 1, n); 312 log->kbuf[n] = '\0'; 313 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 314 log->len_used += n; 315 else 316 log->ubuf = NULL; 317 } 318 319 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 320 { 321 char zero = 0; 322 323 if (!bpf_verifier_log_needed(log)) 324 return; 325 326 log->len_used = new_pos; 327 if (put_user(zero, log->ubuf + new_pos)) 328 log->ubuf = NULL; 329 } 330 331 /* log_level controls verbosity level of eBPF verifier. 332 * bpf_verifier_log_write() is used to dump the verification trace to the log, 333 * so the user can figure out what's wrong with the program 334 */ 335 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 336 const char *fmt, ...) 337 { 338 va_list args; 339 340 if (!bpf_verifier_log_needed(&env->log)) 341 return; 342 343 va_start(args, fmt); 344 bpf_verifier_vlog(&env->log, fmt, args); 345 va_end(args); 346 } 347 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 348 349 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 350 { 351 struct bpf_verifier_env *env = private_data; 352 va_list args; 353 354 if (!bpf_verifier_log_needed(&env->log)) 355 return; 356 357 va_start(args, fmt); 358 bpf_verifier_vlog(&env->log, fmt, args); 359 va_end(args); 360 } 361 362 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 363 const char *fmt, ...) 364 { 365 va_list args; 366 367 if (!bpf_verifier_log_needed(log)) 368 return; 369 370 va_start(args, fmt); 371 bpf_verifier_vlog(log, fmt, args); 372 va_end(args); 373 } 374 EXPORT_SYMBOL_GPL(bpf_log); 375 376 static const char *ltrim(const char *s) 377 { 378 while (isspace(*s)) 379 s++; 380 381 return s; 382 } 383 384 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 385 u32 insn_off, 386 const char *prefix_fmt, ...) 387 { 388 const struct bpf_line_info *linfo; 389 390 if (!bpf_verifier_log_needed(&env->log)) 391 return; 392 393 linfo = find_linfo(env, insn_off); 394 if (!linfo || linfo == env->prev_linfo) 395 return; 396 397 if (prefix_fmt) { 398 va_list args; 399 400 va_start(args, prefix_fmt); 401 bpf_verifier_vlog(&env->log, prefix_fmt, args); 402 va_end(args); 403 } 404 405 verbose(env, "%s\n", 406 ltrim(btf_name_by_offset(env->prog->aux->btf, 407 linfo->line_off))); 408 409 env->prev_linfo = linfo; 410 } 411 412 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 413 struct bpf_reg_state *reg, 414 struct tnum *range, const char *ctx, 415 const char *reg_name) 416 { 417 char tn_buf[48]; 418 419 verbose(env, "At %s the register %s ", ctx, reg_name); 420 if (!tnum_is_unknown(reg->var_off)) { 421 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 422 verbose(env, "has value %s", tn_buf); 423 } else { 424 verbose(env, "has unknown scalar value"); 425 } 426 tnum_strn(tn_buf, sizeof(tn_buf), *range); 427 verbose(env, " should have been in %s\n", tn_buf); 428 } 429 430 static bool type_is_pkt_pointer(enum bpf_reg_type type) 431 { 432 type = base_type(type); 433 return type == PTR_TO_PACKET || 434 type == PTR_TO_PACKET_META; 435 } 436 437 static bool type_is_sk_pointer(enum bpf_reg_type type) 438 { 439 return type == PTR_TO_SOCKET || 440 type == PTR_TO_SOCK_COMMON || 441 type == PTR_TO_TCP_SOCK || 442 type == PTR_TO_XDP_SOCK; 443 } 444 445 static bool reg_type_not_null(enum bpf_reg_type type) 446 { 447 return type == PTR_TO_SOCKET || 448 type == PTR_TO_TCP_SOCK || 449 type == PTR_TO_MAP_VALUE || 450 type == PTR_TO_MAP_KEY || 451 type == PTR_TO_SOCK_COMMON; 452 } 453 454 static bool type_is_ptr_alloc_obj(u32 type) 455 { 456 return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; 457 } 458 459 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 460 { 461 struct btf_record *rec = NULL; 462 struct btf_struct_meta *meta; 463 464 if (reg->type == PTR_TO_MAP_VALUE) { 465 rec = reg->map_ptr->record; 466 } else if (type_is_ptr_alloc_obj(reg->type)) { 467 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 468 if (meta) 469 rec = meta->record; 470 } 471 return rec; 472 } 473 474 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 475 { 476 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); 477 } 478 479 static bool type_is_rdonly_mem(u32 type) 480 { 481 return type & MEM_RDONLY; 482 } 483 484 static bool type_may_be_null(u32 type) 485 { 486 return type & PTR_MAYBE_NULL; 487 } 488 489 static bool is_acquire_function(enum bpf_func_id func_id, 490 const struct bpf_map *map) 491 { 492 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 493 494 if (func_id == BPF_FUNC_sk_lookup_tcp || 495 func_id == BPF_FUNC_sk_lookup_udp || 496 func_id == BPF_FUNC_skc_lookup_tcp || 497 func_id == BPF_FUNC_ringbuf_reserve || 498 func_id == BPF_FUNC_kptr_xchg) 499 return true; 500 501 if (func_id == BPF_FUNC_map_lookup_elem && 502 (map_type == BPF_MAP_TYPE_SOCKMAP || 503 map_type == BPF_MAP_TYPE_SOCKHASH)) 504 return true; 505 506 return false; 507 } 508 509 static bool is_ptr_cast_function(enum bpf_func_id func_id) 510 { 511 return func_id == BPF_FUNC_tcp_sock || 512 func_id == BPF_FUNC_sk_fullsock || 513 func_id == BPF_FUNC_skc_to_tcp_sock || 514 func_id == BPF_FUNC_skc_to_tcp6_sock || 515 func_id == BPF_FUNC_skc_to_udp6_sock || 516 func_id == BPF_FUNC_skc_to_mptcp_sock || 517 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 518 func_id == BPF_FUNC_skc_to_tcp_request_sock; 519 } 520 521 static bool is_dynptr_ref_function(enum bpf_func_id func_id) 522 { 523 return func_id == BPF_FUNC_dynptr_data; 524 } 525 526 static bool is_callback_calling_function(enum bpf_func_id func_id) 527 { 528 return func_id == BPF_FUNC_for_each_map_elem || 529 func_id == BPF_FUNC_timer_set_callback || 530 func_id == BPF_FUNC_find_vma || 531 func_id == BPF_FUNC_loop || 532 func_id == BPF_FUNC_user_ringbuf_drain; 533 } 534 535 static bool is_storage_get_function(enum bpf_func_id func_id) 536 { 537 return func_id == BPF_FUNC_sk_storage_get || 538 func_id == BPF_FUNC_inode_storage_get || 539 func_id == BPF_FUNC_task_storage_get || 540 func_id == BPF_FUNC_cgrp_storage_get; 541 } 542 543 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, 544 const struct bpf_map *map) 545 { 546 int ref_obj_uses = 0; 547 548 if (is_ptr_cast_function(func_id)) 549 ref_obj_uses++; 550 if (is_acquire_function(func_id, map)) 551 ref_obj_uses++; 552 if (is_dynptr_ref_function(func_id)) 553 ref_obj_uses++; 554 555 return ref_obj_uses > 1; 556 } 557 558 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 559 { 560 return BPF_CLASS(insn->code) == BPF_STX && 561 BPF_MODE(insn->code) == BPF_ATOMIC && 562 insn->imm == BPF_CMPXCHG; 563 } 564 565 /* string representation of 'enum bpf_reg_type' 566 * 567 * Note that reg_type_str() can not appear more than once in a single verbose() 568 * statement. 569 */ 570 static const char *reg_type_str(struct bpf_verifier_env *env, 571 enum bpf_reg_type type) 572 { 573 char postfix[16] = {0}, prefix[64] = {0}; 574 static const char * const str[] = { 575 [NOT_INIT] = "?", 576 [SCALAR_VALUE] = "scalar", 577 [PTR_TO_CTX] = "ctx", 578 [CONST_PTR_TO_MAP] = "map_ptr", 579 [PTR_TO_MAP_VALUE] = "map_value", 580 [PTR_TO_STACK] = "fp", 581 [PTR_TO_PACKET] = "pkt", 582 [PTR_TO_PACKET_META] = "pkt_meta", 583 [PTR_TO_PACKET_END] = "pkt_end", 584 [PTR_TO_FLOW_KEYS] = "flow_keys", 585 [PTR_TO_SOCKET] = "sock", 586 [PTR_TO_SOCK_COMMON] = "sock_common", 587 [PTR_TO_TCP_SOCK] = "tcp_sock", 588 [PTR_TO_TP_BUFFER] = "tp_buffer", 589 [PTR_TO_XDP_SOCK] = "xdp_sock", 590 [PTR_TO_BTF_ID] = "ptr_", 591 [PTR_TO_MEM] = "mem", 592 [PTR_TO_BUF] = "buf", 593 [PTR_TO_FUNC] = "func", 594 [PTR_TO_MAP_KEY] = "map_key", 595 [PTR_TO_DYNPTR] = "dynptr_ptr", 596 }; 597 598 if (type & PTR_MAYBE_NULL) { 599 if (base_type(type) == PTR_TO_BTF_ID) 600 strncpy(postfix, "or_null_", 16); 601 else 602 strncpy(postfix, "_or_null", 16); 603 } 604 605 snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", 606 type & MEM_RDONLY ? "rdonly_" : "", 607 type & MEM_RINGBUF ? "ringbuf_" : "", 608 type & MEM_USER ? "user_" : "", 609 type & MEM_PERCPU ? "percpu_" : "", 610 type & MEM_RCU ? "rcu_" : "", 611 type & PTR_UNTRUSTED ? "untrusted_" : "", 612 type & PTR_TRUSTED ? "trusted_" : "" 613 ); 614 615 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", 616 prefix, str[base_type(type)], postfix); 617 return env->type_str_buf; 618 } 619 620 static char slot_type_char[] = { 621 [STACK_INVALID] = '?', 622 [STACK_SPILL] = 'r', 623 [STACK_MISC] = 'm', 624 [STACK_ZERO] = '0', 625 [STACK_DYNPTR] = 'd', 626 }; 627 628 static void print_liveness(struct bpf_verifier_env *env, 629 enum bpf_reg_liveness live) 630 { 631 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 632 verbose(env, "_"); 633 if (live & REG_LIVE_READ) 634 verbose(env, "r"); 635 if (live & REG_LIVE_WRITTEN) 636 verbose(env, "w"); 637 if (live & REG_LIVE_DONE) 638 verbose(env, "D"); 639 } 640 641 static int get_spi(s32 off) 642 { 643 return (-off - 1) / BPF_REG_SIZE; 644 } 645 646 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 647 { 648 int allocated_slots = state->allocated_stack / BPF_REG_SIZE; 649 650 /* We need to check that slots between [spi - nr_slots + 1, spi] are 651 * within [0, allocated_stack). 652 * 653 * Please note that the spi grows downwards. For example, a dynptr 654 * takes the size of two stack slots; the first slot will be at 655 * spi and the second slot will be at spi - 1. 656 */ 657 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 658 } 659 660 static struct bpf_func_state *func(struct bpf_verifier_env *env, 661 const struct bpf_reg_state *reg) 662 { 663 struct bpf_verifier_state *cur = env->cur_state; 664 665 return cur->frame[reg->frameno]; 666 } 667 668 static const char *kernel_type_name(const struct btf* btf, u32 id) 669 { 670 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 671 } 672 673 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) 674 { 675 env->scratched_regs |= 1U << regno; 676 } 677 678 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) 679 { 680 env->scratched_stack_slots |= 1ULL << spi; 681 } 682 683 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) 684 { 685 return (env->scratched_regs >> regno) & 1; 686 } 687 688 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) 689 { 690 return (env->scratched_stack_slots >> regno) & 1; 691 } 692 693 static bool verifier_state_scratched(const struct bpf_verifier_env *env) 694 { 695 return env->scratched_regs || env->scratched_stack_slots; 696 } 697 698 static void mark_verifier_state_clean(struct bpf_verifier_env *env) 699 { 700 env->scratched_regs = 0U; 701 env->scratched_stack_slots = 0ULL; 702 } 703 704 /* Used for printing the entire verifier state. */ 705 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) 706 { 707 env->scratched_regs = ~0U; 708 env->scratched_stack_slots = ~0ULL; 709 } 710 711 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 712 { 713 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 714 case DYNPTR_TYPE_LOCAL: 715 return BPF_DYNPTR_TYPE_LOCAL; 716 case DYNPTR_TYPE_RINGBUF: 717 return BPF_DYNPTR_TYPE_RINGBUF; 718 default: 719 return BPF_DYNPTR_TYPE_INVALID; 720 } 721 } 722 723 static bool dynptr_type_refcounted(enum bpf_dynptr_type type) 724 { 725 return type == BPF_DYNPTR_TYPE_RINGBUF; 726 } 727 728 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 729 enum bpf_arg_type arg_type, int insn_idx) 730 { 731 struct bpf_func_state *state = func(env, reg); 732 enum bpf_dynptr_type type; 733 int spi, i, id; 734 735 spi = get_spi(reg->off); 736 737 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 738 return -EINVAL; 739 740 for (i = 0; i < BPF_REG_SIZE; i++) { 741 state->stack[spi].slot_type[i] = STACK_DYNPTR; 742 state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; 743 } 744 745 type = arg_to_dynptr_type(arg_type); 746 if (type == BPF_DYNPTR_TYPE_INVALID) 747 return -EINVAL; 748 749 state->stack[spi].spilled_ptr.dynptr.first_slot = true; 750 state->stack[spi].spilled_ptr.dynptr.type = type; 751 state->stack[spi - 1].spilled_ptr.dynptr.type = type; 752 753 if (dynptr_type_refcounted(type)) { 754 /* The id is used to track proper releasing */ 755 id = acquire_reference_state(env, insn_idx); 756 if (id < 0) 757 return id; 758 759 state->stack[spi].spilled_ptr.id = id; 760 state->stack[spi - 1].spilled_ptr.id = id; 761 } 762 763 return 0; 764 } 765 766 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 767 { 768 struct bpf_func_state *state = func(env, reg); 769 int spi, i; 770 771 spi = get_spi(reg->off); 772 773 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 774 return -EINVAL; 775 776 for (i = 0; i < BPF_REG_SIZE; i++) { 777 state->stack[spi].slot_type[i] = STACK_INVALID; 778 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 779 } 780 781 /* Invalidate any slices associated with this dynptr */ 782 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 783 release_reference(env, state->stack[spi].spilled_ptr.id); 784 state->stack[spi].spilled_ptr.id = 0; 785 state->stack[spi - 1].spilled_ptr.id = 0; 786 } 787 788 state->stack[spi].spilled_ptr.dynptr.first_slot = false; 789 state->stack[spi].spilled_ptr.dynptr.type = 0; 790 state->stack[spi - 1].spilled_ptr.dynptr.type = 0; 791 792 return 0; 793 } 794 795 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 796 { 797 struct bpf_func_state *state = func(env, reg); 798 int spi = get_spi(reg->off); 799 int i; 800 801 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 802 return true; 803 804 for (i = 0; i < BPF_REG_SIZE; i++) { 805 if (state->stack[spi].slot_type[i] == STACK_DYNPTR || 806 state->stack[spi - 1].slot_type[i] == STACK_DYNPTR) 807 return false; 808 } 809 810 return true; 811 } 812 813 bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, 814 struct bpf_reg_state *reg) 815 { 816 struct bpf_func_state *state = func(env, reg); 817 int spi = get_spi(reg->off); 818 int i; 819 820 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) || 821 !state->stack[spi].spilled_ptr.dynptr.first_slot) 822 return false; 823 824 for (i = 0; i < BPF_REG_SIZE; i++) { 825 if (state->stack[spi].slot_type[i] != STACK_DYNPTR || 826 state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) 827 return false; 828 } 829 830 return true; 831 } 832 833 bool is_dynptr_type_expected(struct bpf_verifier_env *env, 834 struct bpf_reg_state *reg, 835 enum bpf_arg_type arg_type) 836 { 837 struct bpf_func_state *state = func(env, reg); 838 enum bpf_dynptr_type dynptr_type; 839 int spi = get_spi(reg->off); 840 841 /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ 842 if (arg_type == ARG_PTR_TO_DYNPTR) 843 return true; 844 845 dynptr_type = arg_to_dynptr_type(arg_type); 846 847 return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; 848 } 849 850 /* The reg state of a pointer or a bounded scalar was saved when 851 * it was spilled to the stack. 852 */ 853 static bool is_spilled_reg(const struct bpf_stack_state *stack) 854 { 855 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 856 } 857 858 static void scrub_spilled_slot(u8 *stype) 859 { 860 if (*stype != STACK_INVALID) 861 *stype = STACK_MISC; 862 } 863 864 static void print_verifier_state(struct bpf_verifier_env *env, 865 const struct bpf_func_state *state, 866 bool print_all) 867 { 868 const struct bpf_reg_state *reg; 869 enum bpf_reg_type t; 870 int i; 871 872 if (state->frameno) 873 verbose(env, " frame%d:", state->frameno); 874 for (i = 0; i < MAX_BPF_REG; i++) { 875 reg = &state->regs[i]; 876 t = reg->type; 877 if (t == NOT_INIT) 878 continue; 879 if (!print_all && !reg_scratched(env, i)) 880 continue; 881 verbose(env, " R%d", i); 882 print_liveness(env, reg->live); 883 verbose(env, "="); 884 if (t == SCALAR_VALUE && reg->precise) 885 verbose(env, "P"); 886 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 887 tnum_is_const(reg->var_off)) { 888 /* reg->off should be 0 for SCALAR_VALUE */ 889 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 890 verbose(env, "%lld", reg->var_off.value + reg->off); 891 } else { 892 const char *sep = ""; 893 894 verbose(env, "%s", reg_type_str(env, t)); 895 if (base_type(t) == PTR_TO_BTF_ID) 896 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 897 verbose(env, "("); 898 /* 899 * _a stands for append, was shortened to avoid multiline statements below. 900 * This macro is used to output a comma separated list of attributes. 901 */ 902 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) 903 904 if (reg->id) 905 verbose_a("id=%d", reg->id); 906 if (reg->ref_obj_id) 907 verbose_a("ref_obj_id=%d", reg->ref_obj_id); 908 if (t != SCALAR_VALUE) 909 verbose_a("off=%d", reg->off); 910 if (type_is_pkt_pointer(t)) 911 verbose_a("r=%d", reg->range); 912 else if (base_type(t) == CONST_PTR_TO_MAP || 913 base_type(t) == PTR_TO_MAP_KEY || 914 base_type(t) == PTR_TO_MAP_VALUE) 915 verbose_a("ks=%d,vs=%d", 916 reg->map_ptr->key_size, 917 reg->map_ptr->value_size); 918 if (tnum_is_const(reg->var_off)) { 919 /* Typically an immediate SCALAR_VALUE, but 920 * could be a pointer whose offset is too big 921 * for reg->off 922 */ 923 verbose_a("imm=%llx", reg->var_off.value); 924 } else { 925 if (reg->smin_value != reg->umin_value && 926 reg->smin_value != S64_MIN) 927 verbose_a("smin=%lld", (long long)reg->smin_value); 928 if (reg->smax_value != reg->umax_value && 929 reg->smax_value != S64_MAX) 930 verbose_a("smax=%lld", (long long)reg->smax_value); 931 if (reg->umin_value != 0) 932 verbose_a("umin=%llu", (unsigned long long)reg->umin_value); 933 if (reg->umax_value != U64_MAX) 934 verbose_a("umax=%llu", (unsigned long long)reg->umax_value); 935 if (!tnum_is_unknown(reg->var_off)) { 936 char tn_buf[48]; 937 938 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 939 verbose_a("var_off=%s", tn_buf); 940 } 941 if (reg->s32_min_value != reg->smin_value && 942 reg->s32_min_value != S32_MIN) 943 verbose_a("s32_min=%d", (int)(reg->s32_min_value)); 944 if (reg->s32_max_value != reg->smax_value && 945 reg->s32_max_value != S32_MAX) 946 verbose_a("s32_max=%d", (int)(reg->s32_max_value)); 947 if (reg->u32_min_value != reg->umin_value && 948 reg->u32_min_value != U32_MIN) 949 verbose_a("u32_min=%d", (int)(reg->u32_min_value)); 950 if (reg->u32_max_value != reg->umax_value && 951 reg->u32_max_value != U32_MAX) 952 verbose_a("u32_max=%d", (int)(reg->u32_max_value)); 953 } 954 #undef verbose_a 955 956 verbose(env, ")"); 957 } 958 } 959 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 960 char types_buf[BPF_REG_SIZE + 1]; 961 bool valid = false; 962 int j; 963 964 for (j = 0; j < BPF_REG_SIZE; j++) { 965 if (state->stack[i].slot_type[j] != STACK_INVALID) 966 valid = true; 967 types_buf[j] = slot_type_char[ 968 state->stack[i].slot_type[j]]; 969 } 970 types_buf[BPF_REG_SIZE] = 0; 971 if (!valid) 972 continue; 973 if (!print_all && !stack_slot_scratched(env, i)) 974 continue; 975 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 976 print_liveness(env, state->stack[i].spilled_ptr.live); 977 if (is_spilled_reg(&state->stack[i])) { 978 reg = &state->stack[i].spilled_ptr; 979 t = reg->type; 980 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 981 if (t == SCALAR_VALUE && reg->precise) 982 verbose(env, "P"); 983 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 984 verbose(env, "%lld", reg->var_off.value + reg->off); 985 } else { 986 verbose(env, "=%s", types_buf); 987 } 988 } 989 if (state->acquired_refs && state->refs[0].id) { 990 verbose(env, " refs=%d", state->refs[0].id); 991 for (i = 1; i < state->acquired_refs; i++) 992 if (state->refs[i].id) 993 verbose(env, ",%d", state->refs[i].id); 994 } 995 if (state->in_callback_fn) 996 verbose(env, " cb"); 997 if (state->in_async_callback_fn) 998 verbose(env, " async_cb"); 999 verbose(env, "\n"); 1000 mark_verifier_state_clean(env); 1001 } 1002 1003 static inline u32 vlog_alignment(u32 pos) 1004 { 1005 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), 1006 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 1007 } 1008 1009 static void print_insn_state(struct bpf_verifier_env *env, 1010 const struct bpf_func_state *state) 1011 { 1012 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { 1013 /* remove new line character */ 1014 bpf_vlog_reset(&env->log, env->prev_log_len - 1); 1015 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); 1016 } else { 1017 verbose(env, "%d:", env->insn_idx); 1018 } 1019 print_verifier_state(env, state, false); 1020 } 1021 1022 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1023 * small to hold src. This is different from krealloc since we don't want to preserve 1024 * the contents of dst. 1025 * 1026 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1027 * not be allocated. 1028 */ 1029 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1030 { 1031 size_t bytes; 1032 1033 if (ZERO_OR_NULL_PTR(src)) 1034 goto out; 1035 1036 if (unlikely(check_mul_overflow(n, size, &bytes))) 1037 return NULL; 1038 1039 if (ksize(dst) < ksize(src)) { 1040 kfree(dst); 1041 dst = kmalloc_track_caller(kmalloc_size_roundup(bytes), flags); 1042 if (!dst) 1043 return NULL; 1044 } 1045 1046 memcpy(dst, src, bytes); 1047 out: 1048 return dst ? dst : ZERO_SIZE_PTR; 1049 } 1050 1051 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1052 * small to hold new_n items. new items are zeroed out if the array grows. 1053 * 1054 * Contrary to krealloc_array, does not free arr if new_n is zero. 1055 */ 1056 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1057 { 1058 size_t alloc_size; 1059 void *new_arr; 1060 1061 if (!new_n || old_n == new_n) 1062 goto out; 1063 1064 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1065 new_arr = krealloc(arr, alloc_size, GFP_KERNEL); 1066 if (!new_arr) { 1067 kfree(arr); 1068 return NULL; 1069 } 1070 arr = new_arr; 1071 1072 if (new_n > old_n) 1073 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1074 1075 out: 1076 return arr ? arr : ZERO_SIZE_PTR; 1077 } 1078 1079 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1080 { 1081 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1082 sizeof(struct bpf_reference_state), GFP_KERNEL); 1083 if (!dst->refs) 1084 return -ENOMEM; 1085 1086 dst->acquired_refs = src->acquired_refs; 1087 return 0; 1088 } 1089 1090 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1091 { 1092 size_t n = src->allocated_stack / BPF_REG_SIZE; 1093 1094 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1095 GFP_KERNEL); 1096 if (!dst->stack) 1097 return -ENOMEM; 1098 1099 dst->allocated_stack = src->allocated_stack; 1100 return 0; 1101 } 1102 1103 static int resize_reference_state(struct bpf_func_state *state, size_t n) 1104 { 1105 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1106 sizeof(struct bpf_reference_state)); 1107 if (!state->refs) 1108 return -ENOMEM; 1109 1110 state->acquired_refs = n; 1111 return 0; 1112 } 1113 1114 static int grow_stack_state(struct bpf_func_state *state, int size) 1115 { 1116 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 1117 1118 if (old_n >= n) 1119 return 0; 1120 1121 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1122 if (!state->stack) 1123 return -ENOMEM; 1124 1125 state->allocated_stack = size; 1126 return 0; 1127 } 1128 1129 /* Acquire a pointer id from the env and update the state->refs to include 1130 * this new pointer reference. 1131 * On success, returns a valid pointer id to associate with the register 1132 * On failure, returns a negative errno. 1133 */ 1134 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1135 { 1136 struct bpf_func_state *state = cur_func(env); 1137 int new_ofs = state->acquired_refs; 1138 int id, err; 1139 1140 err = resize_reference_state(state, state->acquired_refs + 1); 1141 if (err) 1142 return err; 1143 id = ++env->id_gen; 1144 state->refs[new_ofs].id = id; 1145 state->refs[new_ofs].insn_idx = insn_idx; 1146 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; 1147 1148 return id; 1149 } 1150 1151 /* release function corresponding to acquire_reference_state(). Idempotent. */ 1152 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1153 { 1154 int i, last_idx; 1155 1156 last_idx = state->acquired_refs - 1; 1157 for (i = 0; i < state->acquired_refs; i++) { 1158 if (state->refs[i].id == ptr_id) { 1159 /* Cannot release caller references in callbacks */ 1160 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 1161 return -EINVAL; 1162 if (last_idx && i != last_idx) 1163 memcpy(&state->refs[i], &state->refs[last_idx], 1164 sizeof(*state->refs)); 1165 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1166 state->acquired_refs--; 1167 return 0; 1168 } 1169 } 1170 return -EINVAL; 1171 } 1172 1173 static void free_func_state(struct bpf_func_state *state) 1174 { 1175 if (!state) 1176 return; 1177 kfree(state->refs); 1178 kfree(state->stack); 1179 kfree(state); 1180 } 1181 1182 static void clear_jmp_history(struct bpf_verifier_state *state) 1183 { 1184 kfree(state->jmp_history); 1185 state->jmp_history = NULL; 1186 state->jmp_history_cnt = 0; 1187 } 1188 1189 static void free_verifier_state(struct bpf_verifier_state *state, 1190 bool free_self) 1191 { 1192 int i; 1193 1194 for (i = 0; i <= state->curframe; i++) { 1195 free_func_state(state->frame[i]); 1196 state->frame[i] = NULL; 1197 } 1198 clear_jmp_history(state); 1199 if (free_self) 1200 kfree(state); 1201 } 1202 1203 /* copy verifier state from src to dst growing dst stack space 1204 * when necessary to accommodate larger src stack 1205 */ 1206 static int copy_func_state(struct bpf_func_state *dst, 1207 const struct bpf_func_state *src) 1208 { 1209 int err; 1210 1211 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1212 err = copy_reference_state(dst, src); 1213 if (err) 1214 return err; 1215 return copy_stack_state(dst, src); 1216 } 1217 1218 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1219 const struct bpf_verifier_state *src) 1220 { 1221 struct bpf_func_state *dst; 1222 int i, err; 1223 1224 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1225 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 1226 GFP_USER); 1227 if (!dst_state->jmp_history) 1228 return -ENOMEM; 1229 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1230 1231 /* if dst has more stack frames then src frame, free them */ 1232 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1233 free_func_state(dst_state->frame[i]); 1234 dst_state->frame[i] = NULL; 1235 } 1236 dst_state->speculative = src->speculative; 1237 dst_state->active_rcu_lock = src->active_rcu_lock; 1238 dst_state->curframe = src->curframe; 1239 dst_state->active_lock.ptr = src->active_lock.ptr; 1240 dst_state->active_lock.id = src->active_lock.id; 1241 dst_state->branches = src->branches; 1242 dst_state->parent = src->parent; 1243 dst_state->first_insn_idx = src->first_insn_idx; 1244 dst_state->last_insn_idx = src->last_insn_idx; 1245 for (i = 0; i <= src->curframe; i++) { 1246 dst = dst_state->frame[i]; 1247 if (!dst) { 1248 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1249 if (!dst) 1250 return -ENOMEM; 1251 dst_state->frame[i] = dst; 1252 } 1253 err = copy_func_state(dst, src->frame[i]); 1254 if (err) 1255 return err; 1256 } 1257 return 0; 1258 } 1259 1260 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1261 { 1262 while (st) { 1263 u32 br = --st->branches; 1264 1265 /* WARN_ON(br > 1) technically makes sense here, 1266 * but see comment in push_stack(), hence: 1267 */ 1268 WARN_ONCE((int)br < 0, 1269 "BUG update_branch_counts:branches_to_explore=%d\n", 1270 br); 1271 if (br) 1272 break; 1273 st = st->parent; 1274 } 1275 } 1276 1277 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1278 int *insn_idx, bool pop_log) 1279 { 1280 struct bpf_verifier_state *cur = env->cur_state; 1281 struct bpf_verifier_stack_elem *elem, *head = env->head; 1282 int err; 1283 1284 if (env->head == NULL) 1285 return -ENOENT; 1286 1287 if (cur) { 1288 err = copy_verifier_state(cur, &head->st); 1289 if (err) 1290 return err; 1291 } 1292 if (pop_log) 1293 bpf_vlog_reset(&env->log, head->log_pos); 1294 if (insn_idx) 1295 *insn_idx = head->insn_idx; 1296 if (prev_insn_idx) 1297 *prev_insn_idx = head->prev_insn_idx; 1298 elem = head->next; 1299 free_verifier_state(&head->st, false); 1300 kfree(head); 1301 env->head = elem; 1302 env->stack_size--; 1303 return 0; 1304 } 1305 1306 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1307 int insn_idx, int prev_insn_idx, 1308 bool speculative) 1309 { 1310 struct bpf_verifier_state *cur = env->cur_state; 1311 struct bpf_verifier_stack_elem *elem; 1312 int err; 1313 1314 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1315 if (!elem) 1316 goto err; 1317 1318 elem->insn_idx = insn_idx; 1319 elem->prev_insn_idx = prev_insn_idx; 1320 elem->next = env->head; 1321 elem->log_pos = env->log.len_used; 1322 env->head = elem; 1323 env->stack_size++; 1324 err = copy_verifier_state(&elem->st, cur); 1325 if (err) 1326 goto err; 1327 elem->st.speculative |= speculative; 1328 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1329 verbose(env, "The sequence of %d jumps is too complex.\n", 1330 env->stack_size); 1331 goto err; 1332 } 1333 if (elem->st.parent) { 1334 ++elem->st.parent->branches; 1335 /* WARN_ON(branches > 2) technically makes sense here, 1336 * but 1337 * 1. speculative states will bump 'branches' for non-branch 1338 * instructions 1339 * 2. is_state_visited() heuristics may decide not to create 1340 * a new state for a sequence of branches and all such current 1341 * and cloned states will be pointing to a single parent state 1342 * which might have large 'branches' count. 1343 */ 1344 } 1345 return &elem->st; 1346 err: 1347 free_verifier_state(env->cur_state, true); 1348 env->cur_state = NULL; 1349 /* pop all elements and return */ 1350 while (!pop_stack(env, NULL, NULL, false)); 1351 return NULL; 1352 } 1353 1354 #define CALLER_SAVED_REGS 6 1355 static const int caller_saved[CALLER_SAVED_REGS] = { 1356 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1357 }; 1358 1359 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1360 struct bpf_reg_state *reg); 1361 1362 /* This helper doesn't clear reg->id */ 1363 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1364 { 1365 reg->var_off = tnum_const(imm); 1366 reg->smin_value = (s64)imm; 1367 reg->smax_value = (s64)imm; 1368 reg->umin_value = imm; 1369 reg->umax_value = imm; 1370 1371 reg->s32_min_value = (s32)imm; 1372 reg->s32_max_value = (s32)imm; 1373 reg->u32_min_value = (u32)imm; 1374 reg->u32_max_value = (u32)imm; 1375 } 1376 1377 /* Mark the unknown part of a register (variable offset or scalar value) as 1378 * known to have the value @imm. 1379 */ 1380 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1381 { 1382 /* Clear id, off, and union(map_ptr, range) */ 1383 memset(((u8 *)reg) + sizeof(reg->type), 0, 1384 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1385 ___mark_reg_known(reg, imm); 1386 } 1387 1388 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1389 { 1390 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1391 reg->s32_min_value = (s32)imm; 1392 reg->s32_max_value = (s32)imm; 1393 reg->u32_min_value = (u32)imm; 1394 reg->u32_max_value = (u32)imm; 1395 } 1396 1397 /* Mark the 'variable offset' part of a register as zero. This should be 1398 * used only on registers holding a pointer type. 1399 */ 1400 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1401 { 1402 __mark_reg_known(reg, 0); 1403 } 1404 1405 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1406 { 1407 __mark_reg_known(reg, 0); 1408 reg->type = SCALAR_VALUE; 1409 } 1410 1411 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1412 struct bpf_reg_state *regs, u32 regno) 1413 { 1414 if (WARN_ON(regno >= MAX_BPF_REG)) { 1415 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1416 /* Something bad happened, let's kill all regs */ 1417 for (regno = 0; regno < MAX_BPF_REG; regno++) 1418 __mark_reg_not_init(env, regs + regno); 1419 return; 1420 } 1421 __mark_reg_known_zero(regs + regno); 1422 } 1423 1424 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1425 { 1426 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1427 const struct bpf_map *map = reg->map_ptr; 1428 1429 if (map->inner_map_meta) { 1430 reg->type = CONST_PTR_TO_MAP; 1431 reg->map_ptr = map->inner_map_meta; 1432 /* transfer reg's id which is unique for every map_lookup_elem 1433 * as UID of the inner map. 1434 */ 1435 if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER)) 1436 reg->map_uid = reg->id; 1437 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1438 reg->type = PTR_TO_XDP_SOCK; 1439 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1440 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1441 reg->type = PTR_TO_SOCKET; 1442 } else { 1443 reg->type = PTR_TO_MAP_VALUE; 1444 } 1445 return; 1446 } 1447 1448 reg->type &= ~PTR_MAYBE_NULL; 1449 } 1450 1451 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1452 { 1453 return type_is_pkt_pointer(reg->type); 1454 } 1455 1456 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1457 { 1458 return reg_is_pkt_pointer(reg) || 1459 reg->type == PTR_TO_PACKET_END; 1460 } 1461 1462 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1463 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1464 enum bpf_reg_type which) 1465 { 1466 /* The register can already have a range from prior markings. 1467 * This is fine as long as it hasn't been advanced from its 1468 * origin. 1469 */ 1470 return reg->type == which && 1471 reg->id == 0 && 1472 reg->off == 0 && 1473 tnum_equals_const(reg->var_off, 0); 1474 } 1475 1476 /* Reset the min/max bounds of a register */ 1477 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1478 { 1479 reg->smin_value = S64_MIN; 1480 reg->smax_value = S64_MAX; 1481 reg->umin_value = 0; 1482 reg->umax_value = U64_MAX; 1483 1484 reg->s32_min_value = S32_MIN; 1485 reg->s32_max_value = S32_MAX; 1486 reg->u32_min_value = 0; 1487 reg->u32_max_value = U32_MAX; 1488 } 1489 1490 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1491 { 1492 reg->smin_value = S64_MIN; 1493 reg->smax_value = S64_MAX; 1494 reg->umin_value = 0; 1495 reg->umax_value = U64_MAX; 1496 } 1497 1498 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1499 { 1500 reg->s32_min_value = S32_MIN; 1501 reg->s32_max_value = S32_MAX; 1502 reg->u32_min_value = 0; 1503 reg->u32_max_value = U32_MAX; 1504 } 1505 1506 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1507 { 1508 struct tnum var32_off = tnum_subreg(reg->var_off); 1509 1510 /* min signed is max(sign bit) | min(other bits) */ 1511 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1512 var32_off.value | (var32_off.mask & S32_MIN)); 1513 /* max signed is min(sign bit) | max(other bits) */ 1514 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1515 var32_off.value | (var32_off.mask & S32_MAX)); 1516 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1517 reg->u32_max_value = min(reg->u32_max_value, 1518 (u32)(var32_off.value | var32_off.mask)); 1519 } 1520 1521 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1522 { 1523 /* min signed is max(sign bit) | min(other bits) */ 1524 reg->smin_value = max_t(s64, reg->smin_value, 1525 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1526 /* max signed is min(sign bit) | max(other bits) */ 1527 reg->smax_value = min_t(s64, reg->smax_value, 1528 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1529 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1530 reg->umax_value = min(reg->umax_value, 1531 reg->var_off.value | reg->var_off.mask); 1532 } 1533 1534 static void __update_reg_bounds(struct bpf_reg_state *reg) 1535 { 1536 __update_reg32_bounds(reg); 1537 __update_reg64_bounds(reg); 1538 } 1539 1540 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1541 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1542 { 1543 /* Learn sign from signed bounds. 1544 * If we cannot cross the sign boundary, then signed and unsigned bounds 1545 * are the same, so combine. This works even in the negative case, e.g. 1546 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1547 */ 1548 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1549 reg->s32_min_value = reg->u32_min_value = 1550 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1551 reg->s32_max_value = reg->u32_max_value = 1552 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1553 return; 1554 } 1555 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1556 * boundary, so we must be careful. 1557 */ 1558 if ((s32)reg->u32_max_value >= 0) { 1559 /* Positive. We can't learn anything from the smin, but smax 1560 * is positive, hence safe. 1561 */ 1562 reg->s32_min_value = reg->u32_min_value; 1563 reg->s32_max_value = reg->u32_max_value = 1564 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1565 } else if ((s32)reg->u32_min_value < 0) { 1566 /* Negative. We can't learn anything from the smax, but smin 1567 * is negative, hence safe. 1568 */ 1569 reg->s32_min_value = reg->u32_min_value = 1570 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1571 reg->s32_max_value = reg->u32_max_value; 1572 } 1573 } 1574 1575 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1576 { 1577 /* Learn sign from signed bounds. 1578 * If we cannot cross the sign boundary, then signed and unsigned bounds 1579 * are the same, so combine. This works even in the negative case, e.g. 1580 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1581 */ 1582 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1583 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1584 reg->umin_value); 1585 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1586 reg->umax_value); 1587 return; 1588 } 1589 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1590 * boundary, so we must be careful. 1591 */ 1592 if ((s64)reg->umax_value >= 0) { 1593 /* Positive. We can't learn anything from the smin, but smax 1594 * is positive, hence safe. 1595 */ 1596 reg->smin_value = reg->umin_value; 1597 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1598 reg->umax_value); 1599 } else if ((s64)reg->umin_value < 0) { 1600 /* Negative. We can't learn anything from the smax, but smin 1601 * is negative, hence safe. 1602 */ 1603 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1604 reg->umin_value); 1605 reg->smax_value = reg->umax_value; 1606 } 1607 } 1608 1609 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1610 { 1611 __reg32_deduce_bounds(reg); 1612 __reg64_deduce_bounds(reg); 1613 } 1614 1615 /* Attempts to improve var_off based on unsigned min/max information */ 1616 static void __reg_bound_offset(struct bpf_reg_state *reg) 1617 { 1618 struct tnum var64_off = tnum_intersect(reg->var_off, 1619 tnum_range(reg->umin_value, 1620 reg->umax_value)); 1621 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1622 tnum_range(reg->u32_min_value, 1623 reg->u32_max_value)); 1624 1625 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1626 } 1627 1628 static void reg_bounds_sync(struct bpf_reg_state *reg) 1629 { 1630 /* We might have learned new bounds from the var_off. */ 1631 __update_reg_bounds(reg); 1632 /* We might have learned something about the sign bit. */ 1633 __reg_deduce_bounds(reg); 1634 /* We might have learned some bits from the bounds. */ 1635 __reg_bound_offset(reg); 1636 /* Intersecting with the old var_off might have improved our bounds 1637 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1638 * then new var_off is (0; 0x7f...fc) which improves our umax. 1639 */ 1640 __update_reg_bounds(reg); 1641 } 1642 1643 static bool __reg32_bound_s64(s32 a) 1644 { 1645 return a >= 0 && a <= S32_MAX; 1646 } 1647 1648 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1649 { 1650 reg->umin_value = reg->u32_min_value; 1651 reg->umax_value = reg->u32_max_value; 1652 1653 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 1654 * be positive otherwise set to worse case bounds and refine later 1655 * from tnum. 1656 */ 1657 if (__reg32_bound_s64(reg->s32_min_value) && 1658 __reg32_bound_s64(reg->s32_max_value)) { 1659 reg->smin_value = reg->s32_min_value; 1660 reg->smax_value = reg->s32_max_value; 1661 } else { 1662 reg->smin_value = 0; 1663 reg->smax_value = U32_MAX; 1664 } 1665 } 1666 1667 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1668 { 1669 /* special case when 64-bit register has upper 32-bit register 1670 * zeroed. Typically happens after zext or <<32, >>32 sequence 1671 * allowing us to use 32-bit bounds directly, 1672 */ 1673 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1674 __reg_assign_32_into_64(reg); 1675 } else { 1676 /* Otherwise the best we can do is push lower 32bit known and 1677 * unknown bits into register (var_off set from jmp logic) 1678 * then learn as much as possible from the 64-bit tnum 1679 * known and unknown bits. The previous smin/smax bounds are 1680 * invalid here because of jmp32 compare so mark them unknown 1681 * so they do not impact tnum bounds calculation. 1682 */ 1683 __mark_reg64_unbounded(reg); 1684 } 1685 reg_bounds_sync(reg); 1686 } 1687 1688 static bool __reg64_bound_s32(s64 a) 1689 { 1690 return a >= S32_MIN && a <= S32_MAX; 1691 } 1692 1693 static bool __reg64_bound_u32(u64 a) 1694 { 1695 return a >= U32_MIN && a <= U32_MAX; 1696 } 1697 1698 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1699 { 1700 __mark_reg32_unbounded(reg); 1701 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1702 reg->s32_min_value = (s32)reg->smin_value; 1703 reg->s32_max_value = (s32)reg->smax_value; 1704 } 1705 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1706 reg->u32_min_value = (u32)reg->umin_value; 1707 reg->u32_max_value = (u32)reg->umax_value; 1708 } 1709 reg_bounds_sync(reg); 1710 } 1711 1712 /* Mark a register as having a completely unknown (scalar) value. */ 1713 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1714 struct bpf_reg_state *reg) 1715 { 1716 /* 1717 * Clear type, id, off, and union(map_ptr, range) and 1718 * padding between 'type' and union 1719 */ 1720 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1721 reg->type = SCALAR_VALUE; 1722 reg->var_off = tnum_unknown; 1723 reg->frameno = 0; 1724 reg->precise = !env->bpf_capable; 1725 __mark_reg_unbounded(reg); 1726 } 1727 1728 static void mark_reg_unknown(struct bpf_verifier_env *env, 1729 struct bpf_reg_state *regs, u32 regno) 1730 { 1731 if (WARN_ON(regno >= MAX_BPF_REG)) { 1732 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1733 /* Something bad happened, let's kill all regs except FP */ 1734 for (regno = 0; regno < BPF_REG_FP; regno++) 1735 __mark_reg_not_init(env, regs + regno); 1736 return; 1737 } 1738 __mark_reg_unknown(env, regs + regno); 1739 } 1740 1741 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1742 struct bpf_reg_state *reg) 1743 { 1744 __mark_reg_unknown(env, reg); 1745 reg->type = NOT_INIT; 1746 } 1747 1748 static void mark_reg_not_init(struct bpf_verifier_env *env, 1749 struct bpf_reg_state *regs, u32 regno) 1750 { 1751 if (WARN_ON(regno >= MAX_BPF_REG)) { 1752 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1753 /* Something bad happened, let's kill all regs except FP */ 1754 for (regno = 0; regno < BPF_REG_FP; regno++) 1755 __mark_reg_not_init(env, regs + regno); 1756 return; 1757 } 1758 __mark_reg_not_init(env, regs + regno); 1759 } 1760 1761 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1762 struct bpf_reg_state *regs, u32 regno, 1763 enum bpf_reg_type reg_type, 1764 struct btf *btf, u32 btf_id, 1765 enum bpf_type_flag flag) 1766 { 1767 if (reg_type == SCALAR_VALUE) { 1768 mark_reg_unknown(env, regs, regno); 1769 return; 1770 } 1771 mark_reg_known_zero(env, regs, regno); 1772 regs[regno].type = PTR_TO_BTF_ID | flag; 1773 regs[regno].btf = btf; 1774 regs[regno].btf_id = btf_id; 1775 } 1776 1777 #define DEF_NOT_SUBREG (0) 1778 static void init_reg_state(struct bpf_verifier_env *env, 1779 struct bpf_func_state *state) 1780 { 1781 struct bpf_reg_state *regs = state->regs; 1782 int i; 1783 1784 for (i = 0; i < MAX_BPF_REG; i++) { 1785 mark_reg_not_init(env, regs, i); 1786 regs[i].live = REG_LIVE_NONE; 1787 regs[i].parent = NULL; 1788 regs[i].subreg_def = DEF_NOT_SUBREG; 1789 } 1790 1791 /* frame pointer */ 1792 regs[BPF_REG_FP].type = PTR_TO_STACK; 1793 mark_reg_known_zero(env, regs, BPF_REG_FP); 1794 regs[BPF_REG_FP].frameno = state->frameno; 1795 } 1796 1797 #define BPF_MAIN_FUNC (-1) 1798 static void init_func_state(struct bpf_verifier_env *env, 1799 struct bpf_func_state *state, 1800 int callsite, int frameno, int subprogno) 1801 { 1802 state->callsite = callsite; 1803 state->frameno = frameno; 1804 state->subprogno = subprogno; 1805 state->callback_ret_range = tnum_range(0, 0); 1806 init_reg_state(env, state); 1807 mark_verifier_state_scratched(env); 1808 } 1809 1810 /* Similar to push_stack(), but for async callbacks */ 1811 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 1812 int insn_idx, int prev_insn_idx, 1813 int subprog) 1814 { 1815 struct bpf_verifier_stack_elem *elem; 1816 struct bpf_func_state *frame; 1817 1818 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1819 if (!elem) 1820 goto err; 1821 1822 elem->insn_idx = insn_idx; 1823 elem->prev_insn_idx = prev_insn_idx; 1824 elem->next = env->head; 1825 elem->log_pos = env->log.len_used; 1826 env->head = elem; 1827 env->stack_size++; 1828 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1829 verbose(env, 1830 "The sequence of %d jumps is too complex for async cb.\n", 1831 env->stack_size); 1832 goto err; 1833 } 1834 /* Unlike push_stack() do not copy_verifier_state(). 1835 * The caller state doesn't matter. 1836 * This is async callback. It starts in a fresh stack. 1837 * Initialize it similar to do_check_common(). 1838 */ 1839 elem->st.branches = 1; 1840 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 1841 if (!frame) 1842 goto err; 1843 init_func_state(env, frame, 1844 BPF_MAIN_FUNC /* callsite */, 1845 0 /* frameno within this callchain */, 1846 subprog /* subprog number within this prog */); 1847 elem->st.frame[0] = frame; 1848 return &elem->st; 1849 err: 1850 free_verifier_state(env->cur_state, true); 1851 env->cur_state = NULL; 1852 /* pop all elements and return */ 1853 while (!pop_stack(env, NULL, NULL, false)); 1854 return NULL; 1855 } 1856 1857 1858 enum reg_arg_type { 1859 SRC_OP, /* register is used as source operand */ 1860 DST_OP, /* register is used as destination operand */ 1861 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1862 }; 1863 1864 static int cmp_subprogs(const void *a, const void *b) 1865 { 1866 return ((struct bpf_subprog_info *)a)->start - 1867 ((struct bpf_subprog_info *)b)->start; 1868 } 1869 1870 static int find_subprog(struct bpf_verifier_env *env, int off) 1871 { 1872 struct bpf_subprog_info *p; 1873 1874 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1875 sizeof(env->subprog_info[0]), cmp_subprogs); 1876 if (!p) 1877 return -ENOENT; 1878 return p - env->subprog_info; 1879 1880 } 1881 1882 static int add_subprog(struct bpf_verifier_env *env, int off) 1883 { 1884 int insn_cnt = env->prog->len; 1885 int ret; 1886 1887 if (off >= insn_cnt || off < 0) { 1888 verbose(env, "call to invalid destination\n"); 1889 return -EINVAL; 1890 } 1891 ret = find_subprog(env, off); 1892 if (ret >= 0) 1893 return ret; 1894 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1895 verbose(env, "too many subprograms\n"); 1896 return -E2BIG; 1897 } 1898 /* determine subprog starts. The end is one before the next starts */ 1899 env->subprog_info[env->subprog_cnt++].start = off; 1900 sort(env->subprog_info, env->subprog_cnt, 1901 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1902 return env->subprog_cnt - 1; 1903 } 1904 1905 #define MAX_KFUNC_DESCS 256 1906 #define MAX_KFUNC_BTFS 256 1907 1908 struct bpf_kfunc_desc { 1909 struct btf_func_model func_model; 1910 u32 func_id; 1911 s32 imm; 1912 u16 offset; 1913 }; 1914 1915 struct bpf_kfunc_btf { 1916 struct btf *btf; 1917 struct module *module; 1918 u16 offset; 1919 }; 1920 1921 struct bpf_kfunc_desc_tab { 1922 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 1923 u32 nr_descs; 1924 }; 1925 1926 struct bpf_kfunc_btf_tab { 1927 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 1928 u32 nr_descs; 1929 }; 1930 1931 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 1932 { 1933 const struct bpf_kfunc_desc *d0 = a; 1934 const struct bpf_kfunc_desc *d1 = b; 1935 1936 /* func_id is not greater than BTF_MAX_TYPE */ 1937 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 1938 } 1939 1940 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 1941 { 1942 const struct bpf_kfunc_btf *d0 = a; 1943 const struct bpf_kfunc_btf *d1 = b; 1944 1945 return d0->offset - d1->offset; 1946 } 1947 1948 static const struct bpf_kfunc_desc * 1949 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 1950 { 1951 struct bpf_kfunc_desc desc = { 1952 .func_id = func_id, 1953 .offset = offset, 1954 }; 1955 struct bpf_kfunc_desc_tab *tab; 1956 1957 tab = prog->aux->kfunc_tab; 1958 return bsearch(&desc, tab->descs, tab->nr_descs, 1959 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 1960 } 1961 1962 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 1963 s16 offset) 1964 { 1965 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 1966 struct bpf_kfunc_btf_tab *tab; 1967 struct bpf_kfunc_btf *b; 1968 struct module *mod; 1969 struct btf *btf; 1970 int btf_fd; 1971 1972 tab = env->prog->aux->kfunc_btf_tab; 1973 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 1974 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 1975 if (!b) { 1976 if (tab->nr_descs == MAX_KFUNC_BTFS) { 1977 verbose(env, "too many different module BTFs\n"); 1978 return ERR_PTR(-E2BIG); 1979 } 1980 1981 if (bpfptr_is_null(env->fd_array)) { 1982 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 1983 return ERR_PTR(-EPROTO); 1984 } 1985 1986 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 1987 offset * sizeof(btf_fd), 1988 sizeof(btf_fd))) 1989 return ERR_PTR(-EFAULT); 1990 1991 btf = btf_get_by_fd(btf_fd); 1992 if (IS_ERR(btf)) { 1993 verbose(env, "invalid module BTF fd specified\n"); 1994 return btf; 1995 } 1996 1997 if (!btf_is_module(btf)) { 1998 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 1999 btf_put(btf); 2000 return ERR_PTR(-EINVAL); 2001 } 2002 2003 mod = btf_try_get_module(btf); 2004 if (!mod) { 2005 btf_put(btf); 2006 return ERR_PTR(-ENXIO); 2007 } 2008 2009 b = &tab->descs[tab->nr_descs++]; 2010 b->btf = btf; 2011 b->module = mod; 2012 b->offset = offset; 2013 2014 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2015 kfunc_btf_cmp_by_off, NULL); 2016 } 2017 return b->btf; 2018 } 2019 2020 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2021 { 2022 if (!tab) 2023 return; 2024 2025 while (tab->nr_descs--) { 2026 module_put(tab->descs[tab->nr_descs].module); 2027 btf_put(tab->descs[tab->nr_descs].btf); 2028 } 2029 kfree(tab); 2030 } 2031 2032 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2033 { 2034 if (offset) { 2035 if (offset < 0) { 2036 /* In the future, this can be allowed to increase limit 2037 * of fd index into fd_array, interpreted as u16. 2038 */ 2039 verbose(env, "negative offset disallowed for kernel module function call\n"); 2040 return ERR_PTR(-EINVAL); 2041 } 2042 2043 return __find_kfunc_desc_btf(env, offset); 2044 } 2045 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2046 } 2047 2048 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 2049 { 2050 const struct btf_type *func, *func_proto; 2051 struct bpf_kfunc_btf_tab *btf_tab; 2052 struct bpf_kfunc_desc_tab *tab; 2053 struct bpf_prog_aux *prog_aux; 2054 struct bpf_kfunc_desc *desc; 2055 const char *func_name; 2056 struct btf *desc_btf; 2057 unsigned long call_imm; 2058 unsigned long addr; 2059 int err; 2060 2061 prog_aux = env->prog->aux; 2062 tab = prog_aux->kfunc_tab; 2063 btf_tab = prog_aux->kfunc_btf_tab; 2064 if (!tab) { 2065 if (!btf_vmlinux) { 2066 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 2067 return -ENOTSUPP; 2068 } 2069 2070 if (!env->prog->jit_requested) { 2071 verbose(env, "JIT is required for calling kernel function\n"); 2072 return -ENOTSUPP; 2073 } 2074 2075 if (!bpf_jit_supports_kfunc_call()) { 2076 verbose(env, "JIT does not support calling kernel function\n"); 2077 return -ENOTSUPP; 2078 } 2079 2080 if (!env->prog->gpl_compatible) { 2081 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 2082 return -EINVAL; 2083 } 2084 2085 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 2086 if (!tab) 2087 return -ENOMEM; 2088 prog_aux->kfunc_tab = tab; 2089 } 2090 2091 /* func_id == 0 is always invalid, but instead of returning an error, be 2092 * conservative and wait until the code elimination pass before returning 2093 * error, so that invalid calls that get pruned out can be in BPF programs 2094 * loaded from userspace. It is also required that offset be untouched 2095 * for such calls. 2096 */ 2097 if (!func_id && !offset) 2098 return 0; 2099 2100 if (!btf_tab && offset) { 2101 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 2102 if (!btf_tab) 2103 return -ENOMEM; 2104 prog_aux->kfunc_btf_tab = btf_tab; 2105 } 2106 2107 desc_btf = find_kfunc_desc_btf(env, offset); 2108 if (IS_ERR(desc_btf)) { 2109 verbose(env, "failed to find BTF for kernel function\n"); 2110 return PTR_ERR(desc_btf); 2111 } 2112 2113 if (find_kfunc_desc(env->prog, func_id, offset)) 2114 return 0; 2115 2116 if (tab->nr_descs == MAX_KFUNC_DESCS) { 2117 verbose(env, "too many different kernel function calls\n"); 2118 return -E2BIG; 2119 } 2120 2121 func = btf_type_by_id(desc_btf, func_id); 2122 if (!func || !btf_type_is_func(func)) { 2123 verbose(env, "kernel btf_id %u is not a function\n", 2124 func_id); 2125 return -EINVAL; 2126 } 2127 func_proto = btf_type_by_id(desc_btf, func->type); 2128 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 2129 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 2130 func_id); 2131 return -EINVAL; 2132 } 2133 2134 func_name = btf_name_by_offset(desc_btf, func->name_off); 2135 addr = kallsyms_lookup_name(func_name); 2136 if (!addr) { 2137 verbose(env, "cannot find address for kernel function %s\n", 2138 func_name); 2139 return -EINVAL; 2140 } 2141 2142 call_imm = BPF_CALL_IMM(addr); 2143 /* Check whether or not the relative offset overflows desc->imm */ 2144 if ((unsigned long)(s32)call_imm != call_imm) { 2145 verbose(env, "address of kernel function %s is out of range\n", 2146 func_name); 2147 return -EINVAL; 2148 } 2149 2150 desc = &tab->descs[tab->nr_descs++]; 2151 desc->func_id = func_id; 2152 desc->imm = call_imm; 2153 desc->offset = offset; 2154 err = btf_distill_func_proto(&env->log, desc_btf, 2155 func_proto, func_name, 2156 &desc->func_model); 2157 if (!err) 2158 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2159 kfunc_desc_cmp_by_id_off, NULL); 2160 return err; 2161 } 2162 2163 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 2164 { 2165 const struct bpf_kfunc_desc *d0 = a; 2166 const struct bpf_kfunc_desc *d1 = b; 2167 2168 if (d0->imm > d1->imm) 2169 return 1; 2170 else if (d0->imm < d1->imm) 2171 return -1; 2172 return 0; 2173 } 2174 2175 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 2176 { 2177 struct bpf_kfunc_desc_tab *tab; 2178 2179 tab = prog->aux->kfunc_tab; 2180 if (!tab) 2181 return; 2182 2183 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2184 kfunc_desc_cmp_by_imm, NULL); 2185 } 2186 2187 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2188 { 2189 return !!prog->aux->kfunc_tab; 2190 } 2191 2192 const struct btf_func_model * 2193 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2194 const struct bpf_insn *insn) 2195 { 2196 const struct bpf_kfunc_desc desc = { 2197 .imm = insn->imm, 2198 }; 2199 const struct bpf_kfunc_desc *res; 2200 struct bpf_kfunc_desc_tab *tab; 2201 2202 tab = prog->aux->kfunc_tab; 2203 res = bsearch(&desc, tab->descs, tab->nr_descs, 2204 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 2205 2206 return res ? &res->func_model : NULL; 2207 } 2208 2209 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2210 { 2211 struct bpf_subprog_info *subprog = env->subprog_info; 2212 struct bpf_insn *insn = env->prog->insnsi; 2213 int i, ret, insn_cnt = env->prog->len; 2214 2215 /* Add entry function. */ 2216 ret = add_subprog(env, 0); 2217 if (ret) 2218 return ret; 2219 2220 for (i = 0; i < insn_cnt; i++, insn++) { 2221 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2222 !bpf_pseudo_kfunc_call(insn)) 2223 continue; 2224 2225 if (!env->bpf_capable) { 2226 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2227 return -EPERM; 2228 } 2229 2230 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2231 ret = add_subprog(env, i + insn->imm + 1); 2232 else 2233 ret = add_kfunc_call(env, insn->imm, insn->off); 2234 2235 if (ret < 0) 2236 return ret; 2237 } 2238 2239 /* Add a fake 'exit' subprog which could simplify subprog iteration 2240 * logic. 'subprog_cnt' should not be increased. 2241 */ 2242 subprog[env->subprog_cnt].start = insn_cnt; 2243 2244 if (env->log.level & BPF_LOG_LEVEL2) 2245 for (i = 0; i < env->subprog_cnt; i++) 2246 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2247 2248 return 0; 2249 } 2250 2251 static int check_subprogs(struct bpf_verifier_env *env) 2252 { 2253 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2254 struct bpf_subprog_info *subprog = env->subprog_info; 2255 struct bpf_insn *insn = env->prog->insnsi; 2256 int insn_cnt = env->prog->len; 2257 2258 /* now check that all jumps are within the same subprog */ 2259 subprog_start = subprog[cur_subprog].start; 2260 subprog_end = subprog[cur_subprog + 1].start; 2261 for (i = 0; i < insn_cnt; i++) { 2262 u8 code = insn[i].code; 2263 2264 if (code == (BPF_JMP | BPF_CALL) && 2265 insn[i].imm == BPF_FUNC_tail_call && 2266 insn[i].src_reg != BPF_PSEUDO_CALL) 2267 subprog[cur_subprog].has_tail_call = true; 2268 if (BPF_CLASS(code) == BPF_LD && 2269 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2270 subprog[cur_subprog].has_ld_abs = true; 2271 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2272 goto next; 2273 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2274 goto next; 2275 off = i + insn[i].off + 1; 2276 if (off < subprog_start || off >= subprog_end) { 2277 verbose(env, "jump out of range from insn %d to %d\n", i, off); 2278 return -EINVAL; 2279 } 2280 next: 2281 if (i == subprog_end - 1) { 2282 /* to avoid fall-through from one subprog into another 2283 * the last insn of the subprog should be either exit 2284 * or unconditional jump back 2285 */ 2286 if (code != (BPF_JMP | BPF_EXIT) && 2287 code != (BPF_JMP | BPF_JA)) { 2288 verbose(env, "last insn is not an exit or jmp\n"); 2289 return -EINVAL; 2290 } 2291 subprog_start = subprog_end; 2292 cur_subprog++; 2293 if (cur_subprog < env->subprog_cnt) 2294 subprog_end = subprog[cur_subprog + 1].start; 2295 } 2296 } 2297 return 0; 2298 } 2299 2300 /* Parentage chain of this register (or stack slot) should take care of all 2301 * issues like callee-saved registers, stack slot allocation time, etc. 2302 */ 2303 static int mark_reg_read(struct bpf_verifier_env *env, 2304 const struct bpf_reg_state *state, 2305 struct bpf_reg_state *parent, u8 flag) 2306 { 2307 bool writes = parent == state->parent; /* Observe write marks */ 2308 int cnt = 0; 2309 2310 while (parent) { 2311 /* if read wasn't screened by an earlier write ... */ 2312 if (writes && state->live & REG_LIVE_WRITTEN) 2313 break; 2314 if (parent->live & REG_LIVE_DONE) { 2315 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 2316 reg_type_str(env, parent->type), 2317 parent->var_off.value, parent->off); 2318 return -EFAULT; 2319 } 2320 /* The first condition is more likely to be true than the 2321 * second, checked it first. 2322 */ 2323 if ((parent->live & REG_LIVE_READ) == flag || 2324 parent->live & REG_LIVE_READ64) 2325 /* The parentage chain never changes and 2326 * this parent was already marked as LIVE_READ. 2327 * There is no need to keep walking the chain again and 2328 * keep re-marking all parents as LIVE_READ. 2329 * This case happens when the same register is read 2330 * multiple times without writes into it in-between. 2331 * Also, if parent has the stronger REG_LIVE_READ64 set, 2332 * then no need to set the weak REG_LIVE_READ32. 2333 */ 2334 break; 2335 /* ... then we depend on parent's value */ 2336 parent->live |= flag; 2337 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 2338 if (flag == REG_LIVE_READ64) 2339 parent->live &= ~REG_LIVE_READ32; 2340 state = parent; 2341 parent = state->parent; 2342 writes = true; 2343 cnt++; 2344 } 2345 2346 if (env->longest_mark_read_walk < cnt) 2347 env->longest_mark_read_walk = cnt; 2348 return 0; 2349 } 2350 2351 /* This function is supposed to be used by the following 32-bit optimization 2352 * code only. It returns TRUE if the source or destination register operates 2353 * on 64-bit, otherwise return FALSE. 2354 */ 2355 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 2356 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 2357 { 2358 u8 code, class, op; 2359 2360 code = insn->code; 2361 class = BPF_CLASS(code); 2362 op = BPF_OP(code); 2363 if (class == BPF_JMP) { 2364 /* BPF_EXIT for "main" will reach here. Return TRUE 2365 * conservatively. 2366 */ 2367 if (op == BPF_EXIT) 2368 return true; 2369 if (op == BPF_CALL) { 2370 /* BPF to BPF call will reach here because of marking 2371 * caller saved clobber with DST_OP_NO_MARK for which we 2372 * don't care the register def because they are anyway 2373 * marked as NOT_INIT already. 2374 */ 2375 if (insn->src_reg == BPF_PSEUDO_CALL) 2376 return false; 2377 /* Helper call will reach here because of arg type 2378 * check, conservatively return TRUE. 2379 */ 2380 if (t == SRC_OP) 2381 return true; 2382 2383 return false; 2384 } 2385 } 2386 2387 if (class == BPF_ALU64 || class == BPF_JMP || 2388 /* BPF_END always use BPF_ALU class. */ 2389 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 2390 return true; 2391 2392 if (class == BPF_ALU || class == BPF_JMP32) 2393 return false; 2394 2395 if (class == BPF_LDX) { 2396 if (t != SRC_OP) 2397 return BPF_SIZE(code) == BPF_DW; 2398 /* LDX source must be ptr. */ 2399 return true; 2400 } 2401 2402 if (class == BPF_STX) { 2403 /* BPF_STX (including atomic variants) has multiple source 2404 * operands, one of which is a ptr. Check whether the caller is 2405 * asking about it. 2406 */ 2407 if (t == SRC_OP && reg->type != SCALAR_VALUE) 2408 return true; 2409 return BPF_SIZE(code) == BPF_DW; 2410 } 2411 2412 if (class == BPF_LD) { 2413 u8 mode = BPF_MODE(code); 2414 2415 /* LD_IMM64 */ 2416 if (mode == BPF_IMM) 2417 return true; 2418 2419 /* Both LD_IND and LD_ABS return 32-bit data. */ 2420 if (t != SRC_OP) 2421 return false; 2422 2423 /* Implicit ctx ptr. */ 2424 if (regno == BPF_REG_6) 2425 return true; 2426 2427 /* Explicit source could be any width. */ 2428 return true; 2429 } 2430 2431 if (class == BPF_ST) 2432 /* The only source register for BPF_ST is a ptr. */ 2433 return true; 2434 2435 /* Conservatively return true at default. */ 2436 return true; 2437 } 2438 2439 /* Return the regno defined by the insn, or -1. */ 2440 static int insn_def_regno(const struct bpf_insn *insn) 2441 { 2442 switch (BPF_CLASS(insn->code)) { 2443 case BPF_JMP: 2444 case BPF_JMP32: 2445 case BPF_ST: 2446 return -1; 2447 case BPF_STX: 2448 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2449 (insn->imm & BPF_FETCH)) { 2450 if (insn->imm == BPF_CMPXCHG) 2451 return BPF_REG_0; 2452 else 2453 return insn->src_reg; 2454 } else { 2455 return -1; 2456 } 2457 default: 2458 return insn->dst_reg; 2459 } 2460 } 2461 2462 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 2463 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 2464 { 2465 int dst_reg = insn_def_regno(insn); 2466 2467 if (dst_reg == -1) 2468 return false; 2469 2470 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2471 } 2472 2473 static void mark_insn_zext(struct bpf_verifier_env *env, 2474 struct bpf_reg_state *reg) 2475 { 2476 s32 def_idx = reg->subreg_def; 2477 2478 if (def_idx == DEF_NOT_SUBREG) 2479 return; 2480 2481 env->insn_aux_data[def_idx - 1].zext_dst = true; 2482 /* The dst will be zero extended, so won't be sub-register anymore. */ 2483 reg->subreg_def = DEF_NOT_SUBREG; 2484 } 2485 2486 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2487 enum reg_arg_type t) 2488 { 2489 struct bpf_verifier_state *vstate = env->cur_state; 2490 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2491 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2492 struct bpf_reg_state *reg, *regs = state->regs; 2493 bool rw64; 2494 2495 if (regno >= MAX_BPF_REG) { 2496 verbose(env, "R%d is invalid\n", regno); 2497 return -EINVAL; 2498 } 2499 2500 mark_reg_scratched(env, regno); 2501 2502 reg = ®s[regno]; 2503 rw64 = is_reg64(env, insn, regno, reg, t); 2504 if (t == SRC_OP) { 2505 /* check whether register used as source operand can be read */ 2506 if (reg->type == NOT_INIT) { 2507 verbose(env, "R%d !read_ok\n", regno); 2508 return -EACCES; 2509 } 2510 /* We don't need to worry about FP liveness because it's read-only */ 2511 if (regno == BPF_REG_FP) 2512 return 0; 2513 2514 if (rw64) 2515 mark_insn_zext(env, reg); 2516 2517 return mark_reg_read(env, reg, reg->parent, 2518 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2519 } else { 2520 /* check whether register used as dest operand can be written to */ 2521 if (regno == BPF_REG_FP) { 2522 verbose(env, "frame pointer is read only\n"); 2523 return -EACCES; 2524 } 2525 reg->live |= REG_LIVE_WRITTEN; 2526 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2527 if (t == DST_OP) 2528 mark_reg_unknown(env, regs, regno); 2529 } 2530 return 0; 2531 } 2532 2533 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) 2534 { 2535 env->insn_aux_data[idx].jmp_point = true; 2536 } 2537 2538 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) 2539 { 2540 return env->insn_aux_data[insn_idx].jmp_point; 2541 } 2542 2543 /* for any branch, call, exit record the history of jmps in the given state */ 2544 static int push_jmp_history(struct bpf_verifier_env *env, 2545 struct bpf_verifier_state *cur) 2546 { 2547 u32 cnt = cur->jmp_history_cnt; 2548 struct bpf_idx_pair *p; 2549 size_t alloc_size; 2550 2551 if (!is_jmp_point(env, env->insn_idx)) 2552 return 0; 2553 2554 cnt++; 2555 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); 2556 p = krealloc(cur->jmp_history, alloc_size, GFP_USER); 2557 if (!p) 2558 return -ENOMEM; 2559 p[cnt - 1].idx = env->insn_idx; 2560 p[cnt - 1].prev_idx = env->prev_insn_idx; 2561 cur->jmp_history = p; 2562 cur->jmp_history_cnt = cnt; 2563 return 0; 2564 } 2565 2566 /* Backtrack one insn at a time. If idx is not at the top of recorded 2567 * history then previous instruction came from straight line execution. 2568 */ 2569 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2570 u32 *history) 2571 { 2572 u32 cnt = *history; 2573 2574 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2575 i = st->jmp_history[cnt - 1].prev_idx; 2576 (*history)--; 2577 } else { 2578 i--; 2579 } 2580 return i; 2581 } 2582 2583 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2584 { 2585 const struct btf_type *func; 2586 struct btf *desc_btf; 2587 2588 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2589 return NULL; 2590 2591 desc_btf = find_kfunc_desc_btf(data, insn->off); 2592 if (IS_ERR(desc_btf)) 2593 return "<error>"; 2594 2595 func = btf_type_by_id(desc_btf, insn->imm); 2596 return btf_name_by_offset(desc_btf, func->name_off); 2597 } 2598 2599 /* For given verifier state backtrack_insn() is called from the last insn to 2600 * the first insn. Its purpose is to compute a bitmask of registers and 2601 * stack slots that needs precision in the parent verifier state. 2602 */ 2603 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2604 u32 *reg_mask, u64 *stack_mask) 2605 { 2606 const struct bpf_insn_cbs cbs = { 2607 .cb_call = disasm_kfunc_name, 2608 .cb_print = verbose, 2609 .private_data = env, 2610 }; 2611 struct bpf_insn *insn = env->prog->insnsi + idx; 2612 u8 class = BPF_CLASS(insn->code); 2613 u8 opcode = BPF_OP(insn->code); 2614 u8 mode = BPF_MODE(insn->code); 2615 u32 dreg = 1u << insn->dst_reg; 2616 u32 sreg = 1u << insn->src_reg; 2617 u32 spi; 2618 2619 if (insn->code == 0) 2620 return 0; 2621 if (env->log.level & BPF_LOG_LEVEL2) { 2622 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2623 verbose(env, "%d: ", idx); 2624 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2625 } 2626 2627 if (class == BPF_ALU || class == BPF_ALU64) { 2628 if (!(*reg_mask & dreg)) 2629 return 0; 2630 if (opcode == BPF_MOV) { 2631 if (BPF_SRC(insn->code) == BPF_X) { 2632 /* dreg = sreg 2633 * dreg needs precision after this insn 2634 * sreg needs precision before this insn 2635 */ 2636 *reg_mask &= ~dreg; 2637 *reg_mask |= sreg; 2638 } else { 2639 /* dreg = K 2640 * dreg needs precision after this insn. 2641 * Corresponding register is already marked 2642 * as precise=true in this verifier state. 2643 * No further markings in parent are necessary 2644 */ 2645 *reg_mask &= ~dreg; 2646 } 2647 } else { 2648 if (BPF_SRC(insn->code) == BPF_X) { 2649 /* dreg += sreg 2650 * both dreg and sreg need precision 2651 * before this insn 2652 */ 2653 *reg_mask |= sreg; 2654 } /* else dreg += K 2655 * dreg still needs precision before this insn 2656 */ 2657 } 2658 } else if (class == BPF_LDX) { 2659 if (!(*reg_mask & dreg)) 2660 return 0; 2661 *reg_mask &= ~dreg; 2662 2663 /* scalars can only be spilled into stack w/o losing precision. 2664 * Load from any other memory can be zero extended. 2665 * The desire to keep that precision is already indicated 2666 * by 'precise' mark in corresponding register of this state. 2667 * No further tracking necessary. 2668 */ 2669 if (insn->src_reg != BPF_REG_FP) 2670 return 0; 2671 2672 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2673 * that [fp - off] slot contains scalar that needs to be 2674 * tracked with precision 2675 */ 2676 spi = (-insn->off - 1) / BPF_REG_SIZE; 2677 if (spi >= 64) { 2678 verbose(env, "BUG spi %d\n", spi); 2679 WARN_ONCE(1, "verifier backtracking bug"); 2680 return -EFAULT; 2681 } 2682 *stack_mask |= 1ull << spi; 2683 } else if (class == BPF_STX || class == BPF_ST) { 2684 if (*reg_mask & dreg) 2685 /* stx & st shouldn't be using _scalar_ dst_reg 2686 * to access memory. It means backtracking 2687 * encountered a case of pointer subtraction. 2688 */ 2689 return -ENOTSUPP; 2690 /* scalars can only be spilled into stack */ 2691 if (insn->dst_reg != BPF_REG_FP) 2692 return 0; 2693 spi = (-insn->off - 1) / BPF_REG_SIZE; 2694 if (spi >= 64) { 2695 verbose(env, "BUG spi %d\n", spi); 2696 WARN_ONCE(1, "verifier backtracking bug"); 2697 return -EFAULT; 2698 } 2699 if (!(*stack_mask & (1ull << spi))) 2700 return 0; 2701 *stack_mask &= ~(1ull << spi); 2702 if (class == BPF_STX) 2703 *reg_mask |= sreg; 2704 } else if (class == BPF_JMP || class == BPF_JMP32) { 2705 if (opcode == BPF_CALL) { 2706 if (insn->src_reg == BPF_PSEUDO_CALL) 2707 return -ENOTSUPP; 2708 /* BPF helpers that invoke callback subprogs are 2709 * equivalent to BPF_PSEUDO_CALL above 2710 */ 2711 if (insn->src_reg == 0 && is_callback_calling_function(insn->imm)) 2712 return -ENOTSUPP; 2713 /* regular helper call sets R0 */ 2714 *reg_mask &= ~1; 2715 if (*reg_mask & 0x3f) { 2716 /* if backtracing was looking for registers R1-R5 2717 * they should have been found already. 2718 */ 2719 verbose(env, "BUG regs %x\n", *reg_mask); 2720 WARN_ONCE(1, "verifier backtracking bug"); 2721 return -EFAULT; 2722 } 2723 } else if (opcode == BPF_EXIT) { 2724 return -ENOTSUPP; 2725 } 2726 } else if (class == BPF_LD) { 2727 if (!(*reg_mask & dreg)) 2728 return 0; 2729 *reg_mask &= ~dreg; 2730 /* It's ld_imm64 or ld_abs or ld_ind. 2731 * For ld_imm64 no further tracking of precision 2732 * into parent is necessary 2733 */ 2734 if (mode == BPF_IND || mode == BPF_ABS) 2735 /* to be analyzed */ 2736 return -ENOTSUPP; 2737 } 2738 return 0; 2739 } 2740 2741 /* the scalar precision tracking algorithm: 2742 * . at the start all registers have precise=false. 2743 * . scalar ranges are tracked as normal through alu and jmp insns. 2744 * . once precise value of the scalar register is used in: 2745 * . ptr + scalar alu 2746 * . if (scalar cond K|scalar) 2747 * . helper_call(.., scalar, ...) where ARG_CONST is expected 2748 * backtrack through the verifier states and mark all registers and 2749 * stack slots with spilled constants that these scalar regisers 2750 * should be precise. 2751 * . during state pruning two registers (or spilled stack slots) 2752 * are equivalent if both are not precise. 2753 * 2754 * Note the verifier cannot simply walk register parentage chain, 2755 * since many different registers and stack slots could have been 2756 * used to compute single precise scalar. 2757 * 2758 * The approach of starting with precise=true for all registers and then 2759 * backtrack to mark a register as not precise when the verifier detects 2760 * that program doesn't care about specific value (e.g., when helper 2761 * takes register as ARG_ANYTHING parameter) is not safe. 2762 * 2763 * It's ok to walk single parentage chain of the verifier states. 2764 * It's possible that this backtracking will go all the way till 1st insn. 2765 * All other branches will be explored for needing precision later. 2766 * 2767 * The backtracking needs to deal with cases like: 2768 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 2769 * r9 -= r8 2770 * r5 = r9 2771 * if r5 > 0x79f goto pc+7 2772 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 2773 * r5 += 1 2774 * ... 2775 * call bpf_perf_event_output#25 2776 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 2777 * 2778 * and this case: 2779 * r6 = 1 2780 * call foo // uses callee's r6 inside to compute r0 2781 * r0 += r6 2782 * if r0 == 0 goto 2783 * 2784 * to track above reg_mask/stack_mask needs to be independent for each frame. 2785 * 2786 * Also if parent's curframe > frame where backtracking started, 2787 * the verifier need to mark registers in both frames, otherwise callees 2788 * may incorrectly prune callers. This is similar to 2789 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 2790 * 2791 * For now backtracking falls back into conservative marking. 2792 */ 2793 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 2794 struct bpf_verifier_state *st) 2795 { 2796 struct bpf_func_state *func; 2797 struct bpf_reg_state *reg; 2798 int i, j; 2799 2800 /* big hammer: mark all scalars precise in this path. 2801 * pop_stack may still get !precise scalars. 2802 * We also skip current state and go straight to first parent state, 2803 * because precision markings in current non-checkpointed state are 2804 * not needed. See why in the comment in __mark_chain_precision below. 2805 */ 2806 for (st = st->parent; st; st = st->parent) { 2807 for (i = 0; i <= st->curframe; i++) { 2808 func = st->frame[i]; 2809 for (j = 0; j < BPF_REG_FP; j++) { 2810 reg = &func->regs[j]; 2811 if (reg->type != SCALAR_VALUE) 2812 continue; 2813 reg->precise = true; 2814 } 2815 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2816 if (!is_spilled_reg(&func->stack[j])) 2817 continue; 2818 reg = &func->stack[j].spilled_ptr; 2819 if (reg->type != SCALAR_VALUE) 2820 continue; 2821 reg->precise = true; 2822 } 2823 } 2824 } 2825 } 2826 2827 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 2828 { 2829 struct bpf_func_state *func; 2830 struct bpf_reg_state *reg; 2831 int i, j; 2832 2833 for (i = 0; i <= st->curframe; i++) { 2834 func = st->frame[i]; 2835 for (j = 0; j < BPF_REG_FP; j++) { 2836 reg = &func->regs[j]; 2837 if (reg->type != SCALAR_VALUE) 2838 continue; 2839 reg->precise = false; 2840 } 2841 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2842 if (!is_spilled_reg(&func->stack[j])) 2843 continue; 2844 reg = &func->stack[j].spilled_ptr; 2845 if (reg->type != SCALAR_VALUE) 2846 continue; 2847 reg->precise = false; 2848 } 2849 } 2850 } 2851 2852 /* 2853 * __mark_chain_precision() backtracks BPF program instruction sequence and 2854 * chain of verifier states making sure that register *regno* (if regno >= 0) 2855 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked 2856 * SCALARS, as well as any other registers and slots that contribute to 2857 * a tracked state of given registers/stack slots, depending on specific BPF 2858 * assembly instructions (see backtrack_insns() for exact instruction handling 2859 * logic). This backtracking relies on recorded jmp_history and is able to 2860 * traverse entire chain of parent states. This process ends only when all the 2861 * necessary registers/slots and their transitive dependencies are marked as 2862 * precise. 2863 * 2864 * One important and subtle aspect is that precise marks *do not matter* in 2865 * the currently verified state (current state). It is important to understand 2866 * why this is the case. 2867 * 2868 * First, note that current state is the state that is not yet "checkpointed", 2869 * i.e., it is not yet put into env->explored_states, and it has no children 2870 * states as well. It's ephemeral, and can end up either a) being discarded if 2871 * compatible explored state is found at some point or BPF_EXIT instruction is 2872 * reached or b) checkpointed and put into env->explored_states, branching out 2873 * into one or more children states. 2874 * 2875 * In the former case, precise markings in current state are completely 2876 * ignored by state comparison code (see regsafe() for details). Only 2877 * checkpointed ("old") state precise markings are important, and if old 2878 * state's register/slot is precise, regsafe() assumes current state's 2879 * register/slot as precise and checks value ranges exactly and precisely. If 2880 * states turn out to be compatible, current state's necessary precise 2881 * markings and any required parent states' precise markings are enforced 2882 * after the fact with propagate_precision() logic, after the fact. But it's 2883 * important to realize that in this case, even after marking current state 2884 * registers/slots as precise, we immediately discard current state. So what 2885 * actually matters is any of the precise markings propagated into current 2886 * state's parent states, which are always checkpointed (due to b) case above). 2887 * As such, for scenario a) it doesn't matter if current state has precise 2888 * markings set or not. 2889 * 2890 * Now, for the scenario b), checkpointing and forking into child(ren) 2891 * state(s). Note that before current state gets to checkpointing step, any 2892 * processed instruction always assumes precise SCALAR register/slot 2893 * knowledge: if precise value or range is useful to prune jump branch, BPF 2894 * verifier takes this opportunity enthusiastically. Similarly, when 2895 * register's value is used to calculate offset or memory address, exact 2896 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to 2897 * what we mentioned above about state comparison ignoring precise markings 2898 * during state comparison, BPF verifier ignores and also assumes precise 2899 * markings *at will* during instruction verification process. But as verifier 2900 * assumes precision, it also propagates any precision dependencies across 2901 * parent states, which are not yet finalized, so can be further restricted 2902 * based on new knowledge gained from restrictions enforced by their children 2903 * states. This is so that once those parent states are finalized, i.e., when 2904 * they have no more active children state, state comparison logic in 2905 * is_state_visited() would enforce strict and precise SCALAR ranges, if 2906 * required for correctness. 2907 * 2908 * To build a bit more intuition, note also that once a state is checkpointed, 2909 * the path we took to get to that state is not important. This is crucial 2910 * property for state pruning. When state is checkpointed and finalized at 2911 * some instruction index, it can be correctly and safely used to "short 2912 * circuit" any *compatible* state that reaches exactly the same instruction 2913 * index. I.e., if we jumped to that instruction from a completely different 2914 * code path than original finalized state was derived from, it doesn't 2915 * matter, current state can be discarded because from that instruction 2916 * forward having a compatible state will ensure we will safely reach the 2917 * exit. States describe preconditions for further exploration, but completely 2918 * forget the history of how we got here. 2919 * 2920 * This also means that even if we needed precise SCALAR range to get to 2921 * finalized state, but from that point forward *that same* SCALAR register is 2922 * never used in a precise context (i.e., it's precise value is not needed for 2923 * correctness), it's correct and safe to mark such register as "imprecise" 2924 * (i.e., precise marking set to false). This is what we rely on when we do 2925 * not set precise marking in current state. If no child state requires 2926 * precision for any given SCALAR register, it's safe to dictate that it can 2927 * be imprecise. If any child state does require this register to be precise, 2928 * we'll mark it precise later retroactively during precise markings 2929 * propagation from child state to parent states. 2930 * 2931 * Skipping precise marking setting in current state is a mild version of 2932 * relying on the above observation. But we can utilize this property even 2933 * more aggressively by proactively forgetting any precise marking in the 2934 * current state (which we inherited from the parent state), right before we 2935 * checkpoint it and branch off into new child state. This is done by 2936 * mark_all_scalars_imprecise() to hopefully get more permissive and generic 2937 * finalized states which help in short circuiting more future states. 2938 */ 2939 static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno, 2940 int spi) 2941 { 2942 struct bpf_verifier_state *st = env->cur_state; 2943 int first_idx = st->first_insn_idx; 2944 int last_idx = env->insn_idx; 2945 struct bpf_func_state *func; 2946 struct bpf_reg_state *reg; 2947 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2948 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2949 bool skip_first = true; 2950 bool new_marks = false; 2951 int i, err; 2952 2953 if (!env->bpf_capable) 2954 return 0; 2955 2956 /* Do sanity checks against current state of register and/or stack 2957 * slot, but don't set precise flag in current state, as precision 2958 * tracking in the current state is unnecessary. 2959 */ 2960 func = st->frame[frame]; 2961 if (regno >= 0) { 2962 reg = &func->regs[regno]; 2963 if (reg->type != SCALAR_VALUE) { 2964 WARN_ONCE(1, "backtracing misuse"); 2965 return -EFAULT; 2966 } 2967 new_marks = true; 2968 } 2969 2970 while (spi >= 0) { 2971 if (!is_spilled_reg(&func->stack[spi])) { 2972 stack_mask = 0; 2973 break; 2974 } 2975 reg = &func->stack[spi].spilled_ptr; 2976 if (reg->type != SCALAR_VALUE) { 2977 stack_mask = 0; 2978 break; 2979 } 2980 new_marks = true; 2981 break; 2982 } 2983 2984 if (!new_marks) 2985 return 0; 2986 if (!reg_mask && !stack_mask) 2987 return 0; 2988 2989 for (;;) { 2990 DECLARE_BITMAP(mask, 64); 2991 u32 history = st->jmp_history_cnt; 2992 2993 if (env->log.level & BPF_LOG_LEVEL2) 2994 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2995 2996 if (last_idx < 0) { 2997 /* we are at the entry into subprog, which 2998 * is expected for global funcs, but only if 2999 * requested precise registers are R1-R5 3000 * (which are global func's input arguments) 3001 */ 3002 if (st->curframe == 0 && 3003 st->frame[0]->subprogno > 0 && 3004 st->frame[0]->callsite == BPF_MAIN_FUNC && 3005 stack_mask == 0 && (reg_mask & ~0x3e) == 0) { 3006 bitmap_from_u64(mask, reg_mask); 3007 for_each_set_bit(i, mask, 32) { 3008 reg = &st->frame[0]->regs[i]; 3009 if (reg->type != SCALAR_VALUE) { 3010 reg_mask &= ~(1u << i); 3011 continue; 3012 } 3013 reg->precise = true; 3014 } 3015 return 0; 3016 } 3017 3018 verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n", 3019 st->frame[0]->subprogno, reg_mask, stack_mask); 3020 WARN_ONCE(1, "verifier backtracking bug"); 3021 return -EFAULT; 3022 } 3023 3024 for (i = last_idx;;) { 3025 if (skip_first) { 3026 err = 0; 3027 skip_first = false; 3028 } else { 3029 err = backtrack_insn(env, i, ®_mask, &stack_mask); 3030 } 3031 if (err == -ENOTSUPP) { 3032 mark_all_scalars_precise(env, st); 3033 return 0; 3034 } else if (err) { 3035 return err; 3036 } 3037 if (!reg_mask && !stack_mask) 3038 /* Found assignment(s) into tracked register in this state. 3039 * Since this state is already marked, just return. 3040 * Nothing to be tracked further in the parent state. 3041 */ 3042 return 0; 3043 if (i == first_idx) 3044 break; 3045 i = get_prev_insn_idx(st, i, &history); 3046 if (i >= env->prog->len) { 3047 /* This can happen if backtracking reached insn 0 3048 * and there are still reg_mask or stack_mask 3049 * to backtrack. 3050 * It means the backtracking missed the spot where 3051 * particular register was initialized with a constant. 3052 */ 3053 verbose(env, "BUG backtracking idx %d\n", i); 3054 WARN_ONCE(1, "verifier backtracking bug"); 3055 return -EFAULT; 3056 } 3057 } 3058 st = st->parent; 3059 if (!st) 3060 break; 3061 3062 new_marks = false; 3063 func = st->frame[frame]; 3064 bitmap_from_u64(mask, reg_mask); 3065 for_each_set_bit(i, mask, 32) { 3066 reg = &func->regs[i]; 3067 if (reg->type != SCALAR_VALUE) { 3068 reg_mask &= ~(1u << i); 3069 continue; 3070 } 3071 if (!reg->precise) 3072 new_marks = true; 3073 reg->precise = true; 3074 } 3075 3076 bitmap_from_u64(mask, stack_mask); 3077 for_each_set_bit(i, mask, 64) { 3078 if (i >= func->allocated_stack / BPF_REG_SIZE) { 3079 /* the sequence of instructions: 3080 * 2: (bf) r3 = r10 3081 * 3: (7b) *(u64 *)(r3 -8) = r0 3082 * 4: (79) r4 = *(u64 *)(r10 -8) 3083 * doesn't contain jmps. It's backtracked 3084 * as a single block. 3085 * During backtracking insn 3 is not recognized as 3086 * stack access, so at the end of backtracking 3087 * stack slot fp-8 is still marked in stack_mask. 3088 * However the parent state may not have accessed 3089 * fp-8 and it's "unallocated" stack space. 3090 * In such case fallback to conservative. 3091 */ 3092 mark_all_scalars_precise(env, st); 3093 return 0; 3094 } 3095 3096 if (!is_spilled_reg(&func->stack[i])) { 3097 stack_mask &= ~(1ull << i); 3098 continue; 3099 } 3100 reg = &func->stack[i].spilled_ptr; 3101 if (reg->type != SCALAR_VALUE) { 3102 stack_mask &= ~(1ull << i); 3103 continue; 3104 } 3105 if (!reg->precise) 3106 new_marks = true; 3107 reg->precise = true; 3108 } 3109 if (env->log.level & BPF_LOG_LEVEL2) { 3110 verbose(env, "parent %s regs=%x stack=%llx marks:", 3111 new_marks ? "didn't have" : "already had", 3112 reg_mask, stack_mask); 3113 print_verifier_state(env, func, true); 3114 } 3115 3116 if (!reg_mask && !stack_mask) 3117 break; 3118 if (!new_marks) 3119 break; 3120 3121 last_idx = st->last_insn_idx; 3122 first_idx = st->first_insn_idx; 3123 } 3124 return 0; 3125 } 3126 3127 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 3128 { 3129 return __mark_chain_precision(env, env->cur_state->curframe, regno, -1); 3130 } 3131 3132 static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno) 3133 { 3134 return __mark_chain_precision(env, frame, regno, -1); 3135 } 3136 3137 static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi) 3138 { 3139 return __mark_chain_precision(env, frame, -1, spi); 3140 } 3141 3142 static bool is_spillable_regtype(enum bpf_reg_type type) 3143 { 3144 switch (base_type(type)) { 3145 case PTR_TO_MAP_VALUE: 3146 case PTR_TO_STACK: 3147 case PTR_TO_CTX: 3148 case PTR_TO_PACKET: 3149 case PTR_TO_PACKET_META: 3150 case PTR_TO_PACKET_END: 3151 case PTR_TO_FLOW_KEYS: 3152 case CONST_PTR_TO_MAP: 3153 case PTR_TO_SOCKET: 3154 case PTR_TO_SOCK_COMMON: 3155 case PTR_TO_TCP_SOCK: 3156 case PTR_TO_XDP_SOCK: 3157 case PTR_TO_BTF_ID: 3158 case PTR_TO_BUF: 3159 case PTR_TO_MEM: 3160 case PTR_TO_FUNC: 3161 case PTR_TO_MAP_KEY: 3162 return true; 3163 default: 3164 return false; 3165 } 3166 } 3167 3168 /* Does this register contain a constant zero? */ 3169 static bool register_is_null(struct bpf_reg_state *reg) 3170 { 3171 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 3172 } 3173 3174 static bool register_is_const(struct bpf_reg_state *reg) 3175 { 3176 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 3177 } 3178 3179 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 3180 { 3181 return tnum_is_unknown(reg->var_off) && 3182 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 3183 reg->umin_value == 0 && reg->umax_value == U64_MAX && 3184 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 3185 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 3186 } 3187 3188 static bool register_is_bounded(struct bpf_reg_state *reg) 3189 { 3190 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 3191 } 3192 3193 static bool __is_pointer_value(bool allow_ptr_leaks, 3194 const struct bpf_reg_state *reg) 3195 { 3196 if (allow_ptr_leaks) 3197 return false; 3198 3199 return reg->type != SCALAR_VALUE; 3200 } 3201 3202 static void save_register_state(struct bpf_func_state *state, 3203 int spi, struct bpf_reg_state *reg, 3204 int size) 3205 { 3206 int i; 3207 3208 state->stack[spi].spilled_ptr = *reg; 3209 if (size == BPF_REG_SIZE) 3210 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3211 3212 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 3213 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 3214 3215 /* size < 8 bytes spill */ 3216 for (; i; i--) 3217 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); 3218 } 3219 3220 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 3221 * stack boundary and alignment are checked in check_mem_access() 3222 */ 3223 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 3224 /* stack frame we're writing to */ 3225 struct bpf_func_state *state, 3226 int off, int size, int value_regno, 3227 int insn_idx) 3228 { 3229 struct bpf_func_state *cur; /* state of the current function */ 3230 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 3231 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 3232 struct bpf_reg_state *reg = NULL; 3233 3234 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 3235 if (err) 3236 return err; 3237 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 3238 * so it's aligned access and [off, off + size) are within stack limits 3239 */ 3240 if (!env->allow_ptr_leaks && 3241 state->stack[spi].slot_type[0] == STACK_SPILL && 3242 size != BPF_REG_SIZE) { 3243 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 3244 return -EACCES; 3245 } 3246 3247 cur = env->cur_state->frame[env->cur_state->curframe]; 3248 if (value_regno >= 0) 3249 reg = &cur->regs[value_regno]; 3250 if (!env->bypass_spec_v4) { 3251 bool sanitize = reg && is_spillable_regtype(reg->type); 3252 3253 for (i = 0; i < size; i++) { 3254 if (state->stack[spi].slot_type[i] == STACK_INVALID) { 3255 sanitize = true; 3256 break; 3257 } 3258 } 3259 3260 if (sanitize) 3261 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 3262 } 3263 3264 mark_stack_slot_scratched(env, spi); 3265 if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && 3266 !register_is_null(reg) && env->bpf_capable) { 3267 if (dst_reg != BPF_REG_FP) { 3268 /* The backtracking logic can only recognize explicit 3269 * stack slot address like [fp - 8]. Other spill of 3270 * scalar via different register has to be conservative. 3271 * Backtrack from here and mark all registers as precise 3272 * that contributed into 'reg' being a constant. 3273 */ 3274 err = mark_chain_precision(env, value_regno); 3275 if (err) 3276 return err; 3277 } 3278 save_register_state(state, spi, reg, size); 3279 } else if (reg && is_spillable_regtype(reg->type)) { 3280 /* register containing pointer is being spilled into stack */ 3281 if (size != BPF_REG_SIZE) { 3282 verbose_linfo(env, insn_idx, "; "); 3283 verbose(env, "invalid size of register spill\n"); 3284 return -EACCES; 3285 } 3286 if (state != cur && reg->type == PTR_TO_STACK) { 3287 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 3288 return -EINVAL; 3289 } 3290 save_register_state(state, spi, reg, size); 3291 } else { 3292 u8 type = STACK_MISC; 3293 3294 /* regular write of data into stack destroys any spilled ptr */ 3295 state->stack[spi].spilled_ptr.type = NOT_INIT; 3296 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 3297 if (is_spilled_reg(&state->stack[spi])) 3298 for (i = 0; i < BPF_REG_SIZE; i++) 3299 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 3300 3301 /* only mark the slot as written if all 8 bytes were written 3302 * otherwise read propagation may incorrectly stop too soon 3303 * when stack slots are partially written. 3304 * This heuristic means that read propagation will be 3305 * conservative, since it will add reg_live_read marks 3306 * to stack slots all the way to first state when programs 3307 * writes+reads less than 8 bytes 3308 */ 3309 if (size == BPF_REG_SIZE) 3310 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3311 3312 /* when we zero initialize stack slots mark them as such */ 3313 if (reg && register_is_null(reg)) { 3314 /* backtracking doesn't work for STACK_ZERO yet. */ 3315 err = mark_chain_precision(env, value_regno); 3316 if (err) 3317 return err; 3318 type = STACK_ZERO; 3319 } 3320 3321 /* Mark slots affected by this stack write. */ 3322 for (i = 0; i < size; i++) 3323 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 3324 type; 3325 } 3326 return 0; 3327 } 3328 3329 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 3330 * known to contain a variable offset. 3331 * This function checks whether the write is permitted and conservatively 3332 * tracks the effects of the write, considering that each stack slot in the 3333 * dynamic range is potentially written to. 3334 * 3335 * 'off' includes 'regno->off'. 3336 * 'value_regno' can be -1, meaning that an unknown value is being written to 3337 * the stack. 3338 * 3339 * Spilled pointers in range are not marked as written because we don't know 3340 * what's going to be actually written. This means that read propagation for 3341 * future reads cannot be terminated by this write. 3342 * 3343 * For privileged programs, uninitialized stack slots are considered 3344 * initialized by this write (even though we don't know exactly what offsets 3345 * are going to be written to). The idea is that we don't want the verifier to 3346 * reject future reads that access slots written to through variable offsets. 3347 */ 3348 static int check_stack_write_var_off(struct bpf_verifier_env *env, 3349 /* func where register points to */ 3350 struct bpf_func_state *state, 3351 int ptr_regno, int off, int size, 3352 int value_regno, int insn_idx) 3353 { 3354 struct bpf_func_state *cur; /* state of the current function */ 3355 int min_off, max_off; 3356 int i, err; 3357 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 3358 bool writing_zero = false; 3359 /* set if the fact that we're writing a zero is used to let any 3360 * stack slots remain STACK_ZERO 3361 */ 3362 bool zero_used = false; 3363 3364 cur = env->cur_state->frame[env->cur_state->curframe]; 3365 ptr_reg = &cur->regs[ptr_regno]; 3366 min_off = ptr_reg->smin_value + off; 3367 max_off = ptr_reg->smax_value + off + size; 3368 if (value_regno >= 0) 3369 value_reg = &cur->regs[value_regno]; 3370 if (value_reg && register_is_null(value_reg)) 3371 writing_zero = true; 3372 3373 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 3374 if (err) 3375 return err; 3376 3377 3378 /* Variable offset writes destroy any spilled pointers in range. */ 3379 for (i = min_off; i < max_off; i++) { 3380 u8 new_type, *stype; 3381 int slot, spi; 3382 3383 slot = -i - 1; 3384 spi = slot / BPF_REG_SIZE; 3385 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3386 mark_stack_slot_scratched(env, spi); 3387 3388 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 3389 /* Reject the write if range we may write to has not 3390 * been initialized beforehand. If we didn't reject 3391 * here, the ptr status would be erased below (even 3392 * though not all slots are actually overwritten), 3393 * possibly opening the door to leaks. 3394 * 3395 * We do however catch STACK_INVALID case below, and 3396 * only allow reading possibly uninitialized memory 3397 * later for CAP_PERFMON, as the write may not happen to 3398 * that slot. 3399 */ 3400 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 3401 insn_idx, i); 3402 return -EINVAL; 3403 } 3404 3405 /* Erase all spilled pointers. */ 3406 state->stack[spi].spilled_ptr.type = NOT_INIT; 3407 3408 /* Update the slot type. */ 3409 new_type = STACK_MISC; 3410 if (writing_zero && *stype == STACK_ZERO) { 3411 new_type = STACK_ZERO; 3412 zero_used = true; 3413 } 3414 /* If the slot is STACK_INVALID, we check whether it's OK to 3415 * pretend that it will be initialized by this write. The slot 3416 * might not actually be written to, and so if we mark it as 3417 * initialized future reads might leak uninitialized memory. 3418 * For privileged programs, we will accept such reads to slots 3419 * that may or may not be written because, if we're reject 3420 * them, the error would be too confusing. 3421 */ 3422 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 3423 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 3424 insn_idx, i); 3425 return -EINVAL; 3426 } 3427 *stype = new_type; 3428 } 3429 if (zero_used) { 3430 /* backtracking doesn't work for STACK_ZERO yet. */ 3431 err = mark_chain_precision(env, value_regno); 3432 if (err) 3433 return err; 3434 } 3435 return 0; 3436 } 3437 3438 /* When register 'dst_regno' is assigned some values from stack[min_off, 3439 * max_off), we set the register's type according to the types of the 3440 * respective stack slots. If all the stack values are known to be zeros, then 3441 * so is the destination reg. Otherwise, the register is considered to be 3442 * SCALAR. This function does not deal with register filling; the caller must 3443 * ensure that all spilled registers in the stack range have been marked as 3444 * read. 3445 */ 3446 static void mark_reg_stack_read(struct bpf_verifier_env *env, 3447 /* func where src register points to */ 3448 struct bpf_func_state *ptr_state, 3449 int min_off, int max_off, int dst_regno) 3450 { 3451 struct bpf_verifier_state *vstate = env->cur_state; 3452 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3453 int i, slot, spi; 3454 u8 *stype; 3455 int zeros = 0; 3456 3457 for (i = min_off; i < max_off; i++) { 3458 slot = -i - 1; 3459 spi = slot / BPF_REG_SIZE; 3460 stype = ptr_state->stack[spi].slot_type; 3461 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 3462 break; 3463 zeros++; 3464 } 3465 if (zeros == max_off - min_off) { 3466 /* any access_size read into register is zero extended, 3467 * so the whole register == const_zero 3468 */ 3469 __mark_reg_const_zero(&state->regs[dst_regno]); 3470 /* backtracking doesn't support STACK_ZERO yet, 3471 * so mark it precise here, so that later 3472 * backtracking can stop here. 3473 * Backtracking may not need this if this register 3474 * doesn't participate in pointer adjustment. 3475 * Forward propagation of precise flag is not 3476 * necessary either. This mark is only to stop 3477 * backtracking. Any register that contributed 3478 * to const 0 was marked precise before spill. 3479 */ 3480 state->regs[dst_regno].precise = true; 3481 } else { 3482 /* have read misc data from the stack */ 3483 mark_reg_unknown(env, state->regs, dst_regno); 3484 } 3485 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3486 } 3487 3488 /* Read the stack at 'off' and put the results into the register indicated by 3489 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 3490 * spilled reg. 3491 * 3492 * 'dst_regno' can be -1, meaning that the read value is not going to a 3493 * register. 3494 * 3495 * The access is assumed to be within the current stack bounds. 3496 */ 3497 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 3498 /* func where src register points to */ 3499 struct bpf_func_state *reg_state, 3500 int off, int size, int dst_regno) 3501 { 3502 struct bpf_verifier_state *vstate = env->cur_state; 3503 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3504 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 3505 struct bpf_reg_state *reg; 3506 u8 *stype, type; 3507 3508 stype = reg_state->stack[spi].slot_type; 3509 reg = ®_state->stack[spi].spilled_ptr; 3510 3511 if (is_spilled_reg(®_state->stack[spi])) { 3512 u8 spill_size = 1; 3513 3514 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 3515 spill_size++; 3516 3517 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 3518 if (reg->type != SCALAR_VALUE) { 3519 verbose_linfo(env, env->insn_idx, "; "); 3520 verbose(env, "invalid size of register fill\n"); 3521 return -EACCES; 3522 } 3523 3524 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3525 if (dst_regno < 0) 3526 return 0; 3527 3528 if (!(off % BPF_REG_SIZE) && size == spill_size) { 3529 /* The earlier check_reg_arg() has decided the 3530 * subreg_def for this insn. Save it first. 3531 */ 3532 s32 subreg_def = state->regs[dst_regno].subreg_def; 3533 3534 state->regs[dst_regno] = *reg; 3535 state->regs[dst_regno].subreg_def = subreg_def; 3536 } else { 3537 for (i = 0; i < size; i++) { 3538 type = stype[(slot - i) % BPF_REG_SIZE]; 3539 if (type == STACK_SPILL) 3540 continue; 3541 if (type == STACK_MISC) 3542 continue; 3543 verbose(env, "invalid read from stack off %d+%d size %d\n", 3544 off, i, size); 3545 return -EACCES; 3546 } 3547 mark_reg_unknown(env, state->regs, dst_regno); 3548 } 3549 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3550 return 0; 3551 } 3552 3553 if (dst_regno >= 0) { 3554 /* restore register state from stack */ 3555 state->regs[dst_regno] = *reg; 3556 /* mark reg as written since spilled pointer state likely 3557 * has its liveness marks cleared by is_state_visited() 3558 * which resets stack/reg liveness for state transitions 3559 */ 3560 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3561 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 3562 /* If dst_regno==-1, the caller is asking us whether 3563 * it is acceptable to use this value as a SCALAR_VALUE 3564 * (e.g. for XADD). 3565 * We must not allow unprivileged callers to do that 3566 * with spilled pointers. 3567 */ 3568 verbose(env, "leaking pointer from stack off %d\n", 3569 off); 3570 return -EACCES; 3571 } 3572 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3573 } else { 3574 for (i = 0; i < size; i++) { 3575 type = stype[(slot - i) % BPF_REG_SIZE]; 3576 if (type == STACK_MISC) 3577 continue; 3578 if (type == STACK_ZERO) 3579 continue; 3580 verbose(env, "invalid read from stack off %d+%d size %d\n", 3581 off, i, size); 3582 return -EACCES; 3583 } 3584 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3585 if (dst_regno >= 0) 3586 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 3587 } 3588 return 0; 3589 } 3590 3591 enum bpf_access_src { 3592 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 3593 ACCESS_HELPER = 2, /* the access is performed by a helper */ 3594 }; 3595 3596 static int check_stack_range_initialized(struct bpf_verifier_env *env, 3597 int regno, int off, int access_size, 3598 bool zero_size_allowed, 3599 enum bpf_access_src type, 3600 struct bpf_call_arg_meta *meta); 3601 3602 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 3603 { 3604 return cur_regs(env) + regno; 3605 } 3606 3607 /* Read the stack at 'ptr_regno + off' and put the result into the register 3608 * 'dst_regno'. 3609 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 3610 * but not its variable offset. 3611 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 3612 * 3613 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 3614 * filling registers (i.e. reads of spilled register cannot be detected when 3615 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 3616 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 3617 * offset; for a fixed offset check_stack_read_fixed_off should be used 3618 * instead. 3619 */ 3620 static int check_stack_read_var_off(struct bpf_verifier_env *env, 3621 int ptr_regno, int off, int size, int dst_regno) 3622 { 3623 /* The state of the source register. */ 3624 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3625 struct bpf_func_state *ptr_state = func(env, reg); 3626 int err; 3627 int min_off, max_off; 3628 3629 /* Note that we pass a NULL meta, so raw access will not be permitted. 3630 */ 3631 err = check_stack_range_initialized(env, ptr_regno, off, size, 3632 false, ACCESS_DIRECT, NULL); 3633 if (err) 3634 return err; 3635 3636 min_off = reg->smin_value + off; 3637 max_off = reg->smax_value + off; 3638 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3639 return 0; 3640 } 3641 3642 /* check_stack_read dispatches to check_stack_read_fixed_off or 3643 * check_stack_read_var_off. 3644 * 3645 * The caller must ensure that the offset falls within the allocated stack 3646 * bounds. 3647 * 3648 * 'dst_regno' is a register which will receive the value from the stack. It 3649 * can be -1, meaning that the read value is not going to a register. 3650 */ 3651 static int check_stack_read(struct bpf_verifier_env *env, 3652 int ptr_regno, int off, int size, 3653 int dst_regno) 3654 { 3655 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3656 struct bpf_func_state *state = func(env, reg); 3657 int err; 3658 /* Some accesses are only permitted with a static offset. */ 3659 bool var_off = !tnum_is_const(reg->var_off); 3660 3661 /* The offset is required to be static when reads don't go to a 3662 * register, in order to not leak pointers (see 3663 * check_stack_read_fixed_off). 3664 */ 3665 if (dst_regno < 0 && var_off) { 3666 char tn_buf[48]; 3667 3668 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3669 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3670 tn_buf, off, size); 3671 return -EACCES; 3672 } 3673 /* Variable offset is prohibited for unprivileged mode for simplicity 3674 * since it requires corresponding support in Spectre masking for stack 3675 * ALU. See also retrieve_ptr_limit(). 3676 */ 3677 if (!env->bypass_spec_v1 && var_off) { 3678 char tn_buf[48]; 3679 3680 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3681 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 3682 ptr_regno, tn_buf); 3683 return -EACCES; 3684 } 3685 3686 if (!var_off) { 3687 off += reg->var_off.value; 3688 err = check_stack_read_fixed_off(env, state, off, size, 3689 dst_regno); 3690 } else { 3691 /* Variable offset stack reads need more conservative handling 3692 * than fixed offset ones. Note that dst_regno >= 0 on this 3693 * branch. 3694 */ 3695 err = check_stack_read_var_off(env, ptr_regno, off, size, 3696 dst_regno); 3697 } 3698 return err; 3699 } 3700 3701 3702 /* check_stack_write dispatches to check_stack_write_fixed_off or 3703 * check_stack_write_var_off. 3704 * 3705 * 'ptr_regno' is the register used as a pointer into the stack. 3706 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 3707 * 'value_regno' is the register whose value we're writing to the stack. It can 3708 * be -1, meaning that we're not writing from a register. 3709 * 3710 * The caller must ensure that the offset falls within the maximum stack size. 3711 */ 3712 static int check_stack_write(struct bpf_verifier_env *env, 3713 int ptr_regno, int off, int size, 3714 int value_regno, int insn_idx) 3715 { 3716 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3717 struct bpf_func_state *state = func(env, reg); 3718 int err; 3719 3720 if (tnum_is_const(reg->var_off)) { 3721 off += reg->var_off.value; 3722 err = check_stack_write_fixed_off(env, state, off, size, 3723 value_regno, insn_idx); 3724 } else { 3725 /* Variable offset stack reads need more conservative handling 3726 * than fixed offset ones. 3727 */ 3728 err = check_stack_write_var_off(env, state, 3729 ptr_regno, off, size, 3730 value_regno, insn_idx); 3731 } 3732 return err; 3733 } 3734 3735 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 3736 int off, int size, enum bpf_access_type type) 3737 { 3738 struct bpf_reg_state *regs = cur_regs(env); 3739 struct bpf_map *map = regs[regno].map_ptr; 3740 u32 cap = bpf_map_flags_to_cap(map); 3741 3742 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 3743 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 3744 map->value_size, off, size); 3745 return -EACCES; 3746 } 3747 3748 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 3749 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 3750 map->value_size, off, size); 3751 return -EACCES; 3752 } 3753 3754 return 0; 3755 } 3756 3757 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 3758 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 3759 int off, int size, u32 mem_size, 3760 bool zero_size_allowed) 3761 { 3762 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 3763 struct bpf_reg_state *reg; 3764 3765 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 3766 return 0; 3767 3768 reg = &cur_regs(env)[regno]; 3769 switch (reg->type) { 3770 case PTR_TO_MAP_KEY: 3771 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 3772 mem_size, off, size); 3773 break; 3774 case PTR_TO_MAP_VALUE: 3775 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 3776 mem_size, off, size); 3777 break; 3778 case PTR_TO_PACKET: 3779 case PTR_TO_PACKET_META: 3780 case PTR_TO_PACKET_END: 3781 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 3782 off, size, regno, reg->id, off, mem_size); 3783 break; 3784 case PTR_TO_MEM: 3785 default: 3786 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 3787 mem_size, off, size); 3788 } 3789 3790 return -EACCES; 3791 } 3792 3793 /* check read/write into a memory region with possible variable offset */ 3794 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 3795 int off, int size, u32 mem_size, 3796 bool zero_size_allowed) 3797 { 3798 struct bpf_verifier_state *vstate = env->cur_state; 3799 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3800 struct bpf_reg_state *reg = &state->regs[regno]; 3801 int err; 3802 3803 /* We may have adjusted the register pointing to memory region, so we 3804 * need to try adding each of min_value and max_value to off 3805 * to make sure our theoretical access will be safe. 3806 * 3807 * The minimum value is only important with signed 3808 * comparisons where we can't assume the floor of a 3809 * value is 0. If we are using signed variables for our 3810 * index'es we need to make sure that whatever we use 3811 * will have a set floor within our range. 3812 */ 3813 if (reg->smin_value < 0 && 3814 (reg->smin_value == S64_MIN || 3815 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 3816 reg->smin_value + off < 0)) { 3817 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3818 regno); 3819 return -EACCES; 3820 } 3821 err = __check_mem_access(env, regno, reg->smin_value + off, size, 3822 mem_size, zero_size_allowed); 3823 if (err) { 3824 verbose(env, "R%d min value is outside of the allowed memory range\n", 3825 regno); 3826 return err; 3827 } 3828 3829 /* If we haven't set a max value then we need to bail since we can't be 3830 * sure we won't do bad things. 3831 * If reg->umax_value + off could overflow, treat that as unbounded too. 3832 */ 3833 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 3834 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 3835 regno); 3836 return -EACCES; 3837 } 3838 err = __check_mem_access(env, regno, reg->umax_value + off, size, 3839 mem_size, zero_size_allowed); 3840 if (err) { 3841 verbose(env, "R%d max value is outside of the allowed memory range\n", 3842 regno); 3843 return err; 3844 } 3845 3846 return 0; 3847 } 3848 3849 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 3850 const struct bpf_reg_state *reg, int regno, 3851 bool fixed_off_ok) 3852 { 3853 /* Access to this pointer-typed register or passing it to a helper 3854 * is only allowed in its original, unmodified form. 3855 */ 3856 3857 if (reg->off < 0) { 3858 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 3859 reg_type_str(env, reg->type), regno, reg->off); 3860 return -EACCES; 3861 } 3862 3863 if (!fixed_off_ok && reg->off) { 3864 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 3865 reg_type_str(env, reg->type), regno, reg->off); 3866 return -EACCES; 3867 } 3868 3869 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3870 char tn_buf[48]; 3871 3872 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3873 verbose(env, "variable %s access var_off=%s disallowed\n", 3874 reg_type_str(env, reg->type), tn_buf); 3875 return -EACCES; 3876 } 3877 3878 return 0; 3879 } 3880 3881 int check_ptr_off_reg(struct bpf_verifier_env *env, 3882 const struct bpf_reg_state *reg, int regno) 3883 { 3884 return __check_ptr_off_reg(env, reg, regno, false); 3885 } 3886 3887 static int map_kptr_match_type(struct bpf_verifier_env *env, 3888 struct btf_field *kptr_field, 3889 struct bpf_reg_state *reg, u32 regno) 3890 { 3891 const char *targ_name = kernel_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 3892 int perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED; 3893 const char *reg_name = ""; 3894 3895 /* Only unreferenced case accepts untrusted pointers */ 3896 if (kptr_field->type == BPF_KPTR_UNREF) 3897 perm_flags |= PTR_UNTRUSTED; 3898 3899 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 3900 goto bad_type; 3901 3902 if (!btf_is_kernel(reg->btf)) { 3903 verbose(env, "R%d must point to kernel BTF\n", regno); 3904 return -EINVAL; 3905 } 3906 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 3907 reg_name = kernel_type_name(reg->btf, reg->btf_id); 3908 3909 /* For ref_ptr case, release function check should ensure we get one 3910 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 3911 * normal store of unreferenced kptr, we must ensure var_off is zero. 3912 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 3913 * reg->off and reg->ref_obj_id are not needed here. 3914 */ 3915 if (__check_ptr_off_reg(env, reg, regno, true)) 3916 return -EACCES; 3917 3918 /* A full type match is needed, as BTF can be vmlinux or module BTF, and 3919 * we also need to take into account the reg->off. 3920 * 3921 * We want to support cases like: 3922 * 3923 * struct foo { 3924 * struct bar br; 3925 * struct baz bz; 3926 * }; 3927 * 3928 * struct foo *v; 3929 * v = func(); // PTR_TO_BTF_ID 3930 * val->foo = v; // reg->off is zero, btf and btf_id match type 3931 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 3932 * // first member type of struct after comparison fails 3933 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 3934 * // to match type 3935 * 3936 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 3937 * is zero. We must also ensure that btf_struct_ids_match does not walk 3938 * the struct to match type against first member of struct, i.e. reject 3939 * second case from above. Hence, when type is BPF_KPTR_REF, we set 3940 * strict mode to true for type match. 3941 */ 3942 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 3943 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 3944 kptr_field->type == BPF_KPTR_REF)) 3945 goto bad_type; 3946 return 0; 3947 bad_type: 3948 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 3949 reg_type_str(env, reg->type), reg_name); 3950 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 3951 if (kptr_field->type == BPF_KPTR_UNREF) 3952 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 3953 targ_name); 3954 else 3955 verbose(env, "\n"); 3956 return -EINVAL; 3957 } 3958 3959 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 3960 int value_regno, int insn_idx, 3961 struct btf_field *kptr_field) 3962 { 3963 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3964 int class = BPF_CLASS(insn->code); 3965 struct bpf_reg_state *val_reg; 3966 3967 /* Things we already checked for in check_map_access and caller: 3968 * - Reject cases where variable offset may touch kptr 3969 * - size of access (must be BPF_DW) 3970 * - tnum_is_const(reg->var_off) 3971 * - kptr_field->offset == off + reg->var_off.value 3972 */ 3973 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 3974 if (BPF_MODE(insn->code) != BPF_MEM) { 3975 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 3976 return -EACCES; 3977 } 3978 3979 /* We only allow loading referenced kptr, since it will be marked as 3980 * untrusted, similar to unreferenced kptr. 3981 */ 3982 if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 3983 verbose(env, "store to referenced kptr disallowed\n"); 3984 return -EACCES; 3985 } 3986 3987 if (class == BPF_LDX) { 3988 val_reg = reg_state(env, value_regno); 3989 /* We can simply mark the value_regno receiving the pointer 3990 * value from map as PTR_TO_BTF_ID, with the correct type. 3991 */ 3992 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 3993 kptr_field->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 3994 /* For mark_ptr_or_null_reg */ 3995 val_reg->id = ++env->id_gen; 3996 } else if (class == BPF_STX) { 3997 val_reg = reg_state(env, value_regno); 3998 if (!register_is_null(val_reg) && 3999 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 4000 return -EACCES; 4001 } else if (class == BPF_ST) { 4002 if (insn->imm) { 4003 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 4004 kptr_field->offset); 4005 return -EACCES; 4006 } 4007 } else { 4008 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 4009 return -EACCES; 4010 } 4011 return 0; 4012 } 4013 4014 /* check read/write into a map element with possible variable offset */ 4015 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 4016 int off, int size, bool zero_size_allowed, 4017 enum bpf_access_src src) 4018 { 4019 struct bpf_verifier_state *vstate = env->cur_state; 4020 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4021 struct bpf_reg_state *reg = &state->regs[regno]; 4022 struct bpf_map *map = reg->map_ptr; 4023 struct btf_record *rec; 4024 int err, i; 4025 4026 err = check_mem_region_access(env, regno, off, size, map->value_size, 4027 zero_size_allowed); 4028 if (err) 4029 return err; 4030 4031 if (IS_ERR_OR_NULL(map->record)) 4032 return 0; 4033 rec = map->record; 4034 for (i = 0; i < rec->cnt; i++) { 4035 struct btf_field *field = &rec->fields[i]; 4036 u32 p = field->offset; 4037 4038 /* If any part of a field can be touched by load/store, reject 4039 * this program. To check that [x1, x2) overlaps with [y1, y2), 4040 * it is sufficient to check x1 < y2 && y1 < x2. 4041 */ 4042 if (reg->smin_value + off < p + btf_field_type_size(field->type) && 4043 p < reg->umax_value + off + size) { 4044 switch (field->type) { 4045 case BPF_KPTR_UNREF: 4046 case BPF_KPTR_REF: 4047 if (src != ACCESS_DIRECT) { 4048 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 4049 return -EACCES; 4050 } 4051 if (!tnum_is_const(reg->var_off)) { 4052 verbose(env, "kptr access cannot have variable offset\n"); 4053 return -EACCES; 4054 } 4055 if (p != off + reg->var_off.value) { 4056 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 4057 p, off + reg->var_off.value); 4058 return -EACCES; 4059 } 4060 if (size != bpf_size_to_bytes(BPF_DW)) { 4061 verbose(env, "kptr access size must be BPF_DW\n"); 4062 return -EACCES; 4063 } 4064 break; 4065 default: 4066 verbose(env, "%s cannot be accessed directly by load/store\n", 4067 btf_field_type_name(field->type)); 4068 return -EACCES; 4069 } 4070 } 4071 } 4072 return 0; 4073 } 4074 4075 #define MAX_PACKET_OFF 0xffff 4076 4077 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 4078 const struct bpf_call_arg_meta *meta, 4079 enum bpf_access_type t) 4080 { 4081 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 4082 4083 switch (prog_type) { 4084 /* Program types only with direct read access go here! */ 4085 case BPF_PROG_TYPE_LWT_IN: 4086 case BPF_PROG_TYPE_LWT_OUT: 4087 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 4088 case BPF_PROG_TYPE_SK_REUSEPORT: 4089 case BPF_PROG_TYPE_FLOW_DISSECTOR: 4090 case BPF_PROG_TYPE_CGROUP_SKB: 4091 if (t == BPF_WRITE) 4092 return false; 4093 fallthrough; 4094 4095 /* Program types with direct read + write access go here! */ 4096 case BPF_PROG_TYPE_SCHED_CLS: 4097 case BPF_PROG_TYPE_SCHED_ACT: 4098 case BPF_PROG_TYPE_XDP: 4099 case BPF_PROG_TYPE_LWT_XMIT: 4100 case BPF_PROG_TYPE_SK_SKB: 4101 case BPF_PROG_TYPE_SK_MSG: 4102 if (meta) 4103 return meta->pkt_access; 4104 4105 env->seen_direct_write = true; 4106 return true; 4107 4108 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4109 if (t == BPF_WRITE) 4110 env->seen_direct_write = true; 4111 4112 return true; 4113 4114 default: 4115 return false; 4116 } 4117 } 4118 4119 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 4120 int size, bool zero_size_allowed) 4121 { 4122 struct bpf_reg_state *regs = cur_regs(env); 4123 struct bpf_reg_state *reg = ®s[regno]; 4124 int err; 4125 4126 /* We may have added a variable offset to the packet pointer; but any 4127 * reg->range we have comes after that. We are only checking the fixed 4128 * offset. 4129 */ 4130 4131 /* We don't allow negative numbers, because we aren't tracking enough 4132 * detail to prove they're safe. 4133 */ 4134 if (reg->smin_value < 0) { 4135 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4136 regno); 4137 return -EACCES; 4138 } 4139 4140 err = reg->range < 0 ? -EINVAL : 4141 __check_mem_access(env, regno, off, size, reg->range, 4142 zero_size_allowed); 4143 if (err) { 4144 verbose(env, "R%d offset is outside of the packet\n", regno); 4145 return err; 4146 } 4147 4148 /* __check_mem_access has made sure "off + size - 1" is within u16. 4149 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 4150 * otherwise find_good_pkt_pointers would have refused to set range info 4151 * that __check_mem_access would have rejected this pkt access. 4152 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 4153 */ 4154 env->prog->aux->max_pkt_offset = 4155 max_t(u32, env->prog->aux->max_pkt_offset, 4156 off + reg->umax_value + size - 1); 4157 4158 return err; 4159 } 4160 4161 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 4162 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 4163 enum bpf_access_type t, enum bpf_reg_type *reg_type, 4164 struct btf **btf, u32 *btf_id) 4165 { 4166 struct bpf_insn_access_aux info = { 4167 .reg_type = *reg_type, 4168 .log = &env->log, 4169 }; 4170 4171 if (env->ops->is_valid_access && 4172 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 4173 /* A non zero info.ctx_field_size indicates that this field is a 4174 * candidate for later verifier transformation to load the whole 4175 * field and then apply a mask when accessed with a narrower 4176 * access than actual ctx access size. A zero info.ctx_field_size 4177 * will only allow for whole field access and rejects any other 4178 * type of narrower access. 4179 */ 4180 *reg_type = info.reg_type; 4181 4182 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 4183 *btf = info.btf; 4184 *btf_id = info.btf_id; 4185 } else { 4186 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 4187 } 4188 /* remember the offset of last byte accessed in ctx */ 4189 if (env->prog->aux->max_ctx_offset < off + size) 4190 env->prog->aux->max_ctx_offset = off + size; 4191 return 0; 4192 } 4193 4194 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 4195 return -EACCES; 4196 } 4197 4198 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 4199 int size) 4200 { 4201 if (size < 0 || off < 0 || 4202 (u64)off + size > sizeof(struct bpf_flow_keys)) { 4203 verbose(env, "invalid access to flow keys off=%d size=%d\n", 4204 off, size); 4205 return -EACCES; 4206 } 4207 return 0; 4208 } 4209 4210 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 4211 u32 regno, int off, int size, 4212 enum bpf_access_type t) 4213 { 4214 struct bpf_reg_state *regs = cur_regs(env); 4215 struct bpf_reg_state *reg = ®s[regno]; 4216 struct bpf_insn_access_aux info = {}; 4217 bool valid; 4218 4219 if (reg->smin_value < 0) { 4220 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4221 regno); 4222 return -EACCES; 4223 } 4224 4225 switch (reg->type) { 4226 case PTR_TO_SOCK_COMMON: 4227 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 4228 break; 4229 case PTR_TO_SOCKET: 4230 valid = bpf_sock_is_valid_access(off, size, t, &info); 4231 break; 4232 case PTR_TO_TCP_SOCK: 4233 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 4234 break; 4235 case PTR_TO_XDP_SOCK: 4236 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 4237 break; 4238 default: 4239 valid = false; 4240 } 4241 4242 4243 if (valid) { 4244 env->insn_aux_data[insn_idx].ctx_field_size = 4245 info.ctx_field_size; 4246 return 0; 4247 } 4248 4249 verbose(env, "R%d invalid %s access off=%d size=%d\n", 4250 regno, reg_type_str(env, reg->type), off, size); 4251 4252 return -EACCES; 4253 } 4254 4255 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 4256 { 4257 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 4258 } 4259 4260 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 4261 { 4262 const struct bpf_reg_state *reg = reg_state(env, regno); 4263 4264 return reg->type == PTR_TO_CTX; 4265 } 4266 4267 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 4268 { 4269 const struct bpf_reg_state *reg = reg_state(env, regno); 4270 4271 return type_is_sk_pointer(reg->type); 4272 } 4273 4274 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 4275 { 4276 const struct bpf_reg_state *reg = reg_state(env, regno); 4277 4278 return type_is_pkt_pointer(reg->type); 4279 } 4280 4281 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 4282 { 4283 const struct bpf_reg_state *reg = reg_state(env, regno); 4284 4285 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 4286 return reg->type == PTR_TO_FLOW_KEYS; 4287 } 4288 4289 static bool is_trusted_reg(const struct bpf_reg_state *reg) 4290 { 4291 /* A referenced register is always trusted. */ 4292 if (reg->ref_obj_id) 4293 return true; 4294 4295 /* If a register is not referenced, it is trusted if it has the 4296 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 4297 * other type modifiers may be safe, but we elect to take an opt-in 4298 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 4299 * not. 4300 * 4301 * Eventually, we should make PTR_TRUSTED the single source of truth 4302 * for whether a register is trusted. 4303 */ 4304 return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && 4305 !bpf_type_has_unsafe_modifiers(reg->type); 4306 } 4307 4308 static bool is_rcu_reg(const struct bpf_reg_state *reg) 4309 { 4310 return reg->type & MEM_RCU; 4311 } 4312 4313 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 4314 const struct bpf_reg_state *reg, 4315 int off, int size, bool strict) 4316 { 4317 struct tnum reg_off; 4318 int ip_align; 4319 4320 /* Byte size accesses are always allowed. */ 4321 if (!strict || size == 1) 4322 return 0; 4323 4324 /* For platforms that do not have a Kconfig enabling 4325 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 4326 * NET_IP_ALIGN is universally set to '2'. And on platforms 4327 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 4328 * to this code only in strict mode where we want to emulate 4329 * the NET_IP_ALIGN==2 checking. Therefore use an 4330 * unconditional IP align value of '2'. 4331 */ 4332 ip_align = 2; 4333 4334 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 4335 if (!tnum_is_aligned(reg_off, size)) { 4336 char tn_buf[48]; 4337 4338 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4339 verbose(env, 4340 "misaligned packet access off %d+%s+%d+%d size %d\n", 4341 ip_align, tn_buf, reg->off, off, size); 4342 return -EACCES; 4343 } 4344 4345 return 0; 4346 } 4347 4348 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 4349 const struct bpf_reg_state *reg, 4350 const char *pointer_desc, 4351 int off, int size, bool strict) 4352 { 4353 struct tnum reg_off; 4354 4355 /* Byte size accesses are always allowed. */ 4356 if (!strict || size == 1) 4357 return 0; 4358 4359 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 4360 if (!tnum_is_aligned(reg_off, size)) { 4361 char tn_buf[48]; 4362 4363 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4364 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 4365 pointer_desc, tn_buf, reg->off, off, size); 4366 return -EACCES; 4367 } 4368 4369 return 0; 4370 } 4371 4372 static int check_ptr_alignment(struct bpf_verifier_env *env, 4373 const struct bpf_reg_state *reg, int off, 4374 int size, bool strict_alignment_once) 4375 { 4376 bool strict = env->strict_alignment || strict_alignment_once; 4377 const char *pointer_desc = ""; 4378 4379 switch (reg->type) { 4380 case PTR_TO_PACKET: 4381 case PTR_TO_PACKET_META: 4382 /* Special case, because of NET_IP_ALIGN. Given metadata sits 4383 * right in front, treat it the very same way. 4384 */ 4385 return check_pkt_ptr_alignment(env, reg, off, size, strict); 4386 case PTR_TO_FLOW_KEYS: 4387 pointer_desc = "flow keys "; 4388 break; 4389 case PTR_TO_MAP_KEY: 4390 pointer_desc = "key "; 4391 break; 4392 case PTR_TO_MAP_VALUE: 4393 pointer_desc = "value "; 4394 break; 4395 case PTR_TO_CTX: 4396 pointer_desc = "context "; 4397 break; 4398 case PTR_TO_STACK: 4399 pointer_desc = "stack "; 4400 /* The stack spill tracking logic in check_stack_write_fixed_off() 4401 * and check_stack_read_fixed_off() relies on stack accesses being 4402 * aligned. 4403 */ 4404 strict = true; 4405 break; 4406 case PTR_TO_SOCKET: 4407 pointer_desc = "sock "; 4408 break; 4409 case PTR_TO_SOCK_COMMON: 4410 pointer_desc = "sock_common "; 4411 break; 4412 case PTR_TO_TCP_SOCK: 4413 pointer_desc = "tcp_sock "; 4414 break; 4415 case PTR_TO_XDP_SOCK: 4416 pointer_desc = "xdp_sock "; 4417 break; 4418 default: 4419 break; 4420 } 4421 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 4422 strict); 4423 } 4424 4425 static int update_stack_depth(struct bpf_verifier_env *env, 4426 const struct bpf_func_state *func, 4427 int off) 4428 { 4429 u16 stack = env->subprog_info[func->subprogno].stack_depth; 4430 4431 if (stack >= -off) 4432 return 0; 4433 4434 /* update known max for given subprogram */ 4435 env->subprog_info[func->subprogno].stack_depth = -off; 4436 return 0; 4437 } 4438 4439 /* starting from main bpf function walk all instructions of the function 4440 * and recursively walk all callees that given function can call. 4441 * Ignore jump and exit insns. 4442 * Since recursion is prevented by check_cfg() this algorithm 4443 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 4444 */ 4445 static int check_max_stack_depth(struct bpf_verifier_env *env) 4446 { 4447 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 4448 struct bpf_subprog_info *subprog = env->subprog_info; 4449 struct bpf_insn *insn = env->prog->insnsi; 4450 bool tail_call_reachable = false; 4451 int ret_insn[MAX_CALL_FRAMES]; 4452 int ret_prog[MAX_CALL_FRAMES]; 4453 int j; 4454 4455 process_func: 4456 /* protect against potential stack overflow that might happen when 4457 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 4458 * depth for such case down to 256 so that the worst case scenario 4459 * would result in 8k stack size (32 which is tailcall limit * 256 = 4460 * 8k). 4461 * 4462 * To get the idea what might happen, see an example: 4463 * func1 -> sub rsp, 128 4464 * subfunc1 -> sub rsp, 256 4465 * tailcall1 -> add rsp, 256 4466 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 4467 * subfunc2 -> sub rsp, 64 4468 * subfunc22 -> sub rsp, 128 4469 * tailcall2 -> add rsp, 128 4470 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 4471 * 4472 * tailcall will unwind the current stack frame but it will not get rid 4473 * of caller's stack as shown on the example above. 4474 */ 4475 if (idx && subprog[idx].has_tail_call && depth >= 256) { 4476 verbose(env, 4477 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 4478 depth); 4479 return -EACCES; 4480 } 4481 /* round up to 32-bytes, since this is granularity 4482 * of interpreter stack size 4483 */ 4484 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4485 if (depth > MAX_BPF_STACK) { 4486 verbose(env, "combined stack size of %d calls is %d. Too large\n", 4487 frame + 1, depth); 4488 return -EACCES; 4489 } 4490 continue_func: 4491 subprog_end = subprog[idx + 1].start; 4492 for (; i < subprog_end; i++) { 4493 int next_insn; 4494 4495 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 4496 continue; 4497 /* remember insn and function to return to */ 4498 ret_insn[frame] = i + 1; 4499 ret_prog[frame] = idx; 4500 4501 /* find the callee */ 4502 next_insn = i + insn[i].imm + 1; 4503 idx = find_subprog(env, next_insn); 4504 if (idx < 0) { 4505 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4506 next_insn); 4507 return -EFAULT; 4508 } 4509 if (subprog[idx].is_async_cb) { 4510 if (subprog[idx].has_tail_call) { 4511 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 4512 return -EFAULT; 4513 } 4514 /* async callbacks don't increase bpf prog stack size */ 4515 continue; 4516 } 4517 i = next_insn; 4518 4519 if (subprog[idx].has_tail_call) 4520 tail_call_reachable = true; 4521 4522 frame++; 4523 if (frame >= MAX_CALL_FRAMES) { 4524 verbose(env, "the call stack of %d frames is too deep !\n", 4525 frame); 4526 return -E2BIG; 4527 } 4528 goto process_func; 4529 } 4530 /* if tail call got detected across bpf2bpf calls then mark each of the 4531 * currently present subprog frames as tail call reachable subprogs; 4532 * this info will be utilized by JIT so that we will be preserving the 4533 * tail call counter throughout bpf2bpf calls combined with tailcalls 4534 */ 4535 if (tail_call_reachable) 4536 for (j = 0; j < frame; j++) 4537 subprog[ret_prog[j]].tail_call_reachable = true; 4538 if (subprog[0].tail_call_reachable) 4539 env->prog->aux->tail_call_reachable = true; 4540 4541 /* end of for() loop means the last insn of the 'subprog' 4542 * was reached. Doesn't matter whether it was JA or EXIT 4543 */ 4544 if (frame == 0) 4545 return 0; 4546 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4547 frame--; 4548 i = ret_insn[frame]; 4549 idx = ret_prog[frame]; 4550 goto continue_func; 4551 } 4552 4553 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 4554 static int get_callee_stack_depth(struct bpf_verifier_env *env, 4555 const struct bpf_insn *insn, int idx) 4556 { 4557 int start = idx + insn->imm + 1, subprog; 4558 4559 subprog = find_subprog(env, start); 4560 if (subprog < 0) { 4561 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4562 start); 4563 return -EFAULT; 4564 } 4565 return env->subprog_info[subprog].stack_depth; 4566 } 4567 #endif 4568 4569 static int __check_buffer_access(struct bpf_verifier_env *env, 4570 const char *buf_info, 4571 const struct bpf_reg_state *reg, 4572 int regno, int off, int size) 4573 { 4574 if (off < 0) { 4575 verbose(env, 4576 "R%d invalid %s buffer access: off=%d, size=%d\n", 4577 regno, buf_info, off, size); 4578 return -EACCES; 4579 } 4580 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4581 char tn_buf[48]; 4582 4583 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4584 verbose(env, 4585 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 4586 regno, off, tn_buf); 4587 return -EACCES; 4588 } 4589 4590 return 0; 4591 } 4592 4593 static int check_tp_buffer_access(struct bpf_verifier_env *env, 4594 const struct bpf_reg_state *reg, 4595 int regno, int off, int size) 4596 { 4597 int err; 4598 4599 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 4600 if (err) 4601 return err; 4602 4603 if (off + size > env->prog->aux->max_tp_access) 4604 env->prog->aux->max_tp_access = off + size; 4605 4606 return 0; 4607 } 4608 4609 static int check_buffer_access(struct bpf_verifier_env *env, 4610 const struct bpf_reg_state *reg, 4611 int regno, int off, int size, 4612 bool zero_size_allowed, 4613 u32 *max_access) 4614 { 4615 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 4616 int err; 4617 4618 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 4619 if (err) 4620 return err; 4621 4622 if (off + size > *max_access) 4623 *max_access = off + size; 4624 4625 return 0; 4626 } 4627 4628 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 4629 static void zext_32_to_64(struct bpf_reg_state *reg) 4630 { 4631 reg->var_off = tnum_subreg(reg->var_off); 4632 __reg_assign_32_into_64(reg); 4633 } 4634 4635 /* truncate register to smaller size (in bytes) 4636 * must be called with size < BPF_REG_SIZE 4637 */ 4638 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 4639 { 4640 u64 mask; 4641 4642 /* clear high bits in bit representation */ 4643 reg->var_off = tnum_cast(reg->var_off, size); 4644 4645 /* fix arithmetic bounds */ 4646 mask = ((u64)1 << (size * 8)) - 1; 4647 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 4648 reg->umin_value &= mask; 4649 reg->umax_value &= mask; 4650 } else { 4651 reg->umin_value = 0; 4652 reg->umax_value = mask; 4653 } 4654 reg->smin_value = reg->umin_value; 4655 reg->smax_value = reg->umax_value; 4656 4657 /* If size is smaller than 32bit register the 32bit register 4658 * values are also truncated so we push 64-bit bounds into 4659 * 32-bit bounds. Above were truncated < 32-bits already. 4660 */ 4661 if (size >= 4) 4662 return; 4663 __reg_combine_64_into_32(reg); 4664 } 4665 4666 static bool bpf_map_is_rdonly(const struct bpf_map *map) 4667 { 4668 /* A map is considered read-only if the following condition are true: 4669 * 4670 * 1) BPF program side cannot change any of the map content. The 4671 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 4672 * and was set at map creation time. 4673 * 2) The map value(s) have been initialized from user space by a 4674 * loader and then "frozen", such that no new map update/delete 4675 * operations from syscall side are possible for the rest of 4676 * the map's lifetime from that point onwards. 4677 * 3) Any parallel/pending map update/delete operations from syscall 4678 * side have been completed. Only after that point, it's safe to 4679 * assume that map value(s) are immutable. 4680 */ 4681 return (map->map_flags & BPF_F_RDONLY_PROG) && 4682 READ_ONCE(map->frozen) && 4683 !bpf_map_write_active(map); 4684 } 4685 4686 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 4687 { 4688 void *ptr; 4689 u64 addr; 4690 int err; 4691 4692 err = map->ops->map_direct_value_addr(map, &addr, off); 4693 if (err) 4694 return err; 4695 ptr = (void *)(long)addr + off; 4696 4697 switch (size) { 4698 case sizeof(u8): 4699 *val = (u64)*(u8 *)ptr; 4700 break; 4701 case sizeof(u16): 4702 *val = (u64)*(u16 *)ptr; 4703 break; 4704 case sizeof(u32): 4705 *val = (u64)*(u32 *)ptr; 4706 break; 4707 case sizeof(u64): 4708 *val = *(u64 *)ptr; 4709 break; 4710 default: 4711 return -EINVAL; 4712 } 4713 return 0; 4714 } 4715 4716 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 4717 struct bpf_reg_state *regs, 4718 int regno, int off, int size, 4719 enum bpf_access_type atype, 4720 int value_regno) 4721 { 4722 struct bpf_reg_state *reg = regs + regno; 4723 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 4724 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 4725 enum bpf_type_flag flag = 0; 4726 u32 btf_id; 4727 int ret; 4728 4729 if (!env->allow_ptr_leaks) { 4730 verbose(env, 4731 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4732 tname); 4733 return -EPERM; 4734 } 4735 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 4736 verbose(env, 4737 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 4738 tname); 4739 return -EINVAL; 4740 } 4741 if (off < 0) { 4742 verbose(env, 4743 "R%d is ptr_%s invalid negative access: off=%d\n", 4744 regno, tname, off); 4745 return -EACCES; 4746 } 4747 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4748 char tn_buf[48]; 4749 4750 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4751 verbose(env, 4752 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 4753 regno, tname, off, tn_buf); 4754 return -EACCES; 4755 } 4756 4757 if (reg->type & MEM_USER) { 4758 verbose(env, 4759 "R%d is ptr_%s access user memory: off=%d\n", 4760 regno, tname, off); 4761 return -EACCES; 4762 } 4763 4764 if (reg->type & MEM_PERCPU) { 4765 verbose(env, 4766 "R%d is ptr_%s access percpu memory: off=%d\n", 4767 regno, tname, off); 4768 return -EACCES; 4769 } 4770 4771 if (env->ops->btf_struct_access && !type_is_alloc(reg->type)) { 4772 if (!btf_is_kernel(reg->btf)) { 4773 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); 4774 return -EFAULT; 4775 } 4776 ret = env->ops->btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 4777 } else { 4778 /* Writes are permitted with default btf_struct_access for 4779 * program allocated objects (which always have ref_obj_id > 0), 4780 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. 4781 */ 4782 if (atype != BPF_READ && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 4783 verbose(env, "only read is supported\n"); 4784 return -EACCES; 4785 } 4786 4787 if (type_is_alloc(reg->type) && !reg->ref_obj_id) { 4788 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 4789 return -EFAULT; 4790 } 4791 4792 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 4793 } 4794 4795 if (ret < 0) 4796 return ret; 4797 4798 /* If this is an untrusted pointer, all pointers formed by walking it 4799 * also inherit the untrusted flag. 4800 */ 4801 if (type_flag(reg->type) & PTR_UNTRUSTED) 4802 flag |= PTR_UNTRUSTED; 4803 4804 /* By default any pointer obtained from walking a trusted pointer is 4805 * no longer trusted except the rcu case below. 4806 */ 4807 flag &= ~PTR_TRUSTED; 4808 4809 if (flag & MEM_RCU) { 4810 /* Mark value register as MEM_RCU only if it is protected by 4811 * bpf_rcu_read_lock() and the ptr reg is rcu or trusted. MEM_RCU 4812 * itself can already indicate trustedness inside the rcu 4813 * read lock region. Also mark rcu pointer as PTR_MAYBE_NULL since 4814 * it could be null in some cases. 4815 */ 4816 if (!env->cur_state->active_rcu_lock || 4817 !(is_trusted_reg(reg) || is_rcu_reg(reg))) 4818 flag &= ~MEM_RCU; 4819 else 4820 flag |= PTR_MAYBE_NULL; 4821 } else if (reg->type & MEM_RCU) { 4822 /* ptr (reg) is marked as MEM_RCU, but the struct field is not tagged 4823 * with __rcu. Mark the flag as PTR_UNTRUSTED conservatively. 4824 */ 4825 flag |= PTR_UNTRUSTED; 4826 } 4827 4828 if (atype == BPF_READ && value_regno >= 0) 4829 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 4830 4831 return 0; 4832 } 4833 4834 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 4835 struct bpf_reg_state *regs, 4836 int regno, int off, int size, 4837 enum bpf_access_type atype, 4838 int value_regno) 4839 { 4840 struct bpf_reg_state *reg = regs + regno; 4841 struct bpf_map *map = reg->map_ptr; 4842 struct bpf_reg_state map_reg; 4843 enum bpf_type_flag flag = 0; 4844 const struct btf_type *t; 4845 const char *tname; 4846 u32 btf_id; 4847 int ret; 4848 4849 if (!btf_vmlinux) { 4850 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 4851 return -ENOTSUPP; 4852 } 4853 4854 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 4855 verbose(env, "map_ptr access not supported for map type %d\n", 4856 map->map_type); 4857 return -ENOTSUPP; 4858 } 4859 4860 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 4861 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 4862 4863 if (!env->allow_ptr_leaks) { 4864 verbose(env, 4865 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4866 tname); 4867 return -EPERM; 4868 } 4869 4870 if (off < 0) { 4871 verbose(env, "R%d is %s invalid negative access: off=%d\n", 4872 regno, tname, off); 4873 return -EACCES; 4874 } 4875 4876 if (atype != BPF_READ) { 4877 verbose(env, "only read from %s is supported\n", tname); 4878 return -EACCES; 4879 } 4880 4881 /* Simulate access to a PTR_TO_BTF_ID */ 4882 memset(&map_reg, 0, sizeof(map_reg)); 4883 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); 4884 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag); 4885 if (ret < 0) 4886 return ret; 4887 4888 if (value_regno >= 0) 4889 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 4890 4891 return 0; 4892 } 4893 4894 /* Check that the stack access at the given offset is within bounds. The 4895 * maximum valid offset is -1. 4896 * 4897 * The minimum valid offset is -MAX_BPF_STACK for writes, and 4898 * -state->allocated_stack for reads. 4899 */ 4900 static int check_stack_slot_within_bounds(int off, 4901 struct bpf_func_state *state, 4902 enum bpf_access_type t) 4903 { 4904 int min_valid_off; 4905 4906 if (t == BPF_WRITE) 4907 min_valid_off = -MAX_BPF_STACK; 4908 else 4909 min_valid_off = -state->allocated_stack; 4910 4911 if (off < min_valid_off || off > -1) 4912 return -EACCES; 4913 return 0; 4914 } 4915 4916 /* Check that the stack access at 'regno + off' falls within the maximum stack 4917 * bounds. 4918 * 4919 * 'off' includes `regno->offset`, but not its dynamic part (if any). 4920 */ 4921 static int check_stack_access_within_bounds( 4922 struct bpf_verifier_env *env, 4923 int regno, int off, int access_size, 4924 enum bpf_access_src src, enum bpf_access_type type) 4925 { 4926 struct bpf_reg_state *regs = cur_regs(env); 4927 struct bpf_reg_state *reg = regs + regno; 4928 struct bpf_func_state *state = func(env, reg); 4929 int min_off, max_off; 4930 int err; 4931 char *err_extra; 4932 4933 if (src == ACCESS_HELPER) 4934 /* We don't know if helpers are reading or writing (or both). */ 4935 err_extra = " indirect access to"; 4936 else if (type == BPF_READ) 4937 err_extra = " read from"; 4938 else 4939 err_extra = " write to"; 4940 4941 if (tnum_is_const(reg->var_off)) { 4942 min_off = reg->var_off.value + off; 4943 if (access_size > 0) 4944 max_off = min_off + access_size - 1; 4945 else 4946 max_off = min_off; 4947 } else { 4948 if (reg->smax_value >= BPF_MAX_VAR_OFF || 4949 reg->smin_value <= -BPF_MAX_VAR_OFF) { 4950 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 4951 err_extra, regno); 4952 return -EACCES; 4953 } 4954 min_off = reg->smin_value + off; 4955 if (access_size > 0) 4956 max_off = reg->smax_value + off + access_size - 1; 4957 else 4958 max_off = min_off; 4959 } 4960 4961 err = check_stack_slot_within_bounds(min_off, state, type); 4962 if (!err) 4963 err = check_stack_slot_within_bounds(max_off, state, type); 4964 4965 if (err) { 4966 if (tnum_is_const(reg->var_off)) { 4967 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 4968 err_extra, regno, off, access_size); 4969 } else { 4970 char tn_buf[48]; 4971 4972 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4973 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 4974 err_extra, regno, tn_buf, access_size); 4975 } 4976 } 4977 return err; 4978 } 4979 4980 /* check whether memory at (regno + off) is accessible for t = (read | write) 4981 * if t==write, value_regno is a register which value is stored into memory 4982 * if t==read, value_regno is a register which will receive the value from memory 4983 * if t==write && value_regno==-1, some unknown value is stored into memory 4984 * if t==read && value_regno==-1, don't care what we read from memory 4985 */ 4986 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 4987 int off, int bpf_size, enum bpf_access_type t, 4988 int value_regno, bool strict_alignment_once) 4989 { 4990 struct bpf_reg_state *regs = cur_regs(env); 4991 struct bpf_reg_state *reg = regs + regno; 4992 struct bpf_func_state *state; 4993 int size, err = 0; 4994 4995 size = bpf_size_to_bytes(bpf_size); 4996 if (size < 0) 4997 return size; 4998 4999 /* alignment checks will add in reg->off themselves */ 5000 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 5001 if (err) 5002 return err; 5003 5004 /* for access checks, reg->off is just part of off */ 5005 off += reg->off; 5006 5007 if (reg->type == PTR_TO_MAP_KEY) { 5008 if (t == BPF_WRITE) { 5009 verbose(env, "write to change key R%d not allowed\n", regno); 5010 return -EACCES; 5011 } 5012 5013 err = check_mem_region_access(env, regno, off, size, 5014 reg->map_ptr->key_size, false); 5015 if (err) 5016 return err; 5017 if (value_regno >= 0) 5018 mark_reg_unknown(env, regs, value_regno); 5019 } else if (reg->type == PTR_TO_MAP_VALUE) { 5020 struct btf_field *kptr_field = NULL; 5021 5022 if (t == BPF_WRITE && value_regno >= 0 && 5023 is_pointer_value(env, value_regno)) { 5024 verbose(env, "R%d leaks addr into map\n", value_regno); 5025 return -EACCES; 5026 } 5027 err = check_map_access_type(env, regno, off, size, t); 5028 if (err) 5029 return err; 5030 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 5031 if (err) 5032 return err; 5033 if (tnum_is_const(reg->var_off)) 5034 kptr_field = btf_record_find(reg->map_ptr->record, 5035 off + reg->var_off.value, BPF_KPTR); 5036 if (kptr_field) { 5037 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 5038 } else if (t == BPF_READ && value_regno >= 0) { 5039 struct bpf_map *map = reg->map_ptr; 5040 5041 /* if map is read-only, track its contents as scalars */ 5042 if (tnum_is_const(reg->var_off) && 5043 bpf_map_is_rdonly(map) && 5044 map->ops->map_direct_value_addr) { 5045 int map_off = off + reg->var_off.value; 5046 u64 val = 0; 5047 5048 err = bpf_map_direct_read(map, map_off, size, 5049 &val); 5050 if (err) 5051 return err; 5052 5053 regs[value_regno].type = SCALAR_VALUE; 5054 __mark_reg_known(®s[value_regno], val); 5055 } else { 5056 mark_reg_unknown(env, regs, value_regno); 5057 } 5058 } 5059 } else if (base_type(reg->type) == PTR_TO_MEM) { 5060 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5061 5062 if (type_may_be_null(reg->type)) { 5063 verbose(env, "R%d invalid mem access '%s'\n", regno, 5064 reg_type_str(env, reg->type)); 5065 return -EACCES; 5066 } 5067 5068 if (t == BPF_WRITE && rdonly_mem) { 5069 verbose(env, "R%d cannot write into %s\n", 5070 regno, reg_type_str(env, reg->type)); 5071 return -EACCES; 5072 } 5073 5074 if (t == BPF_WRITE && value_regno >= 0 && 5075 is_pointer_value(env, value_regno)) { 5076 verbose(env, "R%d leaks addr into mem\n", value_regno); 5077 return -EACCES; 5078 } 5079 5080 err = check_mem_region_access(env, regno, off, size, 5081 reg->mem_size, false); 5082 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 5083 mark_reg_unknown(env, regs, value_regno); 5084 } else if (reg->type == PTR_TO_CTX) { 5085 enum bpf_reg_type reg_type = SCALAR_VALUE; 5086 struct btf *btf = NULL; 5087 u32 btf_id = 0; 5088 5089 if (t == BPF_WRITE && value_regno >= 0 && 5090 is_pointer_value(env, value_regno)) { 5091 verbose(env, "R%d leaks addr into ctx\n", value_regno); 5092 return -EACCES; 5093 } 5094 5095 err = check_ptr_off_reg(env, reg, regno); 5096 if (err < 0) 5097 return err; 5098 5099 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 5100 &btf_id); 5101 if (err) 5102 verbose_linfo(env, insn_idx, "; "); 5103 if (!err && t == BPF_READ && value_regno >= 0) { 5104 /* ctx access returns either a scalar, or a 5105 * PTR_TO_PACKET[_META,_END]. In the latter 5106 * case, we know the offset is zero. 5107 */ 5108 if (reg_type == SCALAR_VALUE) { 5109 mark_reg_unknown(env, regs, value_regno); 5110 } else { 5111 mark_reg_known_zero(env, regs, 5112 value_regno); 5113 if (type_may_be_null(reg_type)) 5114 regs[value_regno].id = ++env->id_gen; 5115 /* A load of ctx field could have different 5116 * actual load size with the one encoded in the 5117 * insn. When the dst is PTR, it is for sure not 5118 * a sub-register. 5119 */ 5120 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 5121 if (base_type(reg_type) == PTR_TO_BTF_ID) { 5122 regs[value_regno].btf = btf; 5123 regs[value_regno].btf_id = btf_id; 5124 } 5125 } 5126 regs[value_regno].type = reg_type; 5127 } 5128 5129 } else if (reg->type == PTR_TO_STACK) { 5130 /* Basic bounds checks. */ 5131 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 5132 if (err) 5133 return err; 5134 5135 state = func(env, reg); 5136 err = update_stack_depth(env, state, off); 5137 if (err) 5138 return err; 5139 5140 if (t == BPF_READ) 5141 err = check_stack_read(env, regno, off, size, 5142 value_regno); 5143 else 5144 err = check_stack_write(env, regno, off, size, 5145 value_regno, insn_idx); 5146 } else if (reg_is_pkt_pointer(reg)) { 5147 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 5148 verbose(env, "cannot write into packet\n"); 5149 return -EACCES; 5150 } 5151 if (t == BPF_WRITE && value_regno >= 0 && 5152 is_pointer_value(env, value_regno)) { 5153 verbose(env, "R%d leaks addr into packet\n", 5154 value_regno); 5155 return -EACCES; 5156 } 5157 err = check_packet_access(env, regno, off, size, false); 5158 if (!err && t == BPF_READ && value_regno >= 0) 5159 mark_reg_unknown(env, regs, value_regno); 5160 } else if (reg->type == PTR_TO_FLOW_KEYS) { 5161 if (t == BPF_WRITE && value_regno >= 0 && 5162 is_pointer_value(env, value_regno)) { 5163 verbose(env, "R%d leaks addr into flow keys\n", 5164 value_regno); 5165 return -EACCES; 5166 } 5167 5168 err = check_flow_keys_access(env, off, size); 5169 if (!err && t == BPF_READ && value_regno >= 0) 5170 mark_reg_unknown(env, regs, value_regno); 5171 } else if (type_is_sk_pointer(reg->type)) { 5172 if (t == BPF_WRITE) { 5173 verbose(env, "R%d cannot write into %s\n", 5174 regno, reg_type_str(env, reg->type)); 5175 return -EACCES; 5176 } 5177 err = check_sock_access(env, insn_idx, regno, off, size, t); 5178 if (!err && value_regno >= 0) 5179 mark_reg_unknown(env, regs, value_regno); 5180 } else if (reg->type == PTR_TO_TP_BUFFER) { 5181 err = check_tp_buffer_access(env, reg, regno, off, size); 5182 if (!err && t == BPF_READ && value_regno >= 0) 5183 mark_reg_unknown(env, regs, value_regno); 5184 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 5185 !type_may_be_null(reg->type)) { 5186 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 5187 value_regno); 5188 } else if (reg->type == CONST_PTR_TO_MAP) { 5189 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 5190 value_regno); 5191 } else if (base_type(reg->type) == PTR_TO_BUF) { 5192 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5193 u32 *max_access; 5194 5195 if (rdonly_mem) { 5196 if (t == BPF_WRITE) { 5197 verbose(env, "R%d cannot write into %s\n", 5198 regno, reg_type_str(env, reg->type)); 5199 return -EACCES; 5200 } 5201 max_access = &env->prog->aux->max_rdonly_access; 5202 } else { 5203 max_access = &env->prog->aux->max_rdwr_access; 5204 } 5205 5206 err = check_buffer_access(env, reg, regno, off, size, false, 5207 max_access); 5208 5209 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 5210 mark_reg_unknown(env, regs, value_regno); 5211 } else { 5212 verbose(env, "R%d invalid mem access '%s'\n", regno, 5213 reg_type_str(env, reg->type)); 5214 return -EACCES; 5215 } 5216 5217 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 5218 regs[value_regno].type == SCALAR_VALUE) { 5219 /* b/h/w load zero-extends, mark upper bits as known 0 */ 5220 coerce_reg_to_size(®s[value_regno], size); 5221 } 5222 return err; 5223 } 5224 5225 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 5226 { 5227 int load_reg; 5228 int err; 5229 5230 switch (insn->imm) { 5231 case BPF_ADD: 5232 case BPF_ADD | BPF_FETCH: 5233 case BPF_AND: 5234 case BPF_AND | BPF_FETCH: 5235 case BPF_OR: 5236 case BPF_OR | BPF_FETCH: 5237 case BPF_XOR: 5238 case BPF_XOR | BPF_FETCH: 5239 case BPF_XCHG: 5240 case BPF_CMPXCHG: 5241 break; 5242 default: 5243 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 5244 return -EINVAL; 5245 } 5246 5247 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 5248 verbose(env, "invalid atomic operand size\n"); 5249 return -EINVAL; 5250 } 5251 5252 /* check src1 operand */ 5253 err = check_reg_arg(env, insn->src_reg, SRC_OP); 5254 if (err) 5255 return err; 5256 5257 /* check src2 operand */ 5258 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 5259 if (err) 5260 return err; 5261 5262 if (insn->imm == BPF_CMPXCHG) { 5263 /* Check comparison of R0 with memory location */ 5264 const u32 aux_reg = BPF_REG_0; 5265 5266 err = check_reg_arg(env, aux_reg, SRC_OP); 5267 if (err) 5268 return err; 5269 5270 if (is_pointer_value(env, aux_reg)) { 5271 verbose(env, "R%d leaks addr into mem\n", aux_reg); 5272 return -EACCES; 5273 } 5274 } 5275 5276 if (is_pointer_value(env, insn->src_reg)) { 5277 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 5278 return -EACCES; 5279 } 5280 5281 if (is_ctx_reg(env, insn->dst_reg) || 5282 is_pkt_reg(env, insn->dst_reg) || 5283 is_flow_key_reg(env, insn->dst_reg) || 5284 is_sk_reg(env, insn->dst_reg)) { 5285 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 5286 insn->dst_reg, 5287 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 5288 return -EACCES; 5289 } 5290 5291 if (insn->imm & BPF_FETCH) { 5292 if (insn->imm == BPF_CMPXCHG) 5293 load_reg = BPF_REG_0; 5294 else 5295 load_reg = insn->src_reg; 5296 5297 /* check and record load of old value */ 5298 err = check_reg_arg(env, load_reg, DST_OP); 5299 if (err) 5300 return err; 5301 } else { 5302 /* This instruction accesses a memory location but doesn't 5303 * actually load it into a register. 5304 */ 5305 load_reg = -1; 5306 } 5307 5308 /* Check whether we can read the memory, with second call for fetch 5309 * case to simulate the register fill. 5310 */ 5311 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5312 BPF_SIZE(insn->code), BPF_READ, -1, true); 5313 if (!err && load_reg >= 0) 5314 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5315 BPF_SIZE(insn->code), BPF_READ, load_reg, 5316 true); 5317 if (err) 5318 return err; 5319 5320 /* Check whether we can write into the same memory. */ 5321 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5322 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 5323 if (err) 5324 return err; 5325 5326 return 0; 5327 } 5328 5329 /* When register 'regno' is used to read the stack (either directly or through 5330 * a helper function) make sure that it's within stack boundary and, depending 5331 * on the access type, that all elements of the stack are initialized. 5332 * 5333 * 'off' includes 'regno->off', but not its dynamic part (if any). 5334 * 5335 * All registers that have been spilled on the stack in the slots within the 5336 * read offsets are marked as read. 5337 */ 5338 static int check_stack_range_initialized( 5339 struct bpf_verifier_env *env, int regno, int off, 5340 int access_size, bool zero_size_allowed, 5341 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 5342 { 5343 struct bpf_reg_state *reg = reg_state(env, regno); 5344 struct bpf_func_state *state = func(env, reg); 5345 int err, min_off, max_off, i, j, slot, spi; 5346 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 5347 enum bpf_access_type bounds_check_type; 5348 /* Some accesses can write anything into the stack, others are 5349 * read-only. 5350 */ 5351 bool clobber = false; 5352 5353 if (access_size == 0 && !zero_size_allowed) { 5354 verbose(env, "invalid zero-sized read\n"); 5355 return -EACCES; 5356 } 5357 5358 if (type == ACCESS_HELPER) { 5359 /* The bounds checks for writes are more permissive than for 5360 * reads. However, if raw_mode is not set, we'll do extra 5361 * checks below. 5362 */ 5363 bounds_check_type = BPF_WRITE; 5364 clobber = true; 5365 } else { 5366 bounds_check_type = BPF_READ; 5367 } 5368 err = check_stack_access_within_bounds(env, regno, off, access_size, 5369 type, bounds_check_type); 5370 if (err) 5371 return err; 5372 5373 5374 if (tnum_is_const(reg->var_off)) { 5375 min_off = max_off = reg->var_off.value + off; 5376 } else { 5377 /* Variable offset is prohibited for unprivileged mode for 5378 * simplicity since it requires corresponding support in 5379 * Spectre masking for stack ALU. 5380 * See also retrieve_ptr_limit(). 5381 */ 5382 if (!env->bypass_spec_v1) { 5383 char tn_buf[48]; 5384 5385 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5386 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 5387 regno, err_extra, tn_buf); 5388 return -EACCES; 5389 } 5390 /* Only initialized buffer on stack is allowed to be accessed 5391 * with variable offset. With uninitialized buffer it's hard to 5392 * guarantee that whole memory is marked as initialized on 5393 * helper return since specific bounds are unknown what may 5394 * cause uninitialized stack leaking. 5395 */ 5396 if (meta && meta->raw_mode) 5397 meta = NULL; 5398 5399 min_off = reg->smin_value + off; 5400 max_off = reg->smax_value + off; 5401 } 5402 5403 if (meta && meta->raw_mode) { 5404 meta->access_size = access_size; 5405 meta->regno = regno; 5406 return 0; 5407 } 5408 5409 for (i = min_off; i < max_off + access_size; i++) { 5410 u8 *stype; 5411 5412 slot = -i - 1; 5413 spi = slot / BPF_REG_SIZE; 5414 if (state->allocated_stack <= slot) 5415 goto err; 5416 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 5417 if (*stype == STACK_MISC) 5418 goto mark; 5419 if (*stype == STACK_ZERO) { 5420 if (clobber) { 5421 /* helper can write anything into the stack */ 5422 *stype = STACK_MISC; 5423 } 5424 goto mark; 5425 } 5426 5427 if (is_spilled_reg(&state->stack[spi]) && 5428 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 5429 env->allow_ptr_leaks)) { 5430 if (clobber) { 5431 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 5432 for (j = 0; j < BPF_REG_SIZE; j++) 5433 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 5434 } 5435 goto mark; 5436 } 5437 5438 err: 5439 if (tnum_is_const(reg->var_off)) { 5440 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 5441 err_extra, regno, min_off, i - min_off, access_size); 5442 } else { 5443 char tn_buf[48]; 5444 5445 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5446 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 5447 err_extra, regno, tn_buf, i - min_off, access_size); 5448 } 5449 return -EACCES; 5450 mark: 5451 /* reading any byte out of 8-byte 'spill_slot' will cause 5452 * the whole slot to be marked as 'read' 5453 */ 5454 mark_reg_read(env, &state->stack[spi].spilled_ptr, 5455 state->stack[spi].spilled_ptr.parent, 5456 REG_LIVE_READ64); 5457 /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not 5458 * be sure that whether stack slot is written to or not. Hence, 5459 * we must still conservatively propagate reads upwards even if 5460 * helper may write to the entire memory range. 5461 */ 5462 } 5463 return update_stack_depth(env, state, min_off); 5464 } 5465 5466 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 5467 int access_size, bool zero_size_allowed, 5468 struct bpf_call_arg_meta *meta) 5469 { 5470 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5471 u32 *max_access; 5472 5473 switch (base_type(reg->type)) { 5474 case PTR_TO_PACKET: 5475 case PTR_TO_PACKET_META: 5476 return check_packet_access(env, regno, reg->off, access_size, 5477 zero_size_allowed); 5478 case PTR_TO_MAP_KEY: 5479 if (meta && meta->raw_mode) { 5480 verbose(env, "R%d cannot write into %s\n", regno, 5481 reg_type_str(env, reg->type)); 5482 return -EACCES; 5483 } 5484 return check_mem_region_access(env, regno, reg->off, access_size, 5485 reg->map_ptr->key_size, false); 5486 case PTR_TO_MAP_VALUE: 5487 if (check_map_access_type(env, regno, reg->off, access_size, 5488 meta && meta->raw_mode ? BPF_WRITE : 5489 BPF_READ)) 5490 return -EACCES; 5491 return check_map_access(env, regno, reg->off, access_size, 5492 zero_size_allowed, ACCESS_HELPER); 5493 case PTR_TO_MEM: 5494 if (type_is_rdonly_mem(reg->type)) { 5495 if (meta && meta->raw_mode) { 5496 verbose(env, "R%d cannot write into %s\n", regno, 5497 reg_type_str(env, reg->type)); 5498 return -EACCES; 5499 } 5500 } 5501 return check_mem_region_access(env, regno, reg->off, 5502 access_size, reg->mem_size, 5503 zero_size_allowed); 5504 case PTR_TO_BUF: 5505 if (type_is_rdonly_mem(reg->type)) { 5506 if (meta && meta->raw_mode) { 5507 verbose(env, "R%d cannot write into %s\n", regno, 5508 reg_type_str(env, reg->type)); 5509 return -EACCES; 5510 } 5511 5512 max_access = &env->prog->aux->max_rdonly_access; 5513 } else { 5514 max_access = &env->prog->aux->max_rdwr_access; 5515 } 5516 return check_buffer_access(env, reg, regno, reg->off, 5517 access_size, zero_size_allowed, 5518 max_access); 5519 case PTR_TO_STACK: 5520 return check_stack_range_initialized( 5521 env, 5522 regno, reg->off, access_size, 5523 zero_size_allowed, ACCESS_HELPER, meta); 5524 case PTR_TO_CTX: 5525 /* in case the function doesn't know how to access the context, 5526 * (because we are in a program of type SYSCALL for example), we 5527 * can not statically check its size. 5528 * Dynamically check it now. 5529 */ 5530 if (!env->ops->convert_ctx_access) { 5531 enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; 5532 int offset = access_size - 1; 5533 5534 /* Allow zero-byte read from PTR_TO_CTX */ 5535 if (access_size == 0) 5536 return zero_size_allowed ? 0 : -EACCES; 5537 5538 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, 5539 atype, -1, false); 5540 } 5541 5542 fallthrough; 5543 default: /* scalar_value or invalid ptr */ 5544 /* Allow zero-byte read from NULL, regardless of pointer type */ 5545 if (zero_size_allowed && access_size == 0 && 5546 register_is_null(reg)) 5547 return 0; 5548 5549 verbose(env, "R%d type=%s ", regno, 5550 reg_type_str(env, reg->type)); 5551 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 5552 return -EACCES; 5553 } 5554 } 5555 5556 static int check_mem_size_reg(struct bpf_verifier_env *env, 5557 struct bpf_reg_state *reg, u32 regno, 5558 bool zero_size_allowed, 5559 struct bpf_call_arg_meta *meta) 5560 { 5561 int err; 5562 5563 /* This is used to refine r0 return value bounds for helpers 5564 * that enforce this value as an upper bound on return values. 5565 * See do_refine_retval_range() for helpers that can refine 5566 * the return value. C type of helper is u32 so we pull register 5567 * bound from umax_value however, if negative verifier errors 5568 * out. Only upper bounds can be learned because retval is an 5569 * int type and negative retvals are allowed. 5570 */ 5571 meta->msize_max_value = reg->umax_value; 5572 5573 /* The register is SCALAR_VALUE; the access check 5574 * happens using its boundaries. 5575 */ 5576 if (!tnum_is_const(reg->var_off)) 5577 /* For unprivileged variable accesses, disable raw 5578 * mode so that the program is required to 5579 * initialize all the memory that the helper could 5580 * just partially fill up. 5581 */ 5582 meta = NULL; 5583 5584 if (reg->smin_value < 0) { 5585 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5586 regno); 5587 return -EACCES; 5588 } 5589 5590 if (reg->umin_value == 0) { 5591 err = check_helper_mem_access(env, regno - 1, 0, 5592 zero_size_allowed, 5593 meta); 5594 if (err) 5595 return err; 5596 } 5597 5598 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5599 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5600 regno); 5601 return -EACCES; 5602 } 5603 err = check_helper_mem_access(env, regno - 1, 5604 reg->umax_value, 5605 zero_size_allowed, meta); 5606 if (!err) 5607 err = mark_chain_precision(env, regno); 5608 return err; 5609 } 5610 5611 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5612 u32 regno, u32 mem_size) 5613 { 5614 bool may_be_null = type_may_be_null(reg->type); 5615 struct bpf_reg_state saved_reg; 5616 struct bpf_call_arg_meta meta; 5617 int err; 5618 5619 if (register_is_null(reg)) 5620 return 0; 5621 5622 memset(&meta, 0, sizeof(meta)); 5623 /* Assuming that the register contains a value check if the memory 5624 * access is safe. Temporarily save and restore the register's state as 5625 * the conversion shouldn't be visible to a caller. 5626 */ 5627 if (may_be_null) { 5628 saved_reg = *reg; 5629 mark_ptr_not_null_reg(reg); 5630 } 5631 5632 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 5633 /* Check access for BPF_WRITE */ 5634 meta.raw_mode = true; 5635 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 5636 5637 if (may_be_null) 5638 *reg = saved_reg; 5639 5640 return err; 5641 } 5642 5643 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5644 u32 regno) 5645 { 5646 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 5647 bool may_be_null = type_may_be_null(mem_reg->type); 5648 struct bpf_reg_state saved_reg; 5649 struct bpf_call_arg_meta meta; 5650 int err; 5651 5652 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 5653 5654 memset(&meta, 0, sizeof(meta)); 5655 5656 if (may_be_null) { 5657 saved_reg = *mem_reg; 5658 mark_ptr_not_null_reg(mem_reg); 5659 } 5660 5661 err = check_mem_size_reg(env, reg, regno, true, &meta); 5662 /* Check access for BPF_WRITE */ 5663 meta.raw_mode = true; 5664 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 5665 5666 if (may_be_null) 5667 *mem_reg = saved_reg; 5668 return err; 5669 } 5670 5671 /* Implementation details: 5672 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 5673 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 5674 * Two bpf_map_lookups (even with the same key) will have different reg->id. 5675 * Two separate bpf_obj_new will also have different reg->id. 5676 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 5677 * clears reg->id after value_or_null->value transition, since the verifier only 5678 * cares about the range of access to valid map value pointer and doesn't care 5679 * about actual address of the map element. 5680 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 5681 * reg->id > 0 after value_or_null->value transition. By doing so 5682 * two bpf_map_lookups will be considered two different pointers that 5683 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 5684 * returned from bpf_obj_new. 5685 * The verifier allows taking only one bpf_spin_lock at a time to avoid 5686 * dead-locks. 5687 * Since only one bpf_spin_lock is allowed the checks are simpler than 5688 * reg_is_refcounted() logic. The verifier needs to remember only 5689 * one spin_lock instead of array of acquired_refs. 5690 * cur_state->active_lock remembers which map value element or allocated 5691 * object got locked and clears it after bpf_spin_unlock. 5692 */ 5693 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 5694 bool is_lock) 5695 { 5696 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5697 struct bpf_verifier_state *cur = env->cur_state; 5698 bool is_const = tnum_is_const(reg->var_off); 5699 u64 val = reg->var_off.value; 5700 struct bpf_map *map = NULL; 5701 struct btf *btf = NULL; 5702 struct btf_record *rec; 5703 5704 if (!is_const) { 5705 verbose(env, 5706 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 5707 regno); 5708 return -EINVAL; 5709 } 5710 if (reg->type == PTR_TO_MAP_VALUE) { 5711 map = reg->map_ptr; 5712 if (!map->btf) { 5713 verbose(env, 5714 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 5715 map->name); 5716 return -EINVAL; 5717 } 5718 } else { 5719 btf = reg->btf; 5720 } 5721 5722 rec = reg_btf_record(reg); 5723 if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) { 5724 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", 5725 map ? map->name : "kptr"); 5726 return -EINVAL; 5727 } 5728 if (rec->spin_lock_off != val + reg->off) { 5729 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", 5730 val + reg->off, rec->spin_lock_off); 5731 return -EINVAL; 5732 } 5733 if (is_lock) { 5734 if (cur->active_lock.ptr) { 5735 verbose(env, 5736 "Locking two bpf_spin_locks are not allowed\n"); 5737 return -EINVAL; 5738 } 5739 if (map) 5740 cur->active_lock.ptr = map; 5741 else 5742 cur->active_lock.ptr = btf; 5743 cur->active_lock.id = reg->id; 5744 } else { 5745 struct bpf_func_state *fstate = cur_func(env); 5746 void *ptr; 5747 int i; 5748 5749 if (map) 5750 ptr = map; 5751 else 5752 ptr = btf; 5753 5754 if (!cur->active_lock.ptr) { 5755 verbose(env, "bpf_spin_unlock without taking a lock\n"); 5756 return -EINVAL; 5757 } 5758 if (cur->active_lock.ptr != ptr || 5759 cur->active_lock.id != reg->id) { 5760 verbose(env, "bpf_spin_unlock of different lock\n"); 5761 return -EINVAL; 5762 } 5763 cur->active_lock.ptr = NULL; 5764 cur->active_lock.id = 0; 5765 5766 for (i = fstate->acquired_refs - 1; i >= 0; i--) { 5767 int err; 5768 5769 /* Complain on error because this reference state cannot 5770 * be freed before this point, as bpf_spin_lock critical 5771 * section does not allow functions that release the 5772 * allocated object immediately. 5773 */ 5774 if (!fstate->refs[i].release_on_unlock) 5775 continue; 5776 err = release_reference(env, fstate->refs[i].id); 5777 if (err) { 5778 verbose(env, "failed to release release_on_unlock reference"); 5779 return err; 5780 } 5781 } 5782 } 5783 return 0; 5784 } 5785 5786 static int process_timer_func(struct bpf_verifier_env *env, int regno, 5787 struct bpf_call_arg_meta *meta) 5788 { 5789 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5790 bool is_const = tnum_is_const(reg->var_off); 5791 struct bpf_map *map = reg->map_ptr; 5792 u64 val = reg->var_off.value; 5793 5794 if (!is_const) { 5795 verbose(env, 5796 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 5797 regno); 5798 return -EINVAL; 5799 } 5800 if (!map->btf) { 5801 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 5802 map->name); 5803 return -EINVAL; 5804 } 5805 if (!btf_record_has_field(map->record, BPF_TIMER)) { 5806 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); 5807 return -EINVAL; 5808 } 5809 if (map->record->timer_off != val + reg->off) { 5810 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 5811 val + reg->off, map->record->timer_off); 5812 return -EINVAL; 5813 } 5814 if (meta->map_ptr) { 5815 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 5816 return -EFAULT; 5817 } 5818 meta->map_uid = reg->map_uid; 5819 meta->map_ptr = map; 5820 return 0; 5821 } 5822 5823 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 5824 struct bpf_call_arg_meta *meta) 5825 { 5826 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5827 struct bpf_map *map_ptr = reg->map_ptr; 5828 struct btf_field *kptr_field; 5829 u32 kptr_off; 5830 5831 if (!tnum_is_const(reg->var_off)) { 5832 verbose(env, 5833 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 5834 regno); 5835 return -EINVAL; 5836 } 5837 if (!map_ptr->btf) { 5838 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 5839 map_ptr->name); 5840 return -EINVAL; 5841 } 5842 if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 5843 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 5844 return -EINVAL; 5845 } 5846 5847 meta->map_ptr = map_ptr; 5848 kptr_off = reg->off + reg->var_off.value; 5849 kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 5850 if (!kptr_field) { 5851 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 5852 return -EACCES; 5853 } 5854 if (kptr_field->type != BPF_KPTR_REF) { 5855 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 5856 return -EACCES; 5857 } 5858 meta->kptr_field = kptr_field; 5859 return 0; 5860 } 5861 5862 static bool arg_type_is_mem_size(enum bpf_arg_type type) 5863 { 5864 return type == ARG_CONST_SIZE || 5865 type == ARG_CONST_SIZE_OR_ZERO; 5866 } 5867 5868 static bool arg_type_is_release(enum bpf_arg_type type) 5869 { 5870 return type & OBJ_RELEASE; 5871 } 5872 5873 static bool arg_type_is_dynptr(enum bpf_arg_type type) 5874 { 5875 return base_type(type) == ARG_PTR_TO_DYNPTR; 5876 } 5877 5878 static int int_ptr_type_to_size(enum bpf_arg_type type) 5879 { 5880 if (type == ARG_PTR_TO_INT) 5881 return sizeof(u32); 5882 else if (type == ARG_PTR_TO_LONG) 5883 return sizeof(u64); 5884 5885 return -EINVAL; 5886 } 5887 5888 static int resolve_map_arg_type(struct bpf_verifier_env *env, 5889 const struct bpf_call_arg_meta *meta, 5890 enum bpf_arg_type *arg_type) 5891 { 5892 if (!meta->map_ptr) { 5893 /* kernel subsystem misconfigured verifier */ 5894 verbose(env, "invalid map_ptr to access map->type\n"); 5895 return -EACCES; 5896 } 5897 5898 switch (meta->map_ptr->map_type) { 5899 case BPF_MAP_TYPE_SOCKMAP: 5900 case BPF_MAP_TYPE_SOCKHASH: 5901 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 5902 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 5903 } else { 5904 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 5905 return -EINVAL; 5906 } 5907 break; 5908 case BPF_MAP_TYPE_BLOOM_FILTER: 5909 if (meta->func_id == BPF_FUNC_map_peek_elem) 5910 *arg_type = ARG_PTR_TO_MAP_VALUE; 5911 break; 5912 default: 5913 break; 5914 } 5915 return 0; 5916 } 5917 5918 struct bpf_reg_types { 5919 const enum bpf_reg_type types[10]; 5920 u32 *btf_id; 5921 }; 5922 5923 static const struct bpf_reg_types sock_types = { 5924 .types = { 5925 PTR_TO_SOCK_COMMON, 5926 PTR_TO_SOCKET, 5927 PTR_TO_TCP_SOCK, 5928 PTR_TO_XDP_SOCK, 5929 }, 5930 }; 5931 5932 #ifdef CONFIG_NET 5933 static const struct bpf_reg_types btf_id_sock_common_types = { 5934 .types = { 5935 PTR_TO_SOCK_COMMON, 5936 PTR_TO_SOCKET, 5937 PTR_TO_TCP_SOCK, 5938 PTR_TO_XDP_SOCK, 5939 PTR_TO_BTF_ID, 5940 PTR_TO_BTF_ID | PTR_TRUSTED, 5941 }, 5942 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 5943 }; 5944 #endif 5945 5946 static const struct bpf_reg_types mem_types = { 5947 .types = { 5948 PTR_TO_STACK, 5949 PTR_TO_PACKET, 5950 PTR_TO_PACKET_META, 5951 PTR_TO_MAP_KEY, 5952 PTR_TO_MAP_VALUE, 5953 PTR_TO_MEM, 5954 PTR_TO_MEM | MEM_RINGBUF, 5955 PTR_TO_BUF, 5956 }, 5957 }; 5958 5959 static const struct bpf_reg_types int_ptr_types = { 5960 .types = { 5961 PTR_TO_STACK, 5962 PTR_TO_PACKET, 5963 PTR_TO_PACKET_META, 5964 PTR_TO_MAP_KEY, 5965 PTR_TO_MAP_VALUE, 5966 }, 5967 }; 5968 5969 static const struct bpf_reg_types spin_lock_types = { 5970 .types = { 5971 PTR_TO_MAP_VALUE, 5972 PTR_TO_BTF_ID | MEM_ALLOC, 5973 } 5974 }; 5975 5976 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 5977 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 5978 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 5979 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 5980 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 5981 static const struct bpf_reg_types btf_ptr_types = { 5982 .types = { 5983 PTR_TO_BTF_ID, 5984 PTR_TO_BTF_ID | PTR_TRUSTED, 5985 PTR_TO_BTF_ID | MEM_RCU, 5986 }, 5987 }; 5988 static const struct bpf_reg_types percpu_btf_ptr_types = { 5989 .types = { 5990 PTR_TO_BTF_ID | MEM_PERCPU, 5991 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 5992 } 5993 }; 5994 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 5995 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 5996 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5997 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 5998 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5999 static const struct bpf_reg_types dynptr_types = { 6000 .types = { 6001 PTR_TO_STACK, 6002 PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL, 6003 } 6004 }; 6005 6006 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 6007 [ARG_PTR_TO_MAP_KEY] = &mem_types, 6008 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 6009 [ARG_CONST_SIZE] = &scalar_types, 6010 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 6011 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 6012 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 6013 [ARG_PTR_TO_CTX] = &context_types, 6014 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 6015 #ifdef CONFIG_NET 6016 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 6017 #endif 6018 [ARG_PTR_TO_SOCKET] = &fullsock_types, 6019 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 6020 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 6021 [ARG_PTR_TO_MEM] = &mem_types, 6022 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 6023 [ARG_PTR_TO_INT] = &int_ptr_types, 6024 [ARG_PTR_TO_LONG] = &int_ptr_types, 6025 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 6026 [ARG_PTR_TO_FUNC] = &func_ptr_types, 6027 [ARG_PTR_TO_STACK] = &stack_ptr_types, 6028 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 6029 [ARG_PTR_TO_TIMER] = &timer_types, 6030 [ARG_PTR_TO_KPTR] = &kptr_types, 6031 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 6032 }; 6033 6034 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 6035 enum bpf_arg_type arg_type, 6036 const u32 *arg_btf_id, 6037 struct bpf_call_arg_meta *meta) 6038 { 6039 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6040 enum bpf_reg_type expected, type = reg->type; 6041 const struct bpf_reg_types *compatible; 6042 int i, j; 6043 6044 compatible = compatible_reg_types[base_type(arg_type)]; 6045 if (!compatible) { 6046 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 6047 return -EFAULT; 6048 } 6049 6050 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 6051 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 6052 * 6053 * Same for MAYBE_NULL: 6054 * 6055 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 6056 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 6057 * 6058 * Therefore we fold these flags depending on the arg_type before comparison. 6059 */ 6060 if (arg_type & MEM_RDONLY) 6061 type &= ~MEM_RDONLY; 6062 if (arg_type & PTR_MAYBE_NULL) 6063 type &= ~PTR_MAYBE_NULL; 6064 6065 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 6066 expected = compatible->types[i]; 6067 if (expected == NOT_INIT) 6068 break; 6069 6070 if (type == expected) 6071 goto found; 6072 } 6073 6074 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 6075 for (j = 0; j + 1 < i; j++) 6076 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 6077 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 6078 return -EACCES; 6079 6080 found: 6081 if (reg->type == PTR_TO_BTF_ID || reg->type & PTR_TRUSTED) { 6082 /* For bpf_sk_release, it needs to match against first member 6083 * 'struct sock_common', hence make an exception for it. This 6084 * allows bpf_sk_release to work for multiple socket types. 6085 */ 6086 bool strict_type_match = arg_type_is_release(arg_type) && 6087 meta->func_id != BPF_FUNC_sk_release; 6088 6089 if (!arg_btf_id) { 6090 if (!compatible->btf_id) { 6091 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 6092 return -EFAULT; 6093 } 6094 arg_btf_id = compatible->btf_id; 6095 } 6096 6097 if (meta->func_id == BPF_FUNC_kptr_xchg) { 6098 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 6099 return -EACCES; 6100 } else { 6101 if (arg_btf_id == BPF_PTR_POISON) { 6102 verbose(env, "verifier internal error:"); 6103 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", 6104 regno); 6105 return -EACCES; 6106 } 6107 6108 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 6109 btf_vmlinux, *arg_btf_id, 6110 strict_type_match)) { 6111 verbose(env, "R%d is of type %s but %s is expected\n", 6112 regno, kernel_type_name(reg->btf, reg->btf_id), 6113 kernel_type_name(btf_vmlinux, *arg_btf_id)); 6114 return -EACCES; 6115 } 6116 } 6117 } else if (type_is_alloc(reg->type)) { 6118 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock) { 6119 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); 6120 return -EFAULT; 6121 } 6122 } 6123 6124 return 0; 6125 } 6126 6127 int check_func_arg_reg_off(struct bpf_verifier_env *env, 6128 const struct bpf_reg_state *reg, int regno, 6129 enum bpf_arg_type arg_type) 6130 { 6131 enum bpf_reg_type type = reg->type; 6132 bool fixed_off_ok = false; 6133 6134 switch ((u32)type) { 6135 /* Pointer types where reg offset is explicitly allowed: */ 6136 case PTR_TO_STACK: 6137 if (arg_type_is_dynptr(arg_type) && reg->off % BPF_REG_SIZE) { 6138 verbose(env, "cannot pass in dynptr at an offset\n"); 6139 return -EINVAL; 6140 } 6141 fallthrough; 6142 case PTR_TO_PACKET: 6143 case PTR_TO_PACKET_META: 6144 case PTR_TO_MAP_KEY: 6145 case PTR_TO_MAP_VALUE: 6146 case PTR_TO_MEM: 6147 case PTR_TO_MEM | MEM_RDONLY: 6148 case PTR_TO_MEM | MEM_RINGBUF: 6149 case PTR_TO_BUF: 6150 case PTR_TO_BUF | MEM_RDONLY: 6151 case SCALAR_VALUE: 6152 /* Some of the argument types nevertheless require a 6153 * zero register offset. 6154 */ 6155 if (base_type(arg_type) != ARG_PTR_TO_RINGBUF_MEM) 6156 return 0; 6157 break; 6158 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 6159 * fixed offset. 6160 */ 6161 case PTR_TO_BTF_ID: 6162 case PTR_TO_BTF_ID | MEM_ALLOC: 6163 case PTR_TO_BTF_ID | PTR_TRUSTED: 6164 case PTR_TO_BTF_ID | MEM_RCU: 6165 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_TRUSTED: 6166 /* When referenced PTR_TO_BTF_ID is passed to release function, 6167 * it's fixed offset must be 0. In the other cases, fixed offset 6168 * can be non-zero. 6169 */ 6170 if (arg_type_is_release(arg_type) && reg->off) { 6171 verbose(env, "R%d must have zero offset when passed to release func\n", 6172 regno); 6173 return -EINVAL; 6174 } 6175 /* For arg is release pointer, fixed_off_ok must be false, but 6176 * we already checked and rejected reg->off != 0 above, so set 6177 * to true to allow fixed offset for all other cases. 6178 */ 6179 fixed_off_ok = true; 6180 break; 6181 default: 6182 break; 6183 } 6184 return __check_ptr_off_reg(env, reg, regno, fixed_off_ok); 6185 } 6186 6187 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 6188 { 6189 struct bpf_func_state *state = func(env, reg); 6190 int spi = get_spi(reg->off); 6191 6192 return state->stack[spi].spilled_ptr.id; 6193 } 6194 6195 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 6196 struct bpf_call_arg_meta *meta, 6197 const struct bpf_func_proto *fn) 6198 { 6199 u32 regno = BPF_REG_1 + arg; 6200 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6201 enum bpf_arg_type arg_type = fn->arg_type[arg]; 6202 enum bpf_reg_type type = reg->type; 6203 u32 *arg_btf_id = NULL; 6204 int err = 0; 6205 6206 if (arg_type == ARG_DONTCARE) 6207 return 0; 6208 6209 err = check_reg_arg(env, regno, SRC_OP); 6210 if (err) 6211 return err; 6212 6213 if (arg_type == ARG_ANYTHING) { 6214 if (is_pointer_value(env, regno)) { 6215 verbose(env, "R%d leaks addr into helper function\n", 6216 regno); 6217 return -EACCES; 6218 } 6219 return 0; 6220 } 6221 6222 if (type_is_pkt_pointer(type) && 6223 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 6224 verbose(env, "helper access to the packet is not allowed\n"); 6225 return -EACCES; 6226 } 6227 6228 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { 6229 err = resolve_map_arg_type(env, meta, &arg_type); 6230 if (err) 6231 return err; 6232 } 6233 6234 if (register_is_null(reg) && type_may_be_null(arg_type)) 6235 /* A NULL register has a SCALAR_VALUE type, so skip 6236 * type checking. 6237 */ 6238 goto skip_type_check; 6239 6240 /* arg_btf_id and arg_size are in a union. */ 6241 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || 6242 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 6243 arg_btf_id = fn->arg_btf_id[arg]; 6244 6245 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 6246 if (err) 6247 return err; 6248 6249 err = check_func_arg_reg_off(env, reg, regno, arg_type); 6250 if (err) 6251 return err; 6252 6253 skip_type_check: 6254 if (arg_type_is_release(arg_type)) { 6255 if (arg_type_is_dynptr(arg_type)) { 6256 struct bpf_func_state *state = func(env, reg); 6257 int spi = get_spi(reg->off); 6258 6259 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) || 6260 !state->stack[spi].spilled_ptr.id) { 6261 verbose(env, "arg %d is an unacquired reference\n", regno); 6262 return -EINVAL; 6263 } 6264 } else if (!reg->ref_obj_id && !register_is_null(reg)) { 6265 verbose(env, "R%d must be referenced when passed to release function\n", 6266 regno); 6267 return -EINVAL; 6268 } 6269 if (meta->release_regno) { 6270 verbose(env, "verifier internal error: more than one release argument\n"); 6271 return -EFAULT; 6272 } 6273 meta->release_regno = regno; 6274 } 6275 6276 if (reg->ref_obj_id) { 6277 if (meta->ref_obj_id) { 6278 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 6279 regno, reg->ref_obj_id, 6280 meta->ref_obj_id); 6281 return -EFAULT; 6282 } 6283 meta->ref_obj_id = reg->ref_obj_id; 6284 } 6285 6286 switch (base_type(arg_type)) { 6287 case ARG_CONST_MAP_PTR: 6288 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 6289 if (meta->map_ptr) { 6290 /* Use map_uid (which is unique id of inner map) to reject: 6291 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 6292 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 6293 * if (inner_map1 && inner_map2) { 6294 * timer = bpf_map_lookup_elem(inner_map1); 6295 * if (timer) 6296 * // mismatch would have been allowed 6297 * bpf_timer_init(timer, inner_map2); 6298 * } 6299 * 6300 * Comparing map_ptr is enough to distinguish normal and outer maps. 6301 */ 6302 if (meta->map_ptr != reg->map_ptr || 6303 meta->map_uid != reg->map_uid) { 6304 verbose(env, 6305 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 6306 meta->map_uid, reg->map_uid); 6307 return -EINVAL; 6308 } 6309 } 6310 meta->map_ptr = reg->map_ptr; 6311 meta->map_uid = reg->map_uid; 6312 break; 6313 case ARG_PTR_TO_MAP_KEY: 6314 /* bpf_map_xxx(..., map_ptr, ..., key) call: 6315 * check that [key, key + map->key_size) are within 6316 * stack limits and initialized 6317 */ 6318 if (!meta->map_ptr) { 6319 /* in function declaration map_ptr must come before 6320 * map_key, so that it's verified and known before 6321 * we have to check map_key here. Otherwise it means 6322 * that kernel subsystem misconfigured verifier 6323 */ 6324 verbose(env, "invalid map_ptr to access map->key\n"); 6325 return -EACCES; 6326 } 6327 err = check_helper_mem_access(env, regno, 6328 meta->map_ptr->key_size, false, 6329 NULL); 6330 break; 6331 case ARG_PTR_TO_MAP_VALUE: 6332 if (type_may_be_null(arg_type) && register_is_null(reg)) 6333 return 0; 6334 6335 /* bpf_map_xxx(..., map_ptr, ..., value) call: 6336 * check [value, value + map->value_size) validity 6337 */ 6338 if (!meta->map_ptr) { 6339 /* kernel subsystem misconfigured verifier */ 6340 verbose(env, "invalid map_ptr to access map->value\n"); 6341 return -EACCES; 6342 } 6343 meta->raw_mode = arg_type & MEM_UNINIT; 6344 err = check_helper_mem_access(env, regno, 6345 meta->map_ptr->value_size, false, 6346 meta); 6347 break; 6348 case ARG_PTR_TO_PERCPU_BTF_ID: 6349 if (!reg->btf_id) { 6350 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 6351 return -EACCES; 6352 } 6353 meta->ret_btf = reg->btf; 6354 meta->ret_btf_id = reg->btf_id; 6355 break; 6356 case ARG_PTR_TO_SPIN_LOCK: 6357 if (meta->func_id == BPF_FUNC_spin_lock) { 6358 if (process_spin_lock(env, regno, true)) 6359 return -EACCES; 6360 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 6361 if (process_spin_lock(env, regno, false)) 6362 return -EACCES; 6363 } else { 6364 verbose(env, "verifier internal error\n"); 6365 return -EFAULT; 6366 } 6367 break; 6368 case ARG_PTR_TO_TIMER: 6369 if (process_timer_func(env, regno, meta)) 6370 return -EACCES; 6371 break; 6372 case ARG_PTR_TO_FUNC: 6373 meta->subprogno = reg->subprogno; 6374 break; 6375 case ARG_PTR_TO_MEM: 6376 /* The access to this pointer is only checked when we hit the 6377 * next is_mem_size argument below. 6378 */ 6379 meta->raw_mode = arg_type & MEM_UNINIT; 6380 if (arg_type & MEM_FIXED_SIZE) { 6381 err = check_helper_mem_access(env, regno, 6382 fn->arg_size[arg], false, 6383 meta); 6384 } 6385 break; 6386 case ARG_CONST_SIZE: 6387 err = check_mem_size_reg(env, reg, regno, false, meta); 6388 break; 6389 case ARG_CONST_SIZE_OR_ZERO: 6390 err = check_mem_size_reg(env, reg, regno, true, meta); 6391 break; 6392 case ARG_PTR_TO_DYNPTR: 6393 /* We only need to check for initialized / uninitialized helper 6394 * dynptr args if the dynptr is not PTR_TO_DYNPTR, as the 6395 * assumption is that if it is, that a helper function 6396 * initialized the dynptr on behalf of the BPF program. 6397 */ 6398 if (base_type(reg->type) == PTR_TO_DYNPTR) 6399 break; 6400 if (arg_type & MEM_UNINIT) { 6401 if (!is_dynptr_reg_valid_uninit(env, reg)) { 6402 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 6403 return -EINVAL; 6404 } 6405 6406 /* We only support one dynptr being uninitialized at the moment, 6407 * which is sufficient for the helper functions we have right now. 6408 */ 6409 if (meta->uninit_dynptr_regno) { 6410 verbose(env, "verifier internal error: multiple uninitialized dynptr args\n"); 6411 return -EFAULT; 6412 } 6413 6414 meta->uninit_dynptr_regno = regno; 6415 } else if (!is_dynptr_reg_valid_init(env, reg)) { 6416 verbose(env, 6417 "Expected an initialized dynptr as arg #%d\n", 6418 arg + 1); 6419 return -EINVAL; 6420 } else if (!is_dynptr_type_expected(env, reg, arg_type)) { 6421 const char *err_extra = ""; 6422 6423 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 6424 case DYNPTR_TYPE_LOCAL: 6425 err_extra = "local"; 6426 break; 6427 case DYNPTR_TYPE_RINGBUF: 6428 err_extra = "ringbuf"; 6429 break; 6430 default: 6431 err_extra = "<unknown>"; 6432 break; 6433 } 6434 verbose(env, 6435 "Expected a dynptr of type %s as arg #%d\n", 6436 err_extra, arg + 1); 6437 return -EINVAL; 6438 } 6439 break; 6440 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 6441 if (!tnum_is_const(reg->var_off)) { 6442 verbose(env, "R%d is not a known constant'\n", 6443 regno); 6444 return -EACCES; 6445 } 6446 meta->mem_size = reg->var_off.value; 6447 err = mark_chain_precision(env, regno); 6448 if (err) 6449 return err; 6450 break; 6451 case ARG_PTR_TO_INT: 6452 case ARG_PTR_TO_LONG: 6453 { 6454 int size = int_ptr_type_to_size(arg_type); 6455 6456 err = check_helper_mem_access(env, regno, size, false, meta); 6457 if (err) 6458 return err; 6459 err = check_ptr_alignment(env, reg, 0, size, true); 6460 break; 6461 } 6462 case ARG_PTR_TO_CONST_STR: 6463 { 6464 struct bpf_map *map = reg->map_ptr; 6465 int map_off; 6466 u64 map_addr; 6467 char *str_ptr; 6468 6469 if (!bpf_map_is_rdonly(map)) { 6470 verbose(env, "R%d does not point to a readonly map'\n", regno); 6471 return -EACCES; 6472 } 6473 6474 if (!tnum_is_const(reg->var_off)) { 6475 verbose(env, "R%d is not a constant address'\n", regno); 6476 return -EACCES; 6477 } 6478 6479 if (!map->ops->map_direct_value_addr) { 6480 verbose(env, "no direct value access support for this map type\n"); 6481 return -EACCES; 6482 } 6483 6484 err = check_map_access(env, regno, reg->off, 6485 map->value_size - reg->off, false, 6486 ACCESS_HELPER); 6487 if (err) 6488 return err; 6489 6490 map_off = reg->off + reg->var_off.value; 6491 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 6492 if (err) { 6493 verbose(env, "direct value access on string failed\n"); 6494 return err; 6495 } 6496 6497 str_ptr = (char *)(long)(map_addr); 6498 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 6499 verbose(env, "string is not zero-terminated\n"); 6500 return -EINVAL; 6501 } 6502 break; 6503 } 6504 case ARG_PTR_TO_KPTR: 6505 if (process_kptr_func(env, regno, meta)) 6506 return -EACCES; 6507 break; 6508 } 6509 6510 return err; 6511 } 6512 6513 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 6514 { 6515 enum bpf_attach_type eatype = env->prog->expected_attach_type; 6516 enum bpf_prog_type type = resolve_prog_type(env->prog); 6517 6518 if (func_id != BPF_FUNC_map_update_elem) 6519 return false; 6520 6521 /* It's not possible to get access to a locked struct sock in these 6522 * contexts, so updating is safe. 6523 */ 6524 switch (type) { 6525 case BPF_PROG_TYPE_TRACING: 6526 if (eatype == BPF_TRACE_ITER) 6527 return true; 6528 break; 6529 case BPF_PROG_TYPE_SOCKET_FILTER: 6530 case BPF_PROG_TYPE_SCHED_CLS: 6531 case BPF_PROG_TYPE_SCHED_ACT: 6532 case BPF_PROG_TYPE_XDP: 6533 case BPF_PROG_TYPE_SK_REUSEPORT: 6534 case BPF_PROG_TYPE_FLOW_DISSECTOR: 6535 case BPF_PROG_TYPE_SK_LOOKUP: 6536 return true; 6537 default: 6538 break; 6539 } 6540 6541 verbose(env, "cannot update sockmap in this context\n"); 6542 return false; 6543 } 6544 6545 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 6546 { 6547 return env->prog->jit_requested && 6548 bpf_jit_supports_subprog_tailcalls(); 6549 } 6550 6551 static int check_map_func_compatibility(struct bpf_verifier_env *env, 6552 struct bpf_map *map, int func_id) 6553 { 6554 if (!map) 6555 return 0; 6556 6557 /* We need a two way check, first is from map perspective ... */ 6558 switch (map->map_type) { 6559 case BPF_MAP_TYPE_PROG_ARRAY: 6560 if (func_id != BPF_FUNC_tail_call) 6561 goto error; 6562 break; 6563 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 6564 if (func_id != BPF_FUNC_perf_event_read && 6565 func_id != BPF_FUNC_perf_event_output && 6566 func_id != BPF_FUNC_skb_output && 6567 func_id != BPF_FUNC_perf_event_read_value && 6568 func_id != BPF_FUNC_xdp_output) 6569 goto error; 6570 break; 6571 case BPF_MAP_TYPE_RINGBUF: 6572 if (func_id != BPF_FUNC_ringbuf_output && 6573 func_id != BPF_FUNC_ringbuf_reserve && 6574 func_id != BPF_FUNC_ringbuf_query && 6575 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 6576 func_id != BPF_FUNC_ringbuf_submit_dynptr && 6577 func_id != BPF_FUNC_ringbuf_discard_dynptr) 6578 goto error; 6579 break; 6580 case BPF_MAP_TYPE_USER_RINGBUF: 6581 if (func_id != BPF_FUNC_user_ringbuf_drain) 6582 goto error; 6583 break; 6584 case BPF_MAP_TYPE_STACK_TRACE: 6585 if (func_id != BPF_FUNC_get_stackid) 6586 goto error; 6587 break; 6588 case BPF_MAP_TYPE_CGROUP_ARRAY: 6589 if (func_id != BPF_FUNC_skb_under_cgroup && 6590 func_id != BPF_FUNC_current_task_under_cgroup) 6591 goto error; 6592 break; 6593 case BPF_MAP_TYPE_CGROUP_STORAGE: 6594 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 6595 if (func_id != BPF_FUNC_get_local_storage) 6596 goto error; 6597 break; 6598 case BPF_MAP_TYPE_DEVMAP: 6599 case BPF_MAP_TYPE_DEVMAP_HASH: 6600 if (func_id != BPF_FUNC_redirect_map && 6601 func_id != BPF_FUNC_map_lookup_elem) 6602 goto error; 6603 break; 6604 /* Restrict bpf side of cpumap and xskmap, open when use-cases 6605 * appear. 6606 */ 6607 case BPF_MAP_TYPE_CPUMAP: 6608 if (func_id != BPF_FUNC_redirect_map) 6609 goto error; 6610 break; 6611 case BPF_MAP_TYPE_XSKMAP: 6612 if (func_id != BPF_FUNC_redirect_map && 6613 func_id != BPF_FUNC_map_lookup_elem) 6614 goto error; 6615 break; 6616 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 6617 case BPF_MAP_TYPE_HASH_OF_MAPS: 6618 if (func_id != BPF_FUNC_map_lookup_elem) 6619 goto error; 6620 break; 6621 case BPF_MAP_TYPE_SOCKMAP: 6622 if (func_id != BPF_FUNC_sk_redirect_map && 6623 func_id != BPF_FUNC_sock_map_update && 6624 func_id != BPF_FUNC_map_delete_elem && 6625 func_id != BPF_FUNC_msg_redirect_map && 6626 func_id != BPF_FUNC_sk_select_reuseport && 6627 func_id != BPF_FUNC_map_lookup_elem && 6628 !may_update_sockmap(env, func_id)) 6629 goto error; 6630 break; 6631 case BPF_MAP_TYPE_SOCKHASH: 6632 if (func_id != BPF_FUNC_sk_redirect_hash && 6633 func_id != BPF_FUNC_sock_hash_update && 6634 func_id != BPF_FUNC_map_delete_elem && 6635 func_id != BPF_FUNC_msg_redirect_hash && 6636 func_id != BPF_FUNC_sk_select_reuseport && 6637 func_id != BPF_FUNC_map_lookup_elem && 6638 !may_update_sockmap(env, func_id)) 6639 goto error; 6640 break; 6641 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 6642 if (func_id != BPF_FUNC_sk_select_reuseport) 6643 goto error; 6644 break; 6645 case BPF_MAP_TYPE_QUEUE: 6646 case BPF_MAP_TYPE_STACK: 6647 if (func_id != BPF_FUNC_map_peek_elem && 6648 func_id != BPF_FUNC_map_pop_elem && 6649 func_id != BPF_FUNC_map_push_elem) 6650 goto error; 6651 break; 6652 case BPF_MAP_TYPE_SK_STORAGE: 6653 if (func_id != BPF_FUNC_sk_storage_get && 6654 func_id != BPF_FUNC_sk_storage_delete) 6655 goto error; 6656 break; 6657 case BPF_MAP_TYPE_INODE_STORAGE: 6658 if (func_id != BPF_FUNC_inode_storage_get && 6659 func_id != BPF_FUNC_inode_storage_delete) 6660 goto error; 6661 break; 6662 case BPF_MAP_TYPE_TASK_STORAGE: 6663 if (func_id != BPF_FUNC_task_storage_get && 6664 func_id != BPF_FUNC_task_storage_delete) 6665 goto error; 6666 break; 6667 case BPF_MAP_TYPE_CGRP_STORAGE: 6668 if (func_id != BPF_FUNC_cgrp_storage_get && 6669 func_id != BPF_FUNC_cgrp_storage_delete) 6670 goto error; 6671 break; 6672 case BPF_MAP_TYPE_BLOOM_FILTER: 6673 if (func_id != BPF_FUNC_map_peek_elem && 6674 func_id != BPF_FUNC_map_push_elem) 6675 goto error; 6676 break; 6677 default: 6678 break; 6679 } 6680 6681 /* ... and second from the function itself. */ 6682 switch (func_id) { 6683 case BPF_FUNC_tail_call: 6684 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 6685 goto error; 6686 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 6687 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 6688 return -EINVAL; 6689 } 6690 break; 6691 case BPF_FUNC_perf_event_read: 6692 case BPF_FUNC_perf_event_output: 6693 case BPF_FUNC_perf_event_read_value: 6694 case BPF_FUNC_skb_output: 6695 case BPF_FUNC_xdp_output: 6696 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 6697 goto error; 6698 break; 6699 case BPF_FUNC_ringbuf_output: 6700 case BPF_FUNC_ringbuf_reserve: 6701 case BPF_FUNC_ringbuf_query: 6702 case BPF_FUNC_ringbuf_reserve_dynptr: 6703 case BPF_FUNC_ringbuf_submit_dynptr: 6704 case BPF_FUNC_ringbuf_discard_dynptr: 6705 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 6706 goto error; 6707 break; 6708 case BPF_FUNC_user_ringbuf_drain: 6709 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 6710 goto error; 6711 break; 6712 case BPF_FUNC_get_stackid: 6713 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 6714 goto error; 6715 break; 6716 case BPF_FUNC_current_task_under_cgroup: 6717 case BPF_FUNC_skb_under_cgroup: 6718 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 6719 goto error; 6720 break; 6721 case BPF_FUNC_redirect_map: 6722 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 6723 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 6724 map->map_type != BPF_MAP_TYPE_CPUMAP && 6725 map->map_type != BPF_MAP_TYPE_XSKMAP) 6726 goto error; 6727 break; 6728 case BPF_FUNC_sk_redirect_map: 6729 case BPF_FUNC_msg_redirect_map: 6730 case BPF_FUNC_sock_map_update: 6731 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 6732 goto error; 6733 break; 6734 case BPF_FUNC_sk_redirect_hash: 6735 case BPF_FUNC_msg_redirect_hash: 6736 case BPF_FUNC_sock_hash_update: 6737 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 6738 goto error; 6739 break; 6740 case BPF_FUNC_get_local_storage: 6741 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 6742 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 6743 goto error; 6744 break; 6745 case BPF_FUNC_sk_select_reuseport: 6746 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 6747 map->map_type != BPF_MAP_TYPE_SOCKMAP && 6748 map->map_type != BPF_MAP_TYPE_SOCKHASH) 6749 goto error; 6750 break; 6751 case BPF_FUNC_map_pop_elem: 6752 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6753 map->map_type != BPF_MAP_TYPE_STACK) 6754 goto error; 6755 break; 6756 case BPF_FUNC_map_peek_elem: 6757 case BPF_FUNC_map_push_elem: 6758 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6759 map->map_type != BPF_MAP_TYPE_STACK && 6760 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 6761 goto error; 6762 break; 6763 case BPF_FUNC_map_lookup_percpu_elem: 6764 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 6765 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 6766 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 6767 goto error; 6768 break; 6769 case BPF_FUNC_sk_storage_get: 6770 case BPF_FUNC_sk_storage_delete: 6771 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 6772 goto error; 6773 break; 6774 case BPF_FUNC_inode_storage_get: 6775 case BPF_FUNC_inode_storage_delete: 6776 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 6777 goto error; 6778 break; 6779 case BPF_FUNC_task_storage_get: 6780 case BPF_FUNC_task_storage_delete: 6781 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 6782 goto error; 6783 break; 6784 case BPF_FUNC_cgrp_storage_get: 6785 case BPF_FUNC_cgrp_storage_delete: 6786 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 6787 goto error; 6788 break; 6789 default: 6790 break; 6791 } 6792 6793 return 0; 6794 error: 6795 verbose(env, "cannot pass map_type %d into func %s#%d\n", 6796 map->map_type, func_id_name(func_id), func_id); 6797 return -EINVAL; 6798 } 6799 6800 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 6801 { 6802 int count = 0; 6803 6804 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 6805 count++; 6806 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 6807 count++; 6808 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 6809 count++; 6810 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 6811 count++; 6812 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 6813 count++; 6814 6815 /* We only support one arg being in raw mode at the moment, 6816 * which is sufficient for the helper functions we have 6817 * right now. 6818 */ 6819 return count <= 1; 6820 } 6821 6822 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 6823 { 6824 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 6825 bool has_size = fn->arg_size[arg] != 0; 6826 bool is_next_size = false; 6827 6828 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 6829 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 6830 6831 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 6832 return is_next_size; 6833 6834 return has_size == is_next_size || is_next_size == is_fixed; 6835 } 6836 6837 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 6838 { 6839 /* bpf_xxx(..., buf, len) call will access 'len' 6840 * bytes from memory 'buf'. Both arg types need 6841 * to be paired, so make sure there's no buggy 6842 * helper function specification. 6843 */ 6844 if (arg_type_is_mem_size(fn->arg1_type) || 6845 check_args_pair_invalid(fn, 0) || 6846 check_args_pair_invalid(fn, 1) || 6847 check_args_pair_invalid(fn, 2) || 6848 check_args_pair_invalid(fn, 3) || 6849 check_args_pair_invalid(fn, 4)) 6850 return false; 6851 6852 return true; 6853 } 6854 6855 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 6856 { 6857 int i; 6858 6859 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 6860 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 6861 return !!fn->arg_btf_id[i]; 6862 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 6863 return fn->arg_btf_id[i] == BPF_PTR_POISON; 6864 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 6865 /* arg_btf_id and arg_size are in a union. */ 6866 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 6867 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 6868 return false; 6869 } 6870 6871 return true; 6872 } 6873 6874 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 6875 { 6876 return check_raw_mode_ok(fn) && 6877 check_arg_pair_ok(fn) && 6878 check_btf_id_ok(fn) ? 0 : -EINVAL; 6879 } 6880 6881 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 6882 * are now invalid, so turn them into unknown SCALAR_VALUE. 6883 */ 6884 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 6885 { 6886 struct bpf_func_state *state; 6887 struct bpf_reg_state *reg; 6888 6889 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 6890 if (reg_is_pkt_pointer_any(reg)) 6891 __mark_reg_unknown(env, reg); 6892 })); 6893 } 6894 6895 enum { 6896 AT_PKT_END = -1, 6897 BEYOND_PKT_END = -2, 6898 }; 6899 6900 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 6901 { 6902 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6903 struct bpf_reg_state *reg = &state->regs[regn]; 6904 6905 if (reg->type != PTR_TO_PACKET) 6906 /* PTR_TO_PACKET_META is not supported yet */ 6907 return; 6908 6909 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 6910 * How far beyond pkt_end it goes is unknown. 6911 * if (!range_open) it's the case of pkt >= pkt_end 6912 * if (range_open) it's the case of pkt > pkt_end 6913 * hence this pointer is at least 1 byte bigger than pkt_end 6914 */ 6915 if (range_open) 6916 reg->range = BEYOND_PKT_END; 6917 else 6918 reg->range = AT_PKT_END; 6919 } 6920 6921 /* The pointer with the specified id has released its reference to kernel 6922 * resources. Identify all copies of the same pointer and clear the reference. 6923 */ 6924 static int release_reference(struct bpf_verifier_env *env, 6925 int ref_obj_id) 6926 { 6927 struct bpf_func_state *state; 6928 struct bpf_reg_state *reg; 6929 int err; 6930 6931 err = release_reference_state(cur_func(env), ref_obj_id); 6932 if (err) 6933 return err; 6934 6935 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 6936 if (reg->ref_obj_id == ref_obj_id) { 6937 if (!env->allow_ptr_leaks) 6938 __mark_reg_not_init(env, reg); 6939 else 6940 __mark_reg_unknown(env, reg); 6941 } 6942 })); 6943 6944 return 0; 6945 } 6946 6947 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 6948 struct bpf_reg_state *regs) 6949 { 6950 int i; 6951 6952 /* after the call registers r0 - r5 were scratched */ 6953 for (i = 0; i < CALLER_SAVED_REGS; i++) { 6954 mark_reg_not_init(env, regs, caller_saved[i]); 6955 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 6956 } 6957 } 6958 6959 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 6960 struct bpf_func_state *caller, 6961 struct bpf_func_state *callee, 6962 int insn_idx); 6963 6964 static int set_callee_state(struct bpf_verifier_env *env, 6965 struct bpf_func_state *caller, 6966 struct bpf_func_state *callee, int insn_idx); 6967 6968 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6969 int *insn_idx, int subprog, 6970 set_callee_state_fn set_callee_state_cb) 6971 { 6972 struct bpf_verifier_state *state = env->cur_state; 6973 struct bpf_func_info_aux *func_info_aux; 6974 struct bpf_func_state *caller, *callee; 6975 int err; 6976 bool is_global = false; 6977 6978 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 6979 verbose(env, "the call stack of %d frames is too deep\n", 6980 state->curframe + 2); 6981 return -E2BIG; 6982 } 6983 6984 caller = state->frame[state->curframe]; 6985 if (state->frame[state->curframe + 1]) { 6986 verbose(env, "verifier bug. Frame %d already allocated\n", 6987 state->curframe + 1); 6988 return -EFAULT; 6989 } 6990 6991 func_info_aux = env->prog->aux->func_info_aux; 6992 if (func_info_aux) 6993 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 6994 err = btf_check_subprog_call(env, subprog, caller->regs); 6995 if (err == -EFAULT) 6996 return err; 6997 if (is_global) { 6998 if (err) { 6999 verbose(env, "Caller passes invalid args into func#%d\n", 7000 subprog); 7001 return err; 7002 } else { 7003 if (env->log.level & BPF_LOG_LEVEL) 7004 verbose(env, 7005 "Func#%d is global and valid. Skipping.\n", 7006 subprog); 7007 clear_caller_saved_regs(env, caller->regs); 7008 7009 /* All global functions return a 64-bit SCALAR_VALUE */ 7010 mark_reg_unknown(env, caller->regs, BPF_REG_0); 7011 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7012 7013 /* continue with next insn after call */ 7014 return 0; 7015 } 7016 } 7017 7018 /* set_callee_state is used for direct subprog calls, but we are 7019 * interested in validating only BPF helpers that can call subprogs as 7020 * callbacks 7021 */ 7022 if (set_callee_state_cb != set_callee_state && !is_callback_calling_function(insn->imm)) { 7023 verbose(env, "verifier bug: helper %s#%d is not marked as callback-calling\n", 7024 func_id_name(insn->imm), insn->imm); 7025 return -EFAULT; 7026 } 7027 7028 if (insn->code == (BPF_JMP | BPF_CALL) && 7029 insn->src_reg == 0 && 7030 insn->imm == BPF_FUNC_timer_set_callback) { 7031 struct bpf_verifier_state *async_cb; 7032 7033 /* there is no real recursion here. timer callbacks are async */ 7034 env->subprog_info[subprog].is_async_cb = true; 7035 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 7036 *insn_idx, subprog); 7037 if (!async_cb) 7038 return -EFAULT; 7039 callee = async_cb->frame[0]; 7040 callee->async_entry_cnt = caller->async_entry_cnt + 1; 7041 7042 /* Convert bpf_timer_set_callback() args into timer callback args */ 7043 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7044 if (err) 7045 return err; 7046 7047 clear_caller_saved_regs(env, caller->regs); 7048 mark_reg_unknown(env, caller->regs, BPF_REG_0); 7049 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7050 /* continue with next insn after call */ 7051 return 0; 7052 } 7053 7054 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 7055 if (!callee) 7056 return -ENOMEM; 7057 state->frame[state->curframe + 1] = callee; 7058 7059 /* callee cannot access r0, r6 - r9 for reading and has to write 7060 * into its own stack before reading from it. 7061 * callee can read/write into caller's stack 7062 */ 7063 init_func_state(env, callee, 7064 /* remember the callsite, it will be used by bpf_exit */ 7065 *insn_idx /* callsite */, 7066 state->curframe + 1 /* frameno within this callchain */, 7067 subprog /* subprog number within this prog */); 7068 7069 /* Transfer references to the callee */ 7070 err = copy_reference_state(callee, caller); 7071 if (err) 7072 goto err_out; 7073 7074 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7075 if (err) 7076 goto err_out; 7077 7078 clear_caller_saved_regs(env, caller->regs); 7079 7080 /* only increment it after check_reg_arg() finished */ 7081 state->curframe++; 7082 7083 /* and go analyze first insn of the callee */ 7084 *insn_idx = env->subprog_info[subprog].start - 1; 7085 7086 if (env->log.level & BPF_LOG_LEVEL) { 7087 verbose(env, "caller:\n"); 7088 print_verifier_state(env, caller, true); 7089 verbose(env, "callee:\n"); 7090 print_verifier_state(env, callee, true); 7091 } 7092 return 0; 7093 7094 err_out: 7095 free_func_state(callee); 7096 state->frame[state->curframe + 1] = NULL; 7097 return err; 7098 } 7099 7100 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 7101 struct bpf_func_state *caller, 7102 struct bpf_func_state *callee) 7103 { 7104 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 7105 * void *callback_ctx, u64 flags); 7106 * callback_fn(struct bpf_map *map, void *key, void *value, 7107 * void *callback_ctx); 7108 */ 7109 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7110 7111 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7112 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7113 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7114 7115 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7116 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7117 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7118 7119 /* pointer to stack or null */ 7120 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 7121 7122 /* unused */ 7123 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7124 return 0; 7125 } 7126 7127 static int set_callee_state(struct bpf_verifier_env *env, 7128 struct bpf_func_state *caller, 7129 struct bpf_func_state *callee, int insn_idx) 7130 { 7131 int i; 7132 7133 /* copy r1 - r5 args that callee can access. The copy includes parent 7134 * pointers, which connects us up to the liveness chain 7135 */ 7136 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 7137 callee->regs[i] = caller->regs[i]; 7138 return 0; 7139 } 7140 7141 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7142 int *insn_idx) 7143 { 7144 int subprog, target_insn; 7145 7146 target_insn = *insn_idx + insn->imm + 1; 7147 subprog = find_subprog(env, target_insn); 7148 if (subprog < 0) { 7149 verbose(env, "verifier bug. No program starts at insn %d\n", 7150 target_insn); 7151 return -EFAULT; 7152 } 7153 7154 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 7155 } 7156 7157 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 7158 struct bpf_func_state *caller, 7159 struct bpf_func_state *callee, 7160 int insn_idx) 7161 { 7162 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 7163 struct bpf_map *map; 7164 int err; 7165 7166 if (bpf_map_ptr_poisoned(insn_aux)) { 7167 verbose(env, "tail_call abusing map_ptr\n"); 7168 return -EINVAL; 7169 } 7170 7171 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 7172 if (!map->ops->map_set_for_each_callback_args || 7173 !map->ops->map_for_each_callback) { 7174 verbose(env, "callback function not allowed for map\n"); 7175 return -ENOTSUPP; 7176 } 7177 7178 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 7179 if (err) 7180 return err; 7181 7182 callee->in_callback_fn = true; 7183 callee->callback_ret_range = tnum_range(0, 1); 7184 return 0; 7185 } 7186 7187 static int set_loop_callback_state(struct bpf_verifier_env *env, 7188 struct bpf_func_state *caller, 7189 struct bpf_func_state *callee, 7190 int insn_idx) 7191 { 7192 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 7193 * u64 flags); 7194 * callback_fn(u32 index, void *callback_ctx); 7195 */ 7196 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 7197 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7198 7199 /* unused */ 7200 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7201 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7202 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7203 7204 callee->in_callback_fn = true; 7205 callee->callback_ret_range = tnum_range(0, 1); 7206 return 0; 7207 } 7208 7209 static int set_timer_callback_state(struct bpf_verifier_env *env, 7210 struct bpf_func_state *caller, 7211 struct bpf_func_state *callee, 7212 int insn_idx) 7213 { 7214 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 7215 7216 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 7217 * callback_fn(struct bpf_map *map, void *key, void *value); 7218 */ 7219 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 7220 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 7221 callee->regs[BPF_REG_1].map_ptr = map_ptr; 7222 7223 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7224 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7225 callee->regs[BPF_REG_2].map_ptr = map_ptr; 7226 7227 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7228 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7229 callee->regs[BPF_REG_3].map_ptr = map_ptr; 7230 7231 /* unused */ 7232 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7233 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7234 callee->in_async_callback_fn = true; 7235 callee->callback_ret_range = tnum_range(0, 1); 7236 return 0; 7237 } 7238 7239 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 7240 struct bpf_func_state *caller, 7241 struct bpf_func_state *callee, 7242 int insn_idx) 7243 { 7244 /* bpf_find_vma(struct task_struct *task, u64 addr, 7245 * void *callback_fn, void *callback_ctx, u64 flags) 7246 * (callback_fn)(struct task_struct *task, 7247 * struct vm_area_struct *vma, void *callback_ctx); 7248 */ 7249 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7250 7251 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 7252 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7253 callee->regs[BPF_REG_2].btf = btf_vmlinux; 7254 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], 7255 7256 /* pointer to stack or null */ 7257 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 7258 7259 /* unused */ 7260 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7261 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7262 callee->in_callback_fn = true; 7263 callee->callback_ret_range = tnum_range(0, 1); 7264 return 0; 7265 } 7266 7267 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 7268 struct bpf_func_state *caller, 7269 struct bpf_func_state *callee, 7270 int insn_idx) 7271 { 7272 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 7273 * callback_ctx, u64 flags); 7274 * callback_fn(struct bpf_dynptr_t* dynptr, void *callback_ctx); 7275 */ 7276 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 7277 callee->regs[BPF_REG_1].type = PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL; 7278 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 7279 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7280 7281 /* unused */ 7282 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7283 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7284 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7285 7286 callee->in_callback_fn = true; 7287 callee->callback_ret_range = tnum_range(0, 1); 7288 return 0; 7289 } 7290 7291 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 7292 { 7293 struct bpf_verifier_state *state = env->cur_state; 7294 struct bpf_func_state *caller, *callee; 7295 struct bpf_reg_state *r0; 7296 int err; 7297 7298 callee = state->frame[state->curframe]; 7299 r0 = &callee->regs[BPF_REG_0]; 7300 if (r0->type == PTR_TO_STACK) { 7301 /* technically it's ok to return caller's stack pointer 7302 * (or caller's caller's pointer) back to the caller, 7303 * since these pointers are valid. Only current stack 7304 * pointer will be invalid as soon as function exits, 7305 * but let's be conservative 7306 */ 7307 verbose(env, "cannot return stack pointer to the caller\n"); 7308 return -EINVAL; 7309 } 7310 7311 caller = state->frame[state->curframe - 1]; 7312 if (callee->in_callback_fn) { 7313 /* enforce R0 return value range [0, 1]. */ 7314 struct tnum range = callee->callback_ret_range; 7315 7316 if (r0->type != SCALAR_VALUE) { 7317 verbose(env, "R0 not a scalar value\n"); 7318 return -EACCES; 7319 } 7320 if (!tnum_in(range, r0->var_off)) { 7321 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 7322 return -EINVAL; 7323 } 7324 } else { 7325 /* return to the caller whatever r0 had in the callee */ 7326 caller->regs[BPF_REG_0] = *r0; 7327 } 7328 7329 /* callback_fn frame should have released its own additions to parent's 7330 * reference state at this point, or check_reference_leak would 7331 * complain, hence it must be the same as the caller. There is no need 7332 * to copy it back. 7333 */ 7334 if (!callee->in_callback_fn) { 7335 /* Transfer references to the caller */ 7336 err = copy_reference_state(caller, callee); 7337 if (err) 7338 return err; 7339 } 7340 7341 *insn_idx = callee->callsite + 1; 7342 if (env->log.level & BPF_LOG_LEVEL) { 7343 verbose(env, "returning from callee:\n"); 7344 print_verifier_state(env, callee, true); 7345 verbose(env, "to caller at %d:\n", *insn_idx); 7346 print_verifier_state(env, caller, true); 7347 } 7348 /* clear everything in the callee */ 7349 free_func_state(callee); 7350 state->frame[state->curframe--] = NULL; 7351 return 0; 7352 } 7353 7354 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 7355 int func_id, 7356 struct bpf_call_arg_meta *meta) 7357 { 7358 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 7359 7360 if (ret_type != RET_INTEGER || 7361 (func_id != BPF_FUNC_get_stack && 7362 func_id != BPF_FUNC_get_task_stack && 7363 func_id != BPF_FUNC_probe_read_str && 7364 func_id != BPF_FUNC_probe_read_kernel_str && 7365 func_id != BPF_FUNC_probe_read_user_str)) 7366 return; 7367 7368 ret_reg->smax_value = meta->msize_max_value; 7369 ret_reg->s32_max_value = meta->msize_max_value; 7370 ret_reg->smin_value = -MAX_ERRNO; 7371 ret_reg->s32_min_value = -MAX_ERRNO; 7372 reg_bounds_sync(ret_reg); 7373 } 7374 7375 static int 7376 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 7377 int func_id, int insn_idx) 7378 { 7379 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 7380 struct bpf_map *map = meta->map_ptr; 7381 7382 if (func_id != BPF_FUNC_tail_call && 7383 func_id != BPF_FUNC_map_lookup_elem && 7384 func_id != BPF_FUNC_map_update_elem && 7385 func_id != BPF_FUNC_map_delete_elem && 7386 func_id != BPF_FUNC_map_push_elem && 7387 func_id != BPF_FUNC_map_pop_elem && 7388 func_id != BPF_FUNC_map_peek_elem && 7389 func_id != BPF_FUNC_for_each_map_elem && 7390 func_id != BPF_FUNC_redirect_map && 7391 func_id != BPF_FUNC_map_lookup_percpu_elem) 7392 return 0; 7393 7394 if (map == NULL) { 7395 verbose(env, "kernel subsystem misconfigured verifier\n"); 7396 return -EINVAL; 7397 } 7398 7399 /* In case of read-only, some additional restrictions 7400 * need to be applied in order to prevent altering the 7401 * state of the map from program side. 7402 */ 7403 if ((map->map_flags & BPF_F_RDONLY_PROG) && 7404 (func_id == BPF_FUNC_map_delete_elem || 7405 func_id == BPF_FUNC_map_update_elem || 7406 func_id == BPF_FUNC_map_push_elem || 7407 func_id == BPF_FUNC_map_pop_elem)) { 7408 verbose(env, "write into map forbidden\n"); 7409 return -EACCES; 7410 } 7411 7412 if (!BPF_MAP_PTR(aux->map_ptr_state)) 7413 bpf_map_ptr_store(aux, meta->map_ptr, 7414 !meta->map_ptr->bypass_spec_v1); 7415 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 7416 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 7417 !meta->map_ptr->bypass_spec_v1); 7418 return 0; 7419 } 7420 7421 static int 7422 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 7423 int func_id, int insn_idx) 7424 { 7425 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 7426 struct bpf_reg_state *regs = cur_regs(env), *reg; 7427 struct bpf_map *map = meta->map_ptr; 7428 u64 val, max; 7429 int err; 7430 7431 if (func_id != BPF_FUNC_tail_call) 7432 return 0; 7433 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 7434 verbose(env, "kernel subsystem misconfigured verifier\n"); 7435 return -EINVAL; 7436 } 7437 7438 reg = ®s[BPF_REG_3]; 7439 val = reg->var_off.value; 7440 max = map->max_entries; 7441 7442 if (!(register_is_const(reg) && val < max)) { 7443 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 7444 return 0; 7445 } 7446 7447 err = mark_chain_precision(env, BPF_REG_3); 7448 if (err) 7449 return err; 7450 if (bpf_map_key_unseen(aux)) 7451 bpf_map_key_store(aux, val); 7452 else if (!bpf_map_key_poisoned(aux) && 7453 bpf_map_key_immediate(aux) != val) 7454 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 7455 return 0; 7456 } 7457 7458 static int check_reference_leak(struct bpf_verifier_env *env) 7459 { 7460 struct bpf_func_state *state = cur_func(env); 7461 bool refs_lingering = false; 7462 int i; 7463 7464 if (state->frameno && !state->in_callback_fn) 7465 return 0; 7466 7467 for (i = 0; i < state->acquired_refs; i++) { 7468 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 7469 continue; 7470 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 7471 state->refs[i].id, state->refs[i].insn_idx); 7472 refs_lingering = true; 7473 } 7474 return refs_lingering ? -EINVAL : 0; 7475 } 7476 7477 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 7478 struct bpf_reg_state *regs) 7479 { 7480 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 7481 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 7482 struct bpf_map *fmt_map = fmt_reg->map_ptr; 7483 int err, fmt_map_off, num_args; 7484 u64 fmt_addr; 7485 char *fmt; 7486 7487 /* data must be an array of u64 */ 7488 if (data_len_reg->var_off.value % 8) 7489 return -EINVAL; 7490 num_args = data_len_reg->var_off.value / 8; 7491 7492 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 7493 * and map_direct_value_addr is set. 7494 */ 7495 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 7496 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 7497 fmt_map_off); 7498 if (err) { 7499 verbose(env, "verifier bug\n"); 7500 return -EFAULT; 7501 } 7502 fmt = (char *)(long)fmt_addr + fmt_map_off; 7503 7504 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 7505 * can focus on validating the format specifiers. 7506 */ 7507 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); 7508 if (err < 0) 7509 verbose(env, "Invalid format string\n"); 7510 7511 return err; 7512 } 7513 7514 static int check_get_func_ip(struct bpf_verifier_env *env) 7515 { 7516 enum bpf_prog_type type = resolve_prog_type(env->prog); 7517 int func_id = BPF_FUNC_get_func_ip; 7518 7519 if (type == BPF_PROG_TYPE_TRACING) { 7520 if (!bpf_prog_has_trampoline(env->prog)) { 7521 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 7522 func_id_name(func_id), func_id); 7523 return -ENOTSUPP; 7524 } 7525 return 0; 7526 } else if (type == BPF_PROG_TYPE_KPROBE) { 7527 return 0; 7528 } 7529 7530 verbose(env, "func %s#%d not supported for program type %d\n", 7531 func_id_name(func_id), func_id, type); 7532 return -ENOTSUPP; 7533 } 7534 7535 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 7536 { 7537 return &env->insn_aux_data[env->insn_idx]; 7538 } 7539 7540 static bool loop_flag_is_zero(struct bpf_verifier_env *env) 7541 { 7542 struct bpf_reg_state *regs = cur_regs(env); 7543 struct bpf_reg_state *reg = ®s[BPF_REG_4]; 7544 bool reg_is_null = register_is_null(reg); 7545 7546 if (reg_is_null) 7547 mark_chain_precision(env, BPF_REG_4); 7548 7549 return reg_is_null; 7550 } 7551 7552 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 7553 { 7554 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 7555 7556 if (!state->initialized) { 7557 state->initialized = 1; 7558 state->fit_for_inline = loop_flag_is_zero(env); 7559 state->callback_subprogno = subprogno; 7560 return; 7561 } 7562 7563 if (!state->fit_for_inline) 7564 return; 7565 7566 state->fit_for_inline = (loop_flag_is_zero(env) && 7567 state->callback_subprogno == subprogno); 7568 } 7569 7570 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7571 int *insn_idx_p) 7572 { 7573 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 7574 const struct bpf_func_proto *fn = NULL; 7575 enum bpf_return_type ret_type; 7576 enum bpf_type_flag ret_flag; 7577 struct bpf_reg_state *regs; 7578 struct bpf_call_arg_meta meta; 7579 int insn_idx = *insn_idx_p; 7580 bool changes_data; 7581 int i, err, func_id; 7582 7583 /* find function prototype */ 7584 func_id = insn->imm; 7585 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 7586 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 7587 func_id); 7588 return -EINVAL; 7589 } 7590 7591 if (env->ops->get_func_proto) 7592 fn = env->ops->get_func_proto(func_id, env->prog); 7593 if (!fn) { 7594 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 7595 func_id); 7596 return -EINVAL; 7597 } 7598 7599 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 7600 if (!env->prog->gpl_compatible && fn->gpl_only) { 7601 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 7602 return -EINVAL; 7603 } 7604 7605 if (fn->allowed && !fn->allowed(env->prog)) { 7606 verbose(env, "helper call is not allowed in probe\n"); 7607 return -EINVAL; 7608 } 7609 7610 if (!env->prog->aux->sleepable && fn->might_sleep) { 7611 verbose(env, "helper call might sleep in a non-sleepable prog\n"); 7612 return -EINVAL; 7613 } 7614 7615 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 7616 changes_data = bpf_helper_changes_pkt_data(fn->func); 7617 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 7618 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 7619 func_id_name(func_id), func_id); 7620 return -EINVAL; 7621 } 7622 7623 memset(&meta, 0, sizeof(meta)); 7624 meta.pkt_access = fn->pkt_access; 7625 7626 err = check_func_proto(fn, func_id); 7627 if (err) { 7628 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 7629 func_id_name(func_id), func_id); 7630 return err; 7631 } 7632 7633 if (env->cur_state->active_rcu_lock) { 7634 if (fn->might_sleep) { 7635 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", 7636 func_id_name(func_id), func_id); 7637 return -EINVAL; 7638 } 7639 7640 if (env->prog->aux->sleepable && is_storage_get_function(func_id)) 7641 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 7642 } 7643 7644 meta.func_id = func_id; 7645 /* check args */ 7646 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 7647 err = check_func_arg(env, i, &meta, fn); 7648 if (err) 7649 return err; 7650 } 7651 7652 err = record_func_map(env, &meta, func_id, insn_idx); 7653 if (err) 7654 return err; 7655 7656 err = record_func_key(env, &meta, func_id, insn_idx); 7657 if (err) 7658 return err; 7659 7660 /* Mark slots with STACK_MISC in case of raw mode, stack offset 7661 * is inferred from register state. 7662 */ 7663 for (i = 0; i < meta.access_size; i++) { 7664 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 7665 BPF_WRITE, -1, false); 7666 if (err) 7667 return err; 7668 } 7669 7670 regs = cur_regs(env); 7671 7672 if (meta.uninit_dynptr_regno) { 7673 /* we write BPF_DW bits (8 bytes) at a time */ 7674 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 7675 err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno, 7676 i, BPF_DW, BPF_WRITE, -1, false); 7677 if (err) 7678 return err; 7679 } 7680 7681 err = mark_stack_slots_dynptr(env, ®s[meta.uninit_dynptr_regno], 7682 fn->arg_type[meta.uninit_dynptr_regno - BPF_REG_1], 7683 insn_idx); 7684 if (err) 7685 return err; 7686 } 7687 7688 if (meta.release_regno) { 7689 err = -EINVAL; 7690 if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) 7691 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); 7692 else if (meta.ref_obj_id) 7693 err = release_reference(env, meta.ref_obj_id); 7694 /* meta.ref_obj_id can only be 0 if register that is meant to be 7695 * released is NULL, which must be > R0. 7696 */ 7697 else if (register_is_null(®s[meta.release_regno])) 7698 err = 0; 7699 if (err) { 7700 verbose(env, "func %s#%d reference has not been acquired before\n", 7701 func_id_name(func_id), func_id); 7702 return err; 7703 } 7704 } 7705 7706 switch (func_id) { 7707 case BPF_FUNC_tail_call: 7708 err = check_reference_leak(env); 7709 if (err) { 7710 verbose(env, "tail_call would lead to reference leak\n"); 7711 return err; 7712 } 7713 break; 7714 case BPF_FUNC_get_local_storage: 7715 /* check that flags argument in get_local_storage(map, flags) is 0, 7716 * this is required because get_local_storage() can't return an error. 7717 */ 7718 if (!register_is_null(®s[BPF_REG_2])) { 7719 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 7720 return -EINVAL; 7721 } 7722 break; 7723 case BPF_FUNC_for_each_map_elem: 7724 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7725 set_map_elem_callback_state); 7726 break; 7727 case BPF_FUNC_timer_set_callback: 7728 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7729 set_timer_callback_state); 7730 break; 7731 case BPF_FUNC_find_vma: 7732 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7733 set_find_vma_callback_state); 7734 break; 7735 case BPF_FUNC_snprintf: 7736 err = check_bpf_snprintf_call(env, regs); 7737 break; 7738 case BPF_FUNC_loop: 7739 update_loop_inline_state(env, meta.subprogno); 7740 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7741 set_loop_callback_state); 7742 break; 7743 case BPF_FUNC_dynptr_from_mem: 7744 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 7745 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 7746 reg_type_str(env, regs[BPF_REG_1].type)); 7747 return -EACCES; 7748 } 7749 break; 7750 case BPF_FUNC_set_retval: 7751 if (prog_type == BPF_PROG_TYPE_LSM && 7752 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 7753 if (!env->prog->aux->attach_func_proto->type) { 7754 /* Make sure programs that attach to void 7755 * hooks don't try to modify return value. 7756 */ 7757 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 7758 return -EINVAL; 7759 } 7760 } 7761 break; 7762 case BPF_FUNC_dynptr_data: 7763 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 7764 if (arg_type_is_dynptr(fn->arg_type[i])) { 7765 struct bpf_reg_state *reg = ®s[BPF_REG_1 + i]; 7766 7767 if (meta.ref_obj_id) { 7768 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); 7769 return -EFAULT; 7770 } 7771 7772 if (base_type(reg->type) != PTR_TO_DYNPTR) 7773 /* Find the id of the dynptr we're 7774 * tracking the reference of 7775 */ 7776 meta.ref_obj_id = stack_slot_get_id(env, reg); 7777 break; 7778 } 7779 } 7780 if (i == MAX_BPF_FUNC_REG_ARGS) { 7781 verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n"); 7782 return -EFAULT; 7783 } 7784 break; 7785 case BPF_FUNC_user_ringbuf_drain: 7786 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7787 set_user_ringbuf_callback_state); 7788 break; 7789 } 7790 7791 if (err) 7792 return err; 7793 7794 /* reset caller saved regs */ 7795 for (i = 0; i < CALLER_SAVED_REGS; i++) { 7796 mark_reg_not_init(env, regs, caller_saved[i]); 7797 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 7798 } 7799 7800 /* helper call returns 64-bit value. */ 7801 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7802 7803 /* update return register (already marked as written above) */ 7804 ret_type = fn->ret_type; 7805 ret_flag = type_flag(ret_type); 7806 7807 switch (base_type(ret_type)) { 7808 case RET_INTEGER: 7809 /* sets type to SCALAR_VALUE */ 7810 mark_reg_unknown(env, regs, BPF_REG_0); 7811 break; 7812 case RET_VOID: 7813 regs[BPF_REG_0].type = NOT_INIT; 7814 break; 7815 case RET_PTR_TO_MAP_VALUE: 7816 /* There is no offset yet applied, variable or fixed */ 7817 mark_reg_known_zero(env, regs, BPF_REG_0); 7818 /* remember map_ptr, so that check_map_access() 7819 * can check 'value_size' boundary of memory access 7820 * to map element returned from bpf_map_lookup_elem() 7821 */ 7822 if (meta.map_ptr == NULL) { 7823 verbose(env, 7824 "kernel subsystem misconfigured verifier\n"); 7825 return -EINVAL; 7826 } 7827 regs[BPF_REG_0].map_ptr = meta.map_ptr; 7828 regs[BPF_REG_0].map_uid = meta.map_uid; 7829 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 7830 if (!type_may_be_null(ret_type) && 7831 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 7832 regs[BPF_REG_0].id = ++env->id_gen; 7833 } 7834 break; 7835 case RET_PTR_TO_SOCKET: 7836 mark_reg_known_zero(env, regs, BPF_REG_0); 7837 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 7838 break; 7839 case RET_PTR_TO_SOCK_COMMON: 7840 mark_reg_known_zero(env, regs, BPF_REG_0); 7841 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 7842 break; 7843 case RET_PTR_TO_TCP_SOCK: 7844 mark_reg_known_zero(env, regs, BPF_REG_0); 7845 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 7846 break; 7847 case RET_PTR_TO_MEM: 7848 mark_reg_known_zero(env, regs, BPF_REG_0); 7849 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7850 regs[BPF_REG_0].mem_size = meta.mem_size; 7851 break; 7852 case RET_PTR_TO_MEM_OR_BTF_ID: 7853 { 7854 const struct btf_type *t; 7855 7856 mark_reg_known_zero(env, regs, BPF_REG_0); 7857 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 7858 if (!btf_type_is_struct(t)) { 7859 u32 tsize; 7860 const struct btf_type *ret; 7861 const char *tname; 7862 7863 /* resolve the type size of ksym. */ 7864 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 7865 if (IS_ERR(ret)) { 7866 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 7867 verbose(env, "unable to resolve the size of type '%s': %ld\n", 7868 tname, PTR_ERR(ret)); 7869 return -EINVAL; 7870 } 7871 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7872 regs[BPF_REG_0].mem_size = tsize; 7873 } else { 7874 /* MEM_RDONLY may be carried from ret_flag, but it 7875 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 7876 * it will confuse the check of PTR_TO_BTF_ID in 7877 * check_mem_access(). 7878 */ 7879 ret_flag &= ~MEM_RDONLY; 7880 7881 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7882 regs[BPF_REG_0].btf = meta.ret_btf; 7883 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 7884 } 7885 break; 7886 } 7887 case RET_PTR_TO_BTF_ID: 7888 { 7889 struct btf *ret_btf; 7890 int ret_btf_id; 7891 7892 mark_reg_known_zero(env, regs, BPF_REG_0); 7893 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7894 if (func_id == BPF_FUNC_kptr_xchg) { 7895 ret_btf = meta.kptr_field->kptr.btf; 7896 ret_btf_id = meta.kptr_field->kptr.btf_id; 7897 } else { 7898 if (fn->ret_btf_id == BPF_PTR_POISON) { 7899 verbose(env, "verifier internal error:"); 7900 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", 7901 func_id_name(func_id)); 7902 return -EINVAL; 7903 } 7904 ret_btf = btf_vmlinux; 7905 ret_btf_id = *fn->ret_btf_id; 7906 } 7907 if (ret_btf_id == 0) { 7908 verbose(env, "invalid return type %u of func %s#%d\n", 7909 base_type(ret_type), func_id_name(func_id), 7910 func_id); 7911 return -EINVAL; 7912 } 7913 regs[BPF_REG_0].btf = ret_btf; 7914 regs[BPF_REG_0].btf_id = ret_btf_id; 7915 break; 7916 } 7917 default: 7918 verbose(env, "unknown return type %u of func %s#%d\n", 7919 base_type(ret_type), func_id_name(func_id), func_id); 7920 return -EINVAL; 7921 } 7922 7923 if (type_may_be_null(regs[BPF_REG_0].type)) 7924 regs[BPF_REG_0].id = ++env->id_gen; 7925 7926 if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { 7927 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", 7928 func_id_name(func_id), func_id); 7929 return -EFAULT; 7930 } 7931 7932 if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { 7933 /* For release_reference() */ 7934 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 7935 } else if (is_acquire_function(func_id, meta.map_ptr)) { 7936 int id = acquire_reference_state(env, insn_idx); 7937 7938 if (id < 0) 7939 return id; 7940 /* For mark_ptr_or_null_reg() */ 7941 regs[BPF_REG_0].id = id; 7942 /* For release_reference() */ 7943 regs[BPF_REG_0].ref_obj_id = id; 7944 } 7945 7946 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 7947 7948 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 7949 if (err) 7950 return err; 7951 7952 if ((func_id == BPF_FUNC_get_stack || 7953 func_id == BPF_FUNC_get_task_stack) && 7954 !env->prog->has_callchain_buf) { 7955 const char *err_str; 7956 7957 #ifdef CONFIG_PERF_EVENTS 7958 err = get_callchain_buffers(sysctl_perf_event_max_stack); 7959 err_str = "cannot get callchain buffer for func %s#%d\n"; 7960 #else 7961 err = -ENOTSUPP; 7962 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 7963 #endif 7964 if (err) { 7965 verbose(env, err_str, func_id_name(func_id), func_id); 7966 return err; 7967 } 7968 7969 env->prog->has_callchain_buf = true; 7970 } 7971 7972 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 7973 env->prog->call_get_stack = true; 7974 7975 if (func_id == BPF_FUNC_get_func_ip) { 7976 if (check_get_func_ip(env)) 7977 return -ENOTSUPP; 7978 env->prog->call_get_func_ip = true; 7979 } 7980 7981 if (changes_data) 7982 clear_all_pkt_pointers(env); 7983 return 0; 7984 } 7985 7986 /* mark_btf_func_reg_size() is used when the reg size is determined by 7987 * the BTF func_proto's return value size and argument. 7988 */ 7989 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 7990 size_t reg_size) 7991 { 7992 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 7993 7994 if (regno == BPF_REG_0) { 7995 /* Function return value */ 7996 reg->live |= REG_LIVE_WRITTEN; 7997 reg->subreg_def = reg_size == sizeof(u64) ? 7998 DEF_NOT_SUBREG : env->insn_idx + 1; 7999 } else { 8000 /* Function argument */ 8001 if (reg_size == sizeof(u64)) { 8002 mark_insn_zext(env, reg); 8003 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 8004 } else { 8005 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 8006 } 8007 } 8008 } 8009 8010 struct bpf_kfunc_call_arg_meta { 8011 /* In parameters */ 8012 struct btf *btf; 8013 u32 func_id; 8014 u32 kfunc_flags; 8015 const struct btf_type *func_proto; 8016 const char *func_name; 8017 /* Out parameters */ 8018 u32 ref_obj_id; 8019 u8 release_regno; 8020 bool r0_rdonly; 8021 u32 ret_btf_id; 8022 u64 r0_size; 8023 struct { 8024 u64 value; 8025 bool found; 8026 } arg_constant; 8027 struct { 8028 struct btf *btf; 8029 u32 btf_id; 8030 } arg_obj_drop; 8031 struct { 8032 struct btf_field *field; 8033 } arg_list_head; 8034 }; 8035 8036 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) 8037 { 8038 return meta->kfunc_flags & KF_ACQUIRE; 8039 } 8040 8041 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 8042 { 8043 return meta->kfunc_flags & KF_RET_NULL; 8044 } 8045 8046 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) 8047 { 8048 return meta->kfunc_flags & KF_RELEASE; 8049 } 8050 8051 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta) 8052 { 8053 return meta->kfunc_flags & KF_TRUSTED_ARGS; 8054 } 8055 8056 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) 8057 { 8058 return meta->kfunc_flags & KF_SLEEPABLE; 8059 } 8060 8061 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) 8062 { 8063 return meta->kfunc_flags & KF_DESTRUCTIVE; 8064 } 8065 8066 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) 8067 { 8068 return meta->kfunc_flags & KF_RCU; 8069 } 8070 8071 static bool is_kfunc_arg_kptr_get(struct bpf_kfunc_call_arg_meta *meta, int arg) 8072 { 8073 return arg == 0 && (meta->kfunc_flags & KF_KPTR_GET); 8074 } 8075 8076 static bool __kfunc_param_match_suffix(const struct btf *btf, 8077 const struct btf_param *arg, 8078 const char *suffix) 8079 { 8080 int suffix_len = strlen(suffix), len; 8081 const char *param_name; 8082 8083 /* In the future, this can be ported to use BTF tagging */ 8084 param_name = btf_name_by_offset(btf, arg->name_off); 8085 if (str_is_empty(param_name)) 8086 return false; 8087 len = strlen(param_name); 8088 if (len < suffix_len) 8089 return false; 8090 param_name += len - suffix_len; 8091 return !strncmp(param_name, suffix, suffix_len); 8092 } 8093 8094 static bool is_kfunc_arg_mem_size(const struct btf *btf, 8095 const struct btf_param *arg, 8096 const struct bpf_reg_state *reg) 8097 { 8098 const struct btf_type *t; 8099 8100 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8101 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 8102 return false; 8103 8104 return __kfunc_param_match_suffix(btf, arg, "__sz"); 8105 } 8106 8107 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 8108 { 8109 return __kfunc_param_match_suffix(btf, arg, "__k"); 8110 } 8111 8112 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 8113 { 8114 return __kfunc_param_match_suffix(btf, arg, "__ign"); 8115 } 8116 8117 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 8118 { 8119 return __kfunc_param_match_suffix(btf, arg, "__alloc"); 8120 } 8121 8122 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 8123 const struct btf_param *arg, 8124 const char *name) 8125 { 8126 int len, target_len = strlen(name); 8127 const char *param_name; 8128 8129 param_name = btf_name_by_offset(btf, arg->name_off); 8130 if (str_is_empty(param_name)) 8131 return false; 8132 len = strlen(param_name); 8133 if (len != target_len) 8134 return false; 8135 if (strcmp(param_name, name)) 8136 return false; 8137 8138 return true; 8139 } 8140 8141 enum { 8142 KF_ARG_DYNPTR_ID, 8143 KF_ARG_LIST_HEAD_ID, 8144 KF_ARG_LIST_NODE_ID, 8145 }; 8146 8147 BTF_ID_LIST(kf_arg_btf_ids) 8148 BTF_ID(struct, bpf_dynptr_kern) 8149 BTF_ID(struct, bpf_list_head) 8150 BTF_ID(struct, bpf_list_node) 8151 8152 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 8153 const struct btf_param *arg, int type) 8154 { 8155 const struct btf_type *t; 8156 u32 res_id; 8157 8158 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8159 if (!t) 8160 return false; 8161 if (!btf_type_is_ptr(t)) 8162 return false; 8163 t = btf_type_skip_modifiers(btf, t->type, &res_id); 8164 if (!t) 8165 return false; 8166 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 8167 } 8168 8169 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 8170 { 8171 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 8172 } 8173 8174 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 8175 { 8176 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 8177 } 8178 8179 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 8180 { 8181 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 8182 } 8183 8184 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 8185 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 8186 const struct btf *btf, 8187 const struct btf_type *t, int rec) 8188 { 8189 const struct btf_type *member_type; 8190 const struct btf_member *member; 8191 u32 i; 8192 8193 if (!btf_type_is_struct(t)) 8194 return false; 8195 8196 for_each_member(i, t, member) { 8197 const struct btf_array *array; 8198 8199 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 8200 if (btf_type_is_struct(member_type)) { 8201 if (rec >= 3) { 8202 verbose(env, "max struct nesting depth exceeded\n"); 8203 return false; 8204 } 8205 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) 8206 return false; 8207 continue; 8208 } 8209 if (btf_type_is_array(member_type)) { 8210 array = btf_array(member_type); 8211 if (!array->nelems) 8212 return false; 8213 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 8214 if (!btf_type_is_scalar(member_type)) 8215 return false; 8216 continue; 8217 } 8218 if (!btf_type_is_scalar(member_type)) 8219 return false; 8220 } 8221 return true; 8222 } 8223 8224 8225 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 8226 #ifdef CONFIG_NET 8227 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 8228 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 8229 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 8230 #endif 8231 }; 8232 8233 enum kfunc_ptr_arg_type { 8234 KF_ARG_PTR_TO_CTX, 8235 KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ 8236 KF_ARG_PTR_TO_KPTR, /* PTR_TO_KPTR but type specific */ 8237 KF_ARG_PTR_TO_DYNPTR, 8238 KF_ARG_PTR_TO_LIST_HEAD, 8239 KF_ARG_PTR_TO_LIST_NODE, 8240 KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ 8241 KF_ARG_PTR_TO_MEM, 8242 KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ 8243 }; 8244 8245 enum special_kfunc_type { 8246 KF_bpf_obj_new_impl, 8247 KF_bpf_obj_drop_impl, 8248 KF_bpf_list_push_front, 8249 KF_bpf_list_push_back, 8250 KF_bpf_list_pop_front, 8251 KF_bpf_list_pop_back, 8252 KF_bpf_cast_to_kern_ctx, 8253 KF_bpf_rdonly_cast, 8254 KF_bpf_rcu_read_lock, 8255 KF_bpf_rcu_read_unlock, 8256 }; 8257 8258 BTF_SET_START(special_kfunc_set) 8259 BTF_ID(func, bpf_obj_new_impl) 8260 BTF_ID(func, bpf_obj_drop_impl) 8261 BTF_ID(func, bpf_list_push_front) 8262 BTF_ID(func, bpf_list_push_back) 8263 BTF_ID(func, bpf_list_pop_front) 8264 BTF_ID(func, bpf_list_pop_back) 8265 BTF_ID(func, bpf_cast_to_kern_ctx) 8266 BTF_ID(func, bpf_rdonly_cast) 8267 BTF_SET_END(special_kfunc_set) 8268 8269 BTF_ID_LIST(special_kfunc_list) 8270 BTF_ID(func, bpf_obj_new_impl) 8271 BTF_ID(func, bpf_obj_drop_impl) 8272 BTF_ID(func, bpf_list_push_front) 8273 BTF_ID(func, bpf_list_push_back) 8274 BTF_ID(func, bpf_list_pop_front) 8275 BTF_ID(func, bpf_list_pop_back) 8276 BTF_ID(func, bpf_cast_to_kern_ctx) 8277 BTF_ID(func, bpf_rdonly_cast) 8278 BTF_ID(func, bpf_rcu_read_lock) 8279 BTF_ID(func, bpf_rcu_read_unlock) 8280 8281 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) 8282 { 8283 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; 8284 } 8285 8286 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) 8287 { 8288 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; 8289 } 8290 8291 static enum kfunc_ptr_arg_type 8292 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, 8293 struct bpf_kfunc_call_arg_meta *meta, 8294 const struct btf_type *t, const struct btf_type *ref_t, 8295 const char *ref_tname, const struct btf_param *args, 8296 int argno, int nargs) 8297 { 8298 u32 regno = argno + 1; 8299 struct bpf_reg_state *regs = cur_regs(env); 8300 struct bpf_reg_state *reg = ®s[regno]; 8301 bool arg_mem_size = false; 8302 8303 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) 8304 return KF_ARG_PTR_TO_CTX; 8305 8306 /* In this function, we verify the kfunc's BTF as per the argument type, 8307 * leaving the rest of the verification with respect to the register 8308 * type to our caller. When a set of conditions hold in the BTF type of 8309 * arguments, we resolve it to a known kfunc_ptr_arg_type. 8310 */ 8311 if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) 8312 return KF_ARG_PTR_TO_CTX; 8313 8314 if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) 8315 return KF_ARG_PTR_TO_ALLOC_BTF_ID; 8316 8317 if (is_kfunc_arg_kptr_get(meta, argno)) { 8318 if (!btf_type_is_ptr(ref_t)) { 8319 verbose(env, "arg#0 BTF type must be a double pointer for kptr_get kfunc\n"); 8320 return -EINVAL; 8321 } 8322 ref_t = btf_type_by_id(meta->btf, ref_t->type); 8323 ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); 8324 if (!btf_type_is_struct(ref_t)) { 8325 verbose(env, "kernel function %s args#0 pointer type %s %s is not supported\n", 8326 meta->func_name, btf_type_str(ref_t), ref_tname); 8327 return -EINVAL; 8328 } 8329 return KF_ARG_PTR_TO_KPTR; 8330 } 8331 8332 if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) 8333 return KF_ARG_PTR_TO_DYNPTR; 8334 8335 if (is_kfunc_arg_list_head(meta->btf, &args[argno])) 8336 return KF_ARG_PTR_TO_LIST_HEAD; 8337 8338 if (is_kfunc_arg_list_node(meta->btf, &args[argno])) 8339 return KF_ARG_PTR_TO_LIST_NODE; 8340 8341 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 8342 if (!btf_type_is_struct(ref_t)) { 8343 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", 8344 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 8345 return -EINVAL; 8346 } 8347 return KF_ARG_PTR_TO_BTF_ID; 8348 } 8349 8350 if (argno + 1 < nargs && is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1])) 8351 arg_mem_size = true; 8352 8353 /* This is the catch all argument type of register types supported by 8354 * check_helper_mem_access. However, we only allow when argument type is 8355 * pointer to scalar, or struct composed (recursively) of scalars. When 8356 * arg_mem_size is true, the pointer can be void *. 8357 */ 8358 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && 8359 (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { 8360 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", 8361 argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); 8362 return -EINVAL; 8363 } 8364 return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; 8365 } 8366 8367 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, 8368 struct bpf_reg_state *reg, 8369 const struct btf_type *ref_t, 8370 const char *ref_tname, u32 ref_id, 8371 struct bpf_kfunc_call_arg_meta *meta, 8372 int argno) 8373 { 8374 const struct btf_type *reg_ref_t; 8375 bool strict_type_match = false; 8376 const struct btf *reg_btf; 8377 const char *reg_ref_tname; 8378 u32 reg_ref_id; 8379 8380 if (base_type(reg->type) == PTR_TO_BTF_ID) { 8381 reg_btf = reg->btf; 8382 reg_ref_id = reg->btf_id; 8383 } else { 8384 reg_btf = btf_vmlinux; 8385 reg_ref_id = *reg2btf_ids[base_type(reg->type)]; 8386 } 8387 8388 if (is_kfunc_trusted_args(meta) || (is_kfunc_release(meta) && reg->ref_obj_id)) 8389 strict_type_match = true; 8390 8391 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); 8392 reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); 8393 if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match)) { 8394 verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", 8395 meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, 8396 btf_type_str(reg_ref_t), reg_ref_tname); 8397 return -EINVAL; 8398 } 8399 return 0; 8400 } 8401 8402 static int process_kf_arg_ptr_to_kptr(struct bpf_verifier_env *env, 8403 struct bpf_reg_state *reg, 8404 const struct btf_type *ref_t, 8405 const char *ref_tname, 8406 struct bpf_kfunc_call_arg_meta *meta, 8407 int argno) 8408 { 8409 struct btf_field *kptr_field; 8410 8411 /* check_func_arg_reg_off allows var_off for 8412 * PTR_TO_MAP_VALUE, but we need fixed offset to find 8413 * off_desc. 8414 */ 8415 if (!tnum_is_const(reg->var_off)) { 8416 verbose(env, "arg#0 must have constant offset\n"); 8417 return -EINVAL; 8418 } 8419 8420 kptr_field = btf_record_find(reg->map_ptr->record, reg->off + reg->var_off.value, BPF_KPTR); 8421 if (!kptr_field || kptr_field->type != BPF_KPTR_REF) { 8422 verbose(env, "arg#0 no referenced kptr at map value offset=%llu\n", 8423 reg->off + reg->var_off.value); 8424 return -EINVAL; 8425 } 8426 8427 if (!btf_struct_ids_match(&env->log, meta->btf, ref_t->type, 0, kptr_field->kptr.btf, 8428 kptr_field->kptr.btf_id, true)) { 8429 verbose(env, "kernel function %s args#%d expected pointer to %s %s\n", 8430 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 8431 return -EINVAL; 8432 } 8433 return 0; 8434 } 8435 8436 static int ref_set_release_on_unlock(struct bpf_verifier_env *env, u32 ref_obj_id) 8437 { 8438 struct bpf_func_state *state = cur_func(env); 8439 struct bpf_reg_state *reg; 8440 int i; 8441 8442 /* bpf_spin_lock only allows calling list_push and list_pop, no BPF 8443 * subprogs, no global functions. This means that the references would 8444 * not be released inside the critical section but they may be added to 8445 * the reference state, and the acquired_refs are never copied out for a 8446 * different frame as BPF to BPF calls don't work in bpf_spin_lock 8447 * critical sections. 8448 */ 8449 if (!ref_obj_id) { 8450 verbose(env, "verifier internal error: ref_obj_id is zero for release_on_unlock\n"); 8451 return -EFAULT; 8452 } 8453 for (i = 0; i < state->acquired_refs; i++) { 8454 if (state->refs[i].id == ref_obj_id) { 8455 if (state->refs[i].release_on_unlock) { 8456 verbose(env, "verifier internal error: expected false release_on_unlock"); 8457 return -EFAULT; 8458 } 8459 state->refs[i].release_on_unlock = true; 8460 /* Now mark everyone sharing same ref_obj_id as untrusted */ 8461 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8462 if (reg->ref_obj_id == ref_obj_id) 8463 reg->type |= PTR_UNTRUSTED; 8464 })); 8465 return 0; 8466 } 8467 } 8468 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); 8469 return -EFAULT; 8470 } 8471 8472 /* Implementation details: 8473 * 8474 * Each register points to some region of memory, which we define as an 8475 * allocation. Each allocation may embed a bpf_spin_lock which protects any 8476 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 8477 * allocation. The lock and the data it protects are colocated in the same 8478 * memory region. 8479 * 8480 * Hence, everytime a register holds a pointer value pointing to such 8481 * allocation, the verifier preserves a unique reg->id for it. 8482 * 8483 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 8484 * bpf_spin_lock is called. 8485 * 8486 * To enable this, lock state in the verifier captures two values: 8487 * active_lock.ptr = Register's type specific pointer 8488 * active_lock.id = A unique ID for each register pointer value 8489 * 8490 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 8491 * supported register types. 8492 * 8493 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 8494 * allocated objects is the reg->btf pointer. 8495 * 8496 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 8497 * can establish the provenance of the map value statically for each distinct 8498 * lookup into such maps. They always contain a single map value hence unique 8499 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 8500 * 8501 * So, in case of global variables, they use array maps with max_entries = 1, 8502 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 8503 * into the same map value as max_entries is 1, as described above). 8504 * 8505 * In case of inner map lookups, the inner map pointer has same map_ptr as the 8506 * outer map pointer (in verifier context), but each lookup into an inner map 8507 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 8508 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 8509 * will get different reg->id assigned to each lookup, hence different 8510 * active_lock.id. 8511 * 8512 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 8513 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 8514 * returned from bpf_obj_new. Each allocation receives a new reg->id. 8515 */ 8516 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 8517 { 8518 void *ptr; 8519 u32 id; 8520 8521 switch ((int)reg->type) { 8522 case PTR_TO_MAP_VALUE: 8523 ptr = reg->map_ptr; 8524 break; 8525 case PTR_TO_BTF_ID | MEM_ALLOC: 8526 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_TRUSTED: 8527 ptr = reg->btf; 8528 break; 8529 default: 8530 verbose(env, "verifier internal error: unknown reg type for lock check\n"); 8531 return -EFAULT; 8532 } 8533 id = reg->id; 8534 8535 if (!env->cur_state->active_lock.ptr) 8536 return -EINVAL; 8537 if (env->cur_state->active_lock.ptr != ptr || 8538 env->cur_state->active_lock.id != id) { 8539 verbose(env, "held lock and object are not in the same allocation\n"); 8540 return -EINVAL; 8541 } 8542 return 0; 8543 } 8544 8545 static bool is_bpf_list_api_kfunc(u32 btf_id) 8546 { 8547 return btf_id == special_kfunc_list[KF_bpf_list_push_front] || 8548 btf_id == special_kfunc_list[KF_bpf_list_push_back] || 8549 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 8550 btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 8551 } 8552 8553 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 8554 struct bpf_reg_state *reg, u32 regno, 8555 struct bpf_kfunc_call_arg_meta *meta) 8556 { 8557 struct btf_field *field; 8558 struct btf_record *rec; 8559 u32 list_head_off; 8560 8561 if (meta->btf != btf_vmlinux || !is_bpf_list_api_kfunc(meta->func_id)) { 8562 verbose(env, "verifier internal error: bpf_list_head argument for unknown kfunc\n"); 8563 return -EFAULT; 8564 } 8565 8566 if (!tnum_is_const(reg->var_off)) { 8567 verbose(env, 8568 "R%d doesn't have constant offset. bpf_list_head has to be at the constant offset\n", 8569 regno); 8570 return -EINVAL; 8571 } 8572 8573 rec = reg_btf_record(reg); 8574 list_head_off = reg->off + reg->var_off.value; 8575 field = btf_record_find(rec, list_head_off, BPF_LIST_HEAD); 8576 if (!field) { 8577 verbose(env, "bpf_list_head not found at offset=%u\n", list_head_off); 8578 return -EINVAL; 8579 } 8580 8581 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 8582 if (check_reg_allocation_locked(env, reg)) { 8583 verbose(env, "bpf_spin_lock at off=%d must be held for bpf_list_head\n", 8584 rec->spin_lock_off); 8585 return -EINVAL; 8586 } 8587 8588 if (meta->arg_list_head.field) { 8589 verbose(env, "verifier internal error: repeating bpf_list_head arg\n"); 8590 return -EFAULT; 8591 } 8592 meta->arg_list_head.field = field; 8593 return 0; 8594 } 8595 8596 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 8597 struct bpf_reg_state *reg, u32 regno, 8598 struct bpf_kfunc_call_arg_meta *meta) 8599 { 8600 const struct btf_type *et, *t; 8601 struct btf_field *field; 8602 struct btf_record *rec; 8603 u32 list_node_off; 8604 8605 if (meta->btf != btf_vmlinux || 8606 (meta->func_id != special_kfunc_list[KF_bpf_list_push_front] && 8607 meta->func_id != special_kfunc_list[KF_bpf_list_push_back])) { 8608 verbose(env, "verifier internal error: bpf_list_node argument for unknown kfunc\n"); 8609 return -EFAULT; 8610 } 8611 8612 if (!tnum_is_const(reg->var_off)) { 8613 verbose(env, 8614 "R%d doesn't have constant offset. bpf_list_node has to be at the constant offset\n", 8615 regno); 8616 return -EINVAL; 8617 } 8618 8619 rec = reg_btf_record(reg); 8620 list_node_off = reg->off + reg->var_off.value; 8621 field = btf_record_find(rec, list_node_off, BPF_LIST_NODE); 8622 if (!field || field->offset != list_node_off) { 8623 verbose(env, "bpf_list_node not found at offset=%u\n", list_node_off); 8624 return -EINVAL; 8625 } 8626 8627 field = meta->arg_list_head.field; 8628 8629 et = btf_type_by_id(field->list_head.btf, field->list_head.value_btf_id); 8630 t = btf_type_by_id(reg->btf, reg->btf_id); 8631 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->list_head.btf, 8632 field->list_head.value_btf_id, true)) { 8633 verbose(env, "operation on bpf_list_head expects arg#1 bpf_list_node at offset=%d " 8634 "in struct %s, but arg is at offset=%d in struct %s\n", 8635 field->list_head.node_offset, btf_name_by_offset(field->list_head.btf, et->name_off), 8636 list_node_off, btf_name_by_offset(reg->btf, t->name_off)); 8637 return -EINVAL; 8638 } 8639 8640 if (list_node_off != field->list_head.node_offset) { 8641 verbose(env, "arg#1 offset=%d, but expected bpf_list_node at offset=%d in struct %s\n", 8642 list_node_off, field->list_head.node_offset, 8643 btf_name_by_offset(field->list_head.btf, et->name_off)); 8644 return -EINVAL; 8645 } 8646 /* Set arg#1 for expiration after unlock */ 8647 return ref_set_release_on_unlock(env, reg->ref_obj_id); 8648 } 8649 8650 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta) 8651 { 8652 const char *func_name = meta->func_name, *ref_tname; 8653 const struct btf *btf = meta->btf; 8654 const struct btf_param *args; 8655 u32 i, nargs; 8656 int ret; 8657 8658 args = (const struct btf_param *)(meta->func_proto + 1); 8659 nargs = btf_type_vlen(meta->func_proto); 8660 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 8661 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, 8662 MAX_BPF_FUNC_REG_ARGS); 8663 return -EINVAL; 8664 } 8665 8666 /* Check that BTF function arguments match actual types that the 8667 * verifier sees. 8668 */ 8669 for (i = 0; i < nargs; i++) { 8670 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; 8671 const struct btf_type *t, *ref_t, *resolve_ret; 8672 enum bpf_arg_type arg_type = ARG_DONTCARE; 8673 u32 regno = i + 1, ref_id, type_size; 8674 bool is_ret_buf_sz = false; 8675 int kf_arg_type; 8676 8677 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 8678 8679 if (is_kfunc_arg_ignore(btf, &args[i])) 8680 continue; 8681 8682 if (btf_type_is_scalar(t)) { 8683 if (reg->type != SCALAR_VALUE) { 8684 verbose(env, "R%d is not a scalar\n", regno); 8685 return -EINVAL; 8686 } 8687 8688 if (is_kfunc_arg_constant(meta->btf, &args[i])) { 8689 if (meta->arg_constant.found) { 8690 verbose(env, "verifier internal error: only one constant argument permitted\n"); 8691 return -EFAULT; 8692 } 8693 if (!tnum_is_const(reg->var_off)) { 8694 verbose(env, "R%d must be a known constant\n", regno); 8695 return -EINVAL; 8696 } 8697 ret = mark_chain_precision(env, regno); 8698 if (ret < 0) 8699 return ret; 8700 meta->arg_constant.found = true; 8701 meta->arg_constant.value = reg->var_off.value; 8702 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { 8703 meta->r0_rdonly = true; 8704 is_ret_buf_sz = true; 8705 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { 8706 is_ret_buf_sz = true; 8707 } 8708 8709 if (is_ret_buf_sz) { 8710 if (meta->r0_size) { 8711 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); 8712 return -EINVAL; 8713 } 8714 8715 if (!tnum_is_const(reg->var_off)) { 8716 verbose(env, "R%d is not a const\n", regno); 8717 return -EINVAL; 8718 } 8719 8720 meta->r0_size = reg->var_off.value; 8721 ret = mark_chain_precision(env, regno); 8722 if (ret) 8723 return ret; 8724 } 8725 continue; 8726 } 8727 8728 if (!btf_type_is_ptr(t)) { 8729 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); 8730 return -EINVAL; 8731 } 8732 8733 if (reg->ref_obj_id) { 8734 if (is_kfunc_release(meta) && meta->ref_obj_id) { 8735 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 8736 regno, reg->ref_obj_id, 8737 meta->ref_obj_id); 8738 return -EFAULT; 8739 } 8740 meta->ref_obj_id = reg->ref_obj_id; 8741 if (is_kfunc_release(meta)) 8742 meta->release_regno = regno; 8743 } 8744 8745 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 8746 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 8747 8748 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); 8749 if (kf_arg_type < 0) 8750 return kf_arg_type; 8751 8752 switch (kf_arg_type) { 8753 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 8754 case KF_ARG_PTR_TO_BTF_ID: 8755 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 8756 break; 8757 8758 if (!is_trusted_reg(reg)) { 8759 if (!is_kfunc_rcu(meta)) { 8760 verbose(env, "R%d must be referenced or trusted\n", regno); 8761 return -EINVAL; 8762 } 8763 if (!is_rcu_reg(reg)) { 8764 verbose(env, "R%d must be a rcu pointer\n", regno); 8765 return -EINVAL; 8766 } 8767 } 8768 8769 fallthrough; 8770 case KF_ARG_PTR_TO_CTX: 8771 /* Trusted arguments have the same offset checks as release arguments */ 8772 arg_type |= OBJ_RELEASE; 8773 break; 8774 case KF_ARG_PTR_TO_KPTR: 8775 case KF_ARG_PTR_TO_DYNPTR: 8776 case KF_ARG_PTR_TO_LIST_HEAD: 8777 case KF_ARG_PTR_TO_LIST_NODE: 8778 case KF_ARG_PTR_TO_MEM: 8779 case KF_ARG_PTR_TO_MEM_SIZE: 8780 /* Trusted by default */ 8781 break; 8782 default: 8783 WARN_ON_ONCE(1); 8784 return -EFAULT; 8785 } 8786 8787 if (is_kfunc_release(meta) && reg->ref_obj_id) 8788 arg_type |= OBJ_RELEASE; 8789 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 8790 if (ret < 0) 8791 return ret; 8792 8793 switch (kf_arg_type) { 8794 case KF_ARG_PTR_TO_CTX: 8795 if (reg->type != PTR_TO_CTX) { 8796 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); 8797 return -EINVAL; 8798 } 8799 8800 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 8801 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 8802 if (ret < 0) 8803 return -EINVAL; 8804 meta->ret_btf_id = ret; 8805 } 8806 break; 8807 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 8808 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8809 verbose(env, "arg#%d expected pointer to allocated object\n", i); 8810 return -EINVAL; 8811 } 8812 if (!reg->ref_obj_id) { 8813 verbose(env, "allocated object must be referenced\n"); 8814 return -EINVAL; 8815 } 8816 if (meta->btf == btf_vmlinux && 8817 meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 8818 meta->arg_obj_drop.btf = reg->btf; 8819 meta->arg_obj_drop.btf_id = reg->btf_id; 8820 } 8821 break; 8822 case KF_ARG_PTR_TO_KPTR: 8823 if (reg->type != PTR_TO_MAP_VALUE) { 8824 verbose(env, "arg#0 expected pointer to map value\n"); 8825 return -EINVAL; 8826 } 8827 ret = process_kf_arg_ptr_to_kptr(env, reg, ref_t, ref_tname, meta, i); 8828 if (ret < 0) 8829 return ret; 8830 break; 8831 case KF_ARG_PTR_TO_DYNPTR: 8832 if (reg->type != PTR_TO_STACK) { 8833 verbose(env, "arg#%d expected pointer to stack\n", i); 8834 return -EINVAL; 8835 } 8836 8837 if (!is_dynptr_reg_valid_init(env, reg)) { 8838 verbose(env, "arg#%d pointer type %s %s must be valid and initialized\n", 8839 i, btf_type_str(ref_t), ref_tname); 8840 return -EINVAL; 8841 } 8842 8843 if (!is_dynptr_type_expected(env, reg, ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL)) { 8844 verbose(env, "arg#%d pointer type %s %s points to unsupported dynamic pointer type\n", 8845 i, btf_type_str(ref_t), ref_tname); 8846 return -EINVAL; 8847 } 8848 break; 8849 case KF_ARG_PTR_TO_LIST_HEAD: 8850 if (reg->type != PTR_TO_MAP_VALUE && 8851 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8852 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 8853 return -EINVAL; 8854 } 8855 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 8856 verbose(env, "allocated object must be referenced\n"); 8857 return -EINVAL; 8858 } 8859 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); 8860 if (ret < 0) 8861 return ret; 8862 break; 8863 case KF_ARG_PTR_TO_LIST_NODE: 8864 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8865 verbose(env, "arg#%d expected pointer to allocated object\n", i); 8866 return -EINVAL; 8867 } 8868 if (!reg->ref_obj_id) { 8869 verbose(env, "allocated object must be referenced\n"); 8870 return -EINVAL; 8871 } 8872 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); 8873 if (ret < 0) 8874 return ret; 8875 break; 8876 case KF_ARG_PTR_TO_BTF_ID: 8877 /* Only base_type is checked, further checks are done here */ 8878 if ((base_type(reg->type) != PTR_TO_BTF_ID || 8879 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && 8880 !reg2btf_ids[base_type(reg->type)]) { 8881 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); 8882 verbose(env, "expected %s or socket\n", 8883 reg_type_str(env, base_type(reg->type) | 8884 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 8885 return -EINVAL; 8886 } 8887 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 8888 if (ret < 0) 8889 return ret; 8890 break; 8891 case KF_ARG_PTR_TO_MEM: 8892 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 8893 if (IS_ERR(resolve_ret)) { 8894 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 8895 i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); 8896 return -EINVAL; 8897 } 8898 ret = check_mem_reg(env, reg, regno, type_size); 8899 if (ret < 0) 8900 return ret; 8901 break; 8902 case KF_ARG_PTR_TO_MEM_SIZE: 8903 ret = check_kfunc_mem_size_reg(env, ®s[regno + 1], regno + 1); 8904 if (ret < 0) { 8905 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); 8906 return ret; 8907 } 8908 /* Skip next '__sz' argument */ 8909 i++; 8910 break; 8911 } 8912 } 8913 8914 if (is_kfunc_release(meta) && !meta->release_regno) { 8915 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", 8916 func_name); 8917 return -EINVAL; 8918 } 8919 8920 return 0; 8921 } 8922 8923 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8924 int *insn_idx_p) 8925 { 8926 const struct btf_type *t, *func, *func_proto, *ptr_type; 8927 struct bpf_reg_state *regs = cur_regs(env); 8928 const char *func_name, *ptr_type_name; 8929 bool sleepable, rcu_lock, rcu_unlock; 8930 struct bpf_kfunc_call_arg_meta meta; 8931 u32 i, nargs, func_id, ptr_type_id; 8932 int err, insn_idx = *insn_idx_p; 8933 const struct btf_param *args; 8934 const struct btf_type *ret_t; 8935 struct btf *desc_btf; 8936 u32 *kfunc_flags; 8937 8938 /* skip for now, but return error when we find this in fixup_kfunc_call */ 8939 if (!insn->imm) 8940 return 0; 8941 8942 desc_btf = find_kfunc_desc_btf(env, insn->off); 8943 if (IS_ERR(desc_btf)) 8944 return PTR_ERR(desc_btf); 8945 8946 func_id = insn->imm; 8947 func = btf_type_by_id(desc_btf, func_id); 8948 func_name = btf_name_by_offset(desc_btf, func->name_off); 8949 func_proto = btf_type_by_id(desc_btf, func->type); 8950 8951 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); 8952 if (!kfunc_flags) { 8953 verbose(env, "calling kernel function %s is not allowed\n", 8954 func_name); 8955 return -EACCES; 8956 } 8957 8958 /* Prepare kfunc call metadata */ 8959 memset(&meta, 0, sizeof(meta)); 8960 meta.btf = desc_btf; 8961 meta.func_id = func_id; 8962 meta.kfunc_flags = *kfunc_flags; 8963 meta.func_proto = func_proto; 8964 meta.func_name = func_name; 8965 8966 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 8967 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 8968 return -EACCES; 8969 } 8970 8971 sleepable = is_kfunc_sleepable(&meta); 8972 if (sleepable && !env->prog->aux->sleepable) { 8973 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); 8974 return -EACCES; 8975 } 8976 8977 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 8978 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 8979 if ((rcu_lock || rcu_unlock) && !env->rcu_tag_supported) { 8980 verbose(env, "no vmlinux btf rcu tag support for kfunc %s\n", func_name); 8981 return -EACCES; 8982 } 8983 8984 if (env->cur_state->active_rcu_lock) { 8985 struct bpf_func_state *state; 8986 struct bpf_reg_state *reg; 8987 8988 if (rcu_lock) { 8989 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); 8990 return -EINVAL; 8991 } else if (rcu_unlock) { 8992 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8993 if (reg->type & MEM_RCU) { 8994 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); 8995 reg->type |= PTR_UNTRUSTED; 8996 } 8997 })); 8998 env->cur_state->active_rcu_lock = false; 8999 } else if (sleepable) { 9000 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); 9001 return -EACCES; 9002 } 9003 } else if (rcu_lock) { 9004 env->cur_state->active_rcu_lock = true; 9005 } else if (rcu_unlock) { 9006 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 9007 return -EINVAL; 9008 } 9009 9010 /* Check the arguments */ 9011 err = check_kfunc_args(env, &meta); 9012 if (err < 0) 9013 return err; 9014 /* In case of release function, we get register number of refcounted 9015 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 9016 */ 9017 if (meta.release_regno) { 9018 err = release_reference(env, regs[meta.release_regno].ref_obj_id); 9019 if (err) { 9020 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 9021 func_name, func_id); 9022 return err; 9023 } 9024 } 9025 9026 for (i = 0; i < CALLER_SAVED_REGS; i++) 9027 mark_reg_not_init(env, regs, caller_saved[i]); 9028 9029 /* Check return type */ 9030 t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL); 9031 9032 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 9033 /* Only exception is bpf_obj_new_impl */ 9034 if (meta.btf != btf_vmlinux || meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl]) { 9035 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 9036 return -EINVAL; 9037 } 9038 } 9039 9040 if (btf_type_is_scalar(t)) { 9041 mark_reg_unknown(env, regs, BPF_REG_0); 9042 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 9043 } else if (btf_type_is_ptr(t)) { 9044 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 9045 9046 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 9047 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 9048 struct btf *ret_btf; 9049 u32 ret_btf_id; 9050 9051 if (unlikely(!bpf_global_ma_set)) 9052 return -ENOMEM; 9053 9054 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { 9055 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 9056 return -EINVAL; 9057 } 9058 9059 ret_btf = env->prog->aux->btf; 9060 ret_btf_id = meta.arg_constant.value; 9061 9062 /* This may be NULL due to user not supplying a BTF */ 9063 if (!ret_btf) { 9064 verbose(env, "bpf_obj_new requires prog BTF\n"); 9065 return -EINVAL; 9066 } 9067 9068 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 9069 if (!ret_t || !__btf_type_is_struct(ret_t)) { 9070 verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); 9071 return -EINVAL; 9072 } 9073 9074 mark_reg_known_zero(env, regs, BPF_REG_0); 9075 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 9076 regs[BPF_REG_0].btf = ret_btf; 9077 regs[BPF_REG_0].btf_id = ret_btf_id; 9078 9079 env->insn_aux_data[insn_idx].obj_new_size = ret_t->size; 9080 env->insn_aux_data[insn_idx].kptr_struct_meta = 9081 btf_find_struct_meta(ret_btf, ret_btf_id); 9082 } else if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 9083 env->insn_aux_data[insn_idx].kptr_struct_meta = 9084 btf_find_struct_meta(meta.arg_obj_drop.btf, 9085 meta.arg_obj_drop.btf_id); 9086 } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 9087 meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 9088 struct btf_field *field = meta.arg_list_head.field; 9089 9090 mark_reg_known_zero(env, regs, BPF_REG_0); 9091 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 9092 regs[BPF_REG_0].btf = field->list_head.btf; 9093 regs[BPF_REG_0].btf_id = field->list_head.value_btf_id; 9094 regs[BPF_REG_0].off = field->list_head.node_offset; 9095 } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 9096 mark_reg_known_zero(env, regs, BPF_REG_0); 9097 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 9098 regs[BPF_REG_0].btf = desc_btf; 9099 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 9100 } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 9101 ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value); 9102 if (!ret_t || !btf_type_is_struct(ret_t)) { 9103 verbose(env, 9104 "kfunc bpf_rdonly_cast type ID argument must be of a struct\n"); 9105 return -EINVAL; 9106 } 9107 9108 mark_reg_known_zero(env, regs, BPF_REG_0); 9109 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 9110 regs[BPF_REG_0].btf = desc_btf; 9111 regs[BPF_REG_0].btf_id = meta.arg_constant.value; 9112 } else { 9113 verbose(env, "kernel function %s unhandled dynamic return type\n", 9114 meta.func_name); 9115 return -EFAULT; 9116 } 9117 } else if (!__btf_type_is_struct(ptr_type)) { 9118 if (!meta.r0_size) { 9119 ptr_type_name = btf_name_by_offset(desc_btf, 9120 ptr_type->name_off); 9121 verbose(env, 9122 "kernel function %s returns pointer type %s %s is not supported\n", 9123 func_name, 9124 btf_type_str(ptr_type), 9125 ptr_type_name); 9126 return -EINVAL; 9127 } 9128 9129 mark_reg_known_zero(env, regs, BPF_REG_0); 9130 regs[BPF_REG_0].type = PTR_TO_MEM; 9131 regs[BPF_REG_0].mem_size = meta.r0_size; 9132 9133 if (meta.r0_rdonly) 9134 regs[BPF_REG_0].type |= MEM_RDONLY; 9135 9136 /* Ensures we don't access the memory after a release_reference() */ 9137 if (meta.ref_obj_id) 9138 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 9139 } else { 9140 mark_reg_known_zero(env, regs, BPF_REG_0); 9141 regs[BPF_REG_0].btf = desc_btf; 9142 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 9143 regs[BPF_REG_0].btf_id = ptr_type_id; 9144 } 9145 9146 if (is_kfunc_ret_null(&meta)) { 9147 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 9148 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 9149 regs[BPF_REG_0].id = ++env->id_gen; 9150 } 9151 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 9152 if (is_kfunc_acquire(&meta)) { 9153 int id = acquire_reference_state(env, insn_idx); 9154 9155 if (id < 0) 9156 return id; 9157 if (is_kfunc_ret_null(&meta)) 9158 regs[BPF_REG_0].id = id; 9159 regs[BPF_REG_0].ref_obj_id = id; 9160 } 9161 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 9162 regs[BPF_REG_0].id = ++env->id_gen; 9163 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 9164 9165 nargs = btf_type_vlen(func_proto); 9166 args = (const struct btf_param *)(func_proto + 1); 9167 for (i = 0; i < nargs; i++) { 9168 u32 regno = i + 1; 9169 9170 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 9171 if (btf_type_is_ptr(t)) 9172 mark_btf_func_reg_size(env, regno, sizeof(void *)); 9173 else 9174 /* scalar. ensured by btf_check_kfunc_arg_match() */ 9175 mark_btf_func_reg_size(env, regno, t->size); 9176 } 9177 9178 return 0; 9179 } 9180 9181 static bool signed_add_overflows(s64 a, s64 b) 9182 { 9183 /* Do the add in u64, where overflow is well-defined */ 9184 s64 res = (s64)((u64)a + (u64)b); 9185 9186 if (b < 0) 9187 return res > a; 9188 return res < a; 9189 } 9190 9191 static bool signed_add32_overflows(s32 a, s32 b) 9192 { 9193 /* Do the add in u32, where overflow is well-defined */ 9194 s32 res = (s32)((u32)a + (u32)b); 9195 9196 if (b < 0) 9197 return res > a; 9198 return res < a; 9199 } 9200 9201 static bool signed_sub_overflows(s64 a, s64 b) 9202 { 9203 /* Do the sub in u64, where overflow is well-defined */ 9204 s64 res = (s64)((u64)a - (u64)b); 9205 9206 if (b < 0) 9207 return res < a; 9208 return res > a; 9209 } 9210 9211 static bool signed_sub32_overflows(s32 a, s32 b) 9212 { 9213 /* Do the sub in u32, where overflow is well-defined */ 9214 s32 res = (s32)((u32)a - (u32)b); 9215 9216 if (b < 0) 9217 return res < a; 9218 return res > a; 9219 } 9220 9221 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 9222 const struct bpf_reg_state *reg, 9223 enum bpf_reg_type type) 9224 { 9225 bool known = tnum_is_const(reg->var_off); 9226 s64 val = reg->var_off.value; 9227 s64 smin = reg->smin_value; 9228 9229 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 9230 verbose(env, "math between %s pointer and %lld is not allowed\n", 9231 reg_type_str(env, type), val); 9232 return false; 9233 } 9234 9235 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 9236 verbose(env, "%s pointer offset %d is not allowed\n", 9237 reg_type_str(env, type), reg->off); 9238 return false; 9239 } 9240 9241 if (smin == S64_MIN) { 9242 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 9243 reg_type_str(env, type)); 9244 return false; 9245 } 9246 9247 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 9248 verbose(env, "value %lld makes %s pointer be out of bounds\n", 9249 smin, reg_type_str(env, type)); 9250 return false; 9251 } 9252 9253 return true; 9254 } 9255 9256 enum { 9257 REASON_BOUNDS = -1, 9258 REASON_TYPE = -2, 9259 REASON_PATHS = -3, 9260 REASON_LIMIT = -4, 9261 REASON_STACK = -5, 9262 }; 9263 9264 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 9265 u32 *alu_limit, bool mask_to_left) 9266 { 9267 u32 max = 0, ptr_limit = 0; 9268 9269 switch (ptr_reg->type) { 9270 case PTR_TO_STACK: 9271 /* Offset 0 is out-of-bounds, but acceptable start for the 9272 * left direction, see BPF_REG_FP. Also, unknown scalar 9273 * offset where we would need to deal with min/max bounds is 9274 * currently prohibited for unprivileged. 9275 */ 9276 max = MAX_BPF_STACK + mask_to_left; 9277 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 9278 break; 9279 case PTR_TO_MAP_VALUE: 9280 max = ptr_reg->map_ptr->value_size; 9281 ptr_limit = (mask_to_left ? 9282 ptr_reg->smin_value : 9283 ptr_reg->umax_value) + ptr_reg->off; 9284 break; 9285 default: 9286 return REASON_TYPE; 9287 } 9288 9289 if (ptr_limit >= max) 9290 return REASON_LIMIT; 9291 *alu_limit = ptr_limit; 9292 return 0; 9293 } 9294 9295 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 9296 const struct bpf_insn *insn) 9297 { 9298 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 9299 } 9300 9301 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 9302 u32 alu_state, u32 alu_limit) 9303 { 9304 /* If we arrived here from different branches with different 9305 * state or limits to sanitize, then this won't work. 9306 */ 9307 if (aux->alu_state && 9308 (aux->alu_state != alu_state || 9309 aux->alu_limit != alu_limit)) 9310 return REASON_PATHS; 9311 9312 /* Corresponding fixup done in do_misc_fixups(). */ 9313 aux->alu_state = alu_state; 9314 aux->alu_limit = alu_limit; 9315 return 0; 9316 } 9317 9318 static int sanitize_val_alu(struct bpf_verifier_env *env, 9319 struct bpf_insn *insn) 9320 { 9321 struct bpf_insn_aux_data *aux = cur_aux(env); 9322 9323 if (can_skip_alu_sanitation(env, insn)) 9324 return 0; 9325 9326 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 9327 } 9328 9329 static bool sanitize_needed(u8 opcode) 9330 { 9331 return opcode == BPF_ADD || opcode == BPF_SUB; 9332 } 9333 9334 struct bpf_sanitize_info { 9335 struct bpf_insn_aux_data aux; 9336 bool mask_to_left; 9337 }; 9338 9339 static struct bpf_verifier_state * 9340 sanitize_speculative_path(struct bpf_verifier_env *env, 9341 const struct bpf_insn *insn, 9342 u32 next_idx, u32 curr_idx) 9343 { 9344 struct bpf_verifier_state *branch; 9345 struct bpf_reg_state *regs; 9346 9347 branch = push_stack(env, next_idx, curr_idx, true); 9348 if (branch && insn) { 9349 regs = branch->frame[branch->curframe]->regs; 9350 if (BPF_SRC(insn->code) == BPF_K) { 9351 mark_reg_unknown(env, regs, insn->dst_reg); 9352 } else if (BPF_SRC(insn->code) == BPF_X) { 9353 mark_reg_unknown(env, regs, insn->dst_reg); 9354 mark_reg_unknown(env, regs, insn->src_reg); 9355 } 9356 } 9357 return branch; 9358 } 9359 9360 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 9361 struct bpf_insn *insn, 9362 const struct bpf_reg_state *ptr_reg, 9363 const struct bpf_reg_state *off_reg, 9364 struct bpf_reg_state *dst_reg, 9365 struct bpf_sanitize_info *info, 9366 const bool commit_window) 9367 { 9368 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 9369 struct bpf_verifier_state *vstate = env->cur_state; 9370 bool off_is_imm = tnum_is_const(off_reg->var_off); 9371 bool off_is_neg = off_reg->smin_value < 0; 9372 bool ptr_is_dst_reg = ptr_reg == dst_reg; 9373 u8 opcode = BPF_OP(insn->code); 9374 u32 alu_state, alu_limit; 9375 struct bpf_reg_state tmp; 9376 bool ret; 9377 int err; 9378 9379 if (can_skip_alu_sanitation(env, insn)) 9380 return 0; 9381 9382 /* We already marked aux for masking from non-speculative 9383 * paths, thus we got here in the first place. We only care 9384 * to explore bad access from here. 9385 */ 9386 if (vstate->speculative) 9387 goto do_sim; 9388 9389 if (!commit_window) { 9390 if (!tnum_is_const(off_reg->var_off) && 9391 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 9392 return REASON_BOUNDS; 9393 9394 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 9395 (opcode == BPF_SUB && !off_is_neg); 9396 } 9397 9398 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 9399 if (err < 0) 9400 return err; 9401 9402 if (commit_window) { 9403 /* In commit phase we narrow the masking window based on 9404 * the observed pointer move after the simulated operation. 9405 */ 9406 alu_state = info->aux.alu_state; 9407 alu_limit = abs(info->aux.alu_limit - alu_limit); 9408 } else { 9409 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 9410 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 9411 alu_state |= ptr_is_dst_reg ? 9412 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 9413 9414 /* Limit pruning on unknown scalars to enable deep search for 9415 * potential masking differences from other program paths. 9416 */ 9417 if (!off_is_imm) 9418 env->explore_alu_limits = true; 9419 } 9420 9421 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 9422 if (err < 0) 9423 return err; 9424 do_sim: 9425 /* If we're in commit phase, we're done here given we already 9426 * pushed the truncated dst_reg into the speculative verification 9427 * stack. 9428 * 9429 * Also, when register is a known constant, we rewrite register-based 9430 * operation to immediate-based, and thus do not need masking (and as 9431 * a consequence, do not need to simulate the zero-truncation either). 9432 */ 9433 if (commit_window || off_is_imm) 9434 return 0; 9435 9436 /* Simulate and find potential out-of-bounds access under 9437 * speculative execution from truncation as a result of 9438 * masking when off was not within expected range. If off 9439 * sits in dst, then we temporarily need to move ptr there 9440 * to simulate dst (== 0) +/-= ptr. Needed, for example, 9441 * for cases where we use K-based arithmetic in one direction 9442 * and truncated reg-based in the other in order to explore 9443 * bad access. 9444 */ 9445 if (!ptr_is_dst_reg) { 9446 tmp = *dst_reg; 9447 *dst_reg = *ptr_reg; 9448 } 9449 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 9450 env->insn_idx); 9451 if (!ptr_is_dst_reg && ret) 9452 *dst_reg = tmp; 9453 return !ret ? REASON_STACK : 0; 9454 } 9455 9456 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 9457 { 9458 struct bpf_verifier_state *vstate = env->cur_state; 9459 9460 /* If we simulate paths under speculation, we don't update the 9461 * insn as 'seen' such that when we verify unreachable paths in 9462 * the non-speculative domain, sanitize_dead_code() can still 9463 * rewrite/sanitize them. 9464 */ 9465 if (!vstate->speculative) 9466 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 9467 } 9468 9469 static int sanitize_err(struct bpf_verifier_env *env, 9470 const struct bpf_insn *insn, int reason, 9471 const struct bpf_reg_state *off_reg, 9472 const struct bpf_reg_state *dst_reg) 9473 { 9474 static const char *err = "pointer arithmetic with it prohibited for !root"; 9475 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 9476 u32 dst = insn->dst_reg, src = insn->src_reg; 9477 9478 switch (reason) { 9479 case REASON_BOUNDS: 9480 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 9481 off_reg == dst_reg ? dst : src, err); 9482 break; 9483 case REASON_TYPE: 9484 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 9485 off_reg == dst_reg ? src : dst, err); 9486 break; 9487 case REASON_PATHS: 9488 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 9489 dst, op, err); 9490 break; 9491 case REASON_LIMIT: 9492 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 9493 dst, op, err); 9494 break; 9495 case REASON_STACK: 9496 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 9497 dst, err); 9498 break; 9499 default: 9500 verbose(env, "verifier internal error: unknown reason (%d)\n", 9501 reason); 9502 break; 9503 } 9504 9505 return -EACCES; 9506 } 9507 9508 /* check that stack access falls within stack limits and that 'reg' doesn't 9509 * have a variable offset. 9510 * 9511 * Variable offset is prohibited for unprivileged mode for simplicity since it 9512 * requires corresponding support in Spectre masking for stack ALU. See also 9513 * retrieve_ptr_limit(). 9514 * 9515 * 9516 * 'off' includes 'reg->off'. 9517 */ 9518 static int check_stack_access_for_ptr_arithmetic( 9519 struct bpf_verifier_env *env, 9520 int regno, 9521 const struct bpf_reg_state *reg, 9522 int off) 9523 { 9524 if (!tnum_is_const(reg->var_off)) { 9525 char tn_buf[48]; 9526 9527 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 9528 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 9529 regno, tn_buf, off); 9530 return -EACCES; 9531 } 9532 9533 if (off >= 0 || off < -MAX_BPF_STACK) { 9534 verbose(env, "R%d stack pointer arithmetic goes out of range, " 9535 "prohibited for !root; off=%d\n", regno, off); 9536 return -EACCES; 9537 } 9538 9539 return 0; 9540 } 9541 9542 static int sanitize_check_bounds(struct bpf_verifier_env *env, 9543 const struct bpf_insn *insn, 9544 const struct bpf_reg_state *dst_reg) 9545 { 9546 u32 dst = insn->dst_reg; 9547 9548 /* For unprivileged we require that resulting offset must be in bounds 9549 * in order to be able to sanitize access later on. 9550 */ 9551 if (env->bypass_spec_v1) 9552 return 0; 9553 9554 switch (dst_reg->type) { 9555 case PTR_TO_STACK: 9556 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 9557 dst_reg->off + dst_reg->var_off.value)) 9558 return -EACCES; 9559 break; 9560 case PTR_TO_MAP_VALUE: 9561 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 9562 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 9563 "prohibited for !root\n", dst); 9564 return -EACCES; 9565 } 9566 break; 9567 default: 9568 break; 9569 } 9570 9571 return 0; 9572 } 9573 9574 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 9575 * Caller should also handle BPF_MOV case separately. 9576 * If we return -EACCES, caller may want to try again treating pointer as a 9577 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 9578 */ 9579 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 9580 struct bpf_insn *insn, 9581 const struct bpf_reg_state *ptr_reg, 9582 const struct bpf_reg_state *off_reg) 9583 { 9584 struct bpf_verifier_state *vstate = env->cur_state; 9585 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 9586 struct bpf_reg_state *regs = state->regs, *dst_reg; 9587 bool known = tnum_is_const(off_reg->var_off); 9588 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 9589 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 9590 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 9591 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 9592 struct bpf_sanitize_info info = {}; 9593 u8 opcode = BPF_OP(insn->code); 9594 u32 dst = insn->dst_reg; 9595 int ret; 9596 9597 dst_reg = ®s[dst]; 9598 9599 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 9600 smin_val > smax_val || umin_val > umax_val) { 9601 /* Taint dst register if offset had invalid bounds derived from 9602 * e.g. dead branches. 9603 */ 9604 __mark_reg_unknown(env, dst_reg); 9605 return 0; 9606 } 9607 9608 if (BPF_CLASS(insn->code) != BPF_ALU64) { 9609 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 9610 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 9611 __mark_reg_unknown(env, dst_reg); 9612 return 0; 9613 } 9614 9615 verbose(env, 9616 "R%d 32-bit pointer arithmetic prohibited\n", 9617 dst); 9618 return -EACCES; 9619 } 9620 9621 if (ptr_reg->type & PTR_MAYBE_NULL) { 9622 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 9623 dst, reg_type_str(env, ptr_reg->type)); 9624 return -EACCES; 9625 } 9626 9627 switch (base_type(ptr_reg->type)) { 9628 case CONST_PTR_TO_MAP: 9629 /* smin_val represents the known value */ 9630 if (known && smin_val == 0 && opcode == BPF_ADD) 9631 break; 9632 fallthrough; 9633 case PTR_TO_PACKET_END: 9634 case PTR_TO_SOCKET: 9635 case PTR_TO_SOCK_COMMON: 9636 case PTR_TO_TCP_SOCK: 9637 case PTR_TO_XDP_SOCK: 9638 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 9639 dst, reg_type_str(env, ptr_reg->type)); 9640 return -EACCES; 9641 default: 9642 break; 9643 } 9644 9645 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 9646 * The id may be overwritten later if we create a new variable offset. 9647 */ 9648 dst_reg->type = ptr_reg->type; 9649 dst_reg->id = ptr_reg->id; 9650 9651 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 9652 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 9653 return -EINVAL; 9654 9655 /* pointer types do not carry 32-bit bounds at the moment. */ 9656 __mark_reg32_unbounded(dst_reg); 9657 9658 if (sanitize_needed(opcode)) { 9659 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 9660 &info, false); 9661 if (ret < 0) 9662 return sanitize_err(env, insn, ret, off_reg, dst_reg); 9663 } 9664 9665 switch (opcode) { 9666 case BPF_ADD: 9667 /* We can take a fixed offset as long as it doesn't overflow 9668 * the s32 'off' field 9669 */ 9670 if (known && (ptr_reg->off + smin_val == 9671 (s64)(s32)(ptr_reg->off + smin_val))) { 9672 /* pointer += K. Accumulate it into fixed offset */ 9673 dst_reg->smin_value = smin_ptr; 9674 dst_reg->smax_value = smax_ptr; 9675 dst_reg->umin_value = umin_ptr; 9676 dst_reg->umax_value = umax_ptr; 9677 dst_reg->var_off = ptr_reg->var_off; 9678 dst_reg->off = ptr_reg->off + smin_val; 9679 dst_reg->raw = ptr_reg->raw; 9680 break; 9681 } 9682 /* A new variable offset is created. Note that off_reg->off 9683 * == 0, since it's a scalar. 9684 * dst_reg gets the pointer type and since some positive 9685 * integer value was added to the pointer, give it a new 'id' 9686 * if it's a PTR_TO_PACKET. 9687 * this creates a new 'base' pointer, off_reg (variable) gets 9688 * added into the variable offset, and we copy the fixed offset 9689 * from ptr_reg. 9690 */ 9691 if (signed_add_overflows(smin_ptr, smin_val) || 9692 signed_add_overflows(smax_ptr, smax_val)) { 9693 dst_reg->smin_value = S64_MIN; 9694 dst_reg->smax_value = S64_MAX; 9695 } else { 9696 dst_reg->smin_value = smin_ptr + smin_val; 9697 dst_reg->smax_value = smax_ptr + smax_val; 9698 } 9699 if (umin_ptr + umin_val < umin_ptr || 9700 umax_ptr + umax_val < umax_ptr) { 9701 dst_reg->umin_value = 0; 9702 dst_reg->umax_value = U64_MAX; 9703 } else { 9704 dst_reg->umin_value = umin_ptr + umin_val; 9705 dst_reg->umax_value = umax_ptr + umax_val; 9706 } 9707 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 9708 dst_reg->off = ptr_reg->off; 9709 dst_reg->raw = ptr_reg->raw; 9710 if (reg_is_pkt_pointer(ptr_reg)) { 9711 dst_reg->id = ++env->id_gen; 9712 /* something was added to pkt_ptr, set range to zero */ 9713 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 9714 } 9715 break; 9716 case BPF_SUB: 9717 if (dst_reg == off_reg) { 9718 /* scalar -= pointer. Creates an unknown scalar */ 9719 verbose(env, "R%d tried to subtract pointer from scalar\n", 9720 dst); 9721 return -EACCES; 9722 } 9723 /* We don't allow subtraction from FP, because (according to 9724 * test_verifier.c test "invalid fp arithmetic", JITs might not 9725 * be able to deal with it. 9726 */ 9727 if (ptr_reg->type == PTR_TO_STACK) { 9728 verbose(env, "R%d subtraction from stack pointer prohibited\n", 9729 dst); 9730 return -EACCES; 9731 } 9732 if (known && (ptr_reg->off - smin_val == 9733 (s64)(s32)(ptr_reg->off - smin_val))) { 9734 /* pointer -= K. Subtract it from fixed offset */ 9735 dst_reg->smin_value = smin_ptr; 9736 dst_reg->smax_value = smax_ptr; 9737 dst_reg->umin_value = umin_ptr; 9738 dst_reg->umax_value = umax_ptr; 9739 dst_reg->var_off = ptr_reg->var_off; 9740 dst_reg->id = ptr_reg->id; 9741 dst_reg->off = ptr_reg->off - smin_val; 9742 dst_reg->raw = ptr_reg->raw; 9743 break; 9744 } 9745 /* A new variable offset is created. If the subtrahend is known 9746 * nonnegative, then any reg->range we had before is still good. 9747 */ 9748 if (signed_sub_overflows(smin_ptr, smax_val) || 9749 signed_sub_overflows(smax_ptr, smin_val)) { 9750 /* Overflow possible, we know nothing */ 9751 dst_reg->smin_value = S64_MIN; 9752 dst_reg->smax_value = S64_MAX; 9753 } else { 9754 dst_reg->smin_value = smin_ptr - smax_val; 9755 dst_reg->smax_value = smax_ptr - smin_val; 9756 } 9757 if (umin_ptr < umax_val) { 9758 /* Overflow possible, we know nothing */ 9759 dst_reg->umin_value = 0; 9760 dst_reg->umax_value = U64_MAX; 9761 } else { 9762 /* Cannot overflow (as long as bounds are consistent) */ 9763 dst_reg->umin_value = umin_ptr - umax_val; 9764 dst_reg->umax_value = umax_ptr - umin_val; 9765 } 9766 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 9767 dst_reg->off = ptr_reg->off; 9768 dst_reg->raw = ptr_reg->raw; 9769 if (reg_is_pkt_pointer(ptr_reg)) { 9770 dst_reg->id = ++env->id_gen; 9771 /* something was added to pkt_ptr, set range to zero */ 9772 if (smin_val < 0) 9773 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 9774 } 9775 break; 9776 case BPF_AND: 9777 case BPF_OR: 9778 case BPF_XOR: 9779 /* bitwise ops on pointers are troublesome, prohibit. */ 9780 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 9781 dst, bpf_alu_string[opcode >> 4]); 9782 return -EACCES; 9783 default: 9784 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 9785 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 9786 dst, bpf_alu_string[opcode >> 4]); 9787 return -EACCES; 9788 } 9789 9790 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 9791 return -EINVAL; 9792 reg_bounds_sync(dst_reg); 9793 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 9794 return -EACCES; 9795 if (sanitize_needed(opcode)) { 9796 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 9797 &info, true); 9798 if (ret < 0) 9799 return sanitize_err(env, insn, ret, off_reg, dst_reg); 9800 } 9801 9802 return 0; 9803 } 9804 9805 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 9806 struct bpf_reg_state *src_reg) 9807 { 9808 s32 smin_val = src_reg->s32_min_value; 9809 s32 smax_val = src_reg->s32_max_value; 9810 u32 umin_val = src_reg->u32_min_value; 9811 u32 umax_val = src_reg->u32_max_value; 9812 9813 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 9814 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 9815 dst_reg->s32_min_value = S32_MIN; 9816 dst_reg->s32_max_value = S32_MAX; 9817 } else { 9818 dst_reg->s32_min_value += smin_val; 9819 dst_reg->s32_max_value += smax_val; 9820 } 9821 if (dst_reg->u32_min_value + umin_val < umin_val || 9822 dst_reg->u32_max_value + umax_val < umax_val) { 9823 dst_reg->u32_min_value = 0; 9824 dst_reg->u32_max_value = U32_MAX; 9825 } else { 9826 dst_reg->u32_min_value += umin_val; 9827 dst_reg->u32_max_value += umax_val; 9828 } 9829 } 9830 9831 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 9832 struct bpf_reg_state *src_reg) 9833 { 9834 s64 smin_val = src_reg->smin_value; 9835 s64 smax_val = src_reg->smax_value; 9836 u64 umin_val = src_reg->umin_value; 9837 u64 umax_val = src_reg->umax_value; 9838 9839 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 9840 signed_add_overflows(dst_reg->smax_value, smax_val)) { 9841 dst_reg->smin_value = S64_MIN; 9842 dst_reg->smax_value = S64_MAX; 9843 } else { 9844 dst_reg->smin_value += smin_val; 9845 dst_reg->smax_value += smax_val; 9846 } 9847 if (dst_reg->umin_value + umin_val < umin_val || 9848 dst_reg->umax_value + umax_val < umax_val) { 9849 dst_reg->umin_value = 0; 9850 dst_reg->umax_value = U64_MAX; 9851 } else { 9852 dst_reg->umin_value += umin_val; 9853 dst_reg->umax_value += umax_val; 9854 } 9855 } 9856 9857 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 9858 struct bpf_reg_state *src_reg) 9859 { 9860 s32 smin_val = src_reg->s32_min_value; 9861 s32 smax_val = src_reg->s32_max_value; 9862 u32 umin_val = src_reg->u32_min_value; 9863 u32 umax_val = src_reg->u32_max_value; 9864 9865 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 9866 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 9867 /* Overflow possible, we know nothing */ 9868 dst_reg->s32_min_value = S32_MIN; 9869 dst_reg->s32_max_value = S32_MAX; 9870 } else { 9871 dst_reg->s32_min_value -= smax_val; 9872 dst_reg->s32_max_value -= smin_val; 9873 } 9874 if (dst_reg->u32_min_value < umax_val) { 9875 /* Overflow possible, we know nothing */ 9876 dst_reg->u32_min_value = 0; 9877 dst_reg->u32_max_value = U32_MAX; 9878 } else { 9879 /* Cannot overflow (as long as bounds are consistent) */ 9880 dst_reg->u32_min_value -= umax_val; 9881 dst_reg->u32_max_value -= umin_val; 9882 } 9883 } 9884 9885 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 9886 struct bpf_reg_state *src_reg) 9887 { 9888 s64 smin_val = src_reg->smin_value; 9889 s64 smax_val = src_reg->smax_value; 9890 u64 umin_val = src_reg->umin_value; 9891 u64 umax_val = src_reg->umax_value; 9892 9893 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 9894 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 9895 /* Overflow possible, we know nothing */ 9896 dst_reg->smin_value = S64_MIN; 9897 dst_reg->smax_value = S64_MAX; 9898 } else { 9899 dst_reg->smin_value -= smax_val; 9900 dst_reg->smax_value -= smin_val; 9901 } 9902 if (dst_reg->umin_value < umax_val) { 9903 /* Overflow possible, we know nothing */ 9904 dst_reg->umin_value = 0; 9905 dst_reg->umax_value = U64_MAX; 9906 } else { 9907 /* Cannot overflow (as long as bounds are consistent) */ 9908 dst_reg->umin_value -= umax_val; 9909 dst_reg->umax_value -= umin_val; 9910 } 9911 } 9912 9913 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 9914 struct bpf_reg_state *src_reg) 9915 { 9916 s32 smin_val = src_reg->s32_min_value; 9917 u32 umin_val = src_reg->u32_min_value; 9918 u32 umax_val = src_reg->u32_max_value; 9919 9920 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 9921 /* Ain't nobody got time to multiply that sign */ 9922 __mark_reg32_unbounded(dst_reg); 9923 return; 9924 } 9925 /* Both values are positive, so we can work with unsigned and 9926 * copy the result to signed (unless it exceeds S32_MAX). 9927 */ 9928 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 9929 /* Potential overflow, we know nothing */ 9930 __mark_reg32_unbounded(dst_reg); 9931 return; 9932 } 9933 dst_reg->u32_min_value *= umin_val; 9934 dst_reg->u32_max_value *= umax_val; 9935 if (dst_reg->u32_max_value > S32_MAX) { 9936 /* Overflow possible, we know nothing */ 9937 dst_reg->s32_min_value = S32_MIN; 9938 dst_reg->s32_max_value = S32_MAX; 9939 } else { 9940 dst_reg->s32_min_value = dst_reg->u32_min_value; 9941 dst_reg->s32_max_value = dst_reg->u32_max_value; 9942 } 9943 } 9944 9945 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 9946 struct bpf_reg_state *src_reg) 9947 { 9948 s64 smin_val = src_reg->smin_value; 9949 u64 umin_val = src_reg->umin_value; 9950 u64 umax_val = src_reg->umax_value; 9951 9952 if (smin_val < 0 || dst_reg->smin_value < 0) { 9953 /* Ain't nobody got time to multiply that sign */ 9954 __mark_reg64_unbounded(dst_reg); 9955 return; 9956 } 9957 /* Both values are positive, so we can work with unsigned and 9958 * copy the result to signed (unless it exceeds S64_MAX). 9959 */ 9960 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 9961 /* Potential overflow, we know nothing */ 9962 __mark_reg64_unbounded(dst_reg); 9963 return; 9964 } 9965 dst_reg->umin_value *= umin_val; 9966 dst_reg->umax_value *= umax_val; 9967 if (dst_reg->umax_value > S64_MAX) { 9968 /* Overflow possible, we know nothing */ 9969 dst_reg->smin_value = S64_MIN; 9970 dst_reg->smax_value = S64_MAX; 9971 } else { 9972 dst_reg->smin_value = dst_reg->umin_value; 9973 dst_reg->smax_value = dst_reg->umax_value; 9974 } 9975 } 9976 9977 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 9978 struct bpf_reg_state *src_reg) 9979 { 9980 bool src_known = tnum_subreg_is_const(src_reg->var_off); 9981 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 9982 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 9983 s32 smin_val = src_reg->s32_min_value; 9984 u32 umax_val = src_reg->u32_max_value; 9985 9986 if (src_known && dst_known) { 9987 __mark_reg32_known(dst_reg, var32_off.value); 9988 return; 9989 } 9990 9991 /* We get our minimum from the var_off, since that's inherently 9992 * bitwise. Our maximum is the minimum of the operands' maxima. 9993 */ 9994 dst_reg->u32_min_value = var32_off.value; 9995 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 9996 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 9997 /* Lose signed bounds when ANDing negative numbers, 9998 * ain't nobody got time for that. 9999 */ 10000 dst_reg->s32_min_value = S32_MIN; 10001 dst_reg->s32_max_value = S32_MAX; 10002 } else { 10003 /* ANDing two positives gives a positive, so safe to 10004 * cast result into s64. 10005 */ 10006 dst_reg->s32_min_value = dst_reg->u32_min_value; 10007 dst_reg->s32_max_value = dst_reg->u32_max_value; 10008 } 10009 } 10010 10011 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 10012 struct bpf_reg_state *src_reg) 10013 { 10014 bool src_known = tnum_is_const(src_reg->var_off); 10015 bool dst_known = tnum_is_const(dst_reg->var_off); 10016 s64 smin_val = src_reg->smin_value; 10017 u64 umax_val = src_reg->umax_value; 10018 10019 if (src_known && dst_known) { 10020 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10021 return; 10022 } 10023 10024 /* We get our minimum from the var_off, since that's inherently 10025 * bitwise. Our maximum is the minimum of the operands' maxima. 10026 */ 10027 dst_reg->umin_value = dst_reg->var_off.value; 10028 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 10029 if (dst_reg->smin_value < 0 || smin_val < 0) { 10030 /* Lose signed bounds when ANDing negative numbers, 10031 * ain't nobody got time for that. 10032 */ 10033 dst_reg->smin_value = S64_MIN; 10034 dst_reg->smax_value = S64_MAX; 10035 } else { 10036 /* ANDing two positives gives a positive, so safe to 10037 * cast result into s64. 10038 */ 10039 dst_reg->smin_value = dst_reg->umin_value; 10040 dst_reg->smax_value = dst_reg->umax_value; 10041 } 10042 /* We may learn something more from the var_off */ 10043 __update_reg_bounds(dst_reg); 10044 } 10045 10046 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 10047 struct bpf_reg_state *src_reg) 10048 { 10049 bool src_known = tnum_subreg_is_const(src_reg->var_off); 10050 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 10051 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 10052 s32 smin_val = src_reg->s32_min_value; 10053 u32 umin_val = src_reg->u32_min_value; 10054 10055 if (src_known && dst_known) { 10056 __mark_reg32_known(dst_reg, var32_off.value); 10057 return; 10058 } 10059 10060 /* We get our maximum from the var_off, and our minimum is the 10061 * maximum of the operands' minima 10062 */ 10063 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 10064 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 10065 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 10066 /* Lose signed bounds when ORing negative numbers, 10067 * ain't nobody got time for that. 10068 */ 10069 dst_reg->s32_min_value = S32_MIN; 10070 dst_reg->s32_max_value = S32_MAX; 10071 } else { 10072 /* ORing two positives gives a positive, so safe to 10073 * cast result into s64. 10074 */ 10075 dst_reg->s32_min_value = dst_reg->u32_min_value; 10076 dst_reg->s32_max_value = dst_reg->u32_max_value; 10077 } 10078 } 10079 10080 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 10081 struct bpf_reg_state *src_reg) 10082 { 10083 bool src_known = tnum_is_const(src_reg->var_off); 10084 bool dst_known = tnum_is_const(dst_reg->var_off); 10085 s64 smin_val = src_reg->smin_value; 10086 u64 umin_val = src_reg->umin_value; 10087 10088 if (src_known && dst_known) { 10089 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10090 return; 10091 } 10092 10093 /* We get our maximum from the var_off, and our minimum is the 10094 * maximum of the operands' minima 10095 */ 10096 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 10097 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 10098 if (dst_reg->smin_value < 0 || smin_val < 0) { 10099 /* Lose signed bounds when ORing negative numbers, 10100 * ain't nobody got time for that. 10101 */ 10102 dst_reg->smin_value = S64_MIN; 10103 dst_reg->smax_value = S64_MAX; 10104 } else { 10105 /* ORing two positives gives a positive, so safe to 10106 * cast result into s64. 10107 */ 10108 dst_reg->smin_value = dst_reg->umin_value; 10109 dst_reg->smax_value = dst_reg->umax_value; 10110 } 10111 /* We may learn something more from the var_off */ 10112 __update_reg_bounds(dst_reg); 10113 } 10114 10115 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 10116 struct bpf_reg_state *src_reg) 10117 { 10118 bool src_known = tnum_subreg_is_const(src_reg->var_off); 10119 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 10120 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 10121 s32 smin_val = src_reg->s32_min_value; 10122 10123 if (src_known && dst_known) { 10124 __mark_reg32_known(dst_reg, var32_off.value); 10125 return; 10126 } 10127 10128 /* We get both minimum and maximum from the var32_off. */ 10129 dst_reg->u32_min_value = var32_off.value; 10130 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 10131 10132 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 10133 /* XORing two positive sign numbers gives a positive, 10134 * so safe to cast u32 result into s32. 10135 */ 10136 dst_reg->s32_min_value = dst_reg->u32_min_value; 10137 dst_reg->s32_max_value = dst_reg->u32_max_value; 10138 } else { 10139 dst_reg->s32_min_value = S32_MIN; 10140 dst_reg->s32_max_value = S32_MAX; 10141 } 10142 } 10143 10144 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 10145 struct bpf_reg_state *src_reg) 10146 { 10147 bool src_known = tnum_is_const(src_reg->var_off); 10148 bool dst_known = tnum_is_const(dst_reg->var_off); 10149 s64 smin_val = src_reg->smin_value; 10150 10151 if (src_known && dst_known) { 10152 /* dst_reg->var_off.value has been updated earlier */ 10153 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10154 return; 10155 } 10156 10157 /* We get both minimum and maximum from the var_off. */ 10158 dst_reg->umin_value = dst_reg->var_off.value; 10159 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 10160 10161 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 10162 /* XORing two positive sign numbers gives a positive, 10163 * so safe to cast u64 result into s64. 10164 */ 10165 dst_reg->smin_value = dst_reg->umin_value; 10166 dst_reg->smax_value = dst_reg->umax_value; 10167 } else { 10168 dst_reg->smin_value = S64_MIN; 10169 dst_reg->smax_value = S64_MAX; 10170 } 10171 10172 __update_reg_bounds(dst_reg); 10173 } 10174 10175 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 10176 u64 umin_val, u64 umax_val) 10177 { 10178 /* We lose all sign bit information (except what we can pick 10179 * up from var_off) 10180 */ 10181 dst_reg->s32_min_value = S32_MIN; 10182 dst_reg->s32_max_value = S32_MAX; 10183 /* If we might shift our top bit out, then we know nothing */ 10184 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 10185 dst_reg->u32_min_value = 0; 10186 dst_reg->u32_max_value = U32_MAX; 10187 } else { 10188 dst_reg->u32_min_value <<= umin_val; 10189 dst_reg->u32_max_value <<= umax_val; 10190 } 10191 } 10192 10193 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 10194 struct bpf_reg_state *src_reg) 10195 { 10196 u32 umax_val = src_reg->u32_max_value; 10197 u32 umin_val = src_reg->u32_min_value; 10198 /* u32 alu operation will zext upper bits */ 10199 struct tnum subreg = tnum_subreg(dst_reg->var_off); 10200 10201 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 10202 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 10203 /* Not required but being careful mark reg64 bounds as unknown so 10204 * that we are forced to pick them up from tnum and zext later and 10205 * if some path skips this step we are still safe. 10206 */ 10207 __mark_reg64_unbounded(dst_reg); 10208 __update_reg32_bounds(dst_reg); 10209 } 10210 10211 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 10212 u64 umin_val, u64 umax_val) 10213 { 10214 /* Special case <<32 because it is a common compiler pattern to sign 10215 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 10216 * positive we know this shift will also be positive so we can track 10217 * bounds correctly. Otherwise we lose all sign bit information except 10218 * what we can pick up from var_off. Perhaps we can generalize this 10219 * later to shifts of any length. 10220 */ 10221 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 10222 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 10223 else 10224 dst_reg->smax_value = S64_MAX; 10225 10226 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 10227 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 10228 else 10229 dst_reg->smin_value = S64_MIN; 10230 10231 /* If we might shift our top bit out, then we know nothing */ 10232 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 10233 dst_reg->umin_value = 0; 10234 dst_reg->umax_value = U64_MAX; 10235 } else { 10236 dst_reg->umin_value <<= umin_val; 10237 dst_reg->umax_value <<= umax_val; 10238 } 10239 } 10240 10241 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 10242 struct bpf_reg_state *src_reg) 10243 { 10244 u64 umax_val = src_reg->umax_value; 10245 u64 umin_val = src_reg->umin_value; 10246 10247 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 10248 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 10249 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 10250 10251 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 10252 /* We may learn something more from the var_off */ 10253 __update_reg_bounds(dst_reg); 10254 } 10255 10256 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 10257 struct bpf_reg_state *src_reg) 10258 { 10259 struct tnum subreg = tnum_subreg(dst_reg->var_off); 10260 u32 umax_val = src_reg->u32_max_value; 10261 u32 umin_val = src_reg->u32_min_value; 10262 10263 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 10264 * be negative, then either: 10265 * 1) src_reg might be zero, so the sign bit of the result is 10266 * unknown, so we lose our signed bounds 10267 * 2) it's known negative, thus the unsigned bounds capture the 10268 * signed bounds 10269 * 3) the signed bounds cross zero, so they tell us nothing 10270 * about the result 10271 * If the value in dst_reg is known nonnegative, then again the 10272 * unsigned bounds capture the signed bounds. 10273 * Thus, in all cases it suffices to blow away our signed bounds 10274 * and rely on inferring new ones from the unsigned bounds and 10275 * var_off of the result. 10276 */ 10277 dst_reg->s32_min_value = S32_MIN; 10278 dst_reg->s32_max_value = S32_MAX; 10279 10280 dst_reg->var_off = tnum_rshift(subreg, umin_val); 10281 dst_reg->u32_min_value >>= umax_val; 10282 dst_reg->u32_max_value >>= umin_val; 10283 10284 __mark_reg64_unbounded(dst_reg); 10285 __update_reg32_bounds(dst_reg); 10286 } 10287 10288 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 10289 struct bpf_reg_state *src_reg) 10290 { 10291 u64 umax_val = src_reg->umax_value; 10292 u64 umin_val = src_reg->umin_value; 10293 10294 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 10295 * be negative, then either: 10296 * 1) src_reg might be zero, so the sign bit of the result is 10297 * unknown, so we lose our signed bounds 10298 * 2) it's known negative, thus the unsigned bounds capture the 10299 * signed bounds 10300 * 3) the signed bounds cross zero, so they tell us nothing 10301 * about the result 10302 * If the value in dst_reg is known nonnegative, then again the 10303 * unsigned bounds capture the signed bounds. 10304 * Thus, in all cases it suffices to blow away our signed bounds 10305 * and rely on inferring new ones from the unsigned bounds and 10306 * var_off of the result. 10307 */ 10308 dst_reg->smin_value = S64_MIN; 10309 dst_reg->smax_value = S64_MAX; 10310 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 10311 dst_reg->umin_value >>= umax_val; 10312 dst_reg->umax_value >>= umin_val; 10313 10314 /* Its not easy to operate on alu32 bounds here because it depends 10315 * on bits being shifted in. Take easy way out and mark unbounded 10316 * so we can recalculate later from tnum. 10317 */ 10318 __mark_reg32_unbounded(dst_reg); 10319 __update_reg_bounds(dst_reg); 10320 } 10321 10322 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 10323 struct bpf_reg_state *src_reg) 10324 { 10325 u64 umin_val = src_reg->u32_min_value; 10326 10327 /* Upon reaching here, src_known is true and 10328 * umax_val is equal to umin_val. 10329 */ 10330 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 10331 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 10332 10333 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 10334 10335 /* blow away the dst_reg umin_value/umax_value and rely on 10336 * dst_reg var_off to refine the result. 10337 */ 10338 dst_reg->u32_min_value = 0; 10339 dst_reg->u32_max_value = U32_MAX; 10340 10341 __mark_reg64_unbounded(dst_reg); 10342 __update_reg32_bounds(dst_reg); 10343 } 10344 10345 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 10346 struct bpf_reg_state *src_reg) 10347 { 10348 u64 umin_val = src_reg->umin_value; 10349 10350 /* Upon reaching here, src_known is true and umax_val is equal 10351 * to umin_val. 10352 */ 10353 dst_reg->smin_value >>= umin_val; 10354 dst_reg->smax_value >>= umin_val; 10355 10356 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 10357 10358 /* blow away the dst_reg umin_value/umax_value and rely on 10359 * dst_reg var_off to refine the result. 10360 */ 10361 dst_reg->umin_value = 0; 10362 dst_reg->umax_value = U64_MAX; 10363 10364 /* Its not easy to operate on alu32 bounds here because it depends 10365 * on bits being shifted in from upper 32-bits. Take easy way out 10366 * and mark unbounded so we can recalculate later from tnum. 10367 */ 10368 __mark_reg32_unbounded(dst_reg); 10369 __update_reg_bounds(dst_reg); 10370 } 10371 10372 /* WARNING: This function does calculations on 64-bit values, but the actual 10373 * execution may occur on 32-bit values. Therefore, things like bitshifts 10374 * need extra checks in the 32-bit case. 10375 */ 10376 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 10377 struct bpf_insn *insn, 10378 struct bpf_reg_state *dst_reg, 10379 struct bpf_reg_state src_reg) 10380 { 10381 struct bpf_reg_state *regs = cur_regs(env); 10382 u8 opcode = BPF_OP(insn->code); 10383 bool src_known; 10384 s64 smin_val, smax_val; 10385 u64 umin_val, umax_val; 10386 s32 s32_min_val, s32_max_val; 10387 u32 u32_min_val, u32_max_val; 10388 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 10389 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 10390 int ret; 10391 10392 smin_val = src_reg.smin_value; 10393 smax_val = src_reg.smax_value; 10394 umin_val = src_reg.umin_value; 10395 umax_val = src_reg.umax_value; 10396 10397 s32_min_val = src_reg.s32_min_value; 10398 s32_max_val = src_reg.s32_max_value; 10399 u32_min_val = src_reg.u32_min_value; 10400 u32_max_val = src_reg.u32_max_value; 10401 10402 if (alu32) { 10403 src_known = tnum_subreg_is_const(src_reg.var_off); 10404 if ((src_known && 10405 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 10406 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 10407 /* Taint dst register if offset had invalid bounds 10408 * derived from e.g. dead branches. 10409 */ 10410 __mark_reg_unknown(env, dst_reg); 10411 return 0; 10412 } 10413 } else { 10414 src_known = tnum_is_const(src_reg.var_off); 10415 if ((src_known && 10416 (smin_val != smax_val || umin_val != umax_val)) || 10417 smin_val > smax_val || umin_val > umax_val) { 10418 /* Taint dst register if offset had invalid bounds 10419 * derived from e.g. dead branches. 10420 */ 10421 __mark_reg_unknown(env, dst_reg); 10422 return 0; 10423 } 10424 } 10425 10426 if (!src_known && 10427 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 10428 __mark_reg_unknown(env, dst_reg); 10429 return 0; 10430 } 10431 10432 if (sanitize_needed(opcode)) { 10433 ret = sanitize_val_alu(env, insn); 10434 if (ret < 0) 10435 return sanitize_err(env, insn, ret, NULL, NULL); 10436 } 10437 10438 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 10439 * There are two classes of instructions: The first class we track both 10440 * alu32 and alu64 sign/unsigned bounds independently this provides the 10441 * greatest amount of precision when alu operations are mixed with jmp32 10442 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 10443 * and BPF_OR. This is possible because these ops have fairly easy to 10444 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 10445 * See alu32 verifier tests for examples. The second class of 10446 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 10447 * with regards to tracking sign/unsigned bounds because the bits may 10448 * cross subreg boundaries in the alu64 case. When this happens we mark 10449 * the reg unbounded in the subreg bound space and use the resulting 10450 * tnum to calculate an approximation of the sign/unsigned bounds. 10451 */ 10452 switch (opcode) { 10453 case BPF_ADD: 10454 scalar32_min_max_add(dst_reg, &src_reg); 10455 scalar_min_max_add(dst_reg, &src_reg); 10456 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 10457 break; 10458 case BPF_SUB: 10459 scalar32_min_max_sub(dst_reg, &src_reg); 10460 scalar_min_max_sub(dst_reg, &src_reg); 10461 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 10462 break; 10463 case BPF_MUL: 10464 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 10465 scalar32_min_max_mul(dst_reg, &src_reg); 10466 scalar_min_max_mul(dst_reg, &src_reg); 10467 break; 10468 case BPF_AND: 10469 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 10470 scalar32_min_max_and(dst_reg, &src_reg); 10471 scalar_min_max_and(dst_reg, &src_reg); 10472 break; 10473 case BPF_OR: 10474 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 10475 scalar32_min_max_or(dst_reg, &src_reg); 10476 scalar_min_max_or(dst_reg, &src_reg); 10477 break; 10478 case BPF_XOR: 10479 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 10480 scalar32_min_max_xor(dst_reg, &src_reg); 10481 scalar_min_max_xor(dst_reg, &src_reg); 10482 break; 10483 case BPF_LSH: 10484 if (umax_val >= insn_bitness) { 10485 /* Shifts greater than 31 or 63 are undefined. 10486 * This includes shifts by a negative number. 10487 */ 10488 mark_reg_unknown(env, regs, insn->dst_reg); 10489 break; 10490 } 10491 if (alu32) 10492 scalar32_min_max_lsh(dst_reg, &src_reg); 10493 else 10494 scalar_min_max_lsh(dst_reg, &src_reg); 10495 break; 10496 case BPF_RSH: 10497 if (umax_val >= insn_bitness) { 10498 /* Shifts greater than 31 or 63 are undefined. 10499 * This includes shifts by a negative number. 10500 */ 10501 mark_reg_unknown(env, regs, insn->dst_reg); 10502 break; 10503 } 10504 if (alu32) 10505 scalar32_min_max_rsh(dst_reg, &src_reg); 10506 else 10507 scalar_min_max_rsh(dst_reg, &src_reg); 10508 break; 10509 case BPF_ARSH: 10510 if (umax_val >= insn_bitness) { 10511 /* Shifts greater than 31 or 63 are undefined. 10512 * This includes shifts by a negative number. 10513 */ 10514 mark_reg_unknown(env, regs, insn->dst_reg); 10515 break; 10516 } 10517 if (alu32) 10518 scalar32_min_max_arsh(dst_reg, &src_reg); 10519 else 10520 scalar_min_max_arsh(dst_reg, &src_reg); 10521 break; 10522 default: 10523 mark_reg_unknown(env, regs, insn->dst_reg); 10524 break; 10525 } 10526 10527 /* ALU32 ops are zero extended into 64bit register */ 10528 if (alu32) 10529 zext_32_to_64(dst_reg); 10530 reg_bounds_sync(dst_reg); 10531 return 0; 10532 } 10533 10534 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 10535 * and var_off. 10536 */ 10537 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 10538 struct bpf_insn *insn) 10539 { 10540 struct bpf_verifier_state *vstate = env->cur_state; 10541 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 10542 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 10543 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 10544 u8 opcode = BPF_OP(insn->code); 10545 int err; 10546 10547 dst_reg = ®s[insn->dst_reg]; 10548 src_reg = NULL; 10549 if (dst_reg->type != SCALAR_VALUE) 10550 ptr_reg = dst_reg; 10551 else 10552 /* Make sure ID is cleared otherwise dst_reg min/max could be 10553 * incorrectly propagated into other registers by find_equal_scalars() 10554 */ 10555 dst_reg->id = 0; 10556 if (BPF_SRC(insn->code) == BPF_X) { 10557 src_reg = ®s[insn->src_reg]; 10558 if (src_reg->type != SCALAR_VALUE) { 10559 if (dst_reg->type != SCALAR_VALUE) { 10560 /* Combining two pointers by any ALU op yields 10561 * an arbitrary scalar. Disallow all math except 10562 * pointer subtraction 10563 */ 10564 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 10565 mark_reg_unknown(env, regs, insn->dst_reg); 10566 return 0; 10567 } 10568 verbose(env, "R%d pointer %s pointer prohibited\n", 10569 insn->dst_reg, 10570 bpf_alu_string[opcode >> 4]); 10571 return -EACCES; 10572 } else { 10573 /* scalar += pointer 10574 * This is legal, but we have to reverse our 10575 * src/dest handling in computing the range 10576 */ 10577 err = mark_chain_precision(env, insn->dst_reg); 10578 if (err) 10579 return err; 10580 return adjust_ptr_min_max_vals(env, insn, 10581 src_reg, dst_reg); 10582 } 10583 } else if (ptr_reg) { 10584 /* pointer += scalar */ 10585 err = mark_chain_precision(env, insn->src_reg); 10586 if (err) 10587 return err; 10588 return adjust_ptr_min_max_vals(env, insn, 10589 dst_reg, src_reg); 10590 } else if (dst_reg->precise) { 10591 /* if dst_reg is precise, src_reg should be precise as well */ 10592 err = mark_chain_precision(env, insn->src_reg); 10593 if (err) 10594 return err; 10595 } 10596 } else { 10597 /* Pretend the src is a reg with a known value, since we only 10598 * need to be able to read from this state. 10599 */ 10600 off_reg.type = SCALAR_VALUE; 10601 __mark_reg_known(&off_reg, insn->imm); 10602 src_reg = &off_reg; 10603 if (ptr_reg) /* pointer += K */ 10604 return adjust_ptr_min_max_vals(env, insn, 10605 ptr_reg, src_reg); 10606 } 10607 10608 /* Got here implies adding two SCALAR_VALUEs */ 10609 if (WARN_ON_ONCE(ptr_reg)) { 10610 print_verifier_state(env, state, true); 10611 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 10612 return -EINVAL; 10613 } 10614 if (WARN_ON(!src_reg)) { 10615 print_verifier_state(env, state, true); 10616 verbose(env, "verifier internal error: no src_reg\n"); 10617 return -EINVAL; 10618 } 10619 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 10620 } 10621 10622 /* check validity of 32-bit and 64-bit arithmetic operations */ 10623 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 10624 { 10625 struct bpf_reg_state *regs = cur_regs(env); 10626 u8 opcode = BPF_OP(insn->code); 10627 int err; 10628 10629 if (opcode == BPF_END || opcode == BPF_NEG) { 10630 if (opcode == BPF_NEG) { 10631 if (BPF_SRC(insn->code) != BPF_K || 10632 insn->src_reg != BPF_REG_0 || 10633 insn->off != 0 || insn->imm != 0) { 10634 verbose(env, "BPF_NEG uses reserved fields\n"); 10635 return -EINVAL; 10636 } 10637 } else { 10638 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 10639 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 10640 BPF_CLASS(insn->code) == BPF_ALU64) { 10641 verbose(env, "BPF_END uses reserved fields\n"); 10642 return -EINVAL; 10643 } 10644 } 10645 10646 /* check src operand */ 10647 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10648 if (err) 10649 return err; 10650 10651 if (is_pointer_value(env, insn->dst_reg)) { 10652 verbose(env, "R%d pointer arithmetic prohibited\n", 10653 insn->dst_reg); 10654 return -EACCES; 10655 } 10656 10657 /* check dest operand */ 10658 err = check_reg_arg(env, insn->dst_reg, DST_OP); 10659 if (err) 10660 return err; 10661 10662 } else if (opcode == BPF_MOV) { 10663 10664 if (BPF_SRC(insn->code) == BPF_X) { 10665 if (insn->imm != 0 || insn->off != 0) { 10666 verbose(env, "BPF_MOV uses reserved fields\n"); 10667 return -EINVAL; 10668 } 10669 10670 /* check src operand */ 10671 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10672 if (err) 10673 return err; 10674 } else { 10675 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 10676 verbose(env, "BPF_MOV uses reserved fields\n"); 10677 return -EINVAL; 10678 } 10679 } 10680 10681 /* check dest operand, mark as required later */ 10682 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10683 if (err) 10684 return err; 10685 10686 if (BPF_SRC(insn->code) == BPF_X) { 10687 struct bpf_reg_state *src_reg = regs + insn->src_reg; 10688 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 10689 10690 if (BPF_CLASS(insn->code) == BPF_ALU64) { 10691 /* case: R1 = R2 10692 * copy register state to dest reg 10693 */ 10694 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 10695 /* Assign src and dst registers the same ID 10696 * that will be used by find_equal_scalars() 10697 * to propagate min/max range. 10698 */ 10699 src_reg->id = ++env->id_gen; 10700 *dst_reg = *src_reg; 10701 dst_reg->live |= REG_LIVE_WRITTEN; 10702 dst_reg->subreg_def = DEF_NOT_SUBREG; 10703 } else { 10704 /* R1 = (u32) R2 */ 10705 if (is_pointer_value(env, insn->src_reg)) { 10706 verbose(env, 10707 "R%d partial copy of pointer\n", 10708 insn->src_reg); 10709 return -EACCES; 10710 } else if (src_reg->type == SCALAR_VALUE) { 10711 *dst_reg = *src_reg; 10712 /* Make sure ID is cleared otherwise 10713 * dst_reg min/max could be incorrectly 10714 * propagated into src_reg by find_equal_scalars() 10715 */ 10716 dst_reg->id = 0; 10717 dst_reg->live |= REG_LIVE_WRITTEN; 10718 dst_reg->subreg_def = env->insn_idx + 1; 10719 } else { 10720 mark_reg_unknown(env, regs, 10721 insn->dst_reg); 10722 } 10723 zext_32_to_64(dst_reg); 10724 reg_bounds_sync(dst_reg); 10725 } 10726 } else { 10727 /* case: R = imm 10728 * remember the value we stored into this reg 10729 */ 10730 /* clear any state __mark_reg_known doesn't set */ 10731 mark_reg_unknown(env, regs, insn->dst_reg); 10732 regs[insn->dst_reg].type = SCALAR_VALUE; 10733 if (BPF_CLASS(insn->code) == BPF_ALU64) { 10734 __mark_reg_known(regs + insn->dst_reg, 10735 insn->imm); 10736 } else { 10737 __mark_reg_known(regs + insn->dst_reg, 10738 (u32)insn->imm); 10739 } 10740 } 10741 10742 } else if (opcode > BPF_END) { 10743 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 10744 return -EINVAL; 10745 10746 } else { /* all other ALU ops: and, sub, xor, add, ... */ 10747 10748 if (BPF_SRC(insn->code) == BPF_X) { 10749 if (insn->imm != 0 || insn->off != 0) { 10750 verbose(env, "BPF_ALU uses reserved fields\n"); 10751 return -EINVAL; 10752 } 10753 /* check src1 operand */ 10754 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10755 if (err) 10756 return err; 10757 } else { 10758 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 10759 verbose(env, "BPF_ALU uses reserved fields\n"); 10760 return -EINVAL; 10761 } 10762 } 10763 10764 /* check src2 operand */ 10765 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10766 if (err) 10767 return err; 10768 10769 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 10770 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 10771 verbose(env, "div by zero\n"); 10772 return -EINVAL; 10773 } 10774 10775 if ((opcode == BPF_LSH || opcode == BPF_RSH || 10776 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 10777 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 10778 10779 if (insn->imm < 0 || insn->imm >= size) { 10780 verbose(env, "invalid shift %d\n", insn->imm); 10781 return -EINVAL; 10782 } 10783 } 10784 10785 /* check dest operand */ 10786 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10787 if (err) 10788 return err; 10789 10790 return adjust_reg_min_max_vals(env, insn); 10791 } 10792 10793 return 0; 10794 } 10795 10796 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 10797 struct bpf_reg_state *dst_reg, 10798 enum bpf_reg_type type, 10799 bool range_right_open) 10800 { 10801 struct bpf_func_state *state; 10802 struct bpf_reg_state *reg; 10803 int new_range; 10804 10805 if (dst_reg->off < 0 || 10806 (dst_reg->off == 0 && range_right_open)) 10807 /* This doesn't give us any range */ 10808 return; 10809 10810 if (dst_reg->umax_value > MAX_PACKET_OFF || 10811 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 10812 /* Risk of overflow. For instance, ptr + (1<<63) may be less 10813 * than pkt_end, but that's because it's also less than pkt. 10814 */ 10815 return; 10816 10817 new_range = dst_reg->off; 10818 if (range_right_open) 10819 new_range++; 10820 10821 /* Examples for register markings: 10822 * 10823 * pkt_data in dst register: 10824 * 10825 * r2 = r3; 10826 * r2 += 8; 10827 * if (r2 > pkt_end) goto <handle exception> 10828 * <access okay> 10829 * 10830 * r2 = r3; 10831 * r2 += 8; 10832 * if (r2 < pkt_end) goto <access okay> 10833 * <handle exception> 10834 * 10835 * Where: 10836 * r2 == dst_reg, pkt_end == src_reg 10837 * r2=pkt(id=n,off=8,r=0) 10838 * r3=pkt(id=n,off=0,r=0) 10839 * 10840 * pkt_data in src register: 10841 * 10842 * r2 = r3; 10843 * r2 += 8; 10844 * if (pkt_end >= r2) goto <access okay> 10845 * <handle exception> 10846 * 10847 * r2 = r3; 10848 * r2 += 8; 10849 * if (pkt_end <= r2) goto <handle exception> 10850 * <access okay> 10851 * 10852 * Where: 10853 * pkt_end == dst_reg, r2 == src_reg 10854 * r2=pkt(id=n,off=8,r=0) 10855 * r3=pkt(id=n,off=0,r=0) 10856 * 10857 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 10858 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 10859 * and [r3, r3 + 8-1) respectively is safe to access depending on 10860 * the check. 10861 */ 10862 10863 /* If our ids match, then we must have the same max_value. And we 10864 * don't care about the other reg's fixed offset, since if it's too big 10865 * the range won't allow anything. 10866 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 10867 */ 10868 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 10869 if (reg->type == type && reg->id == dst_reg->id) 10870 /* keep the maximum range already checked */ 10871 reg->range = max(reg->range, new_range); 10872 })); 10873 } 10874 10875 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 10876 { 10877 struct tnum subreg = tnum_subreg(reg->var_off); 10878 s32 sval = (s32)val; 10879 10880 switch (opcode) { 10881 case BPF_JEQ: 10882 if (tnum_is_const(subreg)) 10883 return !!tnum_equals_const(subreg, val); 10884 break; 10885 case BPF_JNE: 10886 if (tnum_is_const(subreg)) 10887 return !tnum_equals_const(subreg, val); 10888 break; 10889 case BPF_JSET: 10890 if ((~subreg.mask & subreg.value) & val) 10891 return 1; 10892 if (!((subreg.mask | subreg.value) & val)) 10893 return 0; 10894 break; 10895 case BPF_JGT: 10896 if (reg->u32_min_value > val) 10897 return 1; 10898 else if (reg->u32_max_value <= val) 10899 return 0; 10900 break; 10901 case BPF_JSGT: 10902 if (reg->s32_min_value > sval) 10903 return 1; 10904 else if (reg->s32_max_value <= sval) 10905 return 0; 10906 break; 10907 case BPF_JLT: 10908 if (reg->u32_max_value < val) 10909 return 1; 10910 else if (reg->u32_min_value >= val) 10911 return 0; 10912 break; 10913 case BPF_JSLT: 10914 if (reg->s32_max_value < sval) 10915 return 1; 10916 else if (reg->s32_min_value >= sval) 10917 return 0; 10918 break; 10919 case BPF_JGE: 10920 if (reg->u32_min_value >= val) 10921 return 1; 10922 else if (reg->u32_max_value < val) 10923 return 0; 10924 break; 10925 case BPF_JSGE: 10926 if (reg->s32_min_value >= sval) 10927 return 1; 10928 else if (reg->s32_max_value < sval) 10929 return 0; 10930 break; 10931 case BPF_JLE: 10932 if (reg->u32_max_value <= val) 10933 return 1; 10934 else if (reg->u32_min_value > val) 10935 return 0; 10936 break; 10937 case BPF_JSLE: 10938 if (reg->s32_max_value <= sval) 10939 return 1; 10940 else if (reg->s32_min_value > sval) 10941 return 0; 10942 break; 10943 } 10944 10945 return -1; 10946 } 10947 10948 10949 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 10950 { 10951 s64 sval = (s64)val; 10952 10953 switch (opcode) { 10954 case BPF_JEQ: 10955 if (tnum_is_const(reg->var_off)) 10956 return !!tnum_equals_const(reg->var_off, val); 10957 break; 10958 case BPF_JNE: 10959 if (tnum_is_const(reg->var_off)) 10960 return !tnum_equals_const(reg->var_off, val); 10961 break; 10962 case BPF_JSET: 10963 if ((~reg->var_off.mask & reg->var_off.value) & val) 10964 return 1; 10965 if (!((reg->var_off.mask | reg->var_off.value) & val)) 10966 return 0; 10967 break; 10968 case BPF_JGT: 10969 if (reg->umin_value > val) 10970 return 1; 10971 else if (reg->umax_value <= val) 10972 return 0; 10973 break; 10974 case BPF_JSGT: 10975 if (reg->smin_value > sval) 10976 return 1; 10977 else if (reg->smax_value <= sval) 10978 return 0; 10979 break; 10980 case BPF_JLT: 10981 if (reg->umax_value < val) 10982 return 1; 10983 else if (reg->umin_value >= val) 10984 return 0; 10985 break; 10986 case BPF_JSLT: 10987 if (reg->smax_value < sval) 10988 return 1; 10989 else if (reg->smin_value >= sval) 10990 return 0; 10991 break; 10992 case BPF_JGE: 10993 if (reg->umin_value >= val) 10994 return 1; 10995 else if (reg->umax_value < val) 10996 return 0; 10997 break; 10998 case BPF_JSGE: 10999 if (reg->smin_value >= sval) 11000 return 1; 11001 else if (reg->smax_value < sval) 11002 return 0; 11003 break; 11004 case BPF_JLE: 11005 if (reg->umax_value <= val) 11006 return 1; 11007 else if (reg->umin_value > val) 11008 return 0; 11009 break; 11010 case BPF_JSLE: 11011 if (reg->smax_value <= sval) 11012 return 1; 11013 else if (reg->smin_value > sval) 11014 return 0; 11015 break; 11016 } 11017 11018 return -1; 11019 } 11020 11021 /* compute branch direction of the expression "if (reg opcode val) goto target;" 11022 * and return: 11023 * 1 - branch will be taken and "goto target" will be executed 11024 * 0 - branch will not be taken and fall-through to next insn 11025 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 11026 * range [0,10] 11027 */ 11028 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 11029 bool is_jmp32) 11030 { 11031 if (__is_pointer_value(false, reg)) { 11032 if (!reg_type_not_null(reg->type)) 11033 return -1; 11034 11035 /* If pointer is valid tests against zero will fail so we can 11036 * use this to direct branch taken. 11037 */ 11038 if (val != 0) 11039 return -1; 11040 11041 switch (opcode) { 11042 case BPF_JEQ: 11043 return 0; 11044 case BPF_JNE: 11045 return 1; 11046 default: 11047 return -1; 11048 } 11049 } 11050 11051 if (is_jmp32) 11052 return is_branch32_taken(reg, val, opcode); 11053 return is_branch64_taken(reg, val, opcode); 11054 } 11055 11056 static int flip_opcode(u32 opcode) 11057 { 11058 /* How can we transform "a <op> b" into "b <op> a"? */ 11059 static const u8 opcode_flip[16] = { 11060 /* these stay the same */ 11061 [BPF_JEQ >> 4] = BPF_JEQ, 11062 [BPF_JNE >> 4] = BPF_JNE, 11063 [BPF_JSET >> 4] = BPF_JSET, 11064 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 11065 [BPF_JGE >> 4] = BPF_JLE, 11066 [BPF_JGT >> 4] = BPF_JLT, 11067 [BPF_JLE >> 4] = BPF_JGE, 11068 [BPF_JLT >> 4] = BPF_JGT, 11069 [BPF_JSGE >> 4] = BPF_JSLE, 11070 [BPF_JSGT >> 4] = BPF_JSLT, 11071 [BPF_JSLE >> 4] = BPF_JSGE, 11072 [BPF_JSLT >> 4] = BPF_JSGT 11073 }; 11074 return opcode_flip[opcode >> 4]; 11075 } 11076 11077 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 11078 struct bpf_reg_state *src_reg, 11079 u8 opcode) 11080 { 11081 struct bpf_reg_state *pkt; 11082 11083 if (src_reg->type == PTR_TO_PACKET_END) { 11084 pkt = dst_reg; 11085 } else if (dst_reg->type == PTR_TO_PACKET_END) { 11086 pkt = src_reg; 11087 opcode = flip_opcode(opcode); 11088 } else { 11089 return -1; 11090 } 11091 11092 if (pkt->range >= 0) 11093 return -1; 11094 11095 switch (opcode) { 11096 case BPF_JLE: 11097 /* pkt <= pkt_end */ 11098 fallthrough; 11099 case BPF_JGT: 11100 /* pkt > pkt_end */ 11101 if (pkt->range == BEYOND_PKT_END) 11102 /* pkt has at last one extra byte beyond pkt_end */ 11103 return opcode == BPF_JGT; 11104 break; 11105 case BPF_JLT: 11106 /* pkt < pkt_end */ 11107 fallthrough; 11108 case BPF_JGE: 11109 /* pkt >= pkt_end */ 11110 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 11111 return opcode == BPF_JGE; 11112 break; 11113 } 11114 return -1; 11115 } 11116 11117 /* Adjusts the register min/max values in the case that the dst_reg is the 11118 * variable register that we are working on, and src_reg is a constant or we're 11119 * simply doing a BPF_K check. 11120 * In JEQ/JNE cases we also adjust the var_off values. 11121 */ 11122 static void reg_set_min_max(struct bpf_reg_state *true_reg, 11123 struct bpf_reg_state *false_reg, 11124 u64 val, u32 val32, 11125 u8 opcode, bool is_jmp32) 11126 { 11127 struct tnum false_32off = tnum_subreg(false_reg->var_off); 11128 struct tnum false_64off = false_reg->var_off; 11129 struct tnum true_32off = tnum_subreg(true_reg->var_off); 11130 struct tnum true_64off = true_reg->var_off; 11131 s64 sval = (s64)val; 11132 s32 sval32 = (s32)val32; 11133 11134 /* If the dst_reg is a pointer, we can't learn anything about its 11135 * variable offset from the compare (unless src_reg were a pointer into 11136 * the same object, but we don't bother with that. 11137 * Since false_reg and true_reg have the same type by construction, we 11138 * only need to check one of them for pointerness. 11139 */ 11140 if (__is_pointer_value(false, false_reg)) 11141 return; 11142 11143 switch (opcode) { 11144 /* JEQ/JNE comparison doesn't change the register equivalence. 11145 * 11146 * r1 = r2; 11147 * if (r1 == 42) goto label; 11148 * ... 11149 * label: // here both r1 and r2 are known to be 42. 11150 * 11151 * Hence when marking register as known preserve it's ID. 11152 */ 11153 case BPF_JEQ: 11154 if (is_jmp32) { 11155 __mark_reg32_known(true_reg, val32); 11156 true_32off = tnum_subreg(true_reg->var_off); 11157 } else { 11158 ___mark_reg_known(true_reg, val); 11159 true_64off = true_reg->var_off; 11160 } 11161 break; 11162 case BPF_JNE: 11163 if (is_jmp32) { 11164 __mark_reg32_known(false_reg, val32); 11165 false_32off = tnum_subreg(false_reg->var_off); 11166 } else { 11167 ___mark_reg_known(false_reg, val); 11168 false_64off = false_reg->var_off; 11169 } 11170 break; 11171 case BPF_JSET: 11172 if (is_jmp32) { 11173 false_32off = tnum_and(false_32off, tnum_const(~val32)); 11174 if (is_power_of_2(val32)) 11175 true_32off = tnum_or(true_32off, 11176 tnum_const(val32)); 11177 } else { 11178 false_64off = tnum_and(false_64off, tnum_const(~val)); 11179 if (is_power_of_2(val)) 11180 true_64off = tnum_or(true_64off, 11181 tnum_const(val)); 11182 } 11183 break; 11184 case BPF_JGE: 11185 case BPF_JGT: 11186 { 11187 if (is_jmp32) { 11188 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 11189 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 11190 11191 false_reg->u32_max_value = min(false_reg->u32_max_value, 11192 false_umax); 11193 true_reg->u32_min_value = max(true_reg->u32_min_value, 11194 true_umin); 11195 } else { 11196 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 11197 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 11198 11199 false_reg->umax_value = min(false_reg->umax_value, false_umax); 11200 true_reg->umin_value = max(true_reg->umin_value, true_umin); 11201 } 11202 break; 11203 } 11204 case BPF_JSGE: 11205 case BPF_JSGT: 11206 { 11207 if (is_jmp32) { 11208 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 11209 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 11210 11211 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 11212 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 11213 } else { 11214 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 11215 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 11216 11217 false_reg->smax_value = min(false_reg->smax_value, false_smax); 11218 true_reg->smin_value = max(true_reg->smin_value, true_smin); 11219 } 11220 break; 11221 } 11222 case BPF_JLE: 11223 case BPF_JLT: 11224 { 11225 if (is_jmp32) { 11226 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 11227 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 11228 11229 false_reg->u32_min_value = max(false_reg->u32_min_value, 11230 false_umin); 11231 true_reg->u32_max_value = min(true_reg->u32_max_value, 11232 true_umax); 11233 } else { 11234 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 11235 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 11236 11237 false_reg->umin_value = max(false_reg->umin_value, false_umin); 11238 true_reg->umax_value = min(true_reg->umax_value, true_umax); 11239 } 11240 break; 11241 } 11242 case BPF_JSLE: 11243 case BPF_JSLT: 11244 { 11245 if (is_jmp32) { 11246 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 11247 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 11248 11249 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 11250 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 11251 } else { 11252 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 11253 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 11254 11255 false_reg->smin_value = max(false_reg->smin_value, false_smin); 11256 true_reg->smax_value = min(true_reg->smax_value, true_smax); 11257 } 11258 break; 11259 } 11260 default: 11261 return; 11262 } 11263 11264 if (is_jmp32) { 11265 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 11266 tnum_subreg(false_32off)); 11267 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 11268 tnum_subreg(true_32off)); 11269 __reg_combine_32_into_64(false_reg); 11270 __reg_combine_32_into_64(true_reg); 11271 } else { 11272 false_reg->var_off = false_64off; 11273 true_reg->var_off = true_64off; 11274 __reg_combine_64_into_32(false_reg); 11275 __reg_combine_64_into_32(true_reg); 11276 } 11277 } 11278 11279 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 11280 * the variable reg. 11281 */ 11282 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 11283 struct bpf_reg_state *false_reg, 11284 u64 val, u32 val32, 11285 u8 opcode, bool is_jmp32) 11286 { 11287 opcode = flip_opcode(opcode); 11288 /* This uses zero as "not present in table"; luckily the zero opcode, 11289 * BPF_JA, can't get here. 11290 */ 11291 if (opcode) 11292 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 11293 } 11294 11295 /* Regs are known to be equal, so intersect their min/max/var_off */ 11296 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 11297 struct bpf_reg_state *dst_reg) 11298 { 11299 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 11300 dst_reg->umin_value); 11301 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 11302 dst_reg->umax_value); 11303 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 11304 dst_reg->smin_value); 11305 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 11306 dst_reg->smax_value); 11307 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 11308 dst_reg->var_off); 11309 reg_bounds_sync(src_reg); 11310 reg_bounds_sync(dst_reg); 11311 } 11312 11313 static void reg_combine_min_max(struct bpf_reg_state *true_src, 11314 struct bpf_reg_state *true_dst, 11315 struct bpf_reg_state *false_src, 11316 struct bpf_reg_state *false_dst, 11317 u8 opcode) 11318 { 11319 switch (opcode) { 11320 case BPF_JEQ: 11321 __reg_combine_min_max(true_src, true_dst); 11322 break; 11323 case BPF_JNE: 11324 __reg_combine_min_max(false_src, false_dst); 11325 break; 11326 } 11327 } 11328 11329 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 11330 struct bpf_reg_state *reg, u32 id, 11331 bool is_null) 11332 { 11333 if (type_may_be_null(reg->type) && reg->id == id && 11334 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 11335 /* Old offset (both fixed and variable parts) should have been 11336 * known-zero, because we don't allow pointer arithmetic on 11337 * pointers that might be NULL. If we see this happening, don't 11338 * convert the register. 11339 * 11340 * But in some cases, some helpers that return local kptrs 11341 * advance offset for the returned pointer. In those cases, it 11342 * is fine to expect to see reg->off. 11343 */ 11344 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) 11345 return; 11346 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL) && WARN_ON_ONCE(reg->off)) 11347 return; 11348 if (is_null) { 11349 reg->type = SCALAR_VALUE; 11350 /* We don't need id and ref_obj_id from this point 11351 * onwards anymore, thus we should better reset it, 11352 * so that state pruning has chances to take effect. 11353 */ 11354 reg->id = 0; 11355 reg->ref_obj_id = 0; 11356 11357 return; 11358 } 11359 11360 mark_ptr_not_null_reg(reg); 11361 11362 if (!reg_may_point_to_spin_lock(reg)) { 11363 /* For not-NULL ptr, reg->ref_obj_id will be reset 11364 * in release_reference(). 11365 * 11366 * reg->id is still used by spin_lock ptr. Other 11367 * than spin_lock ptr type, reg->id can be reset. 11368 */ 11369 reg->id = 0; 11370 } 11371 } 11372 } 11373 11374 /* The logic is similar to find_good_pkt_pointers(), both could eventually 11375 * be folded together at some point. 11376 */ 11377 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 11378 bool is_null) 11379 { 11380 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 11381 struct bpf_reg_state *regs = state->regs, *reg; 11382 u32 ref_obj_id = regs[regno].ref_obj_id; 11383 u32 id = regs[regno].id; 11384 11385 if (ref_obj_id && ref_obj_id == id && is_null) 11386 /* regs[regno] is in the " == NULL" branch. 11387 * No one could have freed the reference state before 11388 * doing the NULL check. 11389 */ 11390 WARN_ON_ONCE(release_reference_state(state, id)); 11391 11392 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 11393 mark_ptr_or_null_reg(state, reg, id, is_null); 11394 })); 11395 } 11396 11397 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 11398 struct bpf_reg_state *dst_reg, 11399 struct bpf_reg_state *src_reg, 11400 struct bpf_verifier_state *this_branch, 11401 struct bpf_verifier_state *other_branch) 11402 { 11403 if (BPF_SRC(insn->code) != BPF_X) 11404 return false; 11405 11406 /* Pointers are always 64-bit. */ 11407 if (BPF_CLASS(insn->code) == BPF_JMP32) 11408 return false; 11409 11410 switch (BPF_OP(insn->code)) { 11411 case BPF_JGT: 11412 if ((dst_reg->type == PTR_TO_PACKET && 11413 src_reg->type == PTR_TO_PACKET_END) || 11414 (dst_reg->type == PTR_TO_PACKET_META && 11415 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11416 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 11417 find_good_pkt_pointers(this_branch, dst_reg, 11418 dst_reg->type, false); 11419 mark_pkt_end(other_branch, insn->dst_reg, true); 11420 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11421 src_reg->type == PTR_TO_PACKET) || 11422 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11423 src_reg->type == PTR_TO_PACKET_META)) { 11424 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 11425 find_good_pkt_pointers(other_branch, src_reg, 11426 src_reg->type, true); 11427 mark_pkt_end(this_branch, insn->src_reg, false); 11428 } else { 11429 return false; 11430 } 11431 break; 11432 case BPF_JLT: 11433 if ((dst_reg->type == PTR_TO_PACKET && 11434 src_reg->type == PTR_TO_PACKET_END) || 11435 (dst_reg->type == PTR_TO_PACKET_META && 11436 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11437 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 11438 find_good_pkt_pointers(other_branch, dst_reg, 11439 dst_reg->type, true); 11440 mark_pkt_end(this_branch, insn->dst_reg, false); 11441 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11442 src_reg->type == PTR_TO_PACKET) || 11443 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11444 src_reg->type == PTR_TO_PACKET_META)) { 11445 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 11446 find_good_pkt_pointers(this_branch, src_reg, 11447 src_reg->type, false); 11448 mark_pkt_end(other_branch, insn->src_reg, true); 11449 } else { 11450 return false; 11451 } 11452 break; 11453 case BPF_JGE: 11454 if ((dst_reg->type == PTR_TO_PACKET && 11455 src_reg->type == PTR_TO_PACKET_END) || 11456 (dst_reg->type == PTR_TO_PACKET_META && 11457 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11458 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 11459 find_good_pkt_pointers(this_branch, dst_reg, 11460 dst_reg->type, true); 11461 mark_pkt_end(other_branch, insn->dst_reg, false); 11462 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11463 src_reg->type == PTR_TO_PACKET) || 11464 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11465 src_reg->type == PTR_TO_PACKET_META)) { 11466 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 11467 find_good_pkt_pointers(other_branch, src_reg, 11468 src_reg->type, false); 11469 mark_pkt_end(this_branch, insn->src_reg, true); 11470 } else { 11471 return false; 11472 } 11473 break; 11474 case BPF_JLE: 11475 if ((dst_reg->type == PTR_TO_PACKET && 11476 src_reg->type == PTR_TO_PACKET_END) || 11477 (dst_reg->type == PTR_TO_PACKET_META && 11478 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11479 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 11480 find_good_pkt_pointers(other_branch, dst_reg, 11481 dst_reg->type, false); 11482 mark_pkt_end(this_branch, insn->dst_reg, true); 11483 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11484 src_reg->type == PTR_TO_PACKET) || 11485 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11486 src_reg->type == PTR_TO_PACKET_META)) { 11487 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 11488 find_good_pkt_pointers(this_branch, src_reg, 11489 src_reg->type, true); 11490 mark_pkt_end(other_branch, insn->src_reg, false); 11491 } else { 11492 return false; 11493 } 11494 break; 11495 default: 11496 return false; 11497 } 11498 11499 return true; 11500 } 11501 11502 static void find_equal_scalars(struct bpf_verifier_state *vstate, 11503 struct bpf_reg_state *known_reg) 11504 { 11505 struct bpf_func_state *state; 11506 struct bpf_reg_state *reg; 11507 11508 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 11509 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 11510 *reg = *known_reg; 11511 })); 11512 } 11513 11514 static int check_cond_jmp_op(struct bpf_verifier_env *env, 11515 struct bpf_insn *insn, int *insn_idx) 11516 { 11517 struct bpf_verifier_state *this_branch = env->cur_state; 11518 struct bpf_verifier_state *other_branch; 11519 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 11520 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 11521 struct bpf_reg_state *eq_branch_regs; 11522 u8 opcode = BPF_OP(insn->code); 11523 bool is_jmp32; 11524 int pred = -1; 11525 int err; 11526 11527 /* Only conditional jumps are expected to reach here. */ 11528 if (opcode == BPF_JA || opcode > BPF_JSLE) { 11529 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 11530 return -EINVAL; 11531 } 11532 11533 if (BPF_SRC(insn->code) == BPF_X) { 11534 if (insn->imm != 0) { 11535 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 11536 return -EINVAL; 11537 } 11538 11539 /* check src1 operand */ 11540 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11541 if (err) 11542 return err; 11543 11544 if (is_pointer_value(env, insn->src_reg)) { 11545 verbose(env, "R%d pointer comparison prohibited\n", 11546 insn->src_reg); 11547 return -EACCES; 11548 } 11549 src_reg = ®s[insn->src_reg]; 11550 } else { 11551 if (insn->src_reg != BPF_REG_0) { 11552 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 11553 return -EINVAL; 11554 } 11555 } 11556 11557 /* check src2 operand */ 11558 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11559 if (err) 11560 return err; 11561 11562 dst_reg = ®s[insn->dst_reg]; 11563 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 11564 11565 if (BPF_SRC(insn->code) == BPF_K) { 11566 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 11567 } else if (src_reg->type == SCALAR_VALUE && 11568 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 11569 pred = is_branch_taken(dst_reg, 11570 tnum_subreg(src_reg->var_off).value, 11571 opcode, 11572 is_jmp32); 11573 } else if (src_reg->type == SCALAR_VALUE && 11574 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 11575 pred = is_branch_taken(dst_reg, 11576 src_reg->var_off.value, 11577 opcode, 11578 is_jmp32); 11579 } else if (reg_is_pkt_pointer_any(dst_reg) && 11580 reg_is_pkt_pointer_any(src_reg) && 11581 !is_jmp32) { 11582 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 11583 } 11584 11585 if (pred >= 0) { 11586 /* If we get here with a dst_reg pointer type it is because 11587 * above is_branch_taken() special cased the 0 comparison. 11588 */ 11589 if (!__is_pointer_value(false, dst_reg)) 11590 err = mark_chain_precision(env, insn->dst_reg); 11591 if (BPF_SRC(insn->code) == BPF_X && !err && 11592 !__is_pointer_value(false, src_reg)) 11593 err = mark_chain_precision(env, insn->src_reg); 11594 if (err) 11595 return err; 11596 } 11597 11598 if (pred == 1) { 11599 /* Only follow the goto, ignore fall-through. If needed, push 11600 * the fall-through branch for simulation under speculative 11601 * execution. 11602 */ 11603 if (!env->bypass_spec_v1 && 11604 !sanitize_speculative_path(env, insn, *insn_idx + 1, 11605 *insn_idx)) 11606 return -EFAULT; 11607 *insn_idx += insn->off; 11608 return 0; 11609 } else if (pred == 0) { 11610 /* Only follow the fall-through branch, since that's where the 11611 * program will go. If needed, push the goto branch for 11612 * simulation under speculative execution. 11613 */ 11614 if (!env->bypass_spec_v1 && 11615 !sanitize_speculative_path(env, insn, 11616 *insn_idx + insn->off + 1, 11617 *insn_idx)) 11618 return -EFAULT; 11619 return 0; 11620 } 11621 11622 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 11623 false); 11624 if (!other_branch) 11625 return -EFAULT; 11626 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 11627 11628 /* detect if we are comparing against a constant value so we can adjust 11629 * our min/max values for our dst register. 11630 * this is only legit if both are scalars (or pointers to the same 11631 * object, I suppose, see the PTR_MAYBE_NULL related if block below), 11632 * because otherwise the different base pointers mean the offsets aren't 11633 * comparable. 11634 */ 11635 if (BPF_SRC(insn->code) == BPF_X) { 11636 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 11637 11638 if (dst_reg->type == SCALAR_VALUE && 11639 src_reg->type == SCALAR_VALUE) { 11640 if (tnum_is_const(src_reg->var_off) || 11641 (is_jmp32 && 11642 tnum_is_const(tnum_subreg(src_reg->var_off)))) 11643 reg_set_min_max(&other_branch_regs[insn->dst_reg], 11644 dst_reg, 11645 src_reg->var_off.value, 11646 tnum_subreg(src_reg->var_off).value, 11647 opcode, is_jmp32); 11648 else if (tnum_is_const(dst_reg->var_off) || 11649 (is_jmp32 && 11650 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 11651 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 11652 src_reg, 11653 dst_reg->var_off.value, 11654 tnum_subreg(dst_reg->var_off).value, 11655 opcode, is_jmp32); 11656 else if (!is_jmp32 && 11657 (opcode == BPF_JEQ || opcode == BPF_JNE)) 11658 /* Comparing for equality, we can combine knowledge */ 11659 reg_combine_min_max(&other_branch_regs[insn->src_reg], 11660 &other_branch_regs[insn->dst_reg], 11661 src_reg, dst_reg, opcode); 11662 if (src_reg->id && 11663 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 11664 find_equal_scalars(this_branch, src_reg); 11665 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 11666 } 11667 11668 } 11669 } else if (dst_reg->type == SCALAR_VALUE) { 11670 reg_set_min_max(&other_branch_regs[insn->dst_reg], 11671 dst_reg, insn->imm, (u32)insn->imm, 11672 opcode, is_jmp32); 11673 } 11674 11675 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 11676 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 11677 find_equal_scalars(this_branch, dst_reg); 11678 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 11679 } 11680 11681 /* if one pointer register is compared to another pointer 11682 * register check if PTR_MAYBE_NULL could be lifted. 11683 * E.g. register A - maybe null 11684 * register B - not null 11685 * for JNE A, B, ... - A is not null in the false branch; 11686 * for JEQ A, B, ... - A is not null in the true branch. 11687 */ 11688 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 11689 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 11690 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type)) { 11691 eq_branch_regs = NULL; 11692 switch (opcode) { 11693 case BPF_JEQ: 11694 eq_branch_regs = other_branch_regs; 11695 break; 11696 case BPF_JNE: 11697 eq_branch_regs = regs; 11698 break; 11699 default: 11700 /* do nothing */ 11701 break; 11702 } 11703 if (eq_branch_regs) { 11704 if (type_may_be_null(src_reg->type)) 11705 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 11706 else 11707 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 11708 } 11709 } 11710 11711 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 11712 * NOTE: these optimizations below are related with pointer comparison 11713 * which will never be JMP32. 11714 */ 11715 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 11716 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 11717 type_may_be_null(dst_reg->type)) { 11718 /* Mark all identical registers in each branch as either 11719 * safe or unknown depending R == 0 or R != 0 conditional. 11720 */ 11721 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 11722 opcode == BPF_JNE); 11723 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 11724 opcode == BPF_JEQ); 11725 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 11726 this_branch, other_branch) && 11727 is_pointer_value(env, insn->dst_reg)) { 11728 verbose(env, "R%d pointer comparison prohibited\n", 11729 insn->dst_reg); 11730 return -EACCES; 11731 } 11732 if (env->log.level & BPF_LOG_LEVEL) 11733 print_insn_state(env, this_branch->frame[this_branch->curframe]); 11734 return 0; 11735 } 11736 11737 /* verify BPF_LD_IMM64 instruction */ 11738 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 11739 { 11740 struct bpf_insn_aux_data *aux = cur_aux(env); 11741 struct bpf_reg_state *regs = cur_regs(env); 11742 struct bpf_reg_state *dst_reg; 11743 struct bpf_map *map; 11744 int err; 11745 11746 if (BPF_SIZE(insn->code) != BPF_DW) { 11747 verbose(env, "invalid BPF_LD_IMM insn\n"); 11748 return -EINVAL; 11749 } 11750 if (insn->off != 0) { 11751 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 11752 return -EINVAL; 11753 } 11754 11755 err = check_reg_arg(env, insn->dst_reg, DST_OP); 11756 if (err) 11757 return err; 11758 11759 dst_reg = ®s[insn->dst_reg]; 11760 if (insn->src_reg == 0) { 11761 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 11762 11763 dst_reg->type = SCALAR_VALUE; 11764 __mark_reg_known(®s[insn->dst_reg], imm); 11765 return 0; 11766 } 11767 11768 /* All special src_reg cases are listed below. From this point onwards 11769 * we either succeed and assign a corresponding dst_reg->type after 11770 * zeroing the offset, or fail and reject the program. 11771 */ 11772 mark_reg_known_zero(env, regs, insn->dst_reg); 11773 11774 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 11775 dst_reg->type = aux->btf_var.reg_type; 11776 switch (base_type(dst_reg->type)) { 11777 case PTR_TO_MEM: 11778 dst_reg->mem_size = aux->btf_var.mem_size; 11779 break; 11780 case PTR_TO_BTF_ID: 11781 dst_reg->btf = aux->btf_var.btf; 11782 dst_reg->btf_id = aux->btf_var.btf_id; 11783 break; 11784 default: 11785 verbose(env, "bpf verifier is misconfigured\n"); 11786 return -EFAULT; 11787 } 11788 return 0; 11789 } 11790 11791 if (insn->src_reg == BPF_PSEUDO_FUNC) { 11792 struct bpf_prog_aux *aux = env->prog->aux; 11793 u32 subprogno = find_subprog(env, 11794 env->insn_idx + insn->imm + 1); 11795 11796 if (!aux->func_info) { 11797 verbose(env, "missing btf func_info\n"); 11798 return -EINVAL; 11799 } 11800 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 11801 verbose(env, "callback function not static\n"); 11802 return -EINVAL; 11803 } 11804 11805 dst_reg->type = PTR_TO_FUNC; 11806 dst_reg->subprogno = subprogno; 11807 return 0; 11808 } 11809 11810 map = env->used_maps[aux->map_index]; 11811 dst_reg->map_ptr = map; 11812 11813 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 11814 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 11815 dst_reg->type = PTR_TO_MAP_VALUE; 11816 dst_reg->off = aux->map_off; 11817 WARN_ON_ONCE(map->max_entries != 1); 11818 /* We want reg->id to be same (0) as map_value is not distinct */ 11819 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 11820 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 11821 dst_reg->type = CONST_PTR_TO_MAP; 11822 } else { 11823 verbose(env, "bpf verifier is misconfigured\n"); 11824 return -EINVAL; 11825 } 11826 11827 return 0; 11828 } 11829 11830 static bool may_access_skb(enum bpf_prog_type type) 11831 { 11832 switch (type) { 11833 case BPF_PROG_TYPE_SOCKET_FILTER: 11834 case BPF_PROG_TYPE_SCHED_CLS: 11835 case BPF_PROG_TYPE_SCHED_ACT: 11836 return true; 11837 default: 11838 return false; 11839 } 11840 } 11841 11842 /* verify safety of LD_ABS|LD_IND instructions: 11843 * - they can only appear in the programs where ctx == skb 11844 * - since they are wrappers of function calls, they scratch R1-R5 registers, 11845 * preserve R6-R9, and store return value into R0 11846 * 11847 * Implicit input: 11848 * ctx == skb == R6 == CTX 11849 * 11850 * Explicit input: 11851 * SRC == any register 11852 * IMM == 32-bit immediate 11853 * 11854 * Output: 11855 * R0 - 8/16/32-bit skb data converted to cpu endianness 11856 */ 11857 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 11858 { 11859 struct bpf_reg_state *regs = cur_regs(env); 11860 static const int ctx_reg = BPF_REG_6; 11861 u8 mode = BPF_MODE(insn->code); 11862 int i, err; 11863 11864 if (!may_access_skb(resolve_prog_type(env->prog))) { 11865 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 11866 return -EINVAL; 11867 } 11868 11869 if (!env->ops->gen_ld_abs) { 11870 verbose(env, "bpf verifier is misconfigured\n"); 11871 return -EINVAL; 11872 } 11873 11874 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 11875 BPF_SIZE(insn->code) == BPF_DW || 11876 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 11877 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 11878 return -EINVAL; 11879 } 11880 11881 /* check whether implicit source operand (register R6) is readable */ 11882 err = check_reg_arg(env, ctx_reg, SRC_OP); 11883 if (err) 11884 return err; 11885 11886 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 11887 * gen_ld_abs() may terminate the program at runtime, leading to 11888 * reference leak. 11889 */ 11890 err = check_reference_leak(env); 11891 if (err) { 11892 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 11893 return err; 11894 } 11895 11896 if (env->cur_state->active_lock.ptr) { 11897 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 11898 return -EINVAL; 11899 } 11900 11901 if (env->cur_state->active_rcu_lock) { 11902 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); 11903 return -EINVAL; 11904 } 11905 11906 if (regs[ctx_reg].type != PTR_TO_CTX) { 11907 verbose(env, 11908 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 11909 return -EINVAL; 11910 } 11911 11912 if (mode == BPF_IND) { 11913 /* check explicit source operand */ 11914 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11915 if (err) 11916 return err; 11917 } 11918 11919 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 11920 if (err < 0) 11921 return err; 11922 11923 /* reset caller saved regs to unreadable */ 11924 for (i = 0; i < CALLER_SAVED_REGS; i++) { 11925 mark_reg_not_init(env, regs, caller_saved[i]); 11926 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 11927 } 11928 11929 /* mark destination R0 register as readable, since it contains 11930 * the value fetched from the packet. 11931 * Already marked as written above. 11932 */ 11933 mark_reg_unknown(env, regs, BPF_REG_0); 11934 /* ld_abs load up to 32-bit skb data. */ 11935 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 11936 return 0; 11937 } 11938 11939 static int check_return_code(struct bpf_verifier_env *env) 11940 { 11941 struct tnum enforce_attach_type_range = tnum_unknown; 11942 const struct bpf_prog *prog = env->prog; 11943 struct bpf_reg_state *reg; 11944 struct tnum range = tnum_range(0, 1); 11945 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 11946 int err; 11947 struct bpf_func_state *frame = env->cur_state->frame[0]; 11948 const bool is_subprog = frame->subprogno; 11949 11950 /* LSM and struct_ops func-ptr's return type could be "void" */ 11951 if (!is_subprog) { 11952 switch (prog_type) { 11953 case BPF_PROG_TYPE_LSM: 11954 if (prog->expected_attach_type == BPF_LSM_CGROUP) 11955 /* See below, can be 0 or 0-1 depending on hook. */ 11956 break; 11957 fallthrough; 11958 case BPF_PROG_TYPE_STRUCT_OPS: 11959 if (!prog->aux->attach_func_proto->type) 11960 return 0; 11961 break; 11962 default: 11963 break; 11964 } 11965 } 11966 11967 /* eBPF calling convention is such that R0 is used 11968 * to return the value from eBPF program. 11969 * Make sure that it's readable at this time 11970 * of bpf_exit, which means that program wrote 11971 * something into it earlier 11972 */ 11973 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 11974 if (err) 11975 return err; 11976 11977 if (is_pointer_value(env, BPF_REG_0)) { 11978 verbose(env, "R0 leaks addr as return value\n"); 11979 return -EACCES; 11980 } 11981 11982 reg = cur_regs(env) + BPF_REG_0; 11983 11984 if (frame->in_async_callback_fn) { 11985 /* enforce return zero from async callbacks like timer */ 11986 if (reg->type != SCALAR_VALUE) { 11987 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 11988 reg_type_str(env, reg->type)); 11989 return -EINVAL; 11990 } 11991 11992 if (!tnum_in(tnum_const(0), reg->var_off)) { 11993 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 11994 return -EINVAL; 11995 } 11996 return 0; 11997 } 11998 11999 if (is_subprog) { 12000 if (reg->type != SCALAR_VALUE) { 12001 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 12002 reg_type_str(env, reg->type)); 12003 return -EINVAL; 12004 } 12005 return 0; 12006 } 12007 12008 switch (prog_type) { 12009 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 12010 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 12011 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 12012 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 12013 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 12014 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 12015 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 12016 range = tnum_range(1, 1); 12017 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 12018 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 12019 range = tnum_range(0, 3); 12020 break; 12021 case BPF_PROG_TYPE_CGROUP_SKB: 12022 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 12023 range = tnum_range(0, 3); 12024 enforce_attach_type_range = tnum_range(2, 3); 12025 } 12026 break; 12027 case BPF_PROG_TYPE_CGROUP_SOCK: 12028 case BPF_PROG_TYPE_SOCK_OPS: 12029 case BPF_PROG_TYPE_CGROUP_DEVICE: 12030 case BPF_PROG_TYPE_CGROUP_SYSCTL: 12031 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 12032 break; 12033 case BPF_PROG_TYPE_RAW_TRACEPOINT: 12034 if (!env->prog->aux->attach_btf_id) 12035 return 0; 12036 range = tnum_const(0); 12037 break; 12038 case BPF_PROG_TYPE_TRACING: 12039 switch (env->prog->expected_attach_type) { 12040 case BPF_TRACE_FENTRY: 12041 case BPF_TRACE_FEXIT: 12042 range = tnum_const(0); 12043 break; 12044 case BPF_TRACE_RAW_TP: 12045 case BPF_MODIFY_RETURN: 12046 return 0; 12047 case BPF_TRACE_ITER: 12048 break; 12049 default: 12050 return -ENOTSUPP; 12051 } 12052 break; 12053 case BPF_PROG_TYPE_SK_LOOKUP: 12054 range = tnum_range(SK_DROP, SK_PASS); 12055 break; 12056 12057 case BPF_PROG_TYPE_LSM: 12058 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 12059 /* Regular BPF_PROG_TYPE_LSM programs can return 12060 * any value. 12061 */ 12062 return 0; 12063 } 12064 if (!env->prog->aux->attach_func_proto->type) { 12065 /* Make sure programs that attach to void 12066 * hooks don't try to modify return value. 12067 */ 12068 range = tnum_range(1, 1); 12069 } 12070 break; 12071 12072 case BPF_PROG_TYPE_EXT: 12073 /* freplace program can return anything as its return value 12074 * depends on the to-be-replaced kernel func or bpf program. 12075 */ 12076 default: 12077 return 0; 12078 } 12079 12080 if (reg->type != SCALAR_VALUE) { 12081 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 12082 reg_type_str(env, reg->type)); 12083 return -EINVAL; 12084 } 12085 12086 if (!tnum_in(range, reg->var_off)) { 12087 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 12088 if (prog->expected_attach_type == BPF_LSM_CGROUP && 12089 prog_type == BPF_PROG_TYPE_LSM && 12090 !prog->aux->attach_func_proto->type) 12091 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 12092 return -EINVAL; 12093 } 12094 12095 if (!tnum_is_unknown(enforce_attach_type_range) && 12096 tnum_in(enforce_attach_type_range, reg->var_off)) 12097 env->prog->enforce_expected_attach_type = 1; 12098 return 0; 12099 } 12100 12101 /* non-recursive DFS pseudo code 12102 * 1 procedure DFS-iterative(G,v): 12103 * 2 label v as discovered 12104 * 3 let S be a stack 12105 * 4 S.push(v) 12106 * 5 while S is not empty 12107 * 6 t <- S.peek() 12108 * 7 if t is what we're looking for: 12109 * 8 return t 12110 * 9 for all edges e in G.adjacentEdges(t) do 12111 * 10 if edge e is already labelled 12112 * 11 continue with the next edge 12113 * 12 w <- G.adjacentVertex(t,e) 12114 * 13 if vertex w is not discovered and not explored 12115 * 14 label e as tree-edge 12116 * 15 label w as discovered 12117 * 16 S.push(w) 12118 * 17 continue at 5 12119 * 18 else if vertex w is discovered 12120 * 19 label e as back-edge 12121 * 20 else 12122 * 21 // vertex w is explored 12123 * 22 label e as forward- or cross-edge 12124 * 23 label t as explored 12125 * 24 S.pop() 12126 * 12127 * convention: 12128 * 0x10 - discovered 12129 * 0x11 - discovered and fall-through edge labelled 12130 * 0x12 - discovered and fall-through and branch edges labelled 12131 * 0x20 - explored 12132 */ 12133 12134 enum { 12135 DISCOVERED = 0x10, 12136 EXPLORED = 0x20, 12137 FALLTHROUGH = 1, 12138 BRANCH = 2, 12139 }; 12140 12141 static u32 state_htab_size(struct bpf_verifier_env *env) 12142 { 12143 return env->prog->len; 12144 } 12145 12146 static struct bpf_verifier_state_list **explored_state( 12147 struct bpf_verifier_env *env, 12148 int idx) 12149 { 12150 struct bpf_verifier_state *cur = env->cur_state; 12151 struct bpf_func_state *state = cur->frame[cur->curframe]; 12152 12153 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 12154 } 12155 12156 static void mark_prune_point(struct bpf_verifier_env *env, int idx) 12157 { 12158 env->insn_aux_data[idx].prune_point = true; 12159 } 12160 12161 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) 12162 { 12163 return env->insn_aux_data[insn_idx].prune_point; 12164 } 12165 12166 enum { 12167 DONE_EXPLORING = 0, 12168 KEEP_EXPLORING = 1, 12169 }; 12170 12171 /* t, w, e - match pseudo-code above: 12172 * t - index of current instruction 12173 * w - next instruction 12174 * e - edge 12175 */ 12176 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 12177 bool loop_ok) 12178 { 12179 int *insn_stack = env->cfg.insn_stack; 12180 int *insn_state = env->cfg.insn_state; 12181 12182 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 12183 return DONE_EXPLORING; 12184 12185 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 12186 return DONE_EXPLORING; 12187 12188 if (w < 0 || w >= env->prog->len) { 12189 verbose_linfo(env, t, "%d: ", t); 12190 verbose(env, "jump out of range from insn %d to %d\n", t, w); 12191 return -EINVAL; 12192 } 12193 12194 if (e == BRANCH) { 12195 /* mark branch target for state pruning */ 12196 mark_prune_point(env, w); 12197 mark_jmp_point(env, w); 12198 } 12199 12200 if (insn_state[w] == 0) { 12201 /* tree-edge */ 12202 insn_state[t] = DISCOVERED | e; 12203 insn_state[w] = DISCOVERED; 12204 if (env->cfg.cur_stack >= env->prog->len) 12205 return -E2BIG; 12206 insn_stack[env->cfg.cur_stack++] = w; 12207 return KEEP_EXPLORING; 12208 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 12209 if (loop_ok && env->bpf_capable) 12210 return DONE_EXPLORING; 12211 verbose_linfo(env, t, "%d: ", t); 12212 verbose_linfo(env, w, "%d: ", w); 12213 verbose(env, "back-edge from insn %d to %d\n", t, w); 12214 return -EINVAL; 12215 } else if (insn_state[w] == EXPLORED) { 12216 /* forward- or cross-edge */ 12217 insn_state[t] = DISCOVERED | e; 12218 } else { 12219 verbose(env, "insn state internal bug\n"); 12220 return -EFAULT; 12221 } 12222 return DONE_EXPLORING; 12223 } 12224 12225 static int visit_func_call_insn(int t, int insn_cnt, 12226 struct bpf_insn *insns, 12227 struct bpf_verifier_env *env, 12228 bool visit_callee) 12229 { 12230 int ret; 12231 12232 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 12233 if (ret) 12234 return ret; 12235 12236 mark_prune_point(env, t + 1); 12237 /* when we exit from subprog, we need to record non-linear history */ 12238 mark_jmp_point(env, t + 1); 12239 12240 if (visit_callee) { 12241 mark_prune_point(env, t); 12242 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 12243 /* It's ok to allow recursion from CFG point of 12244 * view. __check_func_call() will do the actual 12245 * check. 12246 */ 12247 bpf_pseudo_func(insns + t)); 12248 } 12249 return ret; 12250 } 12251 12252 /* Visits the instruction at index t and returns one of the following: 12253 * < 0 - an error occurred 12254 * DONE_EXPLORING - the instruction was fully explored 12255 * KEEP_EXPLORING - there is still work to be done before it is fully explored 12256 */ 12257 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 12258 { 12259 struct bpf_insn *insns = env->prog->insnsi; 12260 int ret; 12261 12262 if (bpf_pseudo_func(insns + t)) 12263 return visit_func_call_insn(t, insn_cnt, insns, env, true); 12264 12265 /* All non-branch instructions have a single fall-through edge. */ 12266 if (BPF_CLASS(insns[t].code) != BPF_JMP && 12267 BPF_CLASS(insns[t].code) != BPF_JMP32) 12268 return push_insn(t, t + 1, FALLTHROUGH, env, false); 12269 12270 switch (BPF_OP(insns[t].code)) { 12271 case BPF_EXIT: 12272 return DONE_EXPLORING; 12273 12274 case BPF_CALL: 12275 if (insns[t].imm == BPF_FUNC_timer_set_callback) 12276 /* Mark this call insn as a prune point to trigger 12277 * is_state_visited() check before call itself is 12278 * processed by __check_func_call(). Otherwise new 12279 * async state will be pushed for further exploration. 12280 */ 12281 mark_prune_point(env, t); 12282 return visit_func_call_insn(t, insn_cnt, insns, env, 12283 insns[t].src_reg == BPF_PSEUDO_CALL); 12284 12285 case BPF_JA: 12286 if (BPF_SRC(insns[t].code) != BPF_K) 12287 return -EINVAL; 12288 12289 /* unconditional jump with single edge */ 12290 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 12291 true); 12292 if (ret) 12293 return ret; 12294 12295 mark_prune_point(env, t + insns[t].off + 1); 12296 mark_jmp_point(env, t + insns[t].off + 1); 12297 12298 return ret; 12299 12300 default: 12301 /* conditional jump with two edges */ 12302 mark_prune_point(env, t); 12303 12304 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 12305 if (ret) 12306 return ret; 12307 12308 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 12309 } 12310 } 12311 12312 /* non-recursive depth-first-search to detect loops in BPF program 12313 * loop == back-edge in directed graph 12314 */ 12315 static int check_cfg(struct bpf_verifier_env *env) 12316 { 12317 int insn_cnt = env->prog->len; 12318 int *insn_stack, *insn_state; 12319 int ret = 0; 12320 int i; 12321 12322 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 12323 if (!insn_state) 12324 return -ENOMEM; 12325 12326 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 12327 if (!insn_stack) { 12328 kvfree(insn_state); 12329 return -ENOMEM; 12330 } 12331 12332 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 12333 insn_stack[0] = 0; /* 0 is the first instruction */ 12334 env->cfg.cur_stack = 1; 12335 12336 while (env->cfg.cur_stack > 0) { 12337 int t = insn_stack[env->cfg.cur_stack - 1]; 12338 12339 ret = visit_insn(t, insn_cnt, env); 12340 switch (ret) { 12341 case DONE_EXPLORING: 12342 insn_state[t] = EXPLORED; 12343 env->cfg.cur_stack--; 12344 break; 12345 case KEEP_EXPLORING: 12346 break; 12347 default: 12348 if (ret > 0) { 12349 verbose(env, "visit_insn internal bug\n"); 12350 ret = -EFAULT; 12351 } 12352 goto err_free; 12353 } 12354 } 12355 12356 if (env->cfg.cur_stack < 0) { 12357 verbose(env, "pop stack internal bug\n"); 12358 ret = -EFAULT; 12359 goto err_free; 12360 } 12361 12362 for (i = 0; i < insn_cnt; i++) { 12363 if (insn_state[i] != EXPLORED) { 12364 verbose(env, "unreachable insn %d\n", i); 12365 ret = -EINVAL; 12366 goto err_free; 12367 } 12368 } 12369 ret = 0; /* cfg looks good */ 12370 12371 err_free: 12372 kvfree(insn_state); 12373 kvfree(insn_stack); 12374 env->cfg.insn_state = env->cfg.insn_stack = NULL; 12375 return ret; 12376 } 12377 12378 static int check_abnormal_return(struct bpf_verifier_env *env) 12379 { 12380 int i; 12381 12382 for (i = 1; i < env->subprog_cnt; i++) { 12383 if (env->subprog_info[i].has_ld_abs) { 12384 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 12385 return -EINVAL; 12386 } 12387 if (env->subprog_info[i].has_tail_call) { 12388 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 12389 return -EINVAL; 12390 } 12391 } 12392 return 0; 12393 } 12394 12395 /* The minimum supported BTF func info size */ 12396 #define MIN_BPF_FUNCINFO_SIZE 8 12397 #define MAX_FUNCINFO_REC_SIZE 252 12398 12399 static int check_btf_func(struct bpf_verifier_env *env, 12400 const union bpf_attr *attr, 12401 bpfptr_t uattr) 12402 { 12403 const struct btf_type *type, *func_proto, *ret_type; 12404 u32 i, nfuncs, urec_size, min_size; 12405 u32 krec_size = sizeof(struct bpf_func_info); 12406 struct bpf_func_info *krecord; 12407 struct bpf_func_info_aux *info_aux = NULL; 12408 struct bpf_prog *prog; 12409 const struct btf *btf; 12410 bpfptr_t urecord; 12411 u32 prev_offset = 0; 12412 bool scalar_return; 12413 int ret = -ENOMEM; 12414 12415 nfuncs = attr->func_info_cnt; 12416 if (!nfuncs) { 12417 if (check_abnormal_return(env)) 12418 return -EINVAL; 12419 return 0; 12420 } 12421 12422 if (nfuncs != env->subprog_cnt) { 12423 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 12424 return -EINVAL; 12425 } 12426 12427 urec_size = attr->func_info_rec_size; 12428 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 12429 urec_size > MAX_FUNCINFO_REC_SIZE || 12430 urec_size % sizeof(u32)) { 12431 verbose(env, "invalid func info rec size %u\n", urec_size); 12432 return -EINVAL; 12433 } 12434 12435 prog = env->prog; 12436 btf = prog->aux->btf; 12437 12438 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 12439 min_size = min_t(u32, krec_size, urec_size); 12440 12441 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 12442 if (!krecord) 12443 return -ENOMEM; 12444 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 12445 if (!info_aux) 12446 goto err_free; 12447 12448 for (i = 0; i < nfuncs; i++) { 12449 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 12450 if (ret) { 12451 if (ret == -E2BIG) { 12452 verbose(env, "nonzero tailing record in func info"); 12453 /* set the size kernel expects so loader can zero 12454 * out the rest of the record. 12455 */ 12456 if (copy_to_bpfptr_offset(uattr, 12457 offsetof(union bpf_attr, func_info_rec_size), 12458 &min_size, sizeof(min_size))) 12459 ret = -EFAULT; 12460 } 12461 goto err_free; 12462 } 12463 12464 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 12465 ret = -EFAULT; 12466 goto err_free; 12467 } 12468 12469 /* check insn_off */ 12470 ret = -EINVAL; 12471 if (i == 0) { 12472 if (krecord[i].insn_off) { 12473 verbose(env, 12474 "nonzero insn_off %u for the first func info record", 12475 krecord[i].insn_off); 12476 goto err_free; 12477 } 12478 } else if (krecord[i].insn_off <= prev_offset) { 12479 verbose(env, 12480 "same or smaller insn offset (%u) than previous func info record (%u)", 12481 krecord[i].insn_off, prev_offset); 12482 goto err_free; 12483 } 12484 12485 if (env->subprog_info[i].start != krecord[i].insn_off) { 12486 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 12487 goto err_free; 12488 } 12489 12490 /* check type_id */ 12491 type = btf_type_by_id(btf, krecord[i].type_id); 12492 if (!type || !btf_type_is_func(type)) { 12493 verbose(env, "invalid type id %d in func info", 12494 krecord[i].type_id); 12495 goto err_free; 12496 } 12497 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 12498 12499 func_proto = btf_type_by_id(btf, type->type); 12500 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 12501 /* btf_func_check() already verified it during BTF load */ 12502 goto err_free; 12503 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 12504 scalar_return = 12505 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); 12506 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 12507 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 12508 goto err_free; 12509 } 12510 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 12511 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 12512 goto err_free; 12513 } 12514 12515 prev_offset = krecord[i].insn_off; 12516 bpfptr_add(&urecord, urec_size); 12517 } 12518 12519 prog->aux->func_info = krecord; 12520 prog->aux->func_info_cnt = nfuncs; 12521 prog->aux->func_info_aux = info_aux; 12522 return 0; 12523 12524 err_free: 12525 kvfree(krecord); 12526 kfree(info_aux); 12527 return ret; 12528 } 12529 12530 static void adjust_btf_func(struct bpf_verifier_env *env) 12531 { 12532 struct bpf_prog_aux *aux = env->prog->aux; 12533 int i; 12534 12535 if (!aux->func_info) 12536 return; 12537 12538 for (i = 0; i < env->subprog_cnt; i++) 12539 aux->func_info[i].insn_off = env->subprog_info[i].start; 12540 } 12541 12542 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 12543 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 12544 12545 static int check_btf_line(struct bpf_verifier_env *env, 12546 const union bpf_attr *attr, 12547 bpfptr_t uattr) 12548 { 12549 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 12550 struct bpf_subprog_info *sub; 12551 struct bpf_line_info *linfo; 12552 struct bpf_prog *prog; 12553 const struct btf *btf; 12554 bpfptr_t ulinfo; 12555 int err; 12556 12557 nr_linfo = attr->line_info_cnt; 12558 if (!nr_linfo) 12559 return 0; 12560 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 12561 return -EINVAL; 12562 12563 rec_size = attr->line_info_rec_size; 12564 if (rec_size < MIN_BPF_LINEINFO_SIZE || 12565 rec_size > MAX_LINEINFO_REC_SIZE || 12566 rec_size & (sizeof(u32) - 1)) 12567 return -EINVAL; 12568 12569 /* Need to zero it in case the userspace may 12570 * pass in a smaller bpf_line_info object. 12571 */ 12572 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 12573 GFP_KERNEL | __GFP_NOWARN); 12574 if (!linfo) 12575 return -ENOMEM; 12576 12577 prog = env->prog; 12578 btf = prog->aux->btf; 12579 12580 s = 0; 12581 sub = env->subprog_info; 12582 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 12583 expected_size = sizeof(struct bpf_line_info); 12584 ncopy = min_t(u32, expected_size, rec_size); 12585 for (i = 0; i < nr_linfo; i++) { 12586 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 12587 if (err) { 12588 if (err == -E2BIG) { 12589 verbose(env, "nonzero tailing record in line_info"); 12590 if (copy_to_bpfptr_offset(uattr, 12591 offsetof(union bpf_attr, line_info_rec_size), 12592 &expected_size, sizeof(expected_size))) 12593 err = -EFAULT; 12594 } 12595 goto err_free; 12596 } 12597 12598 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 12599 err = -EFAULT; 12600 goto err_free; 12601 } 12602 12603 /* 12604 * Check insn_off to ensure 12605 * 1) strictly increasing AND 12606 * 2) bounded by prog->len 12607 * 12608 * The linfo[0].insn_off == 0 check logically falls into 12609 * the later "missing bpf_line_info for func..." case 12610 * because the first linfo[0].insn_off must be the 12611 * first sub also and the first sub must have 12612 * subprog_info[0].start == 0. 12613 */ 12614 if ((i && linfo[i].insn_off <= prev_offset) || 12615 linfo[i].insn_off >= prog->len) { 12616 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 12617 i, linfo[i].insn_off, prev_offset, 12618 prog->len); 12619 err = -EINVAL; 12620 goto err_free; 12621 } 12622 12623 if (!prog->insnsi[linfo[i].insn_off].code) { 12624 verbose(env, 12625 "Invalid insn code at line_info[%u].insn_off\n", 12626 i); 12627 err = -EINVAL; 12628 goto err_free; 12629 } 12630 12631 if (!btf_name_by_offset(btf, linfo[i].line_off) || 12632 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 12633 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 12634 err = -EINVAL; 12635 goto err_free; 12636 } 12637 12638 if (s != env->subprog_cnt) { 12639 if (linfo[i].insn_off == sub[s].start) { 12640 sub[s].linfo_idx = i; 12641 s++; 12642 } else if (sub[s].start < linfo[i].insn_off) { 12643 verbose(env, "missing bpf_line_info for func#%u\n", s); 12644 err = -EINVAL; 12645 goto err_free; 12646 } 12647 } 12648 12649 prev_offset = linfo[i].insn_off; 12650 bpfptr_add(&ulinfo, rec_size); 12651 } 12652 12653 if (s != env->subprog_cnt) { 12654 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 12655 env->subprog_cnt - s, s); 12656 err = -EINVAL; 12657 goto err_free; 12658 } 12659 12660 prog->aux->linfo = linfo; 12661 prog->aux->nr_linfo = nr_linfo; 12662 12663 return 0; 12664 12665 err_free: 12666 kvfree(linfo); 12667 return err; 12668 } 12669 12670 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 12671 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 12672 12673 static int check_core_relo(struct bpf_verifier_env *env, 12674 const union bpf_attr *attr, 12675 bpfptr_t uattr) 12676 { 12677 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 12678 struct bpf_core_relo core_relo = {}; 12679 struct bpf_prog *prog = env->prog; 12680 const struct btf *btf = prog->aux->btf; 12681 struct bpf_core_ctx ctx = { 12682 .log = &env->log, 12683 .btf = btf, 12684 }; 12685 bpfptr_t u_core_relo; 12686 int err; 12687 12688 nr_core_relo = attr->core_relo_cnt; 12689 if (!nr_core_relo) 12690 return 0; 12691 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 12692 return -EINVAL; 12693 12694 rec_size = attr->core_relo_rec_size; 12695 if (rec_size < MIN_CORE_RELO_SIZE || 12696 rec_size > MAX_CORE_RELO_SIZE || 12697 rec_size % sizeof(u32)) 12698 return -EINVAL; 12699 12700 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 12701 expected_size = sizeof(struct bpf_core_relo); 12702 ncopy = min_t(u32, expected_size, rec_size); 12703 12704 /* Unlike func_info and line_info, copy and apply each CO-RE 12705 * relocation record one at a time. 12706 */ 12707 for (i = 0; i < nr_core_relo; i++) { 12708 /* future proofing when sizeof(bpf_core_relo) changes */ 12709 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 12710 if (err) { 12711 if (err == -E2BIG) { 12712 verbose(env, "nonzero tailing record in core_relo"); 12713 if (copy_to_bpfptr_offset(uattr, 12714 offsetof(union bpf_attr, core_relo_rec_size), 12715 &expected_size, sizeof(expected_size))) 12716 err = -EFAULT; 12717 } 12718 break; 12719 } 12720 12721 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 12722 err = -EFAULT; 12723 break; 12724 } 12725 12726 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 12727 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 12728 i, core_relo.insn_off, prog->len); 12729 err = -EINVAL; 12730 break; 12731 } 12732 12733 err = bpf_core_apply(&ctx, &core_relo, i, 12734 &prog->insnsi[core_relo.insn_off / 8]); 12735 if (err) 12736 break; 12737 bpfptr_add(&u_core_relo, rec_size); 12738 } 12739 return err; 12740 } 12741 12742 static int check_btf_info(struct bpf_verifier_env *env, 12743 const union bpf_attr *attr, 12744 bpfptr_t uattr) 12745 { 12746 struct btf *btf; 12747 int err; 12748 12749 if (!attr->func_info_cnt && !attr->line_info_cnt) { 12750 if (check_abnormal_return(env)) 12751 return -EINVAL; 12752 return 0; 12753 } 12754 12755 btf = btf_get_by_fd(attr->prog_btf_fd); 12756 if (IS_ERR(btf)) 12757 return PTR_ERR(btf); 12758 if (btf_is_kernel(btf)) { 12759 btf_put(btf); 12760 return -EACCES; 12761 } 12762 env->prog->aux->btf = btf; 12763 12764 err = check_btf_func(env, attr, uattr); 12765 if (err) 12766 return err; 12767 12768 err = check_btf_line(env, attr, uattr); 12769 if (err) 12770 return err; 12771 12772 err = check_core_relo(env, attr, uattr); 12773 if (err) 12774 return err; 12775 12776 return 0; 12777 } 12778 12779 /* check %cur's range satisfies %old's */ 12780 static bool range_within(struct bpf_reg_state *old, 12781 struct bpf_reg_state *cur) 12782 { 12783 return old->umin_value <= cur->umin_value && 12784 old->umax_value >= cur->umax_value && 12785 old->smin_value <= cur->smin_value && 12786 old->smax_value >= cur->smax_value && 12787 old->u32_min_value <= cur->u32_min_value && 12788 old->u32_max_value >= cur->u32_max_value && 12789 old->s32_min_value <= cur->s32_min_value && 12790 old->s32_max_value >= cur->s32_max_value; 12791 } 12792 12793 /* If in the old state two registers had the same id, then they need to have 12794 * the same id in the new state as well. But that id could be different from 12795 * the old state, so we need to track the mapping from old to new ids. 12796 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 12797 * regs with old id 5 must also have new id 9 for the new state to be safe. But 12798 * regs with a different old id could still have new id 9, we don't care about 12799 * that. 12800 * So we look through our idmap to see if this old id has been seen before. If 12801 * so, we require the new id to match; otherwise, we add the id pair to the map. 12802 */ 12803 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 12804 { 12805 unsigned int i; 12806 12807 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 12808 if (!idmap[i].old) { 12809 /* Reached an empty slot; haven't seen this id before */ 12810 idmap[i].old = old_id; 12811 idmap[i].cur = cur_id; 12812 return true; 12813 } 12814 if (idmap[i].old == old_id) 12815 return idmap[i].cur == cur_id; 12816 } 12817 /* We ran out of idmap slots, which should be impossible */ 12818 WARN_ON_ONCE(1); 12819 return false; 12820 } 12821 12822 static void clean_func_state(struct bpf_verifier_env *env, 12823 struct bpf_func_state *st) 12824 { 12825 enum bpf_reg_liveness live; 12826 int i, j; 12827 12828 for (i = 0; i < BPF_REG_FP; i++) { 12829 live = st->regs[i].live; 12830 /* liveness must not touch this register anymore */ 12831 st->regs[i].live |= REG_LIVE_DONE; 12832 if (!(live & REG_LIVE_READ)) 12833 /* since the register is unused, clear its state 12834 * to make further comparison simpler 12835 */ 12836 __mark_reg_not_init(env, &st->regs[i]); 12837 } 12838 12839 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 12840 live = st->stack[i].spilled_ptr.live; 12841 /* liveness must not touch this stack slot anymore */ 12842 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 12843 if (!(live & REG_LIVE_READ)) { 12844 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 12845 for (j = 0; j < BPF_REG_SIZE; j++) 12846 st->stack[i].slot_type[j] = STACK_INVALID; 12847 } 12848 } 12849 } 12850 12851 static void clean_verifier_state(struct bpf_verifier_env *env, 12852 struct bpf_verifier_state *st) 12853 { 12854 int i; 12855 12856 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 12857 /* all regs in this state in all frames were already marked */ 12858 return; 12859 12860 for (i = 0; i <= st->curframe; i++) 12861 clean_func_state(env, st->frame[i]); 12862 } 12863 12864 /* the parentage chains form a tree. 12865 * the verifier states are added to state lists at given insn and 12866 * pushed into state stack for future exploration. 12867 * when the verifier reaches bpf_exit insn some of the verifer states 12868 * stored in the state lists have their final liveness state already, 12869 * but a lot of states will get revised from liveness point of view when 12870 * the verifier explores other branches. 12871 * Example: 12872 * 1: r0 = 1 12873 * 2: if r1 == 100 goto pc+1 12874 * 3: r0 = 2 12875 * 4: exit 12876 * when the verifier reaches exit insn the register r0 in the state list of 12877 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 12878 * of insn 2 and goes exploring further. At the insn 4 it will walk the 12879 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 12880 * 12881 * Since the verifier pushes the branch states as it sees them while exploring 12882 * the program the condition of walking the branch instruction for the second 12883 * time means that all states below this branch were already explored and 12884 * their final liveness marks are already propagated. 12885 * Hence when the verifier completes the search of state list in is_state_visited() 12886 * we can call this clean_live_states() function to mark all liveness states 12887 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 12888 * will not be used. 12889 * This function also clears the registers and stack for states that !READ 12890 * to simplify state merging. 12891 * 12892 * Important note here that walking the same branch instruction in the callee 12893 * doesn't meant that the states are DONE. The verifier has to compare 12894 * the callsites 12895 */ 12896 static void clean_live_states(struct bpf_verifier_env *env, int insn, 12897 struct bpf_verifier_state *cur) 12898 { 12899 struct bpf_verifier_state_list *sl; 12900 int i; 12901 12902 sl = *explored_state(env, insn); 12903 while (sl) { 12904 if (sl->state.branches) 12905 goto next; 12906 if (sl->state.insn_idx != insn || 12907 sl->state.curframe != cur->curframe) 12908 goto next; 12909 for (i = 0; i <= cur->curframe; i++) 12910 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 12911 goto next; 12912 clean_verifier_state(env, &sl->state); 12913 next: 12914 sl = sl->next; 12915 } 12916 } 12917 12918 /* Returns true if (rold safe implies rcur safe) */ 12919 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 12920 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 12921 { 12922 bool equal; 12923 12924 if (!(rold->live & REG_LIVE_READ)) 12925 /* explored state didn't use this */ 12926 return true; 12927 12928 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 12929 12930 if (rold->type == PTR_TO_STACK) 12931 /* two stack pointers are equal only if they're pointing to 12932 * the same stack frame, since fp-8 in foo != fp-8 in bar 12933 */ 12934 return equal && rold->frameno == rcur->frameno; 12935 12936 if (equal) 12937 return true; 12938 12939 if (rold->type == NOT_INIT) 12940 /* explored state can't have used this */ 12941 return true; 12942 if (rcur->type == NOT_INIT) 12943 return false; 12944 switch (base_type(rold->type)) { 12945 case SCALAR_VALUE: 12946 if (env->explore_alu_limits) 12947 return false; 12948 if (rcur->type == SCALAR_VALUE) { 12949 if (!rold->precise) 12950 return true; 12951 /* new val must satisfy old val knowledge */ 12952 return range_within(rold, rcur) && 12953 tnum_in(rold->var_off, rcur->var_off); 12954 } else { 12955 /* We're trying to use a pointer in place of a scalar. 12956 * Even if the scalar was unbounded, this could lead to 12957 * pointer leaks because scalars are allowed to leak 12958 * while pointers are not. We could make this safe in 12959 * special cases if root is calling us, but it's 12960 * probably not worth the hassle. 12961 */ 12962 return false; 12963 } 12964 case PTR_TO_MAP_KEY: 12965 case PTR_TO_MAP_VALUE: 12966 /* a PTR_TO_MAP_VALUE could be safe to use as a 12967 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 12968 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 12969 * checked, doing so could have affected others with the same 12970 * id, and we can't check for that because we lost the id when 12971 * we converted to a PTR_TO_MAP_VALUE. 12972 */ 12973 if (type_may_be_null(rold->type)) { 12974 if (!type_may_be_null(rcur->type)) 12975 return false; 12976 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 12977 return false; 12978 /* Check our ids match any regs they're supposed to */ 12979 return check_ids(rold->id, rcur->id, idmap); 12980 } 12981 12982 /* If the new min/max/var_off satisfy the old ones and 12983 * everything else matches, we are OK. 12984 * 'id' is not compared, since it's only used for maps with 12985 * bpf_spin_lock inside map element and in such cases if 12986 * the rest of the prog is valid for one map element then 12987 * it's valid for all map elements regardless of the key 12988 * used in bpf_map_lookup() 12989 */ 12990 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 12991 range_within(rold, rcur) && 12992 tnum_in(rold->var_off, rcur->var_off); 12993 case PTR_TO_PACKET_META: 12994 case PTR_TO_PACKET: 12995 if (rcur->type != rold->type) 12996 return false; 12997 /* We must have at least as much range as the old ptr 12998 * did, so that any accesses which were safe before are 12999 * still safe. This is true even if old range < old off, 13000 * since someone could have accessed through (ptr - k), or 13001 * even done ptr -= k in a register, to get a safe access. 13002 */ 13003 if (rold->range > rcur->range) 13004 return false; 13005 /* If the offsets don't match, we can't trust our alignment; 13006 * nor can we be sure that we won't fall out of range. 13007 */ 13008 if (rold->off != rcur->off) 13009 return false; 13010 /* id relations must be preserved */ 13011 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 13012 return false; 13013 /* new val must satisfy old val knowledge */ 13014 return range_within(rold, rcur) && 13015 tnum_in(rold->var_off, rcur->var_off); 13016 case PTR_TO_CTX: 13017 case CONST_PTR_TO_MAP: 13018 case PTR_TO_PACKET_END: 13019 case PTR_TO_FLOW_KEYS: 13020 case PTR_TO_SOCKET: 13021 case PTR_TO_SOCK_COMMON: 13022 case PTR_TO_TCP_SOCK: 13023 case PTR_TO_XDP_SOCK: 13024 /* Only valid matches are exact, which memcmp() above 13025 * would have accepted 13026 */ 13027 default: 13028 /* Don't know what's going on, just say it's not safe */ 13029 return false; 13030 } 13031 13032 /* Shouldn't get here; if we do, say it's not safe */ 13033 WARN_ON_ONCE(1); 13034 return false; 13035 } 13036 13037 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 13038 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 13039 { 13040 int i, spi; 13041 13042 /* walk slots of the explored stack and ignore any additional 13043 * slots in the current stack, since explored(safe) state 13044 * didn't use them 13045 */ 13046 for (i = 0; i < old->allocated_stack; i++) { 13047 spi = i / BPF_REG_SIZE; 13048 13049 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 13050 i += BPF_REG_SIZE - 1; 13051 /* explored state didn't use this */ 13052 continue; 13053 } 13054 13055 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 13056 continue; 13057 13058 /* explored stack has more populated slots than current stack 13059 * and these slots were used 13060 */ 13061 if (i >= cur->allocated_stack) 13062 return false; 13063 13064 /* if old state was safe with misc data in the stack 13065 * it will be safe with zero-initialized stack. 13066 * The opposite is not true 13067 */ 13068 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 13069 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 13070 continue; 13071 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 13072 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 13073 /* Ex: old explored (safe) state has STACK_SPILL in 13074 * this stack slot, but current has STACK_MISC -> 13075 * this verifier states are not equivalent, 13076 * return false to continue verification of this path 13077 */ 13078 return false; 13079 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 13080 continue; 13081 if (!is_spilled_reg(&old->stack[spi])) 13082 continue; 13083 if (!regsafe(env, &old->stack[spi].spilled_ptr, 13084 &cur->stack[spi].spilled_ptr, idmap)) 13085 /* when explored and current stack slot are both storing 13086 * spilled registers, check that stored pointers types 13087 * are the same as well. 13088 * Ex: explored safe path could have stored 13089 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 13090 * but current path has stored: 13091 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 13092 * such verifier states are not equivalent. 13093 * return false to continue verification of this path 13094 */ 13095 return false; 13096 } 13097 return true; 13098 } 13099 13100 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 13101 { 13102 if (old->acquired_refs != cur->acquired_refs) 13103 return false; 13104 return !memcmp(old->refs, cur->refs, 13105 sizeof(*old->refs) * old->acquired_refs); 13106 } 13107 13108 /* compare two verifier states 13109 * 13110 * all states stored in state_list are known to be valid, since 13111 * verifier reached 'bpf_exit' instruction through them 13112 * 13113 * this function is called when verifier exploring different branches of 13114 * execution popped from the state stack. If it sees an old state that has 13115 * more strict register state and more strict stack state then this execution 13116 * branch doesn't need to be explored further, since verifier already 13117 * concluded that more strict state leads to valid finish. 13118 * 13119 * Therefore two states are equivalent if register state is more conservative 13120 * and explored stack state is more conservative than the current one. 13121 * Example: 13122 * explored current 13123 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 13124 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 13125 * 13126 * In other words if current stack state (one being explored) has more 13127 * valid slots than old one that already passed validation, it means 13128 * the verifier can stop exploring and conclude that current state is valid too 13129 * 13130 * Similarly with registers. If explored state has register type as invalid 13131 * whereas register type in current state is meaningful, it means that 13132 * the current state will reach 'bpf_exit' instruction safely 13133 */ 13134 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 13135 struct bpf_func_state *cur) 13136 { 13137 int i; 13138 13139 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 13140 for (i = 0; i < MAX_BPF_REG; i++) 13141 if (!regsafe(env, &old->regs[i], &cur->regs[i], 13142 env->idmap_scratch)) 13143 return false; 13144 13145 if (!stacksafe(env, old, cur, env->idmap_scratch)) 13146 return false; 13147 13148 if (!refsafe(old, cur)) 13149 return false; 13150 13151 return true; 13152 } 13153 13154 static bool states_equal(struct bpf_verifier_env *env, 13155 struct bpf_verifier_state *old, 13156 struct bpf_verifier_state *cur) 13157 { 13158 int i; 13159 13160 if (old->curframe != cur->curframe) 13161 return false; 13162 13163 /* Verification state from speculative execution simulation 13164 * must never prune a non-speculative execution one. 13165 */ 13166 if (old->speculative && !cur->speculative) 13167 return false; 13168 13169 if (old->active_lock.ptr != cur->active_lock.ptr || 13170 old->active_lock.id != cur->active_lock.id) 13171 return false; 13172 13173 if (old->active_rcu_lock != cur->active_rcu_lock) 13174 return false; 13175 13176 /* for states to be equal callsites have to be the same 13177 * and all frame states need to be equivalent 13178 */ 13179 for (i = 0; i <= old->curframe; i++) { 13180 if (old->frame[i]->callsite != cur->frame[i]->callsite) 13181 return false; 13182 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 13183 return false; 13184 } 13185 return true; 13186 } 13187 13188 /* Return 0 if no propagation happened. Return negative error code if error 13189 * happened. Otherwise, return the propagated bit. 13190 */ 13191 static int propagate_liveness_reg(struct bpf_verifier_env *env, 13192 struct bpf_reg_state *reg, 13193 struct bpf_reg_state *parent_reg) 13194 { 13195 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 13196 u8 flag = reg->live & REG_LIVE_READ; 13197 int err; 13198 13199 /* When comes here, read flags of PARENT_REG or REG could be any of 13200 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 13201 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 13202 */ 13203 if (parent_flag == REG_LIVE_READ64 || 13204 /* Or if there is no read flag from REG. */ 13205 !flag || 13206 /* Or if the read flag from REG is the same as PARENT_REG. */ 13207 parent_flag == flag) 13208 return 0; 13209 13210 err = mark_reg_read(env, reg, parent_reg, flag); 13211 if (err) 13212 return err; 13213 13214 return flag; 13215 } 13216 13217 /* A write screens off any subsequent reads; but write marks come from the 13218 * straight-line code between a state and its parent. When we arrive at an 13219 * equivalent state (jump target or such) we didn't arrive by the straight-line 13220 * code, so read marks in the state must propagate to the parent regardless 13221 * of the state's write marks. That's what 'parent == state->parent' comparison 13222 * in mark_reg_read() is for. 13223 */ 13224 static int propagate_liveness(struct bpf_verifier_env *env, 13225 const struct bpf_verifier_state *vstate, 13226 struct bpf_verifier_state *vparent) 13227 { 13228 struct bpf_reg_state *state_reg, *parent_reg; 13229 struct bpf_func_state *state, *parent; 13230 int i, frame, err = 0; 13231 13232 if (vparent->curframe != vstate->curframe) { 13233 WARN(1, "propagate_live: parent frame %d current frame %d\n", 13234 vparent->curframe, vstate->curframe); 13235 return -EFAULT; 13236 } 13237 /* Propagate read liveness of registers... */ 13238 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 13239 for (frame = 0; frame <= vstate->curframe; frame++) { 13240 parent = vparent->frame[frame]; 13241 state = vstate->frame[frame]; 13242 parent_reg = parent->regs; 13243 state_reg = state->regs; 13244 /* We don't need to worry about FP liveness, it's read-only */ 13245 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 13246 err = propagate_liveness_reg(env, &state_reg[i], 13247 &parent_reg[i]); 13248 if (err < 0) 13249 return err; 13250 if (err == REG_LIVE_READ64) 13251 mark_insn_zext(env, &parent_reg[i]); 13252 } 13253 13254 /* Propagate stack slots. */ 13255 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 13256 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 13257 parent_reg = &parent->stack[i].spilled_ptr; 13258 state_reg = &state->stack[i].spilled_ptr; 13259 err = propagate_liveness_reg(env, state_reg, 13260 parent_reg); 13261 if (err < 0) 13262 return err; 13263 } 13264 } 13265 return 0; 13266 } 13267 13268 /* find precise scalars in the previous equivalent state and 13269 * propagate them into the current state 13270 */ 13271 static int propagate_precision(struct bpf_verifier_env *env, 13272 const struct bpf_verifier_state *old) 13273 { 13274 struct bpf_reg_state *state_reg; 13275 struct bpf_func_state *state; 13276 int i, err = 0, fr; 13277 13278 for (fr = old->curframe; fr >= 0; fr--) { 13279 state = old->frame[fr]; 13280 state_reg = state->regs; 13281 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 13282 if (state_reg->type != SCALAR_VALUE || 13283 !state_reg->precise) 13284 continue; 13285 if (env->log.level & BPF_LOG_LEVEL2) 13286 verbose(env, "frame %d: propagating r%d\n", i, fr); 13287 err = mark_chain_precision_frame(env, fr, i); 13288 if (err < 0) 13289 return err; 13290 } 13291 13292 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 13293 if (!is_spilled_reg(&state->stack[i])) 13294 continue; 13295 state_reg = &state->stack[i].spilled_ptr; 13296 if (state_reg->type != SCALAR_VALUE || 13297 !state_reg->precise) 13298 continue; 13299 if (env->log.level & BPF_LOG_LEVEL2) 13300 verbose(env, "frame %d: propagating fp%d\n", 13301 (-i - 1) * BPF_REG_SIZE, fr); 13302 err = mark_chain_precision_stack_frame(env, fr, i); 13303 if (err < 0) 13304 return err; 13305 } 13306 } 13307 return 0; 13308 } 13309 13310 static bool states_maybe_looping(struct bpf_verifier_state *old, 13311 struct bpf_verifier_state *cur) 13312 { 13313 struct bpf_func_state *fold, *fcur; 13314 int i, fr = cur->curframe; 13315 13316 if (old->curframe != fr) 13317 return false; 13318 13319 fold = old->frame[fr]; 13320 fcur = cur->frame[fr]; 13321 for (i = 0; i < MAX_BPF_REG; i++) 13322 if (memcmp(&fold->regs[i], &fcur->regs[i], 13323 offsetof(struct bpf_reg_state, parent))) 13324 return false; 13325 return true; 13326 } 13327 13328 13329 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 13330 { 13331 struct bpf_verifier_state_list *new_sl; 13332 struct bpf_verifier_state_list *sl, **pprev; 13333 struct bpf_verifier_state *cur = env->cur_state, *new; 13334 int i, j, err, states_cnt = 0; 13335 bool add_new_state = env->test_state_freq ? true : false; 13336 13337 /* bpf progs typically have pruning point every 4 instructions 13338 * http://vger.kernel.org/bpfconf2019.html#session-1 13339 * Do not add new state for future pruning if the verifier hasn't seen 13340 * at least 2 jumps and at least 8 instructions. 13341 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 13342 * In tests that amounts to up to 50% reduction into total verifier 13343 * memory consumption and 20% verifier time speedup. 13344 */ 13345 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 13346 env->insn_processed - env->prev_insn_processed >= 8) 13347 add_new_state = true; 13348 13349 pprev = explored_state(env, insn_idx); 13350 sl = *pprev; 13351 13352 clean_live_states(env, insn_idx, cur); 13353 13354 while (sl) { 13355 states_cnt++; 13356 if (sl->state.insn_idx != insn_idx) 13357 goto next; 13358 13359 if (sl->state.branches) { 13360 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 13361 13362 if (frame->in_async_callback_fn && 13363 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 13364 /* Different async_entry_cnt means that the verifier is 13365 * processing another entry into async callback. 13366 * Seeing the same state is not an indication of infinite 13367 * loop or infinite recursion. 13368 * But finding the same state doesn't mean that it's safe 13369 * to stop processing the current state. The previous state 13370 * hasn't yet reached bpf_exit, since state.branches > 0. 13371 * Checking in_async_callback_fn alone is not enough either. 13372 * Since the verifier still needs to catch infinite loops 13373 * inside async callbacks. 13374 */ 13375 } else if (states_maybe_looping(&sl->state, cur) && 13376 states_equal(env, &sl->state, cur)) { 13377 verbose_linfo(env, insn_idx, "; "); 13378 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 13379 return -EINVAL; 13380 } 13381 /* if the verifier is processing a loop, avoid adding new state 13382 * too often, since different loop iterations have distinct 13383 * states and may not help future pruning. 13384 * This threshold shouldn't be too low to make sure that 13385 * a loop with large bound will be rejected quickly. 13386 * The most abusive loop will be: 13387 * r1 += 1 13388 * if r1 < 1000000 goto pc-2 13389 * 1M insn_procssed limit / 100 == 10k peak states. 13390 * This threshold shouldn't be too high either, since states 13391 * at the end of the loop are likely to be useful in pruning. 13392 */ 13393 if (env->jmps_processed - env->prev_jmps_processed < 20 && 13394 env->insn_processed - env->prev_insn_processed < 100) 13395 add_new_state = false; 13396 goto miss; 13397 } 13398 if (states_equal(env, &sl->state, cur)) { 13399 sl->hit_cnt++; 13400 /* reached equivalent register/stack state, 13401 * prune the search. 13402 * Registers read by the continuation are read by us. 13403 * If we have any write marks in env->cur_state, they 13404 * will prevent corresponding reads in the continuation 13405 * from reaching our parent (an explored_state). Our 13406 * own state will get the read marks recorded, but 13407 * they'll be immediately forgotten as we're pruning 13408 * this state and will pop a new one. 13409 */ 13410 err = propagate_liveness(env, &sl->state, cur); 13411 13412 /* if previous state reached the exit with precision and 13413 * current state is equivalent to it (except precsion marks) 13414 * the precision needs to be propagated back in 13415 * the current state. 13416 */ 13417 err = err ? : push_jmp_history(env, cur); 13418 err = err ? : propagate_precision(env, &sl->state); 13419 if (err) 13420 return err; 13421 return 1; 13422 } 13423 miss: 13424 /* when new state is not going to be added do not increase miss count. 13425 * Otherwise several loop iterations will remove the state 13426 * recorded earlier. The goal of these heuristics is to have 13427 * states from some iterations of the loop (some in the beginning 13428 * and some at the end) to help pruning. 13429 */ 13430 if (add_new_state) 13431 sl->miss_cnt++; 13432 /* heuristic to determine whether this state is beneficial 13433 * to keep checking from state equivalence point of view. 13434 * Higher numbers increase max_states_per_insn and verification time, 13435 * but do not meaningfully decrease insn_processed. 13436 */ 13437 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 13438 /* the state is unlikely to be useful. Remove it to 13439 * speed up verification 13440 */ 13441 *pprev = sl->next; 13442 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 13443 u32 br = sl->state.branches; 13444 13445 WARN_ONCE(br, 13446 "BUG live_done but branches_to_explore %d\n", 13447 br); 13448 free_verifier_state(&sl->state, false); 13449 kfree(sl); 13450 env->peak_states--; 13451 } else { 13452 /* cannot free this state, since parentage chain may 13453 * walk it later. Add it for free_list instead to 13454 * be freed at the end of verification 13455 */ 13456 sl->next = env->free_list; 13457 env->free_list = sl; 13458 } 13459 sl = *pprev; 13460 continue; 13461 } 13462 next: 13463 pprev = &sl->next; 13464 sl = *pprev; 13465 } 13466 13467 if (env->max_states_per_insn < states_cnt) 13468 env->max_states_per_insn = states_cnt; 13469 13470 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 13471 return 0; 13472 13473 if (!add_new_state) 13474 return 0; 13475 13476 /* There were no equivalent states, remember the current one. 13477 * Technically the current state is not proven to be safe yet, 13478 * but it will either reach outer most bpf_exit (which means it's safe) 13479 * or it will be rejected. When there are no loops the verifier won't be 13480 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 13481 * again on the way to bpf_exit. 13482 * When looping the sl->state.branches will be > 0 and this state 13483 * will not be considered for equivalence until branches == 0. 13484 */ 13485 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 13486 if (!new_sl) 13487 return -ENOMEM; 13488 env->total_states++; 13489 env->peak_states++; 13490 env->prev_jmps_processed = env->jmps_processed; 13491 env->prev_insn_processed = env->insn_processed; 13492 13493 /* forget precise markings we inherited, see __mark_chain_precision */ 13494 if (env->bpf_capable) 13495 mark_all_scalars_imprecise(env, cur); 13496 13497 /* add new state to the head of linked list */ 13498 new = &new_sl->state; 13499 err = copy_verifier_state(new, cur); 13500 if (err) { 13501 free_verifier_state(new, false); 13502 kfree(new_sl); 13503 return err; 13504 } 13505 new->insn_idx = insn_idx; 13506 WARN_ONCE(new->branches != 1, 13507 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 13508 13509 cur->parent = new; 13510 cur->first_insn_idx = insn_idx; 13511 clear_jmp_history(cur); 13512 new_sl->next = *explored_state(env, insn_idx); 13513 *explored_state(env, insn_idx) = new_sl; 13514 /* connect new state to parentage chain. Current frame needs all 13515 * registers connected. Only r6 - r9 of the callers are alive (pushed 13516 * to the stack implicitly by JITs) so in callers' frames connect just 13517 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 13518 * the state of the call instruction (with WRITTEN set), and r0 comes 13519 * from callee with its full parentage chain, anyway. 13520 */ 13521 /* clear write marks in current state: the writes we did are not writes 13522 * our child did, so they don't screen off its reads from us. 13523 * (There are no read marks in current state, because reads always mark 13524 * their parent and current state never has children yet. Only 13525 * explored_states can get read marks.) 13526 */ 13527 for (j = 0; j <= cur->curframe; j++) { 13528 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 13529 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 13530 for (i = 0; i < BPF_REG_FP; i++) 13531 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 13532 } 13533 13534 /* all stack frames are accessible from callee, clear them all */ 13535 for (j = 0; j <= cur->curframe; j++) { 13536 struct bpf_func_state *frame = cur->frame[j]; 13537 struct bpf_func_state *newframe = new->frame[j]; 13538 13539 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 13540 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 13541 frame->stack[i].spilled_ptr.parent = 13542 &newframe->stack[i].spilled_ptr; 13543 } 13544 } 13545 return 0; 13546 } 13547 13548 /* Return true if it's OK to have the same insn return a different type. */ 13549 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 13550 { 13551 switch (base_type(type)) { 13552 case PTR_TO_CTX: 13553 case PTR_TO_SOCKET: 13554 case PTR_TO_SOCK_COMMON: 13555 case PTR_TO_TCP_SOCK: 13556 case PTR_TO_XDP_SOCK: 13557 case PTR_TO_BTF_ID: 13558 return false; 13559 default: 13560 return true; 13561 } 13562 } 13563 13564 /* If an instruction was previously used with particular pointer types, then we 13565 * need to be careful to avoid cases such as the below, where it may be ok 13566 * for one branch accessing the pointer, but not ok for the other branch: 13567 * 13568 * R1 = sock_ptr 13569 * goto X; 13570 * ... 13571 * R1 = some_other_valid_ptr; 13572 * goto X; 13573 * ... 13574 * R2 = *(u32 *)(R1 + 0); 13575 */ 13576 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 13577 { 13578 return src != prev && (!reg_type_mismatch_ok(src) || 13579 !reg_type_mismatch_ok(prev)); 13580 } 13581 13582 static int do_check(struct bpf_verifier_env *env) 13583 { 13584 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 13585 struct bpf_verifier_state *state = env->cur_state; 13586 struct bpf_insn *insns = env->prog->insnsi; 13587 struct bpf_reg_state *regs; 13588 int insn_cnt = env->prog->len; 13589 bool do_print_state = false; 13590 int prev_insn_idx = -1; 13591 13592 for (;;) { 13593 struct bpf_insn *insn; 13594 u8 class; 13595 int err; 13596 13597 env->prev_insn_idx = prev_insn_idx; 13598 if (env->insn_idx >= insn_cnt) { 13599 verbose(env, "invalid insn idx %d insn_cnt %d\n", 13600 env->insn_idx, insn_cnt); 13601 return -EFAULT; 13602 } 13603 13604 insn = &insns[env->insn_idx]; 13605 class = BPF_CLASS(insn->code); 13606 13607 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 13608 verbose(env, 13609 "BPF program is too large. Processed %d insn\n", 13610 env->insn_processed); 13611 return -E2BIG; 13612 } 13613 13614 state->last_insn_idx = env->prev_insn_idx; 13615 13616 if (is_prune_point(env, env->insn_idx)) { 13617 err = is_state_visited(env, env->insn_idx); 13618 if (err < 0) 13619 return err; 13620 if (err == 1) { 13621 /* found equivalent state, can prune the search */ 13622 if (env->log.level & BPF_LOG_LEVEL) { 13623 if (do_print_state) 13624 verbose(env, "\nfrom %d to %d%s: safe\n", 13625 env->prev_insn_idx, env->insn_idx, 13626 env->cur_state->speculative ? 13627 " (speculative execution)" : ""); 13628 else 13629 verbose(env, "%d: safe\n", env->insn_idx); 13630 } 13631 goto process_bpf_exit; 13632 } 13633 } 13634 13635 if (is_jmp_point(env, env->insn_idx)) { 13636 err = push_jmp_history(env, state); 13637 if (err) 13638 return err; 13639 } 13640 13641 if (signal_pending(current)) 13642 return -EAGAIN; 13643 13644 if (need_resched()) 13645 cond_resched(); 13646 13647 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 13648 verbose(env, "\nfrom %d to %d%s:", 13649 env->prev_insn_idx, env->insn_idx, 13650 env->cur_state->speculative ? 13651 " (speculative execution)" : ""); 13652 print_verifier_state(env, state->frame[state->curframe], true); 13653 do_print_state = false; 13654 } 13655 13656 if (env->log.level & BPF_LOG_LEVEL) { 13657 const struct bpf_insn_cbs cbs = { 13658 .cb_call = disasm_kfunc_name, 13659 .cb_print = verbose, 13660 .private_data = env, 13661 }; 13662 13663 if (verifier_state_scratched(env)) 13664 print_insn_state(env, state->frame[state->curframe]); 13665 13666 verbose_linfo(env, env->insn_idx, "; "); 13667 env->prev_log_len = env->log.len_used; 13668 verbose(env, "%d: ", env->insn_idx); 13669 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 13670 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; 13671 env->prev_log_len = env->log.len_used; 13672 } 13673 13674 if (bpf_prog_is_dev_bound(env->prog->aux)) { 13675 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 13676 env->prev_insn_idx); 13677 if (err) 13678 return err; 13679 } 13680 13681 regs = cur_regs(env); 13682 sanitize_mark_insn_seen(env); 13683 prev_insn_idx = env->insn_idx; 13684 13685 if (class == BPF_ALU || class == BPF_ALU64) { 13686 err = check_alu_op(env, insn); 13687 if (err) 13688 return err; 13689 13690 } else if (class == BPF_LDX) { 13691 enum bpf_reg_type *prev_src_type, src_reg_type; 13692 13693 /* check for reserved fields is already done */ 13694 13695 /* check src operand */ 13696 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13697 if (err) 13698 return err; 13699 13700 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 13701 if (err) 13702 return err; 13703 13704 src_reg_type = regs[insn->src_reg].type; 13705 13706 /* check that memory (src_reg + off) is readable, 13707 * the state of dst_reg will be updated by this func 13708 */ 13709 err = check_mem_access(env, env->insn_idx, insn->src_reg, 13710 insn->off, BPF_SIZE(insn->code), 13711 BPF_READ, insn->dst_reg, false); 13712 if (err) 13713 return err; 13714 13715 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 13716 13717 if (*prev_src_type == NOT_INIT) { 13718 /* saw a valid insn 13719 * dst_reg = *(u32 *)(src_reg + off) 13720 * save type to validate intersecting paths 13721 */ 13722 *prev_src_type = src_reg_type; 13723 13724 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 13725 /* ABuser program is trying to use the same insn 13726 * dst_reg = *(u32*) (src_reg + off) 13727 * with different pointer types: 13728 * src_reg == ctx in one branch and 13729 * src_reg == stack|map in some other branch. 13730 * Reject it. 13731 */ 13732 verbose(env, "same insn cannot be used with different pointers\n"); 13733 return -EINVAL; 13734 } 13735 13736 } else if (class == BPF_STX) { 13737 enum bpf_reg_type *prev_dst_type, dst_reg_type; 13738 13739 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 13740 err = check_atomic(env, env->insn_idx, insn); 13741 if (err) 13742 return err; 13743 env->insn_idx++; 13744 continue; 13745 } 13746 13747 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 13748 verbose(env, "BPF_STX uses reserved fields\n"); 13749 return -EINVAL; 13750 } 13751 13752 /* check src1 operand */ 13753 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13754 if (err) 13755 return err; 13756 /* check src2 operand */ 13757 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 13758 if (err) 13759 return err; 13760 13761 dst_reg_type = regs[insn->dst_reg].type; 13762 13763 /* check that memory (dst_reg + off) is writeable */ 13764 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 13765 insn->off, BPF_SIZE(insn->code), 13766 BPF_WRITE, insn->src_reg, false); 13767 if (err) 13768 return err; 13769 13770 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 13771 13772 if (*prev_dst_type == NOT_INIT) { 13773 *prev_dst_type = dst_reg_type; 13774 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 13775 verbose(env, "same insn cannot be used with different pointers\n"); 13776 return -EINVAL; 13777 } 13778 13779 } else if (class == BPF_ST) { 13780 if (BPF_MODE(insn->code) != BPF_MEM || 13781 insn->src_reg != BPF_REG_0) { 13782 verbose(env, "BPF_ST uses reserved fields\n"); 13783 return -EINVAL; 13784 } 13785 /* check src operand */ 13786 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 13787 if (err) 13788 return err; 13789 13790 if (is_ctx_reg(env, insn->dst_reg)) { 13791 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 13792 insn->dst_reg, 13793 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 13794 return -EACCES; 13795 } 13796 13797 /* check that memory (dst_reg + off) is writeable */ 13798 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 13799 insn->off, BPF_SIZE(insn->code), 13800 BPF_WRITE, -1, false); 13801 if (err) 13802 return err; 13803 13804 } else if (class == BPF_JMP || class == BPF_JMP32) { 13805 u8 opcode = BPF_OP(insn->code); 13806 13807 env->jmps_processed++; 13808 if (opcode == BPF_CALL) { 13809 if (BPF_SRC(insn->code) != BPF_K || 13810 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 13811 && insn->off != 0) || 13812 (insn->src_reg != BPF_REG_0 && 13813 insn->src_reg != BPF_PSEUDO_CALL && 13814 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 13815 insn->dst_reg != BPF_REG_0 || 13816 class == BPF_JMP32) { 13817 verbose(env, "BPF_CALL uses reserved fields\n"); 13818 return -EINVAL; 13819 } 13820 13821 if (env->cur_state->active_lock.ptr) { 13822 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 13823 (insn->src_reg == BPF_PSEUDO_CALL) || 13824 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 13825 (insn->off != 0 || !is_bpf_list_api_kfunc(insn->imm)))) { 13826 verbose(env, "function calls are not allowed while holding a lock\n"); 13827 return -EINVAL; 13828 } 13829 } 13830 if (insn->src_reg == BPF_PSEUDO_CALL) 13831 err = check_func_call(env, insn, &env->insn_idx); 13832 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 13833 err = check_kfunc_call(env, insn, &env->insn_idx); 13834 else 13835 err = check_helper_call(env, insn, &env->insn_idx); 13836 if (err) 13837 return err; 13838 } else if (opcode == BPF_JA) { 13839 if (BPF_SRC(insn->code) != BPF_K || 13840 insn->imm != 0 || 13841 insn->src_reg != BPF_REG_0 || 13842 insn->dst_reg != BPF_REG_0 || 13843 class == BPF_JMP32) { 13844 verbose(env, "BPF_JA uses reserved fields\n"); 13845 return -EINVAL; 13846 } 13847 13848 env->insn_idx += insn->off + 1; 13849 continue; 13850 13851 } else if (opcode == BPF_EXIT) { 13852 if (BPF_SRC(insn->code) != BPF_K || 13853 insn->imm != 0 || 13854 insn->src_reg != BPF_REG_0 || 13855 insn->dst_reg != BPF_REG_0 || 13856 class == BPF_JMP32) { 13857 verbose(env, "BPF_EXIT uses reserved fields\n"); 13858 return -EINVAL; 13859 } 13860 13861 if (env->cur_state->active_lock.ptr) { 13862 verbose(env, "bpf_spin_unlock is missing\n"); 13863 return -EINVAL; 13864 } 13865 13866 if (env->cur_state->active_rcu_lock) { 13867 verbose(env, "bpf_rcu_read_unlock is missing\n"); 13868 return -EINVAL; 13869 } 13870 13871 /* We must do check_reference_leak here before 13872 * prepare_func_exit to handle the case when 13873 * state->curframe > 0, it may be a callback 13874 * function, for which reference_state must 13875 * match caller reference state when it exits. 13876 */ 13877 err = check_reference_leak(env); 13878 if (err) 13879 return err; 13880 13881 if (state->curframe) { 13882 /* exit from nested function */ 13883 err = prepare_func_exit(env, &env->insn_idx); 13884 if (err) 13885 return err; 13886 do_print_state = true; 13887 continue; 13888 } 13889 13890 err = check_return_code(env); 13891 if (err) 13892 return err; 13893 process_bpf_exit: 13894 mark_verifier_state_scratched(env); 13895 update_branch_counts(env, env->cur_state); 13896 err = pop_stack(env, &prev_insn_idx, 13897 &env->insn_idx, pop_log); 13898 if (err < 0) { 13899 if (err != -ENOENT) 13900 return err; 13901 break; 13902 } else { 13903 do_print_state = true; 13904 continue; 13905 } 13906 } else { 13907 err = check_cond_jmp_op(env, insn, &env->insn_idx); 13908 if (err) 13909 return err; 13910 } 13911 } else if (class == BPF_LD) { 13912 u8 mode = BPF_MODE(insn->code); 13913 13914 if (mode == BPF_ABS || mode == BPF_IND) { 13915 err = check_ld_abs(env, insn); 13916 if (err) 13917 return err; 13918 13919 } else if (mode == BPF_IMM) { 13920 err = check_ld_imm(env, insn); 13921 if (err) 13922 return err; 13923 13924 env->insn_idx++; 13925 sanitize_mark_insn_seen(env); 13926 } else { 13927 verbose(env, "invalid BPF_LD mode\n"); 13928 return -EINVAL; 13929 } 13930 } else { 13931 verbose(env, "unknown insn class %d\n", class); 13932 return -EINVAL; 13933 } 13934 13935 env->insn_idx++; 13936 } 13937 13938 return 0; 13939 } 13940 13941 static int find_btf_percpu_datasec(struct btf *btf) 13942 { 13943 const struct btf_type *t; 13944 const char *tname; 13945 int i, n; 13946 13947 /* 13948 * Both vmlinux and module each have their own ".data..percpu" 13949 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 13950 * types to look at only module's own BTF types. 13951 */ 13952 n = btf_nr_types(btf); 13953 if (btf_is_module(btf)) 13954 i = btf_nr_types(btf_vmlinux); 13955 else 13956 i = 1; 13957 13958 for(; i < n; i++) { 13959 t = btf_type_by_id(btf, i); 13960 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 13961 continue; 13962 13963 tname = btf_name_by_offset(btf, t->name_off); 13964 if (!strcmp(tname, ".data..percpu")) 13965 return i; 13966 } 13967 13968 return -ENOENT; 13969 } 13970 13971 /* replace pseudo btf_id with kernel symbol address */ 13972 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 13973 struct bpf_insn *insn, 13974 struct bpf_insn_aux_data *aux) 13975 { 13976 const struct btf_var_secinfo *vsi; 13977 const struct btf_type *datasec; 13978 struct btf_mod_pair *btf_mod; 13979 const struct btf_type *t; 13980 const char *sym_name; 13981 bool percpu = false; 13982 u32 type, id = insn->imm; 13983 struct btf *btf; 13984 s32 datasec_id; 13985 u64 addr; 13986 int i, btf_fd, err; 13987 13988 btf_fd = insn[1].imm; 13989 if (btf_fd) { 13990 btf = btf_get_by_fd(btf_fd); 13991 if (IS_ERR(btf)) { 13992 verbose(env, "invalid module BTF object FD specified.\n"); 13993 return -EINVAL; 13994 } 13995 } else { 13996 if (!btf_vmlinux) { 13997 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 13998 return -EINVAL; 13999 } 14000 btf = btf_vmlinux; 14001 btf_get(btf); 14002 } 14003 14004 t = btf_type_by_id(btf, id); 14005 if (!t) { 14006 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 14007 err = -ENOENT; 14008 goto err_put; 14009 } 14010 14011 if (!btf_type_is_var(t)) { 14012 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 14013 err = -EINVAL; 14014 goto err_put; 14015 } 14016 14017 sym_name = btf_name_by_offset(btf, t->name_off); 14018 addr = kallsyms_lookup_name(sym_name); 14019 if (!addr) { 14020 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 14021 sym_name); 14022 err = -ENOENT; 14023 goto err_put; 14024 } 14025 14026 datasec_id = find_btf_percpu_datasec(btf); 14027 if (datasec_id > 0) { 14028 datasec = btf_type_by_id(btf, datasec_id); 14029 for_each_vsi(i, datasec, vsi) { 14030 if (vsi->type == id) { 14031 percpu = true; 14032 break; 14033 } 14034 } 14035 } 14036 14037 insn[0].imm = (u32)addr; 14038 insn[1].imm = addr >> 32; 14039 14040 type = t->type; 14041 t = btf_type_skip_modifiers(btf, type, NULL); 14042 if (percpu) { 14043 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 14044 aux->btf_var.btf = btf; 14045 aux->btf_var.btf_id = type; 14046 } else if (!btf_type_is_struct(t)) { 14047 const struct btf_type *ret; 14048 const char *tname; 14049 u32 tsize; 14050 14051 /* resolve the type size of ksym. */ 14052 ret = btf_resolve_size(btf, t, &tsize); 14053 if (IS_ERR(ret)) { 14054 tname = btf_name_by_offset(btf, t->name_off); 14055 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 14056 tname, PTR_ERR(ret)); 14057 err = -EINVAL; 14058 goto err_put; 14059 } 14060 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 14061 aux->btf_var.mem_size = tsize; 14062 } else { 14063 aux->btf_var.reg_type = PTR_TO_BTF_ID; 14064 aux->btf_var.btf = btf; 14065 aux->btf_var.btf_id = type; 14066 } 14067 14068 /* check whether we recorded this BTF (and maybe module) already */ 14069 for (i = 0; i < env->used_btf_cnt; i++) { 14070 if (env->used_btfs[i].btf == btf) { 14071 btf_put(btf); 14072 return 0; 14073 } 14074 } 14075 14076 if (env->used_btf_cnt >= MAX_USED_BTFS) { 14077 err = -E2BIG; 14078 goto err_put; 14079 } 14080 14081 btf_mod = &env->used_btfs[env->used_btf_cnt]; 14082 btf_mod->btf = btf; 14083 btf_mod->module = NULL; 14084 14085 /* if we reference variables from kernel module, bump its refcount */ 14086 if (btf_is_module(btf)) { 14087 btf_mod->module = btf_try_get_module(btf); 14088 if (!btf_mod->module) { 14089 err = -ENXIO; 14090 goto err_put; 14091 } 14092 } 14093 14094 env->used_btf_cnt++; 14095 14096 return 0; 14097 err_put: 14098 btf_put(btf); 14099 return err; 14100 } 14101 14102 static bool is_tracing_prog_type(enum bpf_prog_type type) 14103 { 14104 switch (type) { 14105 case BPF_PROG_TYPE_KPROBE: 14106 case BPF_PROG_TYPE_TRACEPOINT: 14107 case BPF_PROG_TYPE_PERF_EVENT: 14108 case BPF_PROG_TYPE_RAW_TRACEPOINT: 14109 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 14110 return true; 14111 default: 14112 return false; 14113 } 14114 } 14115 14116 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 14117 struct bpf_map *map, 14118 struct bpf_prog *prog) 14119 14120 { 14121 enum bpf_prog_type prog_type = resolve_prog_type(prog); 14122 14123 if (btf_record_has_field(map->record, BPF_LIST_HEAD)) { 14124 if (is_tracing_prog_type(prog_type)) { 14125 verbose(env, "tracing progs cannot use bpf_list_head yet\n"); 14126 return -EINVAL; 14127 } 14128 } 14129 14130 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 14131 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 14132 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 14133 return -EINVAL; 14134 } 14135 14136 if (is_tracing_prog_type(prog_type)) { 14137 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 14138 return -EINVAL; 14139 } 14140 14141 if (prog->aux->sleepable) { 14142 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 14143 return -EINVAL; 14144 } 14145 } 14146 14147 if (btf_record_has_field(map->record, BPF_TIMER)) { 14148 if (is_tracing_prog_type(prog_type)) { 14149 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 14150 return -EINVAL; 14151 } 14152 } 14153 14154 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 14155 !bpf_offload_prog_map_match(prog, map)) { 14156 verbose(env, "offload device mismatch between prog and map\n"); 14157 return -EINVAL; 14158 } 14159 14160 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 14161 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 14162 return -EINVAL; 14163 } 14164 14165 if (prog->aux->sleepable) 14166 switch (map->map_type) { 14167 case BPF_MAP_TYPE_HASH: 14168 case BPF_MAP_TYPE_LRU_HASH: 14169 case BPF_MAP_TYPE_ARRAY: 14170 case BPF_MAP_TYPE_PERCPU_HASH: 14171 case BPF_MAP_TYPE_PERCPU_ARRAY: 14172 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 14173 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 14174 case BPF_MAP_TYPE_HASH_OF_MAPS: 14175 case BPF_MAP_TYPE_RINGBUF: 14176 case BPF_MAP_TYPE_USER_RINGBUF: 14177 case BPF_MAP_TYPE_INODE_STORAGE: 14178 case BPF_MAP_TYPE_SK_STORAGE: 14179 case BPF_MAP_TYPE_TASK_STORAGE: 14180 case BPF_MAP_TYPE_CGRP_STORAGE: 14181 break; 14182 default: 14183 verbose(env, 14184 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 14185 return -EINVAL; 14186 } 14187 14188 return 0; 14189 } 14190 14191 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 14192 { 14193 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 14194 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 14195 } 14196 14197 /* find and rewrite pseudo imm in ld_imm64 instructions: 14198 * 14199 * 1. if it accesses map FD, replace it with actual map pointer. 14200 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 14201 * 14202 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 14203 */ 14204 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 14205 { 14206 struct bpf_insn *insn = env->prog->insnsi; 14207 int insn_cnt = env->prog->len; 14208 int i, j, err; 14209 14210 err = bpf_prog_calc_tag(env->prog); 14211 if (err) 14212 return err; 14213 14214 for (i = 0; i < insn_cnt; i++, insn++) { 14215 if (BPF_CLASS(insn->code) == BPF_LDX && 14216 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 14217 verbose(env, "BPF_LDX uses reserved fields\n"); 14218 return -EINVAL; 14219 } 14220 14221 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 14222 struct bpf_insn_aux_data *aux; 14223 struct bpf_map *map; 14224 struct fd f; 14225 u64 addr; 14226 u32 fd; 14227 14228 if (i == insn_cnt - 1 || insn[1].code != 0 || 14229 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 14230 insn[1].off != 0) { 14231 verbose(env, "invalid bpf_ld_imm64 insn\n"); 14232 return -EINVAL; 14233 } 14234 14235 if (insn[0].src_reg == 0) 14236 /* valid generic load 64-bit imm */ 14237 goto next_insn; 14238 14239 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 14240 aux = &env->insn_aux_data[i]; 14241 err = check_pseudo_btf_id(env, insn, aux); 14242 if (err) 14243 return err; 14244 goto next_insn; 14245 } 14246 14247 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 14248 aux = &env->insn_aux_data[i]; 14249 aux->ptr_type = PTR_TO_FUNC; 14250 goto next_insn; 14251 } 14252 14253 /* In final convert_pseudo_ld_imm64() step, this is 14254 * converted into regular 64-bit imm load insn. 14255 */ 14256 switch (insn[0].src_reg) { 14257 case BPF_PSEUDO_MAP_VALUE: 14258 case BPF_PSEUDO_MAP_IDX_VALUE: 14259 break; 14260 case BPF_PSEUDO_MAP_FD: 14261 case BPF_PSEUDO_MAP_IDX: 14262 if (insn[1].imm == 0) 14263 break; 14264 fallthrough; 14265 default: 14266 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 14267 return -EINVAL; 14268 } 14269 14270 switch (insn[0].src_reg) { 14271 case BPF_PSEUDO_MAP_IDX_VALUE: 14272 case BPF_PSEUDO_MAP_IDX: 14273 if (bpfptr_is_null(env->fd_array)) { 14274 verbose(env, "fd_idx without fd_array is invalid\n"); 14275 return -EPROTO; 14276 } 14277 if (copy_from_bpfptr_offset(&fd, env->fd_array, 14278 insn[0].imm * sizeof(fd), 14279 sizeof(fd))) 14280 return -EFAULT; 14281 break; 14282 default: 14283 fd = insn[0].imm; 14284 break; 14285 } 14286 14287 f = fdget(fd); 14288 map = __bpf_map_get(f); 14289 if (IS_ERR(map)) { 14290 verbose(env, "fd %d is not pointing to valid bpf_map\n", 14291 insn[0].imm); 14292 return PTR_ERR(map); 14293 } 14294 14295 err = check_map_prog_compatibility(env, map, env->prog); 14296 if (err) { 14297 fdput(f); 14298 return err; 14299 } 14300 14301 aux = &env->insn_aux_data[i]; 14302 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 14303 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 14304 addr = (unsigned long)map; 14305 } else { 14306 u32 off = insn[1].imm; 14307 14308 if (off >= BPF_MAX_VAR_OFF) { 14309 verbose(env, "direct value offset of %u is not allowed\n", off); 14310 fdput(f); 14311 return -EINVAL; 14312 } 14313 14314 if (!map->ops->map_direct_value_addr) { 14315 verbose(env, "no direct value access support for this map type\n"); 14316 fdput(f); 14317 return -EINVAL; 14318 } 14319 14320 err = map->ops->map_direct_value_addr(map, &addr, off); 14321 if (err) { 14322 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 14323 map->value_size, off); 14324 fdput(f); 14325 return err; 14326 } 14327 14328 aux->map_off = off; 14329 addr += off; 14330 } 14331 14332 insn[0].imm = (u32)addr; 14333 insn[1].imm = addr >> 32; 14334 14335 /* check whether we recorded this map already */ 14336 for (j = 0; j < env->used_map_cnt; j++) { 14337 if (env->used_maps[j] == map) { 14338 aux->map_index = j; 14339 fdput(f); 14340 goto next_insn; 14341 } 14342 } 14343 14344 if (env->used_map_cnt >= MAX_USED_MAPS) { 14345 fdput(f); 14346 return -E2BIG; 14347 } 14348 14349 /* hold the map. If the program is rejected by verifier, 14350 * the map will be released by release_maps() or it 14351 * will be used by the valid program until it's unloaded 14352 * and all maps are released in free_used_maps() 14353 */ 14354 bpf_map_inc(map); 14355 14356 aux->map_index = env->used_map_cnt; 14357 env->used_maps[env->used_map_cnt++] = map; 14358 14359 if (bpf_map_is_cgroup_storage(map) && 14360 bpf_cgroup_storage_assign(env->prog->aux, map)) { 14361 verbose(env, "only one cgroup storage of each type is allowed\n"); 14362 fdput(f); 14363 return -EBUSY; 14364 } 14365 14366 fdput(f); 14367 next_insn: 14368 insn++; 14369 i++; 14370 continue; 14371 } 14372 14373 /* Basic sanity check before we invest more work here. */ 14374 if (!bpf_opcode_in_insntable(insn->code)) { 14375 verbose(env, "unknown opcode %02x\n", insn->code); 14376 return -EINVAL; 14377 } 14378 } 14379 14380 /* now all pseudo BPF_LD_IMM64 instructions load valid 14381 * 'struct bpf_map *' into a register instead of user map_fd. 14382 * These pointers will be used later by verifier to validate map access. 14383 */ 14384 return 0; 14385 } 14386 14387 /* drop refcnt of maps used by the rejected program */ 14388 static void release_maps(struct bpf_verifier_env *env) 14389 { 14390 __bpf_free_used_maps(env->prog->aux, env->used_maps, 14391 env->used_map_cnt); 14392 } 14393 14394 /* drop refcnt of maps used by the rejected program */ 14395 static void release_btfs(struct bpf_verifier_env *env) 14396 { 14397 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 14398 env->used_btf_cnt); 14399 } 14400 14401 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 14402 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 14403 { 14404 struct bpf_insn *insn = env->prog->insnsi; 14405 int insn_cnt = env->prog->len; 14406 int i; 14407 14408 for (i = 0; i < insn_cnt; i++, insn++) { 14409 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 14410 continue; 14411 if (insn->src_reg == BPF_PSEUDO_FUNC) 14412 continue; 14413 insn->src_reg = 0; 14414 } 14415 } 14416 14417 /* single env->prog->insni[off] instruction was replaced with the range 14418 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 14419 * [0, off) and [off, end) to new locations, so the patched range stays zero 14420 */ 14421 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 14422 struct bpf_insn_aux_data *new_data, 14423 struct bpf_prog *new_prog, u32 off, u32 cnt) 14424 { 14425 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 14426 struct bpf_insn *insn = new_prog->insnsi; 14427 u32 old_seen = old_data[off].seen; 14428 u32 prog_len; 14429 int i; 14430 14431 /* aux info at OFF always needs adjustment, no matter fast path 14432 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 14433 * original insn at old prog. 14434 */ 14435 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 14436 14437 if (cnt == 1) 14438 return; 14439 prog_len = new_prog->len; 14440 14441 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 14442 memcpy(new_data + off + cnt - 1, old_data + off, 14443 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 14444 for (i = off; i < off + cnt - 1; i++) { 14445 /* Expand insni[off]'s seen count to the patched range. */ 14446 new_data[i].seen = old_seen; 14447 new_data[i].zext_dst = insn_has_def32(env, insn + i); 14448 } 14449 env->insn_aux_data = new_data; 14450 vfree(old_data); 14451 } 14452 14453 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 14454 { 14455 int i; 14456 14457 if (len == 1) 14458 return; 14459 /* NOTE: fake 'exit' subprog should be updated as well. */ 14460 for (i = 0; i <= env->subprog_cnt; i++) { 14461 if (env->subprog_info[i].start <= off) 14462 continue; 14463 env->subprog_info[i].start += len - 1; 14464 } 14465 } 14466 14467 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 14468 { 14469 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 14470 int i, sz = prog->aux->size_poke_tab; 14471 struct bpf_jit_poke_descriptor *desc; 14472 14473 for (i = 0; i < sz; i++) { 14474 desc = &tab[i]; 14475 if (desc->insn_idx <= off) 14476 continue; 14477 desc->insn_idx += len - 1; 14478 } 14479 } 14480 14481 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 14482 const struct bpf_insn *patch, u32 len) 14483 { 14484 struct bpf_prog *new_prog; 14485 struct bpf_insn_aux_data *new_data = NULL; 14486 14487 if (len > 1) { 14488 new_data = vzalloc(array_size(env->prog->len + len - 1, 14489 sizeof(struct bpf_insn_aux_data))); 14490 if (!new_data) 14491 return NULL; 14492 } 14493 14494 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 14495 if (IS_ERR(new_prog)) { 14496 if (PTR_ERR(new_prog) == -ERANGE) 14497 verbose(env, 14498 "insn %d cannot be patched due to 16-bit range\n", 14499 env->insn_aux_data[off].orig_idx); 14500 vfree(new_data); 14501 return NULL; 14502 } 14503 adjust_insn_aux_data(env, new_data, new_prog, off, len); 14504 adjust_subprog_starts(env, off, len); 14505 adjust_poke_descs(new_prog, off, len); 14506 return new_prog; 14507 } 14508 14509 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 14510 u32 off, u32 cnt) 14511 { 14512 int i, j; 14513 14514 /* find first prog starting at or after off (first to remove) */ 14515 for (i = 0; i < env->subprog_cnt; i++) 14516 if (env->subprog_info[i].start >= off) 14517 break; 14518 /* find first prog starting at or after off + cnt (first to stay) */ 14519 for (j = i; j < env->subprog_cnt; j++) 14520 if (env->subprog_info[j].start >= off + cnt) 14521 break; 14522 /* if j doesn't start exactly at off + cnt, we are just removing 14523 * the front of previous prog 14524 */ 14525 if (env->subprog_info[j].start != off + cnt) 14526 j--; 14527 14528 if (j > i) { 14529 struct bpf_prog_aux *aux = env->prog->aux; 14530 int move; 14531 14532 /* move fake 'exit' subprog as well */ 14533 move = env->subprog_cnt + 1 - j; 14534 14535 memmove(env->subprog_info + i, 14536 env->subprog_info + j, 14537 sizeof(*env->subprog_info) * move); 14538 env->subprog_cnt -= j - i; 14539 14540 /* remove func_info */ 14541 if (aux->func_info) { 14542 move = aux->func_info_cnt - j; 14543 14544 memmove(aux->func_info + i, 14545 aux->func_info + j, 14546 sizeof(*aux->func_info) * move); 14547 aux->func_info_cnt -= j - i; 14548 /* func_info->insn_off is set after all code rewrites, 14549 * in adjust_btf_func() - no need to adjust 14550 */ 14551 } 14552 } else { 14553 /* convert i from "first prog to remove" to "first to adjust" */ 14554 if (env->subprog_info[i].start == off) 14555 i++; 14556 } 14557 14558 /* update fake 'exit' subprog as well */ 14559 for (; i <= env->subprog_cnt; i++) 14560 env->subprog_info[i].start -= cnt; 14561 14562 return 0; 14563 } 14564 14565 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 14566 u32 cnt) 14567 { 14568 struct bpf_prog *prog = env->prog; 14569 u32 i, l_off, l_cnt, nr_linfo; 14570 struct bpf_line_info *linfo; 14571 14572 nr_linfo = prog->aux->nr_linfo; 14573 if (!nr_linfo) 14574 return 0; 14575 14576 linfo = prog->aux->linfo; 14577 14578 /* find first line info to remove, count lines to be removed */ 14579 for (i = 0; i < nr_linfo; i++) 14580 if (linfo[i].insn_off >= off) 14581 break; 14582 14583 l_off = i; 14584 l_cnt = 0; 14585 for (; i < nr_linfo; i++) 14586 if (linfo[i].insn_off < off + cnt) 14587 l_cnt++; 14588 else 14589 break; 14590 14591 /* First live insn doesn't match first live linfo, it needs to "inherit" 14592 * last removed linfo. prog is already modified, so prog->len == off 14593 * means no live instructions after (tail of the program was removed). 14594 */ 14595 if (prog->len != off && l_cnt && 14596 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 14597 l_cnt--; 14598 linfo[--i].insn_off = off + cnt; 14599 } 14600 14601 /* remove the line info which refer to the removed instructions */ 14602 if (l_cnt) { 14603 memmove(linfo + l_off, linfo + i, 14604 sizeof(*linfo) * (nr_linfo - i)); 14605 14606 prog->aux->nr_linfo -= l_cnt; 14607 nr_linfo = prog->aux->nr_linfo; 14608 } 14609 14610 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 14611 for (i = l_off; i < nr_linfo; i++) 14612 linfo[i].insn_off -= cnt; 14613 14614 /* fix up all subprogs (incl. 'exit') which start >= off */ 14615 for (i = 0; i <= env->subprog_cnt; i++) 14616 if (env->subprog_info[i].linfo_idx > l_off) { 14617 /* program may have started in the removed region but 14618 * may not be fully removed 14619 */ 14620 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 14621 env->subprog_info[i].linfo_idx -= l_cnt; 14622 else 14623 env->subprog_info[i].linfo_idx = l_off; 14624 } 14625 14626 return 0; 14627 } 14628 14629 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 14630 { 14631 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14632 unsigned int orig_prog_len = env->prog->len; 14633 int err; 14634 14635 if (bpf_prog_is_dev_bound(env->prog->aux)) 14636 bpf_prog_offload_remove_insns(env, off, cnt); 14637 14638 err = bpf_remove_insns(env->prog, off, cnt); 14639 if (err) 14640 return err; 14641 14642 err = adjust_subprog_starts_after_remove(env, off, cnt); 14643 if (err) 14644 return err; 14645 14646 err = bpf_adj_linfo_after_remove(env, off, cnt); 14647 if (err) 14648 return err; 14649 14650 memmove(aux_data + off, aux_data + off + cnt, 14651 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 14652 14653 return 0; 14654 } 14655 14656 /* The verifier does more data flow analysis than llvm and will not 14657 * explore branches that are dead at run time. Malicious programs can 14658 * have dead code too. Therefore replace all dead at-run-time code 14659 * with 'ja -1'. 14660 * 14661 * Just nops are not optimal, e.g. if they would sit at the end of the 14662 * program and through another bug we would manage to jump there, then 14663 * we'd execute beyond program memory otherwise. Returning exception 14664 * code also wouldn't work since we can have subprogs where the dead 14665 * code could be located. 14666 */ 14667 static void sanitize_dead_code(struct bpf_verifier_env *env) 14668 { 14669 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14670 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 14671 struct bpf_insn *insn = env->prog->insnsi; 14672 const int insn_cnt = env->prog->len; 14673 int i; 14674 14675 for (i = 0; i < insn_cnt; i++) { 14676 if (aux_data[i].seen) 14677 continue; 14678 memcpy(insn + i, &trap, sizeof(trap)); 14679 aux_data[i].zext_dst = false; 14680 } 14681 } 14682 14683 static bool insn_is_cond_jump(u8 code) 14684 { 14685 u8 op; 14686 14687 if (BPF_CLASS(code) == BPF_JMP32) 14688 return true; 14689 14690 if (BPF_CLASS(code) != BPF_JMP) 14691 return false; 14692 14693 op = BPF_OP(code); 14694 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 14695 } 14696 14697 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 14698 { 14699 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14700 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 14701 struct bpf_insn *insn = env->prog->insnsi; 14702 const int insn_cnt = env->prog->len; 14703 int i; 14704 14705 for (i = 0; i < insn_cnt; i++, insn++) { 14706 if (!insn_is_cond_jump(insn->code)) 14707 continue; 14708 14709 if (!aux_data[i + 1].seen) 14710 ja.off = insn->off; 14711 else if (!aux_data[i + 1 + insn->off].seen) 14712 ja.off = 0; 14713 else 14714 continue; 14715 14716 if (bpf_prog_is_dev_bound(env->prog->aux)) 14717 bpf_prog_offload_replace_insn(env, i, &ja); 14718 14719 memcpy(insn, &ja, sizeof(ja)); 14720 } 14721 } 14722 14723 static int opt_remove_dead_code(struct bpf_verifier_env *env) 14724 { 14725 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14726 int insn_cnt = env->prog->len; 14727 int i, err; 14728 14729 for (i = 0; i < insn_cnt; i++) { 14730 int j; 14731 14732 j = 0; 14733 while (i + j < insn_cnt && !aux_data[i + j].seen) 14734 j++; 14735 if (!j) 14736 continue; 14737 14738 err = verifier_remove_insns(env, i, j); 14739 if (err) 14740 return err; 14741 insn_cnt = env->prog->len; 14742 } 14743 14744 return 0; 14745 } 14746 14747 static int opt_remove_nops(struct bpf_verifier_env *env) 14748 { 14749 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 14750 struct bpf_insn *insn = env->prog->insnsi; 14751 int insn_cnt = env->prog->len; 14752 int i, err; 14753 14754 for (i = 0; i < insn_cnt; i++) { 14755 if (memcmp(&insn[i], &ja, sizeof(ja))) 14756 continue; 14757 14758 err = verifier_remove_insns(env, i, 1); 14759 if (err) 14760 return err; 14761 insn_cnt--; 14762 i--; 14763 } 14764 14765 return 0; 14766 } 14767 14768 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 14769 const union bpf_attr *attr) 14770 { 14771 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 14772 struct bpf_insn_aux_data *aux = env->insn_aux_data; 14773 int i, patch_len, delta = 0, len = env->prog->len; 14774 struct bpf_insn *insns = env->prog->insnsi; 14775 struct bpf_prog *new_prog; 14776 bool rnd_hi32; 14777 14778 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 14779 zext_patch[1] = BPF_ZEXT_REG(0); 14780 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 14781 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 14782 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 14783 for (i = 0; i < len; i++) { 14784 int adj_idx = i + delta; 14785 struct bpf_insn insn; 14786 int load_reg; 14787 14788 insn = insns[adj_idx]; 14789 load_reg = insn_def_regno(&insn); 14790 if (!aux[adj_idx].zext_dst) { 14791 u8 code, class; 14792 u32 imm_rnd; 14793 14794 if (!rnd_hi32) 14795 continue; 14796 14797 code = insn.code; 14798 class = BPF_CLASS(code); 14799 if (load_reg == -1) 14800 continue; 14801 14802 /* NOTE: arg "reg" (the fourth one) is only used for 14803 * BPF_STX + SRC_OP, so it is safe to pass NULL 14804 * here. 14805 */ 14806 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 14807 if (class == BPF_LD && 14808 BPF_MODE(code) == BPF_IMM) 14809 i++; 14810 continue; 14811 } 14812 14813 /* ctx load could be transformed into wider load. */ 14814 if (class == BPF_LDX && 14815 aux[adj_idx].ptr_type == PTR_TO_CTX) 14816 continue; 14817 14818 imm_rnd = get_random_u32(); 14819 rnd_hi32_patch[0] = insn; 14820 rnd_hi32_patch[1].imm = imm_rnd; 14821 rnd_hi32_patch[3].dst_reg = load_reg; 14822 patch = rnd_hi32_patch; 14823 patch_len = 4; 14824 goto apply_patch_buffer; 14825 } 14826 14827 /* Add in an zero-extend instruction if a) the JIT has requested 14828 * it or b) it's a CMPXCHG. 14829 * 14830 * The latter is because: BPF_CMPXCHG always loads a value into 14831 * R0, therefore always zero-extends. However some archs' 14832 * equivalent instruction only does this load when the 14833 * comparison is successful. This detail of CMPXCHG is 14834 * orthogonal to the general zero-extension behaviour of the 14835 * CPU, so it's treated independently of bpf_jit_needs_zext. 14836 */ 14837 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 14838 continue; 14839 14840 if (WARN_ON(load_reg == -1)) { 14841 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 14842 return -EFAULT; 14843 } 14844 14845 zext_patch[0] = insn; 14846 zext_patch[1].dst_reg = load_reg; 14847 zext_patch[1].src_reg = load_reg; 14848 patch = zext_patch; 14849 patch_len = 2; 14850 apply_patch_buffer: 14851 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 14852 if (!new_prog) 14853 return -ENOMEM; 14854 env->prog = new_prog; 14855 insns = new_prog->insnsi; 14856 aux = env->insn_aux_data; 14857 delta += patch_len - 1; 14858 } 14859 14860 return 0; 14861 } 14862 14863 /* convert load instructions that access fields of a context type into a 14864 * sequence of instructions that access fields of the underlying structure: 14865 * struct __sk_buff -> struct sk_buff 14866 * struct bpf_sock_ops -> struct sock 14867 */ 14868 static int convert_ctx_accesses(struct bpf_verifier_env *env) 14869 { 14870 const struct bpf_verifier_ops *ops = env->ops; 14871 int i, cnt, size, ctx_field_size, delta = 0; 14872 const int insn_cnt = env->prog->len; 14873 struct bpf_insn insn_buf[16], *insn; 14874 u32 target_size, size_default, off; 14875 struct bpf_prog *new_prog; 14876 enum bpf_access_type type; 14877 bool is_narrower_load; 14878 14879 if (ops->gen_prologue || env->seen_direct_write) { 14880 if (!ops->gen_prologue) { 14881 verbose(env, "bpf verifier is misconfigured\n"); 14882 return -EINVAL; 14883 } 14884 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 14885 env->prog); 14886 if (cnt >= ARRAY_SIZE(insn_buf)) { 14887 verbose(env, "bpf verifier is misconfigured\n"); 14888 return -EINVAL; 14889 } else if (cnt) { 14890 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 14891 if (!new_prog) 14892 return -ENOMEM; 14893 14894 env->prog = new_prog; 14895 delta += cnt - 1; 14896 } 14897 } 14898 14899 if (bpf_prog_is_dev_bound(env->prog->aux)) 14900 return 0; 14901 14902 insn = env->prog->insnsi + delta; 14903 14904 for (i = 0; i < insn_cnt; i++, insn++) { 14905 bpf_convert_ctx_access_t convert_ctx_access; 14906 bool ctx_access; 14907 14908 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 14909 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 14910 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 14911 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 14912 type = BPF_READ; 14913 ctx_access = true; 14914 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 14915 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 14916 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 14917 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 14918 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 14919 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 14920 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 14921 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 14922 type = BPF_WRITE; 14923 ctx_access = BPF_CLASS(insn->code) == BPF_STX; 14924 } else { 14925 continue; 14926 } 14927 14928 if (type == BPF_WRITE && 14929 env->insn_aux_data[i + delta].sanitize_stack_spill) { 14930 struct bpf_insn patch[] = { 14931 *insn, 14932 BPF_ST_NOSPEC(), 14933 }; 14934 14935 cnt = ARRAY_SIZE(patch); 14936 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 14937 if (!new_prog) 14938 return -ENOMEM; 14939 14940 delta += cnt - 1; 14941 env->prog = new_prog; 14942 insn = new_prog->insnsi + i + delta; 14943 continue; 14944 } 14945 14946 if (!ctx_access) 14947 continue; 14948 14949 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 14950 case PTR_TO_CTX: 14951 if (!ops->convert_ctx_access) 14952 continue; 14953 convert_ctx_access = ops->convert_ctx_access; 14954 break; 14955 case PTR_TO_SOCKET: 14956 case PTR_TO_SOCK_COMMON: 14957 convert_ctx_access = bpf_sock_convert_ctx_access; 14958 break; 14959 case PTR_TO_TCP_SOCK: 14960 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 14961 break; 14962 case PTR_TO_XDP_SOCK: 14963 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 14964 break; 14965 case PTR_TO_BTF_ID: 14966 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 14967 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 14968 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot 14969 * be said once it is marked PTR_UNTRUSTED, hence we must handle 14970 * any faults for loads into such types. BPF_WRITE is disallowed 14971 * for this case. 14972 */ 14973 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 14974 if (type == BPF_READ) { 14975 insn->code = BPF_LDX | BPF_PROBE_MEM | 14976 BPF_SIZE((insn)->code); 14977 env->prog->aux->num_exentries++; 14978 } 14979 continue; 14980 default: 14981 continue; 14982 } 14983 14984 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 14985 size = BPF_LDST_BYTES(insn); 14986 14987 /* If the read access is a narrower load of the field, 14988 * convert to a 4/8-byte load, to minimum program type specific 14989 * convert_ctx_access changes. If conversion is successful, 14990 * we will apply proper mask to the result. 14991 */ 14992 is_narrower_load = size < ctx_field_size; 14993 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 14994 off = insn->off; 14995 if (is_narrower_load) { 14996 u8 size_code; 14997 14998 if (type == BPF_WRITE) { 14999 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 15000 return -EINVAL; 15001 } 15002 15003 size_code = BPF_H; 15004 if (ctx_field_size == 4) 15005 size_code = BPF_W; 15006 else if (ctx_field_size == 8) 15007 size_code = BPF_DW; 15008 15009 insn->off = off & ~(size_default - 1); 15010 insn->code = BPF_LDX | BPF_MEM | size_code; 15011 } 15012 15013 target_size = 0; 15014 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 15015 &target_size); 15016 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 15017 (ctx_field_size && !target_size)) { 15018 verbose(env, "bpf verifier is misconfigured\n"); 15019 return -EINVAL; 15020 } 15021 15022 if (is_narrower_load && size < target_size) { 15023 u8 shift = bpf_ctx_narrow_access_offset( 15024 off, size, size_default) * 8; 15025 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 15026 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 15027 return -EINVAL; 15028 } 15029 if (ctx_field_size <= 4) { 15030 if (shift) 15031 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 15032 insn->dst_reg, 15033 shift); 15034 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 15035 (1 << size * 8) - 1); 15036 } else { 15037 if (shift) 15038 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 15039 insn->dst_reg, 15040 shift); 15041 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 15042 (1ULL << size * 8) - 1); 15043 } 15044 } 15045 15046 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15047 if (!new_prog) 15048 return -ENOMEM; 15049 15050 delta += cnt - 1; 15051 15052 /* keep walking new program and skip insns we just inserted */ 15053 env->prog = new_prog; 15054 insn = new_prog->insnsi + i + delta; 15055 } 15056 15057 return 0; 15058 } 15059 15060 static int jit_subprogs(struct bpf_verifier_env *env) 15061 { 15062 struct bpf_prog *prog = env->prog, **func, *tmp; 15063 int i, j, subprog_start, subprog_end = 0, len, subprog; 15064 struct bpf_map *map_ptr; 15065 struct bpf_insn *insn; 15066 void *old_bpf_func; 15067 int err, num_exentries; 15068 15069 if (env->subprog_cnt <= 1) 15070 return 0; 15071 15072 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15073 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 15074 continue; 15075 15076 /* Upon error here we cannot fall back to interpreter but 15077 * need a hard reject of the program. Thus -EFAULT is 15078 * propagated in any case. 15079 */ 15080 subprog = find_subprog(env, i + insn->imm + 1); 15081 if (subprog < 0) { 15082 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 15083 i + insn->imm + 1); 15084 return -EFAULT; 15085 } 15086 /* temporarily remember subprog id inside insn instead of 15087 * aux_data, since next loop will split up all insns into funcs 15088 */ 15089 insn->off = subprog; 15090 /* remember original imm in case JIT fails and fallback 15091 * to interpreter will be needed 15092 */ 15093 env->insn_aux_data[i].call_imm = insn->imm; 15094 /* point imm to __bpf_call_base+1 from JITs point of view */ 15095 insn->imm = 1; 15096 if (bpf_pseudo_func(insn)) 15097 /* jit (e.g. x86_64) may emit fewer instructions 15098 * if it learns a u32 imm is the same as a u64 imm. 15099 * Force a non zero here. 15100 */ 15101 insn[1].imm = 1; 15102 } 15103 15104 err = bpf_prog_alloc_jited_linfo(prog); 15105 if (err) 15106 goto out_undo_insn; 15107 15108 err = -ENOMEM; 15109 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 15110 if (!func) 15111 goto out_undo_insn; 15112 15113 for (i = 0; i < env->subprog_cnt; i++) { 15114 subprog_start = subprog_end; 15115 subprog_end = env->subprog_info[i + 1].start; 15116 15117 len = subprog_end - subprog_start; 15118 /* bpf_prog_run() doesn't call subprogs directly, 15119 * hence main prog stats include the runtime of subprogs. 15120 * subprogs don't have IDs and not reachable via prog_get_next_id 15121 * func[i]->stats will never be accessed and stays NULL 15122 */ 15123 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 15124 if (!func[i]) 15125 goto out_free; 15126 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 15127 len * sizeof(struct bpf_insn)); 15128 func[i]->type = prog->type; 15129 func[i]->len = len; 15130 if (bpf_prog_calc_tag(func[i])) 15131 goto out_free; 15132 func[i]->is_func = 1; 15133 func[i]->aux->func_idx = i; 15134 /* Below members will be freed only at prog->aux */ 15135 func[i]->aux->btf = prog->aux->btf; 15136 func[i]->aux->func_info = prog->aux->func_info; 15137 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 15138 func[i]->aux->poke_tab = prog->aux->poke_tab; 15139 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 15140 15141 for (j = 0; j < prog->aux->size_poke_tab; j++) { 15142 struct bpf_jit_poke_descriptor *poke; 15143 15144 poke = &prog->aux->poke_tab[j]; 15145 if (poke->insn_idx < subprog_end && 15146 poke->insn_idx >= subprog_start) 15147 poke->aux = func[i]->aux; 15148 } 15149 15150 func[i]->aux->name[0] = 'F'; 15151 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 15152 func[i]->jit_requested = 1; 15153 func[i]->blinding_requested = prog->blinding_requested; 15154 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 15155 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 15156 func[i]->aux->linfo = prog->aux->linfo; 15157 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 15158 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 15159 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 15160 num_exentries = 0; 15161 insn = func[i]->insnsi; 15162 for (j = 0; j < func[i]->len; j++, insn++) { 15163 if (BPF_CLASS(insn->code) == BPF_LDX && 15164 BPF_MODE(insn->code) == BPF_PROBE_MEM) 15165 num_exentries++; 15166 } 15167 func[i]->aux->num_exentries = num_exentries; 15168 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 15169 func[i] = bpf_int_jit_compile(func[i]); 15170 if (!func[i]->jited) { 15171 err = -ENOTSUPP; 15172 goto out_free; 15173 } 15174 cond_resched(); 15175 } 15176 15177 /* at this point all bpf functions were successfully JITed 15178 * now populate all bpf_calls with correct addresses and 15179 * run last pass of JIT 15180 */ 15181 for (i = 0; i < env->subprog_cnt; i++) { 15182 insn = func[i]->insnsi; 15183 for (j = 0; j < func[i]->len; j++, insn++) { 15184 if (bpf_pseudo_func(insn)) { 15185 subprog = insn->off; 15186 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 15187 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 15188 continue; 15189 } 15190 if (!bpf_pseudo_call(insn)) 15191 continue; 15192 subprog = insn->off; 15193 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 15194 } 15195 15196 /* we use the aux data to keep a list of the start addresses 15197 * of the JITed images for each function in the program 15198 * 15199 * for some architectures, such as powerpc64, the imm field 15200 * might not be large enough to hold the offset of the start 15201 * address of the callee's JITed image from __bpf_call_base 15202 * 15203 * in such cases, we can lookup the start address of a callee 15204 * by using its subprog id, available from the off field of 15205 * the call instruction, as an index for this list 15206 */ 15207 func[i]->aux->func = func; 15208 func[i]->aux->func_cnt = env->subprog_cnt; 15209 } 15210 for (i = 0; i < env->subprog_cnt; i++) { 15211 old_bpf_func = func[i]->bpf_func; 15212 tmp = bpf_int_jit_compile(func[i]); 15213 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 15214 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 15215 err = -ENOTSUPP; 15216 goto out_free; 15217 } 15218 cond_resched(); 15219 } 15220 15221 /* finally lock prog and jit images for all functions and 15222 * populate kallsysm 15223 */ 15224 for (i = 0; i < env->subprog_cnt; i++) { 15225 bpf_prog_lock_ro(func[i]); 15226 bpf_prog_kallsyms_add(func[i]); 15227 } 15228 15229 /* Last step: make now unused interpreter insns from main 15230 * prog consistent for later dump requests, so they can 15231 * later look the same as if they were interpreted only. 15232 */ 15233 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15234 if (bpf_pseudo_func(insn)) { 15235 insn[0].imm = env->insn_aux_data[i].call_imm; 15236 insn[1].imm = insn->off; 15237 insn->off = 0; 15238 continue; 15239 } 15240 if (!bpf_pseudo_call(insn)) 15241 continue; 15242 insn->off = env->insn_aux_data[i].call_imm; 15243 subprog = find_subprog(env, i + insn->off + 1); 15244 insn->imm = subprog; 15245 } 15246 15247 prog->jited = 1; 15248 prog->bpf_func = func[0]->bpf_func; 15249 prog->jited_len = func[0]->jited_len; 15250 prog->aux->func = func; 15251 prog->aux->func_cnt = env->subprog_cnt; 15252 bpf_prog_jit_attempt_done(prog); 15253 return 0; 15254 out_free: 15255 /* We failed JIT'ing, so at this point we need to unregister poke 15256 * descriptors from subprogs, so that kernel is not attempting to 15257 * patch it anymore as we're freeing the subprog JIT memory. 15258 */ 15259 for (i = 0; i < prog->aux->size_poke_tab; i++) { 15260 map_ptr = prog->aux->poke_tab[i].tail_call.map; 15261 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 15262 } 15263 /* At this point we're guaranteed that poke descriptors are not 15264 * live anymore. We can just unlink its descriptor table as it's 15265 * released with the main prog. 15266 */ 15267 for (i = 0; i < env->subprog_cnt; i++) { 15268 if (!func[i]) 15269 continue; 15270 func[i]->aux->poke_tab = NULL; 15271 bpf_jit_free(func[i]); 15272 } 15273 kfree(func); 15274 out_undo_insn: 15275 /* cleanup main prog to be interpreted */ 15276 prog->jit_requested = 0; 15277 prog->blinding_requested = 0; 15278 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15279 if (!bpf_pseudo_call(insn)) 15280 continue; 15281 insn->off = 0; 15282 insn->imm = env->insn_aux_data[i].call_imm; 15283 } 15284 bpf_prog_jit_attempt_done(prog); 15285 return err; 15286 } 15287 15288 static int fixup_call_args(struct bpf_verifier_env *env) 15289 { 15290 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 15291 struct bpf_prog *prog = env->prog; 15292 struct bpf_insn *insn = prog->insnsi; 15293 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 15294 int i, depth; 15295 #endif 15296 int err = 0; 15297 15298 if (env->prog->jit_requested && 15299 !bpf_prog_is_dev_bound(env->prog->aux)) { 15300 err = jit_subprogs(env); 15301 if (err == 0) 15302 return 0; 15303 if (err == -EFAULT) 15304 return err; 15305 } 15306 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 15307 if (has_kfunc_call) { 15308 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 15309 return -EINVAL; 15310 } 15311 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 15312 /* When JIT fails the progs with bpf2bpf calls and tail_calls 15313 * have to be rejected, since interpreter doesn't support them yet. 15314 */ 15315 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 15316 return -EINVAL; 15317 } 15318 for (i = 0; i < prog->len; i++, insn++) { 15319 if (bpf_pseudo_func(insn)) { 15320 /* When JIT fails the progs with callback calls 15321 * have to be rejected, since interpreter doesn't support them yet. 15322 */ 15323 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 15324 return -EINVAL; 15325 } 15326 15327 if (!bpf_pseudo_call(insn)) 15328 continue; 15329 depth = get_callee_stack_depth(env, insn, i); 15330 if (depth < 0) 15331 return depth; 15332 bpf_patch_call_args(insn, depth); 15333 } 15334 err = 0; 15335 #endif 15336 return err; 15337 } 15338 15339 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 15340 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 15341 { 15342 const struct bpf_kfunc_desc *desc; 15343 15344 if (!insn->imm) { 15345 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 15346 return -EINVAL; 15347 } 15348 15349 /* insn->imm has the btf func_id. Replace it with 15350 * an address (relative to __bpf_base_call). 15351 */ 15352 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 15353 if (!desc) { 15354 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 15355 insn->imm); 15356 return -EFAULT; 15357 } 15358 15359 *cnt = 0; 15360 insn->imm = desc->imm; 15361 if (insn->off) 15362 return 0; 15363 if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 15364 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 15365 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 15366 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 15367 15368 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 15369 insn_buf[1] = addr[0]; 15370 insn_buf[2] = addr[1]; 15371 insn_buf[3] = *insn; 15372 *cnt = 4; 15373 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 15374 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 15375 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 15376 15377 insn_buf[0] = addr[0]; 15378 insn_buf[1] = addr[1]; 15379 insn_buf[2] = *insn; 15380 *cnt = 3; 15381 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 15382 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 15383 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 15384 *cnt = 1; 15385 } 15386 return 0; 15387 } 15388 15389 /* Do various post-verification rewrites in a single program pass. 15390 * These rewrites simplify JIT and interpreter implementations. 15391 */ 15392 static int do_misc_fixups(struct bpf_verifier_env *env) 15393 { 15394 struct bpf_prog *prog = env->prog; 15395 enum bpf_attach_type eatype = prog->expected_attach_type; 15396 enum bpf_prog_type prog_type = resolve_prog_type(prog); 15397 struct bpf_insn *insn = prog->insnsi; 15398 const struct bpf_func_proto *fn; 15399 const int insn_cnt = prog->len; 15400 const struct bpf_map_ops *ops; 15401 struct bpf_insn_aux_data *aux; 15402 struct bpf_insn insn_buf[16]; 15403 struct bpf_prog *new_prog; 15404 struct bpf_map *map_ptr; 15405 int i, ret, cnt, delta = 0; 15406 15407 for (i = 0; i < insn_cnt; i++, insn++) { 15408 /* Make divide-by-zero exceptions impossible. */ 15409 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 15410 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 15411 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 15412 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 15413 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 15414 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 15415 struct bpf_insn *patchlet; 15416 struct bpf_insn chk_and_div[] = { 15417 /* [R,W]x div 0 -> 0 */ 15418 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 15419 BPF_JNE | BPF_K, insn->src_reg, 15420 0, 2, 0), 15421 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 15422 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 15423 *insn, 15424 }; 15425 struct bpf_insn chk_and_mod[] = { 15426 /* [R,W]x mod 0 -> [R,W]x */ 15427 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 15428 BPF_JEQ | BPF_K, insn->src_reg, 15429 0, 1 + (is64 ? 0 : 1), 0), 15430 *insn, 15431 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 15432 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 15433 }; 15434 15435 patchlet = isdiv ? chk_and_div : chk_and_mod; 15436 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 15437 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 15438 15439 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 15440 if (!new_prog) 15441 return -ENOMEM; 15442 15443 delta += cnt - 1; 15444 env->prog = prog = new_prog; 15445 insn = new_prog->insnsi + i + delta; 15446 continue; 15447 } 15448 15449 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 15450 if (BPF_CLASS(insn->code) == BPF_LD && 15451 (BPF_MODE(insn->code) == BPF_ABS || 15452 BPF_MODE(insn->code) == BPF_IND)) { 15453 cnt = env->ops->gen_ld_abs(insn, insn_buf); 15454 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 15455 verbose(env, "bpf verifier is misconfigured\n"); 15456 return -EINVAL; 15457 } 15458 15459 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15460 if (!new_prog) 15461 return -ENOMEM; 15462 15463 delta += cnt - 1; 15464 env->prog = prog = new_prog; 15465 insn = new_prog->insnsi + i + delta; 15466 continue; 15467 } 15468 15469 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 15470 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 15471 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 15472 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 15473 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 15474 struct bpf_insn *patch = &insn_buf[0]; 15475 bool issrc, isneg, isimm; 15476 u32 off_reg; 15477 15478 aux = &env->insn_aux_data[i + delta]; 15479 if (!aux->alu_state || 15480 aux->alu_state == BPF_ALU_NON_POINTER) 15481 continue; 15482 15483 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 15484 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 15485 BPF_ALU_SANITIZE_SRC; 15486 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 15487 15488 off_reg = issrc ? insn->src_reg : insn->dst_reg; 15489 if (isimm) { 15490 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 15491 } else { 15492 if (isneg) 15493 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 15494 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 15495 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 15496 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 15497 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 15498 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 15499 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 15500 } 15501 if (!issrc) 15502 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 15503 insn->src_reg = BPF_REG_AX; 15504 if (isneg) 15505 insn->code = insn->code == code_add ? 15506 code_sub : code_add; 15507 *patch++ = *insn; 15508 if (issrc && isneg && !isimm) 15509 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 15510 cnt = patch - insn_buf; 15511 15512 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15513 if (!new_prog) 15514 return -ENOMEM; 15515 15516 delta += cnt - 1; 15517 env->prog = prog = new_prog; 15518 insn = new_prog->insnsi + i + delta; 15519 continue; 15520 } 15521 15522 if (insn->code != (BPF_JMP | BPF_CALL)) 15523 continue; 15524 if (insn->src_reg == BPF_PSEUDO_CALL) 15525 continue; 15526 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 15527 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 15528 if (ret) 15529 return ret; 15530 if (cnt == 0) 15531 continue; 15532 15533 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15534 if (!new_prog) 15535 return -ENOMEM; 15536 15537 delta += cnt - 1; 15538 env->prog = prog = new_prog; 15539 insn = new_prog->insnsi + i + delta; 15540 continue; 15541 } 15542 15543 if (insn->imm == BPF_FUNC_get_route_realm) 15544 prog->dst_needed = 1; 15545 if (insn->imm == BPF_FUNC_get_prandom_u32) 15546 bpf_user_rnd_init_once(); 15547 if (insn->imm == BPF_FUNC_override_return) 15548 prog->kprobe_override = 1; 15549 if (insn->imm == BPF_FUNC_tail_call) { 15550 /* If we tail call into other programs, we 15551 * cannot make any assumptions since they can 15552 * be replaced dynamically during runtime in 15553 * the program array. 15554 */ 15555 prog->cb_access = 1; 15556 if (!allow_tail_call_in_subprogs(env)) 15557 prog->aux->stack_depth = MAX_BPF_STACK; 15558 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 15559 15560 /* mark bpf_tail_call as different opcode to avoid 15561 * conditional branch in the interpreter for every normal 15562 * call and to prevent accidental JITing by JIT compiler 15563 * that doesn't support bpf_tail_call yet 15564 */ 15565 insn->imm = 0; 15566 insn->code = BPF_JMP | BPF_TAIL_CALL; 15567 15568 aux = &env->insn_aux_data[i + delta]; 15569 if (env->bpf_capable && !prog->blinding_requested && 15570 prog->jit_requested && 15571 !bpf_map_key_poisoned(aux) && 15572 !bpf_map_ptr_poisoned(aux) && 15573 !bpf_map_ptr_unpriv(aux)) { 15574 struct bpf_jit_poke_descriptor desc = { 15575 .reason = BPF_POKE_REASON_TAIL_CALL, 15576 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 15577 .tail_call.key = bpf_map_key_immediate(aux), 15578 .insn_idx = i + delta, 15579 }; 15580 15581 ret = bpf_jit_add_poke_descriptor(prog, &desc); 15582 if (ret < 0) { 15583 verbose(env, "adding tail call poke descriptor failed\n"); 15584 return ret; 15585 } 15586 15587 insn->imm = ret + 1; 15588 continue; 15589 } 15590 15591 if (!bpf_map_ptr_unpriv(aux)) 15592 continue; 15593 15594 /* instead of changing every JIT dealing with tail_call 15595 * emit two extra insns: 15596 * if (index >= max_entries) goto out; 15597 * index &= array->index_mask; 15598 * to avoid out-of-bounds cpu speculation 15599 */ 15600 if (bpf_map_ptr_poisoned(aux)) { 15601 verbose(env, "tail_call abusing map_ptr\n"); 15602 return -EINVAL; 15603 } 15604 15605 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 15606 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 15607 map_ptr->max_entries, 2); 15608 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 15609 container_of(map_ptr, 15610 struct bpf_array, 15611 map)->index_mask); 15612 insn_buf[2] = *insn; 15613 cnt = 3; 15614 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15615 if (!new_prog) 15616 return -ENOMEM; 15617 15618 delta += cnt - 1; 15619 env->prog = prog = new_prog; 15620 insn = new_prog->insnsi + i + delta; 15621 continue; 15622 } 15623 15624 if (insn->imm == BPF_FUNC_timer_set_callback) { 15625 /* The verifier will process callback_fn as many times as necessary 15626 * with different maps and the register states prepared by 15627 * set_timer_callback_state will be accurate. 15628 * 15629 * The following use case is valid: 15630 * map1 is shared by prog1, prog2, prog3. 15631 * prog1 calls bpf_timer_init for some map1 elements 15632 * prog2 calls bpf_timer_set_callback for some map1 elements. 15633 * Those that were not bpf_timer_init-ed will return -EINVAL. 15634 * prog3 calls bpf_timer_start for some map1 elements. 15635 * Those that were not both bpf_timer_init-ed and 15636 * bpf_timer_set_callback-ed will return -EINVAL. 15637 */ 15638 struct bpf_insn ld_addrs[2] = { 15639 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 15640 }; 15641 15642 insn_buf[0] = ld_addrs[0]; 15643 insn_buf[1] = ld_addrs[1]; 15644 insn_buf[2] = *insn; 15645 cnt = 3; 15646 15647 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15648 if (!new_prog) 15649 return -ENOMEM; 15650 15651 delta += cnt - 1; 15652 env->prog = prog = new_prog; 15653 insn = new_prog->insnsi + i + delta; 15654 goto patch_call_imm; 15655 } 15656 15657 if (is_storage_get_function(insn->imm)) { 15658 if (!env->prog->aux->sleepable || 15659 env->insn_aux_data[i + delta].storage_get_func_atomic) 15660 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 15661 else 15662 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 15663 insn_buf[1] = *insn; 15664 cnt = 2; 15665 15666 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15667 if (!new_prog) 15668 return -ENOMEM; 15669 15670 delta += cnt - 1; 15671 env->prog = prog = new_prog; 15672 insn = new_prog->insnsi + i + delta; 15673 goto patch_call_imm; 15674 } 15675 15676 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 15677 * and other inlining handlers are currently limited to 64 bit 15678 * only. 15679 */ 15680 if (prog->jit_requested && BITS_PER_LONG == 64 && 15681 (insn->imm == BPF_FUNC_map_lookup_elem || 15682 insn->imm == BPF_FUNC_map_update_elem || 15683 insn->imm == BPF_FUNC_map_delete_elem || 15684 insn->imm == BPF_FUNC_map_push_elem || 15685 insn->imm == BPF_FUNC_map_pop_elem || 15686 insn->imm == BPF_FUNC_map_peek_elem || 15687 insn->imm == BPF_FUNC_redirect_map || 15688 insn->imm == BPF_FUNC_for_each_map_elem || 15689 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 15690 aux = &env->insn_aux_data[i + delta]; 15691 if (bpf_map_ptr_poisoned(aux)) 15692 goto patch_call_imm; 15693 15694 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 15695 ops = map_ptr->ops; 15696 if (insn->imm == BPF_FUNC_map_lookup_elem && 15697 ops->map_gen_lookup) { 15698 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 15699 if (cnt == -EOPNOTSUPP) 15700 goto patch_map_ops_generic; 15701 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 15702 verbose(env, "bpf verifier is misconfigured\n"); 15703 return -EINVAL; 15704 } 15705 15706 new_prog = bpf_patch_insn_data(env, i + delta, 15707 insn_buf, cnt); 15708 if (!new_prog) 15709 return -ENOMEM; 15710 15711 delta += cnt - 1; 15712 env->prog = prog = new_prog; 15713 insn = new_prog->insnsi + i + delta; 15714 continue; 15715 } 15716 15717 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 15718 (void *(*)(struct bpf_map *map, void *key))NULL)); 15719 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 15720 (int (*)(struct bpf_map *map, void *key))NULL)); 15721 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 15722 (int (*)(struct bpf_map *map, void *key, void *value, 15723 u64 flags))NULL)); 15724 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 15725 (int (*)(struct bpf_map *map, void *value, 15726 u64 flags))NULL)); 15727 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 15728 (int (*)(struct bpf_map *map, void *value))NULL)); 15729 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 15730 (int (*)(struct bpf_map *map, void *value))NULL)); 15731 BUILD_BUG_ON(!__same_type(ops->map_redirect, 15732 (int (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 15733 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 15734 (int (*)(struct bpf_map *map, 15735 bpf_callback_t callback_fn, 15736 void *callback_ctx, 15737 u64 flags))NULL)); 15738 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 15739 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 15740 15741 patch_map_ops_generic: 15742 switch (insn->imm) { 15743 case BPF_FUNC_map_lookup_elem: 15744 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 15745 continue; 15746 case BPF_FUNC_map_update_elem: 15747 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 15748 continue; 15749 case BPF_FUNC_map_delete_elem: 15750 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 15751 continue; 15752 case BPF_FUNC_map_push_elem: 15753 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 15754 continue; 15755 case BPF_FUNC_map_pop_elem: 15756 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 15757 continue; 15758 case BPF_FUNC_map_peek_elem: 15759 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 15760 continue; 15761 case BPF_FUNC_redirect_map: 15762 insn->imm = BPF_CALL_IMM(ops->map_redirect); 15763 continue; 15764 case BPF_FUNC_for_each_map_elem: 15765 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 15766 continue; 15767 case BPF_FUNC_map_lookup_percpu_elem: 15768 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 15769 continue; 15770 } 15771 15772 goto patch_call_imm; 15773 } 15774 15775 /* Implement bpf_jiffies64 inline. */ 15776 if (prog->jit_requested && BITS_PER_LONG == 64 && 15777 insn->imm == BPF_FUNC_jiffies64) { 15778 struct bpf_insn ld_jiffies_addr[2] = { 15779 BPF_LD_IMM64(BPF_REG_0, 15780 (unsigned long)&jiffies), 15781 }; 15782 15783 insn_buf[0] = ld_jiffies_addr[0]; 15784 insn_buf[1] = ld_jiffies_addr[1]; 15785 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 15786 BPF_REG_0, 0); 15787 cnt = 3; 15788 15789 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 15790 cnt); 15791 if (!new_prog) 15792 return -ENOMEM; 15793 15794 delta += cnt - 1; 15795 env->prog = prog = new_prog; 15796 insn = new_prog->insnsi + i + delta; 15797 continue; 15798 } 15799 15800 /* Implement bpf_get_func_arg inline. */ 15801 if (prog_type == BPF_PROG_TYPE_TRACING && 15802 insn->imm == BPF_FUNC_get_func_arg) { 15803 /* Load nr_args from ctx - 8 */ 15804 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15805 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 15806 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 15807 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 15808 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 15809 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 15810 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 15811 insn_buf[7] = BPF_JMP_A(1); 15812 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 15813 cnt = 9; 15814 15815 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15816 if (!new_prog) 15817 return -ENOMEM; 15818 15819 delta += cnt - 1; 15820 env->prog = prog = new_prog; 15821 insn = new_prog->insnsi + i + delta; 15822 continue; 15823 } 15824 15825 /* Implement bpf_get_func_ret inline. */ 15826 if (prog_type == BPF_PROG_TYPE_TRACING && 15827 insn->imm == BPF_FUNC_get_func_ret) { 15828 if (eatype == BPF_TRACE_FEXIT || 15829 eatype == BPF_MODIFY_RETURN) { 15830 /* Load nr_args from ctx - 8 */ 15831 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15832 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 15833 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 15834 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 15835 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 15836 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 15837 cnt = 6; 15838 } else { 15839 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 15840 cnt = 1; 15841 } 15842 15843 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15844 if (!new_prog) 15845 return -ENOMEM; 15846 15847 delta += cnt - 1; 15848 env->prog = prog = new_prog; 15849 insn = new_prog->insnsi + i + delta; 15850 continue; 15851 } 15852 15853 /* Implement get_func_arg_cnt inline. */ 15854 if (prog_type == BPF_PROG_TYPE_TRACING && 15855 insn->imm == BPF_FUNC_get_func_arg_cnt) { 15856 /* Load nr_args from ctx - 8 */ 15857 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15858 15859 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 15860 if (!new_prog) 15861 return -ENOMEM; 15862 15863 env->prog = prog = new_prog; 15864 insn = new_prog->insnsi + i + delta; 15865 continue; 15866 } 15867 15868 /* Implement bpf_get_func_ip inline. */ 15869 if (prog_type == BPF_PROG_TYPE_TRACING && 15870 insn->imm == BPF_FUNC_get_func_ip) { 15871 /* Load IP address from ctx - 16 */ 15872 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 15873 15874 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 15875 if (!new_prog) 15876 return -ENOMEM; 15877 15878 env->prog = prog = new_prog; 15879 insn = new_prog->insnsi + i + delta; 15880 continue; 15881 } 15882 15883 patch_call_imm: 15884 fn = env->ops->get_func_proto(insn->imm, env->prog); 15885 /* all functions that have prototype and verifier allowed 15886 * programs to call them, must be real in-kernel functions 15887 */ 15888 if (!fn->func) { 15889 verbose(env, 15890 "kernel subsystem misconfigured func %s#%d\n", 15891 func_id_name(insn->imm), insn->imm); 15892 return -EFAULT; 15893 } 15894 insn->imm = fn->func - __bpf_call_base; 15895 } 15896 15897 /* Since poke tab is now finalized, publish aux to tracker. */ 15898 for (i = 0; i < prog->aux->size_poke_tab; i++) { 15899 map_ptr = prog->aux->poke_tab[i].tail_call.map; 15900 if (!map_ptr->ops->map_poke_track || 15901 !map_ptr->ops->map_poke_untrack || 15902 !map_ptr->ops->map_poke_run) { 15903 verbose(env, "bpf verifier is misconfigured\n"); 15904 return -EINVAL; 15905 } 15906 15907 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 15908 if (ret < 0) { 15909 verbose(env, "tracking tail call prog failed\n"); 15910 return ret; 15911 } 15912 } 15913 15914 sort_kfunc_descs_by_imm(env->prog); 15915 15916 return 0; 15917 } 15918 15919 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 15920 int position, 15921 s32 stack_base, 15922 u32 callback_subprogno, 15923 u32 *cnt) 15924 { 15925 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 15926 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 15927 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 15928 int reg_loop_max = BPF_REG_6; 15929 int reg_loop_cnt = BPF_REG_7; 15930 int reg_loop_ctx = BPF_REG_8; 15931 15932 struct bpf_prog *new_prog; 15933 u32 callback_start; 15934 u32 call_insn_offset; 15935 s32 callback_offset; 15936 15937 /* This represents an inlined version of bpf_iter.c:bpf_loop, 15938 * be careful to modify this code in sync. 15939 */ 15940 struct bpf_insn insn_buf[] = { 15941 /* Return error and jump to the end of the patch if 15942 * expected number of iterations is too big. 15943 */ 15944 BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), 15945 BPF_MOV32_IMM(BPF_REG_0, -E2BIG), 15946 BPF_JMP_IMM(BPF_JA, 0, 0, 16), 15947 /* spill R6, R7, R8 to use these as loop vars */ 15948 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), 15949 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), 15950 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), 15951 /* initialize loop vars */ 15952 BPF_MOV64_REG(reg_loop_max, BPF_REG_1), 15953 BPF_MOV32_IMM(reg_loop_cnt, 0), 15954 BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), 15955 /* loop header, 15956 * if reg_loop_cnt >= reg_loop_max skip the loop body 15957 */ 15958 BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), 15959 /* callback call, 15960 * correct callback offset would be set after patching 15961 */ 15962 BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), 15963 BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), 15964 BPF_CALL_REL(0), 15965 /* increment loop counter */ 15966 BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), 15967 /* jump to loop header if callback returned 0 */ 15968 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), 15969 /* return value of bpf_loop, 15970 * set R0 to the number of iterations 15971 */ 15972 BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), 15973 /* restore original values of R6, R7, R8 */ 15974 BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), 15975 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), 15976 BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), 15977 }; 15978 15979 *cnt = ARRAY_SIZE(insn_buf); 15980 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); 15981 if (!new_prog) 15982 return new_prog; 15983 15984 /* callback start is known only after patching */ 15985 callback_start = env->subprog_info[callback_subprogno].start; 15986 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 15987 call_insn_offset = position + 12; 15988 callback_offset = callback_start - call_insn_offset - 1; 15989 new_prog->insnsi[call_insn_offset].imm = callback_offset; 15990 15991 return new_prog; 15992 } 15993 15994 static bool is_bpf_loop_call(struct bpf_insn *insn) 15995 { 15996 return insn->code == (BPF_JMP | BPF_CALL) && 15997 insn->src_reg == 0 && 15998 insn->imm == BPF_FUNC_loop; 15999 } 16000 16001 /* For all sub-programs in the program (including main) check 16002 * insn_aux_data to see if there are bpf_loop calls that require 16003 * inlining. If such calls are found the calls are replaced with a 16004 * sequence of instructions produced by `inline_bpf_loop` function and 16005 * subprog stack_depth is increased by the size of 3 registers. 16006 * This stack space is used to spill values of the R6, R7, R8. These 16007 * registers are used to store the loop bound, counter and context 16008 * variables. 16009 */ 16010 static int optimize_bpf_loop(struct bpf_verifier_env *env) 16011 { 16012 struct bpf_subprog_info *subprogs = env->subprog_info; 16013 int i, cur_subprog = 0, cnt, delta = 0; 16014 struct bpf_insn *insn = env->prog->insnsi; 16015 int insn_cnt = env->prog->len; 16016 u16 stack_depth = subprogs[cur_subprog].stack_depth; 16017 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 16018 u16 stack_depth_extra = 0; 16019 16020 for (i = 0; i < insn_cnt; i++, insn++) { 16021 struct bpf_loop_inline_state *inline_state = 16022 &env->insn_aux_data[i + delta].loop_inline_state; 16023 16024 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 16025 struct bpf_prog *new_prog; 16026 16027 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 16028 new_prog = inline_bpf_loop(env, 16029 i + delta, 16030 -(stack_depth + stack_depth_extra), 16031 inline_state->callback_subprogno, 16032 &cnt); 16033 if (!new_prog) 16034 return -ENOMEM; 16035 16036 delta += cnt - 1; 16037 env->prog = new_prog; 16038 insn = new_prog->insnsi + i + delta; 16039 } 16040 16041 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 16042 subprogs[cur_subprog].stack_depth += stack_depth_extra; 16043 cur_subprog++; 16044 stack_depth = subprogs[cur_subprog].stack_depth; 16045 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 16046 stack_depth_extra = 0; 16047 } 16048 } 16049 16050 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 16051 16052 return 0; 16053 } 16054 16055 static void free_states(struct bpf_verifier_env *env) 16056 { 16057 struct bpf_verifier_state_list *sl, *sln; 16058 int i; 16059 16060 sl = env->free_list; 16061 while (sl) { 16062 sln = sl->next; 16063 free_verifier_state(&sl->state, false); 16064 kfree(sl); 16065 sl = sln; 16066 } 16067 env->free_list = NULL; 16068 16069 if (!env->explored_states) 16070 return; 16071 16072 for (i = 0; i < state_htab_size(env); i++) { 16073 sl = env->explored_states[i]; 16074 16075 while (sl) { 16076 sln = sl->next; 16077 free_verifier_state(&sl->state, false); 16078 kfree(sl); 16079 sl = sln; 16080 } 16081 env->explored_states[i] = NULL; 16082 } 16083 } 16084 16085 static int do_check_common(struct bpf_verifier_env *env, int subprog) 16086 { 16087 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 16088 struct bpf_verifier_state *state; 16089 struct bpf_reg_state *regs; 16090 int ret, i; 16091 16092 env->prev_linfo = NULL; 16093 env->pass_cnt++; 16094 16095 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 16096 if (!state) 16097 return -ENOMEM; 16098 state->curframe = 0; 16099 state->speculative = false; 16100 state->branches = 1; 16101 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 16102 if (!state->frame[0]) { 16103 kfree(state); 16104 return -ENOMEM; 16105 } 16106 env->cur_state = state; 16107 init_func_state(env, state->frame[0], 16108 BPF_MAIN_FUNC /* callsite */, 16109 0 /* frameno */, 16110 subprog); 16111 state->first_insn_idx = env->subprog_info[subprog].start; 16112 state->last_insn_idx = -1; 16113 16114 regs = state->frame[state->curframe]->regs; 16115 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 16116 ret = btf_prepare_func_args(env, subprog, regs); 16117 if (ret) 16118 goto out; 16119 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 16120 if (regs[i].type == PTR_TO_CTX) 16121 mark_reg_known_zero(env, regs, i); 16122 else if (regs[i].type == SCALAR_VALUE) 16123 mark_reg_unknown(env, regs, i); 16124 else if (base_type(regs[i].type) == PTR_TO_MEM) { 16125 const u32 mem_size = regs[i].mem_size; 16126 16127 mark_reg_known_zero(env, regs, i); 16128 regs[i].mem_size = mem_size; 16129 regs[i].id = ++env->id_gen; 16130 } 16131 } 16132 } else { 16133 /* 1st arg to a function */ 16134 regs[BPF_REG_1].type = PTR_TO_CTX; 16135 mark_reg_known_zero(env, regs, BPF_REG_1); 16136 ret = btf_check_subprog_arg_match(env, subprog, regs); 16137 if (ret == -EFAULT) 16138 /* unlikely verifier bug. abort. 16139 * ret == 0 and ret < 0 are sadly acceptable for 16140 * main() function due to backward compatibility. 16141 * Like socket filter program may be written as: 16142 * int bpf_prog(struct pt_regs *ctx) 16143 * and never dereference that ctx in the program. 16144 * 'struct pt_regs' is a type mismatch for socket 16145 * filter that should be using 'struct __sk_buff'. 16146 */ 16147 goto out; 16148 } 16149 16150 ret = do_check(env); 16151 out: 16152 /* check for NULL is necessary, since cur_state can be freed inside 16153 * do_check() under memory pressure. 16154 */ 16155 if (env->cur_state) { 16156 free_verifier_state(env->cur_state, true); 16157 env->cur_state = NULL; 16158 } 16159 while (!pop_stack(env, NULL, NULL, false)); 16160 if (!ret && pop_log) 16161 bpf_vlog_reset(&env->log, 0); 16162 free_states(env); 16163 return ret; 16164 } 16165 16166 /* Verify all global functions in a BPF program one by one based on their BTF. 16167 * All global functions must pass verification. Otherwise the whole program is rejected. 16168 * Consider: 16169 * int bar(int); 16170 * int foo(int f) 16171 * { 16172 * return bar(f); 16173 * } 16174 * int bar(int b) 16175 * { 16176 * ... 16177 * } 16178 * foo() will be verified first for R1=any_scalar_value. During verification it 16179 * will be assumed that bar() already verified successfully and call to bar() 16180 * from foo() will be checked for type match only. Later bar() will be verified 16181 * independently to check that it's safe for R1=any_scalar_value. 16182 */ 16183 static int do_check_subprogs(struct bpf_verifier_env *env) 16184 { 16185 struct bpf_prog_aux *aux = env->prog->aux; 16186 int i, ret; 16187 16188 if (!aux->func_info) 16189 return 0; 16190 16191 for (i = 1; i < env->subprog_cnt; i++) { 16192 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 16193 continue; 16194 env->insn_idx = env->subprog_info[i].start; 16195 WARN_ON_ONCE(env->insn_idx == 0); 16196 ret = do_check_common(env, i); 16197 if (ret) { 16198 return ret; 16199 } else if (env->log.level & BPF_LOG_LEVEL) { 16200 verbose(env, 16201 "Func#%d is safe for any args that match its prototype\n", 16202 i); 16203 } 16204 } 16205 return 0; 16206 } 16207 16208 static int do_check_main(struct bpf_verifier_env *env) 16209 { 16210 int ret; 16211 16212 env->insn_idx = 0; 16213 ret = do_check_common(env, 0); 16214 if (!ret) 16215 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 16216 return ret; 16217 } 16218 16219 16220 static void print_verification_stats(struct bpf_verifier_env *env) 16221 { 16222 int i; 16223 16224 if (env->log.level & BPF_LOG_STATS) { 16225 verbose(env, "verification time %lld usec\n", 16226 div_u64(env->verification_time, 1000)); 16227 verbose(env, "stack depth "); 16228 for (i = 0; i < env->subprog_cnt; i++) { 16229 u32 depth = env->subprog_info[i].stack_depth; 16230 16231 verbose(env, "%d", depth); 16232 if (i + 1 < env->subprog_cnt) 16233 verbose(env, "+"); 16234 } 16235 verbose(env, "\n"); 16236 } 16237 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 16238 "total_states %d peak_states %d mark_read %d\n", 16239 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 16240 env->max_states_per_insn, env->total_states, 16241 env->peak_states, env->longest_mark_read_walk); 16242 } 16243 16244 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 16245 { 16246 const struct btf_type *t, *func_proto; 16247 const struct bpf_struct_ops *st_ops; 16248 const struct btf_member *member; 16249 struct bpf_prog *prog = env->prog; 16250 u32 btf_id, member_idx; 16251 const char *mname; 16252 16253 if (!prog->gpl_compatible) { 16254 verbose(env, "struct ops programs must have a GPL compatible license\n"); 16255 return -EINVAL; 16256 } 16257 16258 btf_id = prog->aux->attach_btf_id; 16259 st_ops = bpf_struct_ops_find(btf_id); 16260 if (!st_ops) { 16261 verbose(env, "attach_btf_id %u is not a supported struct\n", 16262 btf_id); 16263 return -ENOTSUPP; 16264 } 16265 16266 t = st_ops->type; 16267 member_idx = prog->expected_attach_type; 16268 if (member_idx >= btf_type_vlen(t)) { 16269 verbose(env, "attach to invalid member idx %u of struct %s\n", 16270 member_idx, st_ops->name); 16271 return -EINVAL; 16272 } 16273 16274 member = &btf_type_member(t)[member_idx]; 16275 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 16276 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 16277 NULL); 16278 if (!func_proto) { 16279 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 16280 mname, member_idx, st_ops->name); 16281 return -EINVAL; 16282 } 16283 16284 if (st_ops->check_member) { 16285 int err = st_ops->check_member(t, member); 16286 16287 if (err) { 16288 verbose(env, "attach to unsupported member %s of struct %s\n", 16289 mname, st_ops->name); 16290 return err; 16291 } 16292 } 16293 16294 prog->aux->attach_func_proto = func_proto; 16295 prog->aux->attach_func_name = mname; 16296 env->ops = st_ops->verifier_ops; 16297 16298 return 0; 16299 } 16300 #define SECURITY_PREFIX "security_" 16301 16302 static int check_attach_modify_return(unsigned long addr, const char *func_name) 16303 { 16304 if (within_error_injection_list(addr) || 16305 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 16306 return 0; 16307 16308 return -EINVAL; 16309 } 16310 16311 /* list of non-sleepable functions that are otherwise on 16312 * ALLOW_ERROR_INJECTION list 16313 */ 16314 BTF_SET_START(btf_non_sleepable_error_inject) 16315 /* Three functions below can be called from sleepable and non-sleepable context. 16316 * Assume non-sleepable from bpf safety point of view. 16317 */ 16318 BTF_ID(func, __filemap_add_folio) 16319 BTF_ID(func, should_fail_alloc_page) 16320 BTF_ID(func, should_failslab) 16321 BTF_SET_END(btf_non_sleepable_error_inject) 16322 16323 static int check_non_sleepable_error_inject(u32 btf_id) 16324 { 16325 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 16326 } 16327 16328 int bpf_check_attach_target(struct bpf_verifier_log *log, 16329 const struct bpf_prog *prog, 16330 const struct bpf_prog *tgt_prog, 16331 u32 btf_id, 16332 struct bpf_attach_target_info *tgt_info) 16333 { 16334 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 16335 const char prefix[] = "btf_trace_"; 16336 int ret = 0, subprog = -1, i; 16337 const struct btf_type *t; 16338 bool conservative = true; 16339 const char *tname; 16340 struct btf *btf; 16341 long addr = 0; 16342 16343 if (!btf_id) { 16344 bpf_log(log, "Tracing programs must provide btf_id\n"); 16345 return -EINVAL; 16346 } 16347 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 16348 if (!btf) { 16349 bpf_log(log, 16350 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 16351 return -EINVAL; 16352 } 16353 t = btf_type_by_id(btf, btf_id); 16354 if (!t) { 16355 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 16356 return -EINVAL; 16357 } 16358 tname = btf_name_by_offset(btf, t->name_off); 16359 if (!tname) { 16360 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 16361 return -EINVAL; 16362 } 16363 if (tgt_prog) { 16364 struct bpf_prog_aux *aux = tgt_prog->aux; 16365 16366 for (i = 0; i < aux->func_info_cnt; i++) 16367 if (aux->func_info[i].type_id == btf_id) { 16368 subprog = i; 16369 break; 16370 } 16371 if (subprog == -1) { 16372 bpf_log(log, "Subprog %s doesn't exist\n", tname); 16373 return -EINVAL; 16374 } 16375 conservative = aux->func_info_aux[subprog].unreliable; 16376 if (prog_extension) { 16377 if (conservative) { 16378 bpf_log(log, 16379 "Cannot replace static functions\n"); 16380 return -EINVAL; 16381 } 16382 if (!prog->jit_requested) { 16383 bpf_log(log, 16384 "Extension programs should be JITed\n"); 16385 return -EINVAL; 16386 } 16387 } 16388 if (!tgt_prog->jited) { 16389 bpf_log(log, "Can attach to only JITed progs\n"); 16390 return -EINVAL; 16391 } 16392 if (tgt_prog->type == prog->type) { 16393 /* Cannot fentry/fexit another fentry/fexit program. 16394 * Cannot attach program extension to another extension. 16395 * It's ok to attach fentry/fexit to extension program. 16396 */ 16397 bpf_log(log, "Cannot recursively attach\n"); 16398 return -EINVAL; 16399 } 16400 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 16401 prog_extension && 16402 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 16403 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 16404 /* Program extensions can extend all program types 16405 * except fentry/fexit. The reason is the following. 16406 * The fentry/fexit programs are used for performance 16407 * analysis, stats and can be attached to any program 16408 * type except themselves. When extension program is 16409 * replacing XDP function it is necessary to allow 16410 * performance analysis of all functions. Both original 16411 * XDP program and its program extension. Hence 16412 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 16413 * allowed. If extending of fentry/fexit was allowed it 16414 * would be possible to create long call chain 16415 * fentry->extension->fentry->extension beyond 16416 * reasonable stack size. Hence extending fentry is not 16417 * allowed. 16418 */ 16419 bpf_log(log, "Cannot extend fentry/fexit\n"); 16420 return -EINVAL; 16421 } 16422 } else { 16423 if (prog_extension) { 16424 bpf_log(log, "Cannot replace kernel functions\n"); 16425 return -EINVAL; 16426 } 16427 } 16428 16429 switch (prog->expected_attach_type) { 16430 case BPF_TRACE_RAW_TP: 16431 if (tgt_prog) { 16432 bpf_log(log, 16433 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 16434 return -EINVAL; 16435 } 16436 if (!btf_type_is_typedef(t)) { 16437 bpf_log(log, "attach_btf_id %u is not a typedef\n", 16438 btf_id); 16439 return -EINVAL; 16440 } 16441 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 16442 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 16443 btf_id, tname); 16444 return -EINVAL; 16445 } 16446 tname += sizeof(prefix) - 1; 16447 t = btf_type_by_id(btf, t->type); 16448 if (!btf_type_is_ptr(t)) 16449 /* should never happen in valid vmlinux build */ 16450 return -EINVAL; 16451 t = btf_type_by_id(btf, t->type); 16452 if (!btf_type_is_func_proto(t)) 16453 /* should never happen in valid vmlinux build */ 16454 return -EINVAL; 16455 16456 break; 16457 case BPF_TRACE_ITER: 16458 if (!btf_type_is_func(t)) { 16459 bpf_log(log, "attach_btf_id %u is not a function\n", 16460 btf_id); 16461 return -EINVAL; 16462 } 16463 t = btf_type_by_id(btf, t->type); 16464 if (!btf_type_is_func_proto(t)) 16465 return -EINVAL; 16466 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 16467 if (ret) 16468 return ret; 16469 break; 16470 default: 16471 if (!prog_extension) 16472 return -EINVAL; 16473 fallthrough; 16474 case BPF_MODIFY_RETURN: 16475 case BPF_LSM_MAC: 16476 case BPF_LSM_CGROUP: 16477 case BPF_TRACE_FENTRY: 16478 case BPF_TRACE_FEXIT: 16479 if (!btf_type_is_func(t)) { 16480 bpf_log(log, "attach_btf_id %u is not a function\n", 16481 btf_id); 16482 return -EINVAL; 16483 } 16484 if (prog_extension && 16485 btf_check_type_match(log, prog, btf, t)) 16486 return -EINVAL; 16487 t = btf_type_by_id(btf, t->type); 16488 if (!btf_type_is_func_proto(t)) 16489 return -EINVAL; 16490 16491 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 16492 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 16493 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 16494 return -EINVAL; 16495 16496 if (tgt_prog && conservative) 16497 t = NULL; 16498 16499 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 16500 if (ret < 0) 16501 return ret; 16502 16503 if (tgt_prog) { 16504 if (subprog == 0) 16505 addr = (long) tgt_prog->bpf_func; 16506 else 16507 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 16508 } else { 16509 addr = kallsyms_lookup_name(tname); 16510 if (!addr) { 16511 bpf_log(log, 16512 "The address of function %s cannot be found\n", 16513 tname); 16514 return -ENOENT; 16515 } 16516 } 16517 16518 if (prog->aux->sleepable) { 16519 ret = -EINVAL; 16520 switch (prog->type) { 16521 case BPF_PROG_TYPE_TRACING: 16522 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 16523 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 16524 */ 16525 if (!check_non_sleepable_error_inject(btf_id) && 16526 within_error_injection_list(addr)) 16527 ret = 0; 16528 break; 16529 case BPF_PROG_TYPE_LSM: 16530 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 16531 * Only some of them are sleepable. 16532 */ 16533 if (bpf_lsm_is_sleepable_hook(btf_id)) 16534 ret = 0; 16535 break; 16536 default: 16537 break; 16538 } 16539 if (ret) { 16540 bpf_log(log, "%s is not sleepable\n", tname); 16541 return ret; 16542 } 16543 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 16544 if (tgt_prog) { 16545 bpf_log(log, "can't modify return codes of BPF programs\n"); 16546 return -EINVAL; 16547 } 16548 ret = check_attach_modify_return(addr, tname); 16549 if (ret) { 16550 bpf_log(log, "%s() is not modifiable\n", tname); 16551 return ret; 16552 } 16553 } 16554 16555 break; 16556 } 16557 tgt_info->tgt_addr = addr; 16558 tgt_info->tgt_name = tname; 16559 tgt_info->tgt_type = t; 16560 return 0; 16561 } 16562 16563 BTF_SET_START(btf_id_deny) 16564 BTF_ID_UNUSED 16565 #ifdef CONFIG_SMP 16566 BTF_ID(func, migrate_disable) 16567 BTF_ID(func, migrate_enable) 16568 #endif 16569 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 16570 BTF_ID(func, rcu_read_unlock_strict) 16571 #endif 16572 BTF_SET_END(btf_id_deny) 16573 16574 static int check_attach_btf_id(struct bpf_verifier_env *env) 16575 { 16576 struct bpf_prog *prog = env->prog; 16577 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 16578 struct bpf_attach_target_info tgt_info = {}; 16579 u32 btf_id = prog->aux->attach_btf_id; 16580 struct bpf_trampoline *tr; 16581 int ret; 16582 u64 key; 16583 16584 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 16585 if (prog->aux->sleepable) 16586 /* attach_btf_id checked to be zero already */ 16587 return 0; 16588 verbose(env, "Syscall programs can only be sleepable\n"); 16589 return -EINVAL; 16590 } 16591 16592 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 16593 prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_KPROBE) { 16594 verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n"); 16595 return -EINVAL; 16596 } 16597 16598 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 16599 return check_struct_ops_btf_id(env); 16600 16601 if (prog->type != BPF_PROG_TYPE_TRACING && 16602 prog->type != BPF_PROG_TYPE_LSM && 16603 prog->type != BPF_PROG_TYPE_EXT) 16604 return 0; 16605 16606 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 16607 if (ret) 16608 return ret; 16609 16610 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 16611 /* to make freplace equivalent to their targets, they need to 16612 * inherit env->ops and expected_attach_type for the rest of the 16613 * verification 16614 */ 16615 env->ops = bpf_verifier_ops[tgt_prog->type]; 16616 prog->expected_attach_type = tgt_prog->expected_attach_type; 16617 } 16618 16619 /* store info about the attachment target that will be used later */ 16620 prog->aux->attach_func_proto = tgt_info.tgt_type; 16621 prog->aux->attach_func_name = tgt_info.tgt_name; 16622 16623 if (tgt_prog) { 16624 prog->aux->saved_dst_prog_type = tgt_prog->type; 16625 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 16626 } 16627 16628 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 16629 prog->aux->attach_btf_trace = true; 16630 return 0; 16631 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 16632 if (!bpf_iter_prog_supported(prog)) 16633 return -EINVAL; 16634 return 0; 16635 } 16636 16637 if (prog->type == BPF_PROG_TYPE_LSM) { 16638 ret = bpf_lsm_verify_prog(&env->log, prog); 16639 if (ret < 0) 16640 return ret; 16641 } else if (prog->type == BPF_PROG_TYPE_TRACING && 16642 btf_id_set_contains(&btf_id_deny, btf_id)) { 16643 return -EINVAL; 16644 } 16645 16646 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 16647 tr = bpf_trampoline_get(key, &tgt_info); 16648 if (!tr) 16649 return -ENOMEM; 16650 16651 prog->aux->dst_trampoline = tr; 16652 return 0; 16653 } 16654 16655 struct btf *bpf_get_btf_vmlinux(void) 16656 { 16657 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 16658 mutex_lock(&bpf_verifier_lock); 16659 if (!btf_vmlinux) 16660 btf_vmlinux = btf_parse_vmlinux(); 16661 mutex_unlock(&bpf_verifier_lock); 16662 } 16663 return btf_vmlinux; 16664 } 16665 16666 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) 16667 { 16668 u64 start_time = ktime_get_ns(); 16669 struct bpf_verifier_env *env; 16670 struct bpf_verifier_log *log; 16671 int i, len, ret = -EINVAL; 16672 bool is_priv; 16673 16674 /* no program is valid */ 16675 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 16676 return -EINVAL; 16677 16678 /* 'struct bpf_verifier_env' can be global, but since it's not small, 16679 * allocate/free it every time bpf_check() is called 16680 */ 16681 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 16682 if (!env) 16683 return -ENOMEM; 16684 log = &env->log; 16685 16686 len = (*prog)->len; 16687 env->insn_aux_data = 16688 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 16689 ret = -ENOMEM; 16690 if (!env->insn_aux_data) 16691 goto err_free_env; 16692 for (i = 0; i < len; i++) 16693 env->insn_aux_data[i].orig_idx = i; 16694 env->prog = *prog; 16695 env->ops = bpf_verifier_ops[env->prog->type]; 16696 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 16697 is_priv = bpf_capable(); 16698 16699 bpf_get_btf_vmlinux(); 16700 16701 /* grab the mutex to protect few globals used by verifier */ 16702 if (!is_priv) 16703 mutex_lock(&bpf_verifier_lock); 16704 16705 if (attr->log_level || attr->log_buf || attr->log_size) { 16706 /* user requested verbose verifier output 16707 * and supplied buffer to store the verification trace 16708 */ 16709 log->level = attr->log_level; 16710 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 16711 log->len_total = attr->log_size; 16712 16713 /* log attributes have to be sane */ 16714 if (!bpf_verifier_log_attr_valid(log)) { 16715 ret = -EINVAL; 16716 goto err_unlock; 16717 } 16718 } 16719 16720 mark_verifier_state_clean(env); 16721 16722 if (IS_ERR(btf_vmlinux)) { 16723 /* Either gcc or pahole or kernel are broken. */ 16724 verbose(env, "in-kernel BTF is malformed\n"); 16725 ret = PTR_ERR(btf_vmlinux); 16726 goto skip_full_check; 16727 } 16728 16729 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 16730 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 16731 env->strict_alignment = true; 16732 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 16733 env->strict_alignment = false; 16734 16735 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 16736 env->allow_uninit_stack = bpf_allow_uninit_stack(); 16737 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 16738 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 16739 env->bpf_capable = bpf_capable(); 16740 env->rcu_tag_supported = btf_vmlinux && 16741 btf_find_by_name_kind(btf_vmlinux, "rcu", BTF_KIND_TYPE_TAG) > 0; 16742 16743 if (is_priv) 16744 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 16745 16746 env->explored_states = kvcalloc(state_htab_size(env), 16747 sizeof(struct bpf_verifier_state_list *), 16748 GFP_USER); 16749 ret = -ENOMEM; 16750 if (!env->explored_states) 16751 goto skip_full_check; 16752 16753 ret = add_subprog_and_kfunc(env); 16754 if (ret < 0) 16755 goto skip_full_check; 16756 16757 ret = check_subprogs(env); 16758 if (ret < 0) 16759 goto skip_full_check; 16760 16761 ret = check_btf_info(env, attr, uattr); 16762 if (ret < 0) 16763 goto skip_full_check; 16764 16765 ret = check_attach_btf_id(env); 16766 if (ret) 16767 goto skip_full_check; 16768 16769 ret = resolve_pseudo_ldimm64(env); 16770 if (ret < 0) 16771 goto skip_full_check; 16772 16773 if (bpf_prog_is_dev_bound(env->prog->aux)) { 16774 ret = bpf_prog_offload_verifier_prep(env->prog); 16775 if (ret) 16776 goto skip_full_check; 16777 } 16778 16779 ret = check_cfg(env); 16780 if (ret < 0) 16781 goto skip_full_check; 16782 16783 ret = do_check_subprogs(env); 16784 ret = ret ?: do_check_main(env); 16785 16786 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 16787 ret = bpf_prog_offload_finalize(env); 16788 16789 skip_full_check: 16790 kvfree(env->explored_states); 16791 16792 if (ret == 0) 16793 ret = check_max_stack_depth(env); 16794 16795 /* instruction rewrites happen after this point */ 16796 if (ret == 0) 16797 ret = optimize_bpf_loop(env); 16798 16799 if (is_priv) { 16800 if (ret == 0) 16801 opt_hard_wire_dead_code_branches(env); 16802 if (ret == 0) 16803 ret = opt_remove_dead_code(env); 16804 if (ret == 0) 16805 ret = opt_remove_nops(env); 16806 } else { 16807 if (ret == 0) 16808 sanitize_dead_code(env); 16809 } 16810 16811 if (ret == 0) 16812 /* program is valid, convert *(u32*)(ctx + off) accesses */ 16813 ret = convert_ctx_accesses(env); 16814 16815 if (ret == 0) 16816 ret = do_misc_fixups(env); 16817 16818 /* do 32-bit optimization after insn patching has done so those patched 16819 * insns could be handled correctly. 16820 */ 16821 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 16822 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 16823 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 16824 : false; 16825 } 16826 16827 if (ret == 0) 16828 ret = fixup_call_args(env); 16829 16830 env->verification_time = ktime_get_ns() - start_time; 16831 print_verification_stats(env); 16832 env->prog->aux->verified_insns = env->insn_processed; 16833 16834 if (log->level && bpf_verifier_log_full(log)) 16835 ret = -ENOSPC; 16836 if (log->level && !log->ubuf) { 16837 ret = -EFAULT; 16838 goto err_release_maps; 16839 } 16840 16841 if (ret) 16842 goto err_release_maps; 16843 16844 if (env->used_map_cnt) { 16845 /* if program passed verifier, update used_maps in bpf_prog_info */ 16846 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 16847 sizeof(env->used_maps[0]), 16848 GFP_KERNEL); 16849 16850 if (!env->prog->aux->used_maps) { 16851 ret = -ENOMEM; 16852 goto err_release_maps; 16853 } 16854 16855 memcpy(env->prog->aux->used_maps, env->used_maps, 16856 sizeof(env->used_maps[0]) * env->used_map_cnt); 16857 env->prog->aux->used_map_cnt = env->used_map_cnt; 16858 } 16859 if (env->used_btf_cnt) { 16860 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 16861 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 16862 sizeof(env->used_btfs[0]), 16863 GFP_KERNEL); 16864 if (!env->prog->aux->used_btfs) { 16865 ret = -ENOMEM; 16866 goto err_release_maps; 16867 } 16868 16869 memcpy(env->prog->aux->used_btfs, env->used_btfs, 16870 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 16871 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 16872 } 16873 if (env->used_map_cnt || env->used_btf_cnt) { 16874 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 16875 * bpf_ld_imm64 instructions 16876 */ 16877 convert_pseudo_ld_imm64(env); 16878 } 16879 16880 adjust_btf_func(env); 16881 16882 err_release_maps: 16883 if (!env->prog->aux->used_maps) 16884 /* if we didn't copy map pointers into bpf_prog_info, release 16885 * them now. Otherwise free_used_maps() will release them. 16886 */ 16887 release_maps(env); 16888 if (!env->prog->aux->used_btfs) 16889 release_btfs(env); 16890 16891 /* extension progs temporarily inherit the attach_type of their targets 16892 for verification purposes, so set it back to zero before returning 16893 */ 16894 if (env->prog->type == BPF_PROG_TYPE_EXT) 16895 env->prog->expected_attach_type = 0; 16896 16897 *prog = env->prog; 16898 err_unlock: 16899 if (!is_priv) 16900 mutex_unlock(&bpf_verifier_lock); 16901 vfree(env->insn_aux_data); 16902 err_free_env: 16903 kfree(env); 16904 return ret; 16905 } 16906