1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 #include <linux/poison.h> 27 28 #include "disasm.h" 29 30 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 31 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 32 [_id] = & _name ## _verifier_ops, 33 #define BPF_MAP_TYPE(_id, _ops) 34 #define BPF_LINK_TYPE(_id, _name) 35 #include <linux/bpf_types.h> 36 #undef BPF_PROG_TYPE 37 #undef BPF_MAP_TYPE 38 #undef BPF_LINK_TYPE 39 }; 40 41 /* bpf_check() is a static code analyzer that walks eBPF program 42 * instruction by instruction and updates register/stack state. 43 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 44 * 45 * The first pass is depth-first-search to check that the program is a DAG. 46 * It rejects the following programs: 47 * - larger than BPF_MAXINSNS insns 48 * - if loop is present (detected via back-edge) 49 * - unreachable insns exist (shouldn't be a forest. program = one function) 50 * - out of bounds or malformed jumps 51 * The second pass is all possible path descent from the 1st insn. 52 * Since it's analyzing all paths through the program, the length of the 53 * analysis is limited to 64k insn, which may be hit even if total number of 54 * insn is less then 4K, but there are too many branches that change stack/regs. 55 * Number of 'branches to be analyzed' is limited to 1k 56 * 57 * On entry to each instruction, each register has a type, and the instruction 58 * changes the types of the registers depending on instruction semantics. 59 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 60 * copied to R1. 61 * 62 * All registers are 64-bit. 63 * R0 - return register 64 * R1-R5 argument passing registers 65 * R6-R9 callee saved registers 66 * R10 - frame pointer read-only 67 * 68 * At the start of BPF program the register R1 contains a pointer to bpf_context 69 * and has type PTR_TO_CTX. 70 * 71 * Verifier tracks arithmetic operations on pointers in case: 72 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 73 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 74 * 1st insn copies R10 (which has FRAME_PTR) type into R1 75 * and 2nd arithmetic instruction is pattern matched to recognize 76 * that it wants to construct a pointer to some element within stack. 77 * So after 2nd insn, the register R1 has type PTR_TO_STACK 78 * (and -20 constant is saved for further stack bounds checking). 79 * Meaning that this reg is a pointer to stack plus known immediate constant. 80 * 81 * Most of the time the registers have SCALAR_VALUE type, which 82 * means the register has some value, but it's not a valid pointer. 83 * (like pointer plus pointer becomes SCALAR_VALUE type) 84 * 85 * When verifier sees load or store instructions the type of base register 86 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 87 * four pointer types recognized by check_mem_access() function. 88 * 89 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 90 * and the range of [ptr, ptr + map's value_size) is accessible. 91 * 92 * registers used to pass values to function calls are checked against 93 * function argument constraints. 94 * 95 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 96 * It means that the register type passed to this function must be 97 * PTR_TO_STACK and it will be used inside the function as 98 * 'pointer to map element key' 99 * 100 * For example the argument constraints for bpf_map_lookup_elem(): 101 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 102 * .arg1_type = ARG_CONST_MAP_PTR, 103 * .arg2_type = ARG_PTR_TO_MAP_KEY, 104 * 105 * ret_type says that this function returns 'pointer to map elem value or null' 106 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 107 * 2nd argument should be a pointer to stack, which will be used inside 108 * the helper function as a pointer to map element key. 109 * 110 * On the kernel side the helper function looks like: 111 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 112 * { 113 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 114 * void *key = (void *) (unsigned long) r2; 115 * void *value; 116 * 117 * here kernel can access 'key' and 'map' pointers safely, knowing that 118 * [key, key + map->key_size) bytes are valid and were initialized on 119 * the stack of eBPF program. 120 * } 121 * 122 * Corresponding eBPF program may look like: 123 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 124 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 125 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 126 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 127 * here verifier looks at prototype of map_lookup_elem() and sees: 128 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 129 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 130 * 131 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 132 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 133 * and were initialized prior to this call. 134 * If it's ok, then verifier allows this BPF_CALL insn and looks at 135 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 136 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 137 * returns either pointer to map value or NULL. 138 * 139 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 140 * insn, the register holding that pointer in the true branch changes state to 141 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 142 * branch. See check_cond_jmp_op(). 143 * 144 * After the call R0 is set to return type of the function and registers R1-R5 145 * are set to NOT_INIT to indicate that they are no longer readable. 146 * 147 * The following reference types represent a potential reference to a kernel 148 * resource which, after first being allocated, must be checked and freed by 149 * the BPF program: 150 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 151 * 152 * When the verifier sees a helper call return a reference type, it allocates a 153 * pointer id for the reference and stores it in the current function state. 154 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 155 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 156 * passes through a NULL-check conditional. For the branch wherein the state is 157 * changed to CONST_IMM, the verifier releases the reference. 158 * 159 * For each helper function that allocates a reference, such as 160 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 161 * bpf_sk_release(). When a reference type passes into the release function, 162 * the verifier also releases the reference. If any unchecked or unreleased 163 * reference remains at the end of the program, the verifier rejects it. 164 */ 165 166 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 167 struct bpf_verifier_stack_elem { 168 /* verifer state is 'st' 169 * before processing instruction 'insn_idx' 170 * and after processing instruction 'prev_insn_idx' 171 */ 172 struct bpf_verifier_state st; 173 int insn_idx; 174 int prev_insn_idx; 175 struct bpf_verifier_stack_elem *next; 176 /* length of verifier log at the time this state was pushed on stack */ 177 u32 log_pos; 178 }; 179 180 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 181 #define BPF_COMPLEXITY_LIMIT_STATES 64 182 183 #define BPF_MAP_KEY_POISON (1ULL << 63) 184 #define BPF_MAP_KEY_SEEN (1ULL << 62) 185 186 #define BPF_MAP_PTR_UNPRIV 1UL 187 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 188 POISON_POINTER_DELTA)) 189 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 190 191 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx); 192 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id); 193 194 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 195 { 196 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 197 } 198 199 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 200 { 201 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 202 } 203 204 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 205 const struct bpf_map *map, bool unpriv) 206 { 207 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 208 unpriv |= bpf_map_ptr_unpriv(aux); 209 aux->map_ptr_state = (unsigned long)map | 210 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 211 } 212 213 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 214 { 215 return aux->map_key_state & BPF_MAP_KEY_POISON; 216 } 217 218 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 219 { 220 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 221 } 222 223 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 224 { 225 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 226 } 227 228 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 229 { 230 bool poisoned = bpf_map_key_poisoned(aux); 231 232 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 233 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 234 } 235 236 static bool bpf_pseudo_call(const struct bpf_insn *insn) 237 { 238 return insn->code == (BPF_JMP | BPF_CALL) && 239 insn->src_reg == BPF_PSEUDO_CALL; 240 } 241 242 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 243 { 244 return insn->code == (BPF_JMP | BPF_CALL) && 245 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 246 } 247 248 struct bpf_call_arg_meta { 249 struct bpf_map *map_ptr; 250 bool raw_mode; 251 bool pkt_access; 252 u8 release_regno; 253 int regno; 254 int access_size; 255 int mem_size; 256 u64 msize_max_value; 257 int ref_obj_id; 258 int map_uid; 259 int func_id; 260 struct btf *btf; 261 u32 btf_id; 262 struct btf *ret_btf; 263 u32 ret_btf_id; 264 u32 subprogno; 265 struct btf_field *kptr_field; 266 u8 uninit_dynptr_regno; 267 }; 268 269 struct btf *btf_vmlinux; 270 271 static DEFINE_MUTEX(bpf_verifier_lock); 272 273 static const struct bpf_line_info * 274 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 275 { 276 const struct bpf_line_info *linfo; 277 const struct bpf_prog *prog; 278 u32 i, nr_linfo; 279 280 prog = env->prog; 281 nr_linfo = prog->aux->nr_linfo; 282 283 if (!nr_linfo || insn_off >= prog->len) 284 return NULL; 285 286 linfo = prog->aux->linfo; 287 for (i = 1; i < nr_linfo; i++) 288 if (insn_off < linfo[i].insn_off) 289 break; 290 291 return &linfo[i - 1]; 292 } 293 294 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 295 va_list args) 296 { 297 unsigned int n; 298 299 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 300 301 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 302 "verifier log line truncated - local buffer too short\n"); 303 304 if (log->level == BPF_LOG_KERNEL) { 305 bool newline = n > 0 && log->kbuf[n - 1] == '\n'; 306 307 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n"); 308 return; 309 } 310 311 n = min(log->len_total - log->len_used - 1, n); 312 log->kbuf[n] = '\0'; 313 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 314 log->len_used += n; 315 else 316 log->ubuf = NULL; 317 } 318 319 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 320 { 321 char zero = 0; 322 323 if (!bpf_verifier_log_needed(log)) 324 return; 325 326 log->len_used = new_pos; 327 if (put_user(zero, log->ubuf + new_pos)) 328 log->ubuf = NULL; 329 } 330 331 /* log_level controls verbosity level of eBPF verifier. 332 * bpf_verifier_log_write() is used to dump the verification trace to the log, 333 * so the user can figure out what's wrong with the program 334 */ 335 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 336 const char *fmt, ...) 337 { 338 va_list args; 339 340 if (!bpf_verifier_log_needed(&env->log)) 341 return; 342 343 va_start(args, fmt); 344 bpf_verifier_vlog(&env->log, fmt, args); 345 va_end(args); 346 } 347 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 348 349 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 350 { 351 struct bpf_verifier_env *env = private_data; 352 va_list args; 353 354 if (!bpf_verifier_log_needed(&env->log)) 355 return; 356 357 va_start(args, fmt); 358 bpf_verifier_vlog(&env->log, fmt, args); 359 va_end(args); 360 } 361 362 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 363 const char *fmt, ...) 364 { 365 va_list args; 366 367 if (!bpf_verifier_log_needed(log)) 368 return; 369 370 va_start(args, fmt); 371 bpf_verifier_vlog(log, fmt, args); 372 va_end(args); 373 } 374 EXPORT_SYMBOL_GPL(bpf_log); 375 376 static const char *ltrim(const char *s) 377 { 378 while (isspace(*s)) 379 s++; 380 381 return s; 382 } 383 384 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 385 u32 insn_off, 386 const char *prefix_fmt, ...) 387 { 388 const struct bpf_line_info *linfo; 389 390 if (!bpf_verifier_log_needed(&env->log)) 391 return; 392 393 linfo = find_linfo(env, insn_off); 394 if (!linfo || linfo == env->prev_linfo) 395 return; 396 397 if (prefix_fmt) { 398 va_list args; 399 400 va_start(args, prefix_fmt); 401 bpf_verifier_vlog(&env->log, prefix_fmt, args); 402 va_end(args); 403 } 404 405 verbose(env, "%s\n", 406 ltrim(btf_name_by_offset(env->prog->aux->btf, 407 linfo->line_off))); 408 409 env->prev_linfo = linfo; 410 } 411 412 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 413 struct bpf_reg_state *reg, 414 struct tnum *range, const char *ctx, 415 const char *reg_name) 416 { 417 char tn_buf[48]; 418 419 verbose(env, "At %s the register %s ", ctx, reg_name); 420 if (!tnum_is_unknown(reg->var_off)) { 421 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 422 verbose(env, "has value %s", tn_buf); 423 } else { 424 verbose(env, "has unknown scalar value"); 425 } 426 tnum_strn(tn_buf, sizeof(tn_buf), *range); 427 verbose(env, " should have been in %s\n", tn_buf); 428 } 429 430 static bool type_is_pkt_pointer(enum bpf_reg_type type) 431 { 432 type = base_type(type); 433 return type == PTR_TO_PACKET || 434 type == PTR_TO_PACKET_META; 435 } 436 437 static bool type_is_sk_pointer(enum bpf_reg_type type) 438 { 439 return type == PTR_TO_SOCKET || 440 type == PTR_TO_SOCK_COMMON || 441 type == PTR_TO_TCP_SOCK || 442 type == PTR_TO_XDP_SOCK; 443 } 444 445 static bool reg_type_not_null(enum bpf_reg_type type) 446 { 447 return type == PTR_TO_SOCKET || 448 type == PTR_TO_TCP_SOCK || 449 type == PTR_TO_MAP_VALUE || 450 type == PTR_TO_MAP_KEY || 451 type == PTR_TO_SOCK_COMMON; 452 } 453 454 static bool type_is_ptr_alloc_obj(u32 type) 455 { 456 return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; 457 } 458 459 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) 460 { 461 struct btf_record *rec = NULL; 462 struct btf_struct_meta *meta; 463 464 if (reg->type == PTR_TO_MAP_VALUE) { 465 rec = reg->map_ptr->record; 466 } else if (type_is_ptr_alloc_obj(reg->type)) { 467 meta = btf_find_struct_meta(reg->btf, reg->btf_id); 468 if (meta) 469 rec = meta->record; 470 } 471 return rec; 472 } 473 474 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 475 { 476 return btf_record_has_field(reg_btf_record(reg), BPF_SPIN_LOCK); 477 } 478 479 static bool type_is_rdonly_mem(u32 type) 480 { 481 return type & MEM_RDONLY; 482 } 483 484 static bool type_may_be_null(u32 type) 485 { 486 return type & PTR_MAYBE_NULL; 487 } 488 489 static bool is_acquire_function(enum bpf_func_id func_id, 490 const struct bpf_map *map) 491 { 492 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 493 494 if (func_id == BPF_FUNC_sk_lookup_tcp || 495 func_id == BPF_FUNC_sk_lookup_udp || 496 func_id == BPF_FUNC_skc_lookup_tcp || 497 func_id == BPF_FUNC_ringbuf_reserve || 498 func_id == BPF_FUNC_kptr_xchg) 499 return true; 500 501 if (func_id == BPF_FUNC_map_lookup_elem && 502 (map_type == BPF_MAP_TYPE_SOCKMAP || 503 map_type == BPF_MAP_TYPE_SOCKHASH)) 504 return true; 505 506 return false; 507 } 508 509 static bool is_ptr_cast_function(enum bpf_func_id func_id) 510 { 511 return func_id == BPF_FUNC_tcp_sock || 512 func_id == BPF_FUNC_sk_fullsock || 513 func_id == BPF_FUNC_skc_to_tcp_sock || 514 func_id == BPF_FUNC_skc_to_tcp6_sock || 515 func_id == BPF_FUNC_skc_to_udp6_sock || 516 func_id == BPF_FUNC_skc_to_mptcp_sock || 517 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 518 func_id == BPF_FUNC_skc_to_tcp_request_sock; 519 } 520 521 static bool is_dynptr_ref_function(enum bpf_func_id func_id) 522 { 523 return func_id == BPF_FUNC_dynptr_data; 524 } 525 526 static bool is_callback_calling_function(enum bpf_func_id func_id) 527 { 528 return func_id == BPF_FUNC_for_each_map_elem || 529 func_id == BPF_FUNC_timer_set_callback || 530 func_id == BPF_FUNC_find_vma || 531 func_id == BPF_FUNC_loop || 532 func_id == BPF_FUNC_user_ringbuf_drain; 533 } 534 535 static bool is_storage_get_function(enum bpf_func_id func_id) 536 { 537 return func_id == BPF_FUNC_sk_storage_get || 538 func_id == BPF_FUNC_inode_storage_get || 539 func_id == BPF_FUNC_task_storage_get || 540 func_id == BPF_FUNC_cgrp_storage_get; 541 } 542 543 static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, 544 const struct bpf_map *map) 545 { 546 int ref_obj_uses = 0; 547 548 if (is_ptr_cast_function(func_id)) 549 ref_obj_uses++; 550 if (is_acquire_function(func_id, map)) 551 ref_obj_uses++; 552 if (is_dynptr_ref_function(func_id)) 553 ref_obj_uses++; 554 555 return ref_obj_uses > 1; 556 } 557 558 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 559 { 560 return BPF_CLASS(insn->code) == BPF_STX && 561 BPF_MODE(insn->code) == BPF_ATOMIC && 562 insn->imm == BPF_CMPXCHG; 563 } 564 565 /* string representation of 'enum bpf_reg_type' 566 * 567 * Note that reg_type_str() can not appear more than once in a single verbose() 568 * statement. 569 */ 570 static const char *reg_type_str(struct bpf_verifier_env *env, 571 enum bpf_reg_type type) 572 { 573 char postfix[16] = {0}, prefix[64] = {0}; 574 static const char * const str[] = { 575 [NOT_INIT] = "?", 576 [SCALAR_VALUE] = "scalar", 577 [PTR_TO_CTX] = "ctx", 578 [CONST_PTR_TO_MAP] = "map_ptr", 579 [PTR_TO_MAP_VALUE] = "map_value", 580 [PTR_TO_STACK] = "fp", 581 [PTR_TO_PACKET] = "pkt", 582 [PTR_TO_PACKET_META] = "pkt_meta", 583 [PTR_TO_PACKET_END] = "pkt_end", 584 [PTR_TO_FLOW_KEYS] = "flow_keys", 585 [PTR_TO_SOCKET] = "sock", 586 [PTR_TO_SOCK_COMMON] = "sock_common", 587 [PTR_TO_TCP_SOCK] = "tcp_sock", 588 [PTR_TO_TP_BUFFER] = "tp_buffer", 589 [PTR_TO_XDP_SOCK] = "xdp_sock", 590 [PTR_TO_BTF_ID] = "ptr_", 591 [PTR_TO_MEM] = "mem", 592 [PTR_TO_BUF] = "buf", 593 [PTR_TO_FUNC] = "func", 594 [PTR_TO_MAP_KEY] = "map_key", 595 [PTR_TO_DYNPTR] = "dynptr_ptr", 596 }; 597 598 if (type & PTR_MAYBE_NULL) { 599 if (base_type(type) == PTR_TO_BTF_ID) 600 strncpy(postfix, "or_null_", 16); 601 else 602 strncpy(postfix, "_or_null", 16); 603 } 604 605 snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s", 606 type & MEM_RDONLY ? "rdonly_" : "", 607 type & MEM_RINGBUF ? "ringbuf_" : "", 608 type & MEM_USER ? "user_" : "", 609 type & MEM_PERCPU ? "percpu_" : "", 610 type & MEM_RCU ? "rcu_" : "", 611 type & PTR_UNTRUSTED ? "untrusted_" : "", 612 type & PTR_TRUSTED ? "trusted_" : "" 613 ); 614 615 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", 616 prefix, str[base_type(type)], postfix); 617 return env->type_str_buf; 618 } 619 620 static char slot_type_char[] = { 621 [STACK_INVALID] = '?', 622 [STACK_SPILL] = 'r', 623 [STACK_MISC] = 'm', 624 [STACK_ZERO] = '0', 625 [STACK_DYNPTR] = 'd', 626 }; 627 628 static void print_liveness(struct bpf_verifier_env *env, 629 enum bpf_reg_liveness live) 630 { 631 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 632 verbose(env, "_"); 633 if (live & REG_LIVE_READ) 634 verbose(env, "r"); 635 if (live & REG_LIVE_WRITTEN) 636 verbose(env, "w"); 637 if (live & REG_LIVE_DONE) 638 verbose(env, "D"); 639 } 640 641 static int get_spi(s32 off) 642 { 643 return (-off - 1) / BPF_REG_SIZE; 644 } 645 646 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots) 647 { 648 int allocated_slots = state->allocated_stack / BPF_REG_SIZE; 649 650 /* We need to check that slots between [spi - nr_slots + 1, spi] are 651 * within [0, allocated_stack). 652 * 653 * Please note that the spi grows downwards. For example, a dynptr 654 * takes the size of two stack slots; the first slot will be at 655 * spi and the second slot will be at spi - 1. 656 */ 657 return spi - nr_slots + 1 >= 0 && spi < allocated_slots; 658 } 659 660 static struct bpf_func_state *func(struct bpf_verifier_env *env, 661 const struct bpf_reg_state *reg) 662 { 663 struct bpf_verifier_state *cur = env->cur_state; 664 665 return cur->frame[reg->frameno]; 666 } 667 668 static const char *kernel_type_name(const struct btf* btf, u32 id) 669 { 670 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 671 } 672 673 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) 674 { 675 env->scratched_regs |= 1U << regno; 676 } 677 678 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) 679 { 680 env->scratched_stack_slots |= 1ULL << spi; 681 } 682 683 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) 684 { 685 return (env->scratched_regs >> regno) & 1; 686 } 687 688 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) 689 { 690 return (env->scratched_stack_slots >> regno) & 1; 691 } 692 693 static bool verifier_state_scratched(const struct bpf_verifier_env *env) 694 { 695 return env->scratched_regs || env->scratched_stack_slots; 696 } 697 698 static void mark_verifier_state_clean(struct bpf_verifier_env *env) 699 { 700 env->scratched_regs = 0U; 701 env->scratched_stack_slots = 0ULL; 702 } 703 704 /* Used for printing the entire verifier state. */ 705 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) 706 { 707 env->scratched_regs = ~0U; 708 env->scratched_stack_slots = ~0ULL; 709 } 710 711 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type) 712 { 713 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 714 case DYNPTR_TYPE_LOCAL: 715 return BPF_DYNPTR_TYPE_LOCAL; 716 case DYNPTR_TYPE_RINGBUF: 717 return BPF_DYNPTR_TYPE_RINGBUF; 718 default: 719 return BPF_DYNPTR_TYPE_INVALID; 720 } 721 } 722 723 static bool dynptr_type_refcounted(enum bpf_dynptr_type type) 724 { 725 return type == BPF_DYNPTR_TYPE_RINGBUF; 726 } 727 728 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 729 enum bpf_arg_type arg_type, int insn_idx) 730 { 731 struct bpf_func_state *state = func(env, reg); 732 enum bpf_dynptr_type type; 733 int spi, i, id; 734 735 spi = get_spi(reg->off); 736 737 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 738 return -EINVAL; 739 740 for (i = 0; i < BPF_REG_SIZE; i++) { 741 state->stack[spi].slot_type[i] = STACK_DYNPTR; 742 state->stack[spi - 1].slot_type[i] = STACK_DYNPTR; 743 } 744 745 type = arg_to_dynptr_type(arg_type); 746 if (type == BPF_DYNPTR_TYPE_INVALID) 747 return -EINVAL; 748 749 state->stack[spi].spilled_ptr.dynptr.first_slot = true; 750 state->stack[spi].spilled_ptr.dynptr.type = type; 751 state->stack[spi - 1].spilled_ptr.dynptr.type = type; 752 753 if (dynptr_type_refcounted(type)) { 754 /* The id is used to track proper releasing */ 755 id = acquire_reference_state(env, insn_idx); 756 if (id < 0) 757 return id; 758 759 state->stack[spi].spilled_ptr.id = id; 760 state->stack[spi - 1].spilled_ptr.id = id; 761 } 762 763 return 0; 764 } 765 766 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 767 { 768 struct bpf_func_state *state = func(env, reg); 769 int spi, i; 770 771 spi = get_spi(reg->off); 772 773 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 774 return -EINVAL; 775 776 for (i = 0; i < BPF_REG_SIZE; i++) { 777 state->stack[spi].slot_type[i] = STACK_INVALID; 778 state->stack[spi - 1].slot_type[i] = STACK_INVALID; 779 } 780 781 /* Invalidate any slices associated with this dynptr */ 782 if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) { 783 release_reference(env, state->stack[spi].spilled_ptr.id); 784 state->stack[spi].spilled_ptr.id = 0; 785 state->stack[spi - 1].spilled_ptr.id = 0; 786 } 787 788 state->stack[spi].spilled_ptr.dynptr.first_slot = false; 789 state->stack[spi].spilled_ptr.dynptr.type = 0; 790 state->stack[spi - 1].spilled_ptr.dynptr.type = 0; 791 792 return 0; 793 } 794 795 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 796 { 797 struct bpf_func_state *state = func(env, reg); 798 int spi = get_spi(reg->off); 799 int i; 800 801 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS)) 802 return true; 803 804 for (i = 0; i < BPF_REG_SIZE; i++) { 805 if (state->stack[spi].slot_type[i] == STACK_DYNPTR || 806 state->stack[spi - 1].slot_type[i] == STACK_DYNPTR) 807 return false; 808 } 809 810 return true; 811 } 812 813 bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, 814 struct bpf_reg_state *reg) 815 { 816 struct bpf_func_state *state = func(env, reg); 817 int spi = get_spi(reg->off); 818 int i; 819 820 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) || 821 !state->stack[spi].spilled_ptr.dynptr.first_slot) 822 return false; 823 824 for (i = 0; i < BPF_REG_SIZE; i++) { 825 if (state->stack[spi].slot_type[i] != STACK_DYNPTR || 826 state->stack[spi - 1].slot_type[i] != STACK_DYNPTR) 827 return false; 828 } 829 830 return true; 831 } 832 833 bool is_dynptr_type_expected(struct bpf_verifier_env *env, 834 struct bpf_reg_state *reg, 835 enum bpf_arg_type arg_type) 836 { 837 struct bpf_func_state *state = func(env, reg); 838 enum bpf_dynptr_type dynptr_type; 839 int spi = get_spi(reg->off); 840 841 /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ 842 if (arg_type == ARG_PTR_TO_DYNPTR) 843 return true; 844 845 dynptr_type = arg_to_dynptr_type(arg_type); 846 847 return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; 848 } 849 850 /* The reg state of a pointer or a bounded scalar was saved when 851 * it was spilled to the stack. 852 */ 853 static bool is_spilled_reg(const struct bpf_stack_state *stack) 854 { 855 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 856 } 857 858 static void scrub_spilled_slot(u8 *stype) 859 { 860 if (*stype != STACK_INVALID) 861 *stype = STACK_MISC; 862 } 863 864 static void print_verifier_state(struct bpf_verifier_env *env, 865 const struct bpf_func_state *state, 866 bool print_all) 867 { 868 const struct bpf_reg_state *reg; 869 enum bpf_reg_type t; 870 int i; 871 872 if (state->frameno) 873 verbose(env, " frame%d:", state->frameno); 874 for (i = 0; i < MAX_BPF_REG; i++) { 875 reg = &state->regs[i]; 876 t = reg->type; 877 if (t == NOT_INIT) 878 continue; 879 if (!print_all && !reg_scratched(env, i)) 880 continue; 881 verbose(env, " R%d", i); 882 print_liveness(env, reg->live); 883 verbose(env, "="); 884 if (t == SCALAR_VALUE && reg->precise) 885 verbose(env, "P"); 886 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 887 tnum_is_const(reg->var_off)) { 888 /* reg->off should be 0 for SCALAR_VALUE */ 889 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 890 verbose(env, "%lld", reg->var_off.value + reg->off); 891 } else { 892 const char *sep = ""; 893 894 verbose(env, "%s", reg_type_str(env, t)); 895 if (base_type(t) == PTR_TO_BTF_ID) 896 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 897 verbose(env, "("); 898 /* 899 * _a stands for append, was shortened to avoid multiline statements below. 900 * This macro is used to output a comma separated list of attributes. 901 */ 902 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) 903 904 if (reg->id) 905 verbose_a("id=%d", reg->id); 906 if (reg->ref_obj_id) 907 verbose_a("ref_obj_id=%d", reg->ref_obj_id); 908 if (t != SCALAR_VALUE) 909 verbose_a("off=%d", reg->off); 910 if (type_is_pkt_pointer(t)) 911 verbose_a("r=%d", reg->range); 912 else if (base_type(t) == CONST_PTR_TO_MAP || 913 base_type(t) == PTR_TO_MAP_KEY || 914 base_type(t) == PTR_TO_MAP_VALUE) 915 verbose_a("ks=%d,vs=%d", 916 reg->map_ptr->key_size, 917 reg->map_ptr->value_size); 918 if (tnum_is_const(reg->var_off)) { 919 /* Typically an immediate SCALAR_VALUE, but 920 * could be a pointer whose offset is too big 921 * for reg->off 922 */ 923 verbose_a("imm=%llx", reg->var_off.value); 924 } else { 925 if (reg->smin_value != reg->umin_value && 926 reg->smin_value != S64_MIN) 927 verbose_a("smin=%lld", (long long)reg->smin_value); 928 if (reg->smax_value != reg->umax_value && 929 reg->smax_value != S64_MAX) 930 verbose_a("smax=%lld", (long long)reg->smax_value); 931 if (reg->umin_value != 0) 932 verbose_a("umin=%llu", (unsigned long long)reg->umin_value); 933 if (reg->umax_value != U64_MAX) 934 verbose_a("umax=%llu", (unsigned long long)reg->umax_value); 935 if (!tnum_is_unknown(reg->var_off)) { 936 char tn_buf[48]; 937 938 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 939 verbose_a("var_off=%s", tn_buf); 940 } 941 if (reg->s32_min_value != reg->smin_value && 942 reg->s32_min_value != S32_MIN) 943 verbose_a("s32_min=%d", (int)(reg->s32_min_value)); 944 if (reg->s32_max_value != reg->smax_value && 945 reg->s32_max_value != S32_MAX) 946 verbose_a("s32_max=%d", (int)(reg->s32_max_value)); 947 if (reg->u32_min_value != reg->umin_value && 948 reg->u32_min_value != U32_MIN) 949 verbose_a("u32_min=%d", (int)(reg->u32_min_value)); 950 if (reg->u32_max_value != reg->umax_value && 951 reg->u32_max_value != U32_MAX) 952 verbose_a("u32_max=%d", (int)(reg->u32_max_value)); 953 } 954 #undef verbose_a 955 956 verbose(env, ")"); 957 } 958 } 959 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 960 char types_buf[BPF_REG_SIZE + 1]; 961 bool valid = false; 962 int j; 963 964 for (j = 0; j < BPF_REG_SIZE; j++) { 965 if (state->stack[i].slot_type[j] != STACK_INVALID) 966 valid = true; 967 types_buf[j] = slot_type_char[ 968 state->stack[i].slot_type[j]]; 969 } 970 types_buf[BPF_REG_SIZE] = 0; 971 if (!valid) 972 continue; 973 if (!print_all && !stack_slot_scratched(env, i)) 974 continue; 975 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 976 print_liveness(env, state->stack[i].spilled_ptr.live); 977 if (is_spilled_reg(&state->stack[i])) { 978 reg = &state->stack[i].spilled_ptr; 979 t = reg->type; 980 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 981 if (t == SCALAR_VALUE && reg->precise) 982 verbose(env, "P"); 983 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 984 verbose(env, "%lld", reg->var_off.value + reg->off); 985 } else { 986 verbose(env, "=%s", types_buf); 987 } 988 } 989 if (state->acquired_refs && state->refs[0].id) { 990 verbose(env, " refs=%d", state->refs[0].id); 991 for (i = 1; i < state->acquired_refs; i++) 992 if (state->refs[i].id) 993 verbose(env, ",%d", state->refs[i].id); 994 } 995 if (state->in_callback_fn) 996 verbose(env, " cb"); 997 if (state->in_async_callback_fn) 998 verbose(env, " async_cb"); 999 verbose(env, "\n"); 1000 mark_verifier_state_clean(env); 1001 } 1002 1003 static inline u32 vlog_alignment(u32 pos) 1004 { 1005 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), 1006 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 1007 } 1008 1009 static void print_insn_state(struct bpf_verifier_env *env, 1010 const struct bpf_func_state *state) 1011 { 1012 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { 1013 /* remove new line character */ 1014 bpf_vlog_reset(&env->log, env->prev_log_len - 1); 1015 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); 1016 } else { 1017 verbose(env, "%d:", env->insn_idx); 1018 } 1019 print_verifier_state(env, state, false); 1020 } 1021 1022 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 1023 * small to hold src. This is different from krealloc since we don't want to preserve 1024 * the contents of dst. 1025 * 1026 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 1027 * not be allocated. 1028 */ 1029 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 1030 { 1031 size_t bytes; 1032 1033 if (ZERO_OR_NULL_PTR(src)) 1034 goto out; 1035 1036 if (unlikely(check_mul_overflow(n, size, &bytes))) 1037 return NULL; 1038 1039 if (ksize(dst) < ksize(src)) { 1040 kfree(dst); 1041 dst = kmalloc_track_caller(kmalloc_size_roundup(bytes), flags); 1042 if (!dst) 1043 return NULL; 1044 } 1045 1046 memcpy(dst, src, bytes); 1047 out: 1048 return dst ? dst : ZERO_SIZE_PTR; 1049 } 1050 1051 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 1052 * small to hold new_n items. new items are zeroed out if the array grows. 1053 * 1054 * Contrary to krealloc_array, does not free arr if new_n is zero. 1055 */ 1056 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 1057 { 1058 size_t alloc_size; 1059 void *new_arr; 1060 1061 if (!new_n || old_n == new_n) 1062 goto out; 1063 1064 alloc_size = kmalloc_size_roundup(size_mul(new_n, size)); 1065 new_arr = krealloc(arr, alloc_size, GFP_KERNEL); 1066 if (!new_arr) { 1067 kfree(arr); 1068 return NULL; 1069 } 1070 arr = new_arr; 1071 1072 if (new_n > old_n) 1073 memset(arr + old_n * size, 0, (new_n - old_n) * size); 1074 1075 out: 1076 return arr ? arr : ZERO_SIZE_PTR; 1077 } 1078 1079 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1080 { 1081 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 1082 sizeof(struct bpf_reference_state), GFP_KERNEL); 1083 if (!dst->refs) 1084 return -ENOMEM; 1085 1086 dst->acquired_refs = src->acquired_refs; 1087 return 0; 1088 } 1089 1090 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 1091 { 1092 size_t n = src->allocated_stack / BPF_REG_SIZE; 1093 1094 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 1095 GFP_KERNEL); 1096 if (!dst->stack) 1097 return -ENOMEM; 1098 1099 dst->allocated_stack = src->allocated_stack; 1100 return 0; 1101 } 1102 1103 static int resize_reference_state(struct bpf_func_state *state, size_t n) 1104 { 1105 state->refs = realloc_array(state->refs, state->acquired_refs, n, 1106 sizeof(struct bpf_reference_state)); 1107 if (!state->refs) 1108 return -ENOMEM; 1109 1110 state->acquired_refs = n; 1111 return 0; 1112 } 1113 1114 static int grow_stack_state(struct bpf_func_state *state, int size) 1115 { 1116 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 1117 1118 if (old_n >= n) 1119 return 0; 1120 1121 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 1122 if (!state->stack) 1123 return -ENOMEM; 1124 1125 state->allocated_stack = size; 1126 return 0; 1127 } 1128 1129 /* Acquire a pointer id from the env and update the state->refs to include 1130 * this new pointer reference. 1131 * On success, returns a valid pointer id to associate with the register 1132 * On failure, returns a negative errno. 1133 */ 1134 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 1135 { 1136 struct bpf_func_state *state = cur_func(env); 1137 int new_ofs = state->acquired_refs; 1138 int id, err; 1139 1140 err = resize_reference_state(state, state->acquired_refs + 1); 1141 if (err) 1142 return err; 1143 id = ++env->id_gen; 1144 state->refs[new_ofs].id = id; 1145 state->refs[new_ofs].insn_idx = insn_idx; 1146 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; 1147 1148 return id; 1149 } 1150 1151 /* release function corresponding to acquire_reference_state(). Idempotent. */ 1152 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 1153 { 1154 int i, last_idx; 1155 1156 last_idx = state->acquired_refs - 1; 1157 for (i = 0; i < state->acquired_refs; i++) { 1158 if (state->refs[i].id == ptr_id) { 1159 /* Cannot release caller references in callbacks */ 1160 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 1161 return -EINVAL; 1162 if (last_idx && i != last_idx) 1163 memcpy(&state->refs[i], &state->refs[last_idx], 1164 sizeof(*state->refs)); 1165 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 1166 state->acquired_refs--; 1167 return 0; 1168 } 1169 } 1170 return -EINVAL; 1171 } 1172 1173 static void free_func_state(struct bpf_func_state *state) 1174 { 1175 if (!state) 1176 return; 1177 kfree(state->refs); 1178 kfree(state->stack); 1179 kfree(state); 1180 } 1181 1182 static void clear_jmp_history(struct bpf_verifier_state *state) 1183 { 1184 kfree(state->jmp_history); 1185 state->jmp_history = NULL; 1186 state->jmp_history_cnt = 0; 1187 } 1188 1189 static void free_verifier_state(struct bpf_verifier_state *state, 1190 bool free_self) 1191 { 1192 int i; 1193 1194 for (i = 0; i <= state->curframe; i++) { 1195 free_func_state(state->frame[i]); 1196 state->frame[i] = NULL; 1197 } 1198 clear_jmp_history(state); 1199 if (free_self) 1200 kfree(state); 1201 } 1202 1203 /* copy verifier state from src to dst growing dst stack space 1204 * when necessary to accommodate larger src stack 1205 */ 1206 static int copy_func_state(struct bpf_func_state *dst, 1207 const struct bpf_func_state *src) 1208 { 1209 int err; 1210 1211 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1212 err = copy_reference_state(dst, src); 1213 if (err) 1214 return err; 1215 return copy_stack_state(dst, src); 1216 } 1217 1218 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1219 const struct bpf_verifier_state *src) 1220 { 1221 struct bpf_func_state *dst; 1222 int i, err; 1223 1224 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1225 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 1226 GFP_USER); 1227 if (!dst_state->jmp_history) 1228 return -ENOMEM; 1229 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1230 1231 /* if dst has more stack frames then src frame, free them */ 1232 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1233 free_func_state(dst_state->frame[i]); 1234 dst_state->frame[i] = NULL; 1235 } 1236 dst_state->speculative = src->speculative; 1237 dst_state->active_rcu_lock = src->active_rcu_lock; 1238 dst_state->curframe = src->curframe; 1239 dst_state->active_lock.ptr = src->active_lock.ptr; 1240 dst_state->active_lock.id = src->active_lock.id; 1241 dst_state->branches = src->branches; 1242 dst_state->parent = src->parent; 1243 dst_state->first_insn_idx = src->first_insn_idx; 1244 dst_state->last_insn_idx = src->last_insn_idx; 1245 for (i = 0; i <= src->curframe; i++) { 1246 dst = dst_state->frame[i]; 1247 if (!dst) { 1248 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1249 if (!dst) 1250 return -ENOMEM; 1251 dst_state->frame[i] = dst; 1252 } 1253 err = copy_func_state(dst, src->frame[i]); 1254 if (err) 1255 return err; 1256 } 1257 return 0; 1258 } 1259 1260 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1261 { 1262 while (st) { 1263 u32 br = --st->branches; 1264 1265 /* WARN_ON(br > 1) technically makes sense here, 1266 * but see comment in push_stack(), hence: 1267 */ 1268 WARN_ONCE((int)br < 0, 1269 "BUG update_branch_counts:branches_to_explore=%d\n", 1270 br); 1271 if (br) 1272 break; 1273 st = st->parent; 1274 } 1275 } 1276 1277 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1278 int *insn_idx, bool pop_log) 1279 { 1280 struct bpf_verifier_state *cur = env->cur_state; 1281 struct bpf_verifier_stack_elem *elem, *head = env->head; 1282 int err; 1283 1284 if (env->head == NULL) 1285 return -ENOENT; 1286 1287 if (cur) { 1288 err = copy_verifier_state(cur, &head->st); 1289 if (err) 1290 return err; 1291 } 1292 if (pop_log) 1293 bpf_vlog_reset(&env->log, head->log_pos); 1294 if (insn_idx) 1295 *insn_idx = head->insn_idx; 1296 if (prev_insn_idx) 1297 *prev_insn_idx = head->prev_insn_idx; 1298 elem = head->next; 1299 free_verifier_state(&head->st, false); 1300 kfree(head); 1301 env->head = elem; 1302 env->stack_size--; 1303 return 0; 1304 } 1305 1306 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1307 int insn_idx, int prev_insn_idx, 1308 bool speculative) 1309 { 1310 struct bpf_verifier_state *cur = env->cur_state; 1311 struct bpf_verifier_stack_elem *elem; 1312 int err; 1313 1314 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1315 if (!elem) 1316 goto err; 1317 1318 elem->insn_idx = insn_idx; 1319 elem->prev_insn_idx = prev_insn_idx; 1320 elem->next = env->head; 1321 elem->log_pos = env->log.len_used; 1322 env->head = elem; 1323 env->stack_size++; 1324 err = copy_verifier_state(&elem->st, cur); 1325 if (err) 1326 goto err; 1327 elem->st.speculative |= speculative; 1328 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1329 verbose(env, "The sequence of %d jumps is too complex.\n", 1330 env->stack_size); 1331 goto err; 1332 } 1333 if (elem->st.parent) { 1334 ++elem->st.parent->branches; 1335 /* WARN_ON(branches > 2) technically makes sense here, 1336 * but 1337 * 1. speculative states will bump 'branches' for non-branch 1338 * instructions 1339 * 2. is_state_visited() heuristics may decide not to create 1340 * a new state for a sequence of branches and all such current 1341 * and cloned states will be pointing to a single parent state 1342 * which might have large 'branches' count. 1343 */ 1344 } 1345 return &elem->st; 1346 err: 1347 free_verifier_state(env->cur_state, true); 1348 env->cur_state = NULL; 1349 /* pop all elements and return */ 1350 while (!pop_stack(env, NULL, NULL, false)); 1351 return NULL; 1352 } 1353 1354 #define CALLER_SAVED_REGS 6 1355 static const int caller_saved[CALLER_SAVED_REGS] = { 1356 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1357 }; 1358 1359 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1360 struct bpf_reg_state *reg); 1361 1362 /* This helper doesn't clear reg->id */ 1363 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1364 { 1365 reg->var_off = tnum_const(imm); 1366 reg->smin_value = (s64)imm; 1367 reg->smax_value = (s64)imm; 1368 reg->umin_value = imm; 1369 reg->umax_value = imm; 1370 1371 reg->s32_min_value = (s32)imm; 1372 reg->s32_max_value = (s32)imm; 1373 reg->u32_min_value = (u32)imm; 1374 reg->u32_max_value = (u32)imm; 1375 } 1376 1377 /* Mark the unknown part of a register (variable offset or scalar value) as 1378 * known to have the value @imm. 1379 */ 1380 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1381 { 1382 /* Clear id, off, and union(map_ptr, range) */ 1383 memset(((u8 *)reg) + sizeof(reg->type), 0, 1384 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1385 ___mark_reg_known(reg, imm); 1386 } 1387 1388 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1389 { 1390 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1391 reg->s32_min_value = (s32)imm; 1392 reg->s32_max_value = (s32)imm; 1393 reg->u32_min_value = (u32)imm; 1394 reg->u32_max_value = (u32)imm; 1395 } 1396 1397 /* Mark the 'variable offset' part of a register as zero. This should be 1398 * used only on registers holding a pointer type. 1399 */ 1400 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1401 { 1402 __mark_reg_known(reg, 0); 1403 } 1404 1405 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1406 { 1407 __mark_reg_known(reg, 0); 1408 reg->type = SCALAR_VALUE; 1409 } 1410 1411 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1412 struct bpf_reg_state *regs, u32 regno) 1413 { 1414 if (WARN_ON(regno >= MAX_BPF_REG)) { 1415 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1416 /* Something bad happened, let's kill all regs */ 1417 for (regno = 0; regno < MAX_BPF_REG; regno++) 1418 __mark_reg_not_init(env, regs + regno); 1419 return; 1420 } 1421 __mark_reg_known_zero(regs + regno); 1422 } 1423 1424 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1425 { 1426 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1427 const struct bpf_map *map = reg->map_ptr; 1428 1429 if (map->inner_map_meta) { 1430 reg->type = CONST_PTR_TO_MAP; 1431 reg->map_ptr = map->inner_map_meta; 1432 /* transfer reg's id which is unique for every map_lookup_elem 1433 * as UID of the inner map. 1434 */ 1435 if (btf_record_has_field(map->inner_map_meta->record, BPF_TIMER)) 1436 reg->map_uid = reg->id; 1437 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1438 reg->type = PTR_TO_XDP_SOCK; 1439 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1440 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1441 reg->type = PTR_TO_SOCKET; 1442 } else { 1443 reg->type = PTR_TO_MAP_VALUE; 1444 } 1445 return; 1446 } 1447 1448 reg->type &= ~PTR_MAYBE_NULL; 1449 } 1450 1451 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1452 { 1453 return type_is_pkt_pointer(reg->type); 1454 } 1455 1456 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1457 { 1458 return reg_is_pkt_pointer(reg) || 1459 reg->type == PTR_TO_PACKET_END; 1460 } 1461 1462 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1463 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1464 enum bpf_reg_type which) 1465 { 1466 /* The register can already have a range from prior markings. 1467 * This is fine as long as it hasn't been advanced from its 1468 * origin. 1469 */ 1470 return reg->type == which && 1471 reg->id == 0 && 1472 reg->off == 0 && 1473 tnum_equals_const(reg->var_off, 0); 1474 } 1475 1476 /* Reset the min/max bounds of a register */ 1477 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1478 { 1479 reg->smin_value = S64_MIN; 1480 reg->smax_value = S64_MAX; 1481 reg->umin_value = 0; 1482 reg->umax_value = U64_MAX; 1483 1484 reg->s32_min_value = S32_MIN; 1485 reg->s32_max_value = S32_MAX; 1486 reg->u32_min_value = 0; 1487 reg->u32_max_value = U32_MAX; 1488 } 1489 1490 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1491 { 1492 reg->smin_value = S64_MIN; 1493 reg->smax_value = S64_MAX; 1494 reg->umin_value = 0; 1495 reg->umax_value = U64_MAX; 1496 } 1497 1498 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1499 { 1500 reg->s32_min_value = S32_MIN; 1501 reg->s32_max_value = S32_MAX; 1502 reg->u32_min_value = 0; 1503 reg->u32_max_value = U32_MAX; 1504 } 1505 1506 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1507 { 1508 struct tnum var32_off = tnum_subreg(reg->var_off); 1509 1510 /* min signed is max(sign bit) | min(other bits) */ 1511 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1512 var32_off.value | (var32_off.mask & S32_MIN)); 1513 /* max signed is min(sign bit) | max(other bits) */ 1514 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1515 var32_off.value | (var32_off.mask & S32_MAX)); 1516 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1517 reg->u32_max_value = min(reg->u32_max_value, 1518 (u32)(var32_off.value | var32_off.mask)); 1519 } 1520 1521 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1522 { 1523 /* min signed is max(sign bit) | min(other bits) */ 1524 reg->smin_value = max_t(s64, reg->smin_value, 1525 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1526 /* max signed is min(sign bit) | max(other bits) */ 1527 reg->smax_value = min_t(s64, reg->smax_value, 1528 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1529 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1530 reg->umax_value = min(reg->umax_value, 1531 reg->var_off.value | reg->var_off.mask); 1532 } 1533 1534 static void __update_reg_bounds(struct bpf_reg_state *reg) 1535 { 1536 __update_reg32_bounds(reg); 1537 __update_reg64_bounds(reg); 1538 } 1539 1540 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1541 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1542 { 1543 /* Learn sign from signed bounds. 1544 * If we cannot cross the sign boundary, then signed and unsigned bounds 1545 * are the same, so combine. This works even in the negative case, e.g. 1546 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1547 */ 1548 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1549 reg->s32_min_value = reg->u32_min_value = 1550 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1551 reg->s32_max_value = reg->u32_max_value = 1552 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1553 return; 1554 } 1555 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1556 * boundary, so we must be careful. 1557 */ 1558 if ((s32)reg->u32_max_value >= 0) { 1559 /* Positive. We can't learn anything from the smin, but smax 1560 * is positive, hence safe. 1561 */ 1562 reg->s32_min_value = reg->u32_min_value; 1563 reg->s32_max_value = reg->u32_max_value = 1564 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1565 } else if ((s32)reg->u32_min_value < 0) { 1566 /* Negative. We can't learn anything from the smax, but smin 1567 * is negative, hence safe. 1568 */ 1569 reg->s32_min_value = reg->u32_min_value = 1570 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1571 reg->s32_max_value = reg->u32_max_value; 1572 } 1573 } 1574 1575 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1576 { 1577 /* Learn sign from signed bounds. 1578 * If we cannot cross the sign boundary, then signed and unsigned bounds 1579 * are the same, so combine. This works even in the negative case, e.g. 1580 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1581 */ 1582 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1583 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1584 reg->umin_value); 1585 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1586 reg->umax_value); 1587 return; 1588 } 1589 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1590 * boundary, so we must be careful. 1591 */ 1592 if ((s64)reg->umax_value >= 0) { 1593 /* Positive. We can't learn anything from the smin, but smax 1594 * is positive, hence safe. 1595 */ 1596 reg->smin_value = reg->umin_value; 1597 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1598 reg->umax_value); 1599 } else if ((s64)reg->umin_value < 0) { 1600 /* Negative. We can't learn anything from the smax, but smin 1601 * is negative, hence safe. 1602 */ 1603 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1604 reg->umin_value); 1605 reg->smax_value = reg->umax_value; 1606 } 1607 } 1608 1609 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1610 { 1611 __reg32_deduce_bounds(reg); 1612 __reg64_deduce_bounds(reg); 1613 } 1614 1615 /* Attempts to improve var_off based on unsigned min/max information */ 1616 static void __reg_bound_offset(struct bpf_reg_state *reg) 1617 { 1618 struct tnum var64_off = tnum_intersect(reg->var_off, 1619 tnum_range(reg->umin_value, 1620 reg->umax_value)); 1621 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1622 tnum_range(reg->u32_min_value, 1623 reg->u32_max_value)); 1624 1625 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1626 } 1627 1628 static void reg_bounds_sync(struct bpf_reg_state *reg) 1629 { 1630 /* We might have learned new bounds from the var_off. */ 1631 __update_reg_bounds(reg); 1632 /* We might have learned something about the sign bit. */ 1633 __reg_deduce_bounds(reg); 1634 /* We might have learned some bits from the bounds. */ 1635 __reg_bound_offset(reg); 1636 /* Intersecting with the old var_off might have improved our bounds 1637 * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1638 * then new var_off is (0; 0x7f...fc) which improves our umax. 1639 */ 1640 __update_reg_bounds(reg); 1641 } 1642 1643 static bool __reg32_bound_s64(s32 a) 1644 { 1645 return a >= 0 && a <= S32_MAX; 1646 } 1647 1648 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1649 { 1650 reg->umin_value = reg->u32_min_value; 1651 reg->umax_value = reg->u32_max_value; 1652 1653 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 1654 * be positive otherwise set to worse case bounds and refine later 1655 * from tnum. 1656 */ 1657 if (__reg32_bound_s64(reg->s32_min_value) && 1658 __reg32_bound_s64(reg->s32_max_value)) { 1659 reg->smin_value = reg->s32_min_value; 1660 reg->smax_value = reg->s32_max_value; 1661 } else { 1662 reg->smin_value = 0; 1663 reg->smax_value = U32_MAX; 1664 } 1665 } 1666 1667 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1668 { 1669 /* special case when 64-bit register has upper 32-bit register 1670 * zeroed. Typically happens after zext or <<32, >>32 sequence 1671 * allowing us to use 32-bit bounds directly, 1672 */ 1673 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1674 __reg_assign_32_into_64(reg); 1675 } else { 1676 /* Otherwise the best we can do is push lower 32bit known and 1677 * unknown bits into register (var_off set from jmp logic) 1678 * then learn as much as possible from the 64-bit tnum 1679 * known and unknown bits. The previous smin/smax bounds are 1680 * invalid here because of jmp32 compare so mark them unknown 1681 * so they do not impact tnum bounds calculation. 1682 */ 1683 __mark_reg64_unbounded(reg); 1684 } 1685 reg_bounds_sync(reg); 1686 } 1687 1688 static bool __reg64_bound_s32(s64 a) 1689 { 1690 return a >= S32_MIN && a <= S32_MAX; 1691 } 1692 1693 static bool __reg64_bound_u32(u64 a) 1694 { 1695 return a >= U32_MIN && a <= U32_MAX; 1696 } 1697 1698 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1699 { 1700 __mark_reg32_unbounded(reg); 1701 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1702 reg->s32_min_value = (s32)reg->smin_value; 1703 reg->s32_max_value = (s32)reg->smax_value; 1704 } 1705 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1706 reg->u32_min_value = (u32)reg->umin_value; 1707 reg->u32_max_value = (u32)reg->umax_value; 1708 } 1709 reg_bounds_sync(reg); 1710 } 1711 1712 /* Mark a register as having a completely unknown (scalar) value. */ 1713 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1714 struct bpf_reg_state *reg) 1715 { 1716 /* 1717 * Clear type, id, off, and union(map_ptr, range) and 1718 * padding between 'type' and union 1719 */ 1720 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1721 reg->type = SCALAR_VALUE; 1722 reg->var_off = tnum_unknown; 1723 reg->frameno = 0; 1724 reg->precise = !env->bpf_capable; 1725 __mark_reg_unbounded(reg); 1726 } 1727 1728 static void mark_reg_unknown(struct bpf_verifier_env *env, 1729 struct bpf_reg_state *regs, u32 regno) 1730 { 1731 if (WARN_ON(regno >= MAX_BPF_REG)) { 1732 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1733 /* Something bad happened, let's kill all regs except FP */ 1734 for (regno = 0; regno < BPF_REG_FP; regno++) 1735 __mark_reg_not_init(env, regs + regno); 1736 return; 1737 } 1738 __mark_reg_unknown(env, regs + regno); 1739 } 1740 1741 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1742 struct bpf_reg_state *reg) 1743 { 1744 __mark_reg_unknown(env, reg); 1745 reg->type = NOT_INIT; 1746 } 1747 1748 static void mark_reg_not_init(struct bpf_verifier_env *env, 1749 struct bpf_reg_state *regs, u32 regno) 1750 { 1751 if (WARN_ON(regno >= MAX_BPF_REG)) { 1752 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1753 /* Something bad happened, let's kill all regs except FP */ 1754 for (regno = 0; regno < BPF_REG_FP; regno++) 1755 __mark_reg_not_init(env, regs + regno); 1756 return; 1757 } 1758 __mark_reg_not_init(env, regs + regno); 1759 } 1760 1761 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1762 struct bpf_reg_state *regs, u32 regno, 1763 enum bpf_reg_type reg_type, 1764 struct btf *btf, u32 btf_id, 1765 enum bpf_type_flag flag) 1766 { 1767 if (reg_type == SCALAR_VALUE) { 1768 mark_reg_unknown(env, regs, regno); 1769 return; 1770 } 1771 mark_reg_known_zero(env, regs, regno); 1772 regs[regno].type = PTR_TO_BTF_ID | flag; 1773 regs[regno].btf = btf; 1774 regs[regno].btf_id = btf_id; 1775 } 1776 1777 #define DEF_NOT_SUBREG (0) 1778 static void init_reg_state(struct bpf_verifier_env *env, 1779 struct bpf_func_state *state) 1780 { 1781 struct bpf_reg_state *regs = state->regs; 1782 int i; 1783 1784 for (i = 0; i < MAX_BPF_REG; i++) { 1785 mark_reg_not_init(env, regs, i); 1786 regs[i].live = REG_LIVE_NONE; 1787 regs[i].parent = NULL; 1788 regs[i].subreg_def = DEF_NOT_SUBREG; 1789 } 1790 1791 /* frame pointer */ 1792 regs[BPF_REG_FP].type = PTR_TO_STACK; 1793 mark_reg_known_zero(env, regs, BPF_REG_FP); 1794 regs[BPF_REG_FP].frameno = state->frameno; 1795 } 1796 1797 #define BPF_MAIN_FUNC (-1) 1798 static void init_func_state(struct bpf_verifier_env *env, 1799 struct bpf_func_state *state, 1800 int callsite, int frameno, int subprogno) 1801 { 1802 state->callsite = callsite; 1803 state->frameno = frameno; 1804 state->subprogno = subprogno; 1805 state->callback_ret_range = tnum_range(0, 0); 1806 init_reg_state(env, state); 1807 mark_verifier_state_scratched(env); 1808 } 1809 1810 /* Similar to push_stack(), but for async callbacks */ 1811 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 1812 int insn_idx, int prev_insn_idx, 1813 int subprog) 1814 { 1815 struct bpf_verifier_stack_elem *elem; 1816 struct bpf_func_state *frame; 1817 1818 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1819 if (!elem) 1820 goto err; 1821 1822 elem->insn_idx = insn_idx; 1823 elem->prev_insn_idx = prev_insn_idx; 1824 elem->next = env->head; 1825 elem->log_pos = env->log.len_used; 1826 env->head = elem; 1827 env->stack_size++; 1828 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1829 verbose(env, 1830 "The sequence of %d jumps is too complex for async cb.\n", 1831 env->stack_size); 1832 goto err; 1833 } 1834 /* Unlike push_stack() do not copy_verifier_state(). 1835 * The caller state doesn't matter. 1836 * This is async callback. It starts in a fresh stack. 1837 * Initialize it similar to do_check_common(). 1838 */ 1839 elem->st.branches = 1; 1840 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 1841 if (!frame) 1842 goto err; 1843 init_func_state(env, frame, 1844 BPF_MAIN_FUNC /* callsite */, 1845 0 /* frameno within this callchain */, 1846 subprog /* subprog number within this prog */); 1847 elem->st.frame[0] = frame; 1848 return &elem->st; 1849 err: 1850 free_verifier_state(env->cur_state, true); 1851 env->cur_state = NULL; 1852 /* pop all elements and return */ 1853 while (!pop_stack(env, NULL, NULL, false)); 1854 return NULL; 1855 } 1856 1857 1858 enum reg_arg_type { 1859 SRC_OP, /* register is used as source operand */ 1860 DST_OP, /* register is used as destination operand */ 1861 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1862 }; 1863 1864 static int cmp_subprogs(const void *a, const void *b) 1865 { 1866 return ((struct bpf_subprog_info *)a)->start - 1867 ((struct bpf_subprog_info *)b)->start; 1868 } 1869 1870 static int find_subprog(struct bpf_verifier_env *env, int off) 1871 { 1872 struct bpf_subprog_info *p; 1873 1874 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1875 sizeof(env->subprog_info[0]), cmp_subprogs); 1876 if (!p) 1877 return -ENOENT; 1878 return p - env->subprog_info; 1879 1880 } 1881 1882 static int add_subprog(struct bpf_verifier_env *env, int off) 1883 { 1884 int insn_cnt = env->prog->len; 1885 int ret; 1886 1887 if (off >= insn_cnt || off < 0) { 1888 verbose(env, "call to invalid destination\n"); 1889 return -EINVAL; 1890 } 1891 ret = find_subprog(env, off); 1892 if (ret >= 0) 1893 return ret; 1894 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1895 verbose(env, "too many subprograms\n"); 1896 return -E2BIG; 1897 } 1898 /* determine subprog starts. The end is one before the next starts */ 1899 env->subprog_info[env->subprog_cnt++].start = off; 1900 sort(env->subprog_info, env->subprog_cnt, 1901 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1902 return env->subprog_cnt - 1; 1903 } 1904 1905 #define MAX_KFUNC_DESCS 256 1906 #define MAX_KFUNC_BTFS 256 1907 1908 struct bpf_kfunc_desc { 1909 struct btf_func_model func_model; 1910 u32 func_id; 1911 s32 imm; 1912 u16 offset; 1913 }; 1914 1915 struct bpf_kfunc_btf { 1916 struct btf *btf; 1917 struct module *module; 1918 u16 offset; 1919 }; 1920 1921 struct bpf_kfunc_desc_tab { 1922 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 1923 u32 nr_descs; 1924 }; 1925 1926 struct bpf_kfunc_btf_tab { 1927 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 1928 u32 nr_descs; 1929 }; 1930 1931 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 1932 { 1933 const struct bpf_kfunc_desc *d0 = a; 1934 const struct bpf_kfunc_desc *d1 = b; 1935 1936 /* func_id is not greater than BTF_MAX_TYPE */ 1937 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 1938 } 1939 1940 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 1941 { 1942 const struct bpf_kfunc_btf *d0 = a; 1943 const struct bpf_kfunc_btf *d1 = b; 1944 1945 return d0->offset - d1->offset; 1946 } 1947 1948 static const struct bpf_kfunc_desc * 1949 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 1950 { 1951 struct bpf_kfunc_desc desc = { 1952 .func_id = func_id, 1953 .offset = offset, 1954 }; 1955 struct bpf_kfunc_desc_tab *tab; 1956 1957 tab = prog->aux->kfunc_tab; 1958 return bsearch(&desc, tab->descs, tab->nr_descs, 1959 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 1960 } 1961 1962 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 1963 s16 offset) 1964 { 1965 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 1966 struct bpf_kfunc_btf_tab *tab; 1967 struct bpf_kfunc_btf *b; 1968 struct module *mod; 1969 struct btf *btf; 1970 int btf_fd; 1971 1972 tab = env->prog->aux->kfunc_btf_tab; 1973 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 1974 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 1975 if (!b) { 1976 if (tab->nr_descs == MAX_KFUNC_BTFS) { 1977 verbose(env, "too many different module BTFs\n"); 1978 return ERR_PTR(-E2BIG); 1979 } 1980 1981 if (bpfptr_is_null(env->fd_array)) { 1982 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 1983 return ERR_PTR(-EPROTO); 1984 } 1985 1986 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 1987 offset * sizeof(btf_fd), 1988 sizeof(btf_fd))) 1989 return ERR_PTR(-EFAULT); 1990 1991 btf = btf_get_by_fd(btf_fd); 1992 if (IS_ERR(btf)) { 1993 verbose(env, "invalid module BTF fd specified\n"); 1994 return btf; 1995 } 1996 1997 if (!btf_is_module(btf)) { 1998 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 1999 btf_put(btf); 2000 return ERR_PTR(-EINVAL); 2001 } 2002 2003 mod = btf_try_get_module(btf); 2004 if (!mod) { 2005 btf_put(btf); 2006 return ERR_PTR(-ENXIO); 2007 } 2008 2009 b = &tab->descs[tab->nr_descs++]; 2010 b->btf = btf; 2011 b->module = mod; 2012 b->offset = offset; 2013 2014 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2015 kfunc_btf_cmp_by_off, NULL); 2016 } 2017 return b->btf; 2018 } 2019 2020 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 2021 { 2022 if (!tab) 2023 return; 2024 2025 while (tab->nr_descs--) { 2026 module_put(tab->descs[tab->nr_descs].module); 2027 btf_put(tab->descs[tab->nr_descs].btf); 2028 } 2029 kfree(tab); 2030 } 2031 2032 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) 2033 { 2034 if (offset) { 2035 if (offset < 0) { 2036 /* In the future, this can be allowed to increase limit 2037 * of fd index into fd_array, interpreted as u16. 2038 */ 2039 verbose(env, "negative offset disallowed for kernel module function call\n"); 2040 return ERR_PTR(-EINVAL); 2041 } 2042 2043 return __find_kfunc_desc_btf(env, offset); 2044 } 2045 return btf_vmlinux ?: ERR_PTR(-ENOENT); 2046 } 2047 2048 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 2049 { 2050 const struct btf_type *func, *func_proto; 2051 struct bpf_kfunc_btf_tab *btf_tab; 2052 struct bpf_kfunc_desc_tab *tab; 2053 struct bpf_prog_aux *prog_aux; 2054 struct bpf_kfunc_desc *desc; 2055 const char *func_name; 2056 struct btf *desc_btf; 2057 unsigned long call_imm; 2058 unsigned long addr; 2059 int err; 2060 2061 prog_aux = env->prog->aux; 2062 tab = prog_aux->kfunc_tab; 2063 btf_tab = prog_aux->kfunc_btf_tab; 2064 if (!tab) { 2065 if (!btf_vmlinux) { 2066 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 2067 return -ENOTSUPP; 2068 } 2069 2070 if (!env->prog->jit_requested) { 2071 verbose(env, "JIT is required for calling kernel function\n"); 2072 return -ENOTSUPP; 2073 } 2074 2075 if (!bpf_jit_supports_kfunc_call()) { 2076 verbose(env, "JIT does not support calling kernel function\n"); 2077 return -ENOTSUPP; 2078 } 2079 2080 if (!env->prog->gpl_compatible) { 2081 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 2082 return -EINVAL; 2083 } 2084 2085 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 2086 if (!tab) 2087 return -ENOMEM; 2088 prog_aux->kfunc_tab = tab; 2089 } 2090 2091 /* func_id == 0 is always invalid, but instead of returning an error, be 2092 * conservative and wait until the code elimination pass before returning 2093 * error, so that invalid calls that get pruned out can be in BPF programs 2094 * loaded from userspace. It is also required that offset be untouched 2095 * for such calls. 2096 */ 2097 if (!func_id && !offset) 2098 return 0; 2099 2100 if (!btf_tab && offset) { 2101 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 2102 if (!btf_tab) 2103 return -ENOMEM; 2104 prog_aux->kfunc_btf_tab = btf_tab; 2105 } 2106 2107 desc_btf = find_kfunc_desc_btf(env, offset); 2108 if (IS_ERR(desc_btf)) { 2109 verbose(env, "failed to find BTF for kernel function\n"); 2110 return PTR_ERR(desc_btf); 2111 } 2112 2113 if (find_kfunc_desc(env->prog, func_id, offset)) 2114 return 0; 2115 2116 if (tab->nr_descs == MAX_KFUNC_DESCS) { 2117 verbose(env, "too many different kernel function calls\n"); 2118 return -E2BIG; 2119 } 2120 2121 func = btf_type_by_id(desc_btf, func_id); 2122 if (!func || !btf_type_is_func(func)) { 2123 verbose(env, "kernel btf_id %u is not a function\n", 2124 func_id); 2125 return -EINVAL; 2126 } 2127 func_proto = btf_type_by_id(desc_btf, func->type); 2128 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 2129 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 2130 func_id); 2131 return -EINVAL; 2132 } 2133 2134 func_name = btf_name_by_offset(desc_btf, func->name_off); 2135 addr = kallsyms_lookup_name(func_name); 2136 if (!addr) { 2137 verbose(env, "cannot find address for kernel function %s\n", 2138 func_name); 2139 return -EINVAL; 2140 } 2141 2142 call_imm = BPF_CALL_IMM(addr); 2143 /* Check whether or not the relative offset overflows desc->imm */ 2144 if ((unsigned long)(s32)call_imm != call_imm) { 2145 verbose(env, "address of kernel function %s is out of range\n", 2146 func_name); 2147 return -EINVAL; 2148 } 2149 2150 desc = &tab->descs[tab->nr_descs++]; 2151 desc->func_id = func_id; 2152 desc->imm = call_imm; 2153 desc->offset = offset; 2154 err = btf_distill_func_proto(&env->log, desc_btf, 2155 func_proto, func_name, 2156 &desc->func_model); 2157 if (!err) 2158 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2159 kfunc_desc_cmp_by_id_off, NULL); 2160 return err; 2161 } 2162 2163 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 2164 { 2165 const struct bpf_kfunc_desc *d0 = a; 2166 const struct bpf_kfunc_desc *d1 = b; 2167 2168 if (d0->imm > d1->imm) 2169 return 1; 2170 else if (d0->imm < d1->imm) 2171 return -1; 2172 return 0; 2173 } 2174 2175 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 2176 { 2177 struct bpf_kfunc_desc_tab *tab; 2178 2179 tab = prog->aux->kfunc_tab; 2180 if (!tab) 2181 return; 2182 2183 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 2184 kfunc_desc_cmp_by_imm, NULL); 2185 } 2186 2187 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2188 { 2189 return !!prog->aux->kfunc_tab; 2190 } 2191 2192 const struct btf_func_model * 2193 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2194 const struct bpf_insn *insn) 2195 { 2196 const struct bpf_kfunc_desc desc = { 2197 .imm = insn->imm, 2198 }; 2199 const struct bpf_kfunc_desc *res; 2200 struct bpf_kfunc_desc_tab *tab; 2201 2202 tab = prog->aux->kfunc_tab; 2203 res = bsearch(&desc, tab->descs, tab->nr_descs, 2204 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 2205 2206 return res ? &res->func_model : NULL; 2207 } 2208 2209 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 2210 { 2211 struct bpf_subprog_info *subprog = env->subprog_info; 2212 struct bpf_insn *insn = env->prog->insnsi; 2213 int i, ret, insn_cnt = env->prog->len; 2214 2215 /* Add entry function. */ 2216 ret = add_subprog(env, 0); 2217 if (ret) 2218 return ret; 2219 2220 for (i = 0; i < insn_cnt; i++, insn++) { 2221 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2222 !bpf_pseudo_kfunc_call(insn)) 2223 continue; 2224 2225 if (!env->bpf_capable) { 2226 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2227 return -EPERM; 2228 } 2229 2230 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2231 ret = add_subprog(env, i + insn->imm + 1); 2232 else 2233 ret = add_kfunc_call(env, insn->imm, insn->off); 2234 2235 if (ret < 0) 2236 return ret; 2237 } 2238 2239 /* Add a fake 'exit' subprog which could simplify subprog iteration 2240 * logic. 'subprog_cnt' should not be increased. 2241 */ 2242 subprog[env->subprog_cnt].start = insn_cnt; 2243 2244 if (env->log.level & BPF_LOG_LEVEL2) 2245 for (i = 0; i < env->subprog_cnt; i++) 2246 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2247 2248 return 0; 2249 } 2250 2251 static int check_subprogs(struct bpf_verifier_env *env) 2252 { 2253 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2254 struct bpf_subprog_info *subprog = env->subprog_info; 2255 struct bpf_insn *insn = env->prog->insnsi; 2256 int insn_cnt = env->prog->len; 2257 2258 /* now check that all jumps are within the same subprog */ 2259 subprog_start = subprog[cur_subprog].start; 2260 subprog_end = subprog[cur_subprog + 1].start; 2261 for (i = 0; i < insn_cnt; i++) { 2262 u8 code = insn[i].code; 2263 2264 if (code == (BPF_JMP | BPF_CALL) && 2265 insn[i].imm == BPF_FUNC_tail_call && 2266 insn[i].src_reg != BPF_PSEUDO_CALL) 2267 subprog[cur_subprog].has_tail_call = true; 2268 if (BPF_CLASS(code) == BPF_LD && 2269 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2270 subprog[cur_subprog].has_ld_abs = true; 2271 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2272 goto next; 2273 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2274 goto next; 2275 off = i + insn[i].off + 1; 2276 if (off < subprog_start || off >= subprog_end) { 2277 verbose(env, "jump out of range from insn %d to %d\n", i, off); 2278 return -EINVAL; 2279 } 2280 next: 2281 if (i == subprog_end - 1) { 2282 /* to avoid fall-through from one subprog into another 2283 * the last insn of the subprog should be either exit 2284 * or unconditional jump back 2285 */ 2286 if (code != (BPF_JMP | BPF_EXIT) && 2287 code != (BPF_JMP | BPF_JA)) { 2288 verbose(env, "last insn is not an exit or jmp\n"); 2289 return -EINVAL; 2290 } 2291 subprog_start = subprog_end; 2292 cur_subprog++; 2293 if (cur_subprog < env->subprog_cnt) 2294 subprog_end = subprog[cur_subprog + 1].start; 2295 } 2296 } 2297 return 0; 2298 } 2299 2300 /* Parentage chain of this register (or stack slot) should take care of all 2301 * issues like callee-saved registers, stack slot allocation time, etc. 2302 */ 2303 static int mark_reg_read(struct bpf_verifier_env *env, 2304 const struct bpf_reg_state *state, 2305 struct bpf_reg_state *parent, u8 flag) 2306 { 2307 bool writes = parent == state->parent; /* Observe write marks */ 2308 int cnt = 0; 2309 2310 while (parent) { 2311 /* if read wasn't screened by an earlier write ... */ 2312 if (writes && state->live & REG_LIVE_WRITTEN) 2313 break; 2314 if (parent->live & REG_LIVE_DONE) { 2315 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 2316 reg_type_str(env, parent->type), 2317 parent->var_off.value, parent->off); 2318 return -EFAULT; 2319 } 2320 /* The first condition is more likely to be true than the 2321 * second, checked it first. 2322 */ 2323 if ((parent->live & REG_LIVE_READ) == flag || 2324 parent->live & REG_LIVE_READ64) 2325 /* The parentage chain never changes and 2326 * this parent was already marked as LIVE_READ. 2327 * There is no need to keep walking the chain again and 2328 * keep re-marking all parents as LIVE_READ. 2329 * This case happens when the same register is read 2330 * multiple times without writes into it in-between. 2331 * Also, if parent has the stronger REG_LIVE_READ64 set, 2332 * then no need to set the weak REG_LIVE_READ32. 2333 */ 2334 break; 2335 /* ... then we depend on parent's value */ 2336 parent->live |= flag; 2337 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 2338 if (flag == REG_LIVE_READ64) 2339 parent->live &= ~REG_LIVE_READ32; 2340 state = parent; 2341 parent = state->parent; 2342 writes = true; 2343 cnt++; 2344 } 2345 2346 if (env->longest_mark_read_walk < cnt) 2347 env->longest_mark_read_walk = cnt; 2348 return 0; 2349 } 2350 2351 /* This function is supposed to be used by the following 32-bit optimization 2352 * code only. It returns TRUE if the source or destination register operates 2353 * on 64-bit, otherwise return FALSE. 2354 */ 2355 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 2356 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 2357 { 2358 u8 code, class, op; 2359 2360 code = insn->code; 2361 class = BPF_CLASS(code); 2362 op = BPF_OP(code); 2363 if (class == BPF_JMP) { 2364 /* BPF_EXIT for "main" will reach here. Return TRUE 2365 * conservatively. 2366 */ 2367 if (op == BPF_EXIT) 2368 return true; 2369 if (op == BPF_CALL) { 2370 /* BPF to BPF call will reach here because of marking 2371 * caller saved clobber with DST_OP_NO_MARK for which we 2372 * don't care the register def because they are anyway 2373 * marked as NOT_INIT already. 2374 */ 2375 if (insn->src_reg == BPF_PSEUDO_CALL) 2376 return false; 2377 /* Helper call will reach here because of arg type 2378 * check, conservatively return TRUE. 2379 */ 2380 if (t == SRC_OP) 2381 return true; 2382 2383 return false; 2384 } 2385 } 2386 2387 if (class == BPF_ALU64 || class == BPF_JMP || 2388 /* BPF_END always use BPF_ALU class. */ 2389 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 2390 return true; 2391 2392 if (class == BPF_ALU || class == BPF_JMP32) 2393 return false; 2394 2395 if (class == BPF_LDX) { 2396 if (t != SRC_OP) 2397 return BPF_SIZE(code) == BPF_DW; 2398 /* LDX source must be ptr. */ 2399 return true; 2400 } 2401 2402 if (class == BPF_STX) { 2403 /* BPF_STX (including atomic variants) has multiple source 2404 * operands, one of which is a ptr. Check whether the caller is 2405 * asking about it. 2406 */ 2407 if (t == SRC_OP && reg->type != SCALAR_VALUE) 2408 return true; 2409 return BPF_SIZE(code) == BPF_DW; 2410 } 2411 2412 if (class == BPF_LD) { 2413 u8 mode = BPF_MODE(code); 2414 2415 /* LD_IMM64 */ 2416 if (mode == BPF_IMM) 2417 return true; 2418 2419 /* Both LD_IND and LD_ABS return 32-bit data. */ 2420 if (t != SRC_OP) 2421 return false; 2422 2423 /* Implicit ctx ptr. */ 2424 if (regno == BPF_REG_6) 2425 return true; 2426 2427 /* Explicit source could be any width. */ 2428 return true; 2429 } 2430 2431 if (class == BPF_ST) 2432 /* The only source register for BPF_ST is a ptr. */ 2433 return true; 2434 2435 /* Conservatively return true at default. */ 2436 return true; 2437 } 2438 2439 /* Return the regno defined by the insn, or -1. */ 2440 static int insn_def_regno(const struct bpf_insn *insn) 2441 { 2442 switch (BPF_CLASS(insn->code)) { 2443 case BPF_JMP: 2444 case BPF_JMP32: 2445 case BPF_ST: 2446 return -1; 2447 case BPF_STX: 2448 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2449 (insn->imm & BPF_FETCH)) { 2450 if (insn->imm == BPF_CMPXCHG) 2451 return BPF_REG_0; 2452 else 2453 return insn->src_reg; 2454 } else { 2455 return -1; 2456 } 2457 default: 2458 return insn->dst_reg; 2459 } 2460 } 2461 2462 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 2463 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 2464 { 2465 int dst_reg = insn_def_regno(insn); 2466 2467 if (dst_reg == -1) 2468 return false; 2469 2470 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2471 } 2472 2473 static void mark_insn_zext(struct bpf_verifier_env *env, 2474 struct bpf_reg_state *reg) 2475 { 2476 s32 def_idx = reg->subreg_def; 2477 2478 if (def_idx == DEF_NOT_SUBREG) 2479 return; 2480 2481 env->insn_aux_data[def_idx - 1].zext_dst = true; 2482 /* The dst will be zero extended, so won't be sub-register anymore. */ 2483 reg->subreg_def = DEF_NOT_SUBREG; 2484 } 2485 2486 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2487 enum reg_arg_type t) 2488 { 2489 struct bpf_verifier_state *vstate = env->cur_state; 2490 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2491 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2492 struct bpf_reg_state *reg, *regs = state->regs; 2493 bool rw64; 2494 2495 if (regno >= MAX_BPF_REG) { 2496 verbose(env, "R%d is invalid\n", regno); 2497 return -EINVAL; 2498 } 2499 2500 mark_reg_scratched(env, regno); 2501 2502 reg = ®s[regno]; 2503 rw64 = is_reg64(env, insn, regno, reg, t); 2504 if (t == SRC_OP) { 2505 /* check whether register used as source operand can be read */ 2506 if (reg->type == NOT_INIT) { 2507 verbose(env, "R%d !read_ok\n", regno); 2508 return -EACCES; 2509 } 2510 /* We don't need to worry about FP liveness because it's read-only */ 2511 if (regno == BPF_REG_FP) 2512 return 0; 2513 2514 if (rw64) 2515 mark_insn_zext(env, reg); 2516 2517 return mark_reg_read(env, reg, reg->parent, 2518 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2519 } else { 2520 /* check whether register used as dest operand can be written to */ 2521 if (regno == BPF_REG_FP) { 2522 verbose(env, "frame pointer is read only\n"); 2523 return -EACCES; 2524 } 2525 reg->live |= REG_LIVE_WRITTEN; 2526 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2527 if (t == DST_OP) 2528 mark_reg_unknown(env, regs, regno); 2529 } 2530 return 0; 2531 } 2532 2533 /* for any branch, call, exit record the history of jmps in the given state */ 2534 static int push_jmp_history(struct bpf_verifier_env *env, 2535 struct bpf_verifier_state *cur) 2536 { 2537 u32 cnt = cur->jmp_history_cnt; 2538 struct bpf_idx_pair *p; 2539 size_t alloc_size; 2540 2541 cnt++; 2542 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p))); 2543 p = krealloc(cur->jmp_history, alloc_size, GFP_USER); 2544 if (!p) 2545 return -ENOMEM; 2546 p[cnt - 1].idx = env->insn_idx; 2547 p[cnt - 1].prev_idx = env->prev_insn_idx; 2548 cur->jmp_history = p; 2549 cur->jmp_history_cnt = cnt; 2550 return 0; 2551 } 2552 2553 /* Backtrack one insn at a time. If idx is not at the top of recorded 2554 * history then previous instruction came from straight line execution. 2555 */ 2556 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2557 u32 *history) 2558 { 2559 u32 cnt = *history; 2560 2561 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2562 i = st->jmp_history[cnt - 1].prev_idx; 2563 (*history)--; 2564 } else { 2565 i--; 2566 } 2567 return i; 2568 } 2569 2570 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2571 { 2572 const struct btf_type *func; 2573 struct btf *desc_btf; 2574 2575 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2576 return NULL; 2577 2578 desc_btf = find_kfunc_desc_btf(data, insn->off); 2579 if (IS_ERR(desc_btf)) 2580 return "<error>"; 2581 2582 func = btf_type_by_id(desc_btf, insn->imm); 2583 return btf_name_by_offset(desc_btf, func->name_off); 2584 } 2585 2586 /* For given verifier state backtrack_insn() is called from the last insn to 2587 * the first insn. Its purpose is to compute a bitmask of registers and 2588 * stack slots that needs precision in the parent verifier state. 2589 */ 2590 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2591 u32 *reg_mask, u64 *stack_mask) 2592 { 2593 const struct bpf_insn_cbs cbs = { 2594 .cb_call = disasm_kfunc_name, 2595 .cb_print = verbose, 2596 .private_data = env, 2597 }; 2598 struct bpf_insn *insn = env->prog->insnsi + idx; 2599 u8 class = BPF_CLASS(insn->code); 2600 u8 opcode = BPF_OP(insn->code); 2601 u8 mode = BPF_MODE(insn->code); 2602 u32 dreg = 1u << insn->dst_reg; 2603 u32 sreg = 1u << insn->src_reg; 2604 u32 spi; 2605 2606 if (insn->code == 0) 2607 return 0; 2608 if (env->log.level & BPF_LOG_LEVEL2) { 2609 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2610 verbose(env, "%d: ", idx); 2611 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2612 } 2613 2614 if (class == BPF_ALU || class == BPF_ALU64) { 2615 if (!(*reg_mask & dreg)) 2616 return 0; 2617 if (opcode == BPF_MOV) { 2618 if (BPF_SRC(insn->code) == BPF_X) { 2619 /* dreg = sreg 2620 * dreg needs precision after this insn 2621 * sreg needs precision before this insn 2622 */ 2623 *reg_mask &= ~dreg; 2624 *reg_mask |= sreg; 2625 } else { 2626 /* dreg = K 2627 * dreg needs precision after this insn. 2628 * Corresponding register is already marked 2629 * as precise=true in this verifier state. 2630 * No further markings in parent are necessary 2631 */ 2632 *reg_mask &= ~dreg; 2633 } 2634 } else { 2635 if (BPF_SRC(insn->code) == BPF_X) { 2636 /* dreg += sreg 2637 * both dreg and sreg need precision 2638 * before this insn 2639 */ 2640 *reg_mask |= sreg; 2641 } /* else dreg += K 2642 * dreg still needs precision before this insn 2643 */ 2644 } 2645 } else if (class == BPF_LDX) { 2646 if (!(*reg_mask & dreg)) 2647 return 0; 2648 *reg_mask &= ~dreg; 2649 2650 /* scalars can only be spilled into stack w/o losing precision. 2651 * Load from any other memory can be zero extended. 2652 * The desire to keep that precision is already indicated 2653 * by 'precise' mark in corresponding register of this state. 2654 * No further tracking necessary. 2655 */ 2656 if (insn->src_reg != BPF_REG_FP) 2657 return 0; 2658 2659 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2660 * that [fp - off] slot contains scalar that needs to be 2661 * tracked with precision 2662 */ 2663 spi = (-insn->off - 1) / BPF_REG_SIZE; 2664 if (spi >= 64) { 2665 verbose(env, "BUG spi %d\n", spi); 2666 WARN_ONCE(1, "verifier backtracking bug"); 2667 return -EFAULT; 2668 } 2669 *stack_mask |= 1ull << spi; 2670 } else if (class == BPF_STX || class == BPF_ST) { 2671 if (*reg_mask & dreg) 2672 /* stx & st shouldn't be using _scalar_ dst_reg 2673 * to access memory. It means backtracking 2674 * encountered a case of pointer subtraction. 2675 */ 2676 return -ENOTSUPP; 2677 /* scalars can only be spilled into stack */ 2678 if (insn->dst_reg != BPF_REG_FP) 2679 return 0; 2680 spi = (-insn->off - 1) / BPF_REG_SIZE; 2681 if (spi >= 64) { 2682 verbose(env, "BUG spi %d\n", spi); 2683 WARN_ONCE(1, "verifier backtracking bug"); 2684 return -EFAULT; 2685 } 2686 if (!(*stack_mask & (1ull << spi))) 2687 return 0; 2688 *stack_mask &= ~(1ull << spi); 2689 if (class == BPF_STX) 2690 *reg_mask |= sreg; 2691 } else if (class == BPF_JMP || class == BPF_JMP32) { 2692 if (opcode == BPF_CALL) { 2693 if (insn->src_reg == BPF_PSEUDO_CALL) 2694 return -ENOTSUPP; 2695 /* BPF helpers that invoke callback subprogs are 2696 * equivalent to BPF_PSEUDO_CALL above 2697 */ 2698 if (insn->src_reg == 0 && is_callback_calling_function(insn->imm)) 2699 return -ENOTSUPP; 2700 /* regular helper call sets R0 */ 2701 *reg_mask &= ~1; 2702 if (*reg_mask & 0x3f) { 2703 /* if backtracing was looking for registers R1-R5 2704 * they should have been found already. 2705 */ 2706 verbose(env, "BUG regs %x\n", *reg_mask); 2707 WARN_ONCE(1, "verifier backtracking bug"); 2708 return -EFAULT; 2709 } 2710 } else if (opcode == BPF_EXIT) { 2711 return -ENOTSUPP; 2712 } 2713 } else if (class == BPF_LD) { 2714 if (!(*reg_mask & dreg)) 2715 return 0; 2716 *reg_mask &= ~dreg; 2717 /* It's ld_imm64 or ld_abs or ld_ind. 2718 * For ld_imm64 no further tracking of precision 2719 * into parent is necessary 2720 */ 2721 if (mode == BPF_IND || mode == BPF_ABS) 2722 /* to be analyzed */ 2723 return -ENOTSUPP; 2724 } 2725 return 0; 2726 } 2727 2728 /* the scalar precision tracking algorithm: 2729 * . at the start all registers have precise=false. 2730 * . scalar ranges are tracked as normal through alu and jmp insns. 2731 * . once precise value of the scalar register is used in: 2732 * . ptr + scalar alu 2733 * . if (scalar cond K|scalar) 2734 * . helper_call(.., scalar, ...) where ARG_CONST is expected 2735 * backtrack through the verifier states and mark all registers and 2736 * stack slots with spilled constants that these scalar regisers 2737 * should be precise. 2738 * . during state pruning two registers (or spilled stack slots) 2739 * are equivalent if both are not precise. 2740 * 2741 * Note the verifier cannot simply walk register parentage chain, 2742 * since many different registers and stack slots could have been 2743 * used to compute single precise scalar. 2744 * 2745 * The approach of starting with precise=true for all registers and then 2746 * backtrack to mark a register as not precise when the verifier detects 2747 * that program doesn't care about specific value (e.g., when helper 2748 * takes register as ARG_ANYTHING parameter) is not safe. 2749 * 2750 * It's ok to walk single parentage chain of the verifier states. 2751 * It's possible that this backtracking will go all the way till 1st insn. 2752 * All other branches will be explored for needing precision later. 2753 * 2754 * The backtracking needs to deal with cases like: 2755 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 2756 * r9 -= r8 2757 * r5 = r9 2758 * if r5 > 0x79f goto pc+7 2759 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 2760 * r5 += 1 2761 * ... 2762 * call bpf_perf_event_output#25 2763 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 2764 * 2765 * and this case: 2766 * r6 = 1 2767 * call foo // uses callee's r6 inside to compute r0 2768 * r0 += r6 2769 * if r0 == 0 goto 2770 * 2771 * to track above reg_mask/stack_mask needs to be independent for each frame. 2772 * 2773 * Also if parent's curframe > frame where backtracking started, 2774 * the verifier need to mark registers in both frames, otherwise callees 2775 * may incorrectly prune callers. This is similar to 2776 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 2777 * 2778 * For now backtracking falls back into conservative marking. 2779 */ 2780 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 2781 struct bpf_verifier_state *st) 2782 { 2783 struct bpf_func_state *func; 2784 struct bpf_reg_state *reg; 2785 int i, j; 2786 2787 /* big hammer: mark all scalars precise in this path. 2788 * pop_stack may still get !precise scalars. 2789 * We also skip current state and go straight to first parent state, 2790 * because precision markings in current non-checkpointed state are 2791 * not needed. See why in the comment in __mark_chain_precision below. 2792 */ 2793 for (st = st->parent; st; st = st->parent) { 2794 for (i = 0; i <= st->curframe; i++) { 2795 func = st->frame[i]; 2796 for (j = 0; j < BPF_REG_FP; j++) { 2797 reg = &func->regs[j]; 2798 if (reg->type != SCALAR_VALUE) 2799 continue; 2800 reg->precise = true; 2801 } 2802 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2803 if (!is_spilled_reg(&func->stack[j])) 2804 continue; 2805 reg = &func->stack[j].spilled_ptr; 2806 if (reg->type != SCALAR_VALUE) 2807 continue; 2808 reg->precise = true; 2809 } 2810 } 2811 } 2812 } 2813 2814 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 2815 { 2816 struct bpf_func_state *func; 2817 struct bpf_reg_state *reg; 2818 int i, j; 2819 2820 for (i = 0; i <= st->curframe; i++) { 2821 func = st->frame[i]; 2822 for (j = 0; j < BPF_REG_FP; j++) { 2823 reg = &func->regs[j]; 2824 if (reg->type != SCALAR_VALUE) 2825 continue; 2826 reg->precise = false; 2827 } 2828 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2829 if (!is_spilled_reg(&func->stack[j])) 2830 continue; 2831 reg = &func->stack[j].spilled_ptr; 2832 if (reg->type != SCALAR_VALUE) 2833 continue; 2834 reg->precise = false; 2835 } 2836 } 2837 } 2838 2839 /* 2840 * __mark_chain_precision() backtracks BPF program instruction sequence and 2841 * chain of verifier states making sure that register *regno* (if regno >= 0) 2842 * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked 2843 * SCALARS, as well as any other registers and slots that contribute to 2844 * a tracked state of given registers/stack slots, depending on specific BPF 2845 * assembly instructions (see backtrack_insns() for exact instruction handling 2846 * logic). This backtracking relies on recorded jmp_history and is able to 2847 * traverse entire chain of parent states. This process ends only when all the 2848 * necessary registers/slots and their transitive dependencies are marked as 2849 * precise. 2850 * 2851 * One important and subtle aspect is that precise marks *do not matter* in 2852 * the currently verified state (current state). It is important to understand 2853 * why this is the case. 2854 * 2855 * First, note that current state is the state that is not yet "checkpointed", 2856 * i.e., it is not yet put into env->explored_states, and it has no children 2857 * states as well. It's ephemeral, and can end up either a) being discarded if 2858 * compatible explored state is found at some point or BPF_EXIT instruction is 2859 * reached or b) checkpointed and put into env->explored_states, branching out 2860 * into one or more children states. 2861 * 2862 * In the former case, precise markings in current state are completely 2863 * ignored by state comparison code (see regsafe() for details). Only 2864 * checkpointed ("old") state precise markings are important, and if old 2865 * state's register/slot is precise, regsafe() assumes current state's 2866 * register/slot as precise and checks value ranges exactly and precisely. If 2867 * states turn out to be compatible, current state's necessary precise 2868 * markings and any required parent states' precise markings are enforced 2869 * after the fact with propagate_precision() logic, after the fact. But it's 2870 * important to realize that in this case, even after marking current state 2871 * registers/slots as precise, we immediately discard current state. So what 2872 * actually matters is any of the precise markings propagated into current 2873 * state's parent states, which are always checkpointed (due to b) case above). 2874 * As such, for scenario a) it doesn't matter if current state has precise 2875 * markings set or not. 2876 * 2877 * Now, for the scenario b), checkpointing and forking into child(ren) 2878 * state(s). Note that before current state gets to checkpointing step, any 2879 * processed instruction always assumes precise SCALAR register/slot 2880 * knowledge: if precise value or range is useful to prune jump branch, BPF 2881 * verifier takes this opportunity enthusiastically. Similarly, when 2882 * register's value is used to calculate offset or memory address, exact 2883 * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to 2884 * what we mentioned above about state comparison ignoring precise markings 2885 * during state comparison, BPF verifier ignores and also assumes precise 2886 * markings *at will* during instruction verification process. But as verifier 2887 * assumes precision, it also propagates any precision dependencies across 2888 * parent states, which are not yet finalized, so can be further restricted 2889 * based on new knowledge gained from restrictions enforced by their children 2890 * states. This is so that once those parent states are finalized, i.e., when 2891 * they have no more active children state, state comparison logic in 2892 * is_state_visited() would enforce strict and precise SCALAR ranges, if 2893 * required for correctness. 2894 * 2895 * To build a bit more intuition, note also that once a state is checkpointed, 2896 * the path we took to get to that state is not important. This is crucial 2897 * property for state pruning. When state is checkpointed and finalized at 2898 * some instruction index, it can be correctly and safely used to "short 2899 * circuit" any *compatible* state that reaches exactly the same instruction 2900 * index. I.e., if we jumped to that instruction from a completely different 2901 * code path than original finalized state was derived from, it doesn't 2902 * matter, current state can be discarded because from that instruction 2903 * forward having a compatible state will ensure we will safely reach the 2904 * exit. States describe preconditions for further exploration, but completely 2905 * forget the history of how we got here. 2906 * 2907 * This also means that even if we needed precise SCALAR range to get to 2908 * finalized state, but from that point forward *that same* SCALAR register is 2909 * never used in a precise context (i.e., it's precise value is not needed for 2910 * correctness), it's correct and safe to mark such register as "imprecise" 2911 * (i.e., precise marking set to false). This is what we rely on when we do 2912 * not set precise marking in current state. If no child state requires 2913 * precision for any given SCALAR register, it's safe to dictate that it can 2914 * be imprecise. If any child state does require this register to be precise, 2915 * we'll mark it precise later retroactively during precise markings 2916 * propagation from child state to parent states. 2917 * 2918 * Skipping precise marking setting in current state is a mild version of 2919 * relying on the above observation. But we can utilize this property even 2920 * more aggressively by proactively forgetting any precise marking in the 2921 * current state (which we inherited from the parent state), right before we 2922 * checkpoint it and branch off into new child state. This is done by 2923 * mark_all_scalars_imprecise() to hopefully get more permissive and generic 2924 * finalized states which help in short circuiting more future states. 2925 */ 2926 static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno, 2927 int spi) 2928 { 2929 struct bpf_verifier_state *st = env->cur_state; 2930 int first_idx = st->first_insn_idx; 2931 int last_idx = env->insn_idx; 2932 struct bpf_func_state *func; 2933 struct bpf_reg_state *reg; 2934 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2935 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2936 bool skip_first = true; 2937 bool new_marks = false; 2938 int i, err; 2939 2940 if (!env->bpf_capable) 2941 return 0; 2942 2943 /* Do sanity checks against current state of register and/or stack 2944 * slot, but don't set precise flag in current state, as precision 2945 * tracking in the current state is unnecessary. 2946 */ 2947 func = st->frame[frame]; 2948 if (regno >= 0) { 2949 reg = &func->regs[regno]; 2950 if (reg->type != SCALAR_VALUE) { 2951 WARN_ONCE(1, "backtracing misuse"); 2952 return -EFAULT; 2953 } 2954 new_marks = true; 2955 } 2956 2957 while (spi >= 0) { 2958 if (!is_spilled_reg(&func->stack[spi])) { 2959 stack_mask = 0; 2960 break; 2961 } 2962 reg = &func->stack[spi].spilled_ptr; 2963 if (reg->type != SCALAR_VALUE) { 2964 stack_mask = 0; 2965 break; 2966 } 2967 new_marks = true; 2968 break; 2969 } 2970 2971 if (!new_marks) 2972 return 0; 2973 if (!reg_mask && !stack_mask) 2974 return 0; 2975 2976 for (;;) { 2977 DECLARE_BITMAP(mask, 64); 2978 u32 history = st->jmp_history_cnt; 2979 2980 if (env->log.level & BPF_LOG_LEVEL2) 2981 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2982 2983 if (last_idx < 0) { 2984 /* we are at the entry into subprog, which 2985 * is expected for global funcs, but only if 2986 * requested precise registers are R1-R5 2987 * (which are global func's input arguments) 2988 */ 2989 if (st->curframe == 0 && 2990 st->frame[0]->subprogno > 0 && 2991 st->frame[0]->callsite == BPF_MAIN_FUNC && 2992 stack_mask == 0 && (reg_mask & ~0x3e) == 0) { 2993 bitmap_from_u64(mask, reg_mask); 2994 for_each_set_bit(i, mask, 32) { 2995 reg = &st->frame[0]->regs[i]; 2996 if (reg->type != SCALAR_VALUE) { 2997 reg_mask &= ~(1u << i); 2998 continue; 2999 } 3000 reg->precise = true; 3001 } 3002 return 0; 3003 } 3004 3005 verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n", 3006 st->frame[0]->subprogno, reg_mask, stack_mask); 3007 WARN_ONCE(1, "verifier backtracking bug"); 3008 return -EFAULT; 3009 } 3010 3011 for (i = last_idx;;) { 3012 if (skip_first) { 3013 err = 0; 3014 skip_first = false; 3015 } else { 3016 err = backtrack_insn(env, i, ®_mask, &stack_mask); 3017 } 3018 if (err == -ENOTSUPP) { 3019 mark_all_scalars_precise(env, st); 3020 return 0; 3021 } else if (err) { 3022 return err; 3023 } 3024 if (!reg_mask && !stack_mask) 3025 /* Found assignment(s) into tracked register in this state. 3026 * Since this state is already marked, just return. 3027 * Nothing to be tracked further in the parent state. 3028 */ 3029 return 0; 3030 if (i == first_idx) 3031 break; 3032 i = get_prev_insn_idx(st, i, &history); 3033 if (i >= env->prog->len) { 3034 /* This can happen if backtracking reached insn 0 3035 * and there are still reg_mask or stack_mask 3036 * to backtrack. 3037 * It means the backtracking missed the spot where 3038 * particular register was initialized with a constant. 3039 */ 3040 verbose(env, "BUG backtracking idx %d\n", i); 3041 WARN_ONCE(1, "verifier backtracking bug"); 3042 return -EFAULT; 3043 } 3044 } 3045 st = st->parent; 3046 if (!st) 3047 break; 3048 3049 new_marks = false; 3050 func = st->frame[frame]; 3051 bitmap_from_u64(mask, reg_mask); 3052 for_each_set_bit(i, mask, 32) { 3053 reg = &func->regs[i]; 3054 if (reg->type != SCALAR_VALUE) { 3055 reg_mask &= ~(1u << i); 3056 continue; 3057 } 3058 if (!reg->precise) 3059 new_marks = true; 3060 reg->precise = true; 3061 } 3062 3063 bitmap_from_u64(mask, stack_mask); 3064 for_each_set_bit(i, mask, 64) { 3065 if (i >= func->allocated_stack / BPF_REG_SIZE) { 3066 /* the sequence of instructions: 3067 * 2: (bf) r3 = r10 3068 * 3: (7b) *(u64 *)(r3 -8) = r0 3069 * 4: (79) r4 = *(u64 *)(r10 -8) 3070 * doesn't contain jmps. It's backtracked 3071 * as a single block. 3072 * During backtracking insn 3 is not recognized as 3073 * stack access, so at the end of backtracking 3074 * stack slot fp-8 is still marked in stack_mask. 3075 * However the parent state may not have accessed 3076 * fp-8 and it's "unallocated" stack space. 3077 * In such case fallback to conservative. 3078 */ 3079 mark_all_scalars_precise(env, st); 3080 return 0; 3081 } 3082 3083 if (!is_spilled_reg(&func->stack[i])) { 3084 stack_mask &= ~(1ull << i); 3085 continue; 3086 } 3087 reg = &func->stack[i].spilled_ptr; 3088 if (reg->type != SCALAR_VALUE) { 3089 stack_mask &= ~(1ull << i); 3090 continue; 3091 } 3092 if (!reg->precise) 3093 new_marks = true; 3094 reg->precise = true; 3095 } 3096 if (env->log.level & BPF_LOG_LEVEL2) { 3097 verbose(env, "parent %s regs=%x stack=%llx marks:", 3098 new_marks ? "didn't have" : "already had", 3099 reg_mask, stack_mask); 3100 print_verifier_state(env, func, true); 3101 } 3102 3103 if (!reg_mask && !stack_mask) 3104 break; 3105 if (!new_marks) 3106 break; 3107 3108 last_idx = st->last_insn_idx; 3109 first_idx = st->first_insn_idx; 3110 } 3111 return 0; 3112 } 3113 3114 int mark_chain_precision(struct bpf_verifier_env *env, int regno) 3115 { 3116 return __mark_chain_precision(env, env->cur_state->curframe, regno, -1); 3117 } 3118 3119 static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno) 3120 { 3121 return __mark_chain_precision(env, frame, regno, -1); 3122 } 3123 3124 static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi) 3125 { 3126 return __mark_chain_precision(env, frame, -1, spi); 3127 } 3128 3129 static bool is_spillable_regtype(enum bpf_reg_type type) 3130 { 3131 switch (base_type(type)) { 3132 case PTR_TO_MAP_VALUE: 3133 case PTR_TO_STACK: 3134 case PTR_TO_CTX: 3135 case PTR_TO_PACKET: 3136 case PTR_TO_PACKET_META: 3137 case PTR_TO_PACKET_END: 3138 case PTR_TO_FLOW_KEYS: 3139 case CONST_PTR_TO_MAP: 3140 case PTR_TO_SOCKET: 3141 case PTR_TO_SOCK_COMMON: 3142 case PTR_TO_TCP_SOCK: 3143 case PTR_TO_XDP_SOCK: 3144 case PTR_TO_BTF_ID: 3145 case PTR_TO_BUF: 3146 case PTR_TO_MEM: 3147 case PTR_TO_FUNC: 3148 case PTR_TO_MAP_KEY: 3149 return true; 3150 default: 3151 return false; 3152 } 3153 } 3154 3155 /* Does this register contain a constant zero? */ 3156 static bool register_is_null(struct bpf_reg_state *reg) 3157 { 3158 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 3159 } 3160 3161 static bool register_is_const(struct bpf_reg_state *reg) 3162 { 3163 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 3164 } 3165 3166 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 3167 { 3168 return tnum_is_unknown(reg->var_off) && 3169 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 3170 reg->umin_value == 0 && reg->umax_value == U64_MAX && 3171 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 3172 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 3173 } 3174 3175 static bool register_is_bounded(struct bpf_reg_state *reg) 3176 { 3177 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 3178 } 3179 3180 static bool __is_pointer_value(bool allow_ptr_leaks, 3181 const struct bpf_reg_state *reg) 3182 { 3183 if (allow_ptr_leaks) 3184 return false; 3185 3186 return reg->type != SCALAR_VALUE; 3187 } 3188 3189 static void save_register_state(struct bpf_func_state *state, 3190 int spi, struct bpf_reg_state *reg, 3191 int size) 3192 { 3193 int i; 3194 3195 state->stack[spi].spilled_ptr = *reg; 3196 if (size == BPF_REG_SIZE) 3197 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3198 3199 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 3200 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 3201 3202 /* size < 8 bytes spill */ 3203 for (; i; i--) 3204 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); 3205 } 3206 3207 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 3208 * stack boundary and alignment are checked in check_mem_access() 3209 */ 3210 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 3211 /* stack frame we're writing to */ 3212 struct bpf_func_state *state, 3213 int off, int size, int value_regno, 3214 int insn_idx) 3215 { 3216 struct bpf_func_state *cur; /* state of the current function */ 3217 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 3218 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 3219 struct bpf_reg_state *reg = NULL; 3220 3221 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 3222 if (err) 3223 return err; 3224 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 3225 * so it's aligned access and [off, off + size) are within stack limits 3226 */ 3227 if (!env->allow_ptr_leaks && 3228 state->stack[spi].slot_type[0] == STACK_SPILL && 3229 size != BPF_REG_SIZE) { 3230 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 3231 return -EACCES; 3232 } 3233 3234 cur = env->cur_state->frame[env->cur_state->curframe]; 3235 if (value_regno >= 0) 3236 reg = &cur->regs[value_regno]; 3237 if (!env->bypass_spec_v4) { 3238 bool sanitize = reg && is_spillable_regtype(reg->type); 3239 3240 for (i = 0; i < size; i++) { 3241 if (state->stack[spi].slot_type[i] == STACK_INVALID) { 3242 sanitize = true; 3243 break; 3244 } 3245 } 3246 3247 if (sanitize) 3248 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 3249 } 3250 3251 mark_stack_slot_scratched(env, spi); 3252 if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && 3253 !register_is_null(reg) && env->bpf_capable) { 3254 if (dst_reg != BPF_REG_FP) { 3255 /* The backtracking logic can only recognize explicit 3256 * stack slot address like [fp - 8]. Other spill of 3257 * scalar via different register has to be conservative. 3258 * Backtrack from here and mark all registers as precise 3259 * that contributed into 'reg' being a constant. 3260 */ 3261 err = mark_chain_precision(env, value_regno); 3262 if (err) 3263 return err; 3264 } 3265 save_register_state(state, spi, reg, size); 3266 } else if (reg && is_spillable_regtype(reg->type)) { 3267 /* register containing pointer is being spilled into stack */ 3268 if (size != BPF_REG_SIZE) { 3269 verbose_linfo(env, insn_idx, "; "); 3270 verbose(env, "invalid size of register spill\n"); 3271 return -EACCES; 3272 } 3273 if (state != cur && reg->type == PTR_TO_STACK) { 3274 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 3275 return -EINVAL; 3276 } 3277 save_register_state(state, spi, reg, size); 3278 } else { 3279 u8 type = STACK_MISC; 3280 3281 /* regular write of data into stack destroys any spilled ptr */ 3282 state->stack[spi].spilled_ptr.type = NOT_INIT; 3283 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 3284 if (is_spilled_reg(&state->stack[spi])) 3285 for (i = 0; i < BPF_REG_SIZE; i++) 3286 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 3287 3288 /* only mark the slot as written if all 8 bytes were written 3289 * otherwise read propagation may incorrectly stop too soon 3290 * when stack slots are partially written. 3291 * This heuristic means that read propagation will be 3292 * conservative, since it will add reg_live_read marks 3293 * to stack slots all the way to first state when programs 3294 * writes+reads less than 8 bytes 3295 */ 3296 if (size == BPF_REG_SIZE) 3297 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 3298 3299 /* when we zero initialize stack slots mark them as such */ 3300 if (reg && register_is_null(reg)) { 3301 /* backtracking doesn't work for STACK_ZERO yet. */ 3302 err = mark_chain_precision(env, value_regno); 3303 if (err) 3304 return err; 3305 type = STACK_ZERO; 3306 } 3307 3308 /* Mark slots affected by this stack write. */ 3309 for (i = 0; i < size; i++) 3310 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 3311 type; 3312 } 3313 return 0; 3314 } 3315 3316 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 3317 * known to contain a variable offset. 3318 * This function checks whether the write is permitted and conservatively 3319 * tracks the effects of the write, considering that each stack slot in the 3320 * dynamic range is potentially written to. 3321 * 3322 * 'off' includes 'regno->off'. 3323 * 'value_regno' can be -1, meaning that an unknown value is being written to 3324 * the stack. 3325 * 3326 * Spilled pointers in range are not marked as written because we don't know 3327 * what's going to be actually written. This means that read propagation for 3328 * future reads cannot be terminated by this write. 3329 * 3330 * For privileged programs, uninitialized stack slots are considered 3331 * initialized by this write (even though we don't know exactly what offsets 3332 * are going to be written to). The idea is that we don't want the verifier to 3333 * reject future reads that access slots written to through variable offsets. 3334 */ 3335 static int check_stack_write_var_off(struct bpf_verifier_env *env, 3336 /* func where register points to */ 3337 struct bpf_func_state *state, 3338 int ptr_regno, int off, int size, 3339 int value_regno, int insn_idx) 3340 { 3341 struct bpf_func_state *cur; /* state of the current function */ 3342 int min_off, max_off; 3343 int i, err; 3344 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 3345 bool writing_zero = false; 3346 /* set if the fact that we're writing a zero is used to let any 3347 * stack slots remain STACK_ZERO 3348 */ 3349 bool zero_used = false; 3350 3351 cur = env->cur_state->frame[env->cur_state->curframe]; 3352 ptr_reg = &cur->regs[ptr_regno]; 3353 min_off = ptr_reg->smin_value + off; 3354 max_off = ptr_reg->smax_value + off + size; 3355 if (value_regno >= 0) 3356 value_reg = &cur->regs[value_regno]; 3357 if (value_reg && register_is_null(value_reg)) 3358 writing_zero = true; 3359 3360 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 3361 if (err) 3362 return err; 3363 3364 3365 /* Variable offset writes destroy any spilled pointers in range. */ 3366 for (i = min_off; i < max_off; i++) { 3367 u8 new_type, *stype; 3368 int slot, spi; 3369 3370 slot = -i - 1; 3371 spi = slot / BPF_REG_SIZE; 3372 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3373 mark_stack_slot_scratched(env, spi); 3374 3375 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { 3376 /* Reject the write if range we may write to has not 3377 * been initialized beforehand. If we didn't reject 3378 * here, the ptr status would be erased below (even 3379 * though not all slots are actually overwritten), 3380 * possibly opening the door to leaks. 3381 * 3382 * We do however catch STACK_INVALID case below, and 3383 * only allow reading possibly uninitialized memory 3384 * later for CAP_PERFMON, as the write may not happen to 3385 * that slot. 3386 */ 3387 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 3388 insn_idx, i); 3389 return -EINVAL; 3390 } 3391 3392 /* Erase all spilled pointers. */ 3393 state->stack[spi].spilled_ptr.type = NOT_INIT; 3394 3395 /* Update the slot type. */ 3396 new_type = STACK_MISC; 3397 if (writing_zero && *stype == STACK_ZERO) { 3398 new_type = STACK_ZERO; 3399 zero_used = true; 3400 } 3401 /* If the slot is STACK_INVALID, we check whether it's OK to 3402 * pretend that it will be initialized by this write. The slot 3403 * might not actually be written to, and so if we mark it as 3404 * initialized future reads might leak uninitialized memory. 3405 * For privileged programs, we will accept such reads to slots 3406 * that may or may not be written because, if we're reject 3407 * them, the error would be too confusing. 3408 */ 3409 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 3410 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 3411 insn_idx, i); 3412 return -EINVAL; 3413 } 3414 *stype = new_type; 3415 } 3416 if (zero_used) { 3417 /* backtracking doesn't work for STACK_ZERO yet. */ 3418 err = mark_chain_precision(env, value_regno); 3419 if (err) 3420 return err; 3421 } 3422 return 0; 3423 } 3424 3425 /* When register 'dst_regno' is assigned some values from stack[min_off, 3426 * max_off), we set the register's type according to the types of the 3427 * respective stack slots. If all the stack values are known to be zeros, then 3428 * so is the destination reg. Otherwise, the register is considered to be 3429 * SCALAR. This function does not deal with register filling; the caller must 3430 * ensure that all spilled registers in the stack range have been marked as 3431 * read. 3432 */ 3433 static void mark_reg_stack_read(struct bpf_verifier_env *env, 3434 /* func where src register points to */ 3435 struct bpf_func_state *ptr_state, 3436 int min_off, int max_off, int dst_regno) 3437 { 3438 struct bpf_verifier_state *vstate = env->cur_state; 3439 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3440 int i, slot, spi; 3441 u8 *stype; 3442 int zeros = 0; 3443 3444 for (i = min_off; i < max_off; i++) { 3445 slot = -i - 1; 3446 spi = slot / BPF_REG_SIZE; 3447 stype = ptr_state->stack[spi].slot_type; 3448 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 3449 break; 3450 zeros++; 3451 } 3452 if (zeros == max_off - min_off) { 3453 /* any access_size read into register is zero extended, 3454 * so the whole register == const_zero 3455 */ 3456 __mark_reg_const_zero(&state->regs[dst_regno]); 3457 /* backtracking doesn't support STACK_ZERO yet, 3458 * so mark it precise here, so that later 3459 * backtracking can stop here. 3460 * Backtracking may not need this if this register 3461 * doesn't participate in pointer adjustment. 3462 * Forward propagation of precise flag is not 3463 * necessary either. This mark is only to stop 3464 * backtracking. Any register that contributed 3465 * to const 0 was marked precise before spill. 3466 */ 3467 state->regs[dst_regno].precise = true; 3468 } else { 3469 /* have read misc data from the stack */ 3470 mark_reg_unknown(env, state->regs, dst_regno); 3471 } 3472 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3473 } 3474 3475 /* Read the stack at 'off' and put the results into the register indicated by 3476 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 3477 * spilled reg. 3478 * 3479 * 'dst_regno' can be -1, meaning that the read value is not going to a 3480 * register. 3481 * 3482 * The access is assumed to be within the current stack bounds. 3483 */ 3484 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 3485 /* func where src register points to */ 3486 struct bpf_func_state *reg_state, 3487 int off, int size, int dst_regno) 3488 { 3489 struct bpf_verifier_state *vstate = env->cur_state; 3490 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3491 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 3492 struct bpf_reg_state *reg; 3493 u8 *stype, type; 3494 3495 stype = reg_state->stack[spi].slot_type; 3496 reg = ®_state->stack[spi].spilled_ptr; 3497 3498 if (is_spilled_reg(®_state->stack[spi])) { 3499 u8 spill_size = 1; 3500 3501 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 3502 spill_size++; 3503 3504 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 3505 if (reg->type != SCALAR_VALUE) { 3506 verbose_linfo(env, env->insn_idx, "; "); 3507 verbose(env, "invalid size of register fill\n"); 3508 return -EACCES; 3509 } 3510 3511 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3512 if (dst_regno < 0) 3513 return 0; 3514 3515 if (!(off % BPF_REG_SIZE) && size == spill_size) { 3516 /* The earlier check_reg_arg() has decided the 3517 * subreg_def for this insn. Save it first. 3518 */ 3519 s32 subreg_def = state->regs[dst_regno].subreg_def; 3520 3521 state->regs[dst_regno] = *reg; 3522 state->regs[dst_regno].subreg_def = subreg_def; 3523 } else { 3524 for (i = 0; i < size; i++) { 3525 type = stype[(slot - i) % BPF_REG_SIZE]; 3526 if (type == STACK_SPILL) 3527 continue; 3528 if (type == STACK_MISC) 3529 continue; 3530 verbose(env, "invalid read from stack off %d+%d size %d\n", 3531 off, i, size); 3532 return -EACCES; 3533 } 3534 mark_reg_unknown(env, state->regs, dst_regno); 3535 } 3536 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3537 return 0; 3538 } 3539 3540 if (dst_regno >= 0) { 3541 /* restore register state from stack */ 3542 state->regs[dst_regno] = *reg; 3543 /* mark reg as written since spilled pointer state likely 3544 * has its liveness marks cleared by is_state_visited() 3545 * which resets stack/reg liveness for state transitions 3546 */ 3547 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3548 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 3549 /* If dst_regno==-1, the caller is asking us whether 3550 * it is acceptable to use this value as a SCALAR_VALUE 3551 * (e.g. for XADD). 3552 * We must not allow unprivileged callers to do that 3553 * with spilled pointers. 3554 */ 3555 verbose(env, "leaking pointer from stack off %d\n", 3556 off); 3557 return -EACCES; 3558 } 3559 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3560 } else { 3561 for (i = 0; i < size; i++) { 3562 type = stype[(slot - i) % BPF_REG_SIZE]; 3563 if (type == STACK_MISC) 3564 continue; 3565 if (type == STACK_ZERO) 3566 continue; 3567 verbose(env, "invalid read from stack off %d+%d size %d\n", 3568 off, i, size); 3569 return -EACCES; 3570 } 3571 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3572 if (dst_regno >= 0) 3573 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 3574 } 3575 return 0; 3576 } 3577 3578 enum bpf_access_src { 3579 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 3580 ACCESS_HELPER = 2, /* the access is performed by a helper */ 3581 }; 3582 3583 static int check_stack_range_initialized(struct bpf_verifier_env *env, 3584 int regno, int off, int access_size, 3585 bool zero_size_allowed, 3586 enum bpf_access_src type, 3587 struct bpf_call_arg_meta *meta); 3588 3589 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 3590 { 3591 return cur_regs(env) + regno; 3592 } 3593 3594 /* Read the stack at 'ptr_regno + off' and put the result into the register 3595 * 'dst_regno'. 3596 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 3597 * but not its variable offset. 3598 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 3599 * 3600 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 3601 * filling registers (i.e. reads of spilled register cannot be detected when 3602 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 3603 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 3604 * offset; for a fixed offset check_stack_read_fixed_off should be used 3605 * instead. 3606 */ 3607 static int check_stack_read_var_off(struct bpf_verifier_env *env, 3608 int ptr_regno, int off, int size, int dst_regno) 3609 { 3610 /* The state of the source register. */ 3611 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3612 struct bpf_func_state *ptr_state = func(env, reg); 3613 int err; 3614 int min_off, max_off; 3615 3616 /* Note that we pass a NULL meta, so raw access will not be permitted. 3617 */ 3618 err = check_stack_range_initialized(env, ptr_regno, off, size, 3619 false, ACCESS_DIRECT, NULL); 3620 if (err) 3621 return err; 3622 3623 min_off = reg->smin_value + off; 3624 max_off = reg->smax_value + off; 3625 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3626 return 0; 3627 } 3628 3629 /* check_stack_read dispatches to check_stack_read_fixed_off or 3630 * check_stack_read_var_off. 3631 * 3632 * The caller must ensure that the offset falls within the allocated stack 3633 * bounds. 3634 * 3635 * 'dst_regno' is a register which will receive the value from the stack. It 3636 * can be -1, meaning that the read value is not going to a register. 3637 */ 3638 static int check_stack_read(struct bpf_verifier_env *env, 3639 int ptr_regno, int off, int size, 3640 int dst_regno) 3641 { 3642 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3643 struct bpf_func_state *state = func(env, reg); 3644 int err; 3645 /* Some accesses are only permitted with a static offset. */ 3646 bool var_off = !tnum_is_const(reg->var_off); 3647 3648 /* The offset is required to be static when reads don't go to a 3649 * register, in order to not leak pointers (see 3650 * check_stack_read_fixed_off). 3651 */ 3652 if (dst_regno < 0 && var_off) { 3653 char tn_buf[48]; 3654 3655 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3656 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3657 tn_buf, off, size); 3658 return -EACCES; 3659 } 3660 /* Variable offset is prohibited for unprivileged mode for simplicity 3661 * since it requires corresponding support in Spectre masking for stack 3662 * ALU. See also retrieve_ptr_limit(). 3663 */ 3664 if (!env->bypass_spec_v1 && var_off) { 3665 char tn_buf[48]; 3666 3667 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3668 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 3669 ptr_regno, tn_buf); 3670 return -EACCES; 3671 } 3672 3673 if (!var_off) { 3674 off += reg->var_off.value; 3675 err = check_stack_read_fixed_off(env, state, off, size, 3676 dst_regno); 3677 } else { 3678 /* Variable offset stack reads need more conservative handling 3679 * than fixed offset ones. Note that dst_regno >= 0 on this 3680 * branch. 3681 */ 3682 err = check_stack_read_var_off(env, ptr_regno, off, size, 3683 dst_regno); 3684 } 3685 return err; 3686 } 3687 3688 3689 /* check_stack_write dispatches to check_stack_write_fixed_off or 3690 * check_stack_write_var_off. 3691 * 3692 * 'ptr_regno' is the register used as a pointer into the stack. 3693 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 3694 * 'value_regno' is the register whose value we're writing to the stack. It can 3695 * be -1, meaning that we're not writing from a register. 3696 * 3697 * The caller must ensure that the offset falls within the maximum stack size. 3698 */ 3699 static int check_stack_write(struct bpf_verifier_env *env, 3700 int ptr_regno, int off, int size, 3701 int value_regno, int insn_idx) 3702 { 3703 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3704 struct bpf_func_state *state = func(env, reg); 3705 int err; 3706 3707 if (tnum_is_const(reg->var_off)) { 3708 off += reg->var_off.value; 3709 err = check_stack_write_fixed_off(env, state, off, size, 3710 value_regno, insn_idx); 3711 } else { 3712 /* Variable offset stack reads need more conservative handling 3713 * than fixed offset ones. 3714 */ 3715 err = check_stack_write_var_off(env, state, 3716 ptr_regno, off, size, 3717 value_regno, insn_idx); 3718 } 3719 return err; 3720 } 3721 3722 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 3723 int off, int size, enum bpf_access_type type) 3724 { 3725 struct bpf_reg_state *regs = cur_regs(env); 3726 struct bpf_map *map = regs[regno].map_ptr; 3727 u32 cap = bpf_map_flags_to_cap(map); 3728 3729 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 3730 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 3731 map->value_size, off, size); 3732 return -EACCES; 3733 } 3734 3735 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 3736 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 3737 map->value_size, off, size); 3738 return -EACCES; 3739 } 3740 3741 return 0; 3742 } 3743 3744 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 3745 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 3746 int off, int size, u32 mem_size, 3747 bool zero_size_allowed) 3748 { 3749 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 3750 struct bpf_reg_state *reg; 3751 3752 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 3753 return 0; 3754 3755 reg = &cur_regs(env)[regno]; 3756 switch (reg->type) { 3757 case PTR_TO_MAP_KEY: 3758 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 3759 mem_size, off, size); 3760 break; 3761 case PTR_TO_MAP_VALUE: 3762 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 3763 mem_size, off, size); 3764 break; 3765 case PTR_TO_PACKET: 3766 case PTR_TO_PACKET_META: 3767 case PTR_TO_PACKET_END: 3768 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 3769 off, size, regno, reg->id, off, mem_size); 3770 break; 3771 case PTR_TO_MEM: 3772 default: 3773 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 3774 mem_size, off, size); 3775 } 3776 3777 return -EACCES; 3778 } 3779 3780 /* check read/write into a memory region with possible variable offset */ 3781 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 3782 int off, int size, u32 mem_size, 3783 bool zero_size_allowed) 3784 { 3785 struct bpf_verifier_state *vstate = env->cur_state; 3786 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3787 struct bpf_reg_state *reg = &state->regs[regno]; 3788 int err; 3789 3790 /* We may have adjusted the register pointing to memory region, so we 3791 * need to try adding each of min_value and max_value to off 3792 * to make sure our theoretical access will be safe. 3793 * 3794 * The minimum value is only important with signed 3795 * comparisons where we can't assume the floor of a 3796 * value is 0. If we are using signed variables for our 3797 * index'es we need to make sure that whatever we use 3798 * will have a set floor within our range. 3799 */ 3800 if (reg->smin_value < 0 && 3801 (reg->smin_value == S64_MIN || 3802 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 3803 reg->smin_value + off < 0)) { 3804 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3805 regno); 3806 return -EACCES; 3807 } 3808 err = __check_mem_access(env, regno, reg->smin_value + off, size, 3809 mem_size, zero_size_allowed); 3810 if (err) { 3811 verbose(env, "R%d min value is outside of the allowed memory range\n", 3812 regno); 3813 return err; 3814 } 3815 3816 /* If we haven't set a max value then we need to bail since we can't be 3817 * sure we won't do bad things. 3818 * If reg->umax_value + off could overflow, treat that as unbounded too. 3819 */ 3820 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 3821 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 3822 regno); 3823 return -EACCES; 3824 } 3825 err = __check_mem_access(env, regno, reg->umax_value + off, size, 3826 mem_size, zero_size_allowed); 3827 if (err) { 3828 verbose(env, "R%d max value is outside of the allowed memory range\n", 3829 regno); 3830 return err; 3831 } 3832 3833 return 0; 3834 } 3835 3836 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 3837 const struct bpf_reg_state *reg, int regno, 3838 bool fixed_off_ok) 3839 { 3840 /* Access to this pointer-typed register or passing it to a helper 3841 * is only allowed in its original, unmodified form. 3842 */ 3843 3844 if (reg->off < 0) { 3845 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 3846 reg_type_str(env, reg->type), regno, reg->off); 3847 return -EACCES; 3848 } 3849 3850 if (!fixed_off_ok && reg->off) { 3851 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 3852 reg_type_str(env, reg->type), regno, reg->off); 3853 return -EACCES; 3854 } 3855 3856 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3857 char tn_buf[48]; 3858 3859 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3860 verbose(env, "variable %s access var_off=%s disallowed\n", 3861 reg_type_str(env, reg->type), tn_buf); 3862 return -EACCES; 3863 } 3864 3865 return 0; 3866 } 3867 3868 int check_ptr_off_reg(struct bpf_verifier_env *env, 3869 const struct bpf_reg_state *reg, int regno) 3870 { 3871 return __check_ptr_off_reg(env, reg, regno, false); 3872 } 3873 3874 static int map_kptr_match_type(struct bpf_verifier_env *env, 3875 struct btf_field *kptr_field, 3876 struct bpf_reg_state *reg, u32 regno) 3877 { 3878 const char *targ_name = kernel_type_name(kptr_field->kptr.btf, kptr_field->kptr.btf_id); 3879 int perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED; 3880 const char *reg_name = ""; 3881 3882 /* Only unreferenced case accepts untrusted pointers */ 3883 if (kptr_field->type == BPF_KPTR_UNREF) 3884 perm_flags |= PTR_UNTRUSTED; 3885 3886 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 3887 goto bad_type; 3888 3889 if (!btf_is_kernel(reg->btf)) { 3890 verbose(env, "R%d must point to kernel BTF\n", regno); 3891 return -EINVAL; 3892 } 3893 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 3894 reg_name = kernel_type_name(reg->btf, reg->btf_id); 3895 3896 /* For ref_ptr case, release function check should ensure we get one 3897 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 3898 * normal store of unreferenced kptr, we must ensure var_off is zero. 3899 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 3900 * reg->off and reg->ref_obj_id are not needed here. 3901 */ 3902 if (__check_ptr_off_reg(env, reg, regno, true)) 3903 return -EACCES; 3904 3905 /* A full type match is needed, as BTF can be vmlinux or module BTF, and 3906 * we also need to take into account the reg->off. 3907 * 3908 * We want to support cases like: 3909 * 3910 * struct foo { 3911 * struct bar br; 3912 * struct baz bz; 3913 * }; 3914 * 3915 * struct foo *v; 3916 * v = func(); // PTR_TO_BTF_ID 3917 * val->foo = v; // reg->off is zero, btf and btf_id match type 3918 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 3919 * // first member type of struct after comparison fails 3920 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 3921 * // to match type 3922 * 3923 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 3924 * is zero. We must also ensure that btf_struct_ids_match does not walk 3925 * the struct to match type against first member of struct, i.e. reject 3926 * second case from above. Hence, when type is BPF_KPTR_REF, we set 3927 * strict mode to true for type match. 3928 */ 3929 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 3930 kptr_field->kptr.btf, kptr_field->kptr.btf_id, 3931 kptr_field->type == BPF_KPTR_REF)) 3932 goto bad_type; 3933 return 0; 3934 bad_type: 3935 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 3936 reg_type_str(env, reg->type), reg_name); 3937 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 3938 if (kptr_field->type == BPF_KPTR_UNREF) 3939 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 3940 targ_name); 3941 else 3942 verbose(env, "\n"); 3943 return -EINVAL; 3944 } 3945 3946 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 3947 int value_regno, int insn_idx, 3948 struct btf_field *kptr_field) 3949 { 3950 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3951 int class = BPF_CLASS(insn->code); 3952 struct bpf_reg_state *val_reg; 3953 3954 /* Things we already checked for in check_map_access and caller: 3955 * - Reject cases where variable offset may touch kptr 3956 * - size of access (must be BPF_DW) 3957 * - tnum_is_const(reg->var_off) 3958 * - kptr_field->offset == off + reg->var_off.value 3959 */ 3960 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 3961 if (BPF_MODE(insn->code) != BPF_MEM) { 3962 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 3963 return -EACCES; 3964 } 3965 3966 /* We only allow loading referenced kptr, since it will be marked as 3967 * untrusted, similar to unreferenced kptr. 3968 */ 3969 if (class != BPF_LDX && kptr_field->type == BPF_KPTR_REF) { 3970 verbose(env, "store to referenced kptr disallowed\n"); 3971 return -EACCES; 3972 } 3973 3974 if (class == BPF_LDX) { 3975 val_reg = reg_state(env, value_regno); 3976 /* We can simply mark the value_regno receiving the pointer 3977 * value from map as PTR_TO_BTF_ID, with the correct type. 3978 */ 3979 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, 3980 kptr_field->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 3981 /* For mark_ptr_or_null_reg */ 3982 val_reg->id = ++env->id_gen; 3983 } else if (class == BPF_STX) { 3984 val_reg = reg_state(env, value_regno); 3985 if (!register_is_null(val_reg) && 3986 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) 3987 return -EACCES; 3988 } else if (class == BPF_ST) { 3989 if (insn->imm) { 3990 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 3991 kptr_field->offset); 3992 return -EACCES; 3993 } 3994 } else { 3995 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 3996 return -EACCES; 3997 } 3998 return 0; 3999 } 4000 4001 /* check read/write into a map element with possible variable offset */ 4002 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 4003 int off, int size, bool zero_size_allowed, 4004 enum bpf_access_src src) 4005 { 4006 struct bpf_verifier_state *vstate = env->cur_state; 4007 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4008 struct bpf_reg_state *reg = &state->regs[regno]; 4009 struct bpf_map *map = reg->map_ptr; 4010 struct btf_record *rec; 4011 int err, i; 4012 4013 err = check_mem_region_access(env, regno, off, size, map->value_size, 4014 zero_size_allowed); 4015 if (err) 4016 return err; 4017 4018 if (IS_ERR_OR_NULL(map->record)) 4019 return 0; 4020 rec = map->record; 4021 for (i = 0; i < rec->cnt; i++) { 4022 struct btf_field *field = &rec->fields[i]; 4023 u32 p = field->offset; 4024 4025 /* If any part of a field can be touched by load/store, reject 4026 * this program. To check that [x1, x2) overlaps with [y1, y2), 4027 * it is sufficient to check x1 < y2 && y1 < x2. 4028 */ 4029 if (reg->smin_value + off < p + btf_field_type_size(field->type) && 4030 p < reg->umax_value + off + size) { 4031 switch (field->type) { 4032 case BPF_KPTR_UNREF: 4033 case BPF_KPTR_REF: 4034 if (src != ACCESS_DIRECT) { 4035 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 4036 return -EACCES; 4037 } 4038 if (!tnum_is_const(reg->var_off)) { 4039 verbose(env, "kptr access cannot have variable offset\n"); 4040 return -EACCES; 4041 } 4042 if (p != off + reg->var_off.value) { 4043 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 4044 p, off + reg->var_off.value); 4045 return -EACCES; 4046 } 4047 if (size != bpf_size_to_bytes(BPF_DW)) { 4048 verbose(env, "kptr access size must be BPF_DW\n"); 4049 return -EACCES; 4050 } 4051 break; 4052 default: 4053 verbose(env, "%s cannot be accessed directly by load/store\n", 4054 btf_field_type_name(field->type)); 4055 return -EACCES; 4056 } 4057 } 4058 } 4059 return 0; 4060 } 4061 4062 #define MAX_PACKET_OFF 0xffff 4063 4064 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 4065 const struct bpf_call_arg_meta *meta, 4066 enum bpf_access_type t) 4067 { 4068 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 4069 4070 switch (prog_type) { 4071 /* Program types only with direct read access go here! */ 4072 case BPF_PROG_TYPE_LWT_IN: 4073 case BPF_PROG_TYPE_LWT_OUT: 4074 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 4075 case BPF_PROG_TYPE_SK_REUSEPORT: 4076 case BPF_PROG_TYPE_FLOW_DISSECTOR: 4077 case BPF_PROG_TYPE_CGROUP_SKB: 4078 if (t == BPF_WRITE) 4079 return false; 4080 fallthrough; 4081 4082 /* Program types with direct read + write access go here! */ 4083 case BPF_PROG_TYPE_SCHED_CLS: 4084 case BPF_PROG_TYPE_SCHED_ACT: 4085 case BPF_PROG_TYPE_XDP: 4086 case BPF_PROG_TYPE_LWT_XMIT: 4087 case BPF_PROG_TYPE_SK_SKB: 4088 case BPF_PROG_TYPE_SK_MSG: 4089 if (meta) 4090 return meta->pkt_access; 4091 4092 env->seen_direct_write = true; 4093 return true; 4094 4095 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 4096 if (t == BPF_WRITE) 4097 env->seen_direct_write = true; 4098 4099 return true; 4100 4101 default: 4102 return false; 4103 } 4104 } 4105 4106 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 4107 int size, bool zero_size_allowed) 4108 { 4109 struct bpf_reg_state *regs = cur_regs(env); 4110 struct bpf_reg_state *reg = ®s[regno]; 4111 int err; 4112 4113 /* We may have added a variable offset to the packet pointer; but any 4114 * reg->range we have comes after that. We are only checking the fixed 4115 * offset. 4116 */ 4117 4118 /* We don't allow negative numbers, because we aren't tracking enough 4119 * detail to prove they're safe. 4120 */ 4121 if (reg->smin_value < 0) { 4122 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4123 regno); 4124 return -EACCES; 4125 } 4126 4127 err = reg->range < 0 ? -EINVAL : 4128 __check_mem_access(env, regno, off, size, reg->range, 4129 zero_size_allowed); 4130 if (err) { 4131 verbose(env, "R%d offset is outside of the packet\n", regno); 4132 return err; 4133 } 4134 4135 /* __check_mem_access has made sure "off + size - 1" is within u16. 4136 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 4137 * otherwise find_good_pkt_pointers would have refused to set range info 4138 * that __check_mem_access would have rejected this pkt access. 4139 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 4140 */ 4141 env->prog->aux->max_pkt_offset = 4142 max_t(u32, env->prog->aux->max_pkt_offset, 4143 off + reg->umax_value + size - 1); 4144 4145 return err; 4146 } 4147 4148 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 4149 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 4150 enum bpf_access_type t, enum bpf_reg_type *reg_type, 4151 struct btf **btf, u32 *btf_id) 4152 { 4153 struct bpf_insn_access_aux info = { 4154 .reg_type = *reg_type, 4155 .log = &env->log, 4156 }; 4157 4158 if (env->ops->is_valid_access && 4159 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 4160 /* A non zero info.ctx_field_size indicates that this field is a 4161 * candidate for later verifier transformation to load the whole 4162 * field and then apply a mask when accessed with a narrower 4163 * access than actual ctx access size. A zero info.ctx_field_size 4164 * will only allow for whole field access and rejects any other 4165 * type of narrower access. 4166 */ 4167 *reg_type = info.reg_type; 4168 4169 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 4170 *btf = info.btf; 4171 *btf_id = info.btf_id; 4172 } else { 4173 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 4174 } 4175 /* remember the offset of last byte accessed in ctx */ 4176 if (env->prog->aux->max_ctx_offset < off + size) 4177 env->prog->aux->max_ctx_offset = off + size; 4178 return 0; 4179 } 4180 4181 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 4182 return -EACCES; 4183 } 4184 4185 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 4186 int size) 4187 { 4188 if (size < 0 || off < 0 || 4189 (u64)off + size > sizeof(struct bpf_flow_keys)) { 4190 verbose(env, "invalid access to flow keys off=%d size=%d\n", 4191 off, size); 4192 return -EACCES; 4193 } 4194 return 0; 4195 } 4196 4197 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 4198 u32 regno, int off, int size, 4199 enum bpf_access_type t) 4200 { 4201 struct bpf_reg_state *regs = cur_regs(env); 4202 struct bpf_reg_state *reg = ®s[regno]; 4203 struct bpf_insn_access_aux info = {}; 4204 bool valid; 4205 4206 if (reg->smin_value < 0) { 4207 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 4208 regno); 4209 return -EACCES; 4210 } 4211 4212 switch (reg->type) { 4213 case PTR_TO_SOCK_COMMON: 4214 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 4215 break; 4216 case PTR_TO_SOCKET: 4217 valid = bpf_sock_is_valid_access(off, size, t, &info); 4218 break; 4219 case PTR_TO_TCP_SOCK: 4220 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 4221 break; 4222 case PTR_TO_XDP_SOCK: 4223 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 4224 break; 4225 default: 4226 valid = false; 4227 } 4228 4229 4230 if (valid) { 4231 env->insn_aux_data[insn_idx].ctx_field_size = 4232 info.ctx_field_size; 4233 return 0; 4234 } 4235 4236 verbose(env, "R%d invalid %s access off=%d size=%d\n", 4237 regno, reg_type_str(env, reg->type), off, size); 4238 4239 return -EACCES; 4240 } 4241 4242 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 4243 { 4244 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 4245 } 4246 4247 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 4248 { 4249 const struct bpf_reg_state *reg = reg_state(env, regno); 4250 4251 return reg->type == PTR_TO_CTX; 4252 } 4253 4254 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 4255 { 4256 const struct bpf_reg_state *reg = reg_state(env, regno); 4257 4258 return type_is_sk_pointer(reg->type); 4259 } 4260 4261 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 4262 { 4263 const struct bpf_reg_state *reg = reg_state(env, regno); 4264 4265 return type_is_pkt_pointer(reg->type); 4266 } 4267 4268 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 4269 { 4270 const struct bpf_reg_state *reg = reg_state(env, regno); 4271 4272 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 4273 return reg->type == PTR_TO_FLOW_KEYS; 4274 } 4275 4276 static bool is_trusted_reg(const struct bpf_reg_state *reg) 4277 { 4278 /* A referenced register is always trusted. */ 4279 if (reg->ref_obj_id) 4280 return true; 4281 4282 /* If a register is not referenced, it is trusted if it has the 4283 * MEM_ALLOC or PTR_TRUSTED type modifiers, and no others. Some of the 4284 * other type modifiers may be safe, but we elect to take an opt-in 4285 * approach here as some (e.g. PTR_UNTRUSTED and PTR_MAYBE_NULL) are 4286 * not. 4287 * 4288 * Eventually, we should make PTR_TRUSTED the single source of truth 4289 * for whether a register is trusted. 4290 */ 4291 return type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS && 4292 !bpf_type_has_unsafe_modifiers(reg->type); 4293 } 4294 4295 static bool is_rcu_reg(const struct bpf_reg_state *reg) 4296 { 4297 return reg->type & MEM_RCU; 4298 } 4299 4300 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 4301 const struct bpf_reg_state *reg, 4302 int off, int size, bool strict) 4303 { 4304 struct tnum reg_off; 4305 int ip_align; 4306 4307 /* Byte size accesses are always allowed. */ 4308 if (!strict || size == 1) 4309 return 0; 4310 4311 /* For platforms that do not have a Kconfig enabling 4312 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 4313 * NET_IP_ALIGN is universally set to '2'. And on platforms 4314 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 4315 * to this code only in strict mode where we want to emulate 4316 * the NET_IP_ALIGN==2 checking. Therefore use an 4317 * unconditional IP align value of '2'. 4318 */ 4319 ip_align = 2; 4320 4321 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 4322 if (!tnum_is_aligned(reg_off, size)) { 4323 char tn_buf[48]; 4324 4325 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4326 verbose(env, 4327 "misaligned packet access off %d+%s+%d+%d size %d\n", 4328 ip_align, tn_buf, reg->off, off, size); 4329 return -EACCES; 4330 } 4331 4332 return 0; 4333 } 4334 4335 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 4336 const struct bpf_reg_state *reg, 4337 const char *pointer_desc, 4338 int off, int size, bool strict) 4339 { 4340 struct tnum reg_off; 4341 4342 /* Byte size accesses are always allowed. */ 4343 if (!strict || size == 1) 4344 return 0; 4345 4346 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 4347 if (!tnum_is_aligned(reg_off, size)) { 4348 char tn_buf[48]; 4349 4350 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4351 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 4352 pointer_desc, tn_buf, reg->off, off, size); 4353 return -EACCES; 4354 } 4355 4356 return 0; 4357 } 4358 4359 static int check_ptr_alignment(struct bpf_verifier_env *env, 4360 const struct bpf_reg_state *reg, int off, 4361 int size, bool strict_alignment_once) 4362 { 4363 bool strict = env->strict_alignment || strict_alignment_once; 4364 const char *pointer_desc = ""; 4365 4366 switch (reg->type) { 4367 case PTR_TO_PACKET: 4368 case PTR_TO_PACKET_META: 4369 /* Special case, because of NET_IP_ALIGN. Given metadata sits 4370 * right in front, treat it the very same way. 4371 */ 4372 return check_pkt_ptr_alignment(env, reg, off, size, strict); 4373 case PTR_TO_FLOW_KEYS: 4374 pointer_desc = "flow keys "; 4375 break; 4376 case PTR_TO_MAP_KEY: 4377 pointer_desc = "key "; 4378 break; 4379 case PTR_TO_MAP_VALUE: 4380 pointer_desc = "value "; 4381 break; 4382 case PTR_TO_CTX: 4383 pointer_desc = "context "; 4384 break; 4385 case PTR_TO_STACK: 4386 pointer_desc = "stack "; 4387 /* The stack spill tracking logic in check_stack_write_fixed_off() 4388 * and check_stack_read_fixed_off() relies on stack accesses being 4389 * aligned. 4390 */ 4391 strict = true; 4392 break; 4393 case PTR_TO_SOCKET: 4394 pointer_desc = "sock "; 4395 break; 4396 case PTR_TO_SOCK_COMMON: 4397 pointer_desc = "sock_common "; 4398 break; 4399 case PTR_TO_TCP_SOCK: 4400 pointer_desc = "tcp_sock "; 4401 break; 4402 case PTR_TO_XDP_SOCK: 4403 pointer_desc = "xdp_sock "; 4404 break; 4405 default: 4406 break; 4407 } 4408 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 4409 strict); 4410 } 4411 4412 static int update_stack_depth(struct bpf_verifier_env *env, 4413 const struct bpf_func_state *func, 4414 int off) 4415 { 4416 u16 stack = env->subprog_info[func->subprogno].stack_depth; 4417 4418 if (stack >= -off) 4419 return 0; 4420 4421 /* update known max for given subprogram */ 4422 env->subprog_info[func->subprogno].stack_depth = -off; 4423 return 0; 4424 } 4425 4426 /* starting from main bpf function walk all instructions of the function 4427 * and recursively walk all callees that given function can call. 4428 * Ignore jump and exit insns. 4429 * Since recursion is prevented by check_cfg() this algorithm 4430 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 4431 */ 4432 static int check_max_stack_depth(struct bpf_verifier_env *env) 4433 { 4434 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 4435 struct bpf_subprog_info *subprog = env->subprog_info; 4436 struct bpf_insn *insn = env->prog->insnsi; 4437 bool tail_call_reachable = false; 4438 int ret_insn[MAX_CALL_FRAMES]; 4439 int ret_prog[MAX_CALL_FRAMES]; 4440 int j; 4441 4442 process_func: 4443 /* protect against potential stack overflow that might happen when 4444 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 4445 * depth for such case down to 256 so that the worst case scenario 4446 * would result in 8k stack size (32 which is tailcall limit * 256 = 4447 * 8k). 4448 * 4449 * To get the idea what might happen, see an example: 4450 * func1 -> sub rsp, 128 4451 * subfunc1 -> sub rsp, 256 4452 * tailcall1 -> add rsp, 256 4453 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 4454 * subfunc2 -> sub rsp, 64 4455 * subfunc22 -> sub rsp, 128 4456 * tailcall2 -> add rsp, 128 4457 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 4458 * 4459 * tailcall will unwind the current stack frame but it will not get rid 4460 * of caller's stack as shown on the example above. 4461 */ 4462 if (idx && subprog[idx].has_tail_call && depth >= 256) { 4463 verbose(env, 4464 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 4465 depth); 4466 return -EACCES; 4467 } 4468 /* round up to 32-bytes, since this is granularity 4469 * of interpreter stack size 4470 */ 4471 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4472 if (depth > MAX_BPF_STACK) { 4473 verbose(env, "combined stack size of %d calls is %d. Too large\n", 4474 frame + 1, depth); 4475 return -EACCES; 4476 } 4477 continue_func: 4478 subprog_end = subprog[idx + 1].start; 4479 for (; i < subprog_end; i++) { 4480 int next_insn; 4481 4482 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 4483 continue; 4484 /* remember insn and function to return to */ 4485 ret_insn[frame] = i + 1; 4486 ret_prog[frame] = idx; 4487 4488 /* find the callee */ 4489 next_insn = i + insn[i].imm + 1; 4490 idx = find_subprog(env, next_insn); 4491 if (idx < 0) { 4492 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4493 next_insn); 4494 return -EFAULT; 4495 } 4496 if (subprog[idx].is_async_cb) { 4497 if (subprog[idx].has_tail_call) { 4498 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 4499 return -EFAULT; 4500 } 4501 /* async callbacks don't increase bpf prog stack size */ 4502 continue; 4503 } 4504 i = next_insn; 4505 4506 if (subprog[idx].has_tail_call) 4507 tail_call_reachable = true; 4508 4509 frame++; 4510 if (frame >= MAX_CALL_FRAMES) { 4511 verbose(env, "the call stack of %d frames is too deep !\n", 4512 frame); 4513 return -E2BIG; 4514 } 4515 goto process_func; 4516 } 4517 /* if tail call got detected across bpf2bpf calls then mark each of the 4518 * currently present subprog frames as tail call reachable subprogs; 4519 * this info will be utilized by JIT so that we will be preserving the 4520 * tail call counter throughout bpf2bpf calls combined with tailcalls 4521 */ 4522 if (tail_call_reachable) 4523 for (j = 0; j < frame; j++) 4524 subprog[ret_prog[j]].tail_call_reachable = true; 4525 if (subprog[0].tail_call_reachable) 4526 env->prog->aux->tail_call_reachable = true; 4527 4528 /* end of for() loop means the last insn of the 'subprog' 4529 * was reached. Doesn't matter whether it was JA or EXIT 4530 */ 4531 if (frame == 0) 4532 return 0; 4533 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4534 frame--; 4535 i = ret_insn[frame]; 4536 idx = ret_prog[frame]; 4537 goto continue_func; 4538 } 4539 4540 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 4541 static int get_callee_stack_depth(struct bpf_verifier_env *env, 4542 const struct bpf_insn *insn, int idx) 4543 { 4544 int start = idx + insn->imm + 1, subprog; 4545 4546 subprog = find_subprog(env, start); 4547 if (subprog < 0) { 4548 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4549 start); 4550 return -EFAULT; 4551 } 4552 return env->subprog_info[subprog].stack_depth; 4553 } 4554 #endif 4555 4556 static int __check_buffer_access(struct bpf_verifier_env *env, 4557 const char *buf_info, 4558 const struct bpf_reg_state *reg, 4559 int regno, int off, int size) 4560 { 4561 if (off < 0) { 4562 verbose(env, 4563 "R%d invalid %s buffer access: off=%d, size=%d\n", 4564 regno, buf_info, off, size); 4565 return -EACCES; 4566 } 4567 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4568 char tn_buf[48]; 4569 4570 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4571 verbose(env, 4572 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 4573 regno, off, tn_buf); 4574 return -EACCES; 4575 } 4576 4577 return 0; 4578 } 4579 4580 static int check_tp_buffer_access(struct bpf_verifier_env *env, 4581 const struct bpf_reg_state *reg, 4582 int regno, int off, int size) 4583 { 4584 int err; 4585 4586 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 4587 if (err) 4588 return err; 4589 4590 if (off + size > env->prog->aux->max_tp_access) 4591 env->prog->aux->max_tp_access = off + size; 4592 4593 return 0; 4594 } 4595 4596 static int check_buffer_access(struct bpf_verifier_env *env, 4597 const struct bpf_reg_state *reg, 4598 int regno, int off, int size, 4599 bool zero_size_allowed, 4600 u32 *max_access) 4601 { 4602 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 4603 int err; 4604 4605 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 4606 if (err) 4607 return err; 4608 4609 if (off + size > *max_access) 4610 *max_access = off + size; 4611 4612 return 0; 4613 } 4614 4615 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 4616 static void zext_32_to_64(struct bpf_reg_state *reg) 4617 { 4618 reg->var_off = tnum_subreg(reg->var_off); 4619 __reg_assign_32_into_64(reg); 4620 } 4621 4622 /* truncate register to smaller size (in bytes) 4623 * must be called with size < BPF_REG_SIZE 4624 */ 4625 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 4626 { 4627 u64 mask; 4628 4629 /* clear high bits in bit representation */ 4630 reg->var_off = tnum_cast(reg->var_off, size); 4631 4632 /* fix arithmetic bounds */ 4633 mask = ((u64)1 << (size * 8)) - 1; 4634 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 4635 reg->umin_value &= mask; 4636 reg->umax_value &= mask; 4637 } else { 4638 reg->umin_value = 0; 4639 reg->umax_value = mask; 4640 } 4641 reg->smin_value = reg->umin_value; 4642 reg->smax_value = reg->umax_value; 4643 4644 /* If size is smaller than 32bit register the 32bit register 4645 * values are also truncated so we push 64-bit bounds into 4646 * 32-bit bounds. Above were truncated < 32-bits already. 4647 */ 4648 if (size >= 4) 4649 return; 4650 __reg_combine_64_into_32(reg); 4651 } 4652 4653 static bool bpf_map_is_rdonly(const struct bpf_map *map) 4654 { 4655 /* A map is considered read-only if the following condition are true: 4656 * 4657 * 1) BPF program side cannot change any of the map content. The 4658 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 4659 * and was set at map creation time. 4660 * 2) The map value(s) have been initialized from user space by a 4661 * loader and then "frozen", such that no new map update/delete 4662 * operations from syscall side are possible for the rest of 4663 * the map's lifetime from that point onwards. 4664 * 3) Any parallel/pending map update/delete operations from syscall 4665 * side have been completed. Only after that point, it's safe to 4666 * assume that map value(s) are immutable. 4667 */ 4668 return (map->map_flags & BPF_F_RDONLY_PROG) && 4669 READ_ONCE(map->frozen) && 4670 !bpf_map_write_active(map); 4671 } 4672 4673 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 4674 { 4675 void *ptr; 4676 u64 addr; 4677 int err; 4678 4679 err = map->ops->map_direct_value_addr(map, &addr, off); 4680 if (err) 4681 return err; 4682 ptr = (void *)(long)addr + off; 4683 4684 switch (size) { 4685 case sizeof(u8): 4686 *val = (u64)*(u8 *)ptr; 4687 break; 4688 case sizeof(u16): 4689 *val = (u64)*(u16 *)ptr; 4690 break; 4691 case sizeof(u32): 4692 *val = (u64)*(u32 *)ptr; 4693 break; 4694 case sizeof(u64): 4695 *val = *(u64 *)ptr; 4696 break; 4697 default: 4698 return -EINVAL; 4699 } 4700 return 0; 4701 } 4702 4703 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 4704 struct bpf_reg_state *regs, 4705 int regno, int off, int size, 4706 enum bpf_access_type atype, 4707 int value_regno) 4708 { 4709 struct bpf_reg_state *reg = regs + regno; 4710 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 4711 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 4712 enum bpf_type_flag flag = 0; 4713 u32 btf_id; 4714 int ret; 4715 4716 if (!env->allow_ptr_leaks) { 4717 verbose(env, 4718 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4719 tname); 4720 return -EPERM; 4721 } 4722 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { 4723 verbose(env, 4724 "Cannot access kernel 'struct %s' from non-GPL compatible program\n", 4725 tname); 4726 return -EINVAL; 4727 } 4728 if (off < 0) { 4729 verbose(env, 4730 "R%d is ptr_%s invalid negative access: off=%d\n", 4731 regno, tname, off); 4732 return -EACCES; 4733 } 4734 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4735 char tn_buf[48]; 4736 4737 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4738 verbose(env, 4739 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 4740 regno, tname, off, tn_buf); 4741 return -EACCES; 4742 } 4743 4744 if (reg->type & MEM_USER) { 4745 verbose(env, 4746 "R%d is ptr_%s access user memory: off=%d\n", 4747 regno, tname, off); 4748 return -EACCES; 4749 } 4750 4751 if (reg->type & MEM_PERCPU) { 4752 verbose(env, 4753 "R%d is ptr_%s access percpu memory: off=%d\n", 4754 regno, tname, off); 4755 return -EACCES; 4756 } 4757 4758 if (env->ops->btf_struct_access && !type_is_alloc(reg->type)) { 4759 if (!btf_is_kernel(reg->btf)) { 4760 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); 4761 return -EFAULT; 4762 } 4763 ret = env->ops->btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 4764 } else { 4765 /* Writes are permitted with default btf_struct_access for 4766 * program allocated objects (which always have ref_obj_id > 0), 4767 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. 4768 */ 4769 if (atype != BPF_READ && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 4770 verbose(env, "only read is supported\n"); 4771 return -EACCES; 4772 } 4773 4774 if (type_is_alloc(reg->type) && !reg->ref_obj_id) { 4775 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); 4776 return -EFAULT; 4777 } 4778 4779 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag); 4780 } 4781 4782 if (ret < 0) 4783 return ret; 4784 4785 /* If this is an untrusted pointer, all pointers formed by walking it 4786 * also inherit the untrusted flag. 4787 */ 4788 if (type_flag(reg->type) & PTR_UNTRUSTED) 4789 flag |= PTR_UNTRUSTED; 4790 4791 /* By default any pointer obtained from walking a trusted pointer is 4792 * no longer trusted except the rcu case below. 4793 */ 4794 flag &= ~PTR_TRUSTED; 4795 4796 if (flag & MEM_RCU) { 4797 /* Mark value register as MEM_RCU only if it is protected by 4798 * bpf_rcu_read_lock() and the ptr reg is rcu or trusted. MEM_RCU 4799 * itself can already indicate trustedness inside the rcu 4800 * read lock region. Also mark rcu pointer as PTR_MAYBE_NULL since 4801 * it could be null in some cases. 4802 */ 4803 if (!env->cur_state->active_rcu_lock || 4804 !(is_trusted_reg(reg) || is_rcu_reg(reg))) 4805 flag &= ~MEM_RCU; 4806 else 4807 flag |= PTR_MAYBE_NULL; 4808 } else if (reg->type & MEM_RCU) { 4809 /* ptr (reg) is marked as MEM_RCU, but the struct field is not tagged 4810 * with __rcu. Mark the flag as PTR_UNTRUSTED conservatively. 4811 */ 4812 flag |= PTR_UNTRUSTED; 4813 } 4814 4815 if (atype == BPF_READ && value_regno >= 0) 4816 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 4817 4818 return 0; 4819 } 4820 4821 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 4822 struct bpf_reg_state *regs, 4823 int regno, int off, int size, 4824 enum bpf_access_type atype, 4825 int value_regno) 4826 { 4827 struct bpf_reg_state *reg = regs + regno; 4828 struct bpf_map *map = reg->map_ptr; 4829 struct bpf_reg_state map_reg; 4830 enum bpf_type_flag flag = 0; 4831 const struct btf_type *t; 4832 const char *tname; 4833 u32 btf_id; 4834 int ret; 4835 4836 if (!btf_vmlinux) { 4837 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 4838 return -ENOTSUPP; 4839 } 4840 4841 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 4842 verbose(env, "map_ptr access not supported for map type %d\n", 4843 map->map_type); 4844 return -ENOTSUPP; 4845 } 4846 4847 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 4848 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 4849 4850 if (!env->allow_ptr_leaks) { 4851 verbose(env, 4852 "'struct %s' access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4853 tname); 4854 return -EPERM; 4855 } 4856 4857 if (off < 0) { 4858 verbose(env, "R%d is %s invalid negative access: off=%d\n", 4859 regno, tname, off); 4860 return -EACCES; 4861 } 4862 4863 if (atype != BPF_READ) { 4864 verbose(env, "only read from %s is supported\n", tname); 4865 return -EACCES; 4866 } 4867 4868 /* Simulate access to a PTR_TO_BTF_ID */ 4869 memset(&map_reg, 0, sizeof(map_reg)); 4870 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); 4871 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag); 4872 if (ret < 0) 4873 return ret; 4874 4875 if (value_regno >= 0) 4876 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 4877 4878 return 0; 4879 } 4880 4881 /* Check that the stack access at the given offset is within bounds. The 4882 * maximum valid offset is -1. 4883 * 4884 * The minimum valid offset is -MAX_BPF_STACK for writes, and 4885 * -state->allocated_stack for reads. 4886 */ 4887 static int check_stack_slot_within_bounds(int off, 4888 struct bpf_func_state *state, 4889 enum bpf_access_type t) 4890 { 4891 int min_valid_off; 4892 4893 if (t == BPF_WRITE) 4894 min_valid_off = -MAX_BPF_STACK; 4895 else 4896 min_valid_off = -state->allocated_stack; 4897 4898 if (off < min_valid_off || off > -1) 4899 return -EACCES; 4900 return 0; 4901 } 4902 4903 /* Check that the stack access at 'regno + off' falls within the maximum stack 4904 * bounds. 4905 * 4906 * 'off' includes `regno->offset`, but not its dynamic part (if any). 4907 */ 4908 static int check_stack_access_within_bounds( 4909 struct bpf_verifier_env *env, 4910 int regno, int off, int access_size, 4911 enum bpf_access_src src, enum bpf_access_type type) 4912 { 4913 struct bpf_reg_state *regs = cur_regs(env); 4914 struct bpf_reg_state *reg = regs + regno; 4915 struct bpf_func_state *state = func(env, reg); 4916 int min_off, max_off; 4917 int err; 4918 char *err_extra; 4919 4920 if (src == ACCESS_HELPER) 4921 /* We don't know if helpers are reading or writing (or both). */ 4922 err_extra = " indirect access to"; 4923 else if (type == BPF_READ) 4924 err_extra = " read from"; 4925 else 4926 err_extra = " write to"; 4927 4928 if (tnum_is_const(reg->var_off)) { 4929 min_off = reg->var_off.value + off; 4930 if (access_size > 0) 4931 max_off = min_off + access_size - 1; 4932 else 4933 max_off = min_off; 4934 } else { 4935 if (reg->smax_value >= BPF_MAX_VAR_OFF || 4936 reg->smin_value <= -BPF_MAX_VAR_OFF) { 4937 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 4938 err_extra, regno); 4939 return -EACCES; 4940 } 4941 min_off = reg->smin_value + off; 4942 if (access_size > 0) 4943 max_off = reg->smax_value + off + access_size - 1; 4944 else 4945 max_off = min_off; 4946 } 4947 4948 err = check_stack_slot_within_bounds(min_off, state, type); 4949 if (!err) 4950 err = check_stack_slot_within_bounds(max_off, state, type); 4951 4952 if (err) { 4953 if (tnum_is_const(reg->var_off)) { 4954 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 4955 err_extra, regno, off, access_size); 4956 } else { 4957 char tn_buf[48]; 4958 4959 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4960 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 4961 err_extra, regno, tn_buf, access_size); 4962 } 4963 } 4964 return err; 4965 } 4966 4967 /* check whether memory at (regno + off) is accessible for t = (read | write) 4968 * if t==write, value_regno is a register which value is stored into memory 4969 * if t==read, value_regno is a register which will receive the value from memory 4970 * if t==write && value_regno==-1, some unknown value is stored into memory 4971 * if t==read && value_regno==-1, don't care what we read from memory 4972 */ 4973 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 4974 int off, int bpf_size, enum bpf_access_type t, 4975 int value_regno, bool strict_alignment_once) 4976 { 4977 struct bpf_reg_state *regs = cur_regs(env); 4978 struct bpf_reg_state *reg = regs + regno; 4979 struct bpf_func_state *state; 4980 int size, err = 0; 4981 4982 size = bpf_size_to_bytes(bpf_size); 4983 if (size < 0) 4984 return size; 4985 4986 /* alignment checks will add in reg->off themselves */ 4987 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 4988 if (err) 4989 return err; 4990 4991 /* for access checks, reg->off is just part of off */ 4992 off += reg->off; 4993 4994 if (reg->type == PTR_TO_MAP_KEY) { 4995 if (t == BPF_WRITE) { 4996 verbose(env, "write to change key R%d not allowed\n", regno); 4997 return -EACCES; 4998 } 4999 5000 err = check_mem_region_access(env, regno, off, size, 5001 reg->map_ptr->key_size, false); 5002 if (err) 5003 return err; 5004 if (value_regno >= 0) 5005 mark_reg_unknown(env, regs, value_regno); 5006 } else if (reg->type == PTR_TO_MAP_VALUE) { 5007 struct btf_field *kptr_field = NULL; 5008 5009 if (t == BPF_WRITE && value_regno >= 0 && 5010 is_pointer_value(env, value_regno)) { 5011 verbose(env, "R%d leaks addr into map\n", value_regno); 5012 return -EACCES; 5013 } 5014 err = check_map_access_type(env, regno, off, size, t); 5015 if (err) 5016 return err; 5017 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 5018 if (err) 5019 return err; 5020 if (tnum_is_const(reg->var_off)) 5021 kptr_field = btf_record_find(reg->map_ptr->record, 5022 off + reg->var_off.value, BPF_KPTR); 5023 if (kptr_field) { 5024 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); 5025 } else if (t == BPF_READ && value_regno >= 0) { 5026 struct bpf_map *map = reg->map_ptr; 5027 5028 /* if map is read-only, track its contents as scalars */ 5029 if (tnum_is_const(reg->var_off) && 5030 bpf_map_is_rdonly(map) && 5031 map->ops->map_direct_value_addr) { 5032 int map_off = off + reg->var_off.value; 5033 u64 val = 0; 5034 5035 err = bpf_map_direct_read(map, map_off, size, 5036 &val); 5037 if (err) 5038 return err; 5039 5040 regs[value_regno].type = SCALAR_VALUE; 5041 __mark_reg_known(®s[value_regno], val); 5042 } else { 5043 mark_reg_unknown(env, regs, value_regno); 5044 } 5045 } 5046 } else if (base_type(reg->type) == PTR_TO_MEM) { 5047 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5048 5049 if (type_may_be_null(reg->type)) { 5050 verbose(env, "R%d invalid mem access '%s'\n", regno, 5051 reg_type_str(env, reg->type)); 5052 return -EACCES; 5053 } 5054 5055 if (t == BPF_WRITE && rdonly_mem) { 5056 verbose(env, "R%d cannot write into %s\n", 5057 regno, reg_type_str(env, reg->type)); 5058 return -EACCES; 5059 } 5060 5061 if (t == BPF_WRITE && value_regno >= 0 && 5062 is_pointer_value(env, value_regno)) { 5063 verbose(env, "R%d leaks addr into mem\n", value_regno); 5064 return -EACCES; 5065 } 5066 5067 err = check_mem_region_access(env, regno, off, size, 5068 reg->mem_size, false); 5069 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 5070 mark_reg_unknown(env, regs, value_regno); 5071 } else if (reg->type == PTR_TO_CTX) { 5072 enum bpf_reg_type reg_type = SCALAR_VALUE; 5073 struct btf *btf = NULL; 5074 u32 btf_id = 0; 5075 5076 if (t == BPF_WRITE && value_regno >= 0 && 5077 is_pointer_value(env, value_regno)) { 5078 verbose(env, "R%d leaks addr into ctx\n", value_regno); 5079 return -EACCES; 5080 } 5081 5082 err = check_ptr_off_reg(env, reg, regno); 5083 if (err < 0) 5084 return err; 5085 5086 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 5087 &btf_id); 5088 if (err) 5089 verbose_linfo(env, insn_idx, "; "); 5090 if (!err && t == BPF_READ && value_regno >= 0) { 5091 /* ctx access returns either a scalar, or a 5092 * PTR_TO_PACKET[_META,_END]. In the latter 5093 * case, we know the offset is zero. 5094 */ 5095 if (reg_type == SCALAR_VALUE) { 5096 mark_reg_unknown(env, regs, value_regno); 5097 } else { 5098 mark_reg_known_zero(env, regs, 5099 value_regno); 5100 if (type_may_be_null(reg_type)) 5101 regs[value_regno].id = ++env->id_gen; 5102 /* A load of ctx field could have different 5103 * actual load size with the one encoded in the 5104 * insn. When the dst is PTR, it is for sure not 5105 * a sub-register. 5106 */ 5107 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 5108 if (base_type(reg_type) == PTR_TO_BTF_ID) { 5109 regs[value_regno].btf = btf; 5110 regs[value_regno].btf_id = btf_id; 5111 } 5112 } 5113 regs[value_regno].type = reg_type; 5114 } 5115 5116 } else if (reg->type == PTR_TO_STACK) { 5117 /* Basic bounds checks. */ 5118 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 5119 if (err) 5120 return err; 5121 5122 state = func(env, reg); 5123 err = update_stack_depth(env, state, off); 5124 if (err) 5125 return err; 5126 5127 if (t == BPF_READ) 5128 err = check_stack_read(env, regno, off, size, 5129 value_regno); 5130 else 5131 err = check_stack_write(env, regno, off, size, 5132 value_regno, insn_idx); 5133 } else if (reg_is_pkt_pointer(reg)) { 5134 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 5135 verbose(env, "cannot write into packet\n"); 5136 return -EACCES; 5137 } 5138 if (t == BPF_WRITE && value_regno >= 0 && 5139 is_pointer_value(env, value_regno)) { 5140 verbose(env, "R%d leaks addr into packet\n", 5141 value_regno); 5142 return -EACCES; 5143 } 5144 err = check_packet_access(env, regno, off, size, false); 5145 if (!err && t == BPF_READ && value_regno >= 0) 5146 mark_reg_unknown(env, regs, value_regno); 5147 } else if (reg->type == PTR_TO_FLOW_KEYS) { 5148 if (t == BPF_WRITE && value_regno >= 0 && 5149 is_pointer_value(env, value_regno)) { 5150 verbose(env, "R%d leaks addr into flow keys\n", 5151 value_regno); 5152 return -EACCES; 5153 } 5154 5155 err = check_flow_keys_access(env, off, size); 5156 if (!err && t == BPF_READ && value_regno >= 0) 5157 mark_reg_unknown(env, regs, value_regno); 5158 } else if (type_is_sk_pointer(reg->type)) { 5159 if (t == BPF_WRITE) { 5160 verbose(env, "R%d cannot write into %s\n", 5161 regno, reg_type_str(env, reg->type)); 5162 return -EACCES; 5163 } 5164 err = check_sock_access(env, insn_idx, regno, off, size, t); 5165 if (!err && value_regno >= 0) 5166 mark_reg_unknown(env, regs, value_regno); 5167 } else if (reg->type == PTR_TO_TP_BUFFER) { 5168 err = check_tp_buffer_access(env, reg, regno, off, size); 5169 if (!err && t == BPF_READ && value_regno >= 0) 5170 mark_reg_unknown(env, regs, value_regno); 5171 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 5172 !type_may_be_null(reg->type)) { 5173 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 5174 value_regno); 5175 } else if (reg->type == CONST_PTR_TO_MAP) { 5176 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 5177 value_regno); 5178 } else if (base_type(reg->type) == PTR_TO_BUF) { 5179 bool rdonly_mem = type_is_rdonly_mem(reg->type); 5180 u32 *max_access; 5181 5182 if (rdonly_mem) { 5183 if (t == BPF_WRITE) { 5184 verbose(env, "R%d cannot write into %s\n", 5185 regno, reg_type_str(env, reg->type)); 5186 return -EACCES; 5187 } 5188 max_access = &env->prog->aux->max_rdonly_access; 5189 } else { 5190 max_access = &env->prog->aux->max_rdwr_access; 5191 } 5192 5193 err = check_buffer_access(env, reg, regno, off, size, false, 5194 max_access); 5195 5196 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 5197 mark_reg_unknown(env, regs, value_regno); 5198 } else { 5199 verbose(env, "R%d invalid mem access '%s'\n", regno, 5200 reg_type_str(env, reg->type)); 5201 return -EACCES; 5202 } 5203 5204 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 5205 regs[value_regno].type == SCALAR_VALUE) { 5206 /* b/h/w load zero-extends, mark upper bits as known 0 */ 5207 coerce_reg_to_size(®s[value_regno], size); 5208 } 5209 return err; 5210 } 5211 5212 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 5213 { 5214 int load_reg; 5215 int err; 5216 5217 switch (insn->imm) { 5218 case BPF_ADD: 5219 case BPF_ADD | BPF_FETCH: 5220 case BPF_AND: 5221 case BPF_AND | BPF_FETCH: 5222 case BPF_OR: 5223 case BPF_OR | BPF_FETCH: 5224 case BPF_XOR: 5225 case BPF_XOR | BPF_FETCH: 5226 case BPF_XCHG: 5227 case BPF_CMPXCHG: 5228 break; 5229 default: 5230 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 5231 return -EINVAL; 5232 } 5233 5234 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 5235 verbose(env, "invalid atomic operand size\n"); 5236 return -EINVAL; 5237 } 5238 5239 /* check src1 operand */ 5240 err = check_reg_arg(env, insn->src_reg, SRC_OP); 5241 if (err) 5242 return err; 5243 5244 /* check src2 operand */ 5245 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 5246 if (err) 5247 return err; 5248 5249 if (insn->imm == BPF_CMPXCHG) { 5250 /* Check comparison of R0 with memory location */ 5251 const u32 aux_reg = BPF_REG_0; 5252 5253 err = check_reg_arg(env, aux_reg, SRC_OP); 5254 if (err) 5255 return err; 5256 5257 if (is_pointer_value(env, aux_reg)) { 5258 verbose(env, "R%d leaks addr into mem\n", aux_reg); 5259 return -EACCES; 5260 } 5261 } 5262 5263 if (is_pointer_value(env, insn->src_reg)) { 5264 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 5265 return -EACCES; 5266 } 5267 5268 if (is_ctx_reg(env, insn->dst_reg) || 5269 is_pkt_reg(env, insn->dst_reg) || 5270 is_flow_key_reg(env, insn->dst_reg) || 5271 is_sk_reg(env, insn->dst_reg)) { 5272 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 5273 insn->dst_reg, 5274 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 5275 return -EACCES; 5276 } 5277 5278 if (insn->imm & BPF_FETCH) { 5279 if (insn->imm == BPF_CMPXCHG) 5280 load_reg = BPF_REG_0; 5281 else 5282 load_reg = insn->src_reg; 5283 5284 /* check and record load of old value */ 5285 err = check_reg_arg(env, load_reg, DST_OP); 5286 if (err) 5287 return err; 5288 } else { 5289 /* This instruction accesses a memory location but doesn't 5290 * actually load it into a register. 5291 */ 5292 load_reg = -1; 5293 } 5294 5295 /* Check whether we can read the memory, with second call for fetch 5296 * case to simulate the register fill. 5297 */ 5298 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5299 BPF_SIZE(insn->code), BPF_READ, -1, true); 5300 if (!err && load_reg >= 0) 5301 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5302 BPF_SIZE(insn->code), BPF_READ, load_reg, 5303 true); 5304 if (err) 5305 return err; 5306 5307 /* Check whether we can write into the same memory. */ 5308 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 5309 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 5310 if (err) 5311 return err; 5312 5313 return 0; 5314 } 5315 5316 /* When register 'regno' is used to read the stack (either directly or through 5317 * a helper function) make sure that it's within stack boundary and, depending 5318 * on the access type, that all elements of the stack are initialized. 5319 * 5320 * 'off' includes 'regno->off', but not its dynamic part (if any). 5321 * 5322 * All registers that have been spilled on the stack in the slots within the 5323 * read offsets are marked as read. 5324 */ 5325 static int check_stack_range_initialized( 5326 struct bpf_verifier_env *env, int regno, int off, 5327 int access_size, bool zero_size_allowed, 5328 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 5329 { 5330 struct bpf_reg_state *reg = reg_state(env, regno); 5331 struct bpf_func_state *state = func(env, reg); 5332 int err, min_off, max_off, i, j, slot, spi; 5333 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 5334 enum bpf_access_type bounds_check_type; 5335 /* Some accesses can write anything into the stack, others are 5336 * read-only. 5337 */ 5338 bool clobber = false; 5339 5340 if (access_size == 0 && !zero_size_allowed) { 5341 verbose(env, "invalid zero-sized read\n"); 5342 return -EACCES; 5343 } 5344 5345 if (type == ACCESS_HELPER) { 5346 /* The bounds checks for writes are more permissive than for 5347 * reads. However, if raw_mode is not set, we'll do extra 5348 * checks below. 5349 */ 5350 bounds_check_type = BPF_WRITE; 5351 clobber = true; 5352 } else { 5353 bounds_check_type = BPF_READ; 5354 } 5355 err = check_stack_access_within_bounds(env, regno, off, access_size, 5356 type, bounds_check_type); 5357 if (err) 5358 return err; 5359 5360 5361 if (tnum_is_const(reg->var_off)) { 5362 min_off = max_off = reg->var_off.value + off; 5363 } else { 5364 /* Variable offset is prohibited for unprivileged mode for 5365 * simplicity since it requires corresponding support in 5366 * Spectre masking for stack ALU. 5367 * See also retrieve_ptr_limit(). 5368 */ 5369 if (!env->bypass_spec_v1) { 5370 char tn_buf[48]; 5371 5372 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5373 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 5374 regno, err_extra, tn_buf); 5375 return -EACCES; 5376 } 5377 /* Only initialized buffer on stack is allowed to be accessed 5378 * with variable offset. With uninitialized buffer it's hard to 5379 * guarantee that whole memory is marked as initialized on 5380 * helper return since specific bounds are unknown what may 5381 * cause uninitialized stack leaking. 5382 */ 5383 if (meta && meta->raw_mode) 5384 meta = NULL; 5385 5386 min_off = reg->smin_value + off; 5387 max_off = reg->smax_value + off; 5388 } 5389 5390 if (meta && meta->raw_mode) { 5391 meta->access_size = access_size; 5392 meta->regno = regno; 5393 return 0; 5394 } 5395 5396 for (i = min_off; i < max_off + access_size; i++) { 5397 u8 *stype; 5398 5399 slot = -i - 1; 5400 spi = slot / BPF_REG_SIZE; 5401 if (state->allocated_stack <= slot) 5402 goto err; 5403 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 5404 if (*stype == STACK_MISC) 5405 goto mark; 5406 if (*stype == STACK_ZERO) { 5407 if (clobber) { 5408 /* helper can write anything into the stack */ 5409 *stype = STACK_MISC; 5410 } 5411 goto mark; 5412 } 5413 5414 if (is_spilled_reg(&state->stack[spi]) && 5415 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 5416 env->allow_ptr_leaks)) { 5417 if (clobber) { 5418 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 5419 for (j = 0; j < BPF_REG_SIZE; j++) 5420 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 5421 } 5422 goto mark; 5423 } 5424 5425 err: 5426 if (tnum_is_const(reg->var_off)) { 5427 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 5428 err_extra, regno, min_off, i - min_off, access_size); 5429 } else { 5430 char tn_buf[48]; 5431 5432 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5433 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 5434 err_extra, regno, tn_buf, i - min_off, access_size); 5435 } 5436 return -EACCES; 5437 mark: 5438 /* reading any byte out of 8-byte 'spill_slot' will cause 5439 * the whole slot to be marked as 'read' 5440 */ 5441 mark_reg_read(env, &state->stack[spi].spilled_ptr, 5442 state->stack[spi].spilled_ptr.parent, 5443 REG_LIVE_READ64); 5444 /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not 5445 * be sure that whether stack slot is written to or not. Hence, 5446 * we must still conservatively propagate reads upwards even if 5447 * helper may write to the entire memory range. 5448 */ 5449 } 5450 return update_stack_depth(env, state, min_off); 5451 } 5452 5453 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 5454 int access_size, bool zero_size_allowed, 5455 struct bpf_call_arg_meta *meta) 5456 { 5457 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5458 u32 *max_access; 5459 5460 switch (base_type(reg->type)) { 5461 case PTR_TO_PACKET: 5462 case PTR_TO_PACKET_META: 5463 return check_packet_access(env, regno, reg->off, access_size, 5464 zero_size_allowed); 5465 case PTR_TO_MAP_KEY: 5466 if (meta && meta->raw_mode) { 5467 verbose(env, "R%d cannot write into %s\n", regno, 5468 reg_type_str(env, reg->type)); 5469 return -EACCES; 5470 } 5471 return check_mem_region_access(env, regno, reg->off, access_size, 5472 reg->map_ptr->key_size, false); 5473 case PTR_TO_MAP_VALUE: 5474 if (check_map_access_type(env, regno, reg->off, access_size, 5475 meta && meta->raw_mode ? BPF_WRITE : 5476 BPF_READ)) 5477 return -EACCES; 5478 return check_map_access(env, regno, reg->off, access_size, 5479 zero_size_allowed, ACCESS_HELPER); 5480 case PTR_TO_MEM: 5481 if (type_is_rdonly_mem(reg->type)) { 5482 if (meta && meta->raw_mode) { 5483 verbose(env, "R%d cannot write into %s\n", regno, 5484 reg_type_str(env, reg->type)); 5485 return -EACCES; 5486 } 5487 } 5488 return check_mem_region_access(env, regno, reg->off, 5489 access_size, reg->mem_size, 5490 zero_size_allowed); 5491 case PTR_TO_BUF: 5492 if (type_is_rdonly_mem(reg->type)) { 5493 if (meta && meta->raw_mode) { 5494 verbose(env, "R%d cannot write into %s\n", regno, 5495 reg_type_str(env, reg->type)); 5496 return -EACCES; 5497 } 5498 5499 max_access = &env->prog->aux->max_rdonly_access; 5500 } else { 5501 max_access = &env->prog->aux->max_rdwr_access; 5502 } 5503 return check_buffer_access(env, reg, regno, reg->off, 5504 access_size, zero_size_allowed, 5505 max_access); 5506 case PTR_TO_STACK: 5507 return check_stack_range_initialized( 5508 env, 5509 regno, reg->off, access_size, 5510 zero_size_allowed, ACCESS_HELPER, meta); 5511 case PTR_TO_CTX: 5512 /* in case the function doesn't know how to access the context, 5513 * (because we are in a program of type SYSCALL for example), we 5514 * can not statically check its size. 5515 * Dynamically check it now. 5516 */ 5517 if (!env->ops->convert_ctx_access) { 5518 enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; 5519 int offset = access_size - 1; 5520 5521 /* Allow zero-byte read from PTR_TO_CTX */ 5522 if (access_size == 0) 5523 return zero_size_allowed ? 0 : -EACCES; 5524 5525 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, 5526 atype, -1, false); 5527 } 5528 5529 fallthrough; 5530 default: /* scalar_value or invalid ptr */ 5531 /* Allow zero-byte read from NULL, regardless of pointer type */ 5532 if (zero_size_allowed && access_size == 0 && 5533 register_is_null(reg)) 5534 return 0; 5535 5536 verbose(env, "R%d type=%s ", regno, 5537 reg_type_str(env, reg->type)); 5538 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 5539 return -EACCES; 5540 } 5541 } 5542 5543 static int check_mem_size_reg(struct bpf_verifier_env *env, 5544 struct bpf_reg_state *reg, u32 regno, 5545 bool zero_size_allowed, 5546 struct bpf_call_arg_meta *meta) 5547 { 5548 int err; 5549 5550 /* This is used to refine r0 return value bounds for helpers 5551 * that enforce this value as an upper bound on return values. 5552 * See do_refine_retval_range() for helpers that can refine 5553 * the return value. C type of helper is u32 so we pull register 5554 * bound from umax_value however, if negative verifier errors 5555 * out. Only upper bounds can be learned because retval is an 5556 * int type and negative retvals are allowed. 5557 */ 5558 meta->msize_max_value = reg->umax_value; 5559 5560 /* The register is SCALAR_VALUE; the access check 5561 * happens using its boundaries. 5562 */ 5563 if (!tnum_is_const(reg->var_off)) 5564 /* For unprivileged variable accesses, disable raw 5565 * mode so that the program is required to 5566 * initialize all the memory that the helper could 5567 * just partially fill up. 5568 */ 5569 meta = NULL; 5570 5571 if (reg->smin_value < 0) { 5572 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5573 regno); 5574 return -EACCES; 5575 } 5576 5577 if (reg->umin_value == 0) { 5578 err = check_helper_mem_access(env, regno - 1, 0, 5579 zero_size_allowed, 5580 meta); 5581 if (err) 5582 return err; 5583 } 5584 5585 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5586 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5587 regno); 5588 return -EACCES; 5589 } 5590 err = check_helper_mem_access(env, regno - 1, 5591 reg->umax_value, 5592 zero_size_allowed, meta); 5593 if (!err) 5594 err = mark_chain_precision(env, regno); 5595 return err; 5596 } 5597 5598 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5599 u32 regno, u32 mem_size) 5600 { 5601 bool may_be_null = type_may_be_null(reg->type); 5602 struct bpf_reg_state saved_reg; 5603 struct bpf_call_arg_meta meta; 5604 int err; 5605 5606 if (register_is_null(reg)) 5607 return 0; 5608 5609 memset(&meta, 0, sizeof(meta)); 5610 /* Assuming that the register contains a value check if the memory 5611 * access is safe. Temporarily save and restore the register's state as 5612 * the conversion shouldn't be visible to a caller. 5613 */ 5614 if (may_be_null) { 5615 saved_reg = *reg; 5616 mark_ptr_not_null_reg(reg); 5617 } 5618 5619 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 5620 /* Check access for BPF_WRITE */ 5621 meta.raw_mode = true; 5622 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 5623 5624 if (may_be_null) 5625 *reg = saved_reg; 5626 5627 return err; 5628 } 5629 5630 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5631 u32 regno) 5632 { 5633 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 5634 bool may_be_null = type_may_be_null(mem_reg->type); 5635 struct bpf_reg_state saved_reg; 5636 struct bpf_call_arg_meta meta; 5637 int err; 5638 5639 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 5640 5641 memset(&meta, 0, sizeof(meta)); 5642 5643 if (may_be_null) { 5644 saved_reg = *mem_reg; 5645 mark_ptr_not_null_reg(mem_reg); 5646 } 5647 5648 err = check_mem_size_reg(env, reg, regno, true, &meta); 5649 /* Check access for BPF_WRITE */ 5650 meta.raw_mode = true; 5651 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 5652 5653 if (may_be_null) 5654 *mem_reg = saved_reg; 5655 return err; 5656 } 5657 5658 /* Implementation details: 5659 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL. 5660 * bpf_obj_new returns PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL. 5661 * Two bpf_map_lookups (even with the same key) will have different reg->id. 5662 * Two separate bpf_obj_new will also have different reg->id. 5663 * For traditional PTR_TO_MAP_VALUE or PTR_TO_BTF_ID | MEM_ALLOC, the verifier 5664 * clears reg->id after value_or_null->value transition, since the verifier only 5665 * cares about the range of access to valid map value pointer and doesn't care 5666 * about actual address of the map element. 5667 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 5668 * reg->id > 0 after value_or_null->value transition. By doing so 5669 * two bpf_map_lookups will be considered two different pointers that 5670 * point to different bpf_spin_locks. Likewise for pointers to allocated objects 5671 * returned from bpf_obj_new. 5672 * The verifier allows taking only one bpf_spin_lock at a time to avoid 5673 * dead-locks. 5674 * Since only one bpf_spin_lock is allowed the checks are simpler than 5675 * reg_is_refcounted() logic. The verifier needs to remember only 5676 * one spin_lock instead of array of acquired_refs. 5677 * cur_state->active_lock remembers which map value element or allocated 5678 * object got locked and clears it after bpf_spin_unlock. 5679 */ 5680 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 5681 bool is_lock) 5682 { 5683 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5684 struct bpf_verifier_state *cur = env->cur_state; 5685 bool is_const = tnum_is_const(reg->var_off); 5686 u64 val = reg->var_off.value; 5687 struct bpf_map *map = NULL; 5688 struct btf *btf = NULL; 5689 struct btf_record *rec; 5690 5691 if (!is_const) { 5692 verbose(env, 5693 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 5694 regno); 5695 return -EINVAL; 5696 } 5697 if (reg->type == PTR_TO_MAP_VALUE) { 5698 map = reg->map_ptr; 5699 if (!map->btf) { 5700 verbose(env, 5701 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 5702 map->name); 5703 return -EINVAL; 5704 } 5705 } else { 5706 btf = reg->btf; 5707 } 5708 5709 rec = reg_btf_record(reg); 5710 if (!btf_record_has_field(rec, BPF_SPIN_LOCK)) { 5711 verbose(env, "%s '%s' has no valid bpf_spin_lock\n", map ? "map" : "local", 5712 map ? map->name : "kptr"); 5713 return -EINVAL; 5714 } 5715 if (rec->spin_lock_off != val + reg->off) { 5716 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock' that is at %d\n", 5717 val + reg->off, rec->spin_lock_off); 5718 return -EINVAL; 5719 } 5720 if (is_lock) { 5721 if (cur->active_lock.ptr) { 5722 verbose(env, 5723 "Locking two bpf_spin_locks are not allowed\n"); 5724 return -EINVAL; 5725 } 5726 if (map) 5727 cur->active_lock.ptr = map; 5728 else 5729 cur->active_lock.ptr = btf; 5730 cur->active_lock.id = reg->id; 5731 } else { 5732 struct bpf_func_state *fstate = cur_func(env); 5733 void *ptr; 5734 int i; 5735 5736 if (map) 5737 ptr = map; 5738 else 5739 ptr = btf; 5740 5741 if (!cur->active_lock.ptr) { 5742 verbose(env, "bpf_spin_unlock without taking a lock\n"); 5743 return -EINVAL; 5744 } 5745 if (cur->active_lock.ptr != ptr || 5746 cur->active_lock.id != reg->id) { 5747 verbose(env, "bpf_spin_unlock of different lock\n"); 5748 return -EINVAL; 5749 } 5750 cur->active_lock.ptr = NULL; 5751 cur->active_lock.id = 0; 5752 5753 for (i = fstate->acquired_refs - 1; i >= 0; i--) { 5754 int err; 5755 5756 /* Complain on error because this reference state cannot 5757 * be freed before this point, as bpf_spin_lock critical 5758 * section does not allow functions that release the 5759 * allocated object immediately. 5760 */ 5761 if (!fstate->refs[i].release_on_unlock) 5762 continue; 5763 err = release_reference(env, fstate->refs[i].id); 5764 if (err) { 5765 verbose(env, "failed to release release_on_unlock reference"); 5766 return err; 5767 } 5768 } 5769 } 5770 return 0; 5771 } 5772 5773 static int process_timer_func(struct bpf_verifier_env *env, int regno, 5774 struct bpf_call_arg_meta *meta) 5775 { 5776 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5777 bool is_const = tnum_is_const(reg->var_off); 5778 struct bpf_map *map = reg->map_ptr; 5779 u64 val = reg->var_off.value; 5780 5781 if (!is_const) { 5782 verbose(env, 5783 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 5784 regno); 5785 return -EINVAL; 5786 } 5787 if (!map->btf) { 5788 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 5789 map->name); 5790 return -EINVAL; 5791 } 5792 if (!btf_record_has_field(map->record, BPF_TIMER)) { 5793 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); 5794 return -EINVAL; 5795 } 5796 if (map->record->timer_off != val + reg->off) { 5797 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 5798 val + reg->off, map->record->timer_off); 5799 return -EINVAL; 5800 } 5801 if (meta->map_ptr) { 5802 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 5803 return -EFAULT; 5804 } 5805 meta->map_uid = reg->map_uid; 5806 meta->map_ptr = map; 5807 return 0; 5808 } 5809 5810 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 5811 struct bpf_call_arg_meta *meta) 5812 { 5813 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5814 struct bpf_map *map_ptr = reg->map_ptr; 5815 struct btf_field *kptr_field; 5816 u32 kptr_off; 5817 5818 if (!tnum_is_const(reg->var_off)) { 5819 verbose(env, 5820 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 5821 regno); 5822 return -EINVAL; 5823 } 5824 if (!map_ptr->btf) { 5825 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 5826 map_ptr->name); 5827 return -EINVAL; 5828 } 5829 if (!btf_record_has_field(map_ptr->record, BPF_KPTR)) { 5830 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 5831 return -EINVAL; 5832 } 5833 5834 meta->map_ptr = map_ptr; 5835 kptr_off = reg->off + reg->var_off.value; 5836 kptr_field = btf_record_find(map_ptr->record, kptr_off, BPF_KPTR); 5837 if (!kptr_field) { 5838 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 5839 return -EACCES; 5840 } 5841 if (kptr_field->type != BPF_KPTR_REF) { 5842 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 5843 return -EACCES; 5844 } 5845 meta->kptr_field = kptr_field; 5846 return 0; 5847 } 5848 5849 static bool arg_type_is_mem_size(enum bpf_arg_type type) 5850 { 5851 return type == ARG_CONST_SIZE || 5852 type == ARG_CONST_SIZE_OR_ZERO; 5853 } 5854 5855 static bool arg_type_is_release(enum bpf_arg_type type) 5856 { 5857 return type & OBJ_RELEASE; 5858 } 5859 5860 static bool arg_type_is_dynptr(enum bpf_arg_type type) 5861 { 5862 return base_type(type) == ARG_PTR_TO_DYNPTR; 5863 } 5864 5865 static int int_ptr_type_to_size(enum bpf_arg_type type) 5866 { 5867 if (type == ARG_PTR_TO_INT) 5868 return sizeof(u32); 5869 else if (type == ARG_PTR_TO_LONG) 5870 return sizeof(u64); 5871 5872 return -EINVAL; 5873 } 5874 5875 static int resolve_map_arg_type(struct bpf_verifier_env *env, 5876 const struct bpf_call_arg_meta *meta, 5877 enum bpf_arg_type *arg_type) 5878 { 5879 if (!meta->map_ptr) { 5880 /* kernel subsystem misconfigured verifier */ 5881 verbose(env, "invalid map_ptr to access map->type\n"); 5882 return -EACCES; 5883 } 5884 5885 switch (meta->map_ptr->map_type) { 5886 case BPF_MAP_TYPE_SOCKMAP: 5887 case BPF_MAP_TYPE_SOCKHASH: 5888 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 5889 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 5890 } else { 5891 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 5892 return -EINVAL; 5893 } 5894 break; 5895 case BPF_MAP_TYPE_BLOOM_FILTER: 5896 if (meta->func_id == BPF_FUNC_map_peek_elem) 5897 *arg_type = ARG_PTR_TO_MAP_VALUE; 5898 break; 5899 default: 5900 break; 5901 } 5902 return 0; 5903 } 5904 5905 struct bpf_reg_types { 5906 const enum bpf_reg_type types[10]; 5907 u32 *btf_id; 5908 }; 5909 5910 static const struct bpf_reg_types sock_types = { 5911 .types = { 5912 PTR_TO_SOCK_COMMON, 5913 PTR_TO_SOCKET, 5914 PTR_TO_TCP_SOCK, 5915 PTR_TO_XDP_SOCK, 5916 }, 5917 }; 5918 5919 #ifdef CONFIG_NET 5920 static const struct bpf_reg_types btf_id_sock_common_types = { 5921 .types = { 5922 PTR_TO_SOCK_COMMON, 5923 PTR_TO_SOCKET, 5924 PTR_TO_TCP_SOCK, 5925 PTR_TO_XDP_SOCK, 5926 PTR_TO_BTF_ID, 5927 PTR_TO_BTF_ID | PTR_TRUSTED, 5928 }, 5929 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 5930 }; 5931 #endif 5932 5933 static const struct bpf_reg_types mem_types = { 5934 .types = { 5935 PTR_TO_STACK, 5936 PTR_TO_PACKET, 5937 PTR_TO_PACKET_META, 5938 PTR_TO_MAP_KEY, 5939 PTR_TO_MAP_VALUE, 5940 PTR_TO_MEM, 5941 PTR_TO_MEM | MEM_RINGBUF, 5942 PTR_TO_BUF, 5943 }, 5944 }; 5945 5946 static const struct bpf_reg_types int_ptr_types = { 5947 .types = { 5948 PTR_TO_STACK, 5949 PTR_TO_PACKET, 5950 PTR_TO_PACKET_META, 5951 PTR_TO_MAP_KEY, 5952 PTR_TO_MAP_VALUE, 5953 }, 5954 }; 5955 5956 static const struct bpf_reg_types spin_lock_types = { 5957 .types = { 5958 PTR_TO_MAP_VALUE, 5959 PTR_TO_BTF_ID | MEM_ALLOC, 5960 } 5961 }; 5962 5963 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 5964 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 5965 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 5966 static const struct bpf_reg_types ringbuf_mem_types = { .types = { PTR_TO_MEM | MEM_RINGBUF } }; 5967 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 5968 static const struct bpf_reg_types btf_ptr_types = { 5969 .types = { 5970 PTR_TO_BTF_ID, 5971 PTR_TO_BTF_ID | PTR_TRUSTED, 5972 PTR_TO_BTF_ID | MEM_RCU, 5973 }, 5974 }; 5975 static const struct bpf_reg_types percpu_btf_ptr_types = { 5976 .types = { 5977 PTR_TO_BTF_ID | MEM_PERCPU, 5978 PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED, 5979 } 5980 }; 5981 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 5982 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 5983 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5984 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 5985 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5986 static const struct bpf_reg_types dynptr_types = { 5987 .types = { 5988 PTR_TO_STACK, 5989 PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL, 5990 } 5991 }; 5992 5993 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 5994 [ARG_PTR_TO_MAP_KEY] = &mem_types, 5995 [ARG_PTR_TO_MAP_VALUE] = &mem_types, 5996 [ARG_CONST_SIZE] = &scalar_types, 5997 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 5998 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 5999 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 6000 [ARG_PTR_TO_CTX] = &context_types, 6001 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 6002 #ifdef CONFIG_NET 6003 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 6004 #endif 6005 [ARG_PTR_TO_SOCKET] = &fullsock_types, 6006 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 6007 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 6008 [ARG_PTR_TO_MEM] = &mem_types, 6009 [ARG_PTR_TO_RINGBUF_MEM] = &ringbuf_mem_types, 6010 [ARG_PTR_TO_INT] = &int_ptr_types, 6011 [ARG_PTR_TO_LONG] = &int_ptr_types, 6012 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 6013 [ARG_PTR_TO_FUNC] = &func_ptr_types, 6014 [ARG_PTR_TO_STACK] = &stack_ptr_types, 6015 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 6016 [ARG_PTR_TO_TIMER] = &timer_types, 6017 [ARG_PTR_TO_KPTR] = &kptr_types, 6018 [ARG_PTR_TO_DYNPTR] = &dynptr_types, 6019 }; 6020 6021 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 6022 enum bpf_arg_type arg_type, 6023 const u32 *arg_btf_id, 6024 struct bpf_call_arg_meta *meta) 6025 { 6026 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6027 enum bpf_reg_type expected, type = reg->type; 6028 const struct bpf_reg_types *compatible; 6029 int i, j; 6030 6031 compatible = compatible_reg_types[base_type(arg_type)]; 6032 if (!compatible) { 6033 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 6034 return -EFAULT; 6035 } 6036 6037 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 6038 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 6039 * 6040 * Same for MAYBE_NULL: 6041 * 6042 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 6043 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 6044 * 6045 * Therefore we fold these flags depending on the arg_type before comparison. 6046 */ 6047 if (arg_type & MEM_RDONLY) 6048 type &= ~MEM_RDONLY; 6049 if (arg_type & PTR_MAYBE_NULL) 6050 type &= ~PTR_MAYBE_NULL; 6051 6052 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 6053 expected = compatible->types[i]; 6054 if (expected == NOT_INIT) 6055 break; 6056 6057 if (type == expected) 6058 goto found; 6059 } 6060 6061 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 6062 for (j = 0; j + 1 < i; j++) 6063 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 6064 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 6065 return -EACCES; 6066 6067 found: 6068 if (reg->type == PTR_TO_BTF_ID || reg->type & PTR_TRUSTED) { 6069 /* For bpf_sk_release, it needs to match against first member 6070 * 'struct sock_common', hence make an exception for it. This 6071 * allows bpf_sk_release to work for multiple socket types. 6072 */ 6073 bool strict_type_match = arg_type_is_release(arg_type) && 6074 meta->func_id != BPF_FUNC_sk_release; 6075 6076 if (!arg_btf_id) { 6077 if (!compatible->btf_id) { 6078 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 6079 return -EFAULT; 6080 } 6081 arg_btf_id = compatible->btf_id; 6082 } 6083 6084 if (meta->func_id == BPF_FUNC_kptr_xchg) { 6085 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) 6086 return -EACCES; 6087 } else { 6088 if (arg_btf_id == BPF_PTR_POISON) { 6089 verbose(env, "verifier internal error:"); 6090 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", 6091 regno); 6092 return -EACCES; 6093 } 6094 6095 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 6096 btf_vmlinux, *arg_btf_id, 6097 strict_type_match)) { 6098 verbose(env, "R%d is of type %s but %s is expected\n", 6099 regno, kernel_type_name(reg->btf, reg->btf_id), 6100 kernel_type_name(btf_vmlinux, *arg_btf_id)); 6101 return -EACCES; 6102 } 6103 } 6104 } else if (type_is_alloc(reg->type)) { 6105 if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock) { 6106 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); 6107 return -EFAULT; 6108 } 6109 } 6110 6111 return 0; 6112 } 6113 6114 int check_func_arg_reg_off(struct bpf_verifier_env *env, 6115 const struct bpf_reg_state *reg, int regno, 6116 enum bpf_arg_type arg_type) 6117 { 6118 enum bpf_reg_type type = reg->type; 6119 bool fixed_off_ok = false; 6120 6121 switch ((u32)type) { 6122 /* Pointer types where reg offset is explicitly allowed: */ 6123 case PTR_TO_STACK: 6124 if (arg_type_is_dynptr(arg_type) && reg->off % BPF_REG_SIZE) { 6125 verbose(env, "cannot pass in dynptr at an offset\n"); 6126 return -EINVAL; 6127 } 6128 fallthrough; 6129 case PTR_TO_PACKET: 6130 case PTR_TO_PACKET_META: 6131 case PTR_TO_MAP_KEY: 6132 case PTR_TO_MAP_VALUE: 6133 case PTR_TO_MEM: 6134 case PTR_TO_MEM | MEM_RDONLY: 6135 case PTR_TO_MEM | MEM_RINGBUF: 6136 case PTR_TO_BUF: 6137 case PTR_TO_BUF | MEM_RDONLY: 6138 case SCALAR_VALUE: 6139 /* Some of the argument types nevertheless require a 6140 * zero register offset. 6141 */ 6142 if (base_type(arg_type) != ARG_PTR_TO_RINGBUF_MEM) 6143 return 0; 6144 break; 6145 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 6146 * fixed offset. 6147 */ 6148 case PTR_TO_BTF_ID: 6149 case PTR_TO_BTF_ID | MEM_ALLOC: 6150 case PTR_TO_BTF_ID | PTR_TRUSTED: 6151 case PTR_TO_BTF_ID | MEM_RCU: 6152 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_TRUSTED: 6153 /* When referenced PTR_TO_BTF_ID is passed to release function, 6154 * it's fixed offset must be 0. In the other cases, fixed offset 6155 * can be non-zero. 6156 */ 6157 if (arg_type_is_release(arg_type) && reg->off) { 6158 verbose(env, "R%d must have zero offset when passed to release func\n", 6159 regno); 6160 return -EINVAL; 6161 } 6162 /* For arg is release pointer, fixed_off_ok must be false, but 6163 * we already checked and rejected reg->off != 0 above, so set 6164 * to true to allow fixed offset for all other cases. 6165 */ 6166 fixed_off_ok = true; 6167 break; 6168 default: 6169 break; 6170 } 6171 return __check_ptr_off_reg(env, reg, regno, fixed_off_ok); 6172 } 6173 6174 static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 6175 { 6176 struct bpf_func_state *state = func(env, reg); 6177 int spi = get_spi(reg->off); 6178 6179 return state->stack[spi].spilled_ptr.id; 6180 } 6181 6182 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 6183 struct bpf_call_arg_meta *meta, 6184 const struct bpf_func_proto *fn) 6185 { 6186 u32 regno = BPF_REG_1 + arg; 6187 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 6188 enum bpf_arg_type arg_type = fn->arg_type[arg]; 6189 enum bpf_reg_type type = reg->type; 6190 u32 *arg_btf_id = NULL; 6191 int err = 0; 6192 6193 if (arg_type == ARG_DONTCARE) 6194 return 0; 6195 6196 err = check_reg_arg(env, regno, SRC_OP); 6197 if (err) 6198 return err; 6199 6200 if (arg_type == ARG_ANYTHING) { 6201 if (is_pointer_value(env, regno)) { 6202 verbose(env, "R%d leaks addr into helper function\n", 6203 regno); 6204 return -EACCES; 6205 } 6206 return 0; 6207 } 6208 6209 if (type_is_pkt_pointer(type) && 6210 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 6211 verbose(env, "helper access to the packet is not allowed\n"); 6212 return -EACCES; 6213 } 6214 6215 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) { 6216 err = resolve_map_arg_type(env, meta, &arg_type); 6217 if (err) 6218 return err; 6219 } 6220 6221 if (register_is_null(reg) && type_may_be_null(arg_type)) 6222 /* A NULL register has a SCALAR_VALUE type, so skip 6223 * type checking. 6224 */ 6225 goto skip_type_check; 6226 6227 /* arg_btf_id and arg_size are in a union. */ 6228 if (base_type(arg_type) == ARG_PTR_TO_BTF_ID || 6229 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) 6230 arg_btf_id = fn->arg_btf_id[arg]; 6231 6232 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); 6233 if (err) 6234 return err; 6235 6236 err = check_func_arg_reg_off(env, reg, regno, arg_type); 6237 if (err) 6238 return err; 6239 6240 skip_type_check: 6241 if (arg_type_is_release(arg_type)) { 6242 if (arg_type_is_dynptr(arg_type)) { 6243 struct bpf_func_state *state = func(env, reg); 6244 int spi = get_spi(reg->off); 6245 6246 if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) || 6247 !state->stack[spi].spilled_ptr.id) { 6248 verbose(env, "arg %d is an unacquired reference\n", regno); 6249 return -EINVAL; 6250 } 6251 } else if (!reg->ref_obj_id && !register_is_null(reg)) { 6252 verbose(env, "R%d must be referenced when passed to release function\n", 6253 regno); 6254 return -EINVAL; 6255 } 6256 if (meta->release_regno) { 6257 verbose(env, "verifier internal error: more than one release argument\n"); 6258 return -EFAULT; 6259 } 6260 meta->release_regno = regno; 6261 } 6262 6263 if (reg->ref_obj_id) { 6264 if (meta->ref_obj_id) { 6265 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 6266 regno, reg->ref_obj_id, 6267 meta->ref_obj_id); 6268 return -EFAULT; 6269 } 6270 meta->ref_obj_id = reg->ref_obj_id; 6271 } 6272 6273 switch (base_type(arg_type)) { 6274 case ARG_CONST_MAP_PTR: 6275 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 6276 if (meta->map_ptr) { 6277 /* Use map_uid (which is unique id of inner map) to reject: 6278 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 6279 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 6280 * if (inner_map1 && inner_map2) { 6281 * timer = bpf_map_lookup_elem(inner_map1); 6282 * if (timer) 6283 * // mismatch would have been allowed 6284 * bpf_timer_init(timer, inner_map2); 6285 * } 6286 * 6287 * Comparing map_ptr is enough to distinguish normal and outer maps. 6288 */ 6289 if (meta->map_ptr != reg->map_ptr || 6290 meta->map_uid != reg->map_uid) { 6291 verbose(env, 6292 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 6293 meta->map_uid, reg->map_uid); 6294 return -EINVAL; 6295 } 6296 } 6297 meta->map_ptr = reg->map_ptr; 6298 meta->map_uid = reg->map_uid; 6299 break; 6300 case ARG_PTR_TO_MAP_KEY: 6301 /* bpf_map_xxx(..., map_ptr, ..., key) call: 6302 * check that [key, key + map->key_size) are within 6303 * stack limits and initialized 6304 */ 6305 if (!meta->map_ptr) { 6306 /* in function declaration map_ptr must come before 6307 * map_key, so that it's verified and known before 6308 * we have to check map_key here. Otherwise it means 6309 * that kernel subsystem misconfigured verifier 6310 */ 6311 verbose(env, "invalid map_ptr to access map->key\n"); 6312 return -EACCES; 6313 } 6314 err = check_helper_mem_access(env, regno, 6315 meta->map_ptr->key_size, false, 6316 NULL); 6317 break; 6318 case ARG_PTR_TO_MAP_VALUE: 6319 if (type_may_be_null(arg_type) && register_is_null(reg)) 6320 return 0; 6321 6322 /* bpf_map_xxx(..., map_ptr, ..., value) call: 6323 * check [value, value + map->value_size) validity 6324 */ 6325 if (!meta->map_ptr) { 6326 /* kernel subsystem misconfigured verifier */ 6327 verbose(env, "invalid map_ptr to access map->value\n"); 6328 return -EACCES; 6329 } 6330 meta->raw_mode = arg_type & MEM_UNINIT; 6331 err = check_helper_mem_access(env, regno, 6332 meta->map_ptr->value_size, false, 6333 meta); 6334 break; 6335 case ARG_PTR_TO_PERCPU_BTF_ID: 6336 if (!reg->btf_id) { 6337 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 6338 return -EACCES; 6339 } 6340 meta->ret_btf = reg->btf; 6341 meta->ret_btf_id = reg->btf_id; 6342 break; 6343 case ARG_PTR_TO_SPIN_LOCK: 6344 if (meta->func_id == BPF_FUNC_spin_lock) { 6345 if (process_spin_lock(env, regno, true)) 6346 return -EACCES; 6347 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 6348 if (process_spin_lock(env, regno, false)) 6349 return -EACCES; 6350 } else { 6351 verbose(env, "verifier internal error\n"); 6352 return -EFAULT; 6353 } 6354 break; 6355 case ARG_PTR_TO_TIMER: 6356 if (process_timer_func(env, regno, meta)) 6357 return -EACCES; 6358 break; 6359 case ARG_PTR_TO_FUNC: 6360 meta->subprogno = reg->subprogno; 6361 break; 6362 case ARG_PTR_TO_MEM: 6363 /* The access to this pointer is only checked when we hit the 6364 * next is_mem_size argument below. 6365 */ 6366 meta->raw_mode = arg_type & MEM_UNINIT; 6367 if (arg_type & MEM_FIXED_SIZE) { 6368 err = check_helper_mem_access(env, regno, 6369 fn->arg_size[arg], false, 6370 meta); 6371 } 6372 break; 6373 case ARG_CONST_SIZE: 6374 err = check_mem_size_reg(env, reg, regno, false, meta); 6375 break; 6376 case ARG_CONST_SIZE_OR_ZERO: 6377 err = check_mem_size_reg(env, reg, regno, true, meta); 6378 break; 6379 case ARG_PTR_TO_DYNPTR: 6380 /* We only need to check for initialized / uninitialized helper 6381 * dynptr args if the dynptr is not PTR_TO_DYNPTR, as the 6382 * assumption is that if it is, that a helper function 6383 * initialized the dynptr on behalf of the BPF program. 6384 */ 6385 if (base_type(reg->type) == PTR_TO_DYNPTR) 6386 break; 6387 if (arg_type & MEM_UNINIT) { 6388 if (!is_dynptr_reg_valid_uninit(env, reg)) { 6389 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); 6390 return -EINVAL; 6391 } 6392 6393 /* We only support one dynptr being uninitialized at the moment, 6394 * which is sufficient for the helper functions we have right now. 6395 */ 6396 if (meta->uninit_dynptr_regno) { 6397 verbose(env, "verifier internal error: multiple uninitialized dynptr args\n"); 6398 return -EFAULT; 6399 } 6400 6401 meta->uninit_dynptr_regno = regno; 6402 } else if (!is_dynptr_reg_valid_init(env, reg)) { 6403 verbose(env, 6404 "Expected an initialized dynptr as arg #%d\n", 6405 arg + 1); 6406 return -EINVAL; 6407 } else if (!is_dynptr_type_expected(env, reg, arg_type)) { 6408 const char *err_extra = ""; 6409 6410 switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { 6411 case DYNPTR_TYPE_LOCAL: 6412 err_extra = "local"; 6413 break; 6414 case DYNPTR_TYPE_RINGBUF: 6415 err_extra = "ringbuf"; 6416 break; 6417 default: 6418 err_extra = "<unknown>"; 6419 break; 6420 } 6421 verbose(env, 6422 "Expected a dynptr of type %s as arg #%d\n", 6423 err_extra, arg + 1); 6424 return -EINVAL; 6425 } 6426 break; 6427 case ARG_CONST_ALLOC_SIZE_OR_ZERO: 6428 if (!tnum_is_const(reg->var_off)) { 6429 verbose(env, "R%d is not a known constant'\n", 6430 regno); 6431 return -EACCES; 6432 } 6433 meta->mem_size = reg->var_off.value; 6434 err = mark_chain_precision(env, regno); 6435 if (err) 6436 return err; 6437 break; 6438 case ARG_PTR_TO_INT: 6439 case ARG_PTR_TO_LONG: 6440 { 6441 int size = int_ptr_type_to_size(arg_type); 6442 6443 err = check_helper_mem_access(env, regno, size, false, meta); 6444 if (err) 6445 return err; 6446 err = check_ptr_alignment(env, reg, 0, size, true); 6447 break; 6448 } 6449 case ARG_PTR_TO_CONST_STR: 6450 { 6451 struct bpf_map *map = reg->map_ptr; 6452 int map_off; 6453 u64 map_addr; 6454 char *str_ptr; 6455 6456 if (!bpf_map_is_rdonly(map)) { 6457 verbose(env, "R%d does not point to a readonly map'\n", regno); 6458 return -EACCES; 6459 } 6460 6461 if (!tnum_is_const(reg->var_off)) { 6462 verbose(env, "R%d is not a constant address'\n", regno); 6463 return -EACCES; 6464 } 6465 6466 if (!map->ops->map_direct_value_addr) { 6467 verbose(env, "no direct value access support for this map type\n"); 6468 return -EACCES; 6469 } 6470 6471 err = check_map_access(env, regno, reg->off, 6472 map->value_size - reg->off, false, 6473 ACCESS_HELPER); 6474 if (err) 6475 return err; 6476 6477 map_off = reg->off + reg->var_off.value; 6478 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 6479 if (err) { 6480 verbose(env, "direct value access on string failed\n"); 6481 return err; 6482 } 6483 6484 str_ptr = (char *)(long)(map_addr); 6485 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 6486 verbose(env, "string is not zero-terminated\n"); 6487 return -EINVAL; 6488 } 6489 break; 6490 } 6491 case ARG_PTR_TO_KPTR: 6492 if (process_kptr_func(env, regno, meta)) 6493 return -EACCES; 6494 break; 6495 } 6496 6497 return err; 6498 } 6499 6500 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 6501 { 6502 enum bpf_attach_type eatype = env->prog->expected_attach_type; 6503 enum bpf_prog_type type = resolve_prog_type(env->prog); 6504 6505 if (func_id != BPF_FUNC_map_update_elem) 6506 return false; 6507 6508 /* It's not possible to get access to a locked struct sock in these 6509 * contexts, so updating is safe. 6510 */ 6511 switch (type) { 6512 case BPF_PROG_TYPE_TRACING: 6513 if (eatype == BPF_TRACE_ITER) 6514 return true; 6515 break; 6516 case BPF_PROG_TYPE_SOCKET_FILTER: 6517 case BPF_PROG_TYPE_SCHED_CLS: 6518 case BPF_PROG_TYPE_SCHED_ACT: 6519 case BPF_PROG_TYPE_XDP: 6520 case BPF_PROG_TYPE_SK_REUSEPORT: 6521 case BPF_PROG_TYPE_FLOW_DISSECTOR: 6522 case BPF_PROG_TYPE_SK_LOOKUP: 6523 return true; 6524 default: 6525 break; 6526 } 6527 6528 verbose(env, "cannot update sockmap in this context\n"); 6529 return false; 6530 } 6531 6532 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 6533 { 6534 return env->prog->jit_requested && 6535 bpf_jit_supports_subprog_tailcalls(); 6536 } 6537 6538 static int check_map_func_compatibility(struct bpf_verifier_env *env, 6539 struct bpf_map *map, int func_id) 6540 { 6541 if (!map) 6542 return 0; 6543 6544 /* We need a two way check, first is from map perspective ... */ 6545 switch (map->map_type) { 6546 case BPF_MAP_TYPE_PROG_ARRAY: 6547 if (func_id != BPF_FUNC_tail_call) 6548 goto error; 6549 break; 6550 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 6551 if (func_id != BPF_FUNC_perf_event_read && 6552 func_id != BPF_FUNC_perf_event_output && 6553 func_id != BPF_FUNC_skb_output && 6554 func_id != BPF_FUNC_perf_event_read_value && 6555 func_id != BPF_FUNC_xdp_output) 6556 goto error; 6557 break; 6558 case BPF_MAP_TYPE_RINGBUF: 6559 if (func_id != BPF_FUNC_ringbuf_output && 6560 func_id != BPF_FUNC_ringbuf_reserve && 6561 func_id != BPF_FUNC_ringbuf_query && 6562 func_id != BPF_FUNC_ringbuf_reserve_dynptr && 6563 func_id != BPF_FUNC_ringbuf_submit_dynptr && 6564 func_id != BPF_FUNC_ringbuf_discard_dynptr) 6565 goto error; 6566 break; 6567 case BPF_MAP_TYPE_USER_RINGBUF: 6568 if (func_id != BPF_FUNC_user_ringbuf_drain) 6569 goto error; 6570 break; 6571 case BPF_MAP_TYPE_STACK_TRACE: 6572 if (func_id != BPF_FUNC_get_stackid) 6573 goto error; 6574 break; 6575 case BPF_MAP_TYPE_CGROUP_ARRAY: 6576 if (func_id != BPF_FUNC_skb_under_cgroup && 6577 func_id != BPF_FUNC_current_task_under_cgroup) 6578 goto error; 6579 break; 6580 case BPF_MAP_TYPE_CGROUP_STORAGE: 6581 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 6582 if (func_id != BPF_FUNC_get_local_storage) 6583 goto error; 6584 break; 6585 case BPF_MAP_TYPE_DEVMAP: 6586 case BPF_MAP_TYPE_DEVMAP_HASH: 6587 if (func_id != BPF_FUNC_redirect_map && 6588 func_id != BPF_FUNC_map_lookup_elem) 6589 goto error; 6590 break; 6591 /* Restrict bpf side of cpumap and xskmap, open when use-cases 6592 * appear. 6593 */ 6594 case BPF_MAP_TYPE_CPUMAP: 6595 if (func_id != BPF_FUNC_redirect_map) 6596 goto error; 6597 break; 6598 case BPF_MAP_TYPE_XSKMAP: 6599 if (func_id != BPF_FUNC_redirect_map && 6600 func_id != BPF_FUNC_map_lookup_elem) 6601 goto error; 6602 break; 6603 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 6604 case BPF_MAP_TYPE_HASH_OF_MAPS: 6605 if (func_id != BPF_FUNC_map_lookup_elem) 6606 goto error; 6607 break; 6608 case BPF_MAP_TYPE_SOCKMAP: 6609 if (func_id != BPF_FUNC_sk_redirect_map && 6610 func_id != BPF_FUNC_sock_map_update && 6611 func_id != BPF_FUNC_map_delete_elem && 6612 func_id != BPF_FUNC_msg_redirect_map && 6613 func_id != BPF_FUNC_sk_select_reuseport && 6614 func_id != BPF_FUNC_map_lookup_elem && 6615 !may_update_sockmap(env, func_id)) 6616 goto error; 6617 break; 6618 case BPF_MAP_TYPE_SOCKHASH: 6619 if (func_id != BPF_FUNC_sk_redirect_hash && 6620 func_id != BPF_FUNC_sock_hash_update && 6621 func_id != BPF_FUNC_map_delete_elem && 6622 func_id != BPF_FUNC_msg_redirect_hash && 6623 func_id != BPF_FUNC_sk_select_reuseport && 6624 func_id != BPF_FUNC_map_lookup_elem && 6625 !may_update_sockmap(env, func_id)) 6626 goto error; 6627 break; 6628 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 6629 if (func_id != BPF_FUNC_sk_select_reuseport) 6630 goto error; 6631 break; 6632 case BPF_MAP_TYPE_QUEUE: 6633 case BPF_MAP_TYPE_STACK: 6634 if (func_id != BPF_FUNC_map_peek_elem && 6635 func_id != BPF_FUNC_map_pop_elem && 6636 func_id != BPF_FUNC_map_push_elem) 6637 goto error; 6638 break; 6639 case BPF_MAP_TYPE_SK_STORAGE: 6640 if (func_id != BPF_FUNC_sk_storage_get && 6641 func_id != BPF_FUNC_sk_storage_delete) 6642 goto error; 6643 break; 6644 case BPF_MAP_TYPE_INODE_STORAGE: 6645 if (func_id != BPF_FUNC_inode_storage_get && 6646 func_id != BPF_FUNC_inode_storage_delete) 6647 goto error; 6648 break; 6649 case BPF_MAP_TYPE_TASK_STORAGE: 6650 if (func_id != BPF_FUNC_task_storage_get && 6651 func_id != BPF_FUNC_task_storage_delete) 6652 goto error; 6653 break; 6654 case BPF_MAP_TYPE_CGRP_STORAGE: 6655 if (func_id != BPF_FUNC_cgrp_storage_get && 6656 func_id != BPF_FUNC_cgrp_storage_delete) 6657 goto error; 6658 break; 6659 case BPF_MAP_TYPE_BLOOM_FILTER: 6660 if (func_id != BPF_FUNC_map_peek_elem && 6661 func_id != BPF_FUNC_map_push_elem) 6662 goto error; 6663 break; 6664 default: 6665 break; 6666 } 6667 6668 /* ... and second from the function itself. */ 6669 switch (func_id) { 6670 case BPF_FUNC_tail_call: 6671 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 6672 goto error; 6673 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 6674 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 6675 return -EINVAL; 6676 } 6677 break; 6678 case BPF_FUNC_perf_event_read: 6679 case BPF_FUNC_perf_event_output: 6680 case BPF_FUNC_perf_event_read_value: 6681 case BPF_FUNC_skb_output: 6682 case BPF_FUNC_xdp_output: 6683 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 6684 goto error; 6685 break; 6686 case BPF_FUNC_ringbuf_output: 6687 case BPF_FUNC_ringbuf_reserve: 6688 case BPF_FUNC_ringbuf_query: 6689 case BPF_FUNC_ringbuf_reserve_dynptr: 6690 case BPF_FUNC_ringbuf_submit_dynptr: 6691 case BPF_FUNC_ringbuf_discard_dynptr: 6692 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 6693 goto error; 6694 break; 6695 case BPF_FUNC_user_ringbuf_drain: 6696 if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) 6697 goto error; 6698 break; 6699 case BPF_FUNC_get_stackid: 6700 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 6701 goto error; 6702 break; 6703 case BPF_FUNC_current_task_under_cgroup: 6704 case BPF_FUNC_skb_under_cgroup: 6705 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 6706 goto error; 6707 break; 6708 case BPF_FUNC_redirect_map: 6709 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 6710 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 6711 map->map_type != BPF_MAP_TYPE_CPUMAP && 6712 map->map_type != BPF_MAP_TYPE_XSKMAP) 6713 goto error; 6714 break; 6715 case BPF_FUNC_sk_redirect_map: 6716 case BPF_FUNC_msg_redirect_map: 6717 case BPF_FUNC_sock_map_update: 6718 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 6719 goto error; 6720 break; 6721 case BPF_FUNC_sk_redirect_hash: 6722 case BPF_FUNC_msg_redirect_hash: 6723 case BPF_FUNC_sock_hash_update: 6724 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 6725 goto error; 6726 break; 6727 case BPF_FUNC_get_local_storage: 6728 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 6729 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 6730 goto error; 6731 break; 6732 case BPF_FUNC_sk_select_reuseport: 6733 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 6734 map->map_type != BPF_MAP_TYPE_SOCKMAP && 6735 map->map_type != BPF_MAP_TYPE_SOCKHASH) 6736 goto error; 6737 break; 6738 case BPF_FUNC_map_pop_elem: 6739 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6740 map->map_type != BPF_MAP_TYPE_STACK) 6741 goto error; 6742 break; 6743 case BPF_FUNC_map_peek_elem: 6744 case BPF_FUNC_map_push_elem: 6745 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6746 map->map_type != BPF_MAP_TYPE_STACK && 6747 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 6748 goto error; 6749 break; 6750 case BPF_FUNC_map_lookup_percpu_elem: 6751 if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY && 6752 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 6753 map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH) 6754 goto error; 6755 break; 6756 case BPF_FUNC_sk_storage_get: 6757 case BPF_FUNC_sk_storage_delete: 6758 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 6759 goto error; 6760 break; 6761 case BPF_FUNC_inode_storage_get: 6762 case BPF_FUNC_inode_storage_delete: 6763 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 6764 goto error; 6765 break; 6766 case BPF_FUNC_task_storage_get: 6767 case BPF_FUNC_task_storage_delete: 6768 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 6769 goto error; 6770 break; 6771 case BPF_FUNC_cgrp_storage_get: 6772 case BPF_FUNC_cgrp_storage_delete: 6773 if (map->map_type != BPF_MAP_TYPE_CGRP_STORAGE) 6774 goto error; 6775 break; 6776 default: 6777 break; 6778 } 6779 6780 return 0; 6781 error: 6782 verbose(env, "cannot pass map_type %d into func %s#%d\n", 6783 map->map_type, func_id_name(func_id), func_id); 6784 return -EINVAL; 6785 } 6786 6787 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 6788 { 6789 int count = 0; 6790 6791 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 6792 count++; 6793 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 6794 count++; 6795 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 6796 count++; 6797 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 6798 count++; 6799 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 6800 count++; 6801 6802 /* We only support one arg being in raw mode at the moment, 6803 * which is sufficient for the helper functions we have 6804 * right now. 6805 */ 6806 return count <= 1; 6807 } 6808 6809 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) 6810 { 6811 bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE; 6812 bool has_size = fn->arg_size[arg] != 0; 6813 bool is_next_size = false; 6814 6815 if (arg + 1 < ARRAY_SIZE(fn->arg_type)) 6816 is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]); 6817 6818 if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM) 6819 return is_next_size; 6820 6821 return has_size == is_next_size || is_next_size == is_fixed; 6822 } 6823 6824 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 6825 { 6826 /* bpf_xxx(..., buf, len) call will access 'len' 6827 * bytes from memory 'buf'. Both arg types need 6828 * to be paired, so make sure there's no buggy 6829 * helper function specification. 6830 */ 6831 if (arg_type_is_mem_size(fn->arg1_type) || 6832 check_args_pair_invalid(fn, 0) || 6833 check_args_pair_invalid(fn, 1) || 6834 check_args_pair_invalid(fn, 2) || 6835 check_args_pair_invalid(fn, 3) || 6836 check_args_pair_invalid(fn, 4)) 6837 return false; 6838 6839 return true; 6840 } 6841 6842 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 6843 { 6844 int i; 6845 6846 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 6847 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) 6848 return !!fn->arg_btf_id[i]; 6849 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) 6850 return fn->arg_btf_id[i] == BPF_PTR_POISON; 6851 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] && 6852 /* arg_btf_id and arg_size are in a union. */ 6853 (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM || 6854 !(fn->arg_type[i] & MEM_FIXED_SIZE))) 6855 return false; 6856 } 6857 6858 return true; 6859 } 6860 6861 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 6862 { 6863 return check_raw_mode_ok(fn) && 6864 check_arg_pair_ok(fn) && 6865 check_btf_id_ok(fn) ? 0 : -EINVAL; 6866 } 6867 6868 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 6869 * are now invalid, so turn them into unknown SCALAR_VALUE. 6870 */ 6871 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 6872 { 6873 struct bpf_func_state *state; 6874 struct bpf_reg_state *reg; 6875 6876 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 6877 if (reg_is_pkt_pointer_any(reg)) 6878 __mark_reg_unknown(env, reg); 6879 })); 6880 } 6881 6882 enum { 6883 AT_PKT_END = -1, 6884 BEYOND_PKT_END = -2, 6885 }; 6886 6887 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 6888 { 6889 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6890 struct bpf_reg_state *reg = &state->regs[regn]; 6891 6892 if (reg->type != PTR_TO_PACKET) 6893 /* PTR_TO_PACKET_META is not supported yet */ 6894 return; 6895 6896 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 6897 * How far beyond pkt_end it goes is unknown. 6898 * if (!range_open) it's the case of pkt >= pkt_end 6899 * if (range_open) it's the case of pkt > pkt_end 6900 * hence this pointer is at least 1 byte bigger than pkt_end 6901 */ 6902 if (range_open) 6903 reg->range = BEYOND_PKT_END; 6904 else 6905 reg->range = AT_PKT_END; 6906 } 6907 6908 /* The pointer with the specified id has released its reference to kernel 6909 * resources. Identify all copies of the same pointer and clear the reference. 6910 */ 6911 static int release_reference(struct bpf_verifier_env *env, 6912 int ref_obj_id) 6913 { 6914 struct bpf_func_state *state; 6915 struct bpf_reg_state *reg; 6916 int err; 6917 6918 err = release_reference_state(cur_func(env), ref_obj_id); 6919 if (err) 6920 return err; 6921 6922 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 6923 if (reg->ref_obj_id == ref_obj_id) { 6924 if (!env->allow_ptr_leaks) 6925 __mark_reg_not_init(env, reg); 6926 else 6927 __mark_reg_unknown(env, reg); 6928 } 6929 })); 6930 6931 return 0; 6932 } 6933 6934 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 6935 struct bpf_reg_state *regs) 6936 { 6937 int i; 6938 6939 /* after the call registers r0 - r5 were scratched */ 6940 for (i = 0; i < CALLER_SAVED_REGS; i++) { 6941 mark_reg_not_init(env, regs, caller_saved[i]); 6942 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 6943 } 6944 } 6945 6946 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 6947 struct bpf_func_state *caller, 6948 struct bpf_func_state *callee, 6949 int insn_idx); 6950 6951 static int set_callee_state(struct bpf_verifier_env *env, 6952 struct bpf_func_state *caller, 6953 struct bpf_func_state *callee, int insn_idx); 6954 6955 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6956 int *insn_idx, int subprog, 6957 set_callee_state_fn set_callee_state_cb) 6958 { 6959 struct bpf_verifier_state *state = env->cur_state; 6960 struct bpf_func_info_aux *func_info_aux; 6961 struct bpf_func_state *caller, *callee; 6962 int err; 6963 bool is_global = false; 6964 6965 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 6966 verbose(env, "the call stack of %d frames is too deep\n", 6967 state->curframe + 2); 6968 return -E2BIG; 6969 } 6970 6971 caller = state->frame[state->curframe]; 6972 if (state->frame[state->curframe + 1]) { 6973 verbose(env, "verifier bug. Frame %d already allocated\n", 6974 state->curframe + 1); 6975 return -EFAULT; 6976 } 6977 6978 func_info_aux = env->prog->aux->func_info_aux; 6979 if (func_info_aux) 6980 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 6981 err = btf_check_subprog_call(env, subprog, caller->regs); 6982 if (err == -EFAULT) 6983 return err; 6984 if (is_global) { 6985 if (err) { 6986 verbose(env, "Caller passes invalid args into func#%d\n", 6987 subprog); 6988 return err; 6989 } else { 6990 if (env->log.level & BPF_LOG_LEVEL) 6991 verbose(env, 6992 "Func#%d is global and valid. Skipping.\n", 6993 subprog); 6994 clear_caller_saved_regs(env, caller->regs); 6995 6996 /* All global functions return a 64-bit SCALAR_VALUE */ 6997 mark_reg_unknown(env, caller->regs, BPF_REG_0); 6998 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 6999 7000 /* continue with next insn after call */ 7001 return 0; 7002 } 7003 } 7004 7005 /* set_callee_state is used for direct subprog calls, but we are 7006 * interested in validating only BPF helpers that can call subprogs as 7007 * callbacks 7008 */ 7009 if (set_callee_state_cb != set_callee_state && !is_callback_calling_function(insn->imm)) { 7010 verbose(env, "verifier bug: helper %s#%d is not marked as callback-calling\n", 7011 func_id_name(insn->imm), insn->imm); 7012 return -EFAULT; 7013 } 7014 7015 if (insn->code == (BPF_JMP | BPF_CALL) && 7016 insn->src_reg == 0 && 7017 insn->imm == BPF_FUNC_timer_set_callback) { 7018 struct bpf_verifier_state *async_cb; 7019 7020 /* there is no real recursion here. timer callbacks are async */ 7021 env->subprog_info[subprog].is_async_cb = true; 7022 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 7023 *insn_idx, subprog); 7024 if (!async_cb) 7025 return -EFAULT; 7026 callee = async_cb->frame[0]; 7027 callee->async_entry_cnt = caller->async_entry_cnt + 1; 7028 7029 /* Convert bpf_timer_set_callback() args into timer callback args */ 7030 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7031 if (err) 7032 return err; 7033 7034 clear_caller_saved_regs(env, caller->regs); 7035 mark_reg_unknown(env, caller->regs, BPF_REG_0); 7036 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7037 /* continue with next insn after call */ 7038 return 0; 7039 } 7040 7041 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 7042 if (!callee) 7043 return -ENOMEM; 7044 state->frame[state->curframe + 1] = callee; 7045 7046 /* callee cannot access r0, r6 - r9 for reading and has to write 7047 * into its own stack before reading from it. 7048 * callee can read/write into caller's stack 7049 */ 7050 init_func_state(env, callee, 7051 /* remember the callsite, it will be used by bpf_exit */ 7052 *insn_idx /* callsite */, 7053 state->curframe + 1 /* frameno within this callchain */, 7054 subprog /* subprog number within this prog */); 7055 7056 /* Transfer references to the callee */ 7057 err = copy_reference_state(callee, caller); 7058 if (err) 7059 goto err_out; 7060 7061 err = set_callee_state_cb(env, caller, callee, *insn_idx); 7062 if (err) 7063 goto err_out; 7064 7065 clear_caller_saved_regs(env, caller->regs); 7066 7067 /* only increment it after check_reg_arg() finished */ 7068 state->curframe++; 7069 7070 /* and go analyze first insn of the callee */ 7071 *insn_idx = env->subprog_info[subprog].start - 1; 7072 7073 if (env->log.level & BPF_LOG_LEVEL) { 7074 verbose(env, "caller:\n"); 7075 print_verifier_state(env, caller, true); 7076 verbose(env, "callee:\n"); 7077 print_verifier_state(env, callee, true); 7078 } 7079 return 0; 7080 7081 err_out: 7082 free_func_state(callee); 7083 state->frame[state->curframe + 1] = NULL; 7084 return err; 7085 } 7086 7087 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 7088 struct bpf_func_state *caller, 7089 struct bpf_func_state *callee) 7090 { 7091 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 7092 * void *callback_ctx, u64 flags); 7093 * callback_fn(struct bpf_map *map, void *key, void *value, 7094 * void *callback_ctx); 7095 */ 7096 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7097 7098 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7099 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7100 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7101 7102 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7103 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7104 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 7105 7106 /* pointer to stack or null */ 7107 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 7108 7109 /* unused */ 7110 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7111 return 0; 7112 } 7113 7114 static int set_callee_state(struct bpf_verifier_env *env, 7115 struct bpf_func_state *caller, 7116 struct bpf_func_state *callee, int insn_idx) 7117 { 7118 int i; 7119 7120 /* copy r1 - r5 args that callee can access. The copy includes parent 7121 * pointers, which connects us up to the liveness chain 7122 */ 7123 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 7124 callee->regs[i] = caller->regs[i]; 7125 return 0; 7126 } 7127 7128 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7129 int *insn_idx) 7130 { 7131 int subprog, target_insn; 7132 7133 target_insn = *insn_idx + insn->imm + 1; 7134 subprog = find_subprog(env, target_insn); 7135 if (subprog < 0) { 7136 verbose(env, "verifier bug. No program starts at insn %d\n", 7137 target_insn); 7138 return -EFAULT; 7139 } 7140 7141 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 7142 } 7143 7144 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 7145 struct bpf_func_state *caller, 7146 struct bpf_func_state *callee, 7147 int insn_idx) 7148 { 7149 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 7150 struct bpf_map *map; 7151 int err; 7152 7153 if (bpf_map_ptr_poisoned(insn_aux)) { 7154 verbose(env, "tail_call abusing map_ptr\n"); 7155 return -EINVAL; 7156 } 7157 7158 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 7159 if (!map->ops->map_set_for_each_callback_args || 7160 !map->ops->map_for_each_callback) { 7161 verbose(env, "callback function not allowed for map\n"); 7162 return -ENOTSUPP; 7163 } 7164 7165 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 7166 if (err) 7167 return err; 7168 7169 callee->in_callback_fn = true; 7170 callee->callback_ret_range = tnum_range(0, 1); 7171 return 0; 7172 } 7173 7174 static int set_loop_callback_state(struct bpf_verifier_env *env, 7175 struct bpf_func_state *caller, 7176 struct bpf_func_state *callee, 7177 int insn_idx) 7178 { 7179 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 7180 * u64 flags); 7181 * callback_fn(u32 index, void *callback_ctx); 7182 */ 7183 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 7184 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7185 7186 /* unused */ 7187 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7188 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7189 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7190 7191 callee->in_callback_fn = true; 7192 callee->callback_ret_range = tnum_range(0, 1); 7193 return 0; 7194 } 7195 7196 static int set_timer_callback_state(struct bpf_verifier_env *env, 7197 struct bpf_func_state *caller, 7198 struct bpf_func_state *callee, 7199 int insn_idx) 7200 { 7201 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 7202 7203 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 7204 * callback_fn(struct bpf_map *map, void *key, void *value); 7205 */ 7206 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 7207 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 7208 callee->regs[BPF_REG_1].map_ptr = map_ptr; 7209 7210 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 7211 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7212 callee->regs[BPF_REG_2].map_ptr = map_ptr; 7213 7214 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 7215 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 7216 callee->regs[BPF_REG_3].map_ptr = map_ptr; 7217 7218 /* unused */ 7219 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7220 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7221 callee->in_async_callback_fn = true; 7222 callee->callback_ret_range = tnum_range(0, 1); 7223 return 0; 7224 } 7225 7226 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 7227 struct bpf_func_state *caller, 7228 struct bpf_func_state *callee, 7229 int insn_idx) 7230 { 7231 /* bpf_find_vma(struct task_struct *task, u64 addr, 7232 * void *callback_fn, void *callback_ctx, u64 flags) 7233 * (callback_fn)(struct task_struct *task, 7234 * struct vm_area_struct *vma, void *callback_ctx); 7235 */ 7236 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 7237 7238 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 7239 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 7240 callee->regs[BPF_REG_2].btf = btf_vmlinux; 7241 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], 7242 7243 /* pointer to stack or null */ 7244 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 7245 7246 /* unused */ 7247 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7248 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7249 callee->in_callback_fn = true; 7250 callee->callback_ret_range = tnum_range(0, 1); 7251 return 0; 7252 } 7253 7254 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, 7255 struct bpf_func_state *caller, 7256 struct bpf_func_state *callee, 7257 int insn_idx) 7258 { 7259 /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void 7260 * callback_ctx, u64 flags); 7261 * callback_fn(struct bpf_dynptr_t* dynptr, void *callback_ctx); 7262 */ 7263 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); 7264 callee->regs[BPF_REG_1].type = PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL; 7265 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 7266 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 7267 7268 /* unused */ 7269 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 7270 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 7271 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 7272 7273 callee->in_callback_fn = true; 7274 callee->callback_ret_range = tnum_range(0, 1); 7275 return 0; 7276 } 7277 7278 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 7279 { 7280 struct bpf_verifier_state *state = env->cur_state; 7281 struct bpf_func_state *caller, *callee; 7282 struct bpf_reg_state *r0; 7283 int err; 7284 7285 callee = state->frame[state->curframe]; 7286 r0 = &callee->regs[BPF_REG_0]; 7287 if (r0->type == PTR_TO_STACK) { 7288 /* technically it's ok to return caller's stack pointer 7289 * (or caller's caller's pointer) back to the caller, 7290 * since these pointers are valid. Only current stack 7291 * pointer will be invalid as soon as function exits, 7292 * but let's be conservative 7293 */ 7294 verbose(env, "cannot return stack pointer to the caller\n"); 7295 return -EINVAL; 7296 } 7297 7298 caller = state->frame[state->curframe - 1]; 7299 if (callee->in_callback_fn) { 7300 /* enforce R0 return value range [0, 1]. */ 7301 struct tnum range = callee->callback_ret_range; 7302 7303 if (r0->type != SCALAR_VALUE) { 7304 verbose(env, "R0 not a scalar value\n"); 7305 return -EACCES; 7306 } 7307 if (!tnum_in(range, r0->var_off)) { 7308 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 7309 return -EINVAL; 7310 } 7311 } else { 7312 /* return to the caller whatever r0 had in the callee */ 7313 caller->regs[BPF_REG_0] = *r0; 7314 } 7315 7316 /* callback_fn frame should have released its own additions to parent's 7317 * reference state at this point, or check_reference_leak would 7318 * complain, hence it must be the same as the caller. There is no need 7319 * to copy it back. 7320 */ 7321 if (!callee->in_callback_fn) { 7322 /* Transfer references to the caller */ 7323 err = copy_reference_state(caller, callee); 7324 if (err) 7325 return err; 7326 } 7327 7328 *insn_idx = callee->callsite + 1; 7329 if (env->log.level & BPF_LOG_LEVEL) { 7330 verbose(env, "returning from callee:\n"); 7331 print_verifier_state(env, callee, true); 7332 verbose(env, "to caller at %d:\n", *insn_idx); 7333 print_verifier_state(env, caller, true); 7334 } 7335 /* clear everything in the callee */ 7336 free_func_state(callee); 7337 state->frame[state->curframe--] = NULL; 7338 return 0; 7339 } 7340 7341 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 7342 int func_id, 7343 struct bpf_call_arg_meta *meta) 7344 { 7345 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 7346 7347 if (ret_type != RET_INTEGER || 7348 (func_id != BPF_FUNC_get_stack && 7349 func_id != BPF_FUNC_get_task_stack && 7350 func_id != BPF_FUNC_probe_read_str && 7351 func_id != BPF_FUNC_probe_read_kernel_str && 7352 func_id != BPF_FUNC_probe_read_user_str)) 7353 return; 7354 7355 ret_reg->smax_value = meta->msize_max_value; 7356 ret_reg->s32_max_value = meta->msize_max_value; 7357 ret_reg->smin_value = -MAX_ERRNO; 7358 ret_reg->s32_min_value = -MAX_ERRNO; 7359 reg_bounds_sync(ret_reg); 7360 } 7361 7362 static int 7363 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 7364 int func_id, int insn_idx) 7365 { 7366 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 7367 struct bpf_map *map = meta->map_ptr; 7368 7369 if (func_id != BPF_FUNC_tail_call && 7370 func_id != BPF_FUNC_map_lookup_elem && 7371 func_id != BPF_FUNC_map_update_elem && 7372 func_id != BPF_FUNC_map_delete_elem && 7373 func_id != BPF_FUNC_map_push_elem && 7374 func_id != BPF_FUNC_map_pop_elem && 7375 func_id != BPF_FUNC_map_peek_elem && 7376 func_id != BPF_FUNC_for_each_map_elem && 7377 func_id != BPF_FUNC_redirect_map && 7378 func_id != BPF_FUNC_map_lookup_percpu_elem) 7379 return 0; 7380 7381 if (map == NULL) { 7382 verbose(env, "kernel subsystem misconfigured verifier\n"); 7383 return -EINVAL; 7384 } 7385 7386 /* In case of read-only, some additional restrictions 7387 * need to be applied in order to prevent altering the 7388 * state of the map from program side. 7389 */ 7390 if ((map->map_flags & BPF_F_RDONLY_PROG) && 7391 (func_id == BPF_FUNC_map_delete_elem || 7392 func_id == BPF_FUNC_map_update_elem || 7393 func_id == BPF_FUNC_map_push_elem || 7394 func_id == BPF_FUNC_map_pop_elem)) { 7395 verbose(env, "write into map forbidden\n"); 7396 return -EACCES; 7397 } 7398 7399 if (!BPF_MAP_PTR(aux->map_ptr_state)) 7400 bpf_map_ptr_store(aux, meta->map_ptr, 7401 !meta->map_ptr->bypass_spec_v1); 7402 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 7403 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 7404 !meta->map_ptr->bypass_spec_v1); 7405 return 0; 7406 } 7407 7408 static int 7409 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 7410 int func_id, int insn_idx) 7411 { 7412 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 7413 struct bpf_reg_state *regs = cur_regs(env), *reg; 7414 struct bpf_map *map = meta->map_ptr; 7415 u64 val, max; 7416 int err; 7417 7418 if (func_id != BPF_FUNC_tail_call) 7419 return 0; 7420 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 7421 verbose(env, "kernel subsystem misconfigured verifier\n"); 7422 return -EINVAL; 7423 } 7424 7425 reg = ®s[BPF_REG_3]; 7426 val = reg->var_off.value; 7427 max = map->max_entries; 7428 7429 if (!(register_is_const(reg) && val < max)) { 7430 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 7431 return 0; 7432 } 7433 7434 err = mark_chain_precision(env, BPF_REG_3); 7435 if (err) 7436 return err; 7437 if (bpf_map_key_unseen(aux)) 7438 bpf_map_key_store(aux, val); 7439 else if (!bpf_map_key_poisoned(aux) && 7440 bpf_map_key_immediate(aux) != val) 7441 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 7442 return 0; 7443 } 7444 7445 static int check_reference_leak(struct bpf_verifier_env *env) 7446 { 7447 struct bpf_func_state *state = cur_func(env); 7448 bool refs_lingering = false; 7449 int i; 7450 7451 if (state->frameno && !state->in_callback_fn) 7452 return 0; 7453 7454 for (i = 0; i < state->acquired_refs; i++) { 7455 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) 7456 continue; 7457 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 7458 state->refs[i].id, state->refs[i].insn_idx); 7459 refs_lingering = true; 7460 } 7461 return refs_lingering ? -EINVAL : 0; 7462 } 7463 7464 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 7465 struct bpf_reg_state *regs) 7466 { 7467 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 7468 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 7469 struct bpf_map *fmt_map = fmt_reg->map_ptr; 7470 int err, fmt_map_off, num_args; 7471 u64 fmt_addr; 7472 char *fmt; 7473 7474 /* data must be an array of u64 */ 7475 if (data_len_reg->var_off.value % 8) 7476 return -EINVAL; 7477 num_args = data_len_reg->var_off.value / 8; 7478 7479 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 7480 * and map_direct_value_addr is set. 7481 */ 7482 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 7483 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 7484 fmt_map_off); 7485 if (err) { 7486 verbose(env, "verifier bug\n"); 7487 return -EFAULT; 7488 } 7489 fmt = (char *)(long)fmt_addr + fmt_map_off; 7490 7491 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 7492 * can focus on validating the format specifiers. 7493 */ 7494 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); 7495 if (err < 0) 7496 verbose(env, "Invalid format string\n"); 7497 7498 return err; 7499 } 7500 7501 static int check_get_func_ip(struct bpf_verifier_env *env) 7502 { 7503 enum bpf_prog_type type = resolve_prog_type(env->prog); 7504 int func_id = BPF_FUNC_get_func_ip; 7505 7506 if (type == BPF_PROG_TYPE_TRACING) { 7507 if (!bpf_prog_has_trampoline(env->prog)) { 7508 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 7509 func_id_name(func_id), func_id); 7510 return -ENOTSUPP; 7511 } 7512 return 0; 7513 } else if (type == BPF_PROG_TYPE_KPROBE) { 7514 return 0; 7515 } 7516 7517 verbose(env, "func %s#%d not supported for program type %d\n", 7518 func_id_name(func_id), func_id, type); 7519 return -ENOTSUPP; 7520 } 7521 7522 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 7523 { 7524 return &env->insn_aux_data[env->insn_idx]; 7525 } 7526 7527 static bool loop_flag_is_zero(struct bpf_verifier_env *env) 7528 { 7529 struct bpf_reg_state *regs = cur_regs(env); 7530 struct bpf_reg_state *reg = ®s[BPF_REG_4]; 7531 bool reg_is_null = register_is_null(reg); 7532 7533 if (reg_is_null) 7534 mark_chain_precision(env, BPF_REG_4); 7535 7536 return reg_is_null; 7537 } 7538 7539 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) 7540 { 7541 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; 7542 7543 if (!state->initialized) { 7544 state->initialized = 1; 7545 state->fit_for_inline = loop_flag_is_zero(env); 7546 state->callback_subprogno = subprogno; 7547 return; 7548 } 7549 7550 if (!state->fit_for_inline) 7551 return; 7552 7553 state->fit_for_inline = (loop_flag_is_zero(env) && 7554 state->callback_subprogno == subprogno); 7555 } 7556 7557 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7558 int *insn_idx_p) 7559 { 7560 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 7561 const struct bpf_func_proto *fn = NULL; 7562 enum bpf_return_type ret_type; 7563 enum bpf_type_flag ret_flag; 7564 struct bpf_reg_state *regs; 7565 struct bpf_call_arg_meta meta; 7566 int insn_idx = *insn_idx_p; 7567 bool changes_data; 7568 int i, err, func_id; 7569 7570 /* find function prototype */ 7571 func_id = insn->imm; 7572 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 7573 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 7574 func_id); 7575 return -EINVAL; 7576 } 7577 7578 if (env->ops->get_func_proto) 7579 fn = env->ops->get_func_proto(func_id, env->prog); 7580 if (!fn) { 7581 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 7582 func_id); 7583 return -EINVAL; 7584 } 7585 7586 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 7587 if (!env->prog->gpl_compatible && fn->gpl_only) { 7588 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 7589 return -EINVAL; 7590 } 7591 7592 if (fn->allowed && !fn->allowed(env->prog)) { 7593 verbose(env, "helper call is not allowed in probe\n"); 7594 return -EINVAL; 7595 } 7596 7597 if (!env->prog->aux->sleepable && fn->might_sleep) { 7598 verbose(env, "helper call might sleep in a non-sleepable prog\n"); 7599 return -EINVAL; 7600 } 7601 7602 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 7603 changes_data = bpf_helper_changes_pkt_data(fn->func); 7604 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 7605 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 7606 func_id_name(func_id), func_id); 7607 return -EINVAL; 7608 } 7609 7610 memset(&meta, 0, sizeof(meta)); 7611 meta.pkt_access = fn->pkt_access; 7612 7613 err = check_func_proto(fn, func_id); 7614 if (err) { 7615 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 7616 func_id_name(func_id), func_id); 7617 return err; 7618 } 7619 7620 if (env->cur_state->active_rcu_lock) { 7621 if (fn->might_sleep) { 7622 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", 7623 func_id_name(func_id), func_id); 7624 return -EINVAL; 7625 } 7626 7627 if (env->prog->aux->sleepable && is_storage_get_function(func_id)) 7628 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; 7629 } 7630 7631 meta.func_id = func_id; 7632 /* check args */ 7633 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 7634 err = check_func_arg(env, i, &meta, fn); 7635 if (err) 7636 return err; 7637 } 7638 7639 err = record_func_map(env, &meta, func_id, insn_idx); 7640 if (err) 7641 return err; 7642 7643 err = record_func_key(env, &meta, func_id, insn_idx); 7644 if (err) 7645 return err; 7646 7647 /* Mark slots with STACK_MISC in case of raw mode, stack offset 7648 * is inferred from register state. 7649 */ 7650 for (i = 0; i < meta.access_size; i++) { 7651 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 7652 BPF_WRITE, -1, false); 7653 if (err) 7654 return err; 7655 } 7656 7657 regs = cur_regs(env); 7658 7659 if (meta.uninit_dynptr_regno) { 7660 /* we write BPF_DW bits (8 bytes) at a time */ 7661 for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) { 7662 err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno, 7663 i, BPF_DW, BPF_WRITE, -1, false); 7664 if (err) 7665 return err; 7666 } 7667 7668 err = mark_stack_slots_dynptr(env, ®s[meta.uninit_dynptr_regno], 7669 fn->arg_type[meta.uninit_dynptr_regno - BPF_REG_1], 7670 insn_idx); 7671 if (err) 7672 return err; 7673 } 7674 7675 if (meta.release_regno) { 7676 err = -EINVAL; 7677 if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) 7678 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); 7679 else if (meta.ref_obj_id) 7680 err = release_reference(env, meta.ref_obj_id); 7681 /* meta.ref_obj_id can only be 0 if register that is meant to be 7682 * released is NULL, which must be > R0. 7683 */ 7684 else if (register_is_null(®s[meta.release_regno])) 7685 err = 0; 7686 if (err) { 7687 verbose(env, "func %s#%d reference has not been acquired before\n", 7688 func_id_name(func_id), func_id); 7689 return err; 7690 } 7691 } 7692 7693 switch (func_id) { 7694 case BPF_FUNC_tail_call: 7695 err = check_reference_leak(env); 7696 if (err) { 7697 verbose(env, "tail_call would lead to reference leak\n"); 7698 return err; 7699 } 7700 break; 7701 case BPF_FUNC_get_local_storage: 7702 /* check that flags argument in get_local_storage(map, flags) is 0, 7703 * this is required because get_local_storage() can't return an error. 7704 */ 7705 if (!register_is_null(®s[BPF_REG_2])) { 7706 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 7707 return -EINVAL; 7708 } 7709 break; 7710 case BPF_FUNC_for_each_map_elem: 7711 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7712 set_map_elem_callback_state); 7713 break; 7714 case BPF_FUNC_timer_set_callback: 7715 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7716 set_timer_callback_state); 7717 break; 7718 case BPF_FUNC_find_vma: 7719 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7720 set_find_vma_callback_state); 7721 break; 7722 case BPF_FUNC_snprintf: 7723 err = check_bpf_snprintf_call(env, regs); 7724 break; 7725 case BPF_FUNC_loop: 7726 update_loop_inline_state(env, meta.subprogno); 7727 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7728 set_loop_callback_state); 7729 break; 7730 case BPF_FUNC_dynptr_from_mem: 7731 if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) { 7732 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", 7733 reg_type_str(env, regs[BPF_REG_1].type)); 7734 return -EACCES; 7735 } 7736 break; 7737 case BPF_FUNC_set_retval: 7738 if (prog_type == BPF_PROG_TYPE_LSM && 7739 env->prog->expected_attach_type == BPF_LSM_CGROUP) { 7740 if (!env->prog->aux->attach_func_proto->type) { 7741 /* Make sure programs that attach to void 7742 * hooks don't try to modify return value. 7743 */ 7744 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 7745 return -EINVAL; 7746 } 7747 } 7748 break; 7749 case BPF_FUNC_dynptr_data: 7750 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 7751 if (arg_type_is_dynptr(fn->arg_type[i])) { 7752 struct bpf_reg_state *reg = ®s[BPF_REG_1 + i]; 7753 7754 if (meta.ref_obj_id) { 7755 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); 7756 return -EFAULT; 7757 } 7758 7759 if (base_type(reg->type) != PTR_TO_DYNPTR) 7760 /* Find the id of the dynptr we're 7761 * tracking the reference of 7762 */ 7763 meta.ref_obj_id = stack_slot_get_id(env, reg); 7764 break; 7765 } 7766 } 7767 if (i == MAX_BPF_FUNC_REG_ARGS) { 7768 verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n"); 7769 return -EFAULT; 7770 } 7771 break; 7772 case BPF_FUNC_user_ringbuf_drain: 7773 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7774 set_user_ringbuf_callback_state); 7775 break; 7776 } 7777 7778 if (err) 7779 return err; 7780 7781 /* reset caller saved regs */ 7782 for (i = 0; i < CALLER_SAVED_REGS; i++) { 7783 mark_reg_not_init(env, regs, caller_saved[i]); 7784 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 7785 } 7786 7787 /* helper call returns 64-bit value. */ 7788 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7789 7790 /* update return register (already marked as written above) */ 7791 ret_type = fn->ret_type; 7792 ret_flag = type_flag(ret_type); 7793 7794 switch (base_type(ret_type)) { 7795 case RET_INTEGER: 7796 /* sets type to SCALAR_VALUE */ 7797 mark_reg_unknown(env, regs, BPF_REG_0); 7798 break; 7799 case RET_VOID: 7800 regs[BPF_REG_0].type = NOT_INIT; 7801 break; 7802 case RET_PTR_TO_MAP_VALUE: 7803 /* There is no offset yet applied, variable or fixed */ 7804 mark_reg_known_zero(env, regs, BPF_REG_0); 7805 /* remember map_ptr, so that check_map_access() 7806 * can check 'value_size' boundary of memory access 7807 * to map element returned from bpf_map_lookup_elem() 7808 */ 7809 if (meta.map_ptr == NULL) { 7810 verbose(env, 7811 "kernel subsystem misconfigured verifier\n"); 7812 return -EINVAL; 7813 } 7814 regs[BPF_REG_0].map_ptr = meta.map_ptr; 7815 regs[BPF_REG_0].map_uid = meta.map_uid; 7816 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 7817 if (!type_may_be_null(ret_type) && 7818 btf_record_has_field(meta.map_ptr->record, BPF_SPIN_LOCK)) { 7819 regs[BPF_REG_0].id = ++env->id_gen; 7820 } 7821 break; 7822 case RET_PTR_TO_SOCKET: 7823 mark_reg_known_zero(env, regs, BPF_REG_0); 7824 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 7825 break; 7826 case RET_PTR_TO_SOCK_COMMON: 7827 mark_reg_known_zero(env, regs, BPF_REG_0); 7828 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 7829 break; 7830 case RET_PTR_TO_TCP_SOCK: 7831 mark_reg_known_zero(env, regs, BPF_REG_0); 7832 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 7833 break; 7834 case RET_PTR_TO_MEM: 7835 mark_reg_known_zero(env, regs, BPF_REG_0); 7836 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7837 regs[BPF_REG_0].mem_size = meta.mem_size; 7838 break; 7839 case RET_PTR_TO_MEM_OR_BTF_ID: 7840 { 7841 const struct btf_type *t; 7842 7843 mark_reg_known_zero(env, regs, BPF_REG_0); 7844 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 7845 if (!btf_type_is_struct(t)) { 7846 u32 tsize; 7847 const struct btf_type *ret; 7848 const char *tname; 7849 7850 /* resolve the type size of ksym. */ 7851 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 7852 if (IS_ERR(ret)) { 7853 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 7854 verbose(env, "unable to resolve the size of type '%s': %ld\n", 7855 tname, PTR_ERR(ret)); 7856 return -EINVAL; 7857 } 7858 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7859 regs[BPF_REG_0].mem_size = tsize; 7860 } else { 7861 /* MEM_RDONLY may be carried from ret_flag, but it 7862 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 7863 * it will confuse the check of PTR_TO_BTF_ID in 7864 * check_mem_access(). 7865 */ 7866 ret_flag &= ~MEM_RDONLY; 7867 7868 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7869 regs[BPF_REG_0].btf = meta.ret_btf; 7870 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 7871 } 7872 break; 7873 } 7874 case RET_PTR_TO_BTF_ID: 7875 { 7876 struct btf *ret_btf; 7877 int ret_btf_id; 7878 7879 mark_reg_known_zero(env, regs, BPF_REG_0); 7880 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7881 if (func_id == BPF_FUNC_kptr_xchg) { 7882 ret_btf = meta.kptr_field->kptr.btf; 7883 ret_btf_id = meta.kptr_field->kptr.btf_id; 7884 } else { 7885 if (fn->ret_btf_id == BPF_PTR_POISON) { 7886 verbose(env, "verifier internal error:"); 7887 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", 7888 func_id_name(func_id)); 7889 return -EINVAL; 7890 } 7891 ret_btf = btf_vmlinux; 7892 ret_btf_id = *fn->ret_btf_id; 7893 } 7894 if (ret_btf_id == 0) { 7895 verbose(env, "invalid return type %u of func %s#%d\n", 7896 base_type(ret_type), func_id_name(func_id), 7897 func_id); 7898 return -EINVAL; 7899 } 7900 regs[BPF_REG_0].btf = ret_btf; 7901 regs[BPF_REG_0].btf_id = ret_btf_id; 7902 break; 7903 } 7904 default: 7905 verbose(env, "unknown return type %u of func %s#%d\n", 7906 base_type(ret_type), func_id_name(func_id), func_id); 7907 return -EINVAL; 7908 } 7909 7910 if (type_may_be_null(regs[BPF_REG_0].type)) 7911 regs[BPF_REG_0].id = ++env->id_gen; 7912 7913 if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { 7914 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", 7915 func_id_name(func_id), func_id); 7916 return -EFAULT; 7917 } 7918 7919 if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { 7920 /* For release_reference() */ 7921 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 7922 } else if (is_acquire_function(func_id, meta.map_ptr)) { 7923 int id = acquire_reference_state(env, insn_idx); 7924 7925 if (id < 0) 7926 return id; 7927 /* For mark_ptr_or_null_reg() */ 7928 regs[BPF_REG_0].id = id; 7929 /* For release_reference() */ 7930 regs[BPF_REG_0].ref_obj_id = id; 7931 } 7932 7933 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 7934 7935 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 7936 if (err) 7937 return err; 7938 7939 if ((func_id == BPF_FUNC_get_stack || 7940 func_id == BPF_FUNC_get_task_stack) && 7941 !env->prog->has_callchain_buf) { 7942 const char *err_str; 7943 7944 #ifdef CONFIG_PERF_EVENTS 7945 err = get_callchain_buffers(sysctl_perf_event_max_stack); 7946 err_str = "cannot get callchain buffer for func %s#%d\n"; 7947 #else 7948 err = -ENOTSUPP; 7949 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 7950 #endif 7951 if (err) { 7952 verbose(env, err_str, func_id_name(func_id), func_id); 7953 return err; 7954 } 7955 7956 env->prog->has_callchain_buf = true; 7957 } 7958 7959 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 7960 env->prog->call_get_stack = true; 7961 7962 if (func_id == BPF_FUNC_get_func_ip) { 7963 if (check_get_func_ip(env)) 7964 return -ENOTSUPP; 7965 env->prog->call_get_func_ip = true; 7966 } 7967 7968 if (changes_data) 7969 clear_all_pkt_pointers(env); 7970 return 0; 7971 } 7972 7973 /* mark_btf_func_reg_size() is used when the reg size is determined by 7974 * the BTF func_proto's return value size and argument. 7975 */ 7976 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 7977 size_t reg_size) 7978 { 7979 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 7980 7981 if (regno == BPF_REG_0) { 7982 /* Function return value */ 7983 reg->live |= REG_LIVE_WRITTEN; 7984 reg->subreg_def = reg_size == sizeof(u64) ? 7985 DEF_NOT_SUBREG : env->insn_idx + 1; 7986 } else { 7987 /* Function argument */ 7988 if (reg_size == sizeof(u64)) { 7989 mark_insn_zext(env, reg); 7990 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 7991 } else { 7992 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 7993 } 7994 } 7995 } 7996 7997 struct bpf_kfunc_call_arg_meta { 7998 /* In parameters */ 7999 struct btf *btf; 8000 u32 func_id; 8001 u32 kfunc_flags; 8002 const struct btf_type *func_proto; 8003 const char *func_name; 8004 /* Out parameters */ 8005 u32 ref_obj_id; 8006 u8 release_regno; 8007 bool r0_rdonly; 8008 u32 ret_btf_id; 8009 u64 r0_size; 8010 struct { 8011 u64 value; 8012 bool found; 8013 } arg_constant; 8014 struct { 8015 struct btf *btf; 8016 u32 btf_id; 8017 } arg_obj_drop; 8018 struct { 8019 struct btf_field *field; 8020 } arg_list_head; 8021 }; 8022 8023 static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) 8024 { 8025 return meta->kfunc_flags & KF_ACQUIRE; 8026 } 8027 8028 static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) 8029 { 8030 return meta->kfunc_flags & KF_RET_NULL; 8031 } 8032 8033 static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) 8034 { 8035 return meta->kfunc_flags & KF_RELEASE; 8036 } 8037 8038 static bool is_kfunc_trusted_args(struct bpf_kfunc_call_arg_meta *meta) 8039 { 8040 return meta->kfunc_flags & KF_TRUSTED_ARGS; 8041 } 8042 8043 static bool is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) 8044 { 8045 return meta->kfunc_flags & KF_SLEEPABLE; 8046 } 8047 8048 static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) 8049 { 8050 return meta->kfunc_flags & KF_DESTRUCTIVE; 8051 } 8052 8053 static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) 8054 { 8055 return meta->kfunc_flags & KF_RCU; 8056 } 8057 8058 static bool is_kfunc_arg_kptr_get(struct bpf_kfunc_call_arg_meta *meta, int arg) 8059 { 8060 return arg == 0 && (meta->kfunc_flags & KF_KPTR_GET); 8061 } 8062 8063 static bool __kfunc_param_match_suffix(const struct btf *btf, 8064 const struct btf_param *arg, 8065 const char *suffix) 8066 { 8067 int suffix_len = strlen(suffix), len; 8068 const char *param_name; 8069 8070 /* In the future, this can be ported to use BTF tagging */ 8071 param_name = btf_name_by_offset(btf, arg->name_off); 8072 if (str_is_empty(param_name)) 8073 return false; 8074 len = strlen(param_name); 8075 if (len < suffix_len) 8076 return false; 8077 param_name += len - suffix_len; 8078 return !strncmp(param_name, suffix, suffix_len); 8079 } 8080 8081 static bool is_kfunc_arg_mem_size(const struct btf *btf, 8082 const struct btf_param *arg, 8083 const struct bpf_reg_state *reg) 8084 { 8085 const struct btf_type *t; 8086 8087 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8088 if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) 8089 return false; 8090 8091 return __kfunc_param_match_suffix(btf, arg, "__sz"); 8092 } 8093 8094 static bool is_kfunc_arg_constant(const struct btf *btf, const struct btf_param *arg) 8095 { 8096 return __kfunc_param_match_suffix(btf, arg, "__k"); 8097 } 8098 8099 static bool is_kfunc_arg_ignore(const struct btf *btf, const struct btf_param *arg) 8100 { 8101 return __kfunc_param_match_suffix(btf, arg, "__ign"); 8102 } 8103 8104 static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) 8105 { 8106 return __kfunc_param_match_suffix(btf, arg, "__alloc"); 8107 } 8108 8109 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, 8110 const struct btf_param *arg, 8111 const char *name) 8112 { 8113 int len, target_len = strlen(name); 8114 const char *param_name; 8115 8116 param_name = btf_name_by_offset(btf, arg->name_off); 8117 if (str_is_empty(param_name)) 8118 return false; 8119 len = strlen(param_name); 8120 if (len != target_len) 8121 return false; 8122 if (strcmp(param_name, name)) 8123 return false; 8124 8125 return true; 8126 } 8127 8128 enum { 8129 KF_ARG_DYNPTR_ID, 8130 KF_ARG_LIST_HEAD_ID, 8131 KF_ARG_LIST_NODE_ID, 8132 }; 8133 8134 BTF_ID_LIST(kf_arg_btf_ids) 8135 BTF_ID(struct, bpf_dynptr_kern) 8136 BTF_ID(struct, bpf_list_head) 8137 BTF_ID(struct, bpf_list_node) 8138 8139 static bool __is_kfunc_ptr_arg_type(const struct btf *btf, 8140 const struct btf_param *arg, int type) 8141 { 8142 const struct btf_type *t; 8143 u32 res_id; 8144 8145 t = btf_type_skip_modifiers(btf, arg->type, NULL); 8146 if (!t) 8147 return false; 8148 if (!btf_type_is_ptr(t)) 8149 return false; 8150 t = btf_type_skip_modifiers(btf, t->type, &res_id); 8151 if (!t) 8152 return false; 8153 return btf_types_are_same(btf, res_id, btf_vmlinux, kf_arg_btf_ids[type]); 8154 } 8155 8156 static bool is_kfunc_arg_dynptr(const struct btf *btf, const struct btf_param *arg) 8157 { 8158 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_DYNPTR_ID); 8159 } 8160 8161 static bool is_kfunc_arg_list_head(const struct btf *btf, const struct btf_param *arg) 8162 { 8163 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_HEAD_ID); 8164 } 8165 8166 static bool is_kfunc_arg_list_node(const struct btf *btf, const struct btf_param *arg) 8167 { 8168 return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_LIST_NODE_ID); 8169 } 8170 8171 /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ 8172 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, 8173 const struct btf *btf, 8174 const struct btf_type *t, int rec) 8175 { 8176 const struct btf_type *member_type; 8177 const struct btf_member *member; 8178 u32 i; 8179 8180 if (!btf_type_is_struct(t)) 8181 return false; 8182 8183 for_each_member(i, t, member) { 8184 const struct btf_array *array; 8185 8186 member_type = btf_type_skip_modifiers(btf, member->type, NULL); 8187 if (btf_type_is_struct(member_type)) { 8188 if (rec >= 3) { 8189 verbose(env, "max struct nesting depth exceeded\n"); 8190 return false; 8191 } 8192 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) 8193 return false; 8194 continue; 8195 } 8196 if (btf_type_is_array(member_type)) { 8197 array = btf_array(member_type); 8198 if (!array->nelems) 8199 return false; 8200 member_type = btf_type_skip_modifiers(btf, array->type, NULL); 8201 if (!btf_type_is_scalar(member_type)) 8202 return false; 8203 continue; 8204 } 8205 if (!btf_type_is_scalar(member_type)) 8206 return false; 8207 } 8208 return true; 8209 } 8210 8211 8212 static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { 8213 #ifdef CONFIG_NET 8214 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK], 8215 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 8216 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP], 8217 #endif 8218 }; 8219 8220 enum kfunc_ptr_arg_type { 8221 KF_ARG_PTR_TO_CTX, 8222 KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ 8223 KF_ARG_PTR_TO_KPTR, /* PTR_TO_KPTR but type specific */ 8224 KF_ARG_PTR_TO_DYNPTR, 8225 KF_ARG_PTR_TO_LIST_HEAD, 8226 KF_ARG_PTR_TO_LIST_NODE, 8227 KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ 8228 KF_ARG_PTR_TO_MEM, 8229 KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ 8230 }; 8231 8232 enum special_kfunc_type { 8233 KF_bpf_obj_new_impl, 8234 KF_bpf_obj_drop_impl, 8235 KF_bpf_list_push_front, 8236 KF_bpf_list_push_back, 8237 KF_bpf_list_pop_front, 8238 KF_bpf_list_pop_back, 8239 KF_bpf_cast_to_kern_ctx, 8240 KF_bpf_rdonly_cast, 8241 KF_bpf_rcu_read_lock, 8242 KF_bpf_rcu_read_unlock, 8243 }; 8244 8245 BTF_SET_START(special_kfunc_set) 8246 BTF_ID(func, bpf_obj_new_impl) 8247 BTF_ID(func, bpf_obj_drop_impl) 8248 BTF_ID(func, bpf_list_push_front) 8249 BTF_ID(func, bpf_list_push_back) 8250 BTF_ID(func, bpf_list_pop_front) 8251 BTF_ID(func, bpf_list_pop_back) 8252 BTF_ID(func, bpf_cast_to_kern_ctx) 8253 BTF_ID(func, bpf_rdonly_cast) 8254 BTF_SET_END(special_kfunc_set) 8255 8256 BTF_ID_LIST(special_kfunc_list) 8257 BTF_ID(func, bpf_obj_new_impl) 8258 BTF_ID(func, bpf_obj_drop_impl) 8259 BTF_ID(func, bpf_list_push_front) 8260 BTF_ID(func, bpf_list_push_back) 8261 BTF_ID(func, bpf_list_pop_front) 8262 BTF_ID(func, bpf_list_pop_back) 8263 BTF_ID(func, bpf_cast_to_kern_ctx) 8264 BTF_ID(func, bpf_rdonly_cast) 8265 BTF_ID(func, bpf_rcu_read_lock) 8266 BTF_ID(func, bpf_rcu_read_unlock) 8267 8268 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) 8269 { 8270 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; 8271 } 8272 8273 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) 8274 { 8275 return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; 8276 } 8277 8278 static enum kfunc_ptr_arg_type 8279 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, 8280 struct bpf_kfunc_call_arg_meta *meta, 8281 const struct btf_type *t, const struct btf_type *ref_t, 8282 const char *ref_tname, const struct btf_param *args, 8283 int argno, int nargs) 8284 { 8285 u32 regno = argno + 1; 8286 struct bpf_reg_state *regs = cur_regs(env); 8287 struct bpf_reg_state *reg = ®s[regno]; 8288 bool arg_mem_size = false; 8289 8290 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) 8291 return KF_ARG_PTR_TO_CTX; 8292 8293 /* In this function, we verify the kfunc's BTF as per the argument type, 8294 * leaving the rest of the verification with respect to the register 8295 * type to our caller. When a set of conditions hold in the BTF type of 8296 * arguments, we resolve it to a known kfunc_ptr_arg_type. 8297 */ 8298 if (btf_get_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) 8299 return KF_ARG_PTR_TO_CTX; 8300 8301 if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno])) 8302 return KF_ARG_PTR_TO_ALLOC_BTF_ID; 8303 8304 if (is_kfunc_arg_kptr_get(meta, argno)) { 8305 if (!btf_type_is_ptr(ref_t)) { 8306 verbose(env, "arg#0 BTF type must be a double pointer for kptr_get kfunc\n"); 8307 return -EINVAL; 8308 } 8309 ref_t = btf_type_by_id(meta->btf, ref_t->type); 8310 ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); 8311 if (!btf_type_is_struct(ref_t)) { 8312 verbose(env, "kernel function %s args#0 pointer type %s %s is not supported\n", 8313 meta->func_name, btf_type_str(ref_t), ref_tname); 8314 return -EINVAL; 8315 } 8316 return KF_ARG_PTR_TO_KPTR; 8317 } 8318 8319 if (is_kfunc_arg_dynptr(meta->btf, &args[argno])) 8320 return KF_ARG_PTR_TO_DYNPTR; 8321 8322 if (is_kfunc_arg_list_head(meta->btf, &args[argno])) 8323 return KF_ARG_PTR_TO_LIST_HEAD; 8324 8325 if (is_kfunc_arg_list_node(meta->btf, &args[argno])) 8326 return KF_ARG_PTR_TO_LIST_NODE; 8327 8328 if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { 8329 if (!btf_type_is_struct(ref_t)) { 8330 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", 8331 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 8332 return -EINVAL; 8333 } 8334 return KF_ARG_PTR_TO_BTF_ID; 8335 } 8336 8337 if (argno + 1 < nargs && is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1])) 8338 arg_mem_size = true; 8339 8340 /* This is the catch all argument type of register types supported by 8341 * check_helper_mem_access. However, we only allow when argument type is 8342 * pointer to scalar, or struct composed (recursively) of scalars. When 8343 * arg_mem_size is true, the pointer can be void *. 8344 */ 8345 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && 8346 (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { 8347 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", 8348 argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); 8349 return -EINVAL; 8350 } 8351 return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; 8352 } 8353 8354 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, 8355 struct bpf_reg_state *reg, 8356 const struct btf_type *ref_t, 8357 const char *ref_tname, u32 ref_id, 8358 struct bpf_kfunc_call_arg_meta *meta, 8359 int argno) 8360 { 8361 const struct btf_type *reg_ref_t; 8362 bool strict_type_match = false; 8363 const struct btf *reg_btf; 8364 const char *reg_ref_tname; 8365 u32 reg_ref_id; 8366 8367 if (base_type(reg->type) == PTR_TO_BTF_ID) { 8368 reg_btf = reg->btf; 8369 reg_ref_id = reg->btf_id; 8370 } else { 8371 reg_btf = btf_vmlinux; 8372 reg_ref_id = *reg2btf_ids[base_type(reg->type)]; 8373 } 8374 8375 if (is_kfunc_trusted_args(meta) || (is_kfunc_release(meta) && reg->ref_obj_id)) 8376 strict_type_match = true; 8377 8378 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); 8379 reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); 8380 if (!btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, strict_type_match)) { 8381 verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", 8382 meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1, 8383 btf_type_str(reg_ref_t), reg_ref_tname); 8384 return -EINVAL; 8385 } 8386 return 0; 8387 } 8388 8389 static int process_kf_arg_ptr_to_kptr(struct bpf_verifier_env *env, 8390 struct bpf_reg_state *reg, 8391 const struct btf_type *ref_t, 8392 const char *ref_tname, 8393 struct bpf_kfunc_call_arg_meta *meta, 8394 int argno) 8395 { 8396 struct btf_field *kptr_field; 8397 8398 /* check_func_arg_reg_off allows var_off for 8399 * PTR_TO_MAP_VALUE, but we need fixed offset to find 8400 * off_desc. 8401 */ 8402 if (!tnum_is_const(reg->var_off)) { 8403 verbose(env, "arg#0 must have constant offset\n"); 8404 return -EINVAL; 8405 } 8406 8407 kptr_field = btf_record_find(reg->map_ptr->record, reg->off + reg->var_off.value, BPF_KPTR); 8408 if (!kptr_field || kptr_field->type != BPF_KPTR_REF) { 8409 verbose(env, "arg#0 no referenced kptr at map value offset=%llu\n", 8410 reg->off + reg->var_off.value); 8411 return -EINVAL; 8412 } 8413 8414 if (!btf_struct_ids_match(&env->log, meta->btf, ref_t->type, 0, kptr_field->kptr.btf, 8415 kptr_field->kptr.btf_id, true)) { 8416 verbose(env, "kernel function %s args#%d expected pointer to %s %s\n", 8417 meta->func_name, argno, btf_type_str(ref_t), ref_tname); 8418 return -EINVAL; 8419 } 8420 return 0; 8421 } 8422 8423 static int ref_set_release_on_unlock(struct bpf_verifier_env *env, u32 ref_obj_id) 8424 { 8425 struct bpf_func_state *state = cur_func(env); 8426 struct bpf_reg_state *reg; 8427 int i; 8428 8429 /* bpf_spin_lock only allows calling list_push and list_pop, no BPF 8430 * subprogs, no global functions. This means that the references would 8431 * not be released inside the critical section but they may be added to 8432 * the reference state, and the acquired_refs are never copied out for a 8433 * different frame as BPF to BPF calls don't work in bpf_spin_lock 8434 * critical sections. 8435 */ 8436 if (!ref_obj_id) { 8437 verbose(env, "verifier internal error: ref_obj_id is zero for release_on_unlock\n"); 8438 return -EFAULT; 8439 } 8440 for (i = 0; i < state->acquired_refs; i++) { 8441 if (state->refs[i].id == ref_obj_id) { 8442 if (state->refs[i].release_on_unlock) { 8443 verbose(env, "verifier internal error: expected false release_on_unlock"); 8444 return -EFAULT; 8445 } 8446 state->refs[i].release_on_unlock = true; 8447 /* Now mark everyone sharing same ref_obj_id as untrusted */ 8448 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8449 if (reg->ref_obj_id == ref_obj_id) 8450 reg->type |= PTR_UNTRUSTED; 8451 })); 8452 return 0; 8453 } 8454 } 8455 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); 8456 return -EFAULT; 8457 } 8458 8459 /* Implementation details: 8460 * 8461 * Each register points to some region of memory, which we define as an 8462 * allocation. Each allocation may embed a bpf_spin_lock which protects any 8463 * special BPF objects (bpf_list_head, bpf_rb_root, etc.) part of the same 8464 * allocation. The lock and the data it protects are colocated in the same 8465 * memory region. 8466 * 8467 * Hence, everytime a register holds a pointer value pointing to such 8468 * allocation, the verifier preserves a unique reg->id for it. 8469 * 8470 * The verifier remembers the lock 'ptr' and the lock 'id' whenever 8471 * bpf_spin_lock is called. 8472 * 8473 * To enable this, lock state in the verifier captures two values: 8474 * active_lock.ptr = Register's type specific pointer 8475 * active_lock.id = A unique ID for each register pointer value 8476 * 8477 * Currently, PTR_TO_MAP_VALUE and PTR_TO_BTF_ID | MEM_ALLOC are the two 8478 * supported register types. 8479 * 8480 * The active_lock.ptr in case of map values is the reg->map_ptr, and in case of 8481 * allocated objects is the reg->btf pointer. 8482 * 8483 * The active_lock.id is non-unique for maps supporting direct_value_addr, as we 8484 * can establish the provenance of the map value statically for each distinct 8485 * lookup into such maps. They always contain a single map value hence unique 8486 * IDs for each pseudo load pessimizes the algorithm and rejects valid programs. 8487 * 8488 * So, in case of global variables, they use array maps with max_entries = 1, 8489 * hence their active_lock.ptr becomes map_ptr and id = 0 (since they all point 8490 * into the same map value as max_entries is 1, as described above). 8491 * 8492 * In case of inner map lookups, the inner map pointer has same map_ptr as the 8493 * outer map pointer (in verifier context), but each lookup into an inner map 8494 * assigns a fresh reg->id to the lookup, so while lookups into distinct inner 8495 * maps from the same outer map share the same map_ptr as active_lock.ptr, they 8496 * will get different reg->id assigned to each lookup, hence different 8497 * active_lock.id. 8498 * 8499 * In case of allocated objects, active_lock.ptr is the reg->btf, and the 8500 * reg->id is a unique ID preserved after the NULL pointer check on the pointer 8501 * returned from bpf_obj_new. Each allocation receives a new reg->id. 8502 */ 8503 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) 8504 { 8505 void *ptr; 8506 u32 id; 8507 8508 switch ((int)reg->type) { 8509 case PTR_TO_MAP_VALUE: 8510 ptr = reg->map_ptr; 8511 break; 8512 case PTR_TO_BTF_ID | MEM_ALLOC: 8513 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_TRUSTED: 8514 ptr = reg->btf; 8515 break; 8516 default: 8517 verbose(env, "verifier internal error: unknown reg type for lock check\n"); 8518 return -EFAULT; 8519 } 8520 id = reg->id; 8521 8522 if (!env->cur_state->active_lock.ptr) 8523 return -EINVAL; 8524 if (env->cur_state->active_lock.ptr != ptr || 8525 env->cur_state->active_lock.id != id) { 8526 verbose(env, "held lock and object are not in the same allocation\n"); 8527 return -EINVAL; 8528 } 8529 return 0; 8530 } 8531 8532 static bool is_bpf_list_api_kfunc(u32 btf_id) 8533 { 8534 return btf_id == special_kfunc_list[KF_bpf_list_push_front] || 8535 btf_id == special_kfunc_list[KF_bpf_list_push_back] || 8536 btf_id == special_kfunc_list[KF_bpf_list_pop_front] || 8537 btf_id == special_kfunc_list[KF_bpf_list_pop_back]; 8538 } 8539 8540 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, 8541 struct bpf_reg_state *reg, u32 regno, 8542 struct bpf_kfunc_call_arg_meta *meta) 8543 { 8544 struct btf_field *field; 8545 struct btf_record *rec; 8546 u32 list_head_off; 8547 8548 if (meta->btf != btf_vmlinux || !is_bpf_list_api_kfunc(meta->func_id)) { 8549 verbose(env, "verifier internal error: bpf_list_head argument for unknown kfunc\n"); 8550 return -EFAULT; 8551 } 8552 8553 if (!tnum_is_const(reg->var_off)) { 8554 verbose(env, 8555 "R%d doesn't have constant offset. bpf_list_head has to be at the constant offset\n", 8556 regno); 8557 return -EINVAL; 8558 } 8559 8560 rec = reg_btf_record(reg); 8561 list_head_off = reg->off + reg->var_off.value; 8562 field = btf_record_find(rec, list_head_off, BPF_LIST_HEAD); 8563 if (!field) { 8564 verbose(env, "bpf_list_head not found at offset=%u\n", list_head_off); 8565 return -EINVAL; 8566 } 8567 8568 /* All functions require bpf_list_head to be protected using a bpf_spin_lock */ 8569 if (check_reg_allocation_locked(env, reg)) { 8570 verbose(env, "bpf_spin_lock at off=%d must be held for bpf_list_head\n", 8571 rec->spin_lock_off); 8572 return -EINVAL; 8573 } 8574 8575 if (meta->arg_list_head.field) { 8576 verbose(env, "verifier internal error: repeating bpf_list_head arg\n"); 8577 return -EFAULT; 8578 } 8579 meta->arg_list_head.field = field; 8580 return 0; 8581 } 8582 8583 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, 8584 struct bpf_reg_state *reg, u32 regno, 8585 struct bpf_kfunc_call_arg_meta *meta) 8586 { 8587 const struct btf_type *et, *t; 8588 struct btf_field *field; 8589 struct btf_record *rec; 8590 u32 list_node_off; 8591 8592 if (meta->btf != btf_vmlinux || 8593 (meta->func_id != special_kfunc_list[KF_bpf_list_push_front] && 8594 meta->func_id != special_kfunc_list[KF_bpf_list_push_back])) { 8595 verbose(env, "verifier internal error: bpf_list_node argument for unknown kfunc\n"); 8596 return -EFAULT; 8597 } 8598 8599 if (!tnum_is_const(reg->var_off)) { 8600 verbose(env, 8601 "R%d doesn't have constant offset. bpf_list_node has to be at the constant offset\n", 8602 regno); 8603 return -EINVAL; 8604 } 8605 8606 rec = reg_btf_record(reg); 8607 list_node_off = reg->off + reg->var_off.value; 8608 field = btf_record_find(rec, list_node_off, BPF_LIST_NODE); 8609 if (!field || field->offset != list_node_off) { 8610 verbose(env, "bpf_list_node not found at offset=%u\n", list_node_off); 8611 return -EINVAL; 8612 } 8613 8614 field = meta->arg_list_head.field; 8615 8616 et = btf_type_by_id(field->list_head.btf, field->list_head.value_btf_id); 8617 t = btf_type_by_id(reg->btf, reg->btf_id); 8618 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->list_head.btf, 8619 field->list_head.value_btf_id, true)) { 8620 verbose(env, "operation on bpf_list_head expects arg#1 bpf_list_node at offset=%d " 8621 "in struct %s, but arg is at offset=%d in struct %s\n", 8622 field->list_head.node_offset, btf_name_by_offset(field->list_head.btf, et->name_off), 8623 list_node_off, btf_name_by_offset(reg->btf, t->name_off)); 8624 return -EINVAL; 8625 } 8626 8627 if (list_node_off != field->list_head.node_offset) { 8628 verbose(env, "arg#1 offset=%d, but expected bpf_list_node at offset=%d in struct %s\n", 8629 list_node_off, field->list_head.node_offset, 8630 btf_name_by_offset(field->list_head.btf, et->name_off)); 8631 return -EINVAL; 8632 } 8633 /* Set arg#1 for expiration after unlock */ 8634 return ref_set_release_on_unlock(env, reg->ref_obj_id); 8635 } 8636 8637 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta) 8638 { 8639 const char *func_name = meta->func_name, *ref_tname; 8640 const struct btf *btf = meta->btf; 8641 const struct btf_param *args; 8642 u32 i, nargs; 8643 int ret; 8644 8645 args = (const struct btf_param *)(meta->func_proto + 1); 8646 nargs = btf_type_vlen(meta->func_proto); 8647 if (nargs > MAX_BPF_FUNC_REG_ARGS) { 8648 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, 8649 MAX_BPF_FUNC_REG_ARGS); 8650 return -EINVAL; 8651 } 8652 8653 /* Check that BTF function arguments match actual types that the 8654 * verifier sees. 8655 */ 8656 for (i = 0; i < nargs; i++) { 8657 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; 8658 const struct btf_type *t, *ref_t, *resolve_ret; 8659 enum bpf_arg_type arg_type = ARG_DONTCARE; 8660 u32 regno = i + 1, ref_id, type_size; 8661 bool is_ret_buf_sz = false; 8662 int kf_arg_type; 8663 8664 t = btf_type_skip_modifiers(btf, args[i].type, NULL); 8665 8666 if (is_kfunc_arg_ignore(btf, &args[i])) 8667 continue; 8668 8669 if (btf_type_is_scalar(t)) { 8670 if (reg->type != SCALAR_VALUE) { 8671 verbose(env, "R%d is not a scalar\n", regno); 8672 return -EINVAL; 8673 } 8674 8675 if (is_kfunc_arg_constant(meta->btf, &args[i])) { 8676 if (meta->arg_constant.found) { 8677 verbose(env, "verifier internal error: only one constant argument permitted\n"); 8678 return -EFAULT; 8679 } 8680 if (!tnum_is_const(reg->var_off)) { 8681 verbose(env, "R%d must be a known constant\n", regno); 8682 return -EINVAL; 8683 } 8684 ret = mark_chain_precision(env, regno); 8685 if (ret < 0) 8686 return ret; 8687 meta->arg_constant.found = true; 8688 meta->arg_constant.value = reg->var_off.value; 8689 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { 8690 meta->r0_rdonly = true; 8691 is_ret_buf_sz = true; 8692 } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { 8693 is_ret_buf_sz = true; 8694 } 8695 8696 if (is_ret_buf_sz) { 8697 if (meta->r0_size) { 8698 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); 8699 return -EINVAL; 8700 } 8701 8702 if (!tnum_is_const(reg->var_off)) { 8703 verbose(env, "R%d is not a const\n", regno); 8704 return -EINVAL; 8705 } 8706 8707 meta->r0_size = reg->var_off.value; 8708 ret = mark_chain_precision(env, regno); 8709 if (ret) 8710 return ret; 8711 } 8712 continue; 8713 } 8714 8715 if (!btf_type_is_ptr(t)) { 8716 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); 8717 return -EINVAL; 8718 } 8719 8720 if (reg->ref_obj_id) { 8721 if (is_kfunc_release(meta) && meta->ref_obj_id) { 8722 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 8723 regno, reg->ref_obj_id, 8724 meta->ref_obj_id); 8725 return -EFAULT; 8726 } 8727 meta->ref_obj_id = reg->ref_obj_id; 8728 if (is_kfunc_release(meta)) 8729 meta->release_regno = regno; 8730 } 8731 8732 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); 8733 ref_tname = btf_name_by_offset(btf, ref_t->name_off); 8734 8735 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); 8736 if (kf_arg_type < 0) 8737 return kf_arg_type; 8738 8739 switch (kf_arg_type) { 8740 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 8741 case KF_ARG_PTR_TO_BTF_ID: 8742 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta)) 8743 break; 8744 8745 if (!is_trusted_reg(reg)) { 8746 if (!is_kfunc_rcu(meta)) { 8747 verbose(env, "R%d must be referenced or trusted\n", regno); 8748 return -EINVAL; 8749 } 8750 if (!is_rcu_reg(reg)) { 8751 verbose(env, "R%d must be a rcu pointer\n", regno); 8752 return -EINVAL; 8753 } 8754 } 8755 8756 fallthrough; 8757 case KF_ARG_PTR_TO_CTX: 8758 /* Trusted arguments have the same offset checks as release arguments */ 8759 arg_type |= OBJ_RELEASE; 8760 break; 8761 case KF_ARG_PTR_TO_KPTR: 8762 case KF_ARG_PTR_TO_DYNPTR: 8763 case KF_ARG_PTR_TO_LIST_HEAD: 8764 case KF_ARG_PTR_TO_LIST_NODE: 8765 case KF_ARG_PTR_TO_MEM: 8766 case KF_ARG_PTR_TO_MEM_SIZE: 8767 /* Trusted by default */ 8768 break; 8769 default: 8770 WARN_ON_ONCE(1); 8771 return -EFAULT; 8772 } 8773 8774 if (is_kfunc_release(meta) && reg->ref_obj_id) 8775 arg_type |= OBJ_RELEASE; 8776 ret = check_func_arg_reg_off(env, reg, regno, arg_type); 8777 if (ret < 0) 8778 return ret; 8779 8780 switch (kf_arg_type) { 8781 case KF_ARG_PTR_TO_CTX: 8782 if (reg->type != PTR_TO_CTX) { 8783 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", i, btf_type_str(t)); 8784 return -EINVAL; 8785 } 8786 8787 if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 8788 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); 8789 if (ret < 0) 8790 return -EINVAL; 8791 meta->ret_btf_id = ret; 8792 } 8793 break; 8794 case KF_ARG_PTR_TO_ALLOC_BTF_ID: 8795 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8796 verbose(env, "arg#%d expected pointer to allocated object\n", i); 8797 return -EINVAL; 8798 } 8799 if (!reg->ref_obj_id) { 8800 verbose(env, "allocated object must be referenced\n"); 8801 return -EINVAL; 8802 } 8803 if (meta->btf == btf_vmlinux && 8804 meta->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 8805 meta->arg_obj_drop.btf = reg->btf; 8806 meta->arg_obj_drop.btf_id = reg->btf_id; 8807 } 8808 break; 8809 case KF_ARG_PTR_TO_KPTR: 8810 if (reg->type != PTR_TO_MAP_VALUE) { 8811 verbose(env, "arg#0 expected pointer to map value\n"); 8812 return -EINVAL; 8813 } 8814 ret = process_kf_arg_ptr_to_kptr(env, reg, ref_t, ref_tname, meta, i); 8815 if (ret < 0) 8816 return ret; 8817 break; 8818 case KF_ARG_PTR_TO_DYNPTR: 8819 if (reg->type != PTR_TO_STACK) { 8820 verbose(env, "arg#%d expected pointer to stack\n", i); 8821 return -EINVAL; 8822 } 8823 8824 if (!is_dynptr_reg_valid_init(env, reg)) { 8825 verbose(env, "arg#%d pointer type %s %s must be valid and initialized\n", 8826 i, btf_type_str(ref_t), ref_tname); 8827 return -EINVAL; 8828 } 8829 8830 if (!is_dynptr_type_expected(env, reg, ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL)) { 8831 verbose(env, "arg#%d pointer type %s %s points to unsupported dynamic pointer type\n", 8832 i, btf_type_str(ref_t), ref_tname); 8833 return -EINVAL; 8834 } 8835 break; 8836 case KF_ARG_PTR_TO_LIST_HEAD: 8837 if (reg->type != PTR_TO_MAP_VALUE && 8838 reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8839 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); 8840 return -EINVAL; 8841 } 8842 if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) { 8843 verbose(env, "allocated object must be referenced\n"); 8844 return -EINVAL; 8845 } 8846 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); 8847 if (ret < 0) 8848 return ret; 8849 break; 8850 case KF_ARG_PTR_TO_LIST_NODE: 8851 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { 8852 verbose(env, "arg#%d expected pointer to allocated object\n", i); 8853 return -EINVAL; 8854 } 8855 if (!reg->ref_obj_id) { 8856 verbose(env, "allocated object must be referenced\n"); 8857 return -EINVAL; 8858 } 8859 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); 8860 if (ret < 0) 8861 return ret; 8862 break; 8863 case KF_ARG_PTR_TO_BTF_ID: 8864 /* Only base_type is checked, further checks are done here */ 8865 if ((base_type(reg->type) != PTR_TO_BTF_ID || 8866 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && 8867 !reg2btf_ids[base_type(reg->type)]) { 8868 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); 8869 verbose(env, "expected %s or socket\n", 8870 reg_type_str(env, base_type(reg->type) | 8871 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); 8872 return -EINVAL; 8873 } 8874 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); 8875 if (ret < 0) 8876 return ret; 8877 break; 8878 case KF_ARG_PTR_TO_MEM: 8879 resolve_ret = btf_resolve_size(btf, ref_t, &type_size); 8880 if (IS_ERR(resolve_ret)) { 8881 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", 8882 i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret)); 8883 return -EINVAL; 8884 } 8885 ret = check_mem_reg(env, reg, regno, type_size); 8886 if (ret < 0) 8887 return ret; 8888 break; 8889 case KF_ARG_PTR_TO_MEM_SIZE: 8890 ret = check_kfunc_mem_size_reg(env, ®s[regno + 1], regno + 1); 8891 if (ret < 0) { 8892 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); 8893 return ret; 8894 } 8895 /* Skip next '__sz' argument */ 8896 i++; 8897 break; 8898 } 8899 } 8900 8901 if (is_kfunc_release(meta) && !meta->release_regno) { 8902 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", 8903 func_name); 8904 return -EINVAL; 8905 } 8906 8907 return 0; 8908 } 8909 8910 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 8911 int *insn_idx_p) 8912 { 8913 const struct btf_type *t, *func, *func_proto, *ptr_type; 8914 struct bpf_reg_state *regs = cur_regs(env); 8915 const char *func_name, *ptr_type_name; 8916 bool sleepable, rcu_lock, rcu_unlock; 8917 struct bpf_kfunc_call_arg_meta meta; 8918 u32 i, nargs, func_id, ptr_type_id; 8919 int err, insn_idx = *insn_idx_p; 8920 const struct btf_param *args; 8921 const struct btf_type *ret_t; 8922 struct btf *desc_btf; 8923 u32 *kfunc_flags; 8924 8925 /* skip for now, but return error when we find this in fixup_kfunc_call */ 8926 if (!insn->imm) 8927 return 0; 8928 8929 desc_btf = find_kfunc_desc_btf(env, insn->off); 8930 if (IS_ERR(desc_btf)) 8931 return PTR_ERR(desc_btf); 8932 8933 func_id = insn->imm; 8934 func = btf_type_by_id(desc_btf, func_id); 8935 func_name = btf_name_by_offset(desc_btf, func->name_off); 8936 func_proto = btf_type_by_id(desc_btf, func->type); 8937 8938 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id); 8939 if (!kfunc_flags) { 8940 verbose(env, "calling kernel function %s is not allowed\n", 8941 func_name); 8942 return -EACCES; 8943 } 8944 8945 /* Prepare kfunc call metadata */ 8946 memset(&meta, 0, sizeof(meta)); 8947 meta.btf = desc_btf; 8948 meta.func_id = func_id; 8949 meta.kfunc_flags = *kfunc_flags; 8950 meta.func_proto = func_proto; 8951 meta.func_name = func_name; 8952 8953 if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { 8954 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); 8955 return -EACCES; 8956 } 8957 8958 sleepable = is_kfunc_sleepable(&meta); 8959 if (sleepable && !env->prog->aux->sleepable) { 8960 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); 8961 return -EACCES; 8962 } 8963 8964 rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); 8965 rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); 8966 if ((rcu_lock || rcu_unlock) && !env->rcu_tag_supported) { 8967 verbose(env, "no vmlinux btf rcu tag support for kfunc %s\n", func_name); 8968 return -EACCES; 8969 } 8970 8971 if (env->cur_state->active_rcu_lock) { 8972 struct bpf_func_state *state; 8973 struct bpf_reg_state *reg; 8974 8975 if (rcu_lock) { 8976 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); 8977 return -EINVAL; 8978 } else if (rcu_unlock) { 8979 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ 8980 if (reg->type & MEM_RCU) { 8981 reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); 8982 reg->type |= PTR_UNTRUSTED; 8983 } 8984 })); 8985 env->cur_state->active_rcu_lock = false; 8986 } else if (sleepable) { 8987 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); 8988 return -EACCES; 8989 } 8990 } else if (rcu_lock) { 8991 env->cur_state->active_rcu_lock = true; 8992 } else if (rcu_unlock) { 8993 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); 8994 return -EINVAL; 8995 } 8996 8997 /* Check the arguments */ 8998 err = check_kfunc_args(env, &meta); 8999 if (err < 0) 9000 return err; 9001 /* In case of release function, we get register number of refcounted 9002 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now. 9003 */ 9004 if (meta.release_regno) { 9005 err = release_reference(env, regs[meta.release_regno].ref_obj_id); 9006 if (err) { 9007 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 9008 func_name, func_id); 9009 return err; 9010 } 9011 } 9012 9013 for (i = 0; i < CALLER_SAVED_REGS; i++) 9014 mark_reg_not_init(env, regs, caller_saved[i]); 9015 9016 /* Check return type */ 9017 t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL); 9018 9019 if (is_kfunc_acquire(&meta) && !btf_type_is_struct_ptr(meta.btf, t)) { 9020 /* Only exception is bpf_obj_new_impl */ 9021 if (meta.btf != btf_vmlinux || meta.func_id != special_kfunc_list[KF_bpf_obj_new_impl]) { 9022 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 9023 return -EINVAL; 9024 } 9025 } 9026 9027 if (btf_type_is_scalar(t)) { 9028 mark_reg_unknown(env, regs, BPF_REG_0); 9029 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 9030 } else if (btf_type_is_ptr(t)) { 9031 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); 9032 9033 if (meta.btf == btf_vmlinux && btf_id_set_contains(&special_kfunc_set, meta.func_id)) { 9034 if (meta.func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 9035 struct btf *ret_btf; 9036 u32 ret_btf_id; 9037 9038 if (unlikely(!bpf_global_ma_set)) 9039 return -ENOMEM; 9040 9041 if (((u64)(u32)meta.arg_constant.value) != meta.arg_constant.value) { 9042 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); 9043 return -EINVAL; 9044 } 9045 9046 ret_btf = env->prog->aux->btf; 9047 ret_btf_id = meta.arg_constant.value; 9048 9049 /* This may be NULL due to user not supplying a BTF */ 9050 if (!ret_btf) { 9051 verbose(env, "bpf_obj_new requires prog BTF\n"); 9052 return -EINVAL; 9053 } 9054 9055 ret_t = btf_type_by_id(ret_btf, ret_btf_id); 9056 if (!ret_t || !__btf_type_is_struct(ret_t)) { 9057 verbose(env, "bpf_obj_new type ID argument must be of a struct\n"); 9058 return -EINVAL; 9059 } 9060 9061 mark_reg_known_zero(env, regs, BPF_REG_0); 9062 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 9063 regs[BPF_REG_0].btf = ret_btf; 9064 regs[BPF_REG_0].btf_id = ret_btf_id; 9065 9066 env->insn_aux_data[insn_idx].obj_new_size = ret_t->size; 9067 env->insn_aux_data[insn_idx].kptr_struct_meta = 9068 btf_find_struct_meta(ret_btf, ret_btf_id); 9069 } else if (meta.func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 9070 env->insn_aux_data[insn_idx].kptr_struct_meta = 9071 btf_find_struct_meta(meta.arg_obj_drop.btf, 9072 meta.arg_obj_drop.btf_id); 9073 } else if (meta.func_id == special_kfunc_list[KF_bpf_list_pop_front] || 9074 meta.func_id == special_kfunc_list[KF_bpf_list_pop_back]) { 9075 struct btf_field *field = meta.arg_list_head.field; 9076 9077 mark_reg_known_zero(env, regs, BPF_REG_0); 9078 regs[BPF_REG_0].type = PTR_TO_BTF_ID | MEM_ALLOC; 9079 regs[BPF_REG_0].btf = field->list_head.btf; 9080 regs[BPF_REG_0].btf_id = field->list_head.value_btf_id; 9081 regs[BPF_REG_0].off = field->list_head.node_offset; 9082 } else if (meta.func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) { 9083 mark_reg_known_zero(env, regs, BPF_REG_0); 9084 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED; 9085 regs[BPF_REG_0].btf = desc_btf; 9086 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 9087 } else if (meta.func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 9088 ret_t = btf_type_by_id(desc_btf, meta.arg_constant.value); 9089 if (!ret_t || !btf_type_is_struct(ret_t)) { 9090 verbose(env, 9091 "kfunc bpf_rdonly_cast type ID argument must be of a struct\n"); 9092 return -EINVAL; 9093 } 9094 9095 mark_reg_known_zero(env, regs, BPF_REG_0); 9096 regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_UNTRUSTED; 9097 regs[BPF_REG_0].btf = desc_btf; 9098 regs[BPF_REG_0].btf_id = meta.arg_constant.value; 9099 } else { 9100 verbose(env, "kernel function %s unhandled dynamic return type\n", 9101 meta.func_name); 9102 return -EFAULT; 9103 } 9104 } else if (!__btf_type_is_struct(ptr_type)) { 9105 if (!meta.r0_size) { 9106 ptr_type_name = btf_name_by_offset(desc_btf, 9107 ptr_type->name_off); 9108 verbose(env, 9109 "kernel function %s returns pointer type %s %s is not supported\n", 9110 func_name, 9111 btf_type_str(ptr_type), 9112 ptr_type_name); 9113 return -EINVAL; 9114 } 9115 9116 mark_reg_known_zero(env, regs, BPF_REG_0); 9117 regs[BPF_REG_0].type = PTR_TO_MEM; 9118 regs[BPF_REG_0].mem_size = meta.r0_size; 9119 9120 if (meta.r0_rdonly) 9121 regs[BPF_REG_0].type |= MEM_RDONLY; 9122 9123 /* Ensures we don't access the memory after a release_reference() */ 9124 if (meta.ref_obj_id) 9125 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 9126 } else { 9127 mark_reg_known_zero(env, regs, BPF_REG_0); 9128 regs[BPF_REG_0].btf = desc_btf; 9129 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 9130 regs[BPF_REG_0].btf_id = ptr_type_id; 9131 } 9132 9133 if (is_kfunc_ret_null(&meta)) { 9134 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 9135 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 9136 regs[BPF_REG_0].id = ++env->id_gen; 9137 } 9138 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 9139 if (is_kfunc_acquire(&meta)) { 9140 int id = acquire_reference_state(env, insn_idx); 9141 9142 if (id < 0) 9143 return id; 9144 if (is_kfunc_ret_null(&meta)) 9145 regs[BPF_REG_0].id = id; 9146 regs[BPF_REG_0].ref_obj_id = id; 9147 } 9148 if (reg_may_point_to_spin_lock(®s[BPF_REG_0]) && !regs[BPF_REG_0].id) 9149 regs[BPF_REG_0].id = ++env->id_gen; 9150 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 9151 9152 nargs = btf_type_vlen(func_proto); 9153 args = (const struct btf_param *)(func_proto + 1); 9154 for (i = 0; i < nargs; i++) { 9155 u32 regno = i + 1; 9156 9157 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 9158 if (btf_type_is_ptr(t)) 9159 mark_btf_func_reg_size(env, regno, sizeof(void *)); 9160 else 9161 /* scalar. ensured by btf_check_kfunc_arg_match() */ 9162 mark_btf_func_reg_size(env, regno, t->size); 9163 } 9164 9165 return 0; 9166 } 9167 9168 static bool signed_add_overflows(s64 a, s64 b) 9169 { 9170 /* Do the add in u64, where overflow is well-defined */ 9171 s64 res = (s64)((u64)a + (u64)b); 9172 9173 if (b < 0) 9174 return res > a; 9175 return res < a; 9176 } 9177 9178 static bool signed_add32_overflows(s32 a, s32 b) 9179 { 9180 /* Do the add in u32, where overflow is well-defined */ 9181 s32 res = (s32)((u32)a + (u32)b); 9182 9183 if (b < 0) 9184 return res > a; 9185 return res < a; 9186 } 9187 9188 static bool signed_sub_overflows(s64 a, s64 b) 9189 { 9190 /* Do the sub in u64, where overflow is well-defined */ 9191 s64 res = (s64)((u64)a - (u64)b); 9192 9193 if (b < 0) 9194 return res < a; 9195 return res > a; 9196 } 9197 9198 static bool signed_sub32_overflows(s32 a, s32 b) 9199 { 9200 /* Do the sub in u32, where overflow is well-defined */ 9201 s32 res = (s32)((u32)a - (u32)b); 9202 9203 if (b < 0) 9204 return res < a; 9205 return res > a; 9206 } 9207 9208 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 9209 const struct bpf_reg_state *reg, 9210 enum bpf_reg_type type) 9211 { 9212 bool known = tnum_is_const(reg->var_off); 9213 s64 val = reg->var_off.value; 9214 s64 smin = reg->smin_value; 9215 9216 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 9217 verbose(env, "math between %s pointer and %lld is not allowed\n", 9218 reg_type_str(env, type), val); 9219 return false; 9220 } 9221 9222 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 9223 verbose(env, "%s pointer offset %d is not allowed\n", 9224 reg_type_str(env, type), reg->off); 9225 return false; 9226 } 9227 9228 if (smin == S64_MIN) { 9229 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 9230 reg_type_str(env, type)); 9231 return false; 9232 } 9233 9234 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 9235 verbose(env, "value %lld makes %s pointer be out of bounds\n", 9236 smin, reg_type_str(env, type)); 9237 return false; 9238 } 9239 9240 return true; 9241 } 9242 9243 enum { 9244 REASON_BOUNDS = -1, 9245 REASON_TYPE = -2, 9246 REASON_PATHS = -3, 9247 REASON_LIMIT = -4, 9248 REASON_STACK = -5, 9249 }; 9250 9251 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 9252 u32 *alu_limit, bool mask_to_left) 9253 { 9254 u32 max = 0, ptr_limit = 0; 9255 9256 switch (ptr_reg->type) { 9257 case PTR_TO_STACK: 9258 /* Offset 0 is out-of-bounds, but acceptable start for the 9259 * left direction, see BPF_REG_FP. Also, unknown scalar 9260 * offset where we would need to deal with min/max bounds is 9261 * currently prohibited for unprivileged. 9262 */ 9263 max = MAX_BPF_STACK + mask_to_left; 9264 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 9265 break; 9266 case PTR_TO_MAP_VALUE: 9267 max = ptr_reg->map_ptr->value_size; 9268 ptr_limit = (mask_to_left ? 9269 ptr_reg->smin_value : 9270 ptr_reg->umax_value) + ptr_reg->off; 9271 break; 9272 default: 9273 return REASON_TYPE; 9274 } 9275 9276 if (ptr_limit >= max) 9277 return REASON_LIMIT; 9278 *alu_limit = ptr_limit; 9279 return 0; 9280 } 9281 9282 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 9283 const struct bpf_insn *insn) 9284 { 9285 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 9286 } 9287 9288 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 9289 u32 alu_state, u32 alu_limit) 9290 { 9291 /* If we arrived here from different branches with different 9292 * state or limits to sanitize, then this won't work. 9293 */ 9294 if (aux->alu_state && 9295 (aux->alu_state != alu_state || 9296 aux->alu_limit != alu_limit)) 9297 return REASON_PATHS; 9298 9299 /* Corresponding fixup done in do_misc_fixups(). */ 9300 aux->alu_state = alu_state; 9301 aux->alu_limit = alu_limit; 9302 return 0; 9303 } 9304 9305 static int sanitize_val_alu(struct bpf_verifier_env *env, 9306 struct bpf_insn *insn) 9307 { 9308 struct bpf_insn_aux_data *aux = cur_aux(env); 9309 9310 if (can_skip_alu_sanitation(env, insn)) 9311 return 0; 9312 9313 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 9314 } 9315 9316 static bool sanitize_needed(u8 opcode) 9317 { 9318 return opcode == BPF_ADD || opcode == BPF_SUB; 9319 } 9320 9321 struct bpf_sanitize_info { 9322 struct bpf_insn_aux_data aux; 9323 bool mask_to_left; 9324 }; 9325 9326 static struct bpf_verifier_state * 9327 sanitize_speculative_path(struct bpf_verifier_env *env, 9328 const struct bpf_insn *insn, 9329 u32 next_idx, u32 curr_idx) 9330 { 9331 struct bpf_verifier_state *branch; 9332 struct bpf_reg_state *regs; 9333 9334 branch = push_stack(env, next_idx, curr_idx, true); 9335 if (branch && insn) { 9336 regs = branch->frame[branch->curframe]->regs; 9337 if (BPF_SRC(insn->code) == BPF_K) { 9338 mark_reg_unknown(env, regs, insn->dst_reg); 9339 } else if (BPF_SRC(insn->code) == BPF_X) { 9340 mark_reg_unknown(env, regs, insn->dst_reg); 9341 mark_reg_unknown(env, regs, insn->src_reg); 9342 } 9343 } 9344 return branch; 9345 } 9346 9347 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 9348 struct bpf_insn *insn, 9349 const struct bpf_reg_state *ptr_reg, 9350 const struct bpf_reg_state *off_reg, 9351 struct bpf_reg_state *dst_reg, 9352 struct bpf_sanitize_info *info, 9353 const bool commit_window) 9354 { 9355 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 9356 struct bpf_verifier_state *vstate = env->cur_state; 9357 bool off_is_imm = tnum_is_const(off_reg->var_off); 9358 bool off_is_neg = off_reg->smin_value < 0; 9359 bool ptr_is_dst_reg = ptr_reg == dst_reg; 9360 u8 opcode = BPF_OP(insn->code); 9361 u32 alu_state, alu_limit; 9362 struct bpf_reg_state tmp; 9363 bool ret; 9364 int err; 9365 9366 if (can_skip_alu_sanitation(env, insn)) 9367 return 0; 9368 9369 /* We already marked aux for masking from non-speculative 9370 * paths, thus we got here in the first place. We only care 9371 * to explore bad access from here. 9372 */ 9373 if (vstate->speculative) 9374 goto do_sim; 9375 9376 if (!commit_window) { 9377 if (!tnum_is_const(off_reg->var_off) && 9378 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 9379 return REASON_BOUNDS; 9380 9381 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 9382 (opcode == BPF_SUB && !off_is_neg); 9383 } 9384 9385 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 9386 if (err < 0) 9387 return err; 9388 9389 if (commit_window) { 9390 /* In commit phase we narrow the masking window based on 9391 * the observed pointer move after the simulated operation. 9392 */ 9393 alu_state = info->aux.alu_state; 9394 alu_limit = abs(info->aux.alu_limit - alu_limit); 9395 } else { 9396 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 9397 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 9398 alu_state |= ptr_is_dst_reg ? 9399 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 9400 9401 /* Limit pruning on unknown scalars to enable deep search for 9402 * potential masking differences from other program paths. 9403 */ 9404 if (!off_is_imm) 9405 env->explore_alu_limits = true; 9406 } 9407 9408 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 9409 if (err < 0) 9410 return err; 9411 do_sim: 9412 /* If we're in commit phase, we're done here given we already 9413 * pushed the truncated dst_reg into the speculative verification 9414 * stack. 9415 * 9416 * Also, when register is a known constant, we rewrite register-based 9417 * operation to immediate-based, and thus do not need masking (and as 9418 * a consequence, do not need to simulate the zero-truncation either). 9419 */ 9420 if (commit_window || off_is_imm) 9421 return 0; 9422 9423 /* Simulate and find potential out-of-bounds access under 9424 * speculative execution from truncation as a result of 9425 * masking when off was not within expected range. If off 9426 * sits in dst, then we temporarily need to move ptr there 9427 * to simulate dst (== 0) +/-= ptr. Needed, for example, 9428 * for cases where we use K-based arithmetic in one direction 9429 * and truncated reg-based in the other in order to explore 9430 * bad access. 9431 */ 9432 if (!ptr_is_dst_reg) { 9433 tmp = *dst_reg; 9434 *dst_reg = *ptr_reg; 9435 } 9436 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 9437 env->insn_idx); 9438 if (!ptr_is_dst_reg && ret) 9439 *dst_reg = tmp; 9440 return !ret ? REASON_STACK : 0; 9441 } 9442 9443 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 9444 { 9445 struct bpf_verifier_state *vstate = env->cur_state; 9446 9447 /* If we simulate paths under speculation, we don't update the 9448 * insn as 'seen' such that when we verify unreachable paths in 9449 * the non-speculative domain, sanitize_dead_code() can still 9450 * rewrite/sanitize them. 9451 */ 9452 if (!vstate->speculative) 9453 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 9454 } 9455 9456 static int sanitize_err(struct bpf_verifier_env *env, 9457 const struct bpf_insn *insn, int reason, 9458 const struct bpf_reg_state *off_reg, 9459 const struct bpf_reg_state *dst_reg) 9460 { 9461 static const char *err = "pointer arithmetic with it prohibited for !root"; 9462 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 9463 u32 dst = insn->dst_reg, src = insn->src_reg; 9464 9465 switch (reason) { 9466 case REASON_BOUNDS: 9467 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 9468 off_reg == dst_reg ? dst : src, err); 9469 break; 9470 case REASON_TYPE: 9471 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 9472 off_reg == dst_reg ? src : dst, err); 9473 break; 9474 case REASON_PATHS: 9475 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 9476 dst, op, err); 9477 break; 9478 case REASON_LIMIT: 9479 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 9480 dst, op, err); 9481 break; 9482 case REASON_STACK: 9483 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 9484 dst, err); 9485 break; 9486 default: 9487 verbose(env, "verifier internal error: unknown reason (%d)\n", 9488 reason); 9489 break; 9490 } 9491 9492 return -EACCES; 9493 } 9494 9495 /* check that stack access falls within stack limits and that 'reg' doesn't 9496 * have a variable offset. 9497 * 9498 * Variable offset is prohibited for unprivileged mode for simplicity since it 9499 * requires corresponding support in Spectre masking for stack ALU. See also 9500 * retrieve_ptr_limit(). 9501 * 9502 * 9503 * 'off' includes 'reg->off'. 9504 */ 9505 static int check_stack_access_for_ptr_arithmetic( 9506 struct bpf_verifier_env *env, 9507 int regno, 9508 const struct bpf_reg_state *reg, 9509 int off) 9510 { 9511 if (!tnum_is_const(reg->var_off)) { 9512 char tn_buf[48]; 9513 9514 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 9515 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 9516 regno, tn_buf, off); 9517 return -EACCES; 9518 } 9519 9520 if (off >= 0 || off < -MAX_BPF_STACK) { 9521 verbose(env, "R%d stack pointer arithmetic goes out of range, " 9522 "prohibited for !root; off=%d\n", regno, off); 9523 return -EACCES; 9524 } 9525 9526 return 0; 9527 } 9528 9529 static int sanitize_check_bounds(struct bpf_verifier_env *env, 9530 const struct bpf_insn *insn, 9531 const struct bpf_reg_state *dst_reg) 9532 { 9533 u32 dst = insn->dst_reg; 9534 9535 /* For unprivileged we require that resulting offset must be in bounds 9536 * in order to be able to sanitize access later on. 9537 */ 9538 if (env->bypass_spec_v1) 9539 return 0; 9540 9541 switch (dst_reg->type) { 9542 case PTR_TO_STACK: 9543 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 9544 dst_reg->off + dst_reg->var_off.value)) 9545 return -EACCES; 9546 break; 9547 case PTR_TO_MAP_VALUE: 9548 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 9549 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 9550 "prohibited for !root\n", dst); 9551 return -EACCES; 9552 } 9553 break; 9554 default: 9555 break; 9556 } 9557 9558 return 0; 9559 } 9560 9561 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 9562 * Caller should also handle BPF_MOV case separately. 9563 * If we return -EACCES, caller may want to try again treating pointer as a 9564 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 9565 */ 9566 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 9567 struct bpf_insn *insn, 9568 const struct bpf_reg_state *ptr_reg, 9569 const struct bpf_reg_state *off_reg) 9570 { 9571 struct bpf_verifier_state *vstate = env->cur_state; 9572 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 9573 struct bpf_reg_state *regs = state->regs, *dst_reg; 9574 bool known = tnum_is_const(off_reg->var_off); 9575 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 9576 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 9577 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 9578 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 9579 struct bpf_sanitize_info info = {}; 9580 u8 opcode = BPF_OP(insn->code); 9581 u32 dst = insn->dst_reg; 9582 int ret; 9583 9584 dst_reg = ®s[dst]; 9585 9586 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 9587 smin_val > smax_val || umin_val > umax_val) { 9588 /* Taint dst register if offset had invalid bounds derived from 9589 * e.g. dead branches. 9590 */ 9591 __mark_reg_unknown(env, dst_reg); 9592 return 0; 9593 } 9594 9595 if (BPF_CLASS(insn->code) != BPF_ALU64) { 9596 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 9597 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 9598 __mark_reg_unknown(env, dst_reg); 9599 return 0; 9600 } 9601 9602 verbose(env, 9603 "R%d 32-bit pointer arithmetic prohibited\n", 9604 dst); 9605 return -EACCES; 9606 } 9607 9608 if (ptr_reg->type & PTR_MAYBE_NULL) { 9609 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 9610 dst, reg_type_str(env, ptr_reg->type)); 9611 return -EACCES; 9612 } 9613 9614 switch (base_type(ptr_reg->type)) { 9615 case CONST_PTR_TO_MAP: 9616 /* smin_val represents the known value */ 9617 if (known && smin_val == 0 && opcode == BPF_ADD) 9618 break; 9619 fallthrough; 9620 case PTR_TO_PACKET_END: 9621 case PTR_TO_SOCKET: 9622 case PTR_TO_SOCK_COMMON: 9623 case PTR_TO_TCP_SOCK: 9624 case PTR_TO_XDP_SOCK: 9625 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 9626 dst, reg_type_str(env, ptr_reg->type)); 9627 return -EACCES; 9628 default: 9629 break; 9630 } 9631 9632 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 9633 * The id may be overwritten later if we create a new variable offset. 9634 */ 9635 dst_reg->type = ptr_reg->type; 9636 dst_reg->id = ptr_reg->id; 9637 9638 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 9639 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 9640 return -EINVAL; 9641 9642 /* pointer types do not carry 32-bit bounds at the moment. */ 9643 __mark_reg32_unbounded(dst_reg); 9644 9645 if (sanitize_needed(opcode)) { 9646 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 9647 &info, false); 9648 if (ret < 0) 9649 return sanitize_err(env, insn, ret, off_reg, dst_reg); 9650 } 9651 9652 switch (opcode) { 9653 case BPF_ADD: 9654 /* We can take a fixed offset as long as it doesn't overflow 9655 * the s32 'off' field 9656 */ 9657 if (known && (ptr_reg->off + smin_val == 9658 (s64)(s32)(ptr_reg->off + smin_val))) { 9659 /* pointer += K. Accumulate it into fixed offset */ 9660 dst_reg->smin_value = smin_ptr; 9661 dst_reg->smax_value = smax_ptr; 9662 dst_reg->umin_value = umin_ptr; 9663 dst_reg->umax_value = umax_ptr; 9664 dst_reg->var_off = ptr_reg->var_off; 9665 dst_reg->off = ptr_reg->off + smin_val; 9666 dst_reg->raw = ptr_reg->raw; 9667 break; 9668 } 9669 /* A new variable offset is created. Note that off_reg->off 9670 * == 0, since it's a scalar. 9671 * dst_reg gets the pointer type and since some positive 9672 * integer value was added to the pointer, give it a new 'id' 9673 * if it's a PTR_TO_PACKET. 9674 * this creates a new 'base' pointer, off_reg (variable) gets 9675 * added into the variable offset, and we copy the fixed offset 9676 * from ptr_reg. 9677 */ 9678 if (signed_add_overflows(smin_ptr, smin_val) || 9679 signed_add_overflows(smax_ptr, smax_val)) { 9680 dst_reg->smin_value = S64_MIN; 9681 dst_reg->smax_value = S64_MAX; 9682 } else { 9683 dst_reg->smin_value = smin_ptr + smin_val; 9684 dst_reg->smax_value = smax_ptr + smax_val; 9685 } 9686 if (umin_ptr + umin_val < umin_ptr || 9687 umax_ptr + umax_val < umax_ptr) { 9688 dst_reg->umin_value = 0; 9689 dst_reg->umax_value = U64_MAX; 9690 } else { 9691 dst_reg->umin_value = umin_ptr + umin_val; 9692 dst_reg->umax_value = umax_ptr + umax_val; 9693 } 9694 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 9695 dst_reg->off = ptr_reg->off; 9696 dst_reg->raw = ptr_reg->raw; 9697 if (reg_is_pkt_pointer(ptr_reg)) { 9698 dst_reg->id = ++env->id_gen; 9699 /* something was added to pkt_ptr, set range to zero */ 9700 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 9701 } 9702 break; 9703 case BPF_SUB: 9704 if (dst_reg == off_reg) { 9705 /* scalar -= pointer. Creates an unknown scalar */ 9706 verbose(env, "R%d tried to subtract pointer from scalar\n", 9707 dst); 9708 return -EACCES; 9709 } 9710 /* We don't allow subtraction from FP, because (according to 9711 * test_verifier.c test "invalid fp arithmetic", JITs might not 9712 * be able to deal with it. 9713 */ 9714 if (ptr_reg->type == PTR_TO_STACK) { 9715 verbose(env, "R%d subtraction from stack pointer prohibited\n", 9716 dst); 9717 return -EACCES; 9718 } 9719 if (known && (ptr_reg->off - smin_val == 9720 (s64)(s32)(ptr_reg->off - smin_val))) { 9721 /* pointer -= K. Subtract it from fixed offset */ 9722 dst_reg->smin_value = smin_ptr; 9723 dst_reg->smax_value = smax_ptr; 9724 dst_reg->umin_value = umin_ptr; 9725 dst_reg->umax_value = umax_ptr; 9726 dst_reg->var_off = ptr_reg->var_off; 9727 dst_reg->id = ptr_reg->id; 9728 dst_reg->off = ptr_reg->off - smin_val; 9729 dst_reg->raw = ptr_reg->raw; 9730 break; 9731 } 9732 /* A new variable offset is created. If the subtrahend is known 9733 * nonnegative, then any reg->range we had before is still good. 9734 */ 9735 if (signed_sub_overflows(smin_ptr, smax_val) || 9736 signed_sub_overflows(smax_ptr, smin_val)) { 9737 /* Overflow possible, we know nothing */ 9738 dst_reg->smin_value = S64_MIN; 9739 dst_reg->smax_value = S64_MAX; 9740 } else { 9741 dst_reg->smin_value = smin_ptr - smax_val; 9742 dst_reg->smax_value = smax_ptr - smin_val; 9743 } 9744 if (umin_ptr < umax_val) { 9745 /* Overflow possible, we know nothing */ 9746 dst_reg->umin_value = 0; 9747 dst_reg->umax_value = U64_MAX; 9748 } else { 9749 /* Cannot overflow (as long as bounds are consistent) */ 9750 dst_reg->umin_value = umin_ptr - umax_val; 9751 dst_reg->umax_value = umax_ptr - umin_val; 9752 } 9753 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 9754 dst_reg->off = ptr_reg->off; 9755 dst_reg->raw = ptr_reg->raw; 9756 if (reg_is_pkt_pointer(ptr_reg)) { 9757 dst_reg->id = ++env->id_gen; 9758 /* something was added to pkt_ptr, set range to zero */ 9759 if (smin_val < 0) 9760 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 9761 } 9762 break; 9763 case BPF_AND: 9764 case BPF_OR: 9765 case BPF_XOR: 9766 /* bitwise ops on pointers are troublesome, prohibit. */ 9767 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 9768 dst, bpf_alu_string[opcode >> 4]); 9769 return -EACCES; 9770 default: 9771 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 9772 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 9773 dst, bpf_alu_string[opcode >> 4]); 9774 return -EACCES; 9775 } 9776 9777 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 9778 return -EINVAL; 9779 reg_bounds_sync(dst_reg); 9780 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 9781 return -EACCES; 9782 if (sanitize_needed(opcode)) { 9783 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 9784 &info, true); 9785 if (ret < 0) 9786 return sanitize_err(env, insn, ret, off_reg, dst_reg); 9787 } 9788 9789 return 0; 9790 } 9791 9792 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 9793 struct bpf_reg_state *src_reg) 9794 { 9795 s32 smin_val = src_reg->s32_min_value; 9796 s32 smax_val = src_reg->s32_max_value; 9797 u32 umin_val = src_reg->u32_min_value; 9798 u32 umax_val = src_reg->u32_max_value; 9799 9800 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 9801 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 9802 dst_reg->s32_min_value = S32_MIN; 9803 dst_reg->s32_max_value = S32_MAX; 9804 } else { 9805 dst_reg->s32_min_value += smin_val; 9806 dst_reg->s32_max_value += smax_val; 9807 } 9808 if (dst_reg->u32_min_value + umin_val < umin_val || 9809 dst_reg->u32_max_value + umax_val < umax_val) { 9810 dst_reg->u32_min_value = 0; 9811 dst_reg->u32_max_value = U32_MAX; 9812 } else { 9813 dst_reg->u32_min_value += umin_val; 9814 dst_reg->u32_max_value += umax_val; 9815 } 9816 } 9817 9818 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 9819 struct bpf_reg_state *src_reg) 9820 { 9821 s64 smin_val = src_reg->smin_value; 9822 s64 smax_val = src_reg->smax_value; 9823 u64 umin_val = src_reg->umin_value; 9824 u64 umax_val = src_reg->umax_value; 9825 9826 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 9827 signed_add_overflows(dst_reg->smax_value, smax_val)) { 9828 dst_reg->smin_value = S64_MIN; 9829 dst_reg->smax_value = S64_MAX; 9830 } else { 9831 dst_reg->smin_value += smin_val; 9832 dst_reg->smax_value += smax_val; 9833 } 9834 if (dst_reg->umin_value + umin_val < umin_val || 9835 dst_reg->umax_value + umax_val < umax_val) { 9836 dst_reg->umin_value = 0; 9837 dst_reg->umax_value = U64_MAX; 9838 } else { 9839 dst_reg->umin_value += umin_val; 9840 dst_reg->umax_value += umax_val; 9841 } 9842 } 9843 9844 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 9845 struct bpf_reg_state *src_reg) 9846 { 9847 s32 smin_val = src_reg->s32_min_value; 9848 s32 smax_val = src_reg->s32_max_value; 9849 u32 umin_val = src_reg->u32_min_value; 9850 u32 umax_val = src_reg->u32_max_value; 9851 9852 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 9853 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 9854 /* Overflow possible, we know nothing */ 9855 dst_reg->s32_min_value = S32_MIN; 9856 dst_reg->s32_max_value = S32_MAX; 9857 } else { 9858 dst_reg->s32_min_value -= smax_val; 9859 dst_reg->s32_max_value -= smin_val; 9860 } 9861 if (dst_reg->u32_min_value < umax_val) { 9862 /* Overflow possible, we know nothing */ 9863 dst_reg->u32_min_value = 0; 9864 dst_reg->u32_max_value = U32_MAX; 9865 } else { 9866 /* Cannot overflow (as long as bounds are consistent) */ 9867 dst_reg->u32_min_value -= umax_val; 9868 dst_reg->u32_max_value -= umin_val; 9869 } 9870 } 9871 9872 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 9873 struct bpf_reg_state *src_reg) 9874 { 9875 s64 smin_val = src_reg->smin_value; 9876 s64 smax_val = src_reg->smax_value; 9877 u64 umin_val = src_reg->umin_value; 9878 u64 umax_val = src_reg->umax_value; 9879 9880 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 9881 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 9882 /* Overflow possible, we know nothing */ 9883 dst_reg->smin_value = S64_MIN; 9884 dst_reg->smax_value = S64_MAX; 9885 } else { 9886 dst_reg->smin_value -= smax_val; 9887 dst_reg->smax_value -= smin_val; 9888 } 9889 if (dst_reg->umin_value < umax_val) { 9890 /* Overflow possible, we know nothing */ 9891 dst_reg->umin_value = 0; 9892 dst_reg->umax_value = U64_MAX; 9893 } else { 9894 /* Cannot overflow (as long as bounds are consistent) */ 9895 dst_reg->umin_value -= umax_val; 9896 dst_reg->umax_value -= umin_val; 9897 } 9898 } 9899 9900 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 9901 struct bpf_reg_state *src_reg) 9902 { 9903 s32 smin_val = src_reg->s32_min_value; 9904 u32 umin_val = src_reg->u32_min_value; 9905 u32 umax_val = src_reg->u32_max_value; 9906 9907 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 9908 /* Ain't nobody got time to multiply that sign */ 9909 __mark_reg32_unbounded(dst_reg); 9910 return; 9911 } 9912 /* Both values are positive, so we can work with unsigned and 9913 * copy the result to signed (unless it exceeds S32_MAX). 9914 */ 9915 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 9916 /* Potential overflow, we know nothing */ 9917 __mark_reg32_unbounded(dst_reg); 9918 return; 9919 } 9920 dst_reg->u32_min_value *= umin_val; 9921 dst_reg->u32_max_value *= umax_val; 9922 if (dst_reg->u32_max_value > S32_MAX) { 9923 /* Overflow possible, we know nothing */ 9924 dst_reg->s32_min_value = S32_MIN; 9925 dst_reg->s32_max_value = S32_MAX; 9926 } else { 9927 dst_reg->s32_min_value = dst_reg->u32_min_value; 9928 dst_reg->s32_max_value = dst_reg->u32_max_value; 9929 } 9930 } 9931 9932 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 9933 struct bpf_reg_state *src_reg) 9934 { 9935 s64 smin_val = src_reg->smin_value; 9936 u64 umin_val = src_reg->umin_value; 9937 u64 umax_val = src_reg->umax_value; 9938 9939 if (smin_val < 0 || dst_reg->smin_value < 0) { 9940 /* Ain't nobody got time to multiply that sign */ 9941 __mark_reg64_unbounded(dst_reg); 9942 return; 9943 } 9944 /* Both values are positive, so we can work with unsigned and 9945 * copy the result to signed (unless it exceeds S64_MAX). 9946 */ 9947 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 9948 /* Potential overflow, we know nothing */ 9949 __mark_reg64_unbounded(dst_reg); 9950 return; 9951 } 9952 dst_reg->umin_value *= umin_val; 9953 dst_reg->umax_value *= umax_val; 9954 if (dst_reg->umax_value > S64_MAX) { 9955 /* Overflow possible, we know nothing */ 9956 dst_reg->smin_value = S64_MIN; 9957 dst_reg->smax_value = S64_MAX; 9958 } else { 9959 dst_reg->smin_value = dst_reg->umin_value; 9960 dst_reg->smax_value = dst_reg->umax_value; 9961 } 9962 } 9963 9964 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 9965 struct bpf_reg_state *src_reg) 9966 { 9967 bool src_known = tnum_subreg_is_const(src_reg->var_off); 9968 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 9969 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 9970 s32 smin_val = src_reg->s32_min_value; 9971 u32 umax_val = src_reg->u32_max_value; 9972 9973 if (src_known && dst_known) { 9974 __mark_reg32_known(dst_reg, var32_off.value); 9975 return; 9976 } 9977 9978 /* We get our minimum from the var_off, since that's inherently 9979 * bitwise. Our maximum is the minimum of the operands' maxima. 9980 */ 9981 dst_reg->u32_min_value = var32_off.value; 9982 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 9983 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 9984 /* Lose signed bounds when ANDing negative numbers, 9985 * ain't nobody got time for that. 9986 */ 9987 dst_reg->s32_min_value = S32_MIN; 9988 dst_reg->s32_max_value = S32_MAX; 9989 } else { 9990 /* ANDing two positives gives a positive, so safe to 9991 * cast result into s64. 9992 */ 9993 dst_reg->s32_min_value = dst_reg->u32_min_value; 9994 dst_reg->s32_max_value = dst_reg->u32_max_value; 9995 } 9996 } 9997 9998 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 9999 struct bpf_reg_state *src_reg) 10000 { 10001 bool src_known = tnum_is_const(src_reg->var_off); 10002 bool dst_known = tnum_is_const(dst_reg->var_off); 10003 s64 smin_val = src_reg->smin_value; 10004 u64 umax_val = src_reg->umax_value; 10005 10006 if (src_known && dst_known) { 10007 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10008 return; 10009 } 10010 10011 /* We get our minimum from the var_off, since that's inherently 10012 * bitwise. Our maximum is the minimum of the operands' maxima. 10013 */ 10014 dst_reg->umin_value = dst_reg->var_off.value; 10015 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 10016 if (dst_reg->smin_value < 0 || smin_val < 0) { 10017 /* Lose signed bounds when ANDing negative numbers, 10018 * ain't nobody got time for that. 10019 */ 10020 dst_reg->smin_value = S64_MIN; 10021 dst_reg->smax_value = S64_MAX; 10022 } else { 10023 /* ANDing two positives gives a positive, so safe to 10024 * cast result into s64. 10025 */ 10026 dst_reg->smin_value = dst_reg->umin_value; 10027 dst_reg->smax_value = dst_reg->umax_value; 10028 } 10029 /* We may learn something more from the var_off */ 10030 __update_reg_bounds(dst_reg); 10031 } 10032 10033 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 10034 struct bpf_reg_state *src_reg) 10035 { 10036 bool src_known = tnum_subreg_is_const(src_reg->var_off); 10037 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 10038 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 10039 s32 smin_val = src_reg->s32_min_value; 10040 u32 umin_val = src_reg->u32_min_value; 10041 10042 if (src_known && dst_known) { 10043 __mark_reg32_known(dst_reg, var32_off.value); 10044 return; 10045 } 10046 10047 /* We get our maximum from the var_off, and our minimum is the 10048 * maximum of the operands' minima 10049 */ 10050 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 10051 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 10052 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 10053 /* Lose signed bounds when ORing negative numbers, 10054 * ain't nobody got time for that. 10055 */ 10056 dst_reg->s32_min_value = S32_MIN; 10057 dst_reg->s32_max_value = S32_MAX; 10058 } else { 10059 /* ORing two positives gives a positive, so safe to 10060 * cast result into s64. 10061 */ 10062 dst_reg->s32_min_value = dst_reg->u32_min_value; 10063 dst_reg->s32_max_value = dst_reg->u32_max_value; 10064 } 10065 } 10066 10067 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 10068 struct bpf_reg_state *src_reg) 10069 { 10070 bool src_known = tnum_is_const(src_reg->var_off); 10071 bool dst_known = tnum_is_const(dst_reg->var_off); 10072 s64 smin_val = src_reg->smin_value; 10073 u64 umin_val = src_reg->umin_value; 10074 10075 if (src_known && dst_known) { 10076 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10077 return; 10078 } 10079 10080 /* We get our maximum from the var_off, and our minimum is the 10081 * maximum of the operands' minima 10082 */ 10083 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 10084 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 10085 if (dst_reg->smin_value < 0 || smin_val < 0) { 10086 /* Lose signed bounds when ORing negative numbers, 10087 * ain't nobody got time for that. 10088 */ 10089 dst_reg->smin_value = S64_MIN; 10090 dst_reg->smax_value = S64_MAX; 10091 } else { 10092 /* ORing two positives gives a positive, so safe to 10093 * cast result into s64. 10094 */ 10095 dst_reg->smin_value = dst_reg->umin_value; 10096 dst_reg->smax_value = dst_reg->umax_value; 10097 } 10098 /* We may learn something more from the var_off */ 10099 __update_reg_bounds(dst_reg); 10100 } 10101 10102 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 10103 struct bpf_reg_state *src_reg) 10104 { 10105 bool src_known = tnum_subreg_is_const(src_reg->var_off); 10106 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 10107 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 10108 s32 smin_val = src_reg->s32_min_value; 10109 10110 if (src_known && dst_known) { 10111 __mark_reg32_known(dst_reg, var32_off.value); 10112 return; 10113 } 10114 10115 /* We get both minimum and maximum from the var32_off. */ 10116 dst_reg->u32_min_value = var32_off.value; 10117 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 10118 10119 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 10120 /* XORing two positive sign numbers gives a positive, 10121 * so safe to cast u32 result into s32. 10122 */ 10123 dst_reg->s32_min_value = dst_reg->u32_min_value; 10124 dst_reg->s32_max_value = dst_reg->u32_max_value; 10125 } else { 10126 dst_reg->s32_min_value = S32_MIN; 10127 dst_reg->s32_max_value = S32_MAX; 10128 } 10129 } 10130 10131 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 10132 struct bpf_reg_state *src_reg) 10133 { 10134 bool src_known = tnum_is_const(src_reg->var_off); 10135 bool dst_known = tnum_is_const(dst_reg->var_off); 10136 s64 smin_val = src_reg->smin_value; 10137 10138 if (src_known && dst_known) { 10139 /* dst_reg->var_off.value has been updated earlier */ 10140 __mark_reg_known(dst_reg, dst_reg->var_off.value); 10141 return; 10142 } 10143 10144 /* We get both minimum and maximum from the var_off. */ 10145 dst_reg->umin_value = dst_reg->var_off.value; 10146 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 10147 10148 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 10149 /* XORing two positive sign numbers gives a positive, 10150 * so safe to cast u64 result into s64. 10151 */ 10152 dst_reg->smin_value = dst_reg->umin_value; 10153 dst_reg->smax_value = dst_reg->umax_value; 10154 } else { 10155 dst_reg->smin_value = S64_MIN; 10156 dst_reg->smax_value = S64_MAX; 10157 } 10158 10159 __update_reg_bounds(dst_reg); 10160 } 10161 10162 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 10163 u64 umin_val, u64 umax_val) 10164 { 10165 /* We lose all sign bit information (except what we can pick 10166 * up from var_off) 10167 */ 10168 dst_reg->s32_min_value = S32_MIN; 10169 dst_reg->s32_max_value = S32_MAX; 10170 /* If we might shift our top bit out, then we know nothing */ 10171 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 10172 dst_reg->u32_min_value = 0; 10173 dst_reg->u32_max_value = U32_MAX; 10174 } else { 10175 dst_reg->u32_min_value <<= umin_val; 10176 dst_reg->u32_max_value <<= umax_val; 10177 } 10178 } 10179 10180 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 10181 struct bpf_reg_state *src_reg) 10182 { 10183 u32 umax_val = src_reg->u32_max_value; 10184 u32 umin_val = src_reg->u32_min_value; 10185 /* u32 alu operation will zext upper bits */ 10186 struct tnum subreg = tnum_subreg(dst_reg->var_off); 10187 10188 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 10189 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 10190 /* Not required but being careful mark reg64 bounds as unknown so 10191 * that we are forced to pick them up from tnum and zext later and 10192 * if some path skips this step we are still safe. 10193 */ 10194 __mark_reg64_unbounded(dst_reg); 10195 __update_reg32_bounds(dst_reg); 10196 } 10197 10198 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 10199 u64 umin_val, u64 umax_val) 10200 { 10201 /* Special case <<32 because it is a common compiler pattern to sign 10202 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 10203 * positive we know this shift will also be positive so we can track 10204 * bounds correctly. Otherwise we lose all sign bit information except 10205 * what we can pick up from var_off. Perhaps we can generalize this 10206 * later to shifts of any length. 10207 */ 10208 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 10209 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 10210 else 10211 dst_reg->smax_value = S64_MAX; 10212 10213 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 10214 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 10215 else 10216 dst_reg->smin_value = S64_MIN; 10217 10218 /* If we might shift our top bit out, then we know nothing */ 10219 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 10220 dst_reg->umin_value = 0; 10221 dst_reg->umax_value = U64_MAX; 10222 } else { 10223 dst_reg->umin_value <<= umin_val; 10224 dst_reg->umax_value <<= umax_val; 10225 } 10226 } 10227 10228 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 10229 struct bpf_reg_state *src_reg) 10230 { 10231 u64 umax_val = src_reg->umax_value; 10232 u64 umin_val = src_reg->umin_value; 10233 10234 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 10235 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 10236 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 10237 10238 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 10239 /* We may learn something more from the var_off */ 10240 __update_reg_bounds(dst_reg); 10241 } 10242 10243 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 10244 struct bpf_reg_state *src_reg) 10245 { 10246 struct tnum subreg = tnum_subreg(dst_reg->var_off); 10247 u32 umax_val = src_reg->u32_max_value; 10248 u32 umin_val = src_reg->u32_min_value; 10249 10250 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 10251 * be negative, then either: 10252 * 1) src_reg might be zero, so the sign bit of the result is 10253 * unknown, so we lose our signed bounds 10254 * 2) it's known negative, thus the unsigned bounds capture the 10255 * signed bounds 10256 * 3) the signed bounds cross zero, so they tell us nothing 10257 * about the result 10258 * If the value in dst_reg is known nonnegative, then again the 10259 * unsigned bounds capture the signed bounds. 10260 * Thus, in all cases it suffices to blow away our signed bounds 10261 * and rely on inferring new ones from the unsigned bounds and 10262 * var_off of the result. 10263 */ 10264 dst_reg->s32_min_value = S32_MIN; 10265 dst_reg->s32_max_value = S32_MAX; 10266 10267 dst_reg->var_off = tnum_rshift(subreg, umin_val); 10268 dst_reg->u32_min_value >>= umax_val; 10269 dst_reg->u32_max_value >>= umin_val; 10270 10271 __mark_reg64_unbounded(dst_reg); 10272 __update_reg32_bounds(dst_reg); 10273 } 10274 10275 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 10276 struct bpf_reg_state *src_reg) 10277 { 10278 u64 umax_val = src_reg->umax_value; 10279 u64 umin_val = src_reg->umin_value; 10280 10281 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 10282 * be negative, then either: 10283 * 1) src_reg might be zero, so the sign bit of the result is 10284 * unknown, so we lose our signed bounds 10285 * 2) it's known negative, thus the unsigned bounds capture the 10286 * signed bounds 10287 * 3) the signed bounds cross zero, so they tell us nothing 10288 * about the result 10289 * If the value in dst_reg is known nonnegative, then again the 10290 * unsigned bounds capture the signed bounds. 10291 * Thus, in all cases it suffices to blow away our signed bounds 10292 * and rely on inferring new ones from the unsigned bounds and 10293 * var_off of the result. 10294 */ 10295 dst_reg->smin_value = S64_MIN; 10296 dst_reg->smax_value = S64_MAX; 10297 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 10298 dst_reg->umin_value >>= umax_val; 10299 dst_reg->umax_value >>= umin_val; 10300 10301 /* Its not easy to operate on alu32 bounds here because it depends 10302 * on bits being shifted in. Take easy way out and mark unbounded 10303 * so we can recalculate later from tnum. 10304 */ 10305 __mark_reg32_unbounded(dst_reg); 10306 __update_reg_bounds(dst_reg); 10307 } 10308 10309 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 10310 struct bpf_reg_state *src_reg) 10311 { 10312 u64 umin_val = src_reg->u32_min_value; 10313 10314 /* Upon reaching here, src_known is true and 10315 * umax_val is equal to umin_val. 10316 */ 10317 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 10318 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 10319 10320 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 10321 10322 /* blow away the dst_reg umin_value/umax_value and rely on 10323 * dst_reg var_off to refine the result. 10324 */ 10325 dst_reg->u32_min_value = 0; 10326 dst_reg->u32_max_value = U32_MAX; 10327 10328 __mark_reg64_unbounded(dst_reg); 10329 __update_reg32_bounds(dst_reg); 10330 } 10331 10332 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 10333 struct bpf_reg_state *src_reg) 10334 { 10335 u64 umin_val = src_reg->umin_value; 10336 10337 /* Upon reaching here, src_known is true and umax_val is equal 10338 * to umin_val. 10339 */ 10340 dst_reg->smin_value >>= umin_val; 10341 dst_reg->smax_value >>= umin_val; 10342 10343 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 10344 10345 /* blow away the dst_reg umin_value/umax_value and rely on 10346 * dst_reg var_off to refine the result. 10347 */ 10348 dst_reg->umin_value = 0; 10349 dst_reg->umax_value = U64_MAX; 10350 10351 /* Its not easy to operate on alu32 bounds here because it depends 10352 * on bits being shifted in from upper 32-bits. Take easy way out 10353 * and mark unbounded so we can recalculate later from tnum. 10354 */ 10355 __mark_reg32_unbounded(dst_reg); 10356 __update_reg_bounds(dst_reg); 10357 } 10358 10359 /* WARNING: This function does calculations on 64-bit values, but the actual 10360 * execution may occur on 32-bit values. Therefore, things like bitshifts 10361 * need extra checks in the 32-bit case. 10362 */ 10363 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 10364 struct bpf_insn *insn, 10365 struct bpf_reg_state *dst_reg, 10366 struct bpf_reg_state src_reg) 10367 { 10368 struct bpf_reg_state *regs = cur_regs(env); 10369 u8 opcode = BPF_OP(insn->code); 10370 bool src_known; 10371 s64 smin_val, smax_val; 10372 u64 umin_val, umax_val; 10373 s32 s32_min_val, s32_max_val; 10374 u32 u32_min_val, u32_max_val; 10375 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 10376 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 10377 int ret; 10378 10379 smin_val = src_reg.smin_value; 10380 smax_val = src_reg.smax_value; 10381 umin_val = src_reg.umin_value; 10382 umax_val = src_reg.umax_value; 10383 10384 s32_min_val = src_reg.s32_min_value; 10385 s32_max_val = src_reg.s32_max_value; 10386 u32_min_val = src_reg.u32_min_value; 10387 u32_max_val = src_reg.u32_max_value; 10388 10389 if (alu32) { 10390 src_known = tnum_subreg_is_const(src_reg.var_off); 10391 if ((src_known && 10392 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 10393 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 10394 /* Taint dst register if offset had invalid bounds 10395 * derived from e.g. dead branches. 10396 */ 10397 __mark_reg_unknown(env, dst_reg); 10398 return 0; 10399 } 10400 } else { 10401 src_known = tnum_is_const(src_reg.var_off); 10402 if ((src_known && 10403 (smin_val != smax_val || umin_val != umax_val)) || 10404 smin_val > smax_val || umin_val > umax_val) { 10405 /* Taint dst register if offset had invalid bounds 10406 * derived from e.g. dead branches. 10407 */ 10408 __mark_reg_unknown(env, dst_reg); 10409 return 0; 10410 } 10411 } 10412 10413 if (!src_known && 10414 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 10415 __mark_reg_unknown(env, dst_reg); 10416 return 0; 10417 } 10418 10419 if (sanitize_needed(opcode)) { 10420 ret = sanitize_val_alu(env, insn); 10421 if (ret < 0) 10422 return sanitize_err(env, insn, ret, NULL, NULL); 10423 } 10424 10425 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 10426 * There are two classes of instructions: The first class we track both 10427 * alu32 and alu64 sign/unsigned bounds independently this provides the 10428 * greatest amount of precision when alu operations are mixed with jmp32 10429 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 10430 * and BPF_OR. This is possible because these ops have fairly easy to 10431 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 10432 * See alu32 verifier tests for examples. The second class of 10433 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 10434 * with regards to tracking sign/unsigned bounds because the bits may 10435 * cross subreg boundaries in the alu64 case. When this happens we mark 10436 * the reg unbounded in the subreg bound space and use the resulting 10437 * tnum to calculate an approximation of the sign/unsigned bounds. 10438 */ 10439 switch (opcode) { 10440 case BPF_ADD: 10441 scalar32_min_max_add(dst_reg, &src_reg); 10442 scalar_min_max_add(dst_reg, &src_reg); 10443 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 10444 break; 10445 case BPF_SUB: 10446 scalar32_min_max_sub(dst_reg, &src_reg); 10447 scalar_min_max_sub(dst_reg, &src_reg); 10448 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 10449 break; 10450 case BPF_MUL: 10451 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 10452 scalar32_min_max_mul(dst_reg, &src_reg); 10453 scalar_min_max_mul(dst_reg, &src_reg); 10454 break; 10455 case BPF_AND: 10456 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 10457 scalar32_min_max_and(dst_reg, &src_reg); 10458 scalar_min_max_and(dst_reg, &src_reg); 10459 break; 10460 case BPF_OR: 10461 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 10462 scalar32_min_max_or(dst_reg, &src_reg); 10463 scalar_min_max_or(dst_reg, &src_reg); 10464 break; 10465 case BPF_XOR: 10466 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 10467 scalar32_min_max_xor(dst_reg, &src_reg); 10468 scalar_min_max_xor(dst_reg, &src_reg); 10469 break; 10470 case BPF_LSH: 10471 if (umax_val >= insn_bitness) { 10472 /* Shifts greater than 31 or 63 are undefined. 10473 * This includes shifts by a negative number. 10474 */ 10475 mark_reg_unknown(env, regs, insn->dst_reg); 10476 break; 10477 } 10478 if (alu32) 10479 scalar32_min_max_lsh(dst_reg, &src_reg); 10480 else 10481 scalar_min_max_lsh(dst_reg, &src_reg); 10482 break; 10483 case BPF_RSH: 10484 if (umax_val >= insn_bitness) { 10485 /* Shifts greater than 31 or 63 are undefined. 10486 * This includes shifts by a negative number. 10487 */ 10488 mark_reg_unknown(env, regs, insn->dst_reg); 10489 break; 10490 } 10491 if (alu32) 10492 scalar32_min_max_rsh(dst_reg, &src_reg); 10493 else 10494 scalar_min_max_rsh(dst_reg, &src_reg); 10495 break; 10496 case BPF_ARSH: 10497 if (umax_val >= insn_bitness) { 10498 /* Shifts greater than 31 or 63 are undefined. 10499 * This includes shifts by a negative number. 10500 */ 10501 mark_reg_unknown(env, regs, insn->dst_reg); 10502 break; 10503 } 10504 if (alu32) 10505 scalar32_min_max_arsh(dst_reg, &src_reg); 10506 else 10507 scalar_min_max_arsh(dst_reg, &src_reg); 10508 break; 10509 default: 10510 mark_reg_unknown(env, regs, insn->dst_reg); 10511 break; 10512 } 10513 10514 /* ALU32 ops are zero extended into 64bit register */ 10515 if (alu32) 10516 zext_32_to_64(dst_reg); 10517 reg_bounds_sync(dst_reg); 10518 return 0; 10519 } 10520 10521 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 10522 * and var_off. 10523 */ 10524 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 10525 struct bpf_insn *insn) 10526 { 10527 struct bpf_verifier_state *vstate = env->cur_state; 10528 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 10529 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 10530 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 10531 u8 opcode = BPF_OP(insn->code); 10532 int err; 10533 10534 dst_reg = ®s[insn->dst_reg]; 10535 src_reg = NULL; 10536 if (dst_reg->type != SCALAR_VALUE) 10537 ptr_reg = dst_reg; 10538 else 10539 /* Make sure ID is cleared otherwise dst_reg min/max could be 10540 * incorrectly propagated into other registers by find_equal_scalars() 10541 */ 10542 dst_reg->id = 0; 10543 if (BPF_SRC(insn->code) == BPF_X) { 10544 src_reg = ®s[insn->src_reg]; 10545 if (src_reg->type != SCALAR_VALUE) { 10546 if (dst_reg->type != SCALAR_VALUE) { 10547 /* Combining two pointers by any ALU op yields 10548 * an arbitrary scalar. Disallow all math except 10549 * pointer subtraction 10550 */ 10551 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 10552 mark_reg_unknown(env, regs, insn->dst_reg); 10553 return 0; 10554 } 10555 verbose(env, "R%d pointer %s pointer prohibited\n", 10556 insn->dst_reg, 10557 bpf_alu_string[opcode >> 4]); 10558 return -EACCES; 10559 } else { 10560 /* scalar += pointer 10561 * This is legal, but we have to reverse our 10562 * src/dest handling in computing the range 10563 */ 10564 err = mark_chain_precision(env, insn->dst_reg); 10565 if (err) 10566 return err; 10567 return adjust_ptr_min_max_vals(env, insn, 10568 src_reg, dst_reg); 10569 } 10570 } else if (ptr_reg) { 10571 /* pointer += scalar */ 10572 err = mark_chain_precision(env, insn->src_reg); 10573 if (err) 10574 return err; 10575 return adjust_ptr_min_max_vals(env, insn, 10576 dst_reg, src_reg); 10577 } else if (dst_reg->precise) { 10578 /* if dst_reg is precise, src_reg should be precise as well */ 10579 err = mark_chain_precision(env, insn->src_reg); 10580 if (err) 10581 return err; 10582 } 10583 } else { 10584 /* Pretend the src is a reg with a known value, since we only 10585 * need to be able to read from this state. 10586 */ 10587 off_reg.type = SCALAR_VALUE; 10588 __mark_reg_known(&off_reg, insn->imm); 10589 src_reg = &off_reg; 10590 if (ptr_reg) /* pointer += K */ 10591 return adjust_ptr_min_max_vals(env, insn, 10592 ptr_reg, src_reg); 10593 } 10594 10595 /* Got here implies adding two SCALAR_VALUEs */ 10596 if (WARN_ON_ONCE(ptr_reg)) { 10597 print_verifier_state(env, state, true); 10598 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 10599 return -EINVAL; 10600 } 10601 if (WARN_ON(!src_reg)) { 10602 print_verifier_state(env, state, true); 10603 verbose(env, "verifier internal error: no src_reg\n"); 10604 return -EINVAL; 10605 } 10606 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 10607 } 10608 10609 /* check validity of 32-bit and 64-bit arithmetic operations */ 10610 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 10611 { 10612 struct bpf_reg_state *regs = cur_regs(env); 10613 u8 opcode = BPF_OP(insn->code); 10614 int err; 10615 10616 if (opcode == BPF_END || opcode == BPF_NEG) { 10617 if (opcode == BPF_NEG) { 10618 if (BPF_SRC(insn->code) != BPF_K || 10619 insn->src_reg != BPF_REG_0 || 10620 insn->off != 0 || insn->imm != 0) { 10621 verbose(env, "BPF_NEG uses reserved fields\n"); 10622 return -EINVAL; 10623 } 10624 } else { 10625 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 10626 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 10627 BPF_CLASS(insn->code) == BPF_ALU64) { 10628 verbose(env, "BPF_END uses reserved fields\n"); 10629 return -EINVAL; 10630 } 10631 } 10632 10633 /* check src operand */ 10634 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10635 if (err) 10636 return err; 10637 10638 if (is_pointer_value(env, insn->dst_reg)) { 10639 verbose(env, "R%d pointer arithmetic prohibited\n", 10640 insn->dst_reg); 10641 return -EACCES; 10642 } 10643 10644 /* check dest operand */ 10645 err = check_reg_arg(env, insn->dst_reg, DST_OP); 10646 if (err) 10647 return err; 10648 10649 } else if (opcode == BPF_MOV) { 10650 10651 if (BPF_SRC(insn->code) == BPF_X) { 10652 if (insn->imm != 0 || insn->off != 0) { 10653 verbose(env, "BPF_MOV uses reserved fields\n"); 10654 return -EINVAL; 10655 } 10656 10657 /* check src operand */ 10658 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10659 if (err) 10660 return err; 10661 } else { 10662 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 10663 verbose(env, "BPF_MOV uses reserved fields\n"); 10664 return -EINVAL; 10665 } 10666 } 10667 10668 /* check dest operand, mark as required later */ 10669 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10670 if (err) 10671 return err; 10672 10673 if (BPF_SRC(insn->code) == BPF_X) { 10674 struct bpf_reg_state *src_reg = regs + insn->src_reg; 10675 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 10676 10677 if (BPF_CLASS(insn->code) == BPF_ALU64) { 10678 /* case: R1 = R2 10679 * copy register state to dest reg 10680 */ 10681 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 10682 /* Assign src and dst registers the same ID 10683 * that will be used by find_equal_scalars() 10684 * to propagate min/max range. 10685 */ 10686 src_reg->id = ++env->id_gen; 10687 *dst_reg = *src_reg; 10688 dst_reg->live |= REG_LIVE_WRITTEN; 10689 dst_reg->subreg_def = DEF_NOT_SUBREG; 10690 } else { 10691 /* R1 = (u32) R2 */ 10692 if (is_pointer_value(env, insn->src_reg)) { 10693 verbose(env, 10694 "R%d partial copy of pointer\n", 10695 insn->src_reg); 10696 return -EACCES; 10697 } else if (src_reg->type == SCALAR_VALUE) { 10698 *dst_reg = *src_reg; 10699 /* Make sure ID is cleared otherwise 10700 * dst_reg min/max could be incorrectly 10701 * propagated into src_reg by find_equal_scalars() 10702 */ 10703 dst_reg->id = 0; 10704 dst_reg->live |= REG_LIVE_WRITTEN; 10705 dst_reg->subreg_def = env->insn_idx + 1; 10706 } else { 10707 mark_reg_unknown(env, regs, 10708 insn->dst_reg); 10709 } 10710 zext_32_to_64(dst_reg); 10711 reg_bounds_sync(dst_reg); 10712 } 10713 } else { 10714 /* case: R = imm 10715 * remember the value we stored into this reg 10716 */ 10717 /* clear any state __mark_reg_known doesn't set */ 10718 mark_reg_unknown(env, regs, insn->dst_reg); 10719 regs[insn->dst_reg].type = SCALAR_VALUE; 10720 if (BPF_CLASS(insn->code) == BPF_ALU64) { 10721 __mark_reg_known(regs + insn->dst_reg, 10722 insn->imm); 10723 } else { 10724 __mark_reg_known(regs + insn->dst_reg, 10725 (u32)insn->imm); 10726 } 10727 } 10728 10729 } else if (opcode > BPF_END) { 10730 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 10731 return -EINVAL; 10732 10733 } else { /* all other ALU ops: and, sub, xor, add, ... */ 10734 10735 if (BPF_SRC(insn->code) == BPF_X) { 10736 if (insn->imm != 0 || insn->off != 0) { 10737 verbose(env, "BPF_ALU uses reserved fields\n"); 10738 return -EINVAL; 10739 } 10740 /* check src1 operand */ 10741 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10742 if (err) 10743 return err; 10744 } else { 10745 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 10746 verbose(env, "BPF_ALU uses reserved fields\n"); 10747 return -EINVAL; 10748 } 10749 } 10750 10751 /* check src2 operand */ 10752 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 10753 if (err) 10754 return err; 10755 10756 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 10757 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 10758 verbose(env, "div by zero\n"); 10759 return -EINVAL; 10760 } 10761 10762 if ((opcode == BPF_LSH || opcode == BPF_RSH || 10763 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 10764 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 10765 10766 if (insn->imm < 0 || insn->imm >= size) { 10767 verbose(env, "invalid shift %d\n", insn->imm); 10768 return -EINVAL; 10769 } 10770 } 10771 10772 /* check dest operand */ 10773 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 10774 if (err) 10775 return err; 10776 10777 return adjust_reg_min_max_vals(env, insn); 10778 } 10779 10780 return 0; 10781 } 10782 10783 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 10784 struct bpf_reg_state *dst_reg, 10785 enum bpf_reg_type type, 10786 bool range_right_open) 10787 { 10788 struct bpf_func_state *state; 10789 struct bpf_reg_state *reg; 10790 int new_range; 10791 10792 if (dst_reg->off < 0 || 10793 (dst_reg->off == 0 && range_right_open)) 10794 /* This doesn't give us any range */ 10795 return; 10796 10797 if (dst_reg->umax_value > MAX_PACKET_OFF || 10798 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 10799 /* Risk of overflow. For instance, ptr + (1<<63) may be less 10800 * than pkt_end, but that's because it's also less than pkt. 10801 */ 10802 return; 10803 10804 new_range = dst_reg->off; 10805 if (range_right_open) 10806 new_range++; 10807 10808 /* Examples for register markings: 10809 * 10810 * pkt_data in dst register: 10811 * 10812 * r2 = r3; 10813 * r2 += 8; 10814 * if (r2 > pkt_end) goto <handle exception> 10815 * <access okay> 10816 * 10817 * r2 = r3; 10818 * r2 += 8; 10819 * if (r2 < pkt_end) goto <access okay> 10820 * <handle exception> 10821 * 10822 * Where: 10823 * r2 == dst_reg, pkt_end == src_reg 10824 * r2=pkt(id=n,off=8,r=0) 10825 * r3=pkt(id=n,off=0,r=0) 10826 * 10827 * pkt_data in src register: 10828 * 10829 * r2 = r3; 10830 * r2 += 8; 10831 * if (pkt_end >= r2) goto <access okay> 10832 * <handle exception> 10833 * 10834 * r2 = r3; 10835 * r2 += 8; 10836 * if (pkt_end <= r2) goto <handle exception> 10837 * <access okay> 10838 * 10839 * Where: 10840 * pkt_end == dst_reg, r2 == src_reg 10841 * r2=pkt(id=n,off=8,r=0) 10842 * r3=pkt(id=n,off=0,r=0) 10843 * 10844 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 10845 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 10846 * and [r3, r3 + 8-1) respectively is safe to access depending on 10847 * the check. 10848 */ 10849 10850 /* If our ids match, then we must have the same max_value. And we 10851 * don't care about the other reg's fixed offset, since if it's too big 10852 * the range won't allow anything. 10853 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 10854 */ 10855 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 10856 if (reg->type == type && reg->id == dst_reg->id) 10857 /* keep the maximum range already checked */ 10858 reg->range = max(reg->range, new_range); 10859 })); 10860 } 10861 10862 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 10863 { 10864 struct tnum subreg = tnum_subreg(reg->var_off); 10865 s32 sval = (s32)val; 10866 10867 switch (opcode) { 10868 case BPF_JEQ: 10869 if (tnum_is_const(subreg)) 10870 return !!tnum_equals_const(subreg, val); 10871 break; 10872 case BPF_JNE: 10873 if (tnum_is_const(subreg)) 10874 return !tnum_equals_const(subreg, val); 10875 break; 10876 case BPF_JSET: 10877 if ((~subreg.mask & subreg.value) & val) 10878 return 1; 10879 if (!((subreg.mask | subreg.value) & val)) 10880 return 0; 10881 break; 10882 case BPF_JGT: 10883 if (reg->u32_min_value > val) 10884 return 1; 10885 else if (reg->u32_max_value <= val) 10886 return 0; 10887 break; 10888 case BPF_JSGT: 10889 if (reg->s32_min_value > sval) 10890 return 1; 10891 else if (reg->s32_max_value <= sval) 10892 return 0; 10893 break; 10894 case BPF_JLT: 10895 if (reg->u32_max_value < val) 10896 return 1; 10897 else if (reg->u32_min_value >= val) 10898 return 0; 10899 break; 10900 case BPF_JSLT: 10901 if (reg->s32_max_value < sval) 10902 return 1; 10903 else if (reg->s32_min_value >= sval) 10904 return 0; 10905 break; 10906 case BPF_JGE: 10907 if (reg->u32_min_value >= val) 10908 return 1; 10909 else if (reg->u32_max_value < val) 10910 return 0; 10911 break; 10912 case BPF_JSGE: 10913 if (reg->s32_min_value >= sval) 10914 return 1; 10915 else if (reg->s32_max_value < sval) 10916 return 0; 10917 break; 10918 case BPF_JLE: 10919 if (reg->u32_max_value <= val) 10920 return 1; 10921 else if (reg->u32_min_value > val) 10922 return 0; 10923 break; 10924 case BPF_JSLE: 10925 if (reg->s32_max_value <= sval) 10926 return 1; 10927 else if (reg->s32_min_value > sval) 10928 return 0; 10929 break; 10930 } 10931 10932 return -1; 10933 } 10934 10935 10936 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 10937 { 10938 s64 sval = (s64)val; 10939 10940 switch (opcode) { 10941 case BPF_JEQ: 10942 if (tnum_is_const(reg->var_off)) 10943 return !!tnum_equals_const(reg->var_off, val); 10944 break; 10945 case BPF_JNE: 10946 if (tnum_is_const(reg->var_off)) 10947 return !tnum_equals_const(reg->var_off, val); 10948 break; 10949 case BPF_JSET: 10950 if ((~reg->var_off.mask & reg->var_off.value) & val) 10951 return 1; 10952 if (!((reg->var_off.mask | reg->var_off.value) & val)) 10953 return 0; 10954 break; 10955 case BPF_JGT: 10956 if (reg->umin_value > val) 10957 return 1; 10958 else if (reg->umax_value <= val) 10959 return 0; 10960 break; 10961 case BPF_JSGT: 10962 if (reg->smin_value > sval) 10963 return 1; 10964 else if (reg->smax_value <= sval) 10965 return 0; 10966 break; 10967 case BPF_JLT: 10968 if (reg->umax_value < val) 10969 return 1; 10970 else if (reg->umin_value >= val) 10971 return 0; 10972 break; 10973 case BPF_JSLT: 10974 if (reg->smax_value < sval) 10975 return 1; 10976 else if (reg->smin_value >= sval) 10977 return 0; 10978 break; 10979 case BPF_JGE: 10980 if (reg->umin_value >= val) 10981 return 1; 10982 else if (reg->umax_value < val) 10983 return 0; 10984 break; 10985 case BPF_JSGE: 10986 if (reg->smin_value >= sval) 10987 return 1; 10988 else if (reg->smax_value < sval) 10989 return 0; 10990 break; 10991 case BPF_JLE: 10992 if (reg->umax_value <= val) 10993 return 1; 10994 else if (reg->umin_value > val) 10995 return 0; 10996 break; 10997 case BPF_JSLE: 10998 if (reg->smax_value <= sval) 10999 return 1; 11000 else if (reg->smin_value > sval) 11001 return 0; 11002 break; 11003 } 11004 11005 return -1; 11006 } 11007 11008 /* compute branch direction of the expression "if (reg opcode val) goto target;" 11009 * and return: 11010 * 1 - branch will be taken and "goto target" will be executed 11011 * 0 - branch will not be taken and fall-through to next insn 11012 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 11013 * range [0,10] 11014 */ 11015 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 11016 bool is_jmp32) 11017 { 11018 if (__is_pointer_value(false, reg)) { 11019 if (!reg_type_not_null(reg->type)) 11020 return -1; 11021 11022 /* If pointer is valid tests against zero will fail so we can 11023 * use this to direct branch taken. 11024 */ 11025 if (val != 0) 11026 return -1; 11027 11028 switch (opcode) { 11029 case BPF_JEQ: 11030 return 0; 11031 case BPF_JNE: 11032 return 1; 11033 default: 11034 return -1; 11035 } 11036 } 11037 11038 if (is_jmp32) 11039 return is_branch32_taken(reg, val, opcode); 11040 return is_branch64_taken(reg, val, opcode); 11041 } 11042 11043 static int flip_opcode(u32 opcode) 11044 { 11045 /* How can we transform "a <op> b" into "b <op> a"? */ 11046 static const u8 opcode_flip[16] = { 11047 /* these stay the same */ 11048 [BPF_JEQ >> 4] = BPF_JEQ, 11049 [BPF_JNE >> 4] = BPF_JNE, 11050 [BPF_JSET >> 4] = BPF_JSET, 11051 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 11052 [BPF_JGE >> 4] = BPF_JLE, 11053 [BPF_JGT >> 4] = BPF_JLT, 11054 [BPF_JLE >> 4] = BPF_JGE, 11055 [BPF_JLT >> 4] = BPF_JGT, 11056 [BPF_JSGE >> 4] = BPF_JSLE, 11057 [BPF_JSGT >> 4] = BPF_JSLT, 11058 [BPF_JSLE >> 4] = BPF_JSGE, 11059 [BPF_JSLT >> 4] = BPF_JSGT 11060 }; 11061 return opcode_flip[opcode >> 4]; 11062 } 11063 11064 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 11065 struct bpf_reg_state *src_reg, 11066 u8 opcode) 11067 { 11068 struct bpf_reg_state *pkt; 11069 11070 if (src_reg->type == PTR_TO_PACKET_END) { 11071 pkt = dst_reg; 11072 } else if (dst_reg->type == PTR_TO_PACKET_END) { 11073 pkt = src_reg; 11074 opcode = flip_opcode(opcode); 11075 } else { 11076 return -1; 11077 } 11078 11079 if (pkt->range >= 0) 11080 return -1; 11081 11082 switch (opcode) { 11083 case BPF_JLE: 11084 /* pkt <= pkt_end */ 11085 fallthrough; 11086 case BPF_JGT: 11087 /* pkt > pkt_end */ 11088 if (pkt->range == BEYOND_PKT_END) 11089 /* pkt has at last one extra byte beyond pkt_end */ 11090 return opcode == BPF_JGT; 11091 break; 11092 case BPF_JLT: 11093 /* pkt < pkt_end */ 11094 fallthrough; 11095 case BPF_JGE: 11096 /* pkt >= pkt_end */ 11097 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 11098 return opcode == BPF_JGE; 11099 break; 11100 } 11101 return -1; 11102 } 11103 11104 /* Adjusts the register min/max values in the case that the dst_reg is the 11105 * variable register that we are working on, and src_reg is a constant or we're 11106 * simply doing a BPF_K check. 11107 * In JEQ/JNE cases we also adjust the var_off values. 11108 */ 11109 static void reg_set_min_max(struct bpf_reg_state *true_reg, 11110 struct bpf_reg_state *false_reg, 11111 u64 val, u32 val32, 11112 u8 opcode, bool is_jmp32) 11113 { 11114 struct tnum false_32off = tnum_subreg(false_reg->var_off); 11115 struct tnum false_64off = false_reg->var_off; 11116 struct tnum true_32off = tnum_subreg(true_reg->var_off); 11117 struct tnum true_64off = true_reg->var_off; 11118 s64 sval = (s64)val; 11119 s32 sval32 = (s32)val32; 11120 11121 /* If the dst_reg is a pointer, we can't learn anything about its 11122 * variable offset from the compare (unless src_reg were a pointer into 11123 * the same object, but we don't bother with that. 11124 * Since false_reg and true_reg have the same type by construction, we 11125 * only need to check one of them for pointerness. 11126 */ 11127 if (__is_pointer_value(false, false_reg)) 11128 return; 11129 11130 switch (opcode) { 11131 /* JEQ/JNE comparison doesn't change the register equivalence. 11132 * 11133 * r1 = r2; 11134 * if (r1 == 42) goto label; 11135 * ... 11136 * label: // here both r1 and r2 are known to be 42. 11137 * 11138 * Hence when marking register as known preserve it's ID. 11139 */ 11140 case BPF_JEQ: 11141 if (is_jmp32) { 11142 __mark_reg32_known(true_reg, val32); 11143 true_32off = tnum_subreg(true_reg->var_off); 11144 } else { 11145 ___mark_reg_known(true_reg, val); 11146 true_64off = true_reg->var_off; 11147 } 11148 break; 11149 case BPF_JNE: 11150 if (is_jmp32) { 11151 __mark_reg32_known(false_reg, val32); 11152 false_32off = tnum_subreg(false_reg->var_off); 11153 } else { 11154 ___mark_reg_known(false_reg, val); 11155 false_64off = false_reg->var_off; 11156 } 11157 break; 11158 case BPF_JSET: 11159 if (is_jmp32) { 11160 false_32off = tnum_and(false_32off, tnum_const(~val32)); 11161 if (is_power_of_2(val32)) 11162 true_32off = tnum_or(true_32off, 11163 tnum_const(val32)); 11164 } else { 11165 false_64off = tnum_and(false_64off, tnum_const(~val)); 11166 if (is_power_of_2(val)) 11167 true_64off = tnum_or(true_64off, 11168 tnum_const(val)); 11169 } 11170 break; 11171 case BPF_JGE: 11172 case BPF_JGT: 11173 { 11174 if (is_jmp32) { 11175 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 11176 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 11177 11178 false_reg->u32_max_value = min(false_reg->u32_max_value, 11179 false_umax); 11180 true_reg->u32_min_value = max(true_reg->u32_min_value, 11181 true_umin); 11182 } else { 11183 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 11184 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 11185 11186 false_reg->umax_value = min(false_reg->umax_value, false_umax); 11187 true_reg->umin_value = max(true_reg->umin_value, true_umin); 11188 } 11189 break; 11190 } 11191 case BPF_JSGE: 11192 case BPF_JSGT: 11193 { 11194 if (is_jmp32) { 11195 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 11196 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 11197 11198 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 11199 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 11200 } else { 11201 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 11202 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 11203 11204 false_reg->smax_value = min(false_reg->smax_value, false_smax); 11205 true_reg->smin_value = max(true_reg->smin_value, true_smin); 11206 } 11207 break; 11208 } 11209 case BPF_JLE: 11210 case BPF_JLT: 11211 { 11212 if (is_jmp32) { 11213 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 11214 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 11215 11216 false_reg->u32_min_value = max(false_reg->u32_min_value, 11217 false_umin); 11218 true_reg->u32_max_value = min(true_reg->u32_max_value, 11219 true_umax); 11220 } else { 11221 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 11222 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 11223 11224 false_reg->umin_value = max(false_reg->umin_value, false_umin); 11225 true_reg->umax_value = min(true_reg->umax_value, true_umax); 11226 } 11227 break; 11228 } 11229 case BPF_JSLE: 11230 case BPF_JSLT: 11231 { 11232 if (is_jmp32) { 11233 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 11234 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 11235 11236 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 11237 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 11238 } else { 11239 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 11240 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 11241 11242 false_reg->smin_value = max(false_reg->smin_value, false_smin); 11243 true_reg->smax_value = min(true_reg->smax_value, true_smax); 11244 } 11245 break; 11246 } 11247 default: 11248 return; 11249 } 11250 11251 if (is_jmp32) { 11252 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 11253 tnum_subreg(false_32off)); 11254 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 11255 tnum_subreg(true_32off)); 11256 __reg_combine_32_into_64(false_reg); 11257 __reg_combine_32_into_64(true_reg); 11258 } else { 11259 false_reg->var_off = false_64off; 11260 true_reg->var_off = true_64off; 11261 __reg_combine_64_into_32(false_reg); 11262 __reg_combine_64_into_32(true_reg); 11263 } 11264 } 11265 11266 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 11267 * the variable reg. 11268 */ 11269 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 11270 struct bpf_reg_state *false_reg, 11271 u64 val, u32 val32, 11272 u8 opcode, bool is_jmp32) 11273 { 11274 opcode = flip_opcode(opcode); 11275 /* This uses zero as "not present in table"; luckily the zero opcode, 11276 * BPF_JA, can't get here. 11277 */ 11278 if (opcode) 11279 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 11280 } 11281 11282 /* Regs are known to be equal, so intersect their min/max/var_off */ 11283 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 11284 struct bpf_reg_state *dst_reg) 11285 { 11286 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 11287 dst_reg->umin_value); 11288 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 11289 dst_reg->umax_value); 11290 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 11291 dst_reg->smin_value); 11292 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 11293 dst_reg->smax_value); 11294 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 11295 dst_reg->var_off); 11296 reg_bounds_sync(src_reg); 11297 reg_bounds_sync(dst_reg); 11298 } 11299 11300 static void reg_combine_min_max(struct bpf_reg_state *true_src, 11301 struct bpf_reg_state *true_dst, 11302 struct bpf_reg_state *false_src, 11303 struct bpf_reg_state *false_dst, 11304 u8 opcode) 11305 { 11306 switch (opcode) { 11307 case BPF_JEQ: 11308 __reg_combine_min_max(true_src, true_dst); 11309 break; 11310 case BPF_JNE: 11311 __reg_combine_min_max(false_src, false_dst); 11312 break; 11313 } 11314 } 11315 11316 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 11317 struct bpf_reg_state *reg, u32 id, 11318 bool is_null) 11319 { 11320 if (type_may_be_null(reg->type) && reg->id == id && 11321 (is_rcu_reg(reg) || !WARN_ON_ONCE(!reg->id))) { 11322 /* Old offset (both fixed and variable parts) should have been 11323 * known-zero, because we don't allow pointer arithmetic on 11324 * pointers that might be NULL. If we see this happening, don't 11325 * convert the register. 11326 * 11327 * But in some cases, some helpers that return local kptrs 11328 * advance offset for the returned pointer. In those cases, it 11329 * is fine to expect to see reg->off. 11330 */ 11331 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || !tnum_equals_const(reg->var_off, 0))) 11332 return; 11333 if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC | PTR_MAYBE_NULL) && WARN_ON_ONCE(reg->off)) 11334 return; 11335 if (is_null) { 11336 reg->type = SCALAR_VALUE; 11337 /* We don't need id and ref_obj_id from this point 11338 * onwards anymore, thus we should better reset it, 11339 * so that state pruning has chances to take effect. 11340 */ 11341 reg->id = 0; 11342 reg->ref_obj_id = 0; 11343 11344 return; 11345 } 11346 11347 mark_ptr_not_null_reg(reg); 11348 11349 if (!reg_may_point_to_spin_lock(reg)) { 11350 /* For not-NULL ptr, reg->ref_obj_id will be reset 11351 * in release_reference(). 11352 * 11353 * reg->id is still used by spin_lock ptr. Other 11354 * than spin_lock ptr type, reg->id can be reset. 11355 */ 11356 reg->id = 0; 11357 } 11358 } 11359 } 11360 11361 /* The logic is similar to find_good_pkt_pointers(), both could eventually 11362 * be folded together at some point. 11363 */ 11364 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 11365 bool is_null) 11366 { 11367 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 11368 struct bpf_reg_state *regs = state->regs, *reg; 11369 u32 ref_obj_id = regs[regno].ref_obj_id; 11370 u32 id = regs[regno].id; 11371 11372 if (ref_obj_id && ref_obj_id == id && is_null) 11373 /* regs[regno] is in the " == NULL" branch. 11374 * No one could have freed the reference state before 11375 * doing the NULL check. 11376 */ 11377 WARN_ON_ONCE(release_reference_state(state, id)); 11378 11379 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 11380 mark_ptr_or_null_reg(state, reg, id, is_null); 11381 })); 11382 } 11383 11384 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 11385 struct bpf_reg_state *dst_reg, 11386 struct bpf_reg_state *src_reg, 11387 struct bpf_verifier_state *this_branch, 11388 struct bpf_verifier_state *other_branch) 11389 { 11390 if (BPF_SRC(insn->code) != BPF_X) 11391 return false; 11392 11393 /* Pointers are always 64-bit. */ 11394 if (BPF_CLASS(insn->code) == BPF_JMP32) 11395 return false; 11396 11397 switch (BPF_OP(insn->code)) { 11398 case BPF_JGT: 11399 if ((dst_reg->type == PTR_TO_PACKET && 11400 src_reg->type == PTR_TO_PACKET_END) || 11401 (dst_reg->type == PTR_TO_PACKET_META && 11402 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11403 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 11404 find_good_pkt_pointers(this_branch, dst_reg, 11405 dst_reg->type, false); 11406 mark_pkt_end(other_branch, insn->dst_reg, true); 11407 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11408 src_reg->type == PTR_TO_PACKET) || 11409 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11410 src_reg->type == PTR_TO_PACKET_META)) { 11411 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 11412 find_good_pkt_pointers(other_branch, src_reg, 11413 src_reg->type, true); 11414 mark_pkt_end(this_branch, insn->src_reg, false); 11415 } else { 11416 return false; 11417 } 11418 break; 11419 case BPF_JLT: 11420 if ((dst_reg->type == PTR_TO_PACKET && 11421 src_reg->type == PTR_TO_PACKET_END) || 11422 (dst_reg->type == PTR_TO_PACKET_META && 11423 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11424 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 11425 find_good_pkt_pointers(other_branch, dst_reg, 11426 dst_reg->type, true); 11427 mark_pkt_end(this_branch, insn->dst_reg, false); 11428 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11429 src_reg->type == PTR_TO_PACKET) || 11430 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11431 src_reg->type == PTR_TO_PACKET_META)) { 11432 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 11433 find_good_pkt_pointers(this_branch, src_reg, 11434 src_reg->type, false); 11435 mark_pkt_end(other_branch, insn->src_reg, true); 11436 } else { 11437 return false; 11438 } 11439 break; 11440 case BPF_JGE: 11441 if ((dst_reg->type == PTR_TO_PACKET && 11442 src_reg->type == PTR_TO_PACKET_END) || 11443 (dst_reg->type == PTR_TO_PACKET_META && 11444 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11445 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 11446 find_good_pkt_pointers(this_branch, dst_reg, 11447 dst_reg->type, true); 11448 mark_pkt_end(other_branch, insn->dst_reg, false); 11449 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11450 src_reg->type == PTR_TO_PACKET) || 11451 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11452 src_reg->type == PTR_TO_PACKET_META)) { 11453 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 11454 find_good_pkt_pointers(other_branch, src_reg, 11455 src_reg->type, false); 11456 mark_pkt_end(this_branch, insn->src_reg, true); 11457 } else { 11458 return false; 11459 } 11460 break; 11461 case BPF_JLE: 11462 if ((dst_reg->type == PTR_TO_PACKET && 11463 src_reg->type == PTR_TO_PACKET_END) || 11464 (dst_reg->type == PTR_TO_PACKET_META && 11465 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 11466 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 11467 find_good_pkt_pointers(other_branch, dst_reg, 11468 dst_reg->type, false); 11469 mark_pkt_end(this_branch, insn->dst_reg, true); 11470 } else if ((dst_reg->type == PTR_TO_PACKET_END && 11471 src_reg->type == PTR_TO_PACKET) || 11472 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 11473 src_reg->type == PTR_TO_PACKET_META)) { 11474 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 11475 find_good_pkt_pointers(this_branch, src_reg, 11476 src_reg->type, true); 11477 mark_pkt_end(other_branch, insn->src_reg, false); 11478 } else { 11479 return false; 11480 } 11481 break; 11482 default: 11483 return false; 11484 } 11485 11486 return true; 11487 } 11488 11489 static void find_equal_scalars(struct bpf_verifier_state *vstate, 11490 struct bpf_reg_state *known_reg) 11491 { 11492 struct bpf_func_state *state; 11493 struct bpf_reg_state *reg; 11494 11495 bpf_for_each_reg_in_vstate(vstate, state, reg, ({ 11496 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 11497 *reg = *known_reg; 11498 })); 11499 } 11500 11501 static int check_cond_jmp_op(struct bpf_verifier_env *env, 11502 struct bpf_insn *insn, int *insn_idx) 11503 { 11504 struct bpf_verifier_state *this_branch = env->cur_state; 11505 struct bpf_verifier_state *other_branch; 11506 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 11507 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 11508 struct bpf_reg_state *eq_branch_regs; 11509 u8 opcode = BPF_OP(insn->code); 11510 bool is_jmp32; 11511 int pred = -1; 11512 int err; 11513 11514 /* Only conditional jumps are expected to reach here. */ 11515 if (opcode == BPF_JA || opcode > BPF_JSLE) { 11516 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 11517 return -EINVAL; 11518 } 11519 11520 if (BPF_SRC(insn->code) == BPF_X) { 11521 if (insn->imm != 0) { 11522 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 11523 return -EINVAL; 11524 } 11525 11526 /* check src1 operand */ 11527 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11528 if (err) 11529 return err; 11530 11531 if (is_pointer_value(env, insn->src_reg)) { 11532 verbose(env, "R%d pointer comparison prohibited\n", 11533 insn->src_reg); 11534 return -EACCES; 11535 } 11536 src_reg = ®s[insn->src_reg]; 11537 } else { 11538 if (insn->src_reg != BPF_REG_0) { 11539 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 11540 return -EINVAL; 11541 } 11542 } 11543 11544 /* check src2 operand */ 11545 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11546 if (err) 11547 return err; 11548 11549 dst_reg = ®s[insn->dst_reg]; 11550 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 11551 11552 if (BPF_SRC(insn->code) == BPF_K) { 11553 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 11554 } else if (src_reg->type == SCALAR_VALUE && 11555 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 11556 pred = is_branch_taken(dst_reg, 11557 tnum_subreg(src_reg->var_off).value, 11558 opcode, 11559 is_jmp32); 11560 } else if (src_reg->type == SCALAR_VALUE && 11561 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 11562 pred = is_branch_taken(dst_reg, 11563 src_reg->var_off.value, 11564 opcode, 11565 is_jmp32); 11566 } else if (reg_is_pkt_pointer_any(dst_reg) && 11567 reg_is_pkt_pointer_any(src_reg) && 11568 !is_jmp32) { 11569 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 11570 } 11571 11572 if (pred >= 0) { 11573 /* If we get here with a dst_reg pointer type it is because 11574 * above is_branch_taken() special cased the 0 comparison. 11575 */ 11576 if (!__is_pointer_value(false, dst_reg)) 11577 err = mark_chain_precision(env, insn->dst_reg); 11578 if (BPF_SRC(insn->code) == BPF_X && !err && 11579 !__is_pointer_value(false, src_reg)) 11580 err = mark_chain_precision(env, insn->src_reg); 11581 if (err) 11582 return err; 11583 } 11584 11585 if (pred == 1) { 11586 /* Only follow the goto, ignore fall-through. If needed, push 11587 * the fall-through branch for simulation under speculative 11588 * execution. 11589 */ 11590 if (!env->bypass_spec_v1 && 11591 !sanitize_speculative_path(env, insn, *insn_idx + 1, 11592 *insn_idx)) 11593 return -EFAULT; 11594 *insn_idx += insn->off; 11595 return 0; 11596 } else if (pred == 0) { 11597 /* Only follow the fall-through branch, since that's where the 11598 * program will go. If needed, push the goto branch for 11599 * simulation under speculative execution. 11600 */ 11601 if (!env->bypass_spec_v1 && 11602 !sanitize_speculative_path(env, insn, 11603 *insn_idx + insn->off + 1, 11604 *insn_idx)) 11605 return -EFAULT; 11606 return 0; 11607 } 11608 11609 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 11610 false); 11611 if (!other_branch) 11612 return -EFAULT; 11613 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 11614 11615 /* detect if we are comparing against a constant value so we can adjust 11616 * our min/max values for our dst register. 11617 * this is only legit if both are scalars (or pointers to the same 11618 * object, I suppose, see the PTR_MAYBE_NULL related if block below), 11619 * because otherwise the different base pointers mean the offsets aren't 11620 * comparable. 11621 */ 11622 if (BPF_SRC(insn->code) == BPF_X) { 11623 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 11624 11625 if (dst_reg->type == SCALAR_VALUE && 11626 src_reg->type == SCALAR_VALUE) { 11627 if (tnum_is_const(src_reg->var_off) || 11628 (is_jmp32 && 11629 tnum_is_const(tnum_subreg(src_reg->var_off)))) 11630 reg_set_min_max(&other_branch_regs[insn->dst_reg], 11631 dst_reg, 11632 src_reg->var_off.value, 11633 tnum_subreg(src_reg->var_off).value, 11634 opcode, is_jmp32); 11635 else if (tnum_is_const(dst_reg->var_off) || 11636 (is_jmp32 && 11637 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 11638 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 11639 src_reg, 11640 dst_reg->var_off.value, 11641 tnum_subreg(dst_reg->var_off).value, 11642 opcode, is_jmp32); 11643 else if (!is_jmp32 && 11644 (opcode == BPF_JEQ || opcode == BPF_JNE)) 11645 /* Comparing for equality, we can combine knowledge */ 11646 reg_combine_min_max(&other_branch_regs[insn->src_reg], 11647 &other_branch_regs[insn->dst_reg], 11648 src_reg, dst_reg, opcode); 11649 if (src_reg->id && 11650 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 11651 find_equal_scalars(this_branch, src_reg); 11652 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 11653 } 11654 11655 } 11656 } else if (dst_reg->type == SCALAR_VALUE) { 11657 reg_set_min_max(&other_branch_regs[insn->dst_reg], 11658 dst_reg, insn->imm, (u32)insn->imm, 11659 opcode, is_jmp32); 11660 } 11661 11662 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 11663 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 11664 find_equal_scalars(this_branch, dst_reg); 11665 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 11666 } 11667 11668 /* if one pointer register is compared to another pointer 11669 * register check if PTR_MAYBE_NULL could be lifted. 11670 * E.g. register A - maybe null 11671 * register B - not null 11672 * for JNE A, B, ... - A is not null in the false branch; 11673 * for JEQ A, B, ... - A is not null in the true branch. 11674 */ 11675 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X && 11676 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && 11677 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type)) { 11678 eq_branch_regs = NULL; 11679 switch (opcode) { 11680 case BPF_JEQ: 11681 eq_branch_regs = other_branch_regs; 11682 break; 11683 case BPF_JNE: 11684 eq_branch_regs = regs; 11685 break; 11686 default: 11687 /* do nothing */ 11688 break; 11689 } 11690 if (eq_branch_regs) { 11691 if (type_may_be_null(src_reg->type)) 11692 mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]); 11693 else 11694 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); 11695 } 11696 } 11697 11698 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 11699 * NOTE: these optimizations below are related with pointer comparison 11700 * which will never be JMP32. 11701 */ 11702 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 11703 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 11704 type_may_be_null(dst_reg->type)) { 11705 /* Mark all identical registers in each branch as either 11706 * safe or unknown depending R == 0 or R != 0 conditional. 11707 */ 11708 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 11709 opcode == BPF_JNE); 11710 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 11711 opcode == BPF_JEQ); 11712 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 11713 this_branch, other_branch) && 11714 is_pointer_value(env, insn->dst_reg)) { 11715 verbose(env, "R%d pointer comparison prohibited\n", 11716 insn->dst_reg); 11717 return -EACCES; 11718 } 11719 if (env->log.level & BPF_LOG_LEVEL) 11720 print_insn_state(env, this_branch->frame[this_branch->curframe]); 11721 return 0; 11722 } 11723 11724 /* verify BPF_LD_IMM64 instruction */ 11725 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 11726 { 11727 struct bpf_insn_aux_data *aux = cur_aux(env); 11728 struct bpf_reg_state *regs = cur_regs(env); 11729 struct bpf_reg_state *dst_reg; 11730 struct bpf_map *map; 11731 int err; 11732 11733 if (BPF_SIZE(insn->code) != BPF_DW) { 11734 verbose(env, "invalid BPF_LD_IMM insn\n"); 11735 return -EINVAL; 11736 } 11737 if (insn->off != 0) { 11738 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 11739 return -EINVAL; 11740 } 11741 11742 err = check_reg_arg(env, insn->dst_reg, DST_OP); 11743 if (err) 11744 return err; 11745 11746 dst_reg = ®s[insn->dst_reg]; 11747 if (insn->src_reg == 0) { 11748 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 11749 11750 dst_reg->type = SCALAR_VALUE; 11751 __mark_reg_known(®s[insn->dst_reg], imm); 11752 return 0; 11753 } 11754 11755 /* All special src_reg cases are listed below. From this point onwards 11756 * we either succeed and assign a corresponding dst_reg->type after 11757 * zeroing the offset, or fail and reject the program. 11758 */ 11759 mark_reg_known_zero(env, regs, insn->dst_reg); 11760 11761 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 11762 dst_reg->type = aux->btf_var.reg_type; 11763 switch (base_type(dst_reg->type)) { 11764 case PTR_TO_MEM: 11765 dst_reg->mem_size = aux->btf_var.mem_size; 11766 break; 11767 case PTR_TO_BTF_ID: 11768 dst_reg->btf = aux->btf_var.btf; 11769 dst_reg->btf_id = aux->btf_var.btf_id; 11770 break; 11771 default: 11772 verbose(env, "bpf verifier is misconfigured\n"); 11773 return -EFAULT; 11774 } 11775 return 0; 11776 } 11777 11778 if (insn->src_reg == BPF_PSEUDO_FUNC) { 11779 struct bpf_prog_aux *aux = env->prog->aux; 11780 u32 subprogno = find_subprog(env, 11781 env->insn_idx + insn->imm + 1); 11782 11783 if (!aux->func_info) { 11784 verbose(env, "missing btf func_info\n"); 11785 return -EINVAL; 11786 } 11787 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 11788 verbose(env, "callback function not static\n"); 11789 return -EINVAL; 11790 } 11791 11792 dst_reg->type = PTR_TO_FUNC; 11793 dst_reg->subprogno = subprogno; 11794 return 0; 11795 } 11796 11797 map = env->used_maps[aux->map_index]; 11798 dst_reg->map_ptr = map; 11799 11800 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 11801 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 11802 dst_reg->type = PTR_TO_MAP_VALUE; 11803 dst_reg->off = aux->map_off; 11804 WARN_ON_ONCE(map->max_entries != 1); 11805 /* We want reg->id to be same (0) as map_value is not distinct */ 11806 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 11807 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 11808 dst_reg->type = CONST_PTR_TO_MAP; 11809 } else { 11810 verbose(env, "bpf verifier is misconfigured\n"); 11811 return -EINVAL; 11812 } 11813 11814 return 0; 11815 } 11816 11817 static bool may_access_skb(enum bpf_prog_type type) 11818 { 11819 switch (type) { 11820 case BPF_PROG_TYPE_SOCKET_FILTER: 11821 case BPF_PROG_TYPE_SCHED_CLS: 11822 case BPF_PROG_TYPE_SCHED_ACT: 11823 return true; 11824 default: 11825 return false; 11826 } 11827 } 11828 11829 /* verify safety of LD_ABS|LD_IND instructions: 11830 * - they can only appear in the programs where ctx == skb 11831 * - since they are wrappers of function calls, they scratch R1-R5 registers, 11832 * preserve R6-R9, and store return value into R0 11833 * 11834 * Implicit input: 11835 * ctx == skb == R6 == CTX 11836 * 11837 * Explicit input: 11838 * SRC == any register 11839 * IMM == 32-bit immediate 11840 * 11841 * Output: 11842 * R0 - 8/16/32-bit skb data converted to cpu endianness 11843 */ 11844 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 11845 { 11846 struct bpf_reg_state *regs = cur_regs(env); 11847 static const int ctx_reg = BPF_REG_6; 11848 u8 mode = BPF_MODE(insn->code); 11849 int i, err; 11850 11851 if (!may_access_skb(resolve_prog_type(env->prog))) { 11852 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 11853 return -EINVAL; 11854 } 11855 11856 if (!env->ops->gen_ld_abs) { 11857 verbose(env, "bpf verifier is misconfigured\n"); 11858 return -EINVAL; 11859 } 11860 11861 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 11862 BPF_SIZE(insn->code) == BPF_DW || 11863 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 11864 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 11865 return -EINVAL; 11866 } 11867 11868 /* check whether implicit source operand (register R6) is readable */ 11869 err = check_reg_arg(env, ctx_reg, SRC_OP); 11870 if (err) 11871 return err; 11872 11873 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 11874 * gen_ld_abs() may terminate the program at runtime, leading to 11875 * reference leak. 11876 */ 11877 err = check_reference_leak(env); 11878 if (err) { 11879 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 11880 return err; 11881 } 11882 11883 if (env->cur_state->active_lock.ptr) { 11884 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 11885 return -EINVAL; 11886 } 11887 11888 if (env->cur_state->active_rcu_lock) { 11889 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n"); 11890 return -EINVAL; 11891 } 11892 11893 if (regs[ctx_reg].type != PTR_TO_CTX) { 11894 verbose(env, 11895 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 11896 return -EINVAL; 11897 } 11898 11899 if (mode == BPF_IND) { 11900 /* check explicit source operand */ 11901 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11902 if (err) 11903 return err; 11904 } 11905 11906 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 11907 if (err < 0) 11908 return err; 11909 11910 /* reset caller saved regs to unreadable */ 11911 for (i = 0; i < CALLER_SAVED_REGS; i++) { 11912 mark_reg_not_init(env, regs, caller_saved[i]); 11913 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 11914 } 11915 11916 /* mark destination R0 register as readable, since it contains 11917 * the value fetched from the packet. 11918 * Already marked as written above. 11919 */ 11920 mark_reg_unknown(env, regs, BPF_REG_0); 11921 /* ld_abs load up to 32-bit skb data. */ 11922 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 11923 return 0; 11924 } 11925 11926 static int check_return_code(struct bpf_verifier_env *env) 11927 { 11928 struct tnum enforce_attach_type_range = tnum_unknown; 11929 const struct bpf_prog *prog = env->prog; 11930 struct bpf_reg_state *reg; 11931 struct tnum range = tnum_range(0, 1); 11932 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 11933 int err; 11934 struct bpf_func_state *frame = env->cur_state->frame[0]; 11935 const bool is_subprog = frame->subprogno; 11936 11937 /* LSM and struct_ops func-ptr's return type could be "void" */ 11938 if (!is_subprog) { 11939 switch (prog_type) { 11940 case BPF_PROG_TYPE_LSM: 11941 if (prog->expected_attach_type == BPF_LSM_CGROUP) 11942 /* See below, can be 0 or 0-1 depending on hook. */ 11943 break; 11944 fallthrough; 11945 case BPF_PROG_TYPE_STRUCT_OPS: 11946 if (!prog->aux->attach_func_proto->type) 11947 return 0; 11948 break; 11949 default: 11950 break; 11951 } 11952 } 11953 11954 /* eBPF calling convention is such that R0 is used 11955 * to return the value from eBPF program. 11956 * Make sure that it's readable at this time 11957 * of bpf_exit, which means that program wrote 11958 * something into it earlier 11959 */ 11960 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 11961 if (err) 11962 return err; 11963 11964 if (is_pointer_value(env, BPF_REG_0)) { 11965 verbose(env, "R0 leaks addr as return value\n"); 11966 return -EACCES; 11967 } 11968 11969 reg = cur_regs(env) + BPF_REG_0; 11970 11971 if (frame->in_async_callback_fn) { 11972 /* enforce return zero from async callbacks like timer */ 11973 if (reg->type != SCALAR_VALUE) { 11974 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 11975 reg_type_str(env, reg->type)); 11976 return -EINVAL; 11977 } 11978 11979 if (!tnum_in(tnum_const(0), reg->var_off)) { 11980 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 11981 return -EINVAL; 11982 } 11983 return 0; 11984 } 11985 11986 if (is_subprog) { 11987 if (reg->type != SCALAR_VALUE) { 11988 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 11989 reg_type_str(env, reg->type)); 11990 return -EINVAL; 11991 } 11992 return 0; 11993 } 11994 11995 switch (prog_type) { 11996 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 11997 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 11998 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 11999 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 12000 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 12001 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 12002 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 12003 range = tnum_range(1, 1); 12004 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 12005 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 12006 range = tnum_range(0, 3); 12007 break; 12008 case BPF_PROG_TYPE_CGROUP_SKB: 12009 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 12010 range = tnum_range(0, 3); 12011 enforce_attach_type_range = tnum_range(2, 3); 12012 } 12013 break; 12014 case BPF_PROG_TYPE_CGROUP_SOCK: 12015 case BPF_PROG_TYPE_SOCK_OPS: 12016 case BPF_PROG_TYPE_CGROUP_DEVICE: 12017 case BPF_PROG_TYPE_CGROUP_SYSCTL: 12018 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 12019 break; 12020 case BPF_PROG_TYPE_RAW_TRACEPOINT: 12021 if (!env->prog->aux->attach_btf_id) 12022 return 0; 12023 range = tnum_const(0); 12024 break; 12025 case BPF_PROG_TYPE_TRACING: 12026 switch (env->prog->expected_attach_type) { 12027 case BPF_TRACE_FENTRY: 12028 case BPF_TRACE_FEXIT: 12029 range = tnum_const(0); 12030 break; 12031 case BPF_TRACE_RAW_TP: 12032 case BPF_MODIFY_RETURN: 12033 return 0; 12034 case BPF_TRACE_ITER: 12035 break; 12036 default: 12037 return -ENOTSUPP; 12038 } 12039 break; 12040 case BPF_PROG_TYPE_SK_LOOKUP: 12041 range = tnum_range(SK_DROP, SK_PASS); 12042 break; 12043 12044 case BPF_PROG_TYPE_LSM: 12045 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { 12046 /* Regular BPF_PROG_TYPE_LSM programs can return 12047 * any value. 12048 */ 12049 return 0; 12050 } 12051 if (!env->prog->aux->attach_func_proto->type) { 12052 /* Make sure programs that attach to void 12053 * hooks don't try to modify return value. 12054 */ 12055 range = tnum_range(1, 1); 12056 } 12057 break; 12058 12059 case BPF_PROG_TYPE_EXT: 12060 /* freplace program can return anything as its return value 12061 * depends on the to-be-replaced kernel func or bpf program. 12062 */ 12063 default: 12064 return 0; 12065 } 12066 12067 if (reg->type != SCALAR_VALUE) { 12068 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 12069 reg_type_str(env, reg->type)); 12070 return -EINVAL; 12071 } 12072 12073 if (!tnum_in(range, reg->var_off)) { 12074 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 12075 if (prog->expected_attach_type == BPF_LSM_CGROUP && 12076 prog_type == BPF_PROG_TYPE_LSM && 12077 !prog->aux->attach_func_proto->type) 12078 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); 12079 return -EINVAL; 12080 } 12081 12082 if (!tnum_is_unknown(enforce_attach_type_range) && 12083 tnum_in(enforce_attach_type_range, reg->var_off)) 12084 env->prog->enforce_expected_attach_type = 1; 12085 return 0; 12086 } 12087 12088 /* non-recursive DFS pseudo code 12089 * 1 procedure DFS-iterative(G,v): 12090 * 2 label v as discovered 12091 * 3 let S be a stack 12092 * 4 S.push(v) 12093 * 5 while S is not empty 12094 * 6 t <- S.peek() 12095 * 7 if t is what we're looking for: 12096 * 8 return t 12097 * 9 for all edges e in G.adjacentEdges(t) do 12098 * 10 if edge e is already labelled 12099 * 11 continue with the next edge 12100 * 12 w <- G.adjacentVertex(t,e) 12101 * 13 if vertex w is not discovered and not explored 12102 * 14 label e as tree-edge 12103 * 15 label w as discovered 12104 * 16 S.push(w) 12105 * 17 continue at 5 12106 * 18 else if vertex w is discovered 12107 * 19 label e as back-edge 12108 * 20 else 12109 * 21 // vertex w is explored 12110 * 22 label e as forward- or cross-edge 12111 * 23 label t as explored 12112 * 24 S.pop() 12113 * 12114 * convention: 12115 * 0x10 - discovered 12116 * 0x11 - discovered and fall-through edge labelled 12117 * 0x12 - discovered and fall-through and branch edges labelled 12118 * 0x20 - explored 12119 */ 12120 12121 enum { 12122 DISCOVERED = 0x10, 12123 EXPLORED = 0x20, 12124 FALLTHROUGH = 1, 12125 BRANCH = 2, 12126 }; 12127 12128 static u32 state_htab_size(struct bpf_verifier_env *env) 12129 { 12130 return env->prog->len; 12131 } 12132 12133 static struct bpf_verifier_state_list **explored_state( 12134 struct bpf_verifier_env *env, 12135 int idx) 12136 { 12137 struct bpf_verifier_state *cur = env->cur_state; 12138 struct bpf_func_state *state = cur->frame[cur->curframe]; 12139 12140 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 12141 } 12142 12143 static void init_explored_state(struct bpf_verifier_env *env, int idx) 12144 { 12145 env->insn_aux_data[idx].prune_point = true; 12146 } 12147 12148 enum { 12149 DONE_EXPLORING = 0, 12150 KEEP_EXPLORING = 1, 12151 }; 12152 12153 /* t, w, e - match pseudo-code above: 12154 * t - index of current instruction 12155 * w - next instruction 12156 * e - edge 12157 */ 12158 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 12159 bool loop_ok) 12160 { 12161 int *insn_stack = env->cfg.insn_stack; 12162 int *insn_state = env->cfg.insn_state; 12163 12164 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 12165 return DONE_EXPLORING; 12166 12167 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 12168 return DONE_EXPLORING; 12169 12170 if (w < 0 || w >= env->prog->len) { 12171 verbose_linfo(env, t, "%d: ", t); 12172 verbose(env, "jump out of range from insn %d to %d\n", t, w); 12173 return -EINVAL; 12174 } 12175 12176 if (e == BRANCH) 12177 /* mark branch target for state pruning */ 12178 init_explored_state(env, w); 12179 12180 if (insn_state[w] == 0) { 12181 /* tree-edge */ 12182 insn_state[t] = DISCOVERED | e; 12183 insn_state[w] = DISCOVERED; 12184 if (env->cfg.cur_stack >= env->prog->len) 12185 return -E2BIG; 12186 insn_stack[env->cfg.cur_stack++] = w; 12187 return KEEP_EXPLORING; 12188 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 12189 if (loop_ok && env->bpf_capable) 12190 return DONE_EXPLORING; 12191 verbose_linfo(env, t, "%d: ", t); 12192 verbose_linfo(env, w, "%d: ", w); 12193 verbose(env, "back-edge from insn %d to %d\n", t, w); 12194 return -EINVAL; 12195 } else if (insn_state[w] == EXPLORED) { 12196 /* forward- or cross-edge */ 12197 insn_state[t] = DISCOVERED | e; 12198 } else { 12199 verbose(env, "insn state internal bug\n"); 12200 return -EFAULT; 12201 } 12202 return DONE_EXPLORING; 12203 } 12204 12205 static int visit_func_call_insn(int t, int insn_cnt, 12206 struct bpf_insn *insns, 12207 struct bpf_verifier_env *env, 12208 bool visit_callee) 12209 { 12210 int ret; 12211 12212 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 12213 if (ret) 12214 return ret; 12215 12216 if (t + 1 < insn_cnt) 12217 init_explored_state(env, t + 1); 12218 if (visit_callee) { 12219 init_explored_state(env, t); 12220 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 12221 /* It's ok to allow recursion from CFG point of 12222 * view. __check_func_call() will do the actual 12223 * check. 12224 */ 12225 bpf_pseudo_func(insns + t)); 12226 } 12227 return ret; 12228 } 12229 12230 /* Visits the instruction at index t and returns one of the following: 12231 * < 0 - an error occurred 12232 * DONE_EXPLORING - the instruction was fully explored 12233 * KEEP_EXPLORING - there is still work to be done before it is fully explored 12234 */ 12235 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 12236 { 12237 struct bpf_insn *insns = env->prog->insnsi; 12238 int ret; 12239 12240 if (bpf_pseudo_func(insns + t)) 12241 return visit_func_call_insn(t, insn_cnt, insns, env, true); 12242 12243 /* All non-branch instructions have a single fall-through edge. */ 12244 if (BPF_CLASS(insns[t].code) != BPF_JMP && 12245 BPF_CLASS(insns[t].code) != BPF_JMP32) 12246 return push_insn(t, t + 1, FALLTHROUGH, env, false); 12247 12248 switch (BPF_OP(insns[t].code)) { 12249 case BPF_EXIT: 12250 return DONE_EXPLORING; 12251 12252 case BPF_CALL: 12253 if (insns[t].imm == BPF_FUNC_timer_set_callback) 12254 /* Mark this call insn to trigger is_state_visited() check 12255 * before call itself is processed by __check_func_call(). 12256 * Otherwise new async state will be pushed for further 12257 * exploration. 12258 */ 12259 init_explored_state(env, t); 12260 return visit_func_call_insn(t, insn_cnt, insns, env, 12261 insns[t].src_reg == BPF_PSEUDO_CALL); 12262 12263 case BPF_JA: 12264 if (BPF_SRC(insns[t].code) != BPF_K) 12265 return -EINVAL; 12266 12267 /* unconditional jump with single edge */ 12268 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 12269 true); 12270 if (ret) 12271 return ret; 12272 12273 /* unconditional jmp is not a good pruning point, 12274 * but it's marked, since backtracking needs 12275 * to record jmp history in is_state_visited(). 12276 */ 12277 init_explored_state(env, t + insns[t].off + 1); 12278 /* tell verifier to check for equivalent states 12279 * after every call and jump 12280 */ 12281 if (t + 1 < insn_cnt) 12282 init_explored_state(env, t + 1); 12283 12284 return ret; 12285 12286 default: 12287 /* conditional jump with two edges */ 12288 init_explored_state(env, t); 12289 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 12290 if (ret) 12291 return ret; 12292 12293 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 12294 } 12295 } 12296 12297 /* non-recursive depth-first-search to detect loops in BPF program 12298 * loop == back-edge in directed graph 12299 */ 12300 static int check_cfg(struct bpf_verifier_env *env) 12301 { 12302 int insn_cnt = env->prog->len; 12303 int *insn_stack, *insn_state; 12304 int ret = 0; 12305 int i; 12306 12307 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 12308 if (!insn_state) 12309 return -ENOMEM; 12310 12311 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 12312 if (!insn_stack) { 12313 kvfree(insn_state); 12314 return -ENOMEM; 12315 } 12316 12317 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 12318 insn_stack[0] = 0; /* 0 is the first instruction */ 12319 env->cfg.cur_stack = 1; 12320 12321 while (env->cfg.cur_stack > 0) { 12322 int t = insn_stack[env->cfg.cur_stack - 1]; 12323 12324 ret = visit_insn(t, insn_cnt, env); 12325 switch (ret) { 12326 case DONE_EXPLORING: 12327 insn_state[t] = EXPLORED; 12328 env->cfg.cur_stack--; 12329 break; 12330 case KEEP_EXPLORING: 12331 break; 12332 default: 12333 if (ret > 0) { 12334 verbose(env, "visit_insn internal bug\n"); 12335 ret = -EFAULT; 12336 } 12337 goto err_free; 12338 } 12339 } 12340 12341 if (env->cfg.cur_stack < 0) { 12342 verbose(env, "pop stack internal bug\n"); 12343 ret = -EFAULT; 12344 goto err_free; 12345 } 12346 12347 for (i = 0; i < insn_cnt; i++) { 12348 if (insn_state[i] != EXPLORED) { 12349 verbose(env, "unreachable insn %d\n", i); 12350 ret = -EINVAL; 12351 goto err_free; 12352 } 12353 } 12354 ret = 0; /* cfg looks good */ 12355 12356 err_free: 12357 kvfree(insn_state); 12358 kvfree(insn_stack); 12359 env->cfg.insn_state = env->cfg.insn_stack = NULL; 12360 return ret; 12361 } 12362 12363 static int check_abnormal_return(struct bpf_verifier_env *env) 12364 { 12365 int i; 12366 12367 for (i = 1; i < env->subprog_cnt; i++) { 12368 if (env->subprog_info[i].has_ld_abs) { 12369 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 12370 return -EINVAL; 12371 } 12372 if (env->subprog_info[i].has_tail_call) { 12373 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 12374 return -EINVAL; 12375 } 12376 } 12377 return 0; 12378 } 12379 12380 /* The minimum supported BTF func info size */ 12381 #define MIN_BPF_FUNCINFO_SIZE 8 12382 #define MAX_FUNCINFO_REC_SIZE 252 12383 12384 static int check_btf_func(struct bpf_verifier_env *env, 12385 const union bpf_attr *attr, 12386 bpfptr_t uattr) 12387 { 12388 const struct btf_type *type, *func_proto, *ret_type; 12389 u32 i, nfuncs, urec_size, min_size; 12390 u32 krec_size = sizeof(struct bpf_func_info); 12391 struct bpf_func_info *krecord; 12392 struct bpf_func_info_aux *info_aux = NULL; 12393 struct bpf_prog *prog; 12394 const struct btf *btf; 12395 bpfptr_t urecord; 12396 u32 prev_offset = 0; 12397 bool scalar_return; 12398 int ret = -ENOMEM; 12399 12400 nfuncs = attr->func_info_cnt; 12401 if (!nfuncs) { 12402 if (check_abnormal_return(env)) 12403 return -EINVAL; 12404 return 0; 12405 } 12406 12407 if (nfuncs != env->subprog_cnt) { 12408 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 12409 return -EINVAL; 12410 } 12411 12412 urec_size = attr->func_info_rec_size; 12413 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 12414 urec_size > MAX_FUNCINFO_REC_SIZE || 12415 urec_size % sizeof(u32)) { 12416 verbose(env, "invalid func info rec size %u\n", urec_size); 12417 return -EINVAL; 12418 } 12419 12420 prog = env->prog; 12421 btf = prog->aux->btf; 12422 12423 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 12424 min_size = min_t(u32, krec_size, urec_size); 12425 12426 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 12427 if (!krecord) 12428 return -ENOMEM; 12429 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 12430 if (!info_aux) 12431 goto err_free; 12432 12433 for (i = 0; i < nfuncs; i++) { 12434 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 12435 if (ret) { 12436 if (ret == -E2BIG) { 12437 verbose(env, "nonzero tailing record in func info"); 12438 /* set the size kernel expects so loader can zero 12439 * out the rest of the record. 12440 */ 12441 if (copy_to_bpfptr_offset(uattr, 12442 offsetof(union bpf_attr, func_info_rec_size), 12443 &min_size, sizeof(min_size))) 12444 ret = -EFAULT; 12445 } 12446 goto err_free; 12447 } 12448 12449 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 12450 ret = -EFAULT; 12451 goto err_free; 12452 } 12453 12454 /* check insn_off */ 12455 ret = -EINVAL; 12456 if (i == 0) { 12457 if (krecord[i].insn_off) { 12458 verbose(env, 12459 "nonzero insn_off %u for the first func info record", 12460 krecord[i].insn_off); 12461 goto err_free; 12462 } 12463 } else if (krecord[i].insn_off <= prev_offset) { 12464 verbose(env, 12465 "same or smaller insn offset (%u) than previous func info record (%u)", 12466 krecord[i].insn_off, prev_offset); 12467 goto err_free; 12468 } 12469 12470 if (env->subprog_info[i].start != krecord[i].insn_off) { 12471 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 12472 goto err_free; 12473 } 12474 12475 /* check type_id */ 12476 type = btf_type_by_id(btf, krecord[i].type_id); 12477 if (!type || !btf_type_is_func(type)) { 12478 verbose(env, "invalid type id %d in func info", 12479 krecord[i].type_id); 12480 goto err_free; 12481 } 12482 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 12483 12484 func_proto = btf_type_by_id(btf, type->type); 12485 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 12486 /* btf_func_check() already verified it during BTF load */ 12487 goto err_free; 12488 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 12489 scalar_return = 12490 btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type); 12491 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 12492 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 12493 goto err_free; 12494 } 12495 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 12496 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 12497 goto err_free; 12498 } 12499 12500 prev_offset = krecord[i].insn_off; 12501 bpfptr_add(&urecord, urec_size); 12502 } 12503 12504 prog->aux->func_info = krecord; 12505 prog->aux->func_info_cnt = nfuncs; 12506 prog->aux->func_info_aux = info_aux; 12507 return 0; 12508 12509 err_free: 12510 kvfree(krecord); 12511 kfree(info_aux); 12512 return ret; 12513 } 12514 12515 static void adjust_btf_func(struct bpf_verifier_env *env) 12516 { 12517 struct bpf_prog_aux *aux = env->prog->aux; 12518 int i; 12519 12520 if (!aux->func_info) 12521 return; 12522 12523 for (i = 0; i < env->subprog_cnt; i++) 12524 aux->func_info[i].insn_off = env->subprog_info[i].start; 12525 } 12526 12527 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 12528 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 12529 12530 static int check_btf_line(struct bpf_verifier_env *env, 12531 const union bpf_attr *attr, 12532 bpfptr_t uattr) 12533 { 12534 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 12535 struct bpf_subprog_info *sub; 12536 struct bpf_line_info *linfo; 12537 struct bpf_prog *prog; 12538 const struct btf *btf; 12539 bpfptr_t ulinfo; 12540 int err; 12541 12542 nr_linfo = attr->line_info_cnt; 12543 if (!nr_linfo) 12544 return 0; 12545 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 12546 return -EINVAL; 12547 12548 rec_size = attr->line_info_rec_size; 12549 if (rec_size < MIN_BPF_LINEINFO_SIZE || 12550 rec_size > MAX_LINEINFO_REC_SIZE || 12551 rec_size & (sizeof(u32) - 1)) 12552 return -EINVAL; 12553 12554 /* Need to zero it in case the userspace may 12555 * pass in a smaller bpf_line_info object. 12556 */ 12557 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 12558 GFP_KERNEL | __GFP_NOWARN); 12559 if (!linfo) 12560 return -ENOMEM; 12561 12562 prog = env->prog; 12563 btf = prog->aux->btf; 12564 12565 s = 0; 12566 sub = env->subprog_info; 12567 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 12568 expected_size = sizeof(struct bpf_line_info); 12569 ncopy = min_t(u32, expected_size, rec_size); 12570 for (i = 0; i < nr_linfo; i++) { 12571 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 12572 if (err) { 12573 if (err == -E2BIG) { 12574 verbose(env, "nonzero tailing record in line_info"); 12575 if (copy_to_bpfptr_offset(uattr, 12576 offsetof(union bpf_attr, line_info_rec_size), 12577 &expected_size, sizeof(expected_size))) 12578 err = -EFAULT; 12579 } 12580 goto err_free; 12581 } 12582 12583 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 12584 err = -EFAULT; 12585 goto err_free; 12586 } 12587 12588 /* 12589 * Check insn_off to ensure 12590 * 1) strictly increasing AND 12591 * 2) bounded by prog->len 12592 * 12593 * The linfo[0].insn_off == 0 check logically falls into 12594 * the later "missing bpf_line_info for func..." case 12595 * because the first linfo[0].insn_off must be the 12596 * first sub also and the first sub must have 12597 * subprog_info[0].start == 0. 12598 */ 12599 if ((i && linfo[i].insn_off <= prev_offset) || 12600 linfo[i].insn_off >= prog->len) { 12601 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 12602 i, linfo[i].insn_off, prev_offset, 12603 prog->len); 12604 err = -EINVAL; 12605 goto err_free; 12606 } 12607 12608 if (!prog->insnsi[linfo[i].insn_off].code) { 12609 verbose(env, 12610 "Invalid insn code at line_info[%u].insn_off\n", 12611 i); 12612 err = -EINVAL; 12613 goto err_free; 12614 } 12615 12616 if (!btf_name_by_offset(btf, linfo[i].line_off) || 12617 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 12618 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 12619 err = -EINVAL; 12620 goto err_free; 12621 } 12622 12623 if (s != env->subprog_cnt) { 12624 if (linfo[i].insn_off == sub[s].start) { 12625 sub[s].linfo_idx = i; 12626 s++; 12627 } else if (sub[s].start < linfo[i].insn_off) { 12628 verbose(env, "missing bpf_line_info for func#%u\n", s); 12629 err = -EINVAL; 12630 goto err_free; 12631 } 12632 } 12633 12634 prev_offset = linfo[i].insn_off; 12635 bpfptr_add(&ulinfo, rec_size); 12636 } 12637 12638 if (s != env->subprog_cnt) { 12639 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 12640 env->subprog_cnt - s, s); 12641 err = -EINVAL; 12642 goto err_free; 12643 } 12644 12645 prog->aux->linfo = linfo; 12646 prog->aux->nr_linfo = nr_linfo; 12647 12648 return 0; 12649 12650 err_free: 12651 kvfree(linfo); 12652 return err; 12653 } 12654 12655 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 12656 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 12657 12658 static int check_core_relo(struct bpf_verifier_env *env, 12659 const union bpf_attr *attr, 12660 bpfptr_t uattr) 12661 { 12662 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 12663 struct bpf_core_relo core_relo = {}; 12664 struct bpf_prog *prog = env->prog; 12665 const struct btf *btf = prog->aux->btf; 12666 struct bpf_core_ctx ctx = { 12667 .log = &env->log, 12668 .btf = btf, 12669 }; 12670 bpfptr_t u_core_relo; 12671 int err; 12672 12673 nr_core_relo = attr->core_relo_cnt; 12674 if (!nr_core_relo) 12675 return 0; 12676 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 12677 return -EINVAL; 12678 12679 rec_size = attr->core_relo_rec_size; 12680 if (rec_size < MIN_CORE_RELO_SIZE || 12681 rec_size > MAX_CORE_RELO_SIZE || 12682 rec_size % sizeof(u32)) 12683 return -EINVAL; 12684 12685 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 12686 expected_size = sizeof(struct bpf_core_relo); 12687 ncopy = min_t(u32, expected_size, rec_size); 12688 12689 /* Unlike func_info and line_info, copy and apply each CO-RE 12690 * relocation record one at a time. 12691 */ 12692 for (i = 0; i < nr_core_relo; i++) { 12693 /* future proofing when sizeof(bpf_core_relo) changes */ 12694 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 12695 if (err) { 12696 if (err == -E2BIG) { 12697 verbose(env, "nonzero tailing record in core_relo"); 12698 if (copy_to_bpfptr_offset(uattr, 12699 offsetof(union bpf_attr, core_relo_rec_size), 12700 &expected_size, sizeof(expected_size))) 12701 err = -EFAULT; 12702 } 12703 break; 12704 } 12705 12706 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 12707 err = -EFAULT; 12708 break; 12709 } 12710 12711 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 12712 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 12713 i, core_relo.insn_off, prog->len); 12714 err = -EINVAL; 12715 break; 12716 } 12717 12718 err = bpf_core_apply(&ctx, &core_relo, i, 12719 &prog->insnsi[core_relo.insn_off / 8]); 12720 if (err) 12721 break; 12722 bpfptr_add(&u_core_relo, rec_size); 12723 } 12724 return err; 12725 } 12726 12727 static int check_btf_info(struct bpf_verifier_env *env, 12728 const union bpf_attr *attr, 12729 bpfptr_t uattr) 12730 { 12731 struct btf *btf; 12732 int err; 12733 12734 if (!attr->func_info_cnt && !attr->line_info_cnt) { 12735 if (check_abnormal_return(env)) 12736 return -EINVAL; 12737 return 0; 12738 } 12739 12740 btf = btf_get_by_fd(attr->prog_btf_fd); 12741 if (IS_ERR(btf)) 12742 return PTR_ERR(btf); 12743 if (btf_is_kernel(btf)) { 12744 btf_put(btf); 12745 return -EACCES; 12746 } 12747 env->prog->aux->btf = btf; 12748 12749 err = check_btf_func(env, attr, uattr); 12750 if (err) 12751 return err; 12752 12753 err = check_btf_line(env, attr, uattr); 12754 if (err) 12755 return err; 12756 12757 err = check_core_relo(env, attr, uattr); 12758 if (err) 12759 return err; 12760 12761 return 0; 12762 } 12763 12764 /* check %cur's range satisfies %old's */ 12765 static bool range_within(struct bpf_reg_state *old, 12766 struct bpf_reg_state *cur) 12767 { 12768 return old->umin_value <= cur->umin_value && 12769 old->umax_value >= cur->umax_value && 12770 old->smin_value <= cur->smin_value && 12771 old->smax_value >= cur->smax_value && 12772 old->u32_min_value <= cur->u32_min_value && 12773 old->u32_max_value >= cur->u32_max_value && 12774 old->s32_min_value <= cur->s32_min_value && 12775 old->s32_max_value >= cur->s32_max_value; 12776 } 12777 12778 /* If in the old state two registers had the same id, then they need to have 12779 * the same id in the new state as well. But that id could be different from 12780 * the old state, so we need to track the mapping from old to new ids. 12781 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 12782 * regs with old id 5 must also have new id 9 for the new state to be safe. But 12783 * regs with a different old id could still have new id 9, we don't care about 12784 * that. 12785 * So we look through our idmap to see if this old id has been seen before. If 12786 * so, we require the new id to match; otherwise, we add the id pair to the map. 12787 */ 12788 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 12789 { 12790 unsigned int i; 12791 12792 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 12793 if (!idmap[i].old) { 12794 /* Reached an empty slot; haven't seen this id before */ 12795 idmap[i].old = old_id; 12796 idmap[i].cur = cur_id; 12797 return true; 12798 } 12799 if (idmap[i].old == old_id) 12800 return idmap[i].cur == cur_id; 12801 } 12802 /* We ran out of idmap slots, which should be impossible */ 12803 WARN_ON_ONCE(1); 12804 return false; 12805 } 12806 12807 static void clean_func_state(struct bpf_verifier_env *env, 12808 struct bpf_func_state *st) 12809 { 12810 enum bpf_reg_liveness live; 12811 int i, j; 12812 12813 for (i = 0; i < BPF_REG_FP; i++) { 12814 live = st->regs[i].live; 12815 /* liveness must not touch this register anymore */ 12816 st->regs[i].live |= REG_LIVE_DONE; 12817 if (!(live & REG_LIVE_READ)) 12818 /* since the register is unused, clear its state 12819 * to make further comparison simpler 12820 */ 12821 __mark_reg_not_init(env, &st->regs[i]); 12822 } 12823 12824 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 12825 live = st->stack[i].spilled_ptr.live; 12826 /* liveness must not touch this stack slot anymore */ 12827 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 12828 if (!(live & REG_LIVE_READ)) { 12829 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 12830 for (j = 0; j < BPF_REG_SIZE; j++) 12831 st->stack[i].slot_type[j] = STACK_INVALID; 12832 } 12833 } 12834 } 12835 12836 static void clean_verifier_state(struct bpf_verifier_env *env, 12837 struct bpf_verifier_state *st) 12838 { 12839 int i; 12840 12841 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 12842 /* all regs in this state in all frames were already marked */ 12843 return; 12844 12845 for (i = 0; i <= st->curframe; i++) 12846 clean_func_state(env, st->frame[i]); 12847 } 12848 12849 /* the parentage chains form a tree. 12850 * the verifier states are added to state lists at given insn and 12851 * pushed into state stack for future exploration. 12852 * when the verifier reaches bpf_exit insn some of the verifer states 12853 * stored in the state lists have their final liveness state already, 12854 * but a lot of states will get revised from liveness point of view when 12855 * the verifier explores other branches. 12856 * Example: 12857 * 1: r0 = 1 12858 * 2: if r1 == 100 goto pc+1 12859 * 3: r0 = 2 12860 * 4: exit 12861 * when the verifier reaches exit insn the register r0 in the state list of 12862 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 12863 * of insn 2 and goes exploring further. At the insn 4 it will walk the 12864 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 12865 * 12866 * Since the verifier pushes the branch states as it sees them while exploring 12867 * the program the condition of walking the branch instruction for the second 12868 * time means that all states below this branch were already explored and 12869 * their final liveness marks are already propagated. 12870 * Hence when the verifier completes the search of state list in is_state_visited() 12871 * we can call this clean_live_states() function to mark all liveness states 12872 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 12873 * will not be used. 12874 * This function also clears the registers and stack for states that !READ 12875 * to simplify state merging. 12876 * 12877 * Important note here that walking the same branch instruction in the callee 12878 * doesn't meant that the states are DONE. The verifier has to compare 12879 * the callsites 12880 */ 12881 static void clean_live_states(struct bpf_verifier_env *env, int insn, 12882 struct bpf_verifier_state *cur) 12883 { 12884 struct bpf_verifier_state_list *sl; 12885 int i; 12886 12887 sl = *explored_state(env, insn); 12888 while (sl) { 12889 if (sl->state.branches) 12890 goto next; 12891 if (sl->state.insn_idx != insn || 12892 sl->state.curframe != cur->curframe) 12893 goto next; 12894 for (i = 0; i <= cur->curframe; i++) 12895 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 12896 goto next; 12897 clean_verifier_state(env, &sl->state); 12898 next: 12899 sl = sl->next; 12900 } 12901 } 12902 12903 /* Returns true if (rold safe implies rcur safe) */ 12904 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 12905 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 12906 { 12907 bool equal; 12908 12909 if (!(rold->live & REG_LIVE_READ)) 12910 /* explored state didn't use this */ 12911 return true; 12912 12913 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 12914 12915 if (rold->type == PTR_TO_STACK) 12916 /* two stack pointers are equal only if they're pointing to 12917 * the same stack frame, since fp-8 in foo != fp-8 in bar 12918 */ 12919 return equal && rold->frameno == rcur->frameno; 12920 12921 if (equal) 12922 return true; 12923 12924 if (rold->type == NOT_INIT) 12925 /* explored state can't have used this */ 12926 return true; 12927 if (rcur->type == NOT_INIT) 12928 return false; 12929 switch (base_type(rold->type)) { 12930 case SCALAR_VALUE: 12931 if (env->explore_alu_limits) 12932 return false; 12933 if (rcur->type == SCALAR_VALUE) { 12934 if (!rold->precise) 12935 return true; 12936 /* new val must satisfy old val knowledge */ 12937 return range_within(rold, rcur) && 12938 tnum_in(rold->var_off, rcur->var_off); 12939 } else { 12940 /* We're trying to use a pointer in place of a scalar. 12941 * Even if the scalar was unbounded, this could lead to 12942 * pointer leaks because scalars are allowed to leak 12943 * while pointers are not. We could make this safe in 12944 * special cases if root is calling us, but it's 12945 * probably not worth the hassle. 12946 */ 12947 return false; 12948 } 12949 case PTR_TO_MAP_KEY: 12950 case PTR_TO_MAP_VALUE: 12951 /* a PTR_TO_MAP_VALUE could be safe to use as a 12952 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 12953 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 12954 * checked, doing so could have affected others with the same 12955 * id, and we can't check for that because we lost the id when 12956 * we converted to a PTR_TO_MAP_VALUE. 12957 */ 12958 if (type_may_be_null(rold->type)) { 12959 if (!type_may_be_null(rcur->type)) 12960 return false; 12961 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 12962 return false; 12963 /* Check our ids match any regs they're supposed to */ 12964 return check_ids(rold->id, rcur->id, idmap); 12965 } 12966 12967 /* If the new min/max/var_off satisfy the old ones and 12968 * everything else matches, we are OK. 12969 * 'id' is not compared, since it's only used for maps with 12970 * bpf_spin_lock inside map element and in such cases if 12971 * the rest of the prog is valid for one map element then 12972 * it's valid for all map elements regardless of the key 12973 * used in bpf_map_lookup() 12974 */ 12975 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 12976 range_within(rold, rcur) && 12977 tnum_in(rold->var_off, rcur->var_off); 12978 case PTR_TO_PACKET_META: 12979 case PTR_TO_PACKET: 12980 if (rcur->type != rold->type) 12981 return false; 12982 /* We must have at least as much range as the old ptr 12983 * did, so that any accesses which were safe before are 12984 * still safe. This is true even if old range < old off, 12985 * since someone could have accessed through (ptr - k), or 12986 * even done ptr -= k in a register, to get a safe access. 12987 */ 12988 if (rold->range > rcur->range) 12989 return false; 12990 /* If the offsets don't match, we can't trust our alignment; 12991 * nor can we be sure that we won't fall out of range. 12992 */ 12993 if (rold->off != rcur->off) 12994 return false; 12995 /* id relations must be preserved */ 12996 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 12997 return false; 12998 /* new val must satisfy old val knowledge */ 12999 return range_within(rold, rcur) && 13000 tnum_in(rold->var_off, rcur->var_off); 13001 case PTR_TO_CTX: 13002 case CONST_PTR_TO_MAP: 13003 case PTR_TO_PACKET_END: 13004 case PTR_TO_FLOW_KEYS: 13005 case PTR_TO_SOCKET: 13006 case PTR_TO_SOCK_COMMON: 13007 case PTR_TO_TCP_SOCK: 13008 case PTR_TO_XDP_SOCK: 13009 /* Only valid matches are exact, which memcmp() above 13010 * would have accepted 13011 */ 13012 default: 13013 /* Don't know what's going on, just say it's not safe */ 13014 return false; 13015 } 13016 13017 /* Shouldn't get here; if we do, say it's not safe */ 13018 WARN_ON_ONCE(1); 13019 return false; 13020 } 13021 13022 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 13023 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 13024 { 13025 int i, spi; 13026 13027 /* walk slots of the explored stack and ignore any additional 13028 * slots in the current stack, since explored(safe) state 13029 * didn't use them 13030 */ 13031 for (i = 0; i < old->allocated_stack; i++) { 13032 spi = i / BPF_REG_SIZE; 13033 13034 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 13035 i += BPF_REG_SIZE - 1; 13036 /* explored state didn't use this */ 13037 continue; 13038 } 13039 13040 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 13041 continue; 13042 13043 /* explored stack has more populated slots than current stack 13044 * and these slots were used 13045 */ 13046 if (i >= cur->allocated_stack) 13047 return false; 13048 13049 /* if old state was safe with misc data in the stack 13050 * it will be safe with zero-initialized stack. 13051 * The opposite is not true 13052 */ 13053 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 13054 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 13055 continue; 13056 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 13057 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 13058 /* Ex: old explored (safe) state has STACK_SPILL in 13059 * this stack slot, but current has STACK_MISC -> 13060 * this verifier states are not equivalent, 13061 * return false to continue verification of this path 13062 */ 13063 return false; 13064 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 13065 continue; 13066 if (!is_spilled_reg(&old->stack[spi])) 13067 continue; 13068 if (!regsafe(env, &old->stack[spi].spilled_ptr, 13069 &cur->stack[spi].spilled_ptr, idmap)) 13070 /* when explored and current stack slot are both storing 13071 * spilled registers, check that stored pointers types 13072 * are the same as well. 13073 * Ex: explored safe path could have stored 13074 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 13075 * but current path has stored: 13076 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 13077 * such verifier states are not equivalent. 13078 * return false to continue verification of this path 13079 */ 13080 return false; 13081 } 13082 return true; 13083 } 13084 13085 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 13086 { 13087 if (old->acquired_refs != cur->acquired_refs) 13088 return false; 13089 return !memcmp(old->refs, cur->refs, 13090 sizeof(*old->refs) * old->acquired_refs); 13091 } 13092 13093 /* compare two verifier states 13094 * 13095 * all states stored in state_list are known to be valid, since 13096 * verifier reached 'bpf_exit' instruction through them 13097 * 13098 * this function is called when verifier exploring different branches of 13099 * execution popped from the state stack. If it sees an old state that has 13100 * more strict register state and more strict stack state then this execution 13101 * branch doesn't need to be explored further, since verifier already 13102 * concluded that more strict state leads to valid finish. 13103 * 13104 * Therefore two states are equivalent if register state is more conservative 13105 * and explored stack state is more conservative than the current one. 13106 * Example: 13107 * explored current 13108 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 13109 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 13110 * 13111 * In other words if current stack state (one being explored) has more 13112 * valid slots than old one that already passed validation, it means 13113 * the verifier can stop exploring and conclude that current state is valid too 13114 * 13115 * Similarly with registers. If explored state has register type as invalid 13116 * whereas register type in current state is meaningful, it means that 13117 * the current state will reach 'bpf_exit' instruction safely 13118 */ 13119 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 13120 struct bpf_func_state *cur) 13121 { 13122 int i; 13123 13124 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 13125 for (i = 0; i < MAX_BPF_REG; i++) 13126 if (!regsafe(env, &old->regs[i], &cur->regs[i], 13127 env->idmap_scratch)) 13128 return false; 13129 13130 if (!stacksafe(env, old, cur, env->idmap_scratch)) 13131 return false; 13132 13133 if (!refsafe(old, cur)) 13134 return false; 13135 13136 return true; 13137 } 13138 13139 static bool states_equal(struct bpf_verifier_env *env, 13140 struct bpf_verifier_state *old, 13141 struct bpf_verifier_state *cur) 13142 { 13143 int i; 13144 13145 if (old->curframe != cur->curframe) 13146 return false; 13147 13148 /* Verification state from speculative execution simulation 13149 * must never prune a non-speculative execution one. 13150 */ 13151 if (old->speculative && !cur->speculative) 13152 return false; 13153 13154 if (old->active_lock.ptr != cur->active_lock.ptr || 13155 old->active_lock.id != cur->active_lock.id) 13156 return false; 13157 13158 if (old->active_rcu_lock != cur->active_rcu_lock) 13159 return false; 13160 13161 /* for states to be equal callsites have to be the same 13162 * and all frame states need to be equivalent 13163 */ 13164 for (i = 0; i <= old->curframe; i++) { 13165 if (old->frame[i]->callsite != cur->frame[i]->callsite) 13166 return false; 13167 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 13168 return false; 13169 } 13170 return true; 13171 } 13172 13173 /* Return 0 if no propagation happened. Return negative error code if error 13174 * happened. Otherwise, return the propagated bit. 13175 */ 13176 static int propagate_liveness_reg(struct bpf_verifier_env *env, 13177 struct bpf_reg_state *reg, 13178 struct bpf_reg_state *parent_reg) 13179 { 13180 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 13181 u8 flag = reg->live & REG_LIVE_READ; 13182 int err; 13183 13184 /* When comes here, read flags of PARENT_REG or REG could be any of 13185 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 13186 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 13187 */ 13188 if (parent_flag == REG_LIVE_READ64 || 13189 /* Or if there is no read flag from REG. */ 13190 !flag || 13191 /* Or if the read flag from REG is the same as PARENT_REG. */ 13192 parent_flag == flag) 13193 return 0; 13194 13195 err = mark_reg_read(env, reg, parent_reg, flag); 13196 if (err) 13197 return err; 13198 13199 return flag; 13200 } 13201 13202 /* A write screens off any subsequent reads; but write marks come from the 13203 * straight-line code between a state and its parent. When we arrive at an 13204 * equivalent state (jump target or such) we didn't arrive by the straight-line 13205 * code, so read marks in the state must propagate to the parent regardless 13206 * of the state's write marks. That's what 'parent == state->parent' comparison 13207 * in mark_reg_read() is for. 13208 */ 13209 static int propagate_liveness(struct bpf_verifier_env *env, 13210 const struct bpf_verifier_state *vstate, 13211 struct bpf_verifier_state *vparent) 13212 { 13213 struct bpf_reg_state *state_reg, *parent_reg; 13214 struct bpf_func_state *state, *parent; 13215 int i, frame, err = 0; 13216 13217 if (vparent->curframe != vstate->curframe) { 13218 WARN(1, "propagate_live: parent frame %d current frame %d\n", 13219 vparent->curframe, vstate->curframe); 13220 return -EFAULT; 13221 } 13222 /* Propagate read liveness of registers... */ 13223 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 13224 for (frame = 0; frame <= vstate->curframe; frame++) { 13225 parent = vparent->frame[frame]; 13226 state = vstate->frame[frame]; 13227 parent_reg = parent->regs; 13228 state_reg = state->regs; 13229 /* We don't need to worry about FP liveness, it's read-only */ 13230 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 13231 err = propagate_liveness_reg(env, &state_reg[i], 13232 &parent_reg[i]); 13233 if (err < 0) 13234 return err; 13235 if (err == REG_LIVE_READ64) 13236 mark_insn_zext(env, &parent_reg[i]); 13237 } 13238 13239 /* Propagate stack slots. */ 13240 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 13241 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 13242 parent_reg = &parent->stack[i].spilled_ptr; 13243 state_reg = &state->stack[i].spilled_ptr; 13244 err = propagate_liveness_reg(env, state_reg, 13245 parent_reg); 13246 if (err < 0) 13247 return err; 13248 } 13249 } 13250 return 0; 13251 } 13252 13253 /* find precise scalars in the previous equivalent state and 13254 * propagate them into the current state 13255 */ 13256 static int propagate_precision(struct bpf_verifier_env *env, 13257 const struct bpf_verifier_state *old) 13258 { 13259 struct bpf_reg_state *state_reg; 13260 struct bpf_func_state *state; 13261 int i, err = 0, fr; 13262 13263 for (fr = old->curframe; fr >= 0; fr--) { 13264 state = old->frame[fr]; 13265 state_reg = state->regs; 13266 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 13267 if (state_reg->type != SCALAR_VALUE || 13268 !state_reg->precise) 13269 continue; 13270 if (env->log.level & BPF_LOG_LEVEL2) 13271 verbose(env, "frame %d: propagating r%d\n", i, fr); 13272 err = mark_chain_precision_frame(env, fr, i); 13273 if (err < 0) 13274 return err; 13275 } 13276 13277 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 13278 if (!is_spilled_reg(&state->stack[i])) 13279 continue; 13280 state_reg = &state->stack[i].spilled_ptr; 13281 if (state_reg->type != SCALAR_VALUE || 13282 !state_reg->precise) 13283 continue; 13284 if (env->log.level & BPF_LOG_LEVEL2) 13285 verbose(env, "frame %d: propagating fp%d\n", 13286 (-i - 1) * BPF_REG_SIZE, fr); 13287 err = mark_chain_precision_stack_frame(env, fr, i); 13288 if (err < 0) 13289 return err; 13290 } 13291 } 13292 return 0; 13293 } 13294 13295 static bool states_maybe_looping(struct bpf_verifier_state *old, 13296 struct bpf_verifier_state *cur) 13297 { 13298 struct bpf_func_state *fold, *fcur; 13299 int i, fr = cur->curframe; 13300 13301 if (old->curframe != fr) 13302 return false; 13303 13304 fold = old->frame[fr]; 13305 fcur = cur->frame[fr]; 13306 for (i = 0; i < MAX_BPF_REG; i++) 13307 if (memcmp(&fold->regs[i], &fcur->regs[i], 13308 offsetof(struct bpf_reg_state, parent))) 13309 return false; 13310 return true; 13311 } 13312 13313 13314 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 13315 { 13316 struct bpf_verifier_state_list *new_sl; 13317 struct bpf_verifier_state_list *sl, **pprev; 13318 struct bpf_verifier_state *cur = env->cur_state, *new; 13319 int i, j, err, states_cnt = 0; 13320 bool add_new_state = env->test_state_freq ? true : false; 13321 13322 cur->last_insn_idx = env->prev_insn_idx; 13323 if (!env->insn_aux_data[insn_idx].prune_point) 13324 /* this 'insn_idx' instruction wasn't marked, so we will not 13325 * be doing state search here 13326 */ 13327 return 0; 13328 13329 /* bpf progs typically have pruning point every 4 instructions 13330 * http://vger.kernel.org/bpfconf2019.html#session-1 13331 * Do not add new state for future pruning if the verifier hasn't seen 13332 * at least 2 jumps and at least 8 instructions. 13333 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 13334 * In tests that amounts to up to 50% reduction into total verifier 13335 * memory consumption and 20% verifier time speedup. 13336 */ 13337 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 13338 env->insn_processed - env->prev_insn_processed >= 8) 13339 add_new_state = true; 13340 13341 pprev = explored_state(env, insn_idx); 13342 sl = *pprev; 13343 13344 clean_live_states(env, insn_idx, cur); 13345 13346 while (sl) { 13347 states_cnt++; 13348 if (sl->state.insn_idx != insn_idx) 13349 goto next; 13350 13351 if (sl->state.branches) { 13352 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 13353 13354 if (frame->in_async_callback_fn && 13355 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 13356 /* Different async_entry_cnt means that the verifier is 13357 * processing another entry into async callback. 13358 * Seeing the same state is not an indication of infinite 13359 * loop or infinite recursion. 13360 * But finding the same state doesn't mean that it's safe 13361 * to stop processing the current state. The previous state 13362 * hasn't yet reached bpf_exit, since state.branches > 0. 13363 * Checking in_async_callback_fn alone is not enough either. 13364 * Since the verifier still needs to catch infinite loops 13365 * inside async callbacks. 13366 */ 13367 } else if (states_maybe_looping(&sl->state, cur) && 13368 states_equal(env, &sl->state, cur)) { 13369 verbose_linfo(env, insn_idx, "; "); 13370 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 13371 return -EINVAL; 13372 } 13373 /* if the verifier is processing a loop, avoid adding new state 13374 * too often, since different loop iterations have distinct 13375 * states and may not help future pruning. 13376 * This threshold shouldn't be too low to make sure that 13377 * a loop with large bound will be rejected quickly. 13378 * The most abusive loop will be: 13379 * r1 += 1 13380 * if r1 < 1000000 goto pc-2 13381 * 1M insn_procssed limit / 100 == 10k peak states. 13382 * This threshold shouldn't be too high either, since states 13383 * at the end of the loop are likely to be useful in pruning. 13384 */ 13385 if (env->jmps_processed - env->prev_jmps_processed < 20 && 13386 env->insn_processed - env->prev_insn_processed < 100) 13387 add_new_state = false; 13388 goto miss; 13389 } 13390 if (states_equal(env, &sl->state, cur)) { 13391 sl->hit_cnt++; 13392 /* reached equivalent register/stack state, 13393 * prune the search. 13394 * Registers read by the continuation are read by us. 13395 * If we have any write marks in env->cur_state, they 13396 * will prevent corresponding reads in the continuation 13397 * from reaching our parent (an explored_state). Our 13398 * own state will get the read marks recorded, but 13399 * they'll be immediately forgotten as we're pruning 13400 * this state and will pop a new one. 13401 */ 13402 err = propagate_liveness(env, &sl->state, cur); 13403 13404 /* if previous state reached the exit with precision and 13405 * current state is equivalent to it (except precsion marks) 13406 * the precision needs to be propagated back in 13407 * the current state. 13408 */ 13409 err = err ? : push_jmp_history(env, cur); 13410 err = err ? : propagate_precision(env, &sl->state); 13411 if (err) 13412 return err; 13413 return 1; 13414 } 13415 miss: 13416 /* when new state is not going to be added do not increase miss count. 13417 * Otherwise several loop iterations will remove the state 13418 * recorded earlier. The goal of these heuristics is to have 13419 * states from some iterations of the loop (some in the beginning 13420 * and some at the end) to help pruning. 13421 */ 13422 if (add_new_state) 13423 sl->miss_cnt++; 13424 /* heuristic to determine whether this state is beneficial 13425 * to keep checking from state equivalence point of view. 13426 * Higher numbers increase max_states_per_insn and verification time, 13427 * but do not meaningfully decrease insn_processed. 13428 */ 13429 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 13430 /* the state is unlikely to be useful. Remove it to 13431 * speed up verification 13432 */ 13433 *pprev = sl->next; 13434 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 13435 u32 br = sl->state.branches; 13436 13437 WARN_ONCE(br, 13438 "BUG live_done but branches_to_explore %d\n", 13439 br); 13440 free_verifier_state(&sl->state, false); 13441 kfree(sl); 13442 env->peak_states--; 13443 } else { 13444 /* cannot free this state, since parentage chain may 13445 * walk it later. Add it for free_list instead to 13446 * be freed at the end of verification 13447 */ 13448 sl->next = env->free_list; 13449 env->free_list = sl; 13450 } 13451 sl = *pprev; 13452 continue; 13453 } 13454 next: 13455 pprev = &sl->next; 13456 sl = *pprev; 13457 } 13458 13459 if (env->max_states_per_insn < states_cnt) 13460 env->max_states_per_insn = states_cnt; 13461 13462 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 13463 return push_jmp_history(env, cur); 13464 13465 if (!add_new_state) 13466 return push_jmp_history(env, cur); 13467 13468 /* There were no equivalent states, remember the current one. 13469 * Technically the current state is not proven to be safe yet, 13470 * but it will either reach outer most bpf_exit (which means it's safe) 13471 * or it will be rejected. When there are no loops the verifier won't be 13472 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 13473 * again on the way to bpf_exit. 13474 * When looping the sl->state.branches will be > 0 and this state 13475 * will not be considered for equivalence until branches == 0. 13476 */ 13477 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 13478 if (!new_sl) 13479 return -ENOMEM; 13480 env->total_states++; 13481 env->peak_states++; 13482 env->prev_jmps_processed = env->jmps_processed; 13483 env->prev_insn_processed = env->insn_processed; 13484 13485 /* forget precise markings we inherited, see __mark_chain_precision */ 13486 if (env->bpf_capable) 13487 mark_all_scalars_imprecise(env, cur); 13488 13489 /* add new state to the head of linked list */ 13490 new = &new_sl->state; 13491 err = copy_verifier_state(new, cur); 13492 if (err) { 13493 free_verifier_state(new, false); 13494 kfree(new_sl); 13495 return err; 13496 } 13497 new->insn_idx = insn_idx; 13498 WARN_ONCE(new->branches != 1, 13499 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 13500 13501 cur->parent = new; 13502 cur->first_insn_idx = insn_idx; 13503 clear_jmp_history(cur); 13504 new_sl->next = *explored_state(env, insn_idx); 13505 *explored_state(env, insn_idx) = new_sl; 13506 /* connect new state to parentage chain. Current frame needs all 13507 * registers connected. Only r6 - r9 of the callers are alive (pushed 13508 * to the stack implicitly by JITs) so in callers' frames connect just 13509 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 13510 * the state of the call instruction (with WRITTEN set), and r0 comes 13511 * from callee with its full parentage chain, anyway. 13512 */ 13513 /* clear write marks in current state: the writes we did are not writes 13514 * our child did, so they don't screen off its reads from us. 13515 * (There are no read marks in current state, because reads always mark 13516 * their parent and current state never has children yet. Only 13517 * explored_states can get read marks.) 13518 */ 13519 for (j = 0; j <= cur->curframe; j++) { 13520 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 13521 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 13522 for (i = 0; i < BPF_REG_FP; i++) 13523 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 13524 } 13525 13526 /* all stack frames are accessible from callee, clear them all */ 13527 for (j = 0; j <= cur->curframe; j++) { 13528 struct bpf_func_state *frame = cur->frame[j]; 13529 struct bpf_func_state *newframe = new->frame[j]; 13530 13531 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 13532 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 13533 frame->stack[i].spilled_ptr.parent = 13534 &newframe->stack[i].spilled_ptr; 13535 } 13536 } 13537 return 0; 13538 } 13539 13540 /* Return true if it's OK to have the same insn return a different type. */ 13541 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 13542 { 13543 switch (base_type(type)) { 13544 case PTR_TO_CTX: 13545 case PTR_TO_SOCKET: 13546 case PTR_TO_SOCK_COMMON: 13547 case PTR_TO_TCP_SOCK: 13548 case PTR_TO_XDP_SOCK: 13549 case PTR_TO_BTF_ID: 13550 return false; 13551 default: 13552 return true; 13553 } 13554 } 13555 13556 /* If an instruction was previously used with particular pointer types, then we 13557 * need to be careful to avoid cases such as the below, where it may be ok 13558 * for one branch accessing the pointer, but not ok for the other branch: 13559 * 13560 * R1 = sock_ptr 13561 * goto X; 13562 * ... 13563 * R1 = some_other_valid_ptr; 13564 * goto X; 13565 * ... 13566 * R2 = *(u32 *)(R1 + 0); 13567 */ 13568 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 13569 { 13570 return src != prev && (!reg_type_mismatch_ok(src) || 13571 !reg_type_mismatch_ok(prev)); 13572 } 13573 13574 static int do_check(struct bpf_verifier_env *env) 13575 { 13576 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 13577 struct bpf_verifier_state *state = env->cur_state; 13578 struct bpf_insn *insns = env->prog->insnsi; 13579 struct bpf_reg_state *regs; 13580 int insn_cnt = env->prog->len; 13581 bool do_print_state = false; 13582 int prev_insn_idx = -1; 13583 13584 for (;;) { 13585 struct bpf_insn *insn; 13586 u8 class; 13587 int err; 13588 13589 env->prev_insn_idx = prev_insn_idx; 13590 if (env->insn_idx >= insn_cnt) { 13591 verbose(env, "invalid insn idx %d insn_cnt %d\n", 13592 env->insn_idx, insn_cnt); 13593 return -EFAULT; 13594 } 13595 13596 insn = &insns[env->insn_idx]; 13597 class = BPF_CLASS(insn->code); 13598 13599 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 13600 verbose(env, 13601 "BPF program is too large. Processed %d insn\n", 13602 env->insn_processed); 13603 return -E2BIG; 13604 } 13605 13606 err = is_state_visited(env, env->insn_idx); 13607 if (err < 0) 13608 return err; 13609 if (err == 1) { 13610 /* found equivalent state, can prune the search */ 13611 if (env->log.level & BPF_LOG_LEVEL) { 13612 if (do_print_state) 13613 verbose(env, "\nfrom %d to %d%s: safe\n", 13614 env->prev_insn_idx, env->insn_idx, 13615 env->cur_state->speculative ? 13616 " (speculative execution)" : ""); 13617 else 13618 verbose(env, "%d: safe\n", env->insn_idx); 13619 } 13620 goto process_bpf_exit; 13621 } 13622 13623 if (signal_pending(current)) 13624 return -EAGAIN; 13625 13626 if (need_resched()) 13627 cond_resched(); 13628 13629 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 13630 verbose(env, "\nfrom %d to %d%s:", 13631 env->prev_insn_idx, env->insn_idx, 13632 env->cur_state->speculative ? 13633 " (speculative execution)" : ""); 13634 print_verifier_state(env, state->frame[state->curframe], true); 13635 do_print_state = false; 13636 } 13637 13638 if (env->log.level & BPF_LOG_LEVEL) { 13639 const struct bpf_insn_cbs cbs = { 13640 .cb_call = disasm_kfunc_name, 13641 .cb_print = verbose, 13642 .private_data = env, 13643 }; 13644 13645 if (verifier_state_scratched(env)) 13646 print_insn_state(env, state->frame[state->curframe]); 13647 13648 verbose_linfo(env, env->insn_idx, "; "); 13649 env->prev_log_len = env->log.len_used; 13650 verbose(env, "%d: ", env->insn_idx); 13651 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 13652 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; 13653 env->prev_log_len = env->log.len_used; 13654 } 13655 13656 if (bpf_prog_is_dev_bound(env->prog->aux)) { 13657 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 13658 env->prev_insn_idx); 13659 if (err) 13660 return err; 13661 } 13662 13663 regs = cur_regs(env); 13664 sanitize_mark_insn_seen(env); 13665 prev_insn_idx = env->insn_idx; 13666 13667 if (class == BPF_ALU || class == BPF_ALU64) { 13668 err = check_alu_op(env, insn); 13669 if (err) 13670 return err; 13671 13672 } else if (class == BPF_LDX) { 13673 enum bpf_reg_type *prev_src_type, src_reg_type; 13674 13675 /* check for reserved fields is already done */ 13676 13677 /* check src operand */ 13678 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13679 if (err) 13680 return err; 13681 13682 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 13683 if (err) 13684 return err; 13685 13686 src_reg_type = regs[insn->src_reg].type; 13687 13688 /* check that memory (src_reg + off) is readable, 13689 * the state of dst_reg will be updated by this func 13690 */ 13691 err = check_mem_access(env, env->insn_idx, insn->src_reg, 13692 insn->off, BPF_SIZE(insn->code), 13693 BPF_READ, insn->dst_reg, false); 13694 if (err) 13695 return err; 13696 13697 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 13698 13699 if (*prev_src_type == NOT_INIT) { 13700 /* saw a valid insn 13701 * dst_reg = *(u32 *)(src_reg + off) 13702 * save type to validate intersecting paths 13703 */ 13704 *prev_src_type = src_reg_type; 13705 13706 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 13707 /* ABuser program is trying to use the same insn 13708 * dst_reg = *(u32*) (src_reg + off) 13709 * with different pointer types: 13710 * src_reg == ctx in one branch and 13711 * src_reg == stack|map in some other branch. 13712 * Reject it. 13713 */ 13714 verbose(env, "same insn cannot be used with different pointers\n"); 13715 return -EINVAL; 13716 } 13717 13718 } else if (class == BPF_STX) { 13719 enum bpf_reg_type *prev_dst_type, dst_reg_type; 13720 13721 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 13722 err = check_atomic(env, env->insn_idx, insn); 13723 if (err) 13724 return err; 13725 env->insn_idx++; 13726 continue; 13727 } 13728 13729 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 13730 verbose(env, "BPF_STX uses reserved fields\n"); 13731 return -EINVAL; 13732 } 13733 13734 /* check src1 operand */ 13735 err = check_reg_arg(env, insn->src_reg, SRC_OP); 13736 if (err) 13737 return err; 13738 /* check src2 operand */ 13739 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 13740 if (err) 13741 return err; 13742 13743 dst_reg_type = regs[insn->dst_reg].type; 13744 13745 /* check that memory (dst_reg + off) is writeable */ 13746 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 13747 insn->off, BPF_SIZE(insn->code), 13748 BPF_WRITE, insn->src_reg, false); 13749 if (err) 13750 return err; 13751 13752 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 13753 13754 if (*prev_dst_type == NOT_INIT) { 13755 *prev_dst_type = dst_reg_type; 13756 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 13757 verbose(env, "same insn cannot be used with different pointers\n"); 13758 return -EINVAL; 13759 } 13760 13761 } else if (class == BPF_ST) { 13762 if (BPF_MODE(insn->code) != BPF_MEM || 13763 insn->src_reg != BPF_REG_0) { 13764 verbose(env, "BPF_ST uses reserved fields\n"); 13765 return -EINVAL; 13766 } 13767 /* check src operand */ 13768 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 13769 if (err) 13770 return err; 13771 13772 if (is_ctx_reg(env, insn->dst_reg)) { 13773 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 13774 insn->dst_reg, 13775 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 13776 return -EACCES; 13777 } 13778 13779 /* check that memory (dst_reg + off) is writeable */ 13780 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 13781 insn->off, BPF_SIZE(insn->code), 13782 BPF_WRITE, -1, false); 13783 if (err) 13784 return err; 13785 13786 } else if (class == BPF_JMP || class == BPF_JMP32) { 13787 u8 opcode = BPF_OP(insn->code); 13788 13789 env->jmps_processed++; 13790 if (opcode == BPF_CALL) { 13791 if (BPF_SRC(insn->code) != BPF_K || 13792 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 13793 && insn->off != 0) || 13794 (insn->src_reg != BPF_REG_0 && 13795 insn->src_reg != BPF_PSEUDO_CALL && 13796 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 13797 insn->dst_reg != BPF_REG_0 || 13798 class == BPF_JMP32) { 13799 verbose(env, "BPF_CALL uses reserved fields\n"); 13800 return -EINVAL; 13801 } 13802 13803 if (env->cur_state->active_lock.ptr) { 13804 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) || 13805 (insn->src_reg == BPF_PSEUDO_CALL) || 13806 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && 13807 (insn->off != 0 || !is_bpf_list_api_kfunc(insn->imm)))) { 13808 verbose(env, "function calls are not allowed while holding a lock\n"); 13809 return -EINVAL; 13810 } 13811 } 13812 if (insn->src_reg == BPF_PSEUDO_CALL) 13813 err = check_func_call(env, insn, &env->insn_idx); 13814 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 13815 err = check_kfunc_call(env, insn, &env->insn_idx); 13816 else 13817 err = check_helper_call(env, insn, &env->insn_idx); 13818 if (err) 13819 return err; 13820 } else if (opcode == BPF_JA) { 13821 if (BPF_SRC(insn->code) != BPF_K || 13822 insn->imm != 0 || 13823 insn->src_reg != BPF_REG_0 || 13824 insn->dst_reg != BPF_REG_0 || 13825 class == BPF_JMP32) { 13826 verbose(env, "BPF_JA uses reserved fields\n"); 13827 return -EINVAL; 13828 } 13829 13830 env->insn_idx += insn->off + 1; 13831 continue; 13832 13833 } else if (opcode == BPF_EXIT) { 13834 if (BPF_SRC(insn->code) != BPF_K || 13835 insn->imm != 0 || 13836 insn->src_reg != BPF_REG_0 || 13837 insn->dst_reg != BPF_REG_0 || 13838 class == BPF_JMP32) { 13839 verbose(env, "BPF_EXIT uses reserved fields\n"); 13840 return -EINVAL; 13841 } 13842 13843 if (env->cur_state->active_lock.ptr) { 13844 verbose(env, "bpf_spin_unlock is missing\n"); 13845 return -EINVAL; 13846 } 13847 13848 if (env->cur_state->active_rcu_lock) { 13849 verbose(env, "bpf_rcu_read_unlock is missing\n"); 13850 return -EINVAL; 13851 } 13852 13853 /* We must do check_reference_leak here before 13854 * prepare_func_exit to handle the case when 13855 * state->curframe > 0, it may be a callback 13856 * function, for which reference_state must 13857 * match caller reference state when it exits. 13858 */ 13859 err = check_reference_leak(env); 13860 if (err) 13861 return err; 13862 13863 if (state->curframe) { 13864 /* exit from nested function */ 13865 err = prepare_func_exit(env, &env->insn_idx); 13866 if (err) 13867 return err; 13868 do_print_state = true; 13869 continue; 13870 } 13871 13872 err = check_return_code(env); 13873 if (err) 13874 return err; 13875 process_bpf_exit: 13876 mark_verifier_state_scratched(env); 13877 update_branch_counts(env, env->cur_state); 13878 err = pop_stack(env, &prev_insn_idx, 13879 &env->insn_idx, pop_log); 13880 if (err < 0) { 13881 if (err != -ENOENT) 13882 return err; 13883 break; 13884 } else { 13885 do_print_state = true; 13886 continue; 13887 } 13888 } else { 13889 err = check_cond_jmp_op(env, insn, &env->insn_idx); 13890 if (err) 13891 return err; 13892 } 13893 } else if (class == BPF_LD) { 13894 u8 mode = BPF_MODE(insn->code); 13895 13896 if (mode == BPF_ABS || mode == BPF_IND) { 13897 err = check_ld_abs(env, insn); 13898 if (err) 13899 return err; 13900 13901 } else if (mode == BPF_IMM) { 13902 err = check_ld_imm(env, insn); 13903 if (err) 13904 return err; 13905 13906 env->insn_idx++; 13907 sanitize_mark_insn_seen(env); 13908 } else { 13909 verbose(env, "invalid BPF_LD mode\n"); 13910 return -EINVAL; 13911 } 13912 } else { 13913 verbose(env, "unknown insn class %d\n", class); 13914 return -EINVAL; 13915 } 13916 13917 env->insn_idx++; 13918 } 13919 13920 return 0; 13921 } 13922 13923 static int find_btf_percpu_datasec(struct btf *btf) 13924 { 13925 const struct btf_type *t; 13926 const char *tname; 13927 int i, n; 13928 13929 /* 13930 * Both vmlinux and module each have their own ".data..percpu" 13931 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 13932 * types to look at only module's own BTF types. 13933 */ 13934 n = btf_nr_types(btf); 13935 if (btf_is_module(btf)) 13936 i = btf_nr_types(btf_vmlinux); 13937 else 13938 i = 1; 13939 13940 for(; i < n; i++) { 13941 t = btf_type_by_id(btf, i); 13942 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 13943 continue; 13944 13945 tname = btf_name_by_offset(btf, t->name_off); 13946 if (!strcmp(tname, ".data..percpu")) 13947 return i; 13948 } 13949 13950 return -ENOENT; 13951 } 13952 13953 /* replace pseudo btf_id with kernel symbol address */ 13954 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 13955 struct bpf_insn *insn, 13956 struct bpf_insn_aux_data *aux) 13957 { 13958 const struct btf_var_secinfo *vsi; 13959 const struct btf_type *datasec; 13960 struct btf_mod_pair *btf_mod; 13961 const struct btf_type *t; 13962 const char *sym_name; 13963 bool percpu = false; 13964 u32 type, id = insn->imm; 13965 struct btf *btf; 13966 s32 datasec_id; 13967 u64 addr; 13968 int i, btf_fd, err; 13969 13970 btf_fd = insn[1].imm; 13971 if (btf_fd) { 13972 btf = btf_get_by_fd(btf_fd); 13973 if (IS_ERR(btf)) { 13974 verbose(env, "invalid module BTF object FD specified.\n"); 13975 return -EINVAL; 13976 } 13977 } else { 13978 if (!btf_vmlinux) { 13979 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 13980 return -EINVAL; 13981 } 13982 btf = btf_vmlinux; 13983 btf_get(btf); 13984 } 13985 13986 t = btf_type_by_id(btf, id); 13987 if (!t) { 13988 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 13989 err = -ENOENT; 13990 goto err_put; 13991 } 13992 13993 if (!btf_type_is_var(t)) { 13994 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 13995 err = -EINVAL; 13996 goto err_put; 13997 } 13998 13999 sym_name = btf_name_by_offset(btf, t->name_off); 14000 addr = kallsyms_lookup_name(sym_name); 14001 if (!addr) { 14002 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 14003 sym_name); 14004 err = -ENOENT; 14005 goto err_put; 14006 } 14007 14008 datasec_id = find_btf_percpu_datasec(btf); 14009 if (datasec_id > 0) { 14010 datasec = btf_type_by_id(btf, datasec_id); 14011 for_each_vsi(i, datasec, vsi) { 14012 if (vsi->type == id) { 14013 percpu = true; 14014 break; 14015 } 14016 } 14017 } 14018 14019 insn[0].imm = (u32)addr; 14020 insn[1].imm = addr >> 32; 14021 14022 type = t->type; 14023 t = btf_type_skip_modifiers(btf, type, NULL); 14024 if (percpu) { 14025 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 14026 aux->btf_var.btf = btf; 14027 aux->btf_var.btf_id = type; 14028 } else if (!btf_type_is_struct(t)) { 14029 const struct btf_type *ret; 14030 const char *tname; 14031 u32 tsize; 14032 14033 /* resolve the type size of ksym. */ 14034 ret = btf_resolve_size(btf, t, &tsize); 14035 if (IS_ERR(ret)) { 14036 tname = btf_name_by_offset(btf, t->name_off); 14037 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 14038 tname, PTR_ERR(ret)); 14039 err = -EINVAL; 14040 goto err_put; 14041 } 14042 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 14043 aux->btf_var.mem_size = tsize; 14044 } else { 14045 aux->btf_var.reg_type = PTR_TO_BTF_ID; 14046 aux->btf_var.btf = btf; 14047 aux->btf_var.btf_id = type; 14048 } 14049 14050 /* check whether we recorded this BTF (and maybe module) already */ 14051 for (i = 0; i < env->used_btf_cnt; i++) { 14052 if (env->used_btfs[i].btf == btf) { 14053 btf_put(btf); 14054 return 0; 14055 } 14056 } 14057 14058 if (env->used_btf_cnt >= MAX_USED_BTFS) { 14059 err = -E2BIG; 14060 goto err_put; 14061 } 14062 14063 btf_mod = &env->used_btfs[env->used_btf_cnt]; 14064 btf_mod->btf = btf; 14065 btf_mod->module = NULL; 14066 14067 /* if we reference variables from kernel module, bump its refcount */ 14068 if (btf_is_module(btf)) { 14069 btf_mod->module = btf_try_get_module(btf); 14070 if (!btf_mod->module) { 14071 err = -ENXIO; 14072 goto err_put; 14073 } 14074 } 14075 14076 env->used_btf_cnt++; 14077 14078 return 0; 14079 err_put: 14080 btf_put(btf); 14081 return err; 14082 } 14083 14084 static bool is_tracing_prog_type(enum bpf_prog_type type) 14085 { 14086 switch (type) { 14087 case BPF_PROG_TYPE_KPROBE: 14088 case BPF_PROG_TYPE_TRACEPOINT: 14089 case BPF_PROG_TYPE_PERF_EVENT: 14090 case BPF_PROG_TYPE_RAW_TRACEPOINT: 14091 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: 14092 return true; 14093 default: 14094 return false; 14095 } 14096 } 14097 14098 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 14099 struct bpf_map *map, 14100 struct bpf_prog *prog) 14101 14102 { 14103 enum bpf_prog_type prog_type = resolve_prog_type(prog); 14104 14105 if (btf_record_has_field(map->record, BPF_LIST_HEAD)) { 14106 if (is_tracing_prog_type(prog_type)) { 14107 verbose(env, "tracing progs cannot use bpf_list_head yet\n"); 14108 return -EINVAL; 14109 } 14110 } 14111 14112 if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { 14113 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 14114 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 14115 return -EINVAL; 14116 } 14117 14118 if (is_tracing_prog_type(prog_type)) { 14119 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 14120 return -EINVAL; 14121 } 14122 14123 if (prog->aux->sleepable) { 14124 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 14125 return -EINVAL; 14126 } 14127 } 14128 14129 if (btf_record_has_field(map->record, BPF_TIMER)) { 14130 if (is_tracing_prog_type(prog_type)) { 14131 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 14132 return -EINVAL; 14133 } 14134 } 14135 14136 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 14137 !bpf_offload_prog_map_match(prog, map)) { 14138 verbose(env, "offload device mismatch between prog and map\n"); 14139 return -EINVAL; 14140 } 14141 14142 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 14143 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 14144 return -EINVAL; 14145 } 14146 14147 if (prog->aux->sleepable) 14148 switch (map->map_type) { 14149 case BPF_MAP_TYPE_HASH: 14150 case BPF_MAP_TYPE_LRU_HASH: 14151 case BPF_MAP_TYPE_ARRAY: 14152 case BPF_MAP_TYPE_PERCPU_HASH: 14153 case BPF_MAP_TYPE_PERCPU_ARRAY: 14154 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 14155 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 14156 case BPF_MAP_TYPE_HASH_OF_MAPS: 14157 case BPF_MAP_TYPE_RINGBUF: 14158 case BPF_MAP_TYPE_USER_RINGBUF: 14159 case BPF_MAP_TYPE_INODE_STORAGE: 14160 case BPF_MAP_TYPE_SK_STORAGE: 14161 case BPF_MAP_TYPE_TASK_STORAGE: 14162 case BPF_MAP_TYPE_CGRP_STORAGE: 14163 break; 14164 default: 14165 verbose(env, 14166 "Sleepable programs can only use array, hash, ringbuf and local storage maps\n"); 14167 return -EINVAL; 14168 } 14169 14170 return 0; 14171 } 14172 14173 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 14174 { 14175 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 14176 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 14177 } 14178 14179 /* find and rewrite pseudo imm in ld_imm64 instructions: 14180 * 14181 * 1. if it accesses map FD, replace it with actual map pointer. 14182 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 14183 * 14184 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 14185 */ 14186 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 14187 { 14188 struct bpf_insn *insn = env->prog->insnsi; 14189 int insn_cnt = env->prog->len; 14190 int i, j, err; 14191 14192 err = bpf_prog_calc_tag(env->prog); 14193 if (err) 14194 return err; 14195 14196 for (i = 0; i < insn_cnt; i++, insn++) { 14197 if (BPF_CLASS(insn->code) == BPF_LDX && 14198 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 14199 verbose(env, "BPF_LDX uses reserved fields\n"); 14200 return -EINVAL; 14201 } 14202 14203 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 14204 struct bpf_insn_aux_data *aux; 14205 struct bpf_map *map; 14206 struct fd f; 14207 u64 addr; 14208 u32 fd; 14209 14210 if (i == insn_cnt - 1 || insn[1].code != 0 || 14211 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 14212 insn[1].off != 0) { 14213 verbose(env, "invalid bpf_ld_imm64 insn\n"); 14214 return -EINVAL; 14215 } 14216 14217 if (insn[0].src_reg == 0) 14218 /* valid generic load 64-bit imm */ 14219 goto next_insn; 14220 14221 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 14222 aux = &env->insn_aux_data[i]; 14223 err = check_pseudo_btf_id(env, insn, aux); 14224 if (err) 14225 return err; 14226 goto next_insn; 14227 } 14228 14229 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 14230 aux = &env->insn_aux_data[i]; 14231 aux->ptr_type = PTR_TO_FUNC; 14232 goto next_insn; 14233 } 14234 14235 /* In final convert_pseudo_ld_imm64() step, this is 14236 * converted into regular 64-bit imm load insn. 14237 */ 14238 switch (insn[0].src_reg) { 14239 case BPF_PSEUDO_MAP_VALUE: 14240 case BPF_PSEUDO_MAP_IDX_VALUE: 14241 break; 14242 case BPF_PSEUDO_MAP_FD: 14243 case BPF_PSEUDO_MAP_IDX: 14244 if (insn[1].imm == 0) 14245 break; 14246 fallthrough; 14247 default: 14248 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 14249 return -EINVAL; 14250 } 14251 14252 switch (insn[0].src_reg) { 14253 case BPF_PSEUDO_MAP_IDX_VALUE: 14254 case BPF_PSEUDO_MAP_IDX: 14255 if (bpfptr_is_null(env->fd_array)) { 14256 verbose(env, "fd_idx without fd_array is invalid\n"); 14257 return -EPROTO; 14258 } 14259 if (copy_from_bpfptr_offset(&fd, env->fd_array, 14260 insn[0].imm * sizeof(fd), 14261 sizeof(fd))) 14262 return -EFAULT; 14263 break; 14264 default: 14265 fd = insn[0].imm; 14266 break; 14267 } 14268 14269 f = fdget(fd); 14270 map = __bpf_map_get(f); 14271 if (IS_ERR(map)) { 14272 verbose(env, "fd %d is not pointing to valid bpf_map\n", 14273 insn[0].imm); 14274 return PTR_ERR(map); 14275 } 14276 14277 err = check_map_prog_compatibility(env, map, env->prog); 14278 if (err) { 14279 fdput(f); 14280 return err; 14281 } 14282 14283 aux = &env->insn_aux_data[i]; 14284 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 14285 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 14286 addr = (unsigned long)map; 14287 } else { 14288 u32 off = insn[1].imm; 14289 14290 if (off >= BPF_MAX_VAR_OFF) { 14291 verbose(env, "direct value offset of %u is not allowed\n", off); 14292 fdput(f); 14293 return -EINVAL; 14294 } 14295 14296 if (!map->ops->map_direct_value_addr) { 14297 verbose(env, "no direct value access support for this map type\n"); 14298 fdput(f); 14299 return -EINVAL; 14300 } 14301 14302 err = map->ops->map_direct_value_addr(map, &addr, off); 14303 if (err) { 14304 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 14305 map->value_size, off); 14306 fdput(f); 14307 return err; 14308 } 14309 14310 aux->map_off = off; 14311 addr += off; 14312 } 14313 14314 insn[0].imm = (u32)addr; 14315 insn[1].imm = addr >> 32; 14316 14317 /* check whether we recorded this map already */ 14318 for (j = 0; j < env->used_map_cnt; j++) { 14319 if (env->used_maps[j] == map) { 14320 aux->map_index = j; 14321 fdput(f); 14322 goto next_insn; 14323 } 14324 } 14325 14326 if (env->used_map_cnt >= MAX_USED_MAPS) { 14327 fdput(f); 14328 return -E2BIG; 14329 } 14330 14331 /* hold the map. If the program is rejected by verifier, 14332 * the map will be released by release_maps() or it 14333 * will be used by the valid program until it's unloaded 14334 * and all maps are released in free_used_maps() 14335 */ 14336 bpf_map_inc(map); 14337 14338 aux->map_index = env->used_map_cnt; 14339 env->used_maps[env->used_map_cnt++] = map; 14340 14341 if (bpf_map_is_cgroup_storage(map) && 14342 bpf_cgroup_storage_assign(env->prog->aux, map)) { 14343 verbose(env, "only one cgroup storage of each type is allowed\n"); 14344 fdput(f); 14345 return -EBUSY; 14346 } 14347 14348 fdput(f); 14349 next_insn: 14350 insn++; 14351 i++; 14352 continue; 14353 } 14354 14355 /* Basic sanity check before we invest more work here. */ 14356 if (!bpf_opcode_in_insntable(insn->code)) { 14357 verbose(env, "unknown opcode %02x\n", insn->code); 14358 return -EINVAL; 14359 } 14360 } 14361 14362 /* now all pseudo BPF_LD_IMM64 instructions load valid 14363 * 'struct bpf_map *' into a register instead of user map_fd. 14364 * These pointers will be used later by verifier to validate map access. 14365 */ 14366 return 0; 14367 } 14368 14369 /* drop refcnt of maps used by the rejected program */ 14370 static void release_maps(struct bpf_verifier_env *env) 14371 { 14372 __bpf_free_used_maps(env->prog->aux, env->used_maps, 14373 env->used_map_cnt); 14374 } 14375 14376 /* drop refcnt of maps used by the rejected program */ 14377 static void release_btfs(struct bpf_verifier_env *env) 14378 { 14379 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 14380 env->used_btf_cnt); 14381 } 14382 14383 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 14384 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 14385 { 14386 struct bpf_insn *insn = env->prog->insnsi; 14387 int insn_cnt = env->prog->len; 14388 int i; 14389 14390 for (i = 0; i < insn_cnt; i++, insn++) { 14391 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 14392 continue; 14393 if (insn->src_reg == BPF_PSEUDO_FUNC) 14394 continue; 14395 insn->src_reg = 0; 14396 } 14397 } 14398 14399 /* single env->prog->insni[off] instruction was replaced with the range 14400 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 14401 * [0, off) and [off, end) to new locations, so the patched range stays zero 14402 */ 14403 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 14404 struct bpf_insn_aux_data *new_data, 14405 struct bpf_prog *new_prog, u32 off, u32 cnt) 14406 { 14407 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 14408 struct bpf_insn *insn = new_prog->insnsi; 14409 u32 old_seen = old_data[off].seen; 14410 u32 prog_len; 14411 int i; 14412 14413 /* aux info at OFF always needs adjustment, no matter fast path 14414 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 14415 * original insn at old prog. 14416 */ 14417 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 14418 14419 if (cnt == 1) 14420 return; 14421 prog_len = new_prog->len; 14422 14423 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 14424 memcpy(new_data + off + cnt - 1, old_data + off, 14425 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 14426 for (i = off; i < off + cnt - 1; i++) { 14427 /* Expand insni[off]'s seen count to the patched range. */ 14428 new_data[i].seen = old_seen; 14429 new_data[i].zext_dst = insn_has_def32(env, insn + i); 14430 } 14431 env->insn_aux_data = new_data; 14432 vfree(old_data); 14433 } 14434 14435 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 14436 { 14437 int i; 14438 14439 if (len == 1) 14440 return; 14441 /* NOTE: fake 'exit' subprog should be updated as well. */ 14442 for (i = 0; i <= env->subprog_cnt; i++) { 14443 if (env->subprog_info[i].start <= off) 14444 continue; 14445 env->subprog_info[i].start += len - 1; 14446 } 14447 } 14448 14449 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 14450 { 14451 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 14452 int i, sz = prog->aux->size_poke_tab; 14453 struct bpf_jit_poke_descriptor *desc; 14454 14455 for (i = 0; i < sz; i++) { 14456 desc = &tab[i]; 14457 if (desc->insn_idx <= off) 14458 continue; 14459 desc->insn_idx += len - 1; 14460 } 14461 } 14462 14463 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 14464 const struct bpf_insn *patch, u32 len) 14465 { 14466 struct bpf_prog *new_prog; 14467 struct bpf_insn_aux_data *new_data = NULL; 14468 14469 if (len > 1) { 14470 new_data = vzalloc(array_size(env->prog->len + len - 1, 14471 sizeof(struct bpf_insn_aux_data))); 14472 if (!new_data) 14473 return NULL; 14474 } 14475 14476 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 14477 if (IS_ERR(new_prog)) { 14478 if (PTR_ERR(new_prog) == -ERANGE) 14479 verbose(env, 14480 "insn %d cannot be patched due to 16-bit range\n", 14481 env->insn_aux_data[off].orig_idx); 14482 vfree(new_data); 14483 return NULL; 14484 } 14485 adjust_insn_aux_data(env, new_data, new_prog, off, len); 14486 adjust_subprog_starts(env, off, len); 14487 adjust_poke_descs(new_prog, off, len); 14488 return new_prog; 14489 } 14490 14491 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 14492 u32 off, u32 cnt) 14493 { 14494 int i, j; 14495 14496 /* find first prog starting at or after off (first to remove) */ 14497 for (i = 0; i < env->subprog_cnt; i++) 14498 if (env->subprog_info[i].start >= off) 14499 break; 14500 /* find first prog starting at or after off + cnt (first to stay) */ 14501 for (j = i; j < env->subprog_cnt; j++) 14502 if (env->subprog_info[j].start >= off + cnt) 14503 break; 14504 /* if j doesn't start exactly at off + cnt, we are just removing 14505 * the front of previous prog 14506 */ 14507 if (env->subprog_info[j].start != off + cnt) 14508 j--; 14509 14510 if (j > i) { 14511 struct bpf_prog_aux *aux = env->prog->aux; 14512 int move; 14513 14514 /* move fake 'exit' subprog as well */ 14515 move = env->subprog_cnt + 1 - j; 14516 14517 memmove(env->subprog_info + i, 14518 env->subprog_info + j, 14519 sizeof(*env->subprog_info) * move); 14520 env->subprog_cnt -= j - i; 14521 14522 /* remove func_info */ 14523 if (aux->func_info) { 14524 move = aux->func_info_cnt - j; 14525 14526 memmove(aux->func_info + i, 14527 aux->func_info + j, 14528 sizeof(*aux->func_info) * move); 14529 aux->func_info_cnt -= j - i; 14530 /* func_info->insn_off is set after all code rewrites, 14531 * in adjust_btf_func() - no need to adjust 14532 */ 14533 } 14534 } else { 14535 /* convert i from "first prog to remove" to "first to adjust" */ 14536 if (env->subprog_info[i].start == off) 14537 i++; 14538 } 14539 14540 /* update fake 'exit' subprog as well */ 14541 for (; i <= env->subprog_cnt; i++) 14542 env->subprog_info[i].start -= cnt; 14543 14544 return 0; 14545 } 14546 14547 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 14548 u32 cnt) 14549 { 14550 struct bpf_prog *prog = env->prog; 14551 u32 i, l_off, l_cnt, nr_linfo; 14552 struct bpf_line_info *linfo; 14553 14554 nr_linfo = prog->aux->nr_linfo; 14555 if (!nr_linfo) 14556 return 0; 14557 14558 linfo = prog->aux->linfo; 14559 14560 /* find first line info to remove, count lines to be removed */ 14561 for (i = 0; i < nr_linfo; i++) 14562 if (linfo[i].insn_off >= off) 14563 break; 14564 14565 l_off = i; 14566 l_cnt = 0; 14567 for (; i < nr_linfo; i++) 14568 if (linfo[i].insn_off < off + cnt) 14569 l_cnt++; 14570 else 14571 break; 14572 14573 /* First live insn doesn't match first live linfo, it needs to "inherit" 14574 * last removed linfo. prog is already modified, so prog->len == off 14575 * means no live instructions after (tail of the program was removed). 14576 */ 14577 if (prog->len != off && l_cnt && 14578 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 14579 l_cnt--; 14580 linfo[--i].insn_off = off + cnt; 14581 } 14582 14583 /* remove the line info which refer to the removed instructions */ 14584 if (l_cnt) { 14585 memmove(linfo + l_off, linfo + i, 14586 sizeof(*linfo) * (nr_linfo - i)); 14587 14588 prog->aux->nr_linfo -= l_cnt; 14589 nr_linfo = prog->aux->nr_linfo; 14590 } 14591 14592 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 14593 for (i = l_off; i < nr_linfo; i++) 14594 linfo[i].insn_off -= cnt; 14595 14596 /* fix up all subprogs (incl. 'exit') which start >= off */ 14597 for (i = 0; i <= env->subprog_cnt; i++) 14598 if (env->subprog_info[i].linfo_idx > l_off) { 14599 /* program may have started in the removed region but 14600 * may not be fully removed 14601 */ 14602 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 14603 env->subprog_info[i].linfo_idx -= l_cnt; 14604 else 14605 env->subprog_info[i].linfo_idx = l_off; 14606 } 14607 14608 return 0; 14609 } 14610 14611 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 14612 { 14613 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14614 unsigned int orig_prog_len = env->prog->len; 14615 int err; 14616 14617 if (bpf_prog_is_dev_bound(env->prog->aux)) 14618 bpf_prog_offload_remove_insns(env, off, cnt); 14619 14620 err = bpf_remove_insns(env->prog, off, cnt); 14621 if (err) 14622 return err; 14623 14624 err = adjust_subprog_starts_after_remove(env, off, cnt); 14625 if (err) 14626 return err; 14627 14628 err = bpf_adj_linfo_after_remove(env, off, cnt); 14629 if (err) 14630 return err; 14631 14632 memmove(aux_data + off, aux_data + off + cnt, 14633 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 14634 14635 return 0; 14636 } 14637 14638 /* The verifier does more data flow analysis than llvm and will not 14639 * explore branches that are dead at run time. Malicious programs can 14640 * have dead code too. Therefore replace all dead at-run-time code 14641 * with 'ja -1'. 14642 * 14643 * Just nops are not optimal, e.g. if they would sit at the end of the 14644 * program and through another bug we would manage to jump there, then 14645 * we'd execute beyond program memory otherwise. Returning exception 14646 * code also wouldn't work since we can have subprogs where the dead 14647 * code could be located. 14648 */ 14649 static void sanitize_dead_code(struct bpf_verifier_env *env) 14650 { 14651 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14652 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 14653 struct bpf_insn *insn = env->prog->insnsi; 14654 const int insn_cnt = env->prog->len; 14655 int i; 14656 14657 for (i = 0; i < insn_cnt; i++) { 14658 if (aux_data[i].seen) 14659 continue; 14660 memcpy(insn + i, &trap, sizeof(trap)); 14661 aux_data[i].zext_dst = false; 14662 } 14663 } 14664 14665 static bool insn_is_cond_jump(u8 code) 14666 { 14667 u8 op; 14668 14669 if (BPF_CLASS(code) == BPF_JMP32) 14670 return true; 14671 14672 if (BPF_CLASS(code) != BPF_JMP) 14673 return false; 14674 14675 op = BPF_OP(code); 14676 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 14677 } 14678 14679 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 14680 { 14681 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14682 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 14683 struct bpf_insn *insn = env->prog->insnsi; 14684 const int insn_cnt = env->prog->len; 14685 int i; 14686 14687 for (i = 0; i < insn_cnt; i++, insn++) { 14688 if (!insn_is_cond_jump(insn->code)) 14689 continue; 14690 14691 if (!aux_data[i + 1].seen) 14692 ja.off = insn->off; 14693 else if (!aux_data[i + 1 + insn->off].seen) 14694 ja.off = 0; 14695 else 14696 continue; 14697 14698 if (bpf_prog_is_dev_bound(env->prog->aux)) 14699 bpf_prog_offload_replace_insn(env, i, &ja); 14700 14701 memcpy(insn, &ja, sizeof(ja)); 14702 } 14703 } 14704 14705 static int opt_remove_dead_code(struct bpf_verifier_env *env) 14706 { 14707 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 14708 int insn_cnt = env->prog->len; 14709 int i, err; 14710 14711 for (i = 0; i < insn_cnt; i++) { 14712 int j; 14713 14714 j = 0; 14715 while (i + j < insn_cnt && !aux_data[i + j].seen) 14716 j++; 14717 if (!j) 14718 continue; 14719 14720 err = verifier_remove_insns(env, i, j); 14721 if (err) 14722 return err; 14723 insn_cnt = env->prog->len; 14724 } 14725 14726 return 0; 14727 } 14728 14729 static int opt_remove_nops(struct bpf_verifier_env *env) 14730 { 14731 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 14732 struct bpf_insn *insn = env->prog->insnsi; 14733 int insn_cnt = env->prog->len; 14734 int i, err; 14735 14736 for (i = 0; i < insn_cnt; i++) { 14737 if (memcmp(&insn[i], &ja, sizeof(ja))) 14738 continue; 14739 14740 err = verifier_remove_insns(env, i, 1); 14741 if (err) 14742 return err; 14743 insn_cnt--; 14744 i--; 14745 } 14746 14747 return 0; 14748 } 14749 14750 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 14751 const union bpf_attr *attr) 14752 { 14753 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 14754 struct bpf_insn_aux_data *aux = env->insn_aux_data; 14755 int i, patch_len, delta = 0, len = env->prog->len; 14756 struct bpf_insn *insns = env->prog->insnsi; 14757 struct bpf_prog *new_prog; 14758 bool rnd_hi32; 14759 14760 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 14761 zext_patch[1] = BPF_ZEXT_REG(0); 14762 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 14763 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 14764 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 14765 for (i = 0; i < len; i++) { 14766 int adj_idx = i + delta; 14767 struct bpf_insn insn; 14768 int load_reg; 14769 14770 insn = insns[adj_idx]; 14771 load_reg = insn_def_regno(&insn); 14772 if (!aux[adj_idx].zext_dst) { 14773 u8 code, class; 14774 u32 imm_rnd; 14775 14776 if (!rnd_hi32) 14777 continue; 14778 14779 code = insn.code; 14780 class = BPF_CLASS(code); 14781 if (load_reg == -1) 14782 continue; 14783 14784 /* NOTE: arg "reg" (the fourth one) is only used for 14785 * BPF_STX + SRC_OP, so it is safe to pass NULL 14786 * here. 14787 */ 14788 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 14789 if (class == BPF_LD && 14790 BPF_MODE(code) == BPF_IMM) 14791 i++; 14792 continue; 14793 } 14794 14795 /* ctx load could be transformed into wider load. */ 14796 if (class == BPF_LDX && 14797 aux[adj_idx].ptr_type == PTR_TO_CTX) 14798 continue; 14799 14800 imm_rnd = get_random_u32(); 14801 rnd_hi32_patch[0] = insn; 14802 rnd_hi32_patch[1].imm = imm_rnd; 14803 rnd_hi32_patch[3].dst_reg = load_reg; 14804 patch = rnd_hi32_patch; 14805 patch_len = 4; 14806 goto apply_patch_buffer; 14807 } 14808 14809 /* Add in an zero-extend instruction if a) the JIT has requested 14810 * it or b) it's a CMPXCHG. 14811 * 14812 * The latter is because: BPF_CMPXCHG always loads a value into 14813 * R0, therefore always zero-extends. However some archs' 14814 * equivalent instruction only does this load when the 14815 * comparison is successful. This detail of CMPXCHG is 14816 * orthogonal to the general zero-extension behaviour of the 14817 * CPU, so it's treated independently of bpf_jit_needs_zext. 14818 */ 14819 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 14820 continue; 14821 14822 if (WARN_ON(load_reg == -1)) { 14823 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 14824 return -EFAULT; 14825 } 14826 14827 zext_patch[0] = insn; 14828 zext_patch[1].dst_reg = load_reg; 14829 zext_patch[1].src_reg = load_reg; 14830 patch = zext_patch; 14831 patch_len = 2; 14832 apply_patch_buffer: 14833 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 14834 if (!new_prog) 14835 return -ENOMEM; 14836 env->prog = new_prog; 14837 insns = new_prog->insnsi; 14838 aux = env->insn_aux_data; 14839 delta += patch_len - 1; 14840 } 14841 14842 return 0; 14843 } 14844 14845 /* convert load instructions that access fields of a context type into a 14846 * sequence of instructions that access fields of the underlying structure: 14847 * struct __sk_buff -> struct sk_buff 14848 * struct bpf_sock_ops -> struct sock 14849 */ 14850 static int convert_ctx_accesses(struct bpf_verifier_env *env) 14851 { 14852 const struct bpf_verifier_ops *ops = env->ops; 14853 int i, cnt, size, ctx_field_size, delta = 0; 14854 const int insn_cnt = env->prog->len; 14855 struct bpf_insn insn_buf[16], *insn; 14856 u32 target_size, size_default, off; 14857 struct bpf_prog *new_prog; 14858 enum bpf_access_type type; 14859 bool is_narrower_load; 14860 14861 if (ops->gen_prologue || env->seen_direct_write) { 14862 if (!ops->gen_prologue) { 14863 verbose(env, "bpf verifier is misconfigured\n"); 14864 return -EINVAL; 14865 } 14866 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 14867 env->prog); 14868 if (cnt >= ARRAY_SIZE(insn_buf)) { 14869 verbose(env, "bpf verifier is misconfigured\n"); 14870 return -EINVAL; 14871 } else if (cnt) { 14872 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 14873 if (!new_prog) 14874 return -ENOMEM; 14875 14876 env->prog = new_prog; 14877 delta += cnt - 1; 14878 } 14879 } 14880 14881 if (bpf_prog_is_dev_bound(env->prog->aux)) 14882 return 0; 14883 14884 insn = env->prog->insnsi + delta; 14885 14886 for (i = 0; i < insn_cnt; i++, insn++) { 14887 bpf_convert_ctx_access_t convert_ctx_access; 14888 bool ctx_access; 14889 14890 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 14891 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 14892 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 14893 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 14894 type = BPF_READ; 14895 ctx_access = true; 14896 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 14897 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 14898 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 14899 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 14900 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 14901 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 14902 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 14903 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 14904 type = BPF_WRITE; 14905 ctx_access = BPF_CLASS(insn->code) == BPF_STX; 14906 } else { 14907 continue; 14908 } 14909 14910 if (type == BPF_WRITE && 14911 env->insn_aux_data[i + delta].sanitize_stack_spill) { 14912 struct bpf_insn patch[] = { 14913 *insn, 14914 BPF_ST_NOSPEC(), 14915 }; 14916 14917 cnt = ARRAY_SIZE(patch); 14918 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 14919 if (!new_prog) 14920 return -ENOMEM; 14921 14922 delta += cnt - 1; 14923 env->prog = new_prog; 14924 insn = new_prog->insnsi + i + delta; 14925 continue; 14926 } 14927 14928 if (!ctx_access) 14929 continue; 14930 14931 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 14932 case PTR_TO_CTX: 14933 if (!ops->convert_ctx_access) 14934 continue; 14935 convert_ctx_access = ops->convert_ctx_access; 14936 break; 14937 case PTR_TO_SOCKET: 14938 case PTR_TO_SOCK_COMMON: 14939 convert_ctx_access = bpf_sock_convert_ctx_access; 14940 break; 14941 case PTR_TO_TCP_SOCK: 14942 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 14943 break; 14944 case PTR_TO_XDP_SOCK: 14945 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 14946 break; 14947 case PTR_TO_BTF_ID: 14948 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 14949 /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike 14950 * PTR_TO_BTF_ID, and an active ref_obj_id, but the same cannot 14951 * be said once it is marked PTR_UNTRUSTED, hence we must handle 14952 * any faults for loads into such types. BPF_WRITE is disallowed 14953 * for this case. 14954 */ 14955 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: 14956 if (type == BPF_READ) { 14957 insn->code = BPF_LDX | BPF_PROBE_MEM | 14958 BPF_SIZE((insn)->code); 14959 env->prog->aux->num_exentries++; 14960 } 14961 continue; 14962 default: 14963 continue; 14964 } 14965 14966 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 14967 size = BPF_LDST_BYTES(insn); 14968 14969 /* If the read access is a narrower load of the field, 14970 * convert to a 4/8-byte load, to minimum program type specific 14971 * convert_ctx_access changes. If conversion is successful, 14972 * we will apply proper mask to the result. 14973 */ 14974 is_narrower_load = size < ctx_field_size; 14975 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 14976 off = insn->off; 14977 if (is_narrower_load) { 14978 u8 size_code; 14979 14980 if (type == BPF_WRITE) { 14981 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 14982 return -EINVAL; 14983 } 14984 14985 size_code = BPF_H; 14986 if (ctx_field_size == 4) 14987 size_code = BPF_W; 14988 else if (ctx_field_size == 8) 14989 size_code = BPF_DW; 14990 14991 insn->off = off & ~(size_default - 1); 14992 insn->code = BPF_LDX | BPF_MEM | size_code; 14993 } 14994 14995 target_size = 0; 14996 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 14997 &target_size); 14998 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 14999 (ctx_field_size && !target_size)) { 15000 verbose(env, "bpf verifier is misconfigured\n"); 15001 return -EINVAL; 15002 } 15003 15004 if (is_narrower_load && size < target_size) { 15005 u8 shift = bpf_ctx_narrow_access_offset( 15006 off, size, size_default) * 8; 15007 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 15008 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 15009 return -EINVAL; 15010 } 15011 if (ctx_field_size <= 4) { 15012 if (shift) 15013 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 15014 insn->dst_reg, 15015 shift); 15016 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 15017 (1 << size * 8) - 1); 15018 } else { 15019 if (shift) 15020 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 15021 insn->dst_reg, 15022 shift); 15023 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 15024 (1ULL << size * 8) - 1); 15025 } 15026 } 15027 15028 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15029 if (!new_prog) 15030 return -ENOMEM; 15031 15032 delta += cnt - 1; 15033 15034 /* keep walking new program and skip insns we just inserted */ 15035 env->prog = new_prog; 15036 insn = new_prog->insnsi + i + delta; 15037 } 15038 15039 return 0; 15040 } 15041 15042 static int jit_subprogs(struct bpf_verifier_env *env) 15043 { 15044 struct bpf_prog *prog = env->prog, **func, *tmp; 15045 int i, j, subprog_start, subprog_end = 0, len, subprog; 15046 struct bpf_map *map_ptr; 15047 struct bpf_insn *insn; 15048 void *old_bpf_func; 15049 int err, num_exentries; 15050 15051 if (env->subprog_cnt <= 1) 15052 return 0; 15053 15054 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15055 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 15056 continue; 15057 15058 /* Upon error here we cannot fall back to interpreter but 15059 * need a hard reject of the program. Thus -EFAULT is 15060 * propagated in any case. 15061 */ 15062 subprog = find_subprog(env, i + insn->imm + 1); 15063 if (subprog < 0) { 15064 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 15065 i + insn->imm + 1); 15066 return -EFAULT; 15067 } 15068 /* temporarily remember subprog id inside insn instead of 15069 * aux_data, since next loop will split up all insns into funcs 15070 */ 15071 insn->off = subprog; 15072 /* remember original imm in case JIT fails and fallback 15073 * to interpreter will be needed 15074 */ 15075 env->insn_aux_data[i].call_imm = insn->imm; 15076 /* point imm to __bpf_call_base+1 from JITs point of view */ 15077 insn->imm = 1; 15078 if (bpf_pseudo_func(insn)) 15079 /* jit (e.g. x86_64) may emit fewer instructions 15080 * if it learns a u32 imm is the same as a u64 imm. 15081 * Force a non zero here. 15082 */ 15083 insn[1].imm = 1; 15084 } 15085 15086 err = bpf_prog_alloc_jited_linfo(prog); 15087 if (err) 15088 goto out_undo_insn; 15089 15090 err = -ENOMEM; 15091 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 15092 if (!func) 15093 goto out_undo_insn; 15094 15095 for (i = 0; i < env->subprog_cnt; i++) { 15096 subprog_start = subprog_end; 15097 subprog_end = env->subprog_info[i + 1].start; 15098 15099 len = subprog_end - subprog_start; 15100 /* bpf_prog_run() doesn't call subprogs directly, 15101 * hence main prog stats include the runtime of subprogs. 15102 * subprogs don't have IDs and not reachable via prog_get_next_id 15103 * func[i]->stats will never be accessed and stays NULL 15104 */ 15105 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 15106 if (!func[i]) 15107 goto out_free; 15108 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 15109 len * sizeof(struct bpf_insn)); 15110 func[i]->type = prog->type; 15111 func[i]->len = len; 15112 if (bpf_prog_calc_tag(func[i])) 15113 goto out_free; 15114 func[i]->is_func = 1; 15115 func[i]->aux->func_idx = i; 15116 /* Below members will be freed only at prog->aux */ 15117 func[i]->aux->btf = prog->aux->btf; 15118 func[i]->aux->func_info = prog->aux->func_info; 15119 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; 15120 func[i]->aux->poke_tab = prog->aux->poke_tab; 15121 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 15122 15123 for (j = 0; j < prog->aux->size_poke_tab; j++) { 15124 struct bpf_jit_poke_descriptor *poke; 15125 15126 poke = &prog->aux->poke_tab[j]; 15127 if (poke->insn_idx < subprog_end && 15128 poke->insn_idx >= subprog_start) 15129 poke->aux = func[i]->aux; 15130 } 15131 15132 func[i]->aux->name[0] = 'F'; 15133 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 15134 func[i]->jit_requested = 1; 15135 func[i]->blinding_requested = prog->blinding_requested; 15136 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 15137 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 15138 func[i]->aux->linfo = prog->aux->linfo; 15139 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 15140 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 15141 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 15142 num_exentries = 0; 15143 insn = func[i]->insnsi; 15144 for (j = 0; j < func[i]->len; j++, insn++) { 15145 if (BPF_CLASS(insn->code) == BPF_LDX && 15146 BPF_MODE(insn->code) == BPF_PROBE_MEM) 15147 num_exentries++; 15148 } 15149 func[i]->aux->num_exentries = num_exentries; 15150 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 15151 func[i] = bpf_int_jit_compile(func[i]); 15152 if (!func[i]->jited) { 15153 err = -ENOTSUPP; 15154 goto out_free; 15155 } 15156 cond_resched(); 15157 } 15158 15159 /* at this point all bpf functions were successfully JITed 15160 * now populate all bpf_calls with correct addresses and 15161 * run last pass of JIT 15162 */ 15163 for (i = 0; i < env->subprog_cnt; i++) { 15164 insn = func[i]->insnsi; 15165 for (j = 0; j < func[i]->len; j++, insn++) { 15166 if (bpf_pseudo_func(insn)) { 15167 subprog = insn->off; 15168 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 15169 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 15170 continue; 15171 } 15172 if (!bpf_pseudo_call(insn)) 15173 continue; 15174 subprog = insn->off; 15175 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 15176 } 15177 15178 /* we use the aux data to keep a list of the start addresses 15179 * of the JITed images for each function in the program 15180 * 15181 * for some architectures, such as powerpc64, the imm field 15182 * might not be large enough to hold the offset of the start 15183 * address of the callee's JITed image from __bpf_call_base 15184 * 15185 * in such cases, we can lookup the start address of a callee 15186 * by using its subprog id, available from the off field of 15187 * the call instruction, as an index for this list 15188 */ 15189 func[i]->aux->func = func; 15190 func[i]->aux->func_cnt = env->subprog_cnt; 15191 } 15192 for (i = 0; i < env->subprog_cnt; i++) { 15193 old_bpf_func = func[i]->bpf_func; 15194 tmp = bpf_int_jit_compile(func[i]); 15195 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 15196 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 15197 err = -ENOTSUPP; 15198 goto out_free; 15199 } 15200 cond_resched(); 15201 } 15202 15203 /* finally lock prog and jit images for all functions and 15204 * populate kallsysm 15205 */ 15206 for (i = 0; i < env->subprog_cnt; i++) { 15207 bpf_prog_lock_ro(func[i]); 15208 bpf_prog_kallsyms_add(func[i]); 15209 } 15210 15211 /* Last step: make now unused interpreter insns from main 15212 * prog consistent for later dump requests, so they can 15213 * later look the same as if they were interpreted only. 15214 */ 15215 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15216 if (bpf_pseudo_func(insn)) { 15217 insn[0].imm = env->insn_aux_data[i].call_imm; 15218 insn[1].imm = insn->off; 15219 insn->off = 0; 15220 continue; 15221 } 15222 if (!bpf_pseudo_call(insn)) 15223 continue; 15224 insn->off = env->insn_aux_data[i].call_imm; 15225 subprog = find_subprog(env, i + insn->off + 1); 15226 insn->imm = subprog; 15227 } 15228 15229 prog->jited = 1; 15230 prog->bpf_func = func[0]->bpf_func; 15231 prog->jited_len = func[0]->jited_len; 15232 prog->aux->func = func; 15233 prog->aux->func_cnt = env->subprog_cnt; 15234 bpf_prog_jit_attempt_done(prog); 15235 return 0; 15236 out_free: 15237 /* We failed JIT'ing, so at this point we need to unregister poke 15238 * descriptors from subprogs, so that kernel is not attempting to 15239 * patch it anymore as we're freeing the subprog JIT memory. 15240 */ 15241 for (i = 0; i < prog->aux->size_poke_tab; i++) { 15242 map_ptr = prog->aux->poke_tab[i].tail_call.map; 15243 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 15244 } 15245 /* At this point we're guaranteed that poke descriptors are not 15246 * live anymore. We can just unlink its descriptor table as it's 15247 * released with the main prog. 15248 */ 15249 for (i = 0; i < env->subprog_cnt; i++) { 15250 if (!func[i]) 15251 continue; 15252 func[i]->aux->poke_tab = NULL; 15253 bpf_jit_free(func[i]); 15254 } 15255 kfree(func); 15256 out_undo_insn: 15257 /* cleanup main prog to be interpreted */ 15258 prog->jit_requested = 0; 15259 prog->blinding_requested = 0; 15260 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 15261 if (!bpf_pseudo_call(insn)) 15262 continue; 15263 insn->off = 0; 15264 insn->imm = env->insn_aux_data[i].call_imm; 15265 } 15266 bpf_prog_jit_attempt_done(prog); 15267 return err; 15268 } 15269 15270 static int fixup_call_args(struct bpf_verifier_env *env) 15271 { 15272 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 15273 struct bpf_prog *prog = env->prog; 15274 struct bpf_insn *insn = prog->insnsi; 15275 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 15276 int i, depth; 15277 #endif 15278 int err = 0; 15279 15280 if (env->prog->jit_requested && 15281 !bpf_prog_is_dev_bound(env->prog->aux)) { 15282 err = jit_subprogs(env); 15283 if (err == 0) 15284 return 0; 15285 if (err == -EFAULT) 15286 return err; 15287 } 15288 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 15289 if (has_kfunc_call) { 15290 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 15291 return -EINVAL; 15292 } 15293 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 15294 /* When JIT fails the progs with bpf2bpf calls and tail_calls 15295 * have to be rejected, since interpreter doesn't support them yet. 15296 */ 15297 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 15298 return -EINVAL; 15299 } 15300 for (i = 0; i < prog->len; i++, insn++) { 15301 if (bpf_pseudo_func(insn)) { 15302 /* When JIT fails the progs with callback calls 15303 * have to be rejected, since interpreter doesn't support them yet. 15304 */ 15305 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 15306 return -EINVAL; 15307 } 15308 15309 if (!bpf_pseudo_call(insn)) 15310 continue; 15311 depth = get_callee_stack_depth(env, insn, i); 15312 if (depth < 0) 15313 return depth; 15314 bpf_patch_call_args(insn, depth); 15315 } 15316 err = 0; 15317 #endif 15318 return err; 15319 } 15320 15321 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 15322 struct bpf_insn *insn_buf, int insn_idx, int *cnt) 15323 { 15324 const struct bpf_kfunc_desc *desc; 15325 15326 if (!insn->imm) { 15327 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 15328 return -EINVAL; 15329 } 15330 15331 /* insn->imm has the btf func_id. Replace it with 15332 * an address (relative to __bpf_base_call). 15333 */ 15334 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 15335 if (!desc) { 15336 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 15337 insn->imm); 15338 return -EFAULT; 15339 } 15340 15341 *cnt = 0; 15342 insn->imm = desc->imm; 15343 if (insn->off) 15344 return 0; 15345 if (desc->func_id == special_kfunc_list[KF_bpf_obj_new_impl]) { 15346 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 15347 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 15348 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; 15349 15350 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_1, obj_new_size); 15351 insn_buf[1] = addr[0]; 15352 insn_buf[2] = addr[1]; 15353 insn_buf[3] = *insn; 15354 *cnt = 4; 15355 } else if (desc->func_id == special_kfunc_list[KF_bpf_obj_drop_impl]) { 15356 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; 15357 struct bpf_insn addr[2] = { BPF_LD_IMM64(BPF_REG_2, (long)kptr_struct_meta) }; 15358 15359 insn_buf[0] = addr[0]; 15360 insn_buf[1] = addr[1]; 15361 insn_buf[2] = *insn; 15362 *cnt = 3; 15363 } else if (desc->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || 15364 desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) { 15365 insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); 15366 *cnt = 1; 15367 } 15368 return 0; 15369 } 15370 15371 /* Do various post-verification rewrites in a single program pass. 15372 * These rewrites simplify JIT and interpreter implementations. 15373 */ 15374 static int do_misc_fixups(struct bpf_verifier_env *env) 15375 { 15376 struct bpf_prog *prog = env->prog; 15377 enum bpf_attach_type eatype = prog->expected_attach_type; 15378 enum bpf_prog_type prog_type = resolve_prog_type(prog); 15379 struct bpf_insn *insn = prog->insnsi; 15380 const struct bpf_func_proto *fn; 15381 const int insn_cnt = prog->len; 15382 const struct bpf_map_ops *ops; 15383 struct bpf_insn_aux_data *aux; 15384 struct bpf_insn insn_buf[16]; 15385 struct bpf_prog *new_prog; 15386 struct bpf_map *map_ptr; 15387 int i, ret, cnt, delta = 0; 15388 15389 for (i = 0; i < insn_cnt; i++, insn++) { 15390 /* Make divide-by-zero exceptions impossible. */ 15391 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 15392 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 15393 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 15394 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 15395 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 15396 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 15397 struct bpf_insn *patchlet; 15398 struct bpf_insn chk_and_div[] = { 15399 /* [R,W]x div 0 -> 0 */ 15400 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 15401 BPF_JNE | BPF_K, insn->src_reg, 15402 0, 2, 0), 15403 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 15404 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 15405 *insn, 15406 }; 15407 struct bpf_insn chk_and_mod[] = { 15408 /* [R,W]x mod 0 -> [R,W]x */ 15409 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 15410 BPF_JEQ | BPF_K, insn->src_reg, 15411 0, 1 + (is64 ? 0 : 1), 0), 15412 *insn, 15413 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 15414 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 15415 }; 15416 15417 patchlet = isdiv ? chk_and_div : chk_and_mod; 15418 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 15419 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 15420 15421 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 15422 if (!new_prog) 15423 return -ENOMEM; 15424 15425 delta += cnt - 1; 15426 env->prog = prog = new_prog; 15427 insn = new_prog->insnsi + i + delta; 15428 continue; 15429 } 15430 15431 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 15432 if (BPF_CLASS(insn->code) == BPF_LD && 15433 (BPF_MODE(insn->code) == BPF_ABS || 15434 BPF_MODE(insn->code) == BPF_IND)) { 15435 cnt = env->ops->gen_ld_abs(insn, insn_buf); 15436 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 15437 verbose(env, "bpf verifier is misconfigured\n"); 15438 return -EINVAL; 15439 } 15440 15441 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15442 if (!new_prog) 15443 return -ENOMEM; 15444 15445 delta += cnt - 1; 15446 env->prog = prog = new_prog; 15447 insn = new_prog->insnsi + i + delta; 15448 continue; 15449 } 15450 15451 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 15452 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 15453 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 15454 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 15455 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 15456 struct bpf_insn *patch = &insn_buf[0]; 15457 bool issrc, isneg, isimm; 15458 u32 off_reg; 15459 15460 aux = &env->insn_aux_data[i + delta]; 15461 if (!aux->alu_state || 15462 aux->alu_state == BPF_ALU_NON_POINTER) 15463 continue; 15464 15465 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 15466 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 15467 BPF_ALU_SANITIZE_SRC; 15468 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 15469 15470 off_reg = issrc ? insn->src_reg : insn->dst_reg; 15471 if (isimm) { 15472 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 15473 } else { 15474 if (isneg) 15475 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 15476 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 15477 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 15478 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 15479 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 15480 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 15481 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 15482 } 15483 if (!issrc) 15484 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 15485 insn->src_reg = BPF_REG_AX; 15486 if (isneg) 15487 insn->code = insn->code == code_add ? 15488 code_sub : code_add; 15489 *patch++ = *insn; 15490 if (issrc && isneg && !isimm) 15491 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 15492 cnt = patch - insn_buf; 15493 15494 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15495 if (!new_prog) 15496 return -ENOMEM; 15497 15498 delta += cnt - 1; 15499 env->prog = prog = new_prog; 15500 insn = new_prog->insnsi + i + delta; 15501 continue; 15502 } 15503 15504 if (insn->code != (BPF_JMP | BPF_CALL)) 15505 continue; 15506 if (insn->src_reg == BPF_PSEUDO_CALL) 15507 continue; 15508 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 15509 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); 15510 if (ret) 15511 return ret; 15512 if (cnt == 0) 15513 continue; 15514 15515 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15516 if (!new_prog) 15517 return -ENOMEM; 15518 15519 delta += cnt - 1; 15520 env->prog = prog = new_prog; 15521 insn = new_prog->insnsi + i + delta; 15522 continue; 15523 } 15524 15525 if (insn->imm == BPF_FUNC_get_route_realm) 15526 prog->dst_needed = 1; 15527 if (insn->imm == BPF_FUNC_get_prandom_u32) 15528 bpf_user_rnd_init_once(); 15529 if (insn->imm == BPF_FUNC_override_return) 15530 prog->kprobe_override = 1; 15531 if (insn->imm == BPF_FUNC_tail_call) { 15532 /* If we tail call into other programs, we 15533 * cannot make any assumptions since they can 15534 * be replaced dynamically during runtime in 15535 * the program array. 15536 */ 15537 prog->cb_access = 1; 15538 if (!allow_tail_call_in_subprogs(env)) 15539 prog->aux->stack_depth = MAX_BPF_STACK; 15540 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 15541 15542 /* mark bpf_tail_call as different opcode to avoid 15543 * conditional branch in the interpreter for every normal 15544 * call and to prevent accidental JITing by JIT compiler 15545 * that doesn't support bpf_tail_call yet 15546 */ 15547 insn->imm = 0; 15548 insn->code = BPF_JMP | BPF_TAIL_CALL; 15549 15550 aux = &env->insn_aux_data[i + delta]; 15551 if (env->bpf_capable && !prog->blinding_requested && 15552 prog->jit_requested && 15553 !bpf_map_key_poisoned(aux) && 15554 !bpf_map_ptr_poisoned(aux) && 15555 !bpf_map_ptr_unpriv(aux)) { 15556 struct bpf_jit_poke_descriptor desc = { 15557 .reason = BPF_POKE_REASON_TAIL_CALL, 15558 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 15559 .tail_call.key = bpf_map_key_immediate(aux), 15560 .insn_idx = i + delta, 15561 }; 15562 15563 ret = bpf_jit_add_poke_descriptor(prog, &desc); 15564 if (ret < 0) { 15565 verbose(env, "adding tail call poke descriptor failed\n"); 15566 return ret; 15567 } 15568 15569 insn->imm = ret + 1; 15570 continue; 15571 } 15572 15573 if (!bpf_map_ptr_unpriv(aux)) 15574 continue; 15575 15576 /* instead of changing every JIT dealing with tail_call 15577 * emit two extra insns: 15578 * if (index >= max_entries) goto out; 15579 * index &= array->index_mask; 15580 * to avoid out-of-bounds cpu speculation 15581 */ 15582 if (bpf_map_ptr_poisoned(aux)) { 15583 verbose(env, "tail_call abusing map_ptr\n"); 15584 return -EINVAL; 15585 } 15586 15587 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 15588 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 15589 map_ptr->max_entries, 2); 15590 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 15591 container_of(map_ptr, 15592 struct bpf_array, 15593 map)->index_mask); 15594 insn_buf[2] = *insn; 15595 cnt = 3; 15596 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15597 if (!new_prog) 15598 return -ENOMEM; 15599 15600 delta += cnt - 1; 15601 env->prog = prog = new_prog; 15602 insn = new_prog->insnsi + i + delta; 15603 continue; 15604 } 15605 15606 if (insn->imm == BPF_FUNC_timer_set_callback) { 15607 /* The verifier will process callback_fn as many times as necessary 15608 * with different maps and the register states prepared by 15609 * set_timer_callback_state will be accurate. 15610 * 15611 * The following use case is valid: 15612 * map1 is shared by prog1, prog2, prog3. 15613 * prog1 calls bpf_timer_init for some map1 elements 15614 * prog2 calls bpf_timer_set_callback for some map1 elements. 15615 * Those that were not bpf_timer_init-ed will return -EINVAL. 15616 * prog3 calls bpf_timer_start for some map1 elements. 15617 * Those that were not both bpf_timer_init-ed and 15618 * bpf_timer_set_callback-ed will return -EINVAL. 15619 */ 15620 struct bpf_insn ld_addrs[2] = { 15621 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 15622 }; 15623 15624 insn_buf[0] = ld_addrs[0]; 15625 insn_buf[1] = ld_addrs[1]; 15626 insn_buf[2] = *insn; 15627 cnt = 3; 15628 15629 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15630 if (!new_prog) 15631 return -ENOMEM; 15632 15633 delta += cnt - 1; 15634 env->prog = prog = new_prog; 15635 insn = new_prog->insnsi + i + delta; 15636 goto patch_call_imm; 15637 } 15638 15639 if (is_storage_get_function(insn->imm)) { 15640 if (!env->prog->aux->sleepable || 15641 env->insn_aux_data[i + delta].storage_get_func_atomic) 15642 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 15643 else 15644 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 15645 insn_buf[1] = *insn; 15646 cnt = 2; 15647 15648 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15649 if (!new_prog) 15650 return -ENOMEM; 15651 15652 delta += cnt - 1; 15653 env->prog = prog = new_prog; 15654 insn = new_prog->insnsi + i + delta; 15655 goto patch_call_imm; 15656 } 15657 15658 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 15659 * and other inlining handlers are currently limited to 64 bit 15660 * only. 15661 */ 15662 if (prog->jit_requested && BITS_PER_LONG == 64 && 15663 (insn->imm == BPF_FUNC_map_lookup_elem || 15664 insn->imm == BPF_FUNC_map_update_elem || 15665 insn->imm == BPF_FUNC_map_delete_elem || 15666 insn->imm == BPF_FUNC_map_push_elem || 15667 insn->imm == BPF_FUNC_map_pop_elem || 15668 insn->imm == BPF_FUNC_map_peek_elem || 15669 insn->imm == BPF_FUNC_redirect_map || 15670 insn->imm == BPF_FUNC_for_each_map_elem || 15671 insn->imm == BPF_FUNC_map_lookup_percpu_elem)) { 15672 aux = &env->insn_aux_data[i + delta]; 15673 if (bpf_map_ptr_poisoned(aux)) 15674 goto patch_call_imm; 15675 15676 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 15677 ops = map_ptr->ops; 15678 if (insn->imm == BPF_FUNC_map_lookup_elem && 15679 ops->map_gen_lookup) { 15680 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 15681 if (cnt == -EOPNOTSUPP) 15682 goto patch_map_ops_generic; 15683 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 15684 verbose(env, "bpf verifier is misconfigured\n"); 15685 return -EINVAL; 15686 } 15687 15688 new_prog = bpf_patch_insn_data(env, i + delta, 15689 insn_buf, cnt); 15690 if (!new_prog) 15691 return -ENOMEM; 15692 15693 delta += cnt - 1; 15694 env->prog = prog = new_prog; 15695 insn = new_prog->insnsi + i + delta; 15696 continue; 15697 } 15698 15699 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 15700 (void *(*)(struct bpf_map *map, void *key))NULL)); 15701 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 15702 (int (*)(struct bpf_map *map, void *key))NULL)); 15703 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 15704 (int (*)(struct bpf_map *map, void *key, void *value, 15705 u64 flags))NULL)); 15706 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 15707 (int (*)(struct bpf_map *map, void *value, 15708 u64 flags))NULL)); 15709 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 15710 (int (*)(struct bpf_map *map, void *value))NULL)); 15711 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 15712 (int (*)(struct bpf_map *map, void *value))NULL)); 15713 BUILD_BUG_ON(!__same_type(ops->map_redirect, 15714 (int (*)(struct bpf_map *map, u64 index, u64 flags))NULL)); 15715 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 15716 (int (*)(struct bpf_map *map, 15717 bpf_callback_t callback_fn, 15718 void *callback_ctx, 15719 u64 flags))NULL)); 15720 BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem, 15721 (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL)); 15722 15723 patch_map_ops_generic: 15724 switch (insn->imm) { 15725 case BPF_FUNC_map_lookup_elem: 15726 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 15727 continue; 15728 case BPF_FUNC_map_update_elem: 15729 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 15730 continue; 15731 case BPF_FUNC_map_delete_elem: 15732 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 15733 continue; 15734 case BPF_FUNC_map_push_elem: 15735 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 15736 continue; 15737 case BPF_FUNC_map_pop_elem: 15738 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 15739 continue; 15740 case BPF_FUNC_map_peek_elem: 15741 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 15742 continue; 15743 case BPF_FUNC_redirect_map: 15744 insn->imm = BPF_CALL_IMM(ops->map_redirect); 15745 continue; 15746 case BPF_FUNC_for_each_map_elem: 15747 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 15748 continue; 15749 case BPF_FUNC_map_lookup_percpu_elem: 15750 insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem); 15751 continue; 15752 } 15753 15754 goto patch_call_imm; 15755 } 15756 15757 /* Implement bpf_jiffies64 inline. */ 15758 if (prog->jit_requested && BITS_PER_LONG == 64 && 15759 insn->imm == BPF_FUNC_jiffies64) { 15760 struct bpf_insn ld_jiffies_addr[2] = { 15761 BPF_LD_IMM64(BPF_REG_0, 15762 (unsigned long)&jiffies), 15763 }; 15764 15765 insn_buf[0] = ld_jiffies_addr[0]; 15766 insn_buf[1] = ld_jiffies_addr[1]; 15767 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 15768 BPF_REG_0, 0); 15769 cnt = 3; 15770 15771 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 15772 cnt); 15773 if (!new_prog) 15774 return -ENOMEM; 15775 15776 delta += cnt - 1; 15777 env->prog = prog = new_prog; 15778 insn = new_prog->insnsi + i + delta; 15779 continue; 15780 } 15781 15782 /* Implement bpf_get_func_arg inline. */ 15783 if (prog_type == BPF_PROG_TYPE_TRACING && 15784 insn->imm == BPF_FUNC_get_func_arg) { 15785 /* Load nr_args from ctx - 8 */ 15786 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15787 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 15788 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 15789 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 15790 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 15791 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 15792 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 15793 insn_buf[7] = BPF_JMP_A(1); 15794 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 15795 cnt = 9; 15796 15797 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15798 if (!new_prog) 15799 return -ENOMEM; 15800 15801 delta += cnt - 1; 15802 env->prog = prog = new_prog; 15803 insn = new_prog->insnsi + i + delta; 15804 continue; 15805 } 15806 15807 /* Implement bpf_get_func_ret inline. */ 15808 if (prog_type == BPF_PROG_TYPE_TRACING && 15809 insn->imm == BPF_FUNC_get_func_ret) { 15810 if (eatype == BPF_TRACE_FEXIT || 15811 eatype == BPF_MODIFY_RETURN) { 15812 /* Load nr_args from ctx - 8 */ 15813 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15814 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 15815 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 15816 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 15817 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 15818 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 15819 cnt = 6; 15820 } else { 15821 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 15822 cnt = 1; 15823 } 15824 15825 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 15826 if (!new_prog) 15827 return -ENOMEM; 15828 15829 delta += cnt - 1; 15830 env->prog = prog = new_prog; 15831 insn = new_prog->insnsi + i + delta; 15832 continue; 15833 } 15834 15835 /* Implement get_func_arg_cnt inline. */ 15836 if (prog_type == BPF_PROG_TYPE_TRACING && 15837 insn->imm == BPF_FUNC_get_func_arg_cnt) { 15838 /* Load nr_args from ctx - 8 */ 15839 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 15840 15841 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 15842 if (!new_prog) 15843 return -ENOMEM; 15844 15845 env->prog = prog = new_prog; 15846 insn = new_prog->insnsi + i + delta; 15847 continue; 15848 } 15849 15850 /* Implement bpf_get_func_ip inline. */ 15851 if (prog_type == BPF_PROG_TYPE_TRACING && 15852 insn->imm == BPF_FUNC_get_func_ip) { 15853 /* Load IP address from ctx - 16 */ 15854 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 15855 15856 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 15857 if (!new_prog) 15858 return -ENOMEM; 15859 15860 env->prog = prog = new_prog; 15861 insn = new_prog->insnsi + i + delta; 15862 continue; 15863 } 15864 15865 patch_call_imm: 15866 fn = env->ops->get_func_proto(insn->imm, env->prog); 15867 /* all functions that have prototype and verifier allowed 15868 * programs to call them, must be real in-kernel functions 15869 */ 15870 if (!fn->func) { 15871 verbose(env, 15872 "kernel subsystem misconfigured func %s#%d\n", 15873 func_id_name(insn->imm), insn->imm); 15874 return -EFAULT; 15875 } 15876 insn->imm = fn->func - __bpf_call_base; 15877 } 15878 15879 /* Since poke tab is now finalized, publish aux to tracker. */ 15880 for (i = 0; i < prog->aux->size_poke_tab; i++) { 15881 map_ptr = prog->aux->poke_tab[i].tail_call.map; 15882 if (!map_ptr->ops->map_poke_track || 15883 !map_ptr->ops->map_poke_untrack || 15884 !map_ptr->ops->map_poke_run) { 15885 verbose(env, "bpf verifier is misconfigured\n"); 15886 return -EINVAL; 15887 } 15888 15889 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 15890 if (ret < 0) { 15891 verbose(env, "tracking tail call prog failed\n"); 15892 return ret; 15893 } 15894 } 15895 15896 sort_kfunc_descs_by_imm(env->prog); 15897 15898 return 0; 15899 } 15900 15901 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, 15902 int position, 15903 s32 stack_base, 15904 u32 callback_subprogno, 15905 u32 *cnt) 15906 { 15907 s32 r6_offset = stack_base + 0 * BPF_REG_SIZE; 15908 s32 r7_offset = stack_base + 1 * BPF_REG_SIZE; 15909 s32 r8_offset = stack_base + 2 * BPF_REG_SIZE; 15910 int reg_loop_max = BPF_REG_6; 15911 int reg_loop_cnt = BPF_REG_7; 15912 int reg_loop_ctx = BPF_REG_8; 15913 15914 struct bpf_prog *new_prog; 15915 u32 callback_start; 15916 u32 call_insn_offset; 15917 s32 callback_offset; 15918 15919 /* This represents an inlined version of bpf_iter.c:bpf_loop, 15920 * be careful to modify this code in sync. 15921 */ 15922 struct bpf_insn insn_buf[] = { 15923 /* Return error and jump to the end of the patch if 15924 * expected number of iterations is too big. 15925 */ 15926 BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2), 15927 BPF_MOV32_IMM(BPF_REG_0, -E2BIG), 15928 BPF_JMP_IMM(BPF_JA, 0, 0, 16), 15929 /* spill R6, R7, R8 to use these as loop vars */ 15930 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset), 15931 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset), 15932 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset), 15933 /* initialize loop vars */ 15934 BPF_MOV64_REG(reg_loop_max, BPF_REG_1), 15935 BPF_MOV32_IMM(reg_loop_cnt, 0), 15936 BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3), 15937 /* loop header, 15938 * if reg_loop_cnt >= reg_loop_max skip the loop body 15939 */ 15940 BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5), 15941 /* callback call, 15942 * correct callback offset would be set after patching 15943 */ 15944 BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt), 15945 BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx), 15946 BPF_CALL_REL(0), 15947 /* increment loop counter */ 15948 BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1), 15949 /* jump to loop header if callback returned 0 */ 15950 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6), 15951 /* return value of bpf_loop, 15952 * set R0 to the number of iterations 15953 */ 15954 BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt), 15955 /* restore original values of R6, R7, R8 */ 15956 BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset), 15957 BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset), 15958 BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset), 15959 }; 15960 15961 *cnt = ARRAY_SIZE(insn_buf); 15962 new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt); 15963 if (!new_prog) 15964 return new_prog; 15965 15966 /* callback start is known only after patching */ 15967 callback_start = env->subprog_info[callback_subprogno].start; 15968 /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */ 15969 call_insn_offset = position + 12; 15970 callback_offset = callback_start - call_insn_offset - 1; 15971 new_prog->insnsi[call_insn_offset].imm = callback_offset; 15972 15973 return new_prog; 15974 } 15975 15976 static bool is_bpf_loop_call(struct bpf_insn *insn) 15977 { 15978 return insn->code == (BPF_JMP | BPF_CALL) && 15979 insn->src_reg == 0 && 15980 insn->imm == BPF_FUNC_loop; 15981 } 15982 15983 /* For all sub-programs in the program (including main) check 15984 * insn_aux_data to see if there are bpf_loop calls that require 15985 * inlining. If such calls are found the calls are replaced with a 15986 * sequence of instructions produced by `inline_bpf_loop` function and 15987 * subprog stack_depth is increased by the size of 3 registers. 15988 * This stack space is used to spill values of the R6, R7, R8. These 15989 * registers are used to store the loop bound, counter and context 15990 * variables. 15991 */ 15992 static int optimize_bpf_loop(struct bpf_verifier_env *env) 15993 { 15994 struct bpf_subprog_info *subprogs = env->subprog_info; 15995 int i, cur_subprog = 0, cnt, delta = 0; 15996 struct bpf_insn *insn = env->prog->insnsi; 15997 int insn_cnt = env->prog->len; 15998 u16 stack_depth = subprogs[cur_subprog].stack_depth; 15999 u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 16000 u16 stack_depth_extra = 0; 16001 16002 for (i = 0; i < insn_cnt; i++, insn++) { 16003 struct bpf_loop_inline_state *inline_state = 16004 &env->insn_aux_data[i + delta].loop_inline_state; 16005 16006 if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) { 16007 struct bpf_prog *new_prog; 16008 16009 stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup; 16010 new_prog = inline_bpf_loop(env, 16011 i + delta, 16012 -(stack_depth + stack_depth_extra), 16013 inline_state->callback_subprogno, 16014 &cnt); 16015 if (!new_prog) 16016 return -ENOMEM; 16017 16018 delta += cnt - 1; 16019 env->prog = new_prog; 16020 insn = new_prog->insnsi + i + delta; 16021 } 16022 16023 if (subprogs[cur_subprog + 1].start == i + delta + 1) { 16024 subprogs[cur_subprog].stack_depth += stack_depth_extra; 16025 cur_subprog++; 16026 stack_depth = subprogs[cur_subprog].stack_depth; 16027 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth; 16028 stack_depth_extra = 0; 16029 } 16030 } 16031 16032 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 16033 16034 return 0; 16035 } 16036 16037 static void free_states(struct bpf_verifier_env *env) 16038 { 16039 struct bpf_verifier_state_list *sl, *sln; 16040 int i; 16041 16042 sl = env->free_list; 16043 while (sl) { 16044 sln = sl->next; 16045 free_verifier_state(&sl->state, false); 16046 kfree(sl); 16047 sl = sln; 16048 } 16049 env->free_list = NULL; 16050 16051 if (!env->explored_states) 16052 return; 16053 16054 for (i = 0; i < state_htab_size(env); i++) { 16055 sl = env->explored_states[i]; 16056 16057 while (sl) { 16058 sln = sl->next; 16059 free_verifier_state(&sl->state, false); 16060 kfree(sl); 16061 sl = sln; 16062 } 16063 env->explored_states[i] = NULL; 16064 } 16065 } 16066 16067 static int do_check_common(struct bpf_verifier_env *env, int subprog) 16068 { 16069 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 16070 struct bpf_verifier_state *state; 16071 struct bpf_reg_state *regs; 16072 int ret, i; 16073 16074 env->prev_linfo = NULL; 16075 env->pass_cnt++; 16076 16077 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 16078 if (!state) 16079 return -ENOMEM; 16080 state->curframe = 0; 16081 state->speculative = false; 16082 state->branches = 1; 16083 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 16084 if (!state->frame[0]) { 16085 kfree(state); 16086 return -ENOMEM; 16087 } 16088 env->cur_state = state; 16089 init_func_state(env, state->frame[0], 16090 BPF_MAIN_FUNC /* callsite */, 16091 0 /* frameno */, 16092 subprog); 16093 state->first_insn_idx = env->subprog_info[subprog].start; 16094 state->last_insn_idx = -1; 16095 16096 regs = state->frame[state->curframe]->regs; 16097 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 16098 ret = btf_prepare_func_args(env, subprog, regs); 16099 if (ret) 16100 goto out; 16101 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 16102 if (regs[i].type == PTR_TO_CTX) 16103 mark_reg_known_zero(env, regs, i); 16104 else if (regs[i].type == SCALAR_VALUE) 16105 mark_reg_unknown(env, regs, i); 16106 else if (base_type(regs[i].type) == PTR_TO_MEM) { 16107 const u32 mem_size = regs[i].mem_size; 16108 16109 mark_reg_known_zero(env, regs, i); 16110 regs[i].mem_size = mem_size; 16111 regs[i].id = ++env->id_gen; 16112 } 16113 } 16114 } else { 16115 /* 1st arg to a function */ 16116 regs[BPF_REG_1].type = PTR_TO_CTX; 16117 mark_reg_known_zero(env, regs, BPF_REG_1); 16118 ret = btf_check_subprog_arg_match(env, subprog, regs); 16119 if (ret == -EFAULT) 16120 /* unlikely verifier bug. abort. 16121 * ret == 0 and ret < 0 are sadly acceptable for 16122 * main() function due to backward compatibility. 16123 * Like socket filter program may be written as: 16124 * int bpf_prog(struct pt_regs *ctx) 16125 * and never dereference that ctx in the program. 16126 * 'struct pt_regs' is a type mismatch for socket 16127 * filter that should be using 'struct __sk_buff'. 16128 */ 16129 goto out; 16130 } 16131 16132 ret = do_check(env); 16133 out: 16134 /* check for NULL is necessary, since cur_state can be freed inside 16135 * do_check() under memory pressure. 16136 */ 16137 if (env->cur_state) { 16138 free_verifier_state(env->cur_state, true); 16139 env->cur_state = NULL; 16140 } 16141 while (!pop_stack(env, NULL, NULL, false)); 16142 if (!ret && pop_log) 16143 bpf_vlog_reset(&env->log, 0); 16144 free_states(env); 16145 return ret; 16146 } 16147 16148 /* Verify all global functions in a BPF program one by one based on their BTF. 16149 * All global functions must pass verification. Otherwise the whole program is rejected. 16150 * Consider: 16151 * int bar(int); 16152 * int foo(int f) 16153 * { 16154 * return bar(f); 16155 * } 16156 * int bar(int b) 16157 * { 16158 * ... 16159 * } 16160 * foo() will be verified first for R1=any_scalar_value. During verification it 16161 * will be assumed that bar() already verified successfully and call to bar() 16162 * from foo() will be checked for type match only. Later bar() will be verified 16163 * independently to check that it's safe for R1=any_scalar_value. 16164 */ 16165 static int do_check_subprogs(struct bpf_verifier_env *env) 16166 { 16167 struct bpf_prog_aux *aux = env->prog->aux; 16168 int i, ret; 16169 16170 if (!aux->func_info) 16171 return 0; 16172 16173 for (i = 1; i < env->subprog_cnt; i++) { 16174 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 16175 continue; 16176 env->insn_idx = env->subprog_info[i].start; 16177 WARN_ON_ONCE(env->insn_idx == 0); 16178 ret = do_check_common(env, i); 16179 if (ret) { 16180 return ret; 16181 } else if (env->log.level & BPF_LOG_LEVEL) { 16182 verbose(env, 16183 "Func#%d is safe for any args that match its prototype\n", 16184 i); 16185 } 16186 } 16187 return 0; 16188 } 16189 16190 static int do_check_main(struct bpf_verifier_env *env) 16191 { 16192 int ret; 16193 16194 env->insn_idx = 0; 16195 ret = do_check_common(env, 0); 16196 if (!ret) 16197 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 16198 return ret; 16199 } 16200 16201 16202 static void print_verification_stats(struct bpf_verifier_env *env) 16203 { 16204 int i; 16205 16206 if (env->log.level & BPF_LOG_STATS) { 16207 verbose(env, "verification time %lld usec\n", 16208 div_u64(env->verification_time, 1000)); 16209 verbose(env, "stack depth "); 16210 for (i = 0; i < env->subprog_cnt; i++) { 16211 u32 depth = env->subprog_info[i].stack_depth; 16212 16213 verbose(env, "%d", depth); 16214 if (i + 1 < env->subprog_cnt) 16215 verbose(env, "+"); 16216 } 16217 verbose(env, "\n"); 16218 } 16219 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 16220 "total_states %d peak_states %d mark_read %d\n", 16221 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 16222 env->max_states_per_insn, env->total_states, 16223 env->peak_states, env->longest_mark_read_walk); 16224 } 16225 16226 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 16227 { 16228 const struct btf_type *t, *func_proto; 16229 const struct bpf_struct_ops *st_ops; 16230 const struct btf_member *member; 16231 struct bpf_prog *prog = env->prog; 16232 u32 btf_id, member_idx; 16233 const char *mname; 16234 16235 if (!prog->gpl_compatible) { 16236 verbose(env, "struct ops programs must have a GPL compatible license\n"); 16237 return -EINVAL; 16238 } 16239 16240 btf_id = prog->aux->attach_btf_id; 16241 st_ops = bpf_struct_ops_find(btf_id); 16242 if (!st_ops) { 16243 verbose(env, "attach_btf_id %u is not a supported struct\n", 16244 btf_id); 16245 return -ENOTSUPP; 16246 } 16247 16248 t = st_ops->type; 16249 member_idx = prog->expected_attach_type; 16250 if (member_idx >= btf_type_vlen(t)) { 16251 verbose(env, "attach to invalid member idx %u of struct %s\n", 16252 member_idx, st_ops->name); 16253 return -EINVAL; 16254 } 16255 16256 member = &btf_type_member(t)[member_idx]; 16257 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 16258 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 16259 NULL); 16260 if (!func_proto) { 16261 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 16262 mname, member_idx, st_ops->name); 16263 return -EINVAL; 16264 } 16265 16266 if (st_ops->check_member) { 16267 int err = st_ops->check_member(t, member); 16268 16269 if (err) { 16270 verbose(env, "attach to unsupported member %s of struct %s\n", 16271 mname, st_ops->name); 16272 return err; 16273 } 16274 } 16275 16276 prog->aux->attach_func_proto = func_proto; 16277 prog->aux->attach_func_name = mname; 16278 env->ops = st_ops->verifier_ops; 16279 16280 return 0; 16281 } 16282 #define SECURITY_PREFIX "security_" 16283 16284 static int check_attach_modify_return(unsigned long addr, const char *func_name) 16285 { 16286 if (within_error_injection_list(addr) || 16287 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 16288 return 0; 16289 16290 return -EINVAL; 16291 } 16292 16293 /* list of non-sleepable functions that are otherwise on 16294 * ALLOW_ERROR_INJECTION list 16295 */ 16296 BTF_SET_START(btf_non_sleepable_error_inject) 16297 /* Three functions below can be called from sleepable and non-sleepable context. 16298 * Assume non-sleepable from bpf safety point of view. 16299 */ 16300 BTF_ID(func, __filemap_add_folio) 16301 BTF_ID(func, should_fail_alloc_page) 16302 BTF_ID(func, should_failslab) 16303 BTF_SET_END(btf_non_sleepable_error_inject) 16304 16305 static int check_non_sleepable_error_inject(u32 btf_id) 16306 { 16307 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 16308 } 16309 16310 int bpf_check_attach_target(struct bpf_verifier_log *log, 16311 const struct bpf_prog *prog, 16312 const struct bpf_prog *tgt_prog, 16313 u32 btf_id, 16314 struct bpf_attach_target_info *tgt_info) 16315 { 16316 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 16317 const char prefix[] = "btf_trace_"; 16318 int ret = 0, subprog = -1, i; 16319 const struct btf_type *t; 16320 bool conservative = true; 16321 const char *tname; 16322 struct btf *btf; 16323 long addr = 0; 16324 16325 if (!btf_id) { 16326 bpf_log(log, "Tracing programs must provide btf_id\n"); 16327 return -EINVAL; 16328 } 16329 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 16330 if (!btf) { 16331 bpf_log(log, 16332 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 16333 return -EINVAL; 16334 } 16335 t = btf_type_by_id(btf, btf_id); 16336 if (!t) { 16337 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 16338 return -EINVAL; 16339 } 16340 tname = btf_name_by_offset(btf, t->name_off); 16341 if (!tname) { 16342 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 16343 return -EINVAL; 16344 } 16345 if (tgt_prog) { 16346 struct bpf_prog_aux *aux = tgt_prog->aux; 16347 16348 for (i = 0; i < aux->func_info_cnt; i++) 16349 if (aux->func_info[i].type_id == btf_id) { 16350 subprog = i; 16351 break; 16352 } 16353 if (subprog == -1) { 16354 bpf_log(log, "Subprog %s doesn't exist\n", tname); 16355 return -EINVAL; 16356 } 16357 conservative = aux->func_info_aux[subprog].unreliable; 16358 if (prog_extension) { 16359 if (conservative) { 16360 bpf_log(log, 16361 "Cannot replace static functions\n"); 16362 return -EINVAL; 16363 } 16364 if (!prog->jit_requested) { 16365 bpf_log(log, 16366 "Extension programs should be JITed\n"); 16367 return -EINVAL; 16368 } 16369 } 16370 if (!tgt_prog->jited) { 16371 bpf_log(log, "Can attach to only JITed progs\n"); 16372 return -EINVAL; 16373 } 16374 if (tgt_prog->type == prog->type) { 16375 /* Cannot fentry/fexit another fentry/fexit program. 16376 * Cannot attach program extension to another extension. 16377 * It's ok to attach fentry/fexit to extension program. 16378 */ 16379 bpf_log(log, "Cannot recursively attach\n"); 16380 return -EINVAL; 16381 } 16382 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 16383 prog_extension && 16384 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 16385 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 16386 /* Program extensions can extend all program types 16387 * except fentry/fexit. The reason is the following. 16388 * The fentry/fexit programs are used for performance 16389 * analysis, stats and can be attached to any program 16390 * type except themselves. When extension program is 16391 * replacing XDP function it is necessary to allow 16392 * performance analysis of all functions. Both original 16393 * XDP program and its program extension. Hence 16394 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 16395 * allowed. If extending of fentry/fexit was allowed it 16396 * would be possible to create long call chain 16397 * fentry->extension->fentry->extension beyond 16398 * reasonable stack size. Hence extending fentry is not 16399 * allowed. 16400 */ 16401 bpf_log(log, "Cannot extend fentry/fexit\n"); 16402 return -EINVAL; 16403 } 16404 } else { 16405 if (prog_extension) { 16406 bpf_log(log, "Cannot replace kernel functions\n"); 16407 return -EINVAL; 16408 } 16409 } 16410 16411 switch (prog->expected_attach_type) { 16412 case BPF_TRACE_RAW_TP: 16413 if (tgt_prog) { 16414 bpf_log(log, 16415 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 16416 return -EINVAL; 16417 } 16418 if (!btf_type_is_typedef(t)) { 16419 bpf_log(log, "attach_btf_id %u is not a typedef\n", 16420 btf_id); 16421 return -EINVAL; 16422 } 16423 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 16424 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 16425 btf_id, tname); 16426 return -EINVAL; 16427 } 16428 tname += sizeof(prefix) - 1; 16429 t = btf_type_by_id(btf, t->type); 16430 if (!btf_type_is_ptr(t)) 16431 /* should never happen in valid vmlinux build */ 16432 return -EINVAL; 16433 t = btf_type_by_id(btf, t->type); 16434 if (!btf_type_is_func_proto(t)) 16435 /* should never happen in valid vmlinux build */ 16436 return -EINVAL; 16437 16438 break; 16439 case BPF_TRACE_ITER: 16440 if (!btf_type_is_func(t)) { 16441 bpf_log(log, "attach_btf_id %u is not a function\n", 16442 btf_id); 16443 return -EINVAL; 16444 } 16445 t = btf_type_by_id(btf, t->type); 16446 if (!btf_type_is_func_proto(t)) 16447 return -EINVAL; 16448 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 16449 if (ret) 16450 return ret; 16451 break; 16452 default: 16453 if (!prog_extension) 16454 return -EINVAL; 16455 fallthrough; 16456 case BPF_MODIFY_RETURN: 16457 case BPF_LSM_MAC: 16458 case BPF_LSM_CGROUP: 16459 case BPF_TRACE_FENTRY: 16460 case BPF_TRACE_FEXIT: 16461 if (!btf_type_is_func(t)) { 16462 bpf_log(log, "attach_btf_id %u is not a function\n", 16463 btf_id); 16464 return -EINVAL; 16465 } 16466 if (prog_extension && 16467 btf_check_type_match(log, prog, btf, t)) 16468 return -EINVAL; 16469 t = btf_type_by_id(btf, t->type); 16470 if (!btf_type_is_func_proto(t)) 16471 return -EINVAL; 16472 16473 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 16474 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 16475 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 16476 return -EINVAL; 16477 16478 if (tgt_prog && conservative) 16479 t = NULL; 16480 16481 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 16482 if (ret < 0) 16483 return ret; 16484 16485 if (tgt_prog) { 16486 if (subprog == 0) 16487 addr = (long) tgt_prog->bpf_func; 16488 else 16489 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 16490 } else { 16491 addr = kallsyms_lookup_name(tname); 16492 if (!addr) { 16493 bpf_log(log, 16494 "The address of function %s cannot be found\n", 16495 tname); 16496 return -ENOENT; 16497 } 16498 } 16499 16500 if (prog->aux->sleepable) { 16501 ret = -EINVAL; 16502 switch (prog->type) { 16503 case BPF_PROG_TYPE_TRACING: 16504 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 16505 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 16506 */ 16507 if (!check_non_sleepable_error_inject(btf_id) && 16508 within_error_injection_list(addr)) 16509 ret = 0; 16510 break; 16511 case BPF_PROG_TYPE_LSM: 16512 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 16513 * Only some of them are sleepable. 16514 */ 16515 if (bpf_lsm_is_sleepable_hook(btf_id)) 16516 ret = 0; 16517 break; 16518 default: 16519 break; 16520 } 16521 if (ret) { 16522 bpf_log(log, "%s is not sleepable\n", tname); 16523 return ret; 16524 } 16525 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 16526 if (tgt_prog) { 16527 bpf_log(log, "can't modify return codes of BPF programs\n"); 16528 return -EINVAL; 16529 } 16530 ret = check_attach_modify_return(addr, tname); 16531 if (ret) { 16532 bpf_log(log, "%s() is not modifiable\n", tname); 16533 return ret; 16534 } 16535 } 16536 16537 break; 16538 } 16539 tgt_info->tgt_addr = addr; 16540 tgt_info->tgt_name = tname; 16541 tgt_info->tgt_type = t; 16542 return 0; 16543 } 16544 16545 BTF_SET_START(btf_id_deny) 16546 BTF_ID_UNUSED 16547 #ifdef CONFIG_SMP 16548 BTF_ID(func, migrate_disable) 16549 BTF_ID(func, migrate_enable) 16550 #endif 16551 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 16552 BTF_ID(func, rcu_read_unlock_strict) 16553 #endif 16554 BTF_SET_END(btf_id_deny) 16555 16556 static int check_attach_btf_id(struct bpf_verifier_env *env) 16557 { 16558 struct bpf_prog *prog = env->prog; 16559 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 16560 struct bpf_attach_target_info tgt_info = {}; 16561 u32 btf_id = prog->aux->attach_btf_id; 16562 struct bpf_trampoline *tr; 16563 int ret; 16564 u64 key; 16565 16566 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 16567 if (prog->aux->sleepable) 16568 /* attach_btf_id checked to be zero already */ 16569 return 0; 16570 verbose(env, "Syscall programs can only be sleepable\n"); 16571 return -EINVAL; 16572 } 16573 16574 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 16575 prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_KPROBE) { 16576 verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n"); 16577 return -EINVAL; 16578 } 16579 16580 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 16581 return check_struct_ops_btf_id(env); 16582 16583 if (prog->type != BPF_PROG_TYPE_TRACING && 16584 prog->type != BPF_PROG_TYPE_LSM && 16585 prog->type != BPF_PROG_TYPE_EXT) 16586 return 0; 16587 16588 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 16589 if (ret) 16590 return ret; 16591 16592 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 16593 /* to make freplace equivalent to their targets, they need to 16594 * inherit env->ops and expected_attach_type for the rest of the 16595 * verification 16596 */ 16597 env->ops = bpf_verifier_ops[tgt_prog->type]; 16598 prog->expected_attach_type = tgt_prog->expected_attach_type; 16599 } 16600 16601 /* store info about the attachment target that will be used later */ 16602 prog->aux->attach_func_proto = tgt_info.tgt_type; 16603 prog->aux->attach_func_name = tgt_info.tgt_name; 16604 16605 if (tgt_prog) { 16606 prog->aux->saved_dst_prog_type = tgt_prog->type; 16607 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 16608 } 16609 16610 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 16611 prog->aux->attach_btf_trace = true; 16612 return 0; 16613 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 16614 if (!bpf_iter_prog_supported(prog)) 16615 return -EINVAL; 16616 return 0; 16617 } 16618 16619 if (prog->type == BPF_PROG_TYPE_LSM) { 16620 ret = bpf_lsm_verify_prog(&env->log, prog); 16621 if (ret < 0) 16622 return ret; 16623 } else if (prog->type == BPF_PROG_TYPE_TRACING && 16624 btf_id_set_contains(&btf_id_deny, btf_id)) { 16625 return -EINVAL; 16626 } 16627 16628 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 16629 tr = bpf_trampoline_get(key, &tgt_info); 16630 if (!tr) 16631 return -ENOMEM; 16632 16633 prog->aux->dst_trampoline = tr; 16634 return 0; 16635 } 16636 16637 struct btf *bpf_get_btf_vmlinux(void) 16638 { 16639 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 16640 mutex_lock(&bpf_verifier_lock); 16641 if (!btf_vmlinux) 16642 btf_vmlinux = btf_parse_vmlinux(); 16643 mutex_unlock(&bpf_verifier_lock); 16644 } 16645 return btf_vmlinux; 16646 } 16647 16648 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) 16649 { 16650 u64 start_time = ktime_get_ns(); 16651 struct bpf_verifier_env *env; 16652 struct bpf_verifier_log *log; 16653 int i, len, ret = -EINVAL; 16654 bool is_priv; 16655 16656 /* no program is valid */ 16657 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 16658 return -EINVAL; 16659 16660 /* 'struct bpf_verifier_env' can be global, but since it's not small, 16661 * allocate/free it every time bpf_check() is called 16662 */ 16663 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 16664 if (!env) 16665 return -ENOMEM; 16666 log = &env->log; 16667 16668 len = (*prog)->len; 16669 env->insn_aux_data = 16670 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 16671 ret = -ENOMEM; 16672 if (!env->insn_aux_data) 16673 goto err_free_env; 16674 for (i = 0; i < len; i++) 16675 env->insn_aux_data[i].orig_idx = i; 16676 env->prog = *prog; 16677 env->ops = bpf_verifier_ops[env->prog->type]; 16678 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 16679 is_priv = bpf_capable(); 16680 16681 bpf_get_btf_vmlinux(); 16682 16683 /* grab the mutex to protect few globals used by verifier */ 16684 if (!is_priv) 16685 mutex_lock(&bpf_verifier_lock); 16686 16687 if (attr->log_level || attr->log_buf || attr->log_size) { 16688 /* user requested verbose verifier output 16689 * and supplied buffer to store the verification trace 16690 */ 16691 log->level = attr->log_level; 16692 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 16693 log->len_total = attr->log_size; 16694 16695 /* log attributes have to be sane */ 16696 if (!bpf_verifier_log_attr_valid(log)) { 16697 ret = -EINVAL; 16698 goto err_unlock; 16699 } 16700 } 16701 16702 mark_verifier_state_clean(env); 16703 16704 if (IS_ERR(btf_vmlinux)) { 16705 /* Either gcc or pahole or kernel are broken. */ 16706 verbose(env, "in-kernel BTF is malformed\n"); 16707 ret = PTR_ERR(btf_vmlinux); 16708 goto skip_full_check; 16709 } 16710 16711 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 16712 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 16713 env->strict_alignment = true; 16714 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 16715 env->strict_alignment = false; 16716 16717 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 16718 env->allow_uninit_stack = bpf_allow_uninit_stack(); 16719 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 16720 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 16721 env->bpf_capable = bpf_capable(); 16722 env->rcu_tag_supported = btf_vmlinux && 16723 btf_find_by_name_kind(btf_vmlinux, "rcu", BTF_KIND_TYPE_TAG) > 0; 16724 16725 if (is_priv) 16726 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 16727 16728 env->explored_states = kvcalloc(state_htab_size(env), 16729 sizeof(struct bpf_verifier_state_list *), 16730 GFP_USER); 16731 ret = -ENOMEM; 16732 if (!env->explored_states) 16733 goto skip_full_check; 16734 16735 ret = add_subprog_and_kfunc(env); 16736 if (ret < 0) 16737 goto skip_full_check; 16738 16739 ret = check_subprogs(env); 16740 if (ret < 0) 16741 goto skip_full_check; 16742 16743 ret = check_btf_info(env, attr, uattr); 16744 if (ret < 0) 16745 goto skip_full_check; 16746 16747 ret = check_attach_btf_id(env); 16748 if (ret) 16749 goto skip_full_check; 16750 16751 ret = resolve_pseudo_ldimm64(env); 16752 if (ret < 0) 16753 goto skip_full_check; 16754 16755 if (bpf_prog_is_dev_bound(env->prog->aux)) { 16756 ret = bpf_prog_offload_verifier_prep(env->prog); 16757 if (ret) 16758 goto skip_full_check; 16759 } 16760 16761 ret = check_cfg(env); 16762 if (ret < 0) 16763 goto skip_full_check; 16764 16765 ret = do_check_subprogs(env); 16766 ret = ret ?: do_check_main(env); 16767 16768 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 16769 ret = bpf_prog_offload_finalize(env); 16770 16771 skip_full_check: 16772 kvfree(env->explored_states); 16773 16774 if (ret == 0) 16775 ret = check_max_stack_depth(env); 16776 16777 /* instruction rewrites happen after this point */ 16778 if (ret == 0) 16779 ret = optimize_bpf_loop(env); 16780 16781 if (is_priv) { 16782 if (ret == 0) 16783 opt_hard_wire_dead_code_branches(env); 16784 if (ret == 0) 16785 ret = opt_remove_dead_code(env); 16786 if (ret == 0) 16787 ret = opt_remove_nops(env); 16788 } else { 16789 if (ret == 0) 16790 sanitize_dead_code(env); 16791 } 16792 16793 if (ret == 0) 16794 /* program is valid, convert *(u32*)(ctx + off) accesses */ 16795 ret = convert_ctx_accesses(env); 16796 16797 if (ret == 0) 16798 ret = do_misc_fixups(env); 16799 16800 /* do 32-bit optimization after insn patching has done so those patched 16801 * insns could be handled correctly. 16802 */ 16803 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 16804 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 16805 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 16806 : false; 16807 } 16808 16809 if (ret == 0) 16810 ret = fixup_call_args(env); 16811 16812 env->verification_time = ktime_get_ns() - start_time; 16813 print_verification_stats(env); 16814 env->prog->aux->verified_insns = env->insn_processed; 16815 16816 if (log->level && bpf_verifier_log_full(log)) 16817 ret = -ENOSPC; 16818 if (log->level && !log->ubuf) { 16819 ret = -EFAULT; 16820 goto err_release_maps; 16821 } 16822 16823 if (ret) 16824 goto err_release_maps; 16825 16826 if (env->used_map_cnt) { 16827 /* if program passed verifier, update used_maps in bpf_prog_info */ 16828 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 16829 sizeof(env->used_maps[0]), 16830 GFP_KERNEL); 16831 16832 if (!env->prog->aux->used_maps) { 16833 ret = -ENOMEM; 16834 goto err_release_maps; 16835 } 16836 16837 memcpy(env->prog->aux->used_maps, env->used_maps, 16838 sizeof(env->used_maps[0]) * env->used_map_cnt); 16839 env->prog->aux->used_map_cnt = env->used_map_cnt; 16840 } 16841 if (env->used_btf_cnt) { 16842 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 16843 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 16844 sizeof(env->used_btfs[0]), 16845 GFP_KERNEL); 16846 if (!env->prog->aux->used_btfs) { 16847 ret = -ENOMEM; 16848 goto err_release_maps; 16849 } 16850 16851 memcpy(env->prog->aux->used_btfs, env->used_btfs, 16852 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 16853 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 16854 } 16855 if (env->used_map_cnt || env->used_btf_cnt) { 16856 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 16857 * bpf_ld_imm64 instructions 16858 */ 16859 convert_pseudo_ld_imm64(env); 16860 } 16861 16862 adjust_btf_func(env); 16863 16864 err_release_maps: 16865 if (!env->prog->aux->used_maps) 16866 /* if we didn't copy map pointers into bpf_prog_info, release 16867 * them now. Otherwise free_used_maps() will release them. 16868 */ 16869 release_maps(env); 16870 if (!env->prog->aux->used_btfs) 16871 release_btfs(env); 16872 16873 /* extension progs temporarily inherit the attach_type of their targets 16874 for verification purposes, so set it back to zero before returning 16875 */ 16876 if (env->prog->type == BPF_PROG_TYPE_EXT) 16877 env->prog->expected_attach_type = 0; 16878 16879 *prog = env->prog; 16880 err_unlock: 16881 if (!is_priv) 16882 mutex_unlock(&bpf_verifier_lock); 16883 vfree(env->insn_aux_data); 16884 err_free_env: 16885 kfree(env); 16886 return ret; 16887 } 16888