1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/bpf-cgroup.h> 8 #include <linux/kernel.h> 9 #include <linux/types.h> 10 #include <linux/slab.h> 11 #include <linux/bpf.h> 12 #include <linux/btf.h> 13 #include <linux/bpf_verifier.h> 14 #include <linux/filter.h> 15 #include <net/netlink.h> 16 #include <linux/file.h> 17 #include <linux/vmalloc.h> 18 #include <linux/stringify.h> 19 #include <linux/bsearch.h> 20 #include <linux/sort.h> 21 #include <linux/perf_event.h> 22 #include <linux/ctype.h> 23 #include <linux/error-injection.h> 24 #include <linux/bpf_lsm.h> 25 #include <linux/btf_ids.h> 26 27 #include "disasm.h" 28 29 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 30 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 31 [_id] = & _name ## _verifier_ops, 32 #define BPF_MAP_TYPE(_id, _ops) 33 #define BPF_LINK_TYPE(_id, _name) 34 #include <linux/bpf_types.h> 35 #undef BPF_PROG_TYPE 36 #undef BPF_MAP_TYPE 37 #undef BPF_LINK_TYPE 38 }; 39 40 /* bpf_check() is a static code analyzer that walks eBPF program 41 * instruction by instruction and updates register/stack state. 42 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 43 * 44 * The first pass is depth-first-search to check that the program is a DAG. 45 * It rejects the following programs: 46 * - larger than BPF_MAXINSNS insns 47 * - if loop is present (detected via back-edge) 48 * - unreachable insns exist (shouldn't be a forest. program = one function) 49 * - out of bounds or malformed jumps 50 * The second pass is all possible path descent from the 1st insn. 51 * Since it's analyzing all paths through the program, the length of the 52 * analysis is limited to 64k insn, which may be hit even if total number of 53 * insn is less then 4K, but there are too many branches that change stack/regs. 54 * Number of 'branches to be analyzed' is limited to 1k 55 * 56 * On entry to each instruction, each register has a type, and the instruction 57 * changes the types of the registers depending on instruction semantics. 58 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 59 * copied to R1. 60 * 61 * All registers are 64-bit. 62 * R0 - return register 63 * R1-R5 argument passing registers 64 * R6-R9 callee saved registers 65 * R10 - frame pointer read-only 66 * 67 * At the start of BPF program the register R1 contains a pointer to bpf_context 68 * and has type PTR_TO_CTX. 69 * 70 * Verifier tracks arithmetic operations on pointers in case: 71 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 72 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 73 * 1st insn copies R10 (which has FRAME_PTR) type into R1 74 * and 2nd arithmetic instruction is pattern matched to recognize 75 * that it wants to construct a pointer to some element within stack. 76 * So after 2nd insn, the register R1 has type PTR_TO_STACK 77 * (and -20 constant is saved for further stack bounds checking). 78 * Meaning that this reg is a pointer to stack plus known immediate constant. 79 * 80 * Most of the time the registers have SCALAR_VALUE type, which 81 * means the register has some value, but it's not a valid pointer. 82 * (like pointer plus pointer becomes SCALAR_VALUE type) 83 * 84 * When verifier sees load or store instructions the type of base register 85 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 86 * four pointer types recognized by check_mem_access() function. 87 * 88 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 89 * and the range of [ptr, ptr + map's value_size) is accessible. 90 * 91 * registers used to pass values to function calls are checked against 92 * function argument constraints. 93 * 94 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 95 * It means that the register type passed to this function must be 96 * PTR_TO_STACK and it will be used inside the function as 97 * 'pointer to map element key' 98 * 99 * For example the argument constraints for bpf_map_lookup_elem(): 100 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 101 * .arg1_type = ARG_CONST_MAP_PTR, 102 * .arg2_type = ARG_PTR_TO_MAP_KEY, 103 * 104 * ret_type says that this function returns 'pointer to map elem value or null' 105 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 106 * 2nd argument should be a pointer to stack, which will be used inside 107 * the helper function as a pointer to map element key. 108 * 109 * On the kernel side the helper function looks like: 110 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 111 * { 112 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 113 * void *key = (void *) (unsigned long) r2; 114 * void *value; 115 * 116 * here kernel can access 'key' and 'map' pointers safely, knowing that 117 * [key, key + map->key_size) bytes are valid and were initialized on 118 * the stack of eBPF program. 119 * } 120 * 121 * Corresponding eBPF program may look like: 122 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 123 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 124 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 125 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 126 * here verifier looks at prototype of map_lookup_elem() and sees: 127 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 128 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 129 * 130 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 131 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 132 * and were initialized prior to this call. 133 * If it's ok, then verifier allows this BPF_CALL insn and looks at 134 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 135 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 136 * returns either pointer to map value or NULL. 137 * 138 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 139 * insn, the register holding that pointer in the true branch changes state to 140 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 141 * branch. See check_cond_jmp_op(). 142 * 143 * After the call R0 is set to return type of the function and registers R1-R5 144 * are set to NOT_INIT to indicate that they are no longer readable. 145 * 146 * The following reference types represent a potential reference to a kernel 147 * resource which, after first being allocated, must be checked and freed by 148 * the BPF program: 149 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 150 * 151 * When the verifier sees a helper call return a reference type, it allocates a 152 * pointer id for the reference and stores it in the current function state. 153 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 154 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 155 * passes through a NULL-check conditional. For the branch wherein the state is 156 * changed to CONST_IMM, the verifier releases the reference. 157 * 158 * For each helper function that allocates a reference, such as 159 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 160 * bpf_sk_release(). When a reference type passes into the release function, 161 * the verifier also releases the reference. If any unchecked or unreleased 162 * reference remains at the end of the program, the verifier rejects it. 163 */ 164 165 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 166 struct bpf_verifier_stack_elem { 167 /* verifer state is 'st' 168 * before processing instruction 'insn_idx' 169 * and after processing instruction 'prev_insn_idx' 170 */ 171 struct bpf_verifier_state st; 172 int insn_idx; 173 int prev_insn_idx; 174 struct bpf_verifier_stack_elem *next; 175 /* length of verifier log at the time this state was pushed on stack */ 176 u32 log_pos; 177 }; 178 179 #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 180 #define BPF_COMPLEXITY_LIMIT_STATES 64 181 182 #define BPF_MAP_KEY_POISON (1ULL << 63) 183 #define BPF_MAP_KEY_SEEN (1ULL << 62) 184 185 #define BPF_MAP_PTR_UNPRIV 1UL 186 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 187 POISON_POINTER_DELTA)) 188 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 189 190 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 191 { 192 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; 193 } 194 195 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 196 { 197 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; 198 } 199 200 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 201 const struct bpf_map *map, bool unpriv) 202 { 203 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 204 unpriv |= bpf_map_ptr_unpriv(aux); 205 aux->map_ptr_state = (unsigned long)map | 206 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 207 } 208 209 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) 210 { 211 return aux->map_key_state & BPF_MAP_KEY_POISON; 212 } 213 214 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) 215 { 216 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); 217 } 218 219 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) 220 { 221 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); 222 } 223 224 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) 225 { 226 bool poisoned = bpf_map_key_poisoned(aux); 227 228 aux->map_key_state = state | BPF_MAP_KEY_SEEN | 229 (poisoned ? BPF_MAP_KEY_POISON : 0ULL); 230 } 231 232 static bool bpf_pseudo_call(const struct bpf_insn *insn) 233 { 234 return insn->code == (BPF_JMP | BPF_CALL) && 235 insn->src_reg == BPF_PSEUDO_CALL; 236 } 237 238 static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) 239 { 240 return insn->code == (BPF_JMP | BPF_CALL) && 241 insn->src_reg == BPF_PSEUDO_KFUNC_CALL; 242 } 243 244 struct bpf_call_arg_meta { 245 struct bpf_map *map_ptr; 246 bool raw_mode; 247 bool pkt_access; 248 u8 release_regno; 249 int regno; 250 int access_size; 251 int mem_size; 252 u64 msize_max_value; 253 int ref_obj_id; 254 int map_uid; 255 int func_id; 256 struct btf *btf; 257 u32 btf_id; 258 struct btf *ret_btf; 259 u32 ret_btf_id; 260 u32 subprogno; 261 struct bpf_map_value_off_desc *kptr_off_desc; 262 }; 263 264 struct btf *btf_vmlinux; 265 266 static DEFINE_MUTEX(bpf_verifier_lock); 267 268 static const struct bpf_line_info * 269 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 270 { 271 const struct bpf_line_info *linfo; 272 const struct bpf_prog *prog; 273 u32 i, nr_linfo; 274 275 prog = env->prog; 276 nr_linfo = prog->aux->nr_linfo; 277 278 if (!nr_linfo || insn_off >= prog->len) 279 return NULL; 280 281 linfo = prog->aux->linfo; 282 for (i = 1; i < nr_linfo; i++) 283 if (insn_off < linfo[i].insn_off) 284 break; 285 286 return &linfo[i - 1]; 287 } 288 289 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 290 va_list args) 291 { 292 unsigned int n; 293 294 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 295 296 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 297 "verifier log line truncated - local buffer too short\n"); 298 299 if (log->level == BPF_LOG_KERNEL) { 300 bool newline = n > 0 && log->kbuf[n - 1] == '\n'; 301 302 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n"); 303 return; 304 } 305 306 n = min(log->len_total - log->len_used - 1, n); 307 log->kbuf[n] = '\0'; 308 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 309 log->len_used += n; 310 else 311 log->ubuf = NULL; 312 } 313 314 static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) 315 { 316 char zero = 0; 317 318 if (!bpf_verifier_log_needed(log)) 319 return; 320 321 log->len_used = new_pos; 322 if (put_user(zero, log->ubuf + new_pos)) 323 log->ubuf = NULL; 324 } 325 326 /* log_level controls verbosity level of eBPF verifier. 327 * bpf_verifier_log_write() is used to dump the verification trace to the log, 328 * so the user can figure out what's wrong with the program 329 */ 330 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 331 const char *fmt, ...) 332 { 333 va_list args; 334 335 if (!bpf_verifier_log_needed(&env->log)) 336 return; 337 338 va_start(args, fmt); 339 bpf_verifier_vlog(&env->log, fmt, args); 340 va_end(args); 341 } 342 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 343 344 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 345 { 346 struct bpf_verifier_env *env = private_data; 347 va_list args; 348 349 if (!bpf_verifier_log_needed(&env->log)) 350 return; 351 352 va_start(args, fmt); 353 bpf_verifier_vlog(&env->log, fmt, args); 354 va_end(args); 355 } 356 357 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, 358 const char *fmt, ...) 359 { 360 va_list args; 361 362 if (!bpf_verifier_log_needed(log)) 363 return; 364 365 va_start(args, fmt); 366 bpf_verifier_vlog(log, fmt, args); 367 va_end(args); 368 } 369 370 static const char *ltrim(const char *s) 371 { 372 while (isspace(*s)) 373 s++; 374 375 return s; 376 } 377 378 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 379 u32 insn_off, 380 const char *prefix_fmt, ...) 381 { 382 const struct bpf_line_info *linfo; 383 384 if (!bpf_verifier_log_needed(&env->log)) 385 return; 386 387 linfo = find_linfo(env, insn_off); 388 if (!linfo || linfo == env->prev_linfo) 389 return; 390 391 if (prefix_fmt) { 392 va_list args; 393 394 va_start(args, prefix_fmt); 395 bpf_verifier_vlog(&env->log, prefix_fmt, args); 396 va_end(args); 397 } 398 399 verbose(env, "%s\n", 400 ltrim(btf_name_by_offset(env->prog->aux->btf, 401 linfo->line_off))); 402 403 env->prev_linfo = linfo; 404 } 405 406 static void verbose_invalid_scalar(struct bpf_verifier_env *env, 407 struct bpf_reg_state *reg, 408 struct tnum *range, const char *ctx, 409 const char *reg_name) 410 { 411 char tn_buf[48]; 412 413 verbose(env, "At %s the register %s ", ctx, reg_name); 414 if (!tnum_is_unknown(reg->var_off)) { 415 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 416 verbose(env, "has value %s", tn_buf); 417 } else { 418 verbose(env, "has unknown scalar value"); 419 } 420 tnum_strn(tn_buf, sizeof(tn_buf), *range); 421 verbose(env, " should have been in %s\n", tn_buf); 422 } 423 424 static bool type_is_pkt_pointer(enum bpf_reg_type type) 425 { 426 return type == PTR_TO_PACKET || 427 type == PTR_TO_PACKET_META; 428 } 429 430 static bool type_is_sk_pointer(enum bpf_reg_type type) 431 { 432 return type == PTR_TO_SOCKET || 433 type == PTR_TO_SOCK_COMMON || 434 type == PTR_TO_TCP_SOCK || 435 type == PTR_TO_XDP_SOCK; 436 } 437 438 static bool reg_type_not_null(enum bpf_reg_type type) 439 { 440 return type == PTR_TO_SOCKET || 441 type == PTR_TO_TCP_SOCK || 442 type == PTR_TO_MAP_VALUE || 443 type == PTR_TO_MAP_KEY || 444 type == PTR_TO_SOCK_COMMON; 445 } 446 447 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 448 { 449 return reg->type == PTR_TO_MAP_VALUE && 450 map_value_has_spin_lock(reg->map_ptr); 451 } 452 453 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) 454 { 455 return base_type(type) == PTR_TO_SOCKET || 456 base_type(type) == PTR_TO_TCP_SOCK || 457 base_type(type) == PTR_TO_MEM || 458 base_type(type) == PTR_TO_BTF_ID; 459 } 460 461 static bool type_is_rdonly_mem(u32 type) 462 { 463 return type & MEM_RDONLY; 464 } 465 466 static bool arg_type_may_be_refcounted(enum bpf_arg_type type) 467 { 468 return type == ARG_PTR_TO_SOCK_COMMON; 469 } 470 471 static bool type_may_be_null(u32 type) 472 { 473 return type & PTR_MAYBE_NULL; 474 } 475 476 static bool may_be_acquire_function(enum bpf_func_id func_id) 477 { 478 return func_id == BPF_FUNC_sk_lookup_tcp || 479 func_id == BPF_FUNC_sk_lookup_udp || 480 func_id == BPF_FUNC_skc_lookup_tcp || 481 func_id == BPF_FUNC_map_lookup_elem || 482 func_id == BPF_FUNC_ringbuf_reserve; 483 } 484 485 static bool is_acquire_function(enum bpf_func_id func_id, 486 const struct bpf_map *map) 487 { 488 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; 489 490 if (func_id == BPF_FUNC_sk_lookup_tcp || 491 func_id == BPF_FUNC_sk_lookup_udp || 492 func_id == BPF_FUNC_skc_lookup_tcp || 493 func_id == BPF_FUNC_ringbuf_reserve || 494 func_id == BPF_FUNC_kptr_xchg) 495 return true; 496 497 if (func_id == BPF_FUNC_map_lookup_elem && 498 (map_type == BPF_MAP_TYPE_SOCKMAP || 499 map_type == BPF_MAP_TYPE_SOCKHASH)) 500 return true; 501 502 return false; 503 } 504 505 static bool is_ptr_cast_function(enum bpf_func_id func_id) 506 { 507 return func_id == BPF_FUNC_tcp_sock || 508 func_id == BPF_FUNC_sk_fullsock || 509 func_id == BPF_FUNC_skc_to_tcp_sock || 510 func_id == BPF_FUNC_skc_to_tcp6_sock || 511 func_id == BPF_FUNC_skc_to_udp6_sock || 512 func_id == BPF_FUNC_skc_to_tcp_timewait_sock || 513 func_id == BPF_FUNC_skc_to_tcp_request_sock; 514 } 515 516 static bool is_cmpxchg_insn(const struct bpf_insn *insn) 517 { 518 return BPF_CLASS(insn->code) == BPF_STX && 519 BPF_MODE(insn->code) == BPF_ATOMIC && 520 insn->imm == BPF_CMPXCHG; 521 } 522 523 /* string representation of 'enum bpf_reg_type' 524 * 525 * Note that reg_type_str() can not appear more than once in a single verbose() 526 * statement. 527 */ 528 static const char *reg_type_str(struct bpf_verifier_env *env, 529 enum bpf_reg_type type) 530 { 531 char postfix[16] = {0}, prefix[32] = {0}; 532 static const char * const str[] = { 533 [NOT_INIT] = "?", 534 [SCALAR_VALUE] = "scalar", 535 [PTR_TO_CTX] = "ctx", 536 [CONST_PTR_TO_MAP] = "map_ptr", 537 [PTR_TO_MAP_VALUE] = "map_value", 538 [PTR_TO_STACK] = "fp", 539 [PTR_TO_PACKET] = "pkt", 540 [PTR_TO_PACKET_META] = "pkt_meta", 541 [PTR_TO_PACKET_END] = "pkt_end", 542 [PTR_TO_FLOW_KEYS] = "flow_keys", 543 [PTR_TO_SOCKET] = "sock", 544 [PTR_TO_SOCK_COMMON] = "sock_common", 545 [PTR_TO_TCP_SOCK] = "tcp_sock", 546 [PTR_TO_TP_BUFFER] = "tp_buffer", 547 [PTR_TO_XDP_SOCK] = "xdp_sock", 548 [PTR_TO_BTF_ID] = "ptr_", 549 [PTR_TO_MEM] = "mem", 550 [PTR_TO_BUF] = "buf", 551 [PTR_TO_FUNC] = "func", 552 [PTR_TO_MAP_KEY] = "map_key", 553 }; 554 555 if (type & PTR_MAYBE_NULL) { 556 if (base_type(type) == PTR_TO_BTF_ID) 557 strncpy(postfix, "or_null_", 16); 558 else 559 strncpy(postfix, "_or_null", 16); 560 } 561 562 if (type & MEM_RDONLY) 563 strncpy(prefix, "rdonly_", 32); 564 if (type & MEM_ALLOC) 565 strncpy(prefix, "alloc_", 32); 566 if (type & MEM_USER) 567 strncpy(prefix, "user_", 32); 568 if (type & MEM_PERCPU) 569 strncpy(prefix, "percpu_", 32); 570 if (type & PTR_UNTRUSTED) 571 strncpy(prefix, "untrusted_", 32); 572 573 snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", 574 prefix, str[base_type(type)], postfix); 575 return env->type_str_buf; 576 } 577 578 static char slot_type_char[] = { 579 [STACK_INVALID] = '?', 580 [STACK_SPILL] = 'r', 581 [STACK_MISC] = 'm', 582 [STACK_ZERO] = '0', 583 }; 584 585 static void print_liveness(struct bpf_verifier_env *env, 586 enum bpf_reg_liveness live) 587 { 588 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 589 verbose(env, "_"); 590 if (live & REG_LIVE_READ) 591 verbose(env, "r"); 592 if (live & REG_LIVE_WRITTEN) 593 verbose(env, "w"); 594 if (live & REG_LIVE_DONE) 595 verbose(env, "D"); 596 } 597 598 static struct bpf_func_state *func(struct bpf_verifier_env *env, 599 const struct bpf_reg_state *reg) 600 { 601 struct bpf_verifier_state *cur = env->cur_state; 602 603 return cur->frame[reg->frameno]; 604 } 605 606 static const char *kernel_type_name(const struct btf* btf, u32 id) 607 { 608 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); 609 } 610 611 static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) 612 { 613 env->scratched_regs |= 1U << regno; 614 } 615 616 static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) 617 { 618 env->scratched_stack_slots |= 1ULL << spi; 619 } 620 621 static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) 622 { 623 return (env->scratched_regs >> regno) & 1; 624 } 625 626 static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) 627 { 628 return (env->scratched_stack_slots >> regno) & 1; 629 } 630 631 static bool verifier_state_scratched(const struct bpf_verifier_env *env) 632 { 633 return env->scratched_regs || env->scratched_stack_slots; 634 } 635 636 static void mark_verifier_state_clean(struct bpf_verifier_env *env) 637 { 638 env->scratched_regs = 0U; 639 env->scratched_stack_slots = 0ULL; 640 } 641 642 /* Used for printing the entire verifier state. */ 643 static void mark_verifier_state_scratched(struct bpf_verifier_env *env) 644 { 645 env->scratched_regs = ~0U; 646 env->scratched_stack_slots = ~0ULL; 647 } 648 649 /* The reg state of a pointer or a bounded scalar was saved when 650 * it was spilled to the stack. 651 */ 652 static bool is_spilled_reg(const struct bpf_stack_state *stack) 653 { 654 return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; 655 } 656 657 static void scrub_spilled_slot(u8 *stype) 658 { 659 if (*stype != STACK_INVALID) 660 *stype = STACK_MISC; 661 } 662 663 static void print_verifier_state(struct bpf_verifier_env *env, 664 const struct bpf_func_state *state, 665 bool print_all) 666 { 667 const struct bpf_reg_state *reg; 668 enum bpf_reg_type t; 669 int i; 670 671 if (state->frameno) 672 verbose(env, " frame%d:", state->frameno); 673 for (i = 0; i < MAX_BPF_REG; i++) { 674 reg = &state->regs[i]; 675 t = reg->type; 676 if (t == NOT_INIT) 677 continue; 678 if (!print_all && !reg_scratched(env, i)) 679 continue; 680 verbose(env, " R%d", i); 681 print_liveness(env, reg->live); 682 verbose(env, "="); 683 if (t == SCALAR_VALUE && reg->precise) 684 verbose(env, "P"); 685 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 686 tnum_is_const(reg->var_off)) { 687 /* reg->off should be 0 for SCALAR_VALUE */ 688 verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 689 verbose(env, "%lld", reg->var_off.value + reg->off); 690 } else { 691 const char *sep = ""; 692 693 verbose(env, "%s", reg_type_str(env, t)); 694 if (base_type(t) == PTR_TO_BTF_ID) 695 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); 696 verbose(env, "("); 697 /* 698 * _a stands for append, was shortened to avoid multiline statements below. 699 * This macro is used to output a comma separated list of attributes. 700 */ 701 #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; }) 702 703 if (reg->id) 704 verbose_a("id=%d", reg->id); 705 if (reg_type_may_be_refcounted_or_null(t) && reg->ref_obj_id) 706 verbose_a("ref_obj_id=%d", reg->ref_obj_id); 707 if (t != SCALAR_VALUE) 708 verbose_a("off=%d", reg->off); 709 if (type_is_pkt_pointer(t)) 710 verbose_a("r=%d", reg->range); 711 else if (base_type(t) == CONST_PTR_TO_MAP || 712 base_type(t) == PTR_TO_MAP_KEY || 713 base_type(t) == PTR_TO_MAP_VALUE) 714 verbose_a("ks=%d,vs=%d", 715 reg->map_ptr->key_size, 716 reg->map_ptr->value_size); 717 if (tnum_is_const(reg->var_off)) { 718 /* Typically an immediate SCALAR_VALUE, but 719 * could be a pointer whose offset is too big 720 * for reg->off 721 */ 722 verbose_a("imm=%llx", reg->var_off.value); 723 } else { 724 if (reg->smin_value != reg->umin_value && 725 reg->smin_value != S64_MIN) 726 verbose_a("smin=%lld", (long long)reg->smin_value); 727 if (reg->smax_value != reg->umax_value && 728 reg->smax_value != S64_MAX) 729 verbose_a("smax=%lld", (long long)reg->smax_value); 730 if (reg->umin_value != 0) 731 verbose_a("umin=%llu", (unsigned long long)reg->umin_value); 732 if (reg->umax_value != U64_MAX) 733 verbose_a("umax=%llu", (unsigned long long)reg->umax_value); 734 if (!tnum_is_unknown(reg->var_off)) { 735 char tn_buf[48]; 736 737 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 738 verbose_a("var_off=%s", tn_buf); 739 } 740 if (reg->s32_min_value != reg->smin_value && 741 reg->s32_min_value != S32_MIN) 742 verbose_a("s32_min=%d", (int)(reg->s32_min_value)); 743 if (reg->s32_max_value != reg->smax_value && 744 reg->s32_max_value != S32_MAX) 745 verbose_a("s32_max=%d", (int)(reg->s32_max_value)); 746 if (reg->u32_min_value != reg->umin_value && 747 reg->u32_min_value != U32_MIN) 748 verbose_a("u32_min=%d", (int)(reg->u32_min_value)); 749 if (reg->u32_max_value != reg->umax_value && 750 reg->u32_max_value != U32_MAX) 751 verbose_a("u32_max=%d", (int)(reg->u32_max_value)); 752 } 753 #undef verbose_a 754 755 verbose(env, ")"); 756 } 757 } 758 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 759 char types_buf[BPF_REG_SIZE + 1]; 760 bool valid = false; 761 int j; 762 763 for (j = 0; j < BPF_REG_SIZE; j++) { 764 if (state->stack[i].slot_type[j] != STACK_INVALID) 765 valid = true; 766 types_buf[j] = slot_type_char[ 767 state->stack[i].slot_type[j]]; 768 } 769 types_buf[BPF_REG_SIZE] = 0; 770 if (!valid) 771 continue; 772 if (!print_all && !stack_slot_scratched(env, i)) 773 continue; 774 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 775 print_liveness(env, state->stack[i].spilled_ptr.live); 776 if (is_spilled_reg(&state->stack[i])) { 777 reg = &state->stack[i].spilled_ptr; 778 t = reg->type; 779 verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t)); 780 if (t == SCALAR_VALUE && reg->precise) 781 verbose(env, "P"); 782 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) 783 verbose(env, "%lld", reg->var_off.value + reg->off); 784 } else { 785 verbose(env, "=%s", types_buf); 786 } 787 } 788 if (state->acquired_refs && state->refs[0].id) { 789 verbose(env, " refs=%d", state->refs[0].id); 790 for (i = 1; i < state->acquired_refs; i++) 791 if (state->refs[i].id) 792 verbose(env, ",%d", state->refs[i].id); 793 } 794 if (state->in_callback_fn) 795 verbose(env, " cb"); 796 if (state->in_async_callback_fn) 797 verbose(env, " async_cb"); 798 verbose(env, "\n"); 799 mark_verifier_state_clean(env); 800 } 801 802 static inline u32 vlog_alignment(u32 pos) 803 { 804 return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), 805 BPF_LOG_MIN_ALIGNMENT) - pos - 1; 806 } 807 808 static void print_insn_state(struct bpf_verifier_env *env, 809 const struct bpf_func_state *state) 810 { 811 if (env->prev_log_len && env->prev_log_len == env->log.len_used) { 812 /* remove new line character */ 813 bpf_vlog_reset(&env->log, env->prev_log_len - 1); 814 verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' '); 815 } else { 816 verbose(env, "%d:", env->insn_idx); 817 } 818 print_verifier_state(env, state, false); 819 } 820 821 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too 822 * small to hold src. This is different from krealloc since we don't want to preserve 823 * the contents of dst. 824 * 825 * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could 826 * not be allocated. 827 */ 828 static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags) 829 { 830 size_t bytes; 831 832 if (ZERO_OR_NULL_PTR(src)) 833 goto out; 834 835 if (unlikely(check_mul_overflow(n, size, &bytes))) 836 return NULL; 837 838 if (ksize(dst) < bytes) { 839 kfree(dst); 840 dst = kmalloc_track_caller(bytes, flags); 841 if (!dst) 842 return NULL; 843 } 844 845 memcpy(dst, src, bytes); 846 out: 847 return dst ? dst : ZERO_SIZE_PTR; 848 } 849 850 /* resize an array from old_n items to new_n items. the array is reallocated if it's too 851 * small to hold new_n items. new items are zeroed out if the array grows. 852 * 853 * Contrary to krealloc_array, does not free arr if new_n is zero. 854 */ 855 static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) 856 { 857 if (!new_n || old_n == new_n) 858 goto out; 859 860 arr = krealloc_array(arr, new_n, size, GFP_KERNEL); 861 if (!arr) 862 return NULL; 863 864 if (new_n > old_n) 865 memset(arr + old_n * size, 0, (new_n - old_n) * size); 866 867 out: 868 return arr ? arr : ZERO_SIZE_PTR; 869 } 870 871 static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 872 { 873 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs, 874 sizeof(struct bpf_reference_state), GFP_KERNEL); 875 if (!dst->refs) 876 return -ENOMEM; 877 878 dst->acquired_refs = src->acquired_refs; 879 return 0; 880 } 881 882 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src) 883 { 884 size_t n = src->allocated_stack / BPF_REG_SIZE; 885 886 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state), 887 GFP_KERNEL); 888 if (!dst->stack) 889 return -ENOMEM; 890 891 dst->allocated_stack = src->allocated_stack; 892 return 0; 893 } 894 895 static int resize_reference_state(struct bpf_func_state *state, size_t n) 896 { 897 state->refs = realloc_array(state->refs, state->acquired_refs, n, 898 sizeof(struct bpf_reference_state)); 899 if (!state->refs) 900 return -ENOMEM; 901 902 state->acquired_refs = n; 903 return 0; 904 } 905 906 static int grow_stack_state(struct bpf_func_state *state, int size) 907 { 908 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE; 909 910 if (old_n >= n) 911 return 0; 912 913 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state)); 914 if (!state->stack) 915 return -ENOMEM; 916 917 state->allocated_stack = size; 918 return 0; 919 } 920 921 /* Acquire a pointer id from the env and update the state->refs to include 922 * this new pointer reference. 923 * On success, returns a valid pointer id to associate with the register 924 * On failure, returns a negative errno. 925 */ 926 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 927 { 928 struct bpf_func_state *state = cur_func(env); 929 int new_ofs = state->acquired_refs; 930 int id, err; 931 932 err = resize_reference_state(state, state->acquired_refs + 1); 933 if (err) 934 return err; 935 id = ++env->id_gen; 936 state->refs[new_ofs].id = id; 937 state->refs[new_ofs].insn_idx = insn_idx; 938 939 return id; 940 } 941 942 /* release function corresponding to acquire_reference_state(). Idempotent. */ 943 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 944 { 945 int i, last_idx; 946 947 last_idx = state->acquired_refs - 1; 948 for (i = 0; i < state->acquired_refs; i++) { 949 if (state->refs[i].id == ptr_id) { 950 if (last_idx && i != last_idx) 951 memcpy(&state->refs[i], &state->refs[last_idx], 952 sizeof(*state->refs)); 953 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 954 state->acquired_refs--; 955 return 0; 956 } 957 } 958 return -EINVAL; 959 } 960 961 static void free_func_state(struct bpf_func_state *state) 962 { 963 if (!state) 964 return; 965 kfree(state->refs); 966 kfree(state->stack); 967 kfree(state); 968 } 969 970 static void clear_jmp_history(struct bpf_verifier_state *state) 971 { 972 kfree(state->jmp_history); 973 state->jmp_history = NULL; 974 state->jmp_history_cnt = 0; 975 } 976 977 static void free_verifier_state(struct bpf_verifier_state *state, 978 bool free_self) 979 { 980 int i; 981 982 for (i = 0; i <= state->curframe; i++) { 983 free_func_state(state->frame[i]); 984 state->frame[i] = NULL; 985 } 986 clear_jmp_history(state); 987 if (free_self) 988 kfree(state); 989 } 990 991 /* copy verifier state from src to dst growing dst stack space 992 * when necessary to accommodate larger src stack 993 */ 994 static int copy_func_state(struct bpf_func_state *dst, 995 const struct bpf_func_state *src) 996 { 997 int err; 998 999 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 1000 err = copy_reference_state(dst, src); 1001 if (err) 1002 return err; 1003 return copy_stack_state(dst, src); 1004 } 1005 1006 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 1007 const struct bpf_verifier_state *src) 1008 { 1009 struct bpf_func_state *dst; 1010 int i, err; 1011 1012 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history, 1013 src->jmp_history_cnt, sizeof(struct bpf_idx_pair), 1014 GFP_USER); 1015 if (!dst_state->jmp_history) 1016 return -ENOMEM; 1017 dst_state->jmp_history_cnt = src->jmp_history_cnt; 1018 1019 /* if dst has more stack frames then src frame, free them */ 1020 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 1021 free_func_state(dst_state->frame[i]); 1022 dst_state->frame[i] = NULL; 1023 } 1024 dst_state->speculative = src->speculative; 1025 dst_state->curframe = src->curframe; 1026 dst_state->active_spin_lock = src->active_spin_lock; 1027 dst_state->branches = src->branches; 1028 dst_state->parent = src->parent; 1029 dst_state->first_insn_idx = src->first_insn_idx; 1030 dst_state->last_insn_idx = src->last_insn_idx; 1031 for (i = 0; i <= src->curframe; i++) { 1032 dst = dst_state->frame[i]; 1033 if (!dst) { 1034 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1035 if (!dst) 1036 return -ENOMEM; 1037 dst_state->frame[i] = dst; 1038 } 1039 err = copy_func_state(dst, src->frame[i]); 1040 if (err) 1041 return err; 1042 } 1043 return 0; 1044 } 1045 1046 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) 1047 { 1048 while (st) { 1049 u32 br = --st->branches; 1050 1051 /* WARN_ON(br > 1) technically makes sense here, 1052 * but see comment in push_stack(), hence: 1053 */ 1054 WARN_ONCE((int)br < 0, 1055 "BUG update_branch_counts:branches_to_explore=%d\n", 1056 br); 1057 if (br) 1058 break; 1059 st = st->parent; 1060 } 1061 } 1062 1063 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 1064 int *insn_idx, bool pop_log) 1065 { 1066 struct bpf_verifier_state *cur = env->cur_state; 1067 struct bpf_verifier_stack_elem *elem, *head = env->head; 1068 int err; 1069 1070 if (env->head == NULL) 1071 return -ENOENT; 1072 1073 if (cur) { 1074 err = copy_verifier_state(cur, &head->st); 1075 if (err) 1076 return err; 1077 } 1078 if (pop_log) 1079 bpf_vlog_reset(&env->log, head->log_pos); 1080 if (insn_idx) 1081 *insn_idx = head->insn_idx; 1082 if (prev_insn_idx) 1083 *prev_insn_idx = head->prev_insn_idx; 1084 elem = head->next; 1085 free_verifier_state(&head->st, false); 1086 kfree(head); 1087 env->head = elem; 1088 env->stack_size--; 1089 return 0; 1090 } 1091 1092 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 1093 int insn_idx, int prev_insn_idx, 1094 bool speculative) 1095 { 1096 struct bpf_verifier_state *cur = env->cur_state; 1097 struct bpf_verifier_stack_elem *elem; 1098 int err; 1099 1100 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1101 if (!elem) 1102 goto err; 1103 1104 elem->insn_idx = insn_idx; 1105 elem->prev_insn_idx = prev_insn_idx; 1106 elem->next = env->head; 1107 elem->log_pos = env->log.len_used; 1108 env->head = elem; 1109 env->stack_size++; 1110 err = copy_verifier_state(&elem->st, cur); 1111 if (err) 1112 goto err; 1113 elem->st.speculative |= speculative; 1114 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1115 verbose(env, "The sequence of %d jumps is too complex.\n", 1116 env->stack_size); 1117 goto err; 1118 } 1119 if (elem->st.parent) { 1120 ++elem->st.parent->branches; 1121 /* WARN_ON(branches > 2) technically makes sense here, 1122 * but 1123 * 1. speculative states will bump 'branches' for non-branch 1124 * instructions 1125 * 2. is_state_visited() heuristics may decide not to create 1126 * a new state for a sequence of branches and all such current 1127 * and cloned states will be pointing to a single parent state 1128 * which might have large 'branches' count. 1129 */ 1130 } 1131 return &elem->st; 1132 err: 1133 free_verifier_state(env->cur_state, true); 1134 env->cur_state = NULL; 1135 /* pop all elements and return */ 1136 while (!pop_stack(env, NULL, NULL, false)); 1137 return NULL; 1138 } 1139 1140 #define CALLER_SAVED_REGS 6 1141 static const int caller_saved[CALLER_SAVED_REGS] = { 1142 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 1143 }; 1144 1145 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1146 struct bpf_reg_state *reg); 1147 1148 /* This helper doesn't clear reg->id */ 1149 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1150 { 1151 reg->var_off = tnum_const(imm); 1152 reg->smin_value = (s64)imm; 1153 reg->smax_value = (s64)imm; 1154 reg->umin_value = imm; 1155 reg->umax_value = imm; 1156 1157 reg->s32_min_value = (s32)imm; 1158 reg->s32_max_value = (s32)imm; 1159 reg->u32_min_value = (u32)imm; 1160 reg->u32_max_value = (u32)imm; 1161 } 1162 1163 /* Mark the unknown part of a register (variable offset or scalar value) as 1164 * known to have the value @imm. 1165 */ 1166 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 1167 { 1168 /* Clear id, off, and union(map_ptr, range) */ 1169 memset(((u8 *)reg) + sizeof(reg->type), 0, 1170 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 1171 ___mark_reg_known(reg, imm); 1172 } 1173 1174 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) 1175 { 1176 reg->var_off = tnum_const_subreg(reg->var_off, imm); 1177 reg->s32_min_value = (s32)imm; 1178 reg->s32_max_value = (s32)imm; 1179 reg->u32_min_value = (u32)imm; 1180 reg->u32_max_value = (u32)imm; 1181 } 1182 1183 /* Mark the 'variable offset' part of a register as zero. This should be 1184 * used only on registers holding a pointer type. 1185 */ 1186 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 1187 { 1188 __mark_reg_known(reg, 0); 1189 } 1190 1191 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 1192 { 1193 __mark_reg_known(reg, 0); 1194 reg->type = SCALAR_VALUE; 1195 } 1196 1197 static void mark_reg_known_zero(struct bpf_verifier_env *env, 1198 struct bpf_reg_state *regs, u32 regno) 1199 { 1200 if (WARN_ON(regno >= MAX_BPF_REG)) { 1201 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 1202 /* Something bad happened, let's kill all regs */ 1203 for (regno = 0; regno < MAX_BPF_REG; regno++) 1204 __mark_reg_not_init(env, regs + regno); 1205 return; 1206 } 1207 __mark_reg_known_zero(regs + regno); 1208 } 1209 1210 static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) 1211 { 1212 if (base_type(reg->type) == PTR_TO_MAP_VALUE) { 1213 const struct bpf_map *map = reg->map_ptr; 1214 1215 if (map->inner_map_meta) { 1216 reg->type = CONST_PTR_TO_MAP; 1217 reg->map_ptr = map->inner_map_meta; 1218 /* transfer reg's id which is unique for every map_lookup_elem 1219 * as UID of the inner map. 1220 */ 1221 if (map_value_has_timer(map->inner_map_meta)) 1222 reg->map_uid = reg->id; 1223 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { 1224 reg->type = PTR_TO_XDP_SOCK; 1225 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || 1226 map->map_type == BPF_MAP_TYPE_SOCKHASH) { 1227 reg->type = PTR_TO_SOCKET; 1228 } else { 1229 reg->type = PTR_TO_MAP_VALUE; 1230 } 1231 return; 1232 } 1233 1234 reg->type &= ~PTR_MAYBE_NULL; 1235 } 1236 1237 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 1238 { 1239 return type_is_pkt_pointer(reg->type); 1240 } 1241 1242 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 1243 { 1244 return reg_is_pkt_pointer(reg) || 1245 reg->type == PTR_TO_PACKET_END; 1246 } 1247 1248 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 1249 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 1250 enum bpf_reg_type which) 1251 { 1252 /* The register can already have a range from prior markings. 1253 * This is fine as long as it hasn't been advanced from its 1254 * origin. 1255 */ 1256 return reg->type == which && 1257 reg->id == 0 && 1258 reg->off == 0 && 1259 tnum_equals_const(reg->var_off, 0); 1260 } 1261 1262 /* Reset the min/max bounds of a register */ 1263 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 1264 { 1265 reg->smin_value = S64_MIN; 1266 reg->smax_value = S64_MAX; 1267 reg->umin_value = 0; 1268 reg->umax_value = U64_MAX; 1269 1270 reg->s32_min_value = S32_MIN; 1271 reg->s32_max_value = S32_MAX; 1272 reg->u32_min_value = 0; 1273 reg->u32_max_value = U32_MAX; 1274 } 1275 1276 static void __mark_reg64_unbounded(struct bpf_reg_state *reg) 1277 { 1278 reg->smin_value = S64_MIN; 1279 reg->smax_value = S64_MAX; 1280 reg->umin_value = 0; 1281 reg->umax_value = U64_MAX; 1282 } 1283 1284 static void __mark_reg32_unbounded(struct bpf_reg_state *reg) 1285 { 1286 reg->s32_min_value = S32_MIN; 1287 reg->s32_max_value = S32_MAX; 1288 reg->u32_min_value = 0; 1289 reg->u32_max_value = U32_MAX; 1290 } 1291 1292 static void __update_reg32_bounds(struct bpf_reg_state *reg) 1293 { 1294 struct tnum var32_off = tnum_subreg(reg->var_off); 1295 1296 /* min signed is max(sign bit) | min(other bits) */ 1297 reg->s32_min_value = max_t(s32, reg->s32_min_value, 1298 var32_off.value | (var32_off.mask & S32_MIN)); 1299 /* max signed is min(sign bit) | max(other bits) */ 1300 reg->s32_max_value = min_t(s32, reg->s32_max_value, 1301 var32_off.value | (var32_off.mask & S32_MAX)); 1302 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); 1303 reg->u32_max_value = min(reg->u32_max_value, 1304 (u32)(var32_off.value | var32_off.mask)); 1305 } 1306 1307 static void __update_reg64_bounds(struct bpf_reg_state *reg) 1308 { 1309 /* min signed is max(sign bit) | min(other bits) */ 1310 reg->smin_value = max_t(s64, reg->smin_value, 1311 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 1312 /* max signed is min(sign bit) | max(other bits) */ 1313 reg->smax_value = min_t(s64, reg->smax_value, 1314 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 1315 reg->umin_value = max(reg->umin_value, reg->var_off.value); 1316 reg->umax_value = min(reg->umax_value, 1317 reg->var_off.value | reg->var_off.mask); 1318 } 1319 1320 static void __update_reg_bounds(struct bpf_reg_state *reg) 1321 { 1322 __update_reg32_bounds(reg); 1323 __update_reg64_bounds(reg); 1324 } 1325 1326 /* Uses signed min/max values to inform unsigned, and vice-versa */ 1327 static void __reg32_deduce_bounds(struct bpf_reg_state *reg) 1328 { 1329 /* Learn sign from signed bounds. 1330 * If we cannot cross the sign boundary, then signed and unsigned bounds 1331 * are the same, so combine. This works even in the negative case, e.g. 1332 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1333 */ 1334 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { 1335 reg->s32_min_value = reg->u32_min_value = 1336 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1337 reg->s32_max_value = reg->u32_max_value = 1338 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1339 return; 1340 } 1341 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1342 * boundary, so we must be careful. 1343 */ 1344 if ((s32)reg->u32_max_value >= 0) { 1345 /* Positive. We can't learn anything from the smin, but smax 1346 * is positive, hence safe. 1347 */ 1348 reg->s32_min_value = reg->u32_min_value; 1349 reg->s32_max_value = reg->u32_max_value = 1350 min_t(u32, reg->s32_max_value, reg->u32_max_value); 1351 } else if ((s32)reg->u32_min_value < 0) { 1352 /* Negative. We can't learn anything from the smax, but smin 1353 * is negative, hence safe. 1354 */ 1355 reg->s32_min_value = reg->u32_min_value = 1356 max_t(u32, reg->s32_min_value, reg->u32_min_value); 1357 reg->s32_max_value = reg->u32_max_value; 1358 } 1359 } 1360 1361 static void __reg64_deduce_bounds(struct bpf_reg_state *reg) 1362 { 1363 /* Learn sign from signed bounds. 1364 * If we cannot cross the sign boundary, then signed and unsigned bounds 1365 * are the same, so combine. This works even in the negative case, e.g. 1366 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 1367 */ 1368 if (reg->smin_value >= 0 || reg->smax_value < 0) { 1369 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1370 reg->umin_value); 1371 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1372 reg->umax_value); 1373 return; 1374 } 1375 /* Learn sign from unsigned bounds. Signed bounds cross the sign 1376 * boundary, so we must be careful. 1377 */ 1378 if ((s64)reg->umax_value >= 0) { 1379 /* Positive. We can't learn anything from the smin, but smax 1380 * is positive, hence safe. 1381 */ 1382 reg->smin_value = reg->umin_value; 1383 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 1384 reg->umax_value); 1385 } else if ((s64)reg->umin_value < 0) { 1386 /* Negative. We can't learn anything from the smax, but smin 1387 * is negative, hence safe. 1388 */ 1389 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 1390 reg->umin_value); 1391 reg->smax_value = reg->umax_value; 1392 } 1393 } 1394 1395 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 1396 { 1397 __reg32_deduce_bounds(reg); 1398 __reg64_deduce_bounds(reg); 1399 } 1400 1401 /* Attempts to improve var_off based on unsigned min/max information */ 1402 static void __reg_bound_offset(struct bpf_reg_state *reg) 1403 { 1404 struct tnum var64_off = tnum_intersect(reg->var_off, 1405 tnum_range(reg->umin_value, 1406 reg->umax_value)); 1407 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), 1408 tnum_range(reg->u32_min_value, 1409 reg->u32_max_value)); 1410 1411 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); 1412 } 1413 1414 static bool __reg32_bound_s64(s32 a) 1415 { 1416 return a >= 0 && a <= S32_MAX; 1417 } 1418 1419 static void __reg_assign_32_into_64(struct bpf_reg_state *reg) 1420 { 1421 reg->umin_value = reg->u32_min_value; 1422 reg->umax_value = reg->u32_max_value; 1423 1424 /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must 1425 * be positive otherwise set to worse case bounds and refine later 1426 * from tnum. 1427 */ 1428 if (__reg32_bound_s64(reg->s32_min_value) && 1429 __reg32_bound_s64(reg->s32_max_value)) { 1430 reg->smin_value = reg->s32_min_value; 1431 reg->smax_value = reg->s32_max_value; 1432 } else { 1433 reg->smin_value = 0; 1434 reg->smax_value = U32_MAX; 1435 } 1436 } 1437 1438 static void __reg_combine_32_into_64(struct bpf_reg_state *reg) 1439 { 1440 /* special case when 64-bit register has upper 32-bit register 1441 * zeroed. Typically happens after zext or <<32, >>32 sequence 1442 * allowing us to use 32-bit bounds directly, 1443 */ 1444 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { 1445 __reg_assign_32_into_64(reg); 1446 } else { 1447 /* Otherwise the best we can do is push lower 32bit known and 1448 * unknown bits into register (var_off set from jmp logic) 1449 * then learn as much as possible from the 64-bit tnum 1450 * known and unknown bits. The previous smin/smax bounds are 1451 * invalid here because of jmp32 compare so mark them unknown 1452 * so they do not impact tnum bounds calculation. 1453 */ 1454 __mark_reg64_unbounded(reg); 1455 __update_reg_bounds(reg); 1456 } 1457 1458 /* Intersecting with the old var_off might have improved our bounds 1459 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1460 * then new var_off is (0; 0x7f...fc) which improves our umax. 1461 */ 1462 __reg_deduce_bounds(reg); 1463 __reg_bound_offset(reg); 1464 __update_reg_bounds(reg); 1465 } 1466 1467 static bool __reg64_bound_s32(s64 a) 1468 { 1469 return a >= S32_MIN && a <= S32_MAX; 1470 } 1471 1472 static bool __reg64_bound_u32(u64 a) 1473 { 1474 return a >= U32_MIN && a <= U32_MAX; 1475 } 1476 1477 static void __reg_combine_64_into_32(struct bpf_reg_state *reg) 1478 { 1479 __mark_reg32_unbounded(reg); 1480 1481 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { 1482 reg->s32_min_value = (s32)reg->smin_value; 1483 reg->s32_max_value = (s32)reg->smax_value; 1484 } 1485 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) { 1486 reg->u32_min_value = (u32)reg->umin_value; 1487 reg->u32_max_value = (u32)reg->umax_value; 1488 } 1489 1490 /* Intersecting with the old var_off might have improved our bounds 1491 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 1492 * then new var_off is (0; 0x7f...fc) which improves our umax. 1493 */ 1494 __reg_deduce_bounds(reg); 1495 __reg_bound_offset(reg); 1496 __update_reg_bounds(reg); 1497 } 1498 1499 /* Mark a register as having a completely unknown (scalar) value. */ 1500 static void __mark_reg_unknown(const struct bpf_verifier_env *env, 1501 struct bpf_reg_state *reg) 1502 { 1503 /* 1504 * Clear type, id, off, and union(map_ptr, range) and 1505 * padding between 'type' and union 1506 */ 1507 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 1508 reg->type = SCALAR_VALUE; 1509 reg->var_off = tnum_unknown; 1510 reg->frameno = 0; 1511 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; 1512 __mark_reg_unbounded(reg); 1513 } 1514 1515 static void mark_reg_unknown(struct bpf_verifier_env *env, 1516 struct bpf_reg_state *regs, u32 regno) 1517 { 1518 if (WARN_ON(regno >= MAX_BPF_REG)) { 1519 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 1520 /* Something bad happened, let's kill all regs except FP */ 1521 for (regno = 0; regno < BPF_REG_FP; regno++) 1522 __mark_reg_not_init(env, regs + regno); 1523 return; 1524 } 1525 __mark_reg_unknown(env, regs + regno); 1526 } 1527 1528 static void __mark_reg_not_init(const struct bpf_verifier_env *env, 1529 struct bpf_reg_state *reg) 1530 { 1531 __mark_reg_unknown(env, reg); 1532 reg->type = NOT_INIT; 1533 } 1534 1535 static void mark_reg_not_init(struct bpf_verifier_env *env, 1536 struct bpf_reg_state *regs, u32 regno) 1537 { 1538 if (WARN_ON(regno >= MAX_BPF_REG)) { 1539 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 1540 /* Something bad happened, let's kill all regs except FP */ 1541 for (regno = 0; regno < BPF_REG_FP; regno++) 1542 __mark_reg_not_init(env, regs + regno); 1543 return; 1544 } 1545 __mark_reg_not_init(env, regs + regno); 1546 } 1547 1548 static void mark_btf_ld_reg(struct bpf_verifier_env *env, 1549 struct bpf_reg_state *regs, u32 regno, 1550 enum bpf_reg_type reg_type, 1551 struct btf *btf, u32 btf_id, 1552 enum bpf_type_flag flag) 1553 { 1554 if (reg_type == SCALAR_VALUE) { 1555 mark_reg_unknown(env, regs, regno); 1556 return; 1557 } 1558 mark_reg_known_zero(env, regs, regno); 1559 regs[regno].type = PTR_TO_BTF_ID | flag; 1560 regs[regno].btf = btf; 1561 regs[regno].btf_id = btf_id; 1562 } 1563 1564 #define DEF_NOT_SUBREG (0) 1565 static void init_reg_state(struct bpf_verifier_env *env, 1566 struct bpf_func_state *state) 1567 { 1568 struct bpf_reg_state *regs = state->regs; 1569 int i; 1570 1571 for (i = 0; i < MAX_BPF_REG; i++) { 1572 mark_reg_not_init(env, regs, i); 1573 regs[i].live = REG_LIVE_NONE; 1574 regs[i].parent = NULL; 1575 regs[i].subreg_def = DEF_NOT_SUBREG; 1576 } 1577 1578 /* frame pointer */ 1579 regs[BPF_REG_FP].type = PTR_TO_STACK; 1580 mark_reg_known_zero(env, regs, BPF_REG_FP); 1581 regs[BPF_REG_FP].frameno = state->frameno; 1582 } 1583 1584 #define BPF_MAIN_FUNC (-1) 1585 static void init_func_state(struct bpf_verifier_env *env, 1586 struct bpf_func_state *state, 1587 int callsite, int frameno, int subprogno) 1588 { 1589 state->callsite = callsite; 1590 state->frameno = frameno; 1591 state->subprogno = subprogno; 1592 init_reg_state(env, state); 1593 mark_verifier_state_scratched(env); 1594 } 1595 1596 /* Similar to push_stack(), but for async callbacks */ 1597 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, 1598 int insn_idx, int prev_insn_idx, 1599 int subprog) 1600 { 1601 struct bpf_verifier_stack_elem *elem; 1602 struct bpf_func_state *frame; 1603 1604 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 1605 if (!elem) 1606 goto err; 1607 1608 elem->insn_idx = insn_idx; 1609 elem->prev_insn_idx = prev_insn_idx; 1610 elem->next = env->head; 1611 elem->log_pos = env->log.len_used; 1612 env->head = elem; 1613 env->stack_size++; 1614 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { 1615 verbose(env, 1616 "The sequence of %d jumps is too complex for async cb.\n", 1617 env->stack_size); 1618 goto err; 1619 } 1620 /* Unlike push_stack() do not copy_verifier_state(). 1621 * The caller state doesn't matter. 1622 * This is async callback. It starts in a fresh stack. 1623 * Initialize it similar to do_check_common(). 1624 */ 1625 elem->st.branches = 1; 1626 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 1627 if (!frame) 1628 goto err; 1629 init_func_state(env, frame, 1630 BPF_MAIN_FUNC /* callsite */, 1631 0 /* frameno within this callchain */, 1632 subprog /* subprog number within this prog */); 1633 elem->st.frame[0] = frame; 1634 return &elem->st; 1635 err: 1636 free_verifier_state(env->cur_state, true); 1637 env->cur_state = NULL; 1638 /* pop all elements and return */ 1639 while (!pop_stack(env, NULL, NULL, false)); 1640 return NULL; 1641 } 1642 1643 1644 enum reg_arg_type { 1645 SRC_OP, /* register is used as source operand */ 1646 DST_OP, /* register is used as destination operand */ 1647 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1648 }; 1649 1650 static int cmp_subprogs(const void *a, const void *b) 1651 { 1652 return ((struct bpf_subprog_info *)a)->start - 1653 ((struct bpf_subprog_info *)b)->start; 1654 } 1655 1656 static int find_subprog(struct bpf_verifier_env *env, int off) 1657 { 1658 struct bpf_subprog_info *p; 1659 1660 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1661 sizeof(env->subprog_info[0]), cmp_subprogs); 1662 if (!p) 1663 return -ENOENT; 1664 return p - env->subprog_info; 1665 1666 } 1667 1668 static int add_subprog(struct bpf_verifier_env *env, int off) 1669 { 1670 int insn_cnt = env->prog->len; 1671 int ret; 1672 1673 if (off >= insn_cnt || off < 0) { 1674 verbose(env, "call to invalid destination\n"); 1675 return -EINVAL; 1676 } 1677 ret = find_subprog(env, off); 1678 if (ret >= 0) 1679 return ret; 1680 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1681 verbose(env, "too many subprograms\n"); 1682 return -E2BIG; 1683 } 1684 /* determine subprog starts. The end is one before the next starts */ 1685 env->subprog_info[env->subprog_cnt++].start = off; 1686 sort(env->subprog_info, env->subprog_cnt, 1687 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1688 return env->subprog_cnt - 1; 1689 } 1690 1691 #define MAX_KFUNC_DESCS 256 1692 #define MAX_KFUNC_BTFS 256 1693 1694 struct bpf_kfunc_desc { 1695 struct btf_func_model func_model; 1696 u32 func_id; 1697 s32 imm; 1698 u16 offset; 1699 }; 1700 1701 struct bpf_kfunc_btf { 1702 struct btf *btf; 1703 struct module *module; 1704 u16 offset; 1705 }; 1706 1707 struct bpf_kfunc_desc_tab { 1708 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; 1709 u32 nr_descs; 1710 }; 1711 1712 struct bpf_kfunc_btf_tab { 1713 struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS]; 1714 u32 nr_descs; 1715 }; 1716 1717 static int kfunc_desc_cmp_by_id_off(const void *a, const void *b) 1718 { 1719 const struct bpf_kfunc_desc *d0 = a; 1720 const struct bpf_kfunc_desc *d1 = b; 1721 1722 /* func_id is not greater than BTF_MAX_TYPE */ 1723 return d0->func_id - d1->func_id ?: d0->offset - d1->offset; 1724 } 1725 1726 static int kfunc_btf_cmp_by_off(const void *a, const void *b) 1727 { 1728 const struct bpf_kfunc_btf *d0 = a; 1729 const struct bpf_kfunc_btf *d1 = b; 1730 1731 return d0->offset - d1->offset; 1732 } 1733 1734 static const struct bpf_kfunc_desc * 1735 find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset) 1736 { 1737 struct bpf_kfunc_desc desc = { 1738 .func_id = func_id, 1739 .offset = offset, 1740 }; 1741 struct bpf_kfunc_desc_tab *tab; 1742 1743 tab = prog->aux->kfunc_tab; 1744 return bsearch(&desc, tab->descs, tab->nr_descs, 1745 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off); 1746 } 1747 1748 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, 1749 s16 offset) 1750 { 1751 struct bpf_kfunc_btf kf_btf = { .offset = offset }; 1752 struct bpf_kfunc_btf_tab *tab; 1753 struct bpf_kfunc_btf *b; 1754 struct module *mod; 1755 struct btf *btf; 1756 int btf_fd; 1757 1758 tab = env->prog->aux->kfunc_btf_tab; 1759 b = bsearch(&kf_btf, tab->descs, tab->nr_descs, 1760 sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); 1761 if (!b) { 1762 if (tab->nr_descs == MAX_KFUNC_BTFS) { 1763 verbose(env, "too many different module BTFs\n"); 1764 return ERR_PTR(-E2BIG); 1765 } 1766 1767 if (bpfptr_is_null(env->fd_array)) { 1768 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); 1769 return ERR_PTR(-EPROTO); 1770 } 1771 1772 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, 1773 offset * sizeof(btf_fd), 1774 sizeof(btf_fd))) 1775 return ERR_PTR(-EFAULT); 1776 1777 btf = btf_get_by_fd(btf_fd); 1778 if (IS_ERR(btf)) { 1779 verbose(env, "invalid module BTF fd specified\n"); 1780 return btf; 1781 } 1782 1783 if (!btf_is_module(btf)) { 1784 verbose(env, "BTF fd for kfunc is not a module BTF\n"); 1785 btf_put(btf); 1786 return ERR_PTR(-EINVAL); 1787 } 1788 1789 mod = btf_try_get_module(btf); 1790 if (!mod) { 1791 btf_put(btf); 1792 return ERR_PTR(-ENXIO); 1793 } 1794 1795 b = &tab->descs[tab->nr_descs++]; 1796 b->btf = btf; 1797 b->module = mod; 1798 b->offset = offset; 1799 1800 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1801 kfunc_btf_cmp_by_off, NULL); 1802 } 1803 return b->btf; 1804 } 1805 1806 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab) 1807 { 1808 if (!tab) 1809 return; 1810 1811 while (tab->nr_descs--) { 1812 module_put(tab->descs[tab->nr_descs].module); 1813 btf_put(tab->descs[tab->nr_descs].btf); 1814 } 1815 kfree(tab); 1816 } 1817 1818 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, 1819 u32 func_id, s16 offset) 1820 { 1821 if (offset) { 1822 if (offset < 0) { 1823 /* In the future, this can be allowed to increase limit 1824 * of fd index into fd_array, interpreted as u16. 1825 */ 1826 verbose(env, "negative offset disallowed for kernel module function call\n"); 1827 return ERR_PTR(-EINVAL); 1828 } 1829 1830 return __find_kfunc_desc_btf(env, offset); 1831 } 1832 return btf_vmlinux ?: ERR_PTR(-ENOENT); 1833 } 1834 1835 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) 1836 { 1837 const struct btf_type *func, *func_proto; 1838 struct bpf_kfunc_btf_tab *btf_tab; 1839 struct bpf_kfunc_desc_tab *tab; 1840 struct bpf_prog_aux *prog_aux; 1841 struct bpf_kfunc_desc *desc; 1842 const char *func_name; 1843 struct btf *desc_btf; 1844 unsigned long call_imm; 1845 unsigned long addr; 1846 int err; 1847 1848 prog_aux = env->prog->aux; 1849 tab = prog_aux->kfunc_tab; 1850 btf_tab = prog_aux->kfunc_btf_tab; 1851 if (!tab) { 1852 if (!btf_vmlinux) { 1853 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); 1854 return -ENOTSUPP; 1855 } 1856 1857 if (!env->prog->jit_requested) { 1858 verbose(env, "JIT is required for calling kernel function\n"); 1859 return -ENOTSUPP; 1860 } 1861 1862 if (!bpf_jit_supports_kfunc_call()) { 1863 verbose(env, "JIT does not support calling kernel function\n"); 1864 return -ENOTSUPP; 1865 } 1866 1867 if (!env->prog->gpl_compatible) { 1868 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); 1869 return -EINVAL; 1870 } 1871 1872 tab = kzalloc(sizeof(*tab), GFP_KERNEL); 1873 if (!tab) 1874 return -ENOMEM; 1875 prog_aux->kfunc_tab = tab; 1876 } 1877 1878 /* func_id == 0 is always invalid, but instead of returning an error, be 1879 * conservative and wait until the code elimination pass before returning 1880 * error, so that invalid calls that get pruned out can be in BPF programs 1881 * loaded from userspace. It is also required that offset be untouched 1882 * for such calls. 1883 */ 1884 if (!func_id && !offset) 1885 return 0; 1886 1887 if (!btf_tab && offset) { 1888 btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL); 1889 if (!btf_tab) 1890 return -ENOMEM; 1891 prog_aux->kfunc_btf_tab = btf_tab; 1892 } 1893 1894 desc_btf = find_kfunc_desc_btf(env, func_id, offset); 1895 if (IS_ERR(desc_btf)) { 1896 verbose(env, "failed to find BTF for kernel function\n"); 1897 return PTR_ERR(desc_btf); 1898 } 1899 1900 if (find_kfunc_desc(env->prog, func_id, offset)) 1901 return 0; 1902 1903 if (tab->nr_descs == MAX_KFUNC_DESCS) { 1904 verbose(env, "too many different kernel function calls\n"); 1905 return -E2BIG; 1906 } 1907 1908 func = btf_type_by_id(desc_btf, func_id); 1909 if (!func || !btf_type_is_func(func)) { 1910 verbose(env, "kernel btf_id %u is not a function\n", 1911 func_id); 1912 return -EINVAL; 1913 } 1914 func_proto = btf_type_by_id(desc_btf, func->type); 1915 if (!func_proto || !btf_type_is_func_proto(func_proto)) { 1916 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", 1917 func_id); 1918 return -EINVAL; 1919 } 1920 1921 func_name = btf_name_by_offset(desc_btf, func->name_off); 1922 addr = kallsyms_lookup_name(func_name); 1923 if (!addr) { 1924 verbose(env, "cannot find address for kernel function %s\n", 1925 func_name); 1926 return -EINVAL; 1927 } 1928 1929 call_imm = BPF_CALL_IMM(addr); 1930 /* Check whether or not the relative offset overflows desc->imm */ 1931 if ((unsigned long)(s32)call_imm != call_imm) { 1932 verbose(env, "address of kernel function %s is out of range\n", 1933 func_name); 1934 return -EINVAL; 1935 } 1936 1937 desc = &tab->descs[tab->nr_descs++]; 1938 desc->func_id = func_id; 1939 desc->imm = call_imm; 1940 desc->offset = offset; 1941 err = btf_distill_func_proto(&env->log, desc_btf, 1942 func_proto, func_name, 1943 &desc->func_model); 1944 if (!err) 1945 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1946 kfunc_desc_cmp_by_id_off, NULL); 1947 return err; 1948 } 1949 1950 static int kfunc_desc_cmp_by_imm(const void *a, const void *b) 1951 { 1952 const struct bpf_kfunc_desc *d0 = a; 1953 const struct bpf_kfunc_desc *d1 = b; 1954 1955 if (d0->imm > d1->imm) 1956 return 1; 1957 else if (d0->imm < d1->imm) 1958 return -1; 1959 return 0; 1960 } 1961 1962 static void sort_kfunc_descs_by_imm(struct bpf_prog *prog) 1963 { 1964 struct bpf_kfunc_desc_tab *tab; 1965 1966 tab = prog->aux->kfunc_tab; 1967 if (!tab) 1968 return; 1969 1970 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), 1971 kfunc_desc_cmp_by_imm, NULL); 1972 } 1973 1974 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 1975 { 1976 return !!prog->aux->kfunc_tab; 1977 } 1978 1979 const struct btf_func_model * 1980 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 1981 const struct bpf_insn *insn) 1982 { 1983 const struct bpf_kfunc_desc desc = { 1984 .imm = insn->imm, 1985 }; 1986 const struct bpf_kfunc_desc *res; 1987 struct bpf_kfunc_desc_tab *tab; 1988 1989 tab = prog->aux->kfunc_tab; 1990 res = bsearch(&desc, tab->descs, tab->nr_descs, 1991 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm); 1992 1993 return res ? &res->func_model : NULL; 1994 } 1995 1996 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) 1997 { 1998 struct bpf_subprog_info *subprog = env->subprog_info; 1999 struct bpf_insn *insn = env->prog->insnsi; 2000 int i, ret, insn_cnt = env->prog->len; 2001 2002 /* Add entry function. */ 2003 ret = add_subprog(env, 0); 2004 if (ret) 2005 return ret; 2006 2007 for (i = 0; i < insn_cnt; i++, insn++) { 2008 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && 2009 !bpf_pseudo_kfunc_call(insn)) 2010 continue; 2011 2012 if (!env->bpf_capable) { 2013 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); 2014 return -EPERM; 2015 } 2016 2017 if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) 2018 ret = add_subprog(env, i + insn->imm + 1); 2019 else 2020 ret = add_kfunc_call(env, insn->imm, insn->off); 2021 2022 if (ret < 0) 2023 return ret; 2024 } 2025 2026 /* Add a fake 'exit' subprog which could simplify subprog iteration 2027 * logic. 'subprog_cnt' should not be increased. 2028 */ 2029 subprog[env->subprog_cnt].start = insn_cnt; 2030 2031 if (env->log.level & BPF_LOG_LEVEL2) 2032 for (i = 0; i < env->subprog_cnt; i++) 2033 verbose(env, "func#%d @%d\n", i, subprog[i].start); 2034 2035 return 0; 2036 } 2037 2038 static int check_subprogs(struct bpf_verifier_env *env) 2039 { 2040 int i, subprog_start, subprog_end, off, cur_subprog = 0; 2041 struct bpf_subprog_info *subprog = env->subprog_info; 2042 struct bpf_insn *insn = env->prog->insnsi; 2043 int insn_cnt = env->prog->len; 2044 2045 /* now check that all jumps are within the same subprog */ 2046 subprog_start = subprog[cur_subprog].start; 2047 subprog_end = subprog[cur_subprog + 1].start; 2048 for (i = 0; i < insn_cnt; i++) { 2049 u8 code = insn[i].code; 2050 2051 if (code == (BPF_JMP | BPF_CALL) && 2052 insn[i].imm == BPF_FUNC_tail_call && 2053 insn[i].src_reg != BPF_PSEUDO_CALL) 2054 subprog[cur_subprog].has_tail_call = true; 2055 if (BPF_CLASS(code) == BPF_LD && 2056 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) 2057 subprog[cur_subprog].has_ld_abs = true; 2058 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 2059 goto next; 2060 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 2061 goto next; 2062 off = i + insn[i].off + 1; 2063 if (off < subprog_start || off >= subprog_end) { 2064 verbose(env, "jump out of range from insn %d to %d\n", i, off); 2065 return -EINVAL; 2066 } 2067 next: 2068 if (i == subprog_end - 1) { 2069 /* to avoid fall-through from one subprog into another 2070 * the last insn of the subprog should be either exit 2071 * or unconditional jump back 2072 */ 2073 if (code != (BPF_JMP | BPF_EXIT) && 2074 code != (BPF_JMP | BPF_JA)) { 2075 verbose(env, "last insn is not an exit or jmp\n"); 2076 return -EINVAL; 2077 } 2078 subprog_start = subprog_end; 2079 cur_subprog++; 2080 if (cur_subprog < env->subprog_cnt) 2081 subprog_end = subprog[cur_subprog + 1].start; 2082 } 2083 } 2084 return 0; 2085 } 2086 2087 /* Parentage chain of this register (or stack slot) should take care of all 2088 * issues like callee-saved registers, stack slot allocation time, etc. 2089 */ 2090 static int mark_reg_read(struct bpf_verifier_env *env, 2091 const struct bpf_reg_state *state, 2092 struct bpf_reg_state *parent, u8 flag) 2093 { 2094 bool writes = parent == state->parent; /* Observe write marks */ 2095 int cnt = 0; 2096 2097 while (parent) { 2098 /* if read wasn't screened by an earlier write ... */ 2099 if (writes && state->live & REG_LIVE_WRITTEN) 2100 break; 2101 if (parent->live & REG_LIVE_DONE) { 2102 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 2103 reg_type_str(env, parent->type), 2104 parent->var_off.value, parent->off); 2105 return -EFAULT; 2106 } 2107 /* The first condition is more likely to be true than the 2108 * second, checked it first. 2109 */ 2110 if ((parent->live & REG_LIVE_READ) == flag || 2111 parent->live & REG_LIVE_READ64) 2112 /* The parentage chain never changes and 2113 * this parent was already marked as LIVE_READ. 2114 * There is no need to keep walking the chain again and 2115 * keep re-marking all parents as LIVE_READ. 2116 * This case happens when the same register is read 2117 * multiple times without writes into it in-between. 2118 * Also, if parent has the stronger REG_LIVE_READ64 set, 2119 * then no need to set the weak REG_LIVE_READ32. 2120 */ 2121 break; 2122 /* ... then we depend on parent's value */ 2123 parent->live |= flag; 2124 /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ 2125 if (flag == REG_LIVE_READ64) 2126 parent->live &= ~REG_LIVE_READ32; 2127 state = parent; 2128 parent = state->parent; 2129 writes = true; 2130 cnt++; 2131 } 2132 2133 if (env->longest_mark_read_walk < cnt) 2134 env->longest_mark_read_walk = cnt; 2135 return 0; 2136 } 2137 2138 /* This function is supposed to be used by the following 32-bit optimization 2139 * code only. It returns TRUE if the source or destination register operates 2140 * on 64-bit, otherwise return FALSE. 2141 */ 2142 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, 2143 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) 2144 { 2145 u8 code, class, op; 2146 2147 code = insn->code; 2148 class = BPF_CLASS(code); 2149 op = BPF_OP(code); 2150 if (class == BPF_JMP) { 2151 /* BPF_EXIT for "main" will reach here. Return TRUE 2152 * conservatively. 2153 */ 2154 if (op == BPF_EXIT) 2155 return true; 2156 if (op == BPF_CALL) { 2157 /* BPF to BPF call will reach here because of marking 2158 * caller saved clobber with DST_OP_NO_MARK for which we 2159 * don't care the register def because they are anyway 2160 * marked as NOT_INIT already. 2161 */ 2162 if (insn->src_reg == BPF_PSEUDO_CALL) 2163 return false; 2164 /* Helper call will reach here because of arg type 2165 * check, conservatively return TRUE. 2166 */ 2167 if (t == SRC_OP) 2168 return true; 2169 2170 return false; 2171 } 2172 } 2173 2174 if (class == BPF_ALU64 || class == BPF_JMP || 2175 /* BPF_END always use BPF_ALU class. */ 2176 (class == BPF_ALU && op == BPF_END && insn->imm == 64)) 2177 return true; 2178 2179 if (class == BPF_ALU || class == BPF_JMP32) 2180 return false; 2181 2182 if (class == BPF_LDX) { 2183 if (t != SRC_OP) 2184 return BPF_SIZE(code) == BPF_DW; 2185 /* LDX source must be ptr. */ 2186 return true; 2187 } 2188 2189 if (class == BPF_STX) { 2190 /* BPF_STX (including atomic variants) has multiple source 2191 * operands, one of which is a ptr. Check whether the caller is 2192 * asking about it. 2193 */ 2194 if (t == SRC_OP && reg->type != SCALAR_VALUE) 2195 return true; 2196 return BPF_SIZE(code) == BPF_DW; 2197 } 2198 2199 if (class == BPF_LD) { 2200 u8 mode = BPF_MODE(code); 2201 2202 /* LD_IMM64 */ 2203 if (mode == BPF_IMM) 2204 return true; 2205 2206 /* Both LD_IND and LD_ABS return 32-bit data. */ 2207 if (t != SRC_OP) 2208 return false; 2209 2210 /* Implicit ctx ptr. */ 2211 if (regno == BPF_REG_6) 2212 return true; 2213 2214 /* Explicit source could be any width. */ 2215 return true; 2216 } 2217 2218 if (class == BPF_ST) 2219 /* The only source register for BPF_ST is a ptr. */ 2220 return true; 2221 2222 /* Conservatively return true at default. */ 2223 return true; 2224 } 2225 2226 /* Return the regno defined by the insn, or -1. */ 2227 static int insn_def_regno(const struct bpf_insn *insn) 2228 { 2229 switch (BPF_CLASS(insn->code)) { 2230 case BPF_JMP: 2231 case BPF_JMP32: 2232 case BPF_ST: 2233 return -1; 2234 case BPF_STX: 2235 if (BPF_MODE(insn->code) == BPF_ATOMIC && 2236 (insn->imm & BPF_FETCH)) { 2237 if (insn->imm == BPF_CMPXCHG) 2238 return BPF_REG_0; 2239 else 2240 return insn->src_reg; 2241 } else { 2242 return -1; 2243 } 2244 default: 2245 return insn->dst_reg; 2246 } 2247 } 2248 2249 /* Return TRUE if INSN has defined any 32-bit value explicitly. */ 2250 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) 2251 { 2252 int dst_reg = insn_def_regno(insn); 2253 2254 if (dst_reg == -1) 2255 return false; 2256 2257 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); 2258 } 2259 2260 static void mark_insn_zext(struct bpf_verifier_env *env, 2261 struct bpf_reg_state *reg) 2262 { 2263 s32 def_idx = reg->subreg_def; 2264 2265 if (def_idx == DEF_NOT_SUBREG) 2266 return; 2267 2268 env->insn_aux_data[def_idx - 1].zext_dst = true; 2269 /* The dst will be zero extended, so won't be sub-register anymore. */ 2270 reg->subreg_def = DEF_NOT_SUBREG; 2271 } 2272 2273 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 2274 enum reg_arg_type t) 2275 { 2276 struct bpf_verifier_state *vstate = env->cur_state; 2277 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 2278 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; 2279 struct bpf_reg_state *reg, *regs = state->regs; 2280 bool rw64; 2281 2282 if (regno >= MAX_BPF_REG) { 2283 verbose(env, "R%d is invalid\n", regno); 2284 return -EINVAL; 2285 } 2286 2287 mark_reg_scratched(env, regno); 2288 2289 reg = ®s[regno]; 2290 rw64 = is_reg64(env, insn, regno, reg, t); 2291 if (t == SRC_OP) { 2292 /* check whether register used as source operand can be read */ 2293 if (reg->type == NOT_INIT) { 2294 verbose(env, "R%d !read_ok\n", regno); 2295 return -EACCES; 2296 } 2297 /* We don't need to worry about FP liveness because it's read-only */ 2298 if (regno == BPF_REG_FP) 2299 return 0; 2300 2301 if (rw64) 2302 mark_insn_zext(env, reg); 2303 2304 return mark_reg_read(env, reg, reg->parent, 2305 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); 2306 } else { 2307 /* check whether register used as dest operand can be written to */ 2308 if (regno == BPF_REG_FP) { 2309 verbose(env, "frame pointer is read only\n"); 2310 return -EACCES; 2311 } 2312 reg->live |= REG_LIVE_WRITTEN; 2313 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; 2314 if (t == DST_OP) 2315 mark_reg_unknown(env, regs, regno); 2316 } 2317 return 0; 2318 } 2319 2320 /* for any branch, call, exit record the history of jmps in the given state */ 2321 static int push_jmp_history(struct bpf_verifier_env *env, 2322 struct bpf_verifier_state *cur) 2323 { 2324 u32 cnt = cur->jmp_history_cnt; 2325 struct bpf_idx_pair *p; 2326 2327 cnt++; 2328 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); 2329 if (!p) 2330 return -ENOMEM; 2331 p[cnt - 1].idx = env->insn_idx; 2332 p[cnt - 1].prev_idx = env->prev_insn_idx; 2333 cur->jmp_history = p; 2334 cur->jmp_history_cnt = cnt; 2335 return 0; 2336 } 2337 2338 /* Backtrack one insn at a time. If idx is not at the top of recorded 2339 * history then previous instruction came from straight line execution. 2340 */ 2341 static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, 2342 u32 *history) 2343 { 2344 u32 cnt = *history; 2345 2346 if (cnt && st->jmp_history[cnt - 1].idx == i) { 2347 i = st->jmp_history[cnt - 1].prev_idx; 2348 (*history)--; 2349 } else { 2350 i--; 2351 } 2352 return i; 2353 } 2354 2355 static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) 2356 { 2357 const struct btf_type *func; 2358 struct btf *desc_btf; 2359 2360 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) 2361 return NULL; 2362 2363 desc_btf = find_kfunc_desc_btf(data, insn->imm, insn->off); 2364 if (IS_ERR(desc_btf)) 2365 return "<error>"; 2366 2367 func = btf_type_by_id(desc_btf, insn->imm); 2368 return btf_name_by_offset(desc_btf, func->name_off); 2369 } 2370 2371 /* For given verifier state backtrack_insn() is called from the last insn to 2372 * the first insn. Its purpose is to compute a bitmask of registers and 2373 * stack slots that needs precision in the parent verifier state. 2374 */ 2375 static int backtrack_insn(struct bpf_verifier_env *env, int idx, 2376 u32 *reg_mask, u64 *stack_mask) 2377 { 2378 const struct bpf_insn_cbs cbs = { 2379 .cb_call = disasm_kfunc_name, 2380 .cb_print = verbose, 2381 .private_data = env, 2382 }; 2383 struct bpf_insn *insn = env->prog->insnsi + idx; 2384 u8 class = BPF_CLASS(insn->code); 2385 u8 opcode = BPF_OP(insn->code); 2386 u8 mode = BPF_MODE(insn->code); 2387 u32 dreg = 1u << insn->dst_reg; 2388 u32 sreg = 1u << insn->src_reg; 2389 u32 spi; 2390 2391 if (insn->code == 0) 2392 return 0; 2393 if (env->log.level & BPF_LOG_LEVEL2) { 2394 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); 2395 verbose(env, "%d: ", idx); 2396 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 2397 } 2398 2399 if (class == BPF_ALU || class == BPF_ALU64) { 2400 if (!(*reg_mask & dreg)) 2401 return 0; 2402 if (opcode == BPF_MOV) { 2403 if (BPF_SRC(insn->code) == BPF_X) { 2404 /* dreg = sreg 2405 * dreg needs precision after this insn 2406 * sreg needs precision before this insn 2407 */ 2408 *reg_mask &= ~dreg; 2409 *reg_mask |= sreg; 2410 } else { 2411 /* dreg = K 2412 * dreg needs precision after this insn. 2413 * Corresponding register is already marked 2414 * as precise=true in this verifier state. 2415 * No further markings in parent are necessary 2416 */ 2417 *reg_mask &= ~dreg; 2418 } 2419 } else { 2420 if (BPF_SRC(insn->code) == BPF_X) { 2421 /* dreg += sreg 2422 * both dreg and sreg need precision 2423 * before this insn 2424 */ 2425 *reg_mask |= sreg; 2426 } /* else dreg += K 2427 * dreg still needs precision before this insn 2428 */ 2429 } 2430 } else if (class == BPF_LDX) { 2431 if (!(*reg_mask & dreg)) 2432 return 0; 2433 *reg_mask &= ~dreg; 2434 2435 /* scalars can only be spilled into stack w/o losing precision. 2436 * Load from any other memory can be zero extended. 2437 * The desire to keep that precision is already indicated 2438 * by 'precise' mark in corresponding register of this state. 2439 * No further tracking necessary. 2440 */ 2441 if (insn->src_reg != BPF_REG_FP) 2442 return 0; 2443 2444 /* dreg = *(u64 *)[fp - off] was a fill from the stack. 2445 * that [fp - off] slot contains scalar that needs to be 2446 * tracked with precision 2447 */ 2448 spi = (-insn->off - 1) / BPF_REG_SIZE; 2449 if (spi >= 64) { 2450 verbose(env, "BUG spi %d\n", spi); 2451 WARN_ONCE(1, "verifier backtracking bug"); 2452 return -EFAULT; 2453 } 2454 *stack_mask |= 1ull << spi; 2455 } else if (class == BPF_STX || class == BPF_ST) { 2456 if (*reg_mask & dreg) 2457 /* stx & st shouldn't be using _scalar_ dst_reg 2458 * to access memory. It means backtracking 2459 * encountered a case of pointer subtraction. 2460 */ 2461 return -ENOTSUPP; 2462 /* scalars can only be spilled into stack */ 2463 if (insn->dst_reg != BPF_REG_FP) 2464 return 0; 2465 spi = (-insn->off - 1) / BPF_REG_SIZE; 2466 if (spi >= 64) { 2467 verbose(env, "BUG spi %d\n", spi); 2468 WARN_ONCE(1, "verifier backtracking bug"); 2469 return -EFAULT; 2470 } 2471 if (!(*stack_mask & (1ull << spi))) 2472 return 0; 2473 *stack_mask &= ~(1ull << spi); 2474 if (class == BPF_STX) 2475 *reg_mask |= sreg; 2476 } else if (class == BPF_JMP || class == BPF_JMP32) { 2477 if (opcode == BPF_CALL) { 2478 if (insn->src_reg == BPF_PSEUDO_CALL) 2479 return -ENOTSUPP; 2480 /* regular helper call sets R0 */ 2481 *reg_mask &= ~1; 2482 if (*reg_mask & 0x3f) { 2483 /* if backtracing was looking for registers R1-R5 2484 * they should have been found already. 2485 */ 2486 verbose(env, "BUG regs %x\n", *reg_mask); 2487 WARN_ONCE(1, "verifier backtracking bug"); 2488 return -EFAULT; 2489 } 2490 } else if (opcode == BPF_EXIT) { 2491 return -ENOTSUPP; 2492 } 2493 } else if (class == BPF_LD) { 2494 if (!(*reg_mask & dreg)) 2495 return 0; 2496 *reg_mask &= ~dreg; 2497 /* It's ld_imm64 or ld_abs or ld_ind. 2498 * For ld_imm64 no further tracking of precision 2499 * into parent is necessary 2500 */ 2501 if (mode == BPF_IND || mode == BPF_ABS) 2502 /* to be analyzed */ 2503 return -ENOTSUPP; 2504 } 2505 return 0; 2506 } 2507 2508 /* the scalar precision tracking algorithm: 2509 * . at the start all registers have precise=false. 2510 * . scalar ranges are tracked as normal through alu and jmp insns. 2511 * . once precise value of the scalar register is used in: 2512 * . ptr + scalar alu 2513 * . if (scalar cond K|scalar) 2514 * . helper_call(.., scalar, ...) where ARG_CONST is expected 2515 * backtrack through the verifier states and mark all registers and 2516 * stack slots with spilled constants that these scalar regisers 2517 * should be precise. 2518 * . during state pruning two registers (or spilled stack slots) 2519 * are equivalent if both are not precise. 2520 * 2521 * Note the verifier cannot simply walk register parentage chain, 2522 * since many different registers and stack slots could have been 2523 * used to compute single precise scalar. 2524 * 2525 * The approach of starting with precise=true for all registers and then 2526 * backtrack to mark a register as not precise when the verifier detects 2527 * that program doesn't care about specific value (e.g., when helper 2528 * takes register as ARG_ANYTHING parameter) is not safe. 2529 * 2530 * It's ok to walk single parentage chain of the verifier states. 2531 * It's possible that this backtracking will go all the way till 1st insn. 2532 * All other branches will be explored for needing precision later. 2533 * 2534 * The backtracking needs to deal with cases like: 2535 * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) 2536 * r9 -= r8 2537 * r5 = r9 2538 * if r5 > 0x79f goto pc+7 2539 * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) 2540 * r5 += 1 2541 * ... 2542 * call bpf_perf_event_output#25 2543 * where .arg5_type = ARG_CONST_SIZE_OR_ZERO 2544 * 2545 * and this case: 2546 * r6 = 1 2547 * call foo // uses callee's r6 inside to compute r0 2548 * r0 += r6 2549 * if r0 == 0 goto 2550 * 2551 * to track above reg_mask/stack_mask needs to be independent for each frame. 2552 * 2553 * Also if parent's curframe > frame where backtracking started, 2554 * the verifier need to mark registers in both frames, otherwise callees 2555 * may incorrectly prune callers. This is similar to 2556 * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") 2557 * 2558 * For now backtracking falls back into conservative marking. 2559 */ 2560 static void mark_all_scalars_precise(struct bpf_verifier_env *env, 2561 struct bpf_verifier_state *st) 2562 { 2563 struct bpf_func_state *func; 2564 struct bpf_reg_state *reg; 2565 int i, j; 2566 2567 /* big hammer: mark all scalars precise in this path. 2568 * pop_stack may still get !precise scalars. 2569 */ 2570 for (; st; st = st->parent) 2571 for (i = 0; i <= st->curframe; i++) { 2572 func = st->frame[i]; 2573 for (j = 0; j < BPF_REG_FP; j++) { 2574 reg = &func->regs[j]; 2575 if (reg->type != SCALAR_VALUE) 2576 continue; 2577 reg->precise = true; 2578 } 2579 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { 2580 if (!is_spilled_reg(&func->stack[j])) 2581 continue; 2582 reg = &func->stack[j].spilled_ptr; 2583 if (reg->type != SCALAR_VALUE) 2584 continue; 2585 reg->precise = true; 2586 } 2587 } 2588 } 2589 2590 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, 2591 int spi) 2592 { 2593 struct bpf_verifier_state *st = env->cur_state; 2594 int first_idx = st->first_insn_idx; 2595 int last_idx = env->insn_idx; 2596 struct bpf_func_state *func; 2597 struct bpf_reg_state *reg; 2598 u32 reg_mask = regno >= 0 ? 1u << regno : 0; 2599 u64 stack_mask = spi >= 0 ? 1ull << spi : 0; 2600 bool skip_first = true; 2601 bool new_marks = false; 2602 int i, err; 2603 2604 if (!env->bpf_capable) 2605 return 0; 2606 2607 func = st->frame[st->curframe]; 2608 if (regno >= 0) { 2609 reg = &func->regs[regno]; 2610 if (reg->type != SCALAR_VALUE) { 2611 WARN_ONCE(1, "backtracing misuse"); 2612 return -EFAULT; 2613 } 2614 if (!reg->precise) 2615 new_marks = true; 2616 else 2617 reg_mask = 0; 2618 reg->precise = true; 2619 } 2620 2621 while (spi >= 0) { 2622 if (!is_spilled_reg(&func->stack[spi])) { 2623 stack_mask = 0; 2624 break; 2625 } 2626 reg = &func->stack[spi].spilled_ptr; 2627 if (reg->type != SCALAR_VALUE) { 2628 stack_mask = 0; 2629 break; 2630 } 2631 if (!reg->precise) 2632 new_marks = true; 2633 else 2634 stack_mask = 0; 2635 reg->precise = true; 2636 break; 2637 } 2638 2639 if (!new_marks) 2640 return 0; 2641 if (!reg_mask && !stack_mask) 2642 return 0; 2643 for (;;) { 2644 DECLARE_BITMAP(mask, 64); 2645 u32 history = st->jmp_history_cnt; 2646 2647 if (env->log.level & BPF_LOG_LEVEL2) 2648 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); 2649 for (i = last_idx;;) { 2650 if (skip_first) { 2651 err = 0; 2652 skip_first = false; 2653 } else { 2654 err = backtrack_insn(env, i, ®_mask, &stack_mask); 2655 } 2656 if (err == -ENOTSUPP) { 2657 mark_all_scalars_precise(env, st); 2658 return 0; 2659 } else if (err) { 2660 return err; 2661 } 2662 if (!reg_mask && !stack_mask) 2663 /* Found assignment(s) into tracked register in this state. 2664 * Since this state is already marked, just return. 2665 * Nothing to be tracked further in the parent state. 2666 */ 2667 return 0; 2668 if (i == first_idx) 2669 break; 2670 i = get_prev_insn_idx(st, i, &history); 2671 if (i >= env->prog->len) { 2672 /* This can happen if backtracking reached insn 0 2673 * and there are still reg_mask or stack_mask 2674 * to backtrack. 2675 * It means the backtracking missed the spot where 2676 * particular register was initialized with a constant. 2677 */ 2678 verbose(env, "BUG backtracking idx %d\n", i); 2679 WARN_ONCE(1, "verifier backtracking bug"); 2680 return -EFAULT; 2681 } 2682 } 2683 st = st->parent; 2684 if (!st) 2685 break; 2686 2687 new_marks = false; 2688 func = st->frame[st->curframe]; 2689 bitmap_from_u64(mask, reg_mask); 2690 for_each_set_bit(i, mask, 32) { 2691 reg = &func->regs[i]; 2692 if (reg->type != SCALAR_VALUE) { 2693 reg_mask &= ~(1u << i); 2694 continue; 2695 } 2696 if (!reg->precise) 2697 new_marks = true; 2698 reg->precise = true; 2699 } 2700 2701 bitmap_from_u64(mask, stack_mask); 2702 for_each_set_bit(i, mask, 64) { 2703 if (i >= func->allocated_stack / BPF_REG_SIZE) { 2704 /* the sequence of instructions: 2705 * 2: (bf) r3 = r10 2706 * 3: (7b) *(u64 *)(r3 -8) = r0 2707 * 4: (79) r4 = *(u64 *)(r10 -8) 2708 * doesn't contain jmps. It's backtracked 2709 * as a single block. 2710 * During backtracking insn 3 is not recognized as 2711 * stack access, so at the end of backtracking 2712 * stack slot fp-8 is still marked in stack_mask. 2713 * However the parent state may not have accessed 2714 * fp-8 and it's "unallocated" stack space. 2715 * In such case fallback to conservative. 2716 */ 2717 mark_all_scalars_precise(env, st); 2718 return 0; 2719 } 2720 2721 if (!is_spilled_reg(&func->stack[i])) { 2722 stack_mask &= ~(1ull << i); 2723 continue; 2724 } 2725 reg = &func->stack[i].spilled_ptr; 2726 if (reg->type != SCALAR_VALUE) { 2727 stack_mask &= ~(1ull << i); 2728 continue; 2729 } 2730 if (!reg->precise) 2731 new_marks = true; 2732 reg->precise = true; 2733 } 2734 if (env->log.level & BPF_LOG_LEVEL2) { 2735 verbose(env, "parent %s regs=%x stack=%llx marks:", 2736 new_marks ? "didn't have" : "already had", 2737 reg_mask, stack_mask); 2738 print_verifier_state(env, func, true); 2739 } 2740 2741 if (!reg_mask && !stack_mask) 2742 break; 2743 if (!new_marks) 2744 break; 2745 2746 last_idx = st->last_insn_idx; 2747 first_idx = st->first_insn_idx; 2748 } 2749 return 0; 2750 } 2751 2752 static int mark_chain_precision(struct bpf_verifier_env *env, int regno) 2753 { 2754 return __mark_chain_precision(env, regno, -1); 2755 } 2756 2757 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) 2758 { 2759 return __mark_chain_precision(env, -1, spi); 2760 } 2761 2762 static bool is_spillable_regtype(enum bpf_reg_type type) 2763 { 2764 switch (base_type(type)) { 2765 case PTR_TO_MAP_VALUE: 2766 case PTR_TO_STACK: 2767 case PTR_TO_CTX: 2768 case PTR_TO_PACKET: 2769 case PTR_TO_PACKET_META: 2770 case PTR_TO_PACKET_END: 2771 case PTR_TO_FLOW_KEYS: 2772 case CONST_PTR_TO_MAP: 2773 case PTR_TO_SOCKET: 2774 case PTR_TO_SOCK_COMMON: 2775 case PTR_TO_TCP_SOCK: 2776 case PTR_TO_XDP_SOCK: 2777 case PTR_TO_BTF_ID: 2778 case PTR_TO_BUF: 2779 case PTR_TO_MEM: 2780 case PTR_TO_FUNC: 2781 case PTR_TO_MAP_KEY: 2782 return true; 2783 default: 2784 return false; 2785 } 2786 } 2787 2788 /* Does this register contain a constant zero? */ 2789 static bool register_is_null(struct bpf_reg_state *reg) 2790 { 2791 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 2792 } 2793 2794 static bool register_is_const(struct bpf_reg_state *reg) 2795 { 2796 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); 2797 } 2798 2799 static bool __is_scalar_unbounded(struct bpf_reg_state *reg) 2800 { 2801 return tnum_is_unknown(reg->var_off) && 2802 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && 2803 reg->umin_value == 0 && reg->umax_value == U64_MAX && 2804 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && 2805 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; 2806 } 2807 2808 static bool register_is_bounded(struct bpf_reg_state *reg) 2809 { 2810 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); 2811 } 2812 2813 static bool __is_pointer_value(bool allow_ptr_leaks, 2814 const struct bpf_reg_state *reg) 2815 { 2816 if (allow_ptr_leaks) 2817 return false; 2818 2819 return reg->type != SCALAR_VALUE; 2820 } 2821 2822 static void save_register_state(struct bpf_func_state *state, 2823 int spi, struct bpf_reg_state *reg, 2824 int size) 2825 { 2826 int i; 2827 2828 state->stack[spi].spilled_ptr = *reg; 2829 if (size == BPF_REG_SIZE) 2830 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2831 2832 for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) 2833 state->stack[spi].slot_type[i - 1] = STACK_SPILL; 2834 2835 /* size < 8 bytes spill */ 2836 for (; i; i--) 2837 scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]); 2838 } 2839 2840 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, 2841 * stack boundary and alignment are checked in check_mem_access() 2842 */ 2843 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, 2844 /* stack frame we're writing to */ 2845 struct bpf_func_state *state, 2846 int off, int size, int value_regno, 2847 int insn_idx) 2848 { 2849 struct bpf_func_state *cur; /* state of the current function */ 2850 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 2851 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; 2852 struct bpf_reg_state *reg = NULL; 2853 2854 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE)); 2855 if (err) 2856 return err; 2857 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 2858 * so it's aligned access and [off, off + size) are within stack limits 2859 */ 2860 if (!env->allow_ptr_leaks && 2861 state->stack[spi].slot_type[0] == STACK_SPILL && 2862 size != BPF_REG_SIZE) { 2863 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 2864 return -EACCES; 2865 } 2866 2867 cur = env->cur_state->frame[env->cur_state->curframe]; 2868 if (value_regno >= 0) 2869 reg = &cur->regs[value_regno]; 2870 if (!env->bypass_spec_v4) { 2871 bool sanitize = reg && is_spillable_regtype(reg->type); 2872 2873 for (i = 0; i < size; i++) { 2874 if (state->stack[spi].slot_type[i] == STACK_INVALID) { 2875 sanitize = true; 2876 break; 2877 } 2878 } 2879 2880 if (sanitize) 2881 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; 2882 } 2883 2884 mark_stack_slot_scratched(env, spi); 2885 if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) && 2886 !register_is_null(reg) && env->bpf_capable) { 2887 if (dst_reg != BPF_REG_FP) { 2888 /* The backtracking logic can only recognize explicit 2889 * stack slot address like [fp - 8]. Other spill of 2890 * scalar via different register has to be conservative. 2891 * Backtrack from here and mark all registers as precise 2892 * that contributed into 'reg' being a constant. 2893 */ 2894 err = mark_chain_precision(env, value_regno); 2895 if (err) 2896 return err; 2897 } 2898 save_register_state(state, spi, reg, size); 2899 } else if (reg && is_spillable_regtype(reg->type)) { 2900 /* register containing pointer is being spilled into stack */ 2901 if (size != BPF_REG_SIZE) { 2902 verbose_linfo(env, insn_idx, "; "); 2903 verbose(env, "invalid size of register spill\n"); 2904 return -EACCES; 2905 } 2906 if (state != cur && reg->type == PTR_TO_STACK) { 2907 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 2908 return -EINVAL; 2909 } 2910 save_register_state(state, spi, reg, size); 2911 } else { 2912 u8 type = STACK_MISC; 2913 2914 /* regular write of data into stack destroys any spilled ptr */ 2915 state->stack[spi].spilled_ptr.type = NOT_INIT; 2916 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 2917 if (is_spilled_reg(&state->stack[spi])) 2918 for (i = 0; i < BPF_REG_SIZE; i++) 2919 scrub_spilled_slot(&state->stack[spi].slot_type[i]); 2920 2921 /* only mark the slot as written if all 8 bytes were written 2922 * otherwise read propagation may incorrectly stop too soon 2923 * when stack slots are partially written. 2924 * This heuristic means that read propagation will be 2925 * conservative, since it will add reg_live_read marks 2926 * to stack slots all the way to first state when programs 2927 * writes+reads less than 8 bytes 2928 */ 2929 if (size == BPF_REG_SIZE) 2930 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 2931 2932 /* when we zero initialize stack slots mark them as such */ 2933 if (reg && register_is_null(reg)) { 2934 /* backtracking doesn't work for STACK_ZERO yet. */ 2935 err = mark_chain_precision(env, value_regno); 2936 if (err) 2937 return err; 2938 type = STACK_ZERO; 2939 } 2940 2941 /* Mark slots affected by this stack write. */ 2942 for (i = 0; i < size; i++) 2943 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 2944 type; 2945 } 2946 return 0; 2947 } 2948 2949 /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is 2950 * known to contain a variable offset. 2951 * This function checks whether the write is permitted and conservatively 2952 * tracks the effects of the write, considering that each stack slot in the 2953 * dynamic range is potentially written to. 2954 * 2955 * 'off' includes 'regno->off'. 2956 * 'value_regno' can be -1, meaning that an unknown value is being written to 2957 * the stack. 2958 * 2959 * Spilled pointers in range are not marked as written because we don't know 2960 * what's going to be actually written. This means that read propagation for 2961 * future reads cannot be terminated by this write. 2962 * 2963 * For privileged programs, uninitialized stack slots are considered 2964 * initialized by this write (even though we don't know exactly what offsets 2965 * are going to be written to). The idea is that we don't want the verifier to 2966 * reject future reads that access slots written to through variable offsets. 2967 */ 2968 static int check_stack_write_var_off(struct bpf_verifier_env *env, 2969 /* func where register points to */ 2970 struct bpf_func_state *state, 2971 int ptr_regno, int off, int size, 2972 int value_regno, int insn_idx) 2973 { 2974 struct bpf_func_state *cur; /* state of the current function */ 2975 int min_off, max_off; 2976 int i, err; 2977 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; 2978 bool writing_zero = false; 2979 /* set if the fact that we're writing a zero is used to let any 2980 * stack slots remain STACK_ZERO 2981 */ 2982 bool zero_used = false; 2983 2984 cur = env->cur_state->frame[env->cur_state->curframe]; 2985 ptr_reg = &cur->regs[ptr_regno]; 2986 min_off = ptr_reg->smin_value + off; 2987 max_off = ptr_reg->smax_value + off + size; 2988 if (value_regno >= 0) 2989 value_reg = &cur->regs[value_regno]; 2990 if (value_reg && register_is_null(value_reg)) 2991 writing_zero = true; 2992 2993 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE)); 2994 if (err) 2995 return err; 2996 2997 2998 /* Variable offset writes destroy any spilled pointers in range. */ 2999 for (i = min_off; i < max_off; i++) { 3000 u8 new_type, *stype; 3001 int slot, spi; 3002 3003 slot = -i - 1; 3004 spi = slot / BPF_REG_SIZE; 3005 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 3006 mark_stack_slot_scratched(env, spi); 3007 3008 if (!env->allow_ptr_leaks 3009 && *stype != NOT_INIT 3010 && *stype != SCALAR_VALUE) { 3011 /* Reject the write if there's are spilled pointers in 3012 * range. If we didn't reject here, the ptr status 3013 * would be erased below (even though not all slots are 3014 * actually overwritten), possibly opening the door to 3015 * leaks. 3016 */ 3017 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", 3018 insn_idx, i); 3019 return -EINVAL; 3020 } 3021 3022 /* Erase all spilled pointers. */ 3023 state->stack[spi].spilled_ptr.type = NOT_INIT; 3024 3025 /* Update the slot type. */ 3026 new_type = STACK_MISC; 3027 if (writing_zero && *stype == STACK_ZERO) { 3028 new_type = STACK_ZERO; 3029 zero_used = true; 3030 } 3031 /* If the slot is STACK_INVALID, we check whether it's OK to 3032 * pretend that it will be initialized by this write. The slot 3033 * might not actually be written to, and so if we mark it as 3034 * initialized future reads might leak uninitialized memory. 3035 * For privileged programs, we will accept such reads to slots 3036 * that may or may not be written because, if we're reject 3037 * them, the error would be too confusing. 3038 */ 3039 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { 3040 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", 3041 insn_idx, i); 3042 return -EINVAL; 3043 } 3044 *stype = new_type; 3045 } 3046 if (zero_used) { 3047 /* backtracking doesn't work for STACK_ZERO yet. */ 3048 err = mark_chain_precision(env, value_regno); 3049 if (err) 3050 return err; 3051 } 3052 return 0; 3053 } 3054 3055 /* When register 'dst_regno' is assigned some values from stack[min_off, 3056 * max_off), we set the register's type according to the types of the 3057 * respective stack slots. If all the stack values are known to be zeros, then 3058 * so is the destination reg. Otherwise, the register is considered to be 3059 * SCALAR. This function does not deal with register filling; the caller must 3060 * ensure that all spilled registers in the stack range have been marked as 3061 * read. 3062 */ 3063 static void mark_reg_stack_read(struct bpf_verifier_env *env, 3064 /* func where src register points to */ 3065 struct bpf_func_state *ptr_state, 3066 int min_off, int max_off, int dst_regno) 3067 { 3068 struct bpf_verifier_state *vstate = env->cur_state; 3069 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3070 int i, slot, spi; 3071 u8 *stype; 3072 int zeros = 0; 3073 3074 for (i = min_off; i < max_off; i++) { 3075 slot = -i - 1; 3076 spi = slot / BPF_REG_SIZE; 3077 stype = ptr_state->stack[spi].slot_type; 3078 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) 3079 break; 3080 zeros++; 3081 } 3082 if (zeros == max_off - min_off) { 3083 /* any access_size read into register is zero extended, 3084 * so the whole register == const_zero 3085 */ 3086 __mark_reg_const_zero(&state->regs[dst_regno]); 3087 /* backtracking doesn't support STACK_ZERO yet, 3088 * so mark it precise here, so that later 3089 * backtracking can stop here. 3090 * Backtracking may not need this if this register 3091 * doesn't participate in pointer adjustment. 3092 * Forward propagation of precise flag is not 3093 * necessary either. This mark is only to stop 3094 * backtracking. Any register that contributed 3095 * to const 0 was marked precise before spill. 3096 */ 3097 state->regs[dst_regno].precise = true; 3098 } else { 3099 /* have read misc data from the stack */ 3100 mark_reg_unknown(env, state->regs, dst_regno); 3101 } 3102 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3103 } 3104 3105 /* Read the stack at 'off' and put the results into the register indicated by 3106 * 'dst_regno'. It handles reg filling if the addressed stack slot is a 3107 * spilled reg. 3108 * 3109 * 'dst_regno' can be -1, meaning that the read value is not going to a 3110 * register. 3111 * 3112 * The access is assumed to be within the current stack bounds. 3113 */ 3114 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, 3115 /* func where src register points to */ 3116 struct bpf_func_state *reg_state, 3117 int off, int size, int dst_regno) 3118 { 3119 struct bpf_verifier_state *vstate = env->cur_state; 3120 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3121 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 3122 struct bpf_reg_state *reg; 3123 u8 *stype, type; 3124 3125 stype = reg_state->stack[spi].slot_type; 3126 reg = ®_state->stack[spi].spilled_ptr; 3127 3128 if (is_spilled_reg(®_state->stack[spi])) { 3129 u8 spill_size = 1; 3130 3131 for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--) 3132 spill_size++; 3133 3134 if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) { 3135 if (reg->type != SCALAR_VALUE) { 3136 verbose_linfo(env, env->insn_idx, "; "); 3137 verbose(env, "invalid size of register fill\n"); 3138 return -EACCES; 3139 } 3140 3141 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3142 if (dst_regno < 0) 3143 return 0; 3144 3145 if (!(off % BPF_REG_SIZE) && size == spill_size) { 3146 /* The earlier check_reg_arg() has decided the 3147 * subreg_def for this insn. Save it first. 3148 */ 3149 s32 subreg_def = state->regs[dst_regno].subreg_def; 3150 3151 state->regs[dst_regno] = *reg; 3152 state->regs[dst_regno].subreg_def = subreg_def; 3153 } else { 3154 for (i = 0; i < size; i++) { 3155 type = stype[(slot - i) % BPF_REG_SIZE]; 3156 if (type == STACK_SPILL) 3157 continue; 3158 if (type == STACK_MISC) 3159 continue; 3160 verbose(env, "invalid read from stack off %d+%d size %d\n", 3161 off, i, size); 3162 return -EACCES; 3163 } 3164 mark_reg_unknown(env, state->regs, dst_regno); 3165 } 3166 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3167 return 0; 3168 } 3169 3170 if (dst_regno >= 0) { 3171 /* restore register state from stack */ 3172 state->regs[dst_regno] = *reg; 3173 /* mark reg as written since spilled pointer state likely 3174 * has its liveness marks cleared by is_state_visited() 3175 * which resets stack/reg liveness for state transitions 3176 */ 3177 state->regs[dst_regno].live |= REG_LIVE_WRITTEN; 3178 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { 3179 /* If dst_regno==-1, the caller is asking us whether 3180 * it is acceptable to use this value as a SCALAR_VALUE 3181 * (e.g. for XADD). 3182 * We must not allow unprivileged callers to do that 3183 * with spilled pointers. 3184 */ 3185 verbose(env, "leaking pointer from stack off %d\n", 3186 off); 3187 return -EACCES; 3188 } 3189 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3190 } else { 3191 for (i = 0; i < size; i++) { 3192 type = stype[(slot - i) % BPF_REG_SIZE]; 3193 if (type == STACK_MISC) 3194 continue; 3195 if (type == STACK_ZERO) 3196 continue; 3197 verbose(env, "invalid read from stack off %d+%d size %d\n", 3198 off, i, size); 3199 return -EACCES; 3200 } 3201 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 3202 if (dst_regno >= 0) 3203 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); 3204 } 3205 return 0; 3206 } 3207 3208 enum bpf_access_src { 3209 ACCESS_DIRECT = 1, /* the access is performed by an instruction */ 3210 ACCESS_HELPER = 2, /* the access is performed by a helper */ 3211 }; 3212 3213 static int check_stack_range_initialized(struct bpf_verifier_env *env, 3214 int regno, int off, int access_size, 3215 bool zero_size_allowed, 3216 enum bpf_access_src type, 3217 struct bpf_call_arg_meta *meta); 3218 3219 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 3220 { 3221 return cur_regs(env) + regno; 3222 } 3223 3224 /* Read the stack at 'ptr_regno + off' and put the result into the register 3225 * 'dst_regno'. 3226 * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), 3227 * but not its variable offset. 3228 * 'size' is assumed to be <= reg size and the access is assumed to be aligned. 3229 * 3230 * As opposed to check_stack_read_fixed_off, this function doesn't deal with 3231 * filling registers (i.e. reads of spilled register cannot be detected when 3232 * the offset is not fixed). We conservatively mark 'dst_regno' as containing 3233 * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable 3234 * offset; for a fixed offset check_stack_read_fixed_off should be used 3235 * instead. 3236 */ 3237 static int check_stack_read_var_off(struct bpf_verifier_env *env, 3238 int ptr_regno, int off, int size, int dst_regno) 3239 { 3240 /* The state of the source register. */ 3241 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3242 struct bpf_func_state *ptr_state = func(env, reg); 3243 int err; 3244 int min_off, max_off; 3245 3246 /* Note that we pass a NULL meta, so raw access will not be permitted. 3247 */ 3248 err = check_stack_range_initialized(env, ptr_regno, off, size, 3249 false, ACCESS_DIRECT, NULL); 3250 if (err) 3251 return err; 3252 3253 min_off = reg->smin_value + off; 3254 max_off = reg->smax_value + off; 3255 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); 3256 return 0; 3257 } 3258 3259 /* check_stack_read dispatches to check_stack_read_fixed_off or 3260 * check_stack_read_var_off. 3261 * 3262 * The caller must ensure that the offset falls within the allocated stack 3263 * bounds. 3264 * 3265 * 'dst_regno' is a register which will receive the value from the stack. It 3266 * can be -1, meaning that the read value is not going to a register. 3267 */ 3268 static int check_stack_read(struct bpf_verifier_env *env, 3269 int ptr_regno, int off, int size, 3270 int dst_regno) 3271 { 3272 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3273 struct bpf_func_state *state = func(env, reg); 3274 int err; 3275 /* Some accesses are only permitted with a static offset. */ 3276 bool var_off = !tnum_is_const(reg->var_off); 3277 3278 /* The offset is required to be static when reads don't go to a 3279 * register, in order to not leak pointers (see 3280 * check_stack_read_fixed_off). 3281 */ 3282 if (dst_regno < 0 && var_off) { 3283 char tn_buf[48]; 3284 3285 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3286 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", 3287 tn_buf, off, size); 3288 return -EACCES; 3289 } 3290 /* Variable offset is prohibited for unprivileged mode for simplicity 3291 * since it requires corresponding support in Spectre masking for stack 3292 * ALU. See also retrieve_ptr_limit(). 3293 */ 3294 if (!env->bypass_spec_v1 && var_off) { 3295 char tn_buf[48]; 3296 3297 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3298 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", 3299 ptr_regno, tn_buf); 3300 return -EACCES; 3301 } 3302 3303 if (!var_off) { 3304 off += reg->var_off.value; 3305 err = check_stack_read_fixed_off(env, state, off, size, 3306 dst_regno); 3307 } else { 3308 /* Variable offset stack reads need more conservative handling 3309 * than fixed offset ones. Note that dst_regno >= 0 on this 3310 * branch. 3311 */ 3312 err = check_stack_read_var_off(env, ptr_regno, off, size, 3313 dst_regno); 3314 } 3315 return err; 3316 } 3317 3318 3319 /* check_stack_write dispatches to check_stack_write_fixed_off or 3320 * check_stack_write_var_off. 3321 * 3322 * 'ptr_regno' is the register used as a pointer into the stack. 3323 * 'off' includes 'ptr_regno->off', but not its variable offset (if any). 3324 * 'value_regno' is the register whose value we're writing to the stack. It can 3325 * be -1, meaning that we're not writing from a register. 3326 * 3327 * The caller must ensure that the offset falls within the maximum stack size. 3328 */ 3329 static int check_stack_write(struct bpf_verifier_env *env, 3330 int ptr_regno, int off, int size, 3331 int value_regno, int insn_idx) 3332 { 3333 struct bpf_reg_state *reg = reg_state(env, ptr_regno); 3334 struct bpf_func_state *state = func(env, reg); 3335 int err; 3336 3337 if (tnum_is_const(reg->var_off)) { 3338 off += reg->var_off.value; 3339 err = check_stack_write_fixed_off(env, state, off, size, 3340 value_regno, insn_idx); 3341 } else { 3342 /* Variable offset stack reads need more conservative handling 3343 * than fixed offset ones. 3344 */ 3345 err = check_stack_write_var_off(env, state, 3346 ptr_regno, off, size, 3347 value_regno, insn_idx); 3348 } 3349 return err; 3350 } 3351 3352 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 3353 int off, int size, enum bpf_access_type type) 3354 { 3355 struct bpf_reg_state *regs = cur_regs(env); 3356 struct bpf_map *map = regs[regno].map_ptr; 3357 u32 cap = bpf_map_flags_to_cap(map); 3358 3359 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 3360 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 3361 map->value_size, off, size); 3362 return -EACCES; 3363 } 3364 3365 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 3366 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 3367 map->value_size, off, size); 3368 return -EACCES; 3369 } 3370 3371 return 0; 3372 } 3373 3374 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ 3375 static int __check_mem_access(struct bpf_verifier_env *env, int regno, 3376 int off, int size, u32 mem_size, 3377 bool zero_size_allowed) 3378 { 3379 bool size_ok = size > 0 || (size == 0 && zero_size_allowed); 3380 struct bpf_reg_state *reg; 3381 3382 if (off >= 0 && size_ok && (u64)off + size <= mem_size) 3383 return 0; 3384 3385 reg = &cur_regs(env)[regno]; 3386 switch (reg->type) { 3387 case PTR_TO_MAP_KEY: 3388 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", 3389 mem_size, off, size); 3390 break; 3391 case PTR_TO_MAP_VALUE: 3392 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 3393 mem_size, off, size); 3394 break; 3395 case PTR_TO_PACKET: 3396 case PTR_TO_PACKET_META: 3397 case PTR_TO_PACKET_END: 3398 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 3399 off, size, regno, reg->id, off, mem_size); 3400 break; 3401 case PTR_TO_MEM: 3402 default: 3403 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", 3404 mem_size, off, size); 3405 } 3406 3407 return -EACCES; 3408 } 3409 3410 /* check read/write into a memory region with possible variable offset */ 3411 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, 3412 int off, int size, u32 mem_size, 3413 bool zero_size_allowed) 3414 { 3415 struct bpf_verifier_state *vstate = env->cur_state; 3416 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3417 struct bpf_reg_state *reg = &state->regs[regno]; 3418 int err; 3419 3420 /* We may have adjusted the register pointing to memory region, so we 3421 * need to try adding each of min_value and max_value to off 3422 * to make sure our theoretical access will be safe. 3423 * 3424 * The minimum value is only important with signed 3425 * comparisons where we can't assume the floor of a 3426 * value is 0. If we are using signed variables for our 3427 * index'es we need to make sure that whatever we use 3428 * will have a set floor within our range. 3429 */ 3430 if (reg->smin_value < 0 && 3431 (reg->smin_value == S64_MIN || 3432 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 3433 reg->smin_value + off < 0)) { 3434 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3435 regno); 3436 return -EACCES; 3437 } 3438 err = __check_mem_access(env, regno, reg->smin_value + off, size, 3439 mem_size, zero_size_allowed); 3440 if (err) { 3441 verbose(env, "R%d min value is outside of the allowed memory range\n", 3442 regno); 3443 return err; 3444 } 3445 3446 /* If we haven't set a max value then we need to bail since we can't be 3447 * sure we won't do bad things. 3448 * If reg->umax_value + off could overflow, treat that as unbounded too. 3449 */ 3450 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 3451 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", 3452 regno); 3453 return -EACCES; 3454 } 3455 err = __check_mem_access(env, regno, reg->umax_value + off, size, 3456 mem_size, zero_size_allowed); 3457 if (err) { 3458 verbose(env, "R%d max value is outside of the allowed memory range\n", 3459 regno); 3460 return err; 3461 } 3462 3463 return 0; 3464 } 3465 3466 static int __check_ptr_off_reg(struct bpf_verifier_env *env, 3467 const struct bpf_reg_state *reg, int regno, 3468 bool fixed_off_ok) 3469 { 3470 /* Access to this pointer-typed register or passing it to a helper 3471 * is only allowed in its original, unmodified form. 3472 */ 3473 3474 if (reg->off < 0) { 3475 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", 3476 reg_type_str(env, reg->type), regno, reg->off); 3477 return -EACCES; 3478 } 3479 3480 if (!fixed_off_ok && reg->off) { 3481 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", 3482 reg_type_str(env, reg->type), regno, reg->off); 3483 return -EACCES; 3484 } 3485 3486 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 3487 char tn_buf[48]; 3488 3489 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3490 verbose(env, "variable %s access var_off=%s disallowed\n", 3491 reg_type_str(env, reg->type), tn_buf); 3492 return -EACCES; 3493 } 3494 3495 return 0; 3496 } 3497 3498 int check_ptr_off_reg(struct bpf_verifier_env *env, 3499 const struct bpf_reg_state *reg, int regno) 3500 { 3501 return __check_ptr_off_reg(env, reg, regno, false); 3502 } 3503 3504 static int map_kptr_match_type(struct bpf_verifier_env *env, 3505 struct bpf_map_value_off_desc *off_desc, 3506 struct bpf_reg_state *reg, u32 regno) 3507 { 3508 const char *targ_name = kernel_type_name(off_desc->kptr.btf, off_desc->kptr.btf_id); 3509 int perm_flags = PTR_MAYBE_NULL; 3510 const char *reg_name = ""; 3511 3512 /* Only unreferenced case accepts untrusted pointers */ 3513 if (off_desc->type == BPF_KPTR_UNREF) 3514 perm_flags |= PTR_UNTRUSTED; 3515 3516 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) 3517 goto bad_type; 3518 3519 if (!btf_is_kernel(reg->btf)) { 3520 verbose(env, "R%d must point to kernel BTF\n", regno); 3521 return -EINVAL; 3522 } 3523 /* We need to verify reg->type and reg->btf, before accessing reg->btf */ 3524 reg_name = kernel_type_name(reg->btf, reg->btf_id); 3525 3526 /* For ref_ptr case, release function check should ensure we get one 3527 * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the 3528 * normal store of unreferenced kptr, we must ensure var_off is zero. 3529 * Since ref_ptr cannot be accessed directly by BPF insns, checks for 3530 * reg->off and reg->ref_obj_id are not needed here. 3531 */ 3532 if (__check_ptr_off_reg(env, reg, regno, true)) 3533 return -EACCES; 3534 3535 /* A full type match is needed, as BTF can be vmlinux or module BTF, and 3536 * we also need to take into account the reg->off. 3537 * 3538 * We want to support cases like: 3539 * 3540 * struct foo { 3541 * struct bar br; 3542 * struct baz bz; 3543 * }; 3544 * 3545 * struct foo *v; 3546 * v = func(); // PTR_TO_BTF_ID 3547 * val->foo = v; // reg->off is zero, btf and btf_id match type 3548 * val->bar = &v->br; // reg->off is still zero, but we need to retry with 3549 * // first member type of struct after comparison fails 3550 * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked 3551 * // to match type 3552 * 3553 * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off 3554 * is zero. We must also ensure that btf_struct_ids_match does not walk 3555 * the struct to match type against first member of struct, i.e. reject 3556 * second case from above. Hence, when type is BPF_KPTR_REF, we set 3557 * strict mode to true for type match. 3558 */ 3559 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 3560 off_desc->kptr.btf, off_desc->kptr.btf_id, 3561 off_desc->type == BPF_KPTR_REF)) 3562 goto bad_type; 3563 return 0; 3564 bad_type: 3565 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, 3566 reg_type_str(env, reg->type), reg_name); 3567 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); 3568 if (off_desc->type == BPF_KPTR_UNREF) 3569 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), 3570 targ_name); 3571 else 3572 verbose(env, "\n"); 3573 return -EINVAL; 3574 } 3575 3576 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, 3577 int value_regno, int insn_idx, 3578 struct bpf_map_value_off_desc *off_desc) 3579 { 3580 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; 3581 int class = BPF_CLASS(insn->code); 3582 struct bpf_reg_state *val_reg; 3583 3584 /* Things we already checked for in check_map_access and caller: 3585 * - Reject cases where variable offset may touch kptr 3586 * - size of access (must be BPF_DW) 3587 * - tnum_is_const(reg->var_off) 3588 * - off_desc->offset == off + reg->var_off.value 3589 */ 3590 /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */ 3591 if (BPF_MODE(insn->code) != BPF_MEM) { 3592 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); 3593 return -EACCES; 3594 } 3595 3596 /* We only allow loading referenced kptr, since it will be marked as 3597 * untrusted, similar to unreferenced kptr. 3598 */ 3599 if (class != BPF_LDX && off_desc->type == BPF_KPTR_REF) { 3600 verbose(env, "store to referenced kptr disallowed\n"); 3601 return -EACCES; 3602 } 3603 3604 if (class == BPF_LDX) { 3605 val_reg = reg_state(env, value_regno); 3606 /* We can simply mark the value_regno receiving the pointer 3607 * value from map as PTR_TO_BTF_ID, with the correct type. 3608 */ 3609 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, off_desc->kptr.btf, 3610 off_desc->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED); 3611 /* For mark_ptr_or_null_reg */ 3612 val_reg->id = ++env->id_gen; 3613 } else if (class == BPF_STX) { 3614 val_reg = reg_state(env, value_regno); 3615 if (!register_is_null(val_reg) && 3616 map_kptr_match_type(env, off_desc, val_reg, value_regno)) 3617 return -EACCES; 3618 } else if (class == BPF_ST) { 3619 if (insn->imm) { 3620 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", 3621 off_desc->offset); 3622 return -EACCES; 3623 } 3624 } else { 3625 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); 3626 return -EACCES; 3627 } 3628 return 0; 3629 } 3630 3631 /* check read/write into a map element with possible variable offset */ 3632 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 3633 int off, int size, bool zero_size_allowed, 3634 enum bpf_access_src src) 3635 { 3636 struct bpf_verifier_state *vstate = env->cur_state; 3637 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3638 struct bpf_reg_state *reg = &state->regs[regno]; 3639 struct bpf_map *map = reg->map_ptr; 3640 int err; 3641 3642 err = check_mem_region_access(env, regno, off, size, map->value_size, 3643 zero_size_allowed); 3644 if (err) 3645 return err; 3646 3647 if (map_value_has_spin_lock(map)) { 3648 u32 lock = map->spin_lock_off; 3649 3650 /* if any part of struct bpf_spin_lock can be touched by 3651 * load/store reject this program. 3652 * To check that [x1, x2) overlaps with [y1, y2) 3653 * it is sufficient to check x1 < y2 && y1 < x2. 3654 */ 3655 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && 3656 lock < reg->umax_value + off + size) { 3657 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); 3658 return -EACCES; 3659 } 3660 } 3661 if (map_value_has_timer(map)) { 3662 u32 t = map->timer_off; 3663 3664 if (reg->smin_value + off < t + sizeof(struct bpf_timer) && 3665 t < reg->umax_value + off + size) { 3666 verbose(env, "bpf_timer cannot be accessed directly by load/store\n"); 3667 return -EACCES; 3668 } 3669 } 3670 if (map_value_has_kptrs(map)) { 3671 struct bpf_map_value_off *tab = map->kptr_off_tab; 3672 int i; 3673 3674 for (i = 0; i < tab->nr_off; i++) { 3675 u32 p = tab->off[i].offset; 3676 3677 if (reg->smin_value + off < p + sizeof(u64) && 3678 p < reg->umax_value + off + size) { 3679 if (src != ACCESS_DIRECT) { 3680 verbose(env, "kptr cannot be accessed indirectly by helper\n"); 3681 return -EACCES; 3682 } 3683 if (!tnum_is_const(reg->var_off)) { 3684 verbose(env, "kptr access cannot have variable offset\n"); 3685 return -EACCES; 3686 } 3687 if (p != off + reg->var_off.value) { 3688 verbose(env, "kptr access misaligned expected=%u off=%llu\n", 3689 p, off + reg->var_off.value); 3690 return -EACCES; 3691 } 3692 if (size != bpf_size_to_bytes(BPF_DW)) { 3693 verbose(env, "kptr access size must be BPF_DW\n"); 3694 return -EACCES; 3695 } 3696 break; 3697 } 3698 } 3699 } 3700 return err; 3701 } 3702 3703 #define MAX_PACKET_OFF 0xffff 3704 3705 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 3706 const struct bpf_call_arg_meta *meta, 3707 enum bpf_access_type t) 3708 { 3709 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 3710 3711 switch (prog_type) { 3712 /* Program types only with direct read access go here! */ 3713 case BPF_PROG_TYPE_LWT_IN: 3714 case BPF_PROG_TYPE_LWT_OUT: 3715 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 3716 case BPF_PROG_TYPE_SK_REUSEPORT: 3717 case BPF_PROG_TYPE_FLOW_DISSECTOR: 3718 case BPF_PROG_TYPE_CGROUP_SKB: 3719 if (t == BPF_WRITE) 3720 return false; 3721 fallthrough; 3722 3723 /* Program types with direct read + write access go here! */ 3724 case BPF_PROG_TYPE_SCHED_CLS: 3725 case BPF_PROG_TYPE_SCHED_ACT: 3726 case BPF_PROG_TYPE_XDP: 3727 case BPF_PROG_TYPE_LWT_XMIT: 3728 case BPF_PROG_TYPE_SK_SKB: 3729 case BPF_PROG_TYPE_SK_MSG: 3730 if (meta) 3731 return meta->pkt_access; 3732 3733 env->seen_direct_write = true; 3734 return true; 3735 3736 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 3737 if (t == BPF_WRITE) 3738 env->seen_direct_write = true; 3739 3740 return true; 3741 3742 default: 3743 return false; 3744 } 3745 } 3746 3747 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 3748 int size, bool zero_size_allowed) 3749 { 3750 struct bpf_reg_state *regs = cur_regs(env); 3751 struct bpf_reg_state *reg = ®s[regno]; 3752 int err; 3753 3754 /* We may have added a variable offset to the packet pointer; but any 3755 * reg->range we have comes after that. We are only checking the fixed 3756 * offset. 3757 */ 3758 3759 /* We don't allow negative numbers, because we aren't tracking enough 3760 * detail to prove they're safe. 3761 */ 3762 if (reg->smin_value < 0) { 3763 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3764 regno); 3765 return -EACCES; 3766 } 3767 3768 err = reg->range < 0 ? -EINVAL : 3769 __check_mem_access(env, regno, off, size, reg->range, 3770 zero_size_allowed); 3771 if (err) { 3772 verbose(env, "R%d offset is outside of the packet\n", regno); 3773 return err; 3774 } 3775 3776 /* __check_mem_access has made sure "off + size - 1" is within u16. 3777 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 3778 * otherwise find_good_pkt_pointers would have refused to set range info 3779 * that __check_mem_access would have rejected this pkt access. 3780 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 3781 */ 3782 env->prog->aux->max_pkt_offset = 3783 max_t(u32, env->prog->aux->max_pkt_offset, 3784 off + reg->umax_value + size - 1); 3785 3786 return err; 3787 } 3788 3789 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 3790 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 3791 enum bpf_access_type t, enum bpf_reg_type *reg_type, 3792 struct btf **btf, u32 *btf_id) 3793 { 3794 struct bpf_insn_access_aux info = { 3795 .reg_type = *reg_type, 3796 .log = &env->log, 3797 }; 3798 3799 if (env->ops->is_valid_access && 3800 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 3801 /* A non zero info.ctx_field_size indicates that this field is a 3802 * candidate for later verifier transformation to load the whole 3803 * field and then apply a mask when accessed with a narrower 3804 * access than actual ctx access size. A zero info.ctx_field_size 3805 * will only allow for whole field access and rejects any other 3806 * type of narrower access. 3807 */ 3808 *reg_type = info.reg_type; 3809 3810 if (base_type(*reg_type) == PTR_TO_BTF_ID) { 3811 *btf = info.btf; 3812 *btf_id = info.btf_id; 3813 } else { 3814 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 3815 } 3816 /* remember the offset of last byte accessed in ctx */ 3817 if (env->prog->aux->max_ctx_offset < off + size) 3818 env->prog->aux->max_ctx_offset = off + size; 3819 return 0; 3820 } 3821 3822 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 3823 return -EACCES; 3824 } 3825 3826 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 3827 int size) 3828 { 3829 if (size < 0 || off < 0 || 3830 (u64)off + size > sizeof(struct bpf_flow_keys)) { 3831 verbose(env, "invalid access to flow keys off=%d size=%d\n", 3832 off, size); 3833 return -EACCES; 3834 } 3835 return 0; 3836 } 3837 3838 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 3839 u32 regno, int off, int size, 3840 enum bpf_access_type t) 3841 { 3842 struct bpf_reg_state *regs = cur_regs(env); 3843 struct bpf_reg_state *reg = ®s[regno]; 3844 struct bpf_insn_access_aux info = {}; 3845 bool valid; 3846 3847 if (reg->smin_value < 0) { 3848 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 3849 regno); 3850 return -EACCES; 3851 } 3852 3853 switch (reg->type) { 3854 case PTR_TO_SOCK_COMMON: 3855 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 3856 break; 3857 case PTR_TO_SOCKET: 3858 valid = bpf_sock_is_valid_access(off, size, t, &info); 3859 break; 3860 case PTR_TO_TCP_SOCK: 3861 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 3862 break; 3863 case PTR_TO_XDP_SOCK: 3864 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); 3865 break; 3866 default: 3867 valid = false; 3868 } 3869 3870 3871 if (valid) { 3872 env->insn_aux_data[insn_idx].ctx_field_size = 3873 info.ctx_field_size; 3874 return 0; 3875 } 3876 3877 verbose(env, "R%d invalid %s access off=%d size=%d\n", 3878 regno, reg_type_str(env, reg->type), off, size); 3879 3880 return -EACCES; 3881 } 3882 3883 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 3884 { 3885 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 3886 } 3887 3888 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 3889 { 3890 const struct bpf_reg_state *reg = reg_state(env, regno); 3891 3892 return reg->type == PTR_TO_CTX; 3893 } 3894 3895 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 3896 { 3897 const struct bpf_reg_state *reg = reg_state(env, regno); 3898 3899 return type_is_sk_pointer(reg->type); 3900 } 3901 3902 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 3903 { 3904 const struct bpf_reg_state *reg = reg_state(env, regno); 3905 3906 return type_is_pkt_pointer(reg->type); 3907 } 3908 3909 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 3910 { 3911 const struct bpf_reg_state *reg = reg_state(env, regno); 3912 3913 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 3914 return reg->type == PTR_TO_FLOW_KEYS; 3915 } 3916 3917 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 3918 const struct bpf_reg_state *reg, 3919 int off, int size, bool strict) 3920 { 3921 struct tnum reg_off; 3922 int ip_align; 3923 3924 /* Byte size accesses are always allowed. */ 3925 if (!strict || size == 1) 3926 return 0; 3927 3928 /* For platforms that do not have a Kconfig enabling 3929 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 3930 * NET_IP_ALIGN is universally set to '2'. And on platforms 3931 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 3932 * to this code only in strict mode where we want to emulate 3933 * the NET_IP_ALIGN==2 checking. Therefore use an 3934 * unconditional IP align value of '2'. 3935 */ 3936 ip_align = 2; 3937 3938 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 3939 if (!tnum_is_aligned(reg_off, size)) { 3940 char tn_buf[48]; 3941 3942 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3943 verbose(env, 3944 "misaligned packet access off %d+%s+%d+%d size %d\n", 3945 ip_align, tn_buf, reg->off, off, size); 3946 return -EACCES; 3947 } 3948 3949 return 0; 3950 } 3951 3952 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 3953 const struct bpf_reg_state *reg, 3954 const char *pointer_desc, 3955 int off, int size, bool strict) 3956 { 3957 struct tnum reg_off; 3958 3959 /* Byte size accesses are always allowed. */ 3960 if (!strict || size == 1) 3961 return 0; 3962 3963 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 3964 if (!tnum_is_aligned(reg_off, size)) { 3965 char tn_buf[48]; 3966 3967 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 3968 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 3969 pointer_desc, tn_buf, reg->off, off, size); 3970 return -EACCES; 3971 } 3972 3973 return 0; 3974 } 3975 3976 static int check_ptr_alignment(struct bpf_verifier_env *env, 3977 const struct bpf_reg_state *reg, int off, 3978 int size, bool strict_alignment_once) 3979 { 3980 bool strict = env->strict_alignment || strict_alignment_once; 3981 const char *pointer_desc = ""; 3982 3983 switch (reg->type) { 3984 case PTR_TO_PACKET: 3985 case PTR_TO_PACKET_META: 3986 /* Special case, because of NET_IP_ALIGN. Given metadata sits 3987 * right in front, treat it the very same way. 3988 */ 3989 return check_pkt_ptr_alignment(env, reg, off, size, strict); 3990 case PTR_TO_FLOW_KEYS: 3991 pointer_desc = "flow keys "; 3992 break; 3993 case PTR_TO_MAP_KEY: 3994 pointer_desc = "key "; 3995 break; 3996 case PTR_TO_MAP_VALUE: 3997 pointer_desc = "value "; 3998 break; 3999 case PTR_TO_CTX: 4000 pointer_desc = "context "; 4001 break; 4002 case PTR_TO_STACK: 4003 pointer_desc = "stack "; 4004 /* The stack spill tracking logic in check_stack_write_fixed_off() 4005 * and check_stack_read_fixed_off() relies on stack accesses being 4006 * aligned. 4007 */ 4008 strict = true; 4009 break; 4010 case PTR_TO_SOCKET: 4011 pointer_desc = "sock "; 4012 break; 4013 case PTR_TO_SOCK_COMMON: 4014 pointer_desc = "sock_common "; 4015 break; 4016 case PTR_TO_TCP_SOCK: 4017 pointer_desc = "tcp_sock "; 4018 break; 4019 case PTR_TO_XDP_SOCK: 4020 pointer_desc = "xdp_sock "; 4021 break; 4022 default: 4023 break; 4024 } 4025 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 4026 strict); 4027 } 4028 4029 static int update_stack_depth(struct bpf_verifier_env *env, 4030 const struct bpf_func_state *func, 4031 int off) 4032 { 4033 u16 stack = env->subprog_info[func->subprogno].stack_depth; 4034 4035 if (stack >= -off) 4036 return 0; 4037 4038 /* update known max for given subprogram */ 4039 env->subprog_info[func->subprogno].stack_depth = -off; 4040 return 0; 4041 } 4042 4043 /* starting from main bpf function walk all instructions of the function 4044 * and recursively walk all callees that given function can call. 4045 * Ignore jump and exit insns. 4046 * Since recursion is prevented by check_cfg() this algorithm 4047 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 4048 */ 4049 static int check_max_stack_depth(struct bpf_verifier_env *env) 4050 { 4051 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 4052 struct bpf_subprog_info *subprog = env->subprog_info; 4053 struct bpf_insn *insn = env->prog->insnsi; 4054 bool tail_call_reachable = false; 4055 int ret_insn[MAX_CALL_FRAMES]; 4056 int ret_prog[MAX_CALL_FRAMES]; 4057 int j; 4058 4059 process_func: 4060 /* protect against potential stack overflow that might happen when 4061 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack 4062 * depth for such case down to 256 so that the worst case scenario 4063 * would result in 8k stack size (32 which is tailcall limit * 256 = 4064 * 8k). 4065 * 4066 * To get the idea what might happen, see an example: 4067 * func1 -> sub rsp, 128 4068 * subfunc1 -> sub rsp, 256 4069 * tailcall1 -> add rsp, 256 4070 * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) 4071 * subfunc2 -> sub rsp, 64 4072 * subfunc22 -> sub rsp, 128 4073 * tailcall2 -> add rsp, 128 4074 * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) 4075 * 4076 * tailcall will unwind the current stack frame but it will not get rid 4077 * of caller's stack as shown on the example above. 4078 */ 4079 if (idx && subprog[idx].has_tail_call && depth >= 256) { 4080 verbose(env, 4081 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", 4082 depth); 4083 return -EACCES; 4084 } 4085 /* round up to 32-bytes, since this is granularity 4086 * of interpreter stack size 4087 */ 4088 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4089 if (depth > MAX_BPF_STACK) { 4090 verbose(env, "combined stack size of %d calls is %d. Too large\n", 4091 frame + 1, depth); 4092 return -EACCES; 4093 } 4094 continue_func: 4095 subprog_end = subprog[idx + 1].start; 4096 for (; i < subprog_end; i++) { 4097 int next_insn; 4098 4099 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) 4100 continue; 4101 /* remember insn and function to return to */ 4102 ret_insn[frame] = i + 1; 4103 ret_prog[frame] = idx; 4104 4105 /* find the callee */ 4106 next_insn = i + insn[i].imm + 1; 4107 idx = find_subprog(env, next_insn); 4108 if (idx < 0) { 4109 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4110 next_insn); 4111 return -EFAULT; 4112 } 4113 if (subprog[idx].is_async_cb) { 4114 if (subprog[idx].has_tail_call) { 4115 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); 4116 return -EFAULT; 4117 } 4118 /* async callbacks don't increase bpf prog stack size */ 4119 continue; 4120 } 4121 i = next_insn; 4122 4123 if (subprog[idx].has_tail_call) 4124 tail_call_reachable = true; 4125 4126 frame++; 4127 if (frame >= MAX_CALL_FRAMES) { 4128 verbose(env, "the call stack of %d frames is too deep !\n", 4129 frame); 4130 return -E2BIG; 4131 } 4132 goto process_func; 4133 } 4134 /* if tail call got detected across bpf2bpf calls then mark each of the 4135 * currently present subprog frames as tail call reachable subprogs; 4136 * this info will be utilized by JIT so that we will be preserving the 4137 * tail call counter throughout bpf2bpf calls combined with tailcalls 4138 */ 4139 if (tail_call_reachable) 4140 for (j = 0; j < frame; j++) 4141 subprog[ret_prog[j]].tail_call_reachable = true; 4142 if (subprog[0].tail_call_reachable) 4143 env->prog->aux->tail_call_reachable = true; 4144 4145 /* end of for() loop means the last insn of the 'subprog' 4146 * was reached. Doesn't matter whether it was JA or EXIT 4147 */ 4148 if (frame == 0) 4149 return 0; 4150 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 4151 frame--; 4152 i = ret_insn[frame]; 4153 idx = ret_prog[frame]; 4154 goto continue_func; 4155 } 4156 4157 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 4158 static int get_callee_stack_depth(struct bpf_verifier_env *env, 4159 const struct bpf_insn *insn, int idx) 4160 { 4161 int start = idx + insn->imm + 1, subprog; 4162 4163 subprog = find_subprog(env, start); 4164 if (subprog < 0) { 4165 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 4166 start); 4167 return -EFAULT; 4168 } 4169 return env->subprog_info[subprog].stack_depth; 4170 } 4171 #endif 4172 4173 static int __check_buffer_access(struct bpf_verifier_env *env, 4174 const char *buf_info, 4175 const struct bpf_reg_state *reg, 4176 int regno, int off, int size) 4177 { 4178 if (off < 0) { 4179 verbose(env, 4180 "R%d invalid %s buffer access: off=%d, size=%d\n", 4181 regno, buf_info, off, size); 4182 return -EACCES; 4183 } 4184 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4185 char tn_buf[48]; 4186 4187 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4188 verbose(env, 4189 "R%d invalid variable buffer offset: off=%d, var_off=%s\n", 4190 regno, off, tn_buf); 4191 return -EACCES; 4192 } 4193 4194 return 0; 4195 } 4196 4197 static int check_tp_buffer_access(struct bpf_verifier_env *env, 4198 const struct bpf_reg_state *reg, 4199 int regno, int off, int size) 4200 { 4201 int err; 4202 4203 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); 4204 if (err) 4205 return err; 4206 4207 if (off + size > env->prog->aux->max_tp_access) 4208 env->prog->aux->max_tp_access = off + size; 4209 4210 return 0; 4211 } 4212 4213 static int check_buffer_access(struct bpf_verifier_env *env, 4214 const struct bpf_reg_state *reg, 4215 int regno, int off, int size, 4216 bool zero_size_allowed, 4217 u32 *max_access) 4218 { 4219 const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr"; 4220 int err; 4221 4222 err = __check_buffer_access(env, buf_info, reg, regno, off, size); 4223 if (err) 4224 return err; 4225 4226 if (off + size > *max_access) 4227 *max_access = off + size; 4228 4229 return 0; 4230 } 4231 4232 /* BPF architecture zero extends alu32 ops into 64-bit registesr */ 4233 static void zext_32_to_64(struct bpf_reg_state *reg) 4234 { 4235 reg->var_off = tnum_subreg(reg->var_off); 4236 __reg_assign_32_into_64(reg); 4237 } 4238 4239 /* truncate register to smaller size (in bytes) 4240 * must be called with size < BPF_REG_SIZE 4241 */ 4242 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 4243 { 4244 u64 mask; 4245 4246 /* clear high bits in bit representation */ 4247 reg->var_off = tnum_cast(reg->var_off, size); 4248 4249 /* fix arithmetic bounds */ 4250 mask = ((u64)1 << (size * 8)) - 1; 4251 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 4252 reg->umin_value &= mask; 4253 reg->umax_value &= mask; 4254 } else { 4255 reg->umin_value = 0; 4256 reg->umax_value = mask; 4257 } 4258 reg->smin_value = reg->umin_value; 4259 reg->smax_value = reg->umax_value; 4260 4261 /* If size is smaller than 32bit register the 32bit register 4262 * values are also truncated so we push 64-bit bounds into 4263 * 32-bit bounds. Above were truncated < 32-bits already. 4264 */ 4265 if (size >= 4) 4266 return; 4267 __reg_combine_64_into_32(reg); 4268 } 4269 4270 static bool bpf_map_is_rdonly(const struct bpf_map *map) 4271 { 4272 /* A map is considered read-only if the following condition are true: 4273 * 4274 * 1) BPF program side cannot change any of the map content. The 4275 * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map 4276 * and was set at map creation time. 4277 * 2) The map value(s) have been initialized from user space by a 4278 * loader and then "frozen", such that no new map update/delete 4279 * operations from syscall side are possible for the rest of 4280 * the map's lifetime from that point onwards. 4281 * 3) Any parallel/pending map update/delete operations from syscall 4282 * side have been completed. Only after that point, it's safe to 4283 * assume that map value(s) are immutable. 4284 */ 4285 return (map->map_flags & BPF_F_RDONLY_PROG) && 4286 READ_ONCE(map->frozen) && 4287 !bpf_map_write_active(map); 4288 } 4289 4290 static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) 4291 { 4292 void *ptr; 4293 u64 addr; 4294 int err; 4295 4296 err = map->ops->map_direct_value_addr(map, &addr, off); 4297 if (err) 4298 return err; 4299 ptr = (void *)(long)addr + off; 4300 4301 switch (size) { 4302 case sizeof(u8): 4303 *val = (u64)*(u8 *)ptr; 4304 break; 4305 case sizeof(u16): 4306 *val = (u64)*(u16 *)ptr; 4307 break; 4308 case sizeof(u32): 4309 *val = (u64)*(u32 *)ptr; 4310 break; 4311 case sizeof(u64): 4312 *val = *(u64 *)ptr; 4313 break; 4314 default: 4315 return -EINVAL; 4316 } 4317 return 0; 4318 } 4319 4320 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, 4321 struct bpf_reg_state *regs, 4322 int regno, int off, int size, 4323 enum bpf_access_type atype, 4324 int value_regno) 4325 { 4326 struct bpf_reg_state *reg = regs + regno; 4327 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); 4328 const char *tname = btf_name_by_offset(reg->btf, t->name_off); 4329 enum bpf_type_flag flag = 0; 4330 u32 btf_id; 4331 int ret; 4332 4333 if (off < 0) { 4334 verbose(env, 4335 "R%d is ptr_%s invalid negative access: off=%d\n", 4336 regno, tname, off); 4337 return -EACCES; 4338 } 4339 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 4340 char tn_buf[48]; 4341 4342 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4343 verbose(env, 4344 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", 4345 regno, tname, off, tn_buf); 4346 return -EACCES; 4347 } 4348 4349 if (reg->type & MEM_USER) { 4350 verbose(env, 4351 "R%d is ptr_%s access user memory: off=%d\n", 4352 regno, tname, off); 4353 return -EACCES; 4354 } 4355 4356 if (reg->type & MEM_PERCPU) { 4357 verbose(env, 4358 "R%d is ptr_%s access percpu memory: off=%d\n", 4359 regno, tname, off); 4360 return -EACCES; 4361 } 4362 4363 if (env->ops->btf_struct_access) { 4364 ret = env->ops->btf_struct_access(&env->log, reg->btf, t, 4365 off, size, atype, &btf_id, &flag); 4366 } else { 4367 if (atype != BPF_READ) { 4368 verbose(env, "only read is supported\n"); 4369 return -EACCES; 4370 } 4371 4372 ret = btf_struct_access(&env->log, reg->btf, t, off, size, 4373 atype, &btf_id, &flag); 4374 } 4375 4376 if (ret < 0) 4377 return ret; 4378 4379 /* If this is an untrusted pointer, all pointers formed by walking it 4380 * also inherit the untrusted flag. 4381 */ 4382 if (type_flag(reg->type) & PTR_UNTRUSTED) 4383 flag |= PTR_UNTRUSTED; 4384 4385 if (atype == BPF_READ && value_regno >= 0) 4386 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); 4387 4388 return 0; 4389 } 4390 4391 static int check_ptr_to_map_access(struct bpf_verifier_env *env, 4392 struct bpf_reg_state *regs, 4393 int regno, int off, int size, 4394 enum bpf_access_type atype, 4395 int value_regno) 4396 { 4397 struct bpf_reg_state *reg = regs + regno; 4398 struct bpf_map *map = reg->map_ptr; 4399 enum bpf_type_flag flag = 0; 4400 const struct btf_type *t; 4401 const char *tname; 4402 u32 btf_id; 4403 int ret; 4404 4405 if (!btf_vmlinux) { 4406 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); 4407 return -ENOTSUPP; 4408 } 4409 4410 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { 4411 verbose(env, "map_ptr access not supported for map type %d\n", 4412 map->map_type); 4413 return -ENOTSUPP; 4414 } 4415 4416 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); 4417 tname = btf_name_by_offset(btf_vmlinux, t->name_off); 4418 4419 if (!env->allow_ptr_to_map_access) { 4420 verbose(env, 4421 "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", 4422 tname); 4423 return -EPERM; 4424 } 4425 4426 if (off < 0) { 4427 verbose(env, "R%d is %s invalid negative access: off=%d\n", 4428 regno, tname, off); 4429 return -EACCES; 4430 } 4431 4432 if (atype != BPF_READ) { 4433 verbose(env, "only read from %s is supported\n", tname); 4434 return -EACCES; 4435 } 4436 4437 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id, &flag); 4438 if (ret < 0) 4439 return ret; 4440 4441 if (value_regno >= 0) 4442 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); 4443 4444 return 0; 4445 } 4446 4447 /* Check that the stack access at the given offset is within bounds. The 4448 * maximum valid offset is -1. 4449 * 4450 * The minimum valid offset is -MAX_BPF_STACK for writes, and 4451 * -state->allocated_stack for reads. 4452 */ 4453 static int check_stack_slot_within_bounds(int off, 4454 struct bpf_func_state *state, 4455 enum bpf_access_type t) 4456 { 4457 int min_valid_off; 4458 4459 if (t == BPF_WRITE) 4460 min_valid_off = -MAX_BPF_STACK; 4461 else 4462 min_valid_off = -state->allocated_stack; 4463 4464 if (off < min_valid_off || off > -1) 4465 return -EACCES; 4466 return 0; 4467 } 4468 4469 /* Check that the stack access at 'regno + off' falls within the maximum stack 4470 * bounds. 4471 * 4472 * 'off' includes `regno->offset`, but not its dynamic part (if any). 4473 */ 4474 static int check_stack_access_within_bounds( 4475 struct bpf_verifier_env *env, 4476 int regno, int off, int access_size, 4477 enum bpf_access_src src, enum bpf_access_type type) 4478 { 4479 struct bpf_reg_state *regs = cur_regs(env); 4480 struct bpf_reg_state *reg = regs + regno; 4481 struct bpf_func_state *state = func(env, reg); 4482 int min_off, max_off; 4483 int err; 4484 char *err_extra; 4485 4486 if (src == ACCESS_HELPER) 4487 /* We don't know if helpers are reading or writing (or both). */ 4488 err_extra = " indirect access to"; 4489 else if (type == BPF_READ) 4490 err_extra = " read from"; 4491 else 4492 err_extra = " write to"; 4493 4494 if (tnum_is_const(reg->var_off)) { 4495 min_off = reg->var_off.value + off; 4496 if (access_size > 0) 4497 max_off = min_off + access_size - 1; 4498 else 4499 max_off = min_off; 4500 } else { 4501 if (reg->smax_value >= BPF_MAX_VAR_OFF || 4502 reg->smin_value <= -BPF_MAX_VAR_OFF) { 4503 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", 4504 err_extra, regno); 4505 return -EACCES; 4506 } 4507 min_off = reg->smin_value + off; 4508 if (access_size > 0) 4509 max_off = reg->smax_value + off + access_size - 1; 4510 else 4511 max_off = min_off; 4512 } 4513 4514 err = check_stack_slot_within_bounds(min_off, state, type); 4515 if (!err) 4516 err = check_stack_slot_within_bounds(max_off, state, type); 4517 4518 if (err) { 4519 if (tnum_is_const(reg->var_off)) { 4520 verbose(env, "invalid%s stack R%d off=%d size=%d\n", 4521 err_extra, regno, off, access_size); 4522 } else { 4523 char tn_buf[48]; 4524 4525 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4526 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", 4527 err_extra, regno, tn_buf, access_size); 4528 } 4529 } 4530 return err; 4531 } 4532 4533 /* check whether memory at (regno + off) is accessible for t = (read | write) 4534 * if t==write, value_regno is a register which value is stored into memory 4535 * if t==read, value_regno is a register which will receive the value from memory 4536 * if t==write && value_regno==-1, some unknown value is stored into memory 4537 * if t==read && value_regno==-1, don't care what we read from memory 4538 */ 4539 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 4540 int off, int bpf_size, enum bpf_access_type t, 4541 int value_regno, bool strict_alignment_once) 4542 { 4543 struct bpf_reg_state *regs = cur_regs(env); 4544 struct bpf_reg_state *reg = regs + regno; 4545 struct bpf_func_state *state; 4546 int size, err = 0; 4547 4548 size = bpf_size_to_bytes(bpf_size); 4549 if (size < 0) 4550 return size; 4551 4552 /* alignment checks will add in reg->off themselves */ 4553 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 4554 if (err) 4555 return err; 4556 4557 /* for access checks, reg->off is just part of off */ 4558 off += reg->off; 4559 4560 if (reg->type == PTR_TO_MAP_KEY) { 4561 if (t == BPF_WRITE) { 4562 verbose(env, "write to change key R%d not allowed\n", regno); 4563 return -EACCES; 4564 } 4565 4566 err = check_mem_region_access(env, regno, off, size, 4567 reg->map_ptr->key_size, false); 4568 if (err) 4569 return err; 4570 if (value_regno >= 0) 4571 mark_reg_unknown(env, regs, value_regno); 4572 } else if (reg->type == PTR_TO_MAP_VALUE) { 4573 struct bpf_map_value_off_desc *kptr_off_desc = NULL; 4574 4575 if (t == BPF_WRITE && value_regno >= 0 && 4576 is_pointer_value(env, value_regno)) { 4577 verbose(env, "R%d leaks addr into map\n", value_regno); 4578 return -EACCES; 4579 } 4580 err = check_map_access_type(env, regno, off, size, t); 4581 if (err) 4582 return err; 4583 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); 4584 if (err) 4585 return err; 4586 if (tnum_is_const(reg->var_off)) 4587 kptr_off_desc = bpf_map_kptr_off_contains(reg->map_ptr, 4588 off + reg->var_off.value); 4589 if (kptr_off_desc) { 4590 err = check_map_kptr_access(env, regno, value_regno, insn_idx, 4591 kptr_off_desc); 4592 } else if (t == BPF_READ && value_regno >= 0) { 4593 struct bpf_map *map = reg->map_ptr; 4594 4595 /* if map is read-only, track its contents as scalars */ 4596 if (tnum_is_const(reg->var_off) && 4597 bpf_map_is_rdonly(map) && 4598 map->ops->map_direct_value_addr) { 4599 int map_off = off + reg->var_off.value; 4600 u64 val = 0; 4601 4602 err = bpf_map_direct_read(map, map_off, size, 4603 &val); 4604 if (err) 4605 return err; 4606 4607 regs[value_regno].type = SCALAR_VALUE; 4608 __mark_reg_known(®s[value_regno], val); 4609 } else { 4610 mark_reg_unknown(env, regs, value_regno); 4611 } 4612 } 4613 } else if (base_type(reg->type) == PTR_TO_MEM) { 4614 bool rdonly_mem = type_is_rdonly_mem(reg->type); 4615 4616 if (type_may_be_null(reg->type)) { 4617 verbose(env, "R%d invalid mem access '%s'\n", regno, 4618 reg_type_str(env, reg->type)); 4619 return -EACCES; 4620 } 4621 4622 if (t == BPF_WRITE && rdonly_mem) { 4623 verbose(env, "R%d cannot write into %s\n", 4624 regno, reg_type_str(env, reg->type)); 4625 return -EACCES; 4626 } 4627 4628 if (t == BPF_WRITE && value_regno >= 0 && 4629 is_pointer_value(env, value_regno)) { 4630 verbose(env, "R%d leaks addr into mem\n", value_regno); 4631 return -EACCES; 4632 } 4633 4634 err = check_mem_region_access(env, regno, off, size, 4635 reg->mem_size, false); 4636 if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem)) 4637 mark_reg_unknown(env, regs, value_regno); 4638 } else if (reg->type == PTR_TO_CTX) { 4639 enum bpf_reg_type reg_type = SCALAR_VALUE; 4640 struct btf *btf = NULL; 4641 u32 btf_id = 0; 4642 4643 if (t == BPF_WRITE && value_regno >= 0 && 4644 is_pointer_value(env, value_regno)) { 4645 verbose(env, "R%d leaks addr into ctx\n", value_regno); 4646 return -EACCES; 4647 } 4648 4649 err = check_ptr_off_reg(env, reg, regno); 4650 if (err < 0) 4651 return err; 4652 4653 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, 4654 &btf_id); 4655 if (err) 4656 verbose_linfo(env, insn_idx, "; "); 4657 if (!err && t == BPF_READ && value_regno >= 0) { 4658 /* ctx access returns either a scalar, or a 4659 * PTR_TO_PACKET[_META,_END]. In the latter 4660 * case, we know the offset is zero. 4661 */ 4662 if (reg_type == SCALAR_VALUE) { 4663 mark_reg_unknown(env, regs, value_regno); 4664 } else { 4665 mark_reg_known_zero(env, regs, 4666 value_regno); 4667 if (type_may_be_null(reg_type)) 4668 regs[value_regno].id = ++env->id_gen; 4669 /* A load of ctx field could have different 4670 * actual load size with the one encoded in the 4671 * insn. When the dst is PTR, it is for sure not 4672 * a sub-register. 4673 */ 4674 regs[value_regno].subreg_def = DEF_NOT_SUBREG; 4675 if (base_type(reg_type) == PTR_TO_BTF_ID) { 4676 regs[value_regno].btf = btf; 4677 regs[value_regno].btf_id = btf_id; 4678 } 4679 } 4680 regs[value_regno].type = reg_type; 4681 } 4682 4683 } else if (reg->type == PTR_TO_STACK) { 4684 /* Basic bounds checks. */ 4685 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); 4686 if (err) 4687 return err; 4688 4689 state = func(env, reg); 4690 err = update_stack_depth(env, state, off); 4691 if (err) 4692 return err; 4693 4694 if (t == BPF_READ) 4695 err = check_stack_read(env, regno, off, size, 4696 value_regno); 4697 else 4698 err = check_stack_write(env, regno, off, size, 4699 value_regno, insn_idx); 4700 } else if (reg_is_pkt_pointer(reg)) { 4701 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 4702 verbose(env, "cannot write into packet\n"); 4703 return -EACCES; 4704 } 4705 if (t == BPF_WRITE && value_regno >= 0 && 4706 is_pointer_value(env, value_regno)) { 4707 verbose(env, "R%d leaks addr into packet\n", 4708 value_regno); 4709 return -EACCES; 4710 } 4711 err = check_packet_access(env, regno, off, size, false); 4712 if (!err && t == BPF_READ && value_regno >= 0) 4713 mark_reg_unknown(env, regs, value_regno); 4714 } else if (reg->type == PTR_TO_FLOW_KEYS) { 4715 if (t == BPF_WRITE && value_regno >= 0 && 4716 is_pointer_value(env, value_regno)) { 4717 verbose(env, "R%d leaks addr into flow keys\n", 4718 value_regno); 4719 return -EACCES; 4720 } 4721 4722 err = check_flow_keys_access(env, off, size); 4723 if (!err && t == BPF_READ && value_regno >= 0) 4724 mark_reg_unknown(env, regs, value_regno); 4725 } else if (type_is_sk_pointer(reg->type)) { 4726 if (t == BPF_WRITE) { 4727 verbose(env, "R%d cannot write into %s\n", 4728 regno, reg_type_str(env, reg->type)); 4729 return -EACCES; 4730 } 4731 err = check_sock_access(env, insn_idx, regno, off, size, t); 4732 if (!err && value_regno >= 0) 4733 mark_reg_unknown(env, regs, value_regno); 4734 } else if (reg->type == PTR_TO_TP_BUFFER) { 4735 err = check_tp_buffer_access(env, reg, regno, off, size); 4736 if (!err && t == BPF_READ && value_regno >= 0) 4737 mark_reg_unknown(env, regs, value_regno); 4738 } else if (base_type(reg->type) == PTR_TO_BTF_ID && 4739 !type_may_be_null(reg->type)) { 4740 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, 4741 value_regno); 4742 } else if (reg->type == CONST_PTR_TO_MAP) { 4743 err = check_ptr_to_map_access(env, regs, regno, off, size, t, 4744 value_regno); 4745 } else if (base_type(reg->type) == PTR_TO_BUF) { 4746 bool rdonly_mem = type_is_rdonly_mem(reg->type); 4747 u32 *max_access; 4748 4749 if (rdonly_mem) { 4750 if (t == BPF_WRITE) { 4751 verbose(env, "R%d cannot write into %s\n", 4752 regno, reg_type_str(env, reg->type)); 4753 return -EACCES; 4754 } 4755 max_access = &env->prog->aux->max_rdonly_access; 4756 } else { 4757 max_access = &env->prog->aux->max_rdwr_access; 4758 } 4759 4760 err = check_buffer_access(env, reg, regno, off, size, false, 4761 max_access); 4762 4763 if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ)) 4764 mark_reg_unknown(env, regs, value_regno); 4765 } else { 4766 verbose(env, "R%d invalid mem access '%s'\n", regno, 4767 reg_type_str(env, reg->type)); 4768 return -EACCES; 4769 } 4770 4771 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 4772 regs[value_regno].type == SCALAR_VALUE) { 4773 /* b/h/w load zero-extends, mark upper bits as known 0 */ 4774 coerce_reg_to_size(®s[value_regno], size); 4775 } 4776 return err; 4777 } 4778 4779 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 4780 { 4781 int load_reg; 4782 int err; 4783 4784 switch (insn->imm) { 4785 case BPF_ADD: 4786 case BPF_ADD | BPF_FETCH: 4787 case BPF_AND: 4788 case BPF_AND | BPF_FETCH: 4789 case BPF_OR: 4790 case BPF_OR | BPF_FETCH: 4791 case BPF_XOR: 4792 case BPF_XOR | BPF_FETCH: 4793 case BPF_XCHG: 4794 case BPF_CMPXCHG: 4795 break; 4796 default: 4797 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); 4798 return -EINVAL; 4799 } 4800 4801 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { 4802 verbose(env, "invalid atomic operand size\n"); 4803 return -EINVAL; 4804 } 4805 4806 /* check src1 operand */ 4807 err = check_reg_arg(env, insn->src_reg, SRC_OP); 4808 if (err) 4809 return err; 4810 4811 /* check src2 operand */ 4812 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 4813 if (err) 4814 return err; 4815 4816 if (insn->imm == BPF_CMPXCHG) { 4817 /* Check comparison of R0 with memory location */ 4818 const u32 aux_reg = BPF_REG_0; 4819 4820 err = check_reg_arg(env, aux_reg, SRC_OP); 4821 if (err) 4822 return err; 4823 4824 if (is_pointer_value(env, aux_reg)) { 4825 verbose(env, "R%d leaks addr into mem\n", aux_reg); 4826 return -EACCES; 4827 } 4828 } 4829 4830 if (is_pointer_value(env, insn->src_reg)) { 4831 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 4832 return -EACCES; 4833 } 4834 4835 if (is_ctx_reg(env, insn->dst_reg) || 4836 is_pkt_reg(env, insn->dst_reg) || 4837 is_flow_key_reg(env, insn->dst_reg) || 4838 is_sk_reg(env, insn->dst_reg)) { 4839 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", 4840 insn->dst_reg, 4841 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 4842 return -EACCES; 4843 } 4844 4845 if (insn->imm & BPF_FETCH) { 4846 if (insn->imm == BPF_CMPXCHG) 4847 load_reg = BPF_REG_0; 4848 else 4849 load_reg = insn->src_reg; 4850 4851 /* check and record load of old value */ 4852 err = check_reg_arg(env, load_reg, DST_OP); 4853 if (err) 4854 return err; 4855 } else { 4856 /* This instruction accesses a memory location but doesn't 4857 * actually load it into a register. 4858 */ 4859 load_reg = -1; 4860 } 4861 4862 /* Check whether we can read the memory, with second call for fetch 4863 * case to simulate the register fill. 4864 */ 4865 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4866 BPF_SIZE(insn->code), BPF_READ, -1, true); 4867 if (!err && load_reg >= 0) 4868 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4869 BPF_SIZE(insn->code), BPF_READ, load_reg, 4870 true); 4871 if (err) 4872 return err; 4873 4874 /* Check whether we can write into the same memory. */ 4875 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 4876 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 4877 if (err) 4878 return err; 4879 4880 return 0; 4881 } 4882 4883 /* When register 'regno' is used to read the stack (either directly or through 4884 * a helper function) make sure that it's within stack boundary and, depending 4885 * on the access type, that all elements of the stack are initialized. 4886 * 4887 * 'off' includes 'regno->off', but not its dynamic part (if any). 4888 * 4889 * All registers that have been spilled on the stack in the slots within the 4890 * read offsets are marked as read. 4891 */ 4892 static int check_stack_range_initialized( 4893 struct bpf_verifier_env *env, int regno, int off, 4894 int access_size, bool zero_size_allowed, 4895 enum bpf_access_src type, struct bpf_call_arg_meta *meta) 4896 { 4897 struct bpf_reg_state *reg = reg_state(env, regno); 4898 struct bpf_func_state *state = func(env, reg); 4899 int err, min_off, max_off, i, j, slot, spi; 4900 char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; 4901 enum bpf_access_type bounds_check_type; 4902 /* Some accesses can write anything into the stack, others are 4903 * read-only. 4904 */ 4905 bool clobber = false; 4906 4907 if (access_size == 0 && !zero_size_allowed) { 4908 verbose(env, "invalid zero-sized read\n"); 4909 return -EACCES; 4910 } 4911 4912 if (type == ACCESS_HELPER) { 4913 /* The bounds checks for writes are more permissive than for 4914 * reads. However, if raw_mode is not set, we'll do extra 4915 * checks below. 4916 */ 4917 bounds_check_type = BPF_WRITE; 4918 clobber = true; 4919 } else { 4920 bounds_check_type = BPF_READ; 4921 } 4922 err = check_stack_access_within_bounds(env, regno, off, access_size, 4923 type, bounds_check_type); 4924 if (err) 4925 return err; 4926 4927 4928 if (tnum_is_const(reg->var_off)) { 4929 min_off = max_off = reg->var_off.value + off; 4930 } else { 4931 /* Variable offset is prohibited for unprivileged mode for 4932 * simplicity since it requires corresponding support in 4933 * Spectre masking for stack ALU. 4934 * See also retrieve_ptr_limit(). 4935 */ 4936 if (!env->bypass_spec_v1) { 4937 char tn_buf[48]; 4938 4939 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 4940 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", 4941 regno, err_extra, tn_buf); 4942 return -EACCES; 4943 } 4944 /* Only initialized buffer on stack is allowed to be accessed 4945 * with variable offset. With uninitialized buffer it's hard to 4946 * guarantee that whole memory is marked as initialized on 4947 * helper return since specific bounds are unknown what may 4948 * cause uninitialized stack leaking. 4949 */ 4950 if (meta && meta->raw_mode) 4951 meta = NULL; 4952 4953 min_off = reg->smin_value + off; 4954 max_off = reg->smax_value + off; 4955 } 4956 4957 if (meta && meta->raw_mode) { 4958 meta->access_size = access_size; 4959 meta->regno = regno; 4960 return 0; 4961 } 4962 4963 for (i = min_off; i < max_off + access_size; i++) { 4964 u8 *stype; 4965 4966 slot = -i - 1; 4967 spi = slot / BPF_REG_SIZE; 4968 if (state->allocated_stack <= slot) 4969 goto err; 4970 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 4971 if (*stype == STACK_MISC) 4972 goto mark; 4973 if (*stype == STACK_ZERO) { 4974 if (clobber) { 4975 /* helper can write anything into the stack */ 4976 *stype = STACK_MISC; 4977 } 4978 goto mark; 4979 } 4980 4981 if (is_spilled_reg(&state->stack[spi]) && 4982 base_type(state->stack[spi].spilled_ptr.type) == PTR_TO_BTF_ID) 4983 goto mark; 4984 4985 if (is_spilled_reg(&state->stack[spi]) && 4986 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || 4987 env->allow_ptr_leaks)) { 4988 if (clobber) { 4989 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); 4990 for (j = 0; j < BPF_REG_SIZE; j++) 4991 scrub_spilled_slot(&state->stack[spi].slot_type[j]); 4992 } 4993 goto mark; 4994 } 4995 4996 err: 4997 if (tnum_is_const(reg->var_off)) { 4998 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", 4999 err_extra, regno, min_off, i - min_off, access_size); 5000 } else { 5001 char tn_buf[48]; 5002 5003 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5004 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", 5005 err_extra, regno, tn_buf, i - min_off, access_size); 5006 } 5007 return -EACCES; 5008 mark: 5009 /* reading any byte out of 8-byte 'spill_slot' will cause 5010 * the whole slot to be marked as 'read' 5011 */ 5012 mark_reg_read(env, &state->stack[spi].spilled_ptr, 5013 state->stack[spi].spilled_ptr.parent, 5014 REG_LIVE_READ64); 5015 } 5016 return update_stack_depth(env, state, min_off); 5017 } 5018 5019 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 5020 int access_size, bool zero_size_allowed, 5021 struct bpf_call_arg_meta *meta) 5022 { 5023 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5024 u32 *max_access; 5025 5026 switch (base_type(reg->type)) { 5027 case PTR_TO_PACKET: 5028 case PTR_TO_PACKET_META: 5029 return check_packet_access(env, regno, reg->off, access_size, 5030 zero_size_allowed); 5031 case PTR_TO_MAP_KEY: 5032 if (meta && meta->raw_mode) { 5033 verbose(env, "R%d cannot write into %s\n", regno, 5034 reg_type_str(env, reg->type)); 5035 return -EACCES; 5036 } 5037 return check_mem_region_access(env, regno, reg->off, access_size, 5038 reg->map_ptr->key_size, false); 5039 case PTR_TO_MAP_VALUE: 5040 if (check_map_access_type(env, regno, reg->off, access_size, 5041 meta && meta->raw_mode ? BPF_WRITE : 5042 BPF_READ)) 5043 return -EACCES; 5044 return check_map_access(env, regno, reg->off, access_size, 5045 zero_size_allowed, ACCESS_HELPER); 5046 case PTR_TO_MEM: 5047 if (type_is_rdonly_mem(reg->type)) { 5048 if (meta && meta->raw_mode) { 5049 verbose(env, "R%d cannot write into %s\n", regno, 5050 reg_type_str(env, reg->type)); 5051 return -EACCES; 5052 } 5053 } 5054 return check_mem_region_access(env, regno, reg->off, 5055 access_size, reg->mem_size, 5056 zero_size_allowed); 5057 case PTR_TO_BUF: 5058 if (type_is_rdonly_mem(reg->type)) { 5059 if (meta && meta->raw_mode) { 5060 verbose(env, "R%d cannot write into %s\n", regno, 5061 reg_type_str(env, reg->type)); 5062 return -EACCES; 5063 } 5064 5065 max_access = &env->prog->aux->max_rdonly_access; 5066 } else { 5067 max_access = &env->prog->aux->max_rdwr_access; 5068 } 5069 return check_buffer_access(env, reg, regno, reg->off, 5070 access_size, zero_size_allowed, 5071 max_access); 5072 case PTR_TO_STACK: 5073 return check_stack_range_initialized( 5074 env, 5075 regno, reg->off, access_size, 5076 zero_size_allowed, ACCESS_HELPER, meta); 5077 default: /* scalar_value or invalid ptr */ 5078 /* Allow zero-byte read from NULL, regardless of pointer type */ 5079 if (zero_size_allowed && access_size == 0 && 5080 register_is_null(reg)) 5081 return 0; 5082 5083 verbose(env, "R%d type=%s ", regno, 5084 reg_type_str(env, reg->type)); 5085 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); 5086 return -EACCES; 5087 } 5088 } 5089 5090 static int check_mem_size_reg(struct bpf_verifier_env *env, 5091 struct bpf_reg_state *reg, u32 regno, 5092 bool zero_size_allowed, 5093 struct bpf_call_arg_meta *meta) 5094 { 5095 int err; 5096 5097 /* This is used to refine r0 return value bounds for helpers 5098 * that enforce this value as an upper bound on return values. 5099 * See do_refine_retval_range() for helpers that can refine 5100 * the return value. C type of helper is u32 so we pull register 5101 * bound from umax_value however, if negative verifier errors 5102 * out. Only upper bounds can be learned because retval is an 5103 * int type and negative retvals are allowed. 5104 */ 5105 meta->msize_max_value = reg->umax_value; 5106 5107 /* The register is SCALAR_VALUE; the access check 5108 * happens using its boundaries. 5109 */ 5110 if (!tnum_is_const(reg->var_off)) 5111 /* For unprivileged variable accesses, disable raw 5112 * mode so that the program is required to 5113 * initialize all the memory that the helper could 5114 * just partially fill up. 5115 */ 5116 meta = NULL; 5117 5118 if (reg->smin_value < 0) { 5119 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 5120 regno); 5121 return -EACCES; 5122 } 5123 5124 if (reg->umin_value == 0) { 5125 err = check_helper_mem_access(env, regno - 1, 0, 5126 zero_size_allowed, 5127 meta); 5128 if (err) 5129 return err; 5130 } 5131 5132 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 5133 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 5134 regno); 5135 return -EACCES; 5136 } 5137 err = check_helper_mem_access(env, regno - 1, 5138 reg->umax_value, 5139 zero_size_allowed, meta); 5140 if (!err) 5141 err = mark_chain_precision(env, regno); 5142 return err; 5143 } 5144 5145 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5146 u32 regno, u32 mem_size) 5147 { 5148 bool may_be_null = type_may_be_null(reg->type); 5149 struct bpf_reg_state saved_reg; 5150 struct bpf_call_arg_meta meta; 5151 int err; 5152 5153 if (register_is_null(reg)) 5154 return 0; 5155 5156 memset(&meta, 0, sizeof(meta)); 5157 /* Assuming that the register contains a value check if the memory 5158 * access is safe. Temporarily save and restore the register's state as 5159 * the conversion shouldn't be visible to a caller. 5160 */ 5161 if (may_be_null) { 5162 saved_reg = *reg; 5163 mark_ptr_not_null_reg(reg); 5164 } 5165 5166 err = check_helper_mem_access(env, regno, mem_size, true, &meta); 5167 /* Check access for BPF_WRITE */ 5168 meta.raw_mode = true; 5169 err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta); 5170 5171 if (may_be_null) 5172 *reg = saved_reg; 5173 5174 return err; 5175 } 5176 5177 int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, 5178 u32 regno) 5179 { 5180 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; 5181 bool may_be_null = type_may_be_null(mem_reg->type); 5182 struct bpf_reg_state saved_reg; 5183 struct bpf_call_arg_meta meta; 5184 int err; 5185 5186 WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5); 5187 5188 memset(&meta, 0, sizeof(meta)); 5189 5190 if (may_be_null) { 5191 saved_reg = *mem_reg; 5192 mark_ptr_not_null_reg(mem_reg); 5193 } 5194 5195 err = check_mem_size_reg(env, reg, regno, true, &meta); 5196 /* Check access for BPF_WRITE */ 5197 meta.raw_mode = true; 5198 err = err ?: check_mem_size_reg(env, reg, regno, true, &meta); 5199 5200 if (may_be_null) 5201 *mem_reg = saved_reg; 5202 return err; 5203 } 5204 5205 /* Implementation details: 5206 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL 5207 * Two bpf_map_lookups (even with the same key) will have different reg->id. 5208 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after 5209 * value_or_null->value transition, since the verifier only cares about 5210 * the range of access to valid map value pointer and doesn't care about actual 5211 * address of the map element. 5212 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 5213 * reg->id > 0 after value_or_null->value transition. By doing so 5214 * two bpf_map_lookups will be considered two different pointers that 5215 * point to different bpf_spin_locks. 5216 * The verifier allows taking only one bpf_spin_lock at a time to avoid 5217 * dead-locks. 5218 * Since only one bpf_spin_lock is allowed the checks are simpler than 5219 * reg_is_refcounted() logic. The verifier needs to remember only 5220 * one spin_lock instead of array of acquired_refs. 5221 * cur_state->active_spin_lock remembers which map value element got locked 5222 * and clears it after bpf_spin_unlock. 5223 */ 5224 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 5225 bool is_lock) 5226 { 5227 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5228 struct bpf_verifier_state *cur = env->cur_state; 5229 bool is_const = tnum_is_const(reg->var_off); 5230 struct bpf_map *map = reg->map_ptr; 5231 u64 val = reg->var_off.value; 5232 5233 if (!is_const) { 5234 verbose(env, 5235 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 5236 regno); 5237 return -EINVAL; 5238 } 5239 if (!map->btf) { 5240 verbose(env, 5241 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 5242 map->name); 5243 return -EINVAL; 5244 } 5245 if (!map_value_has_spin_lock(map)) { 5246 if (map->spin_lock_off == -E2BIG) 5247 verbose(env, 5248 "map '%s' has more than one 'struct bpf_spin_lock'\n", 5249 map->name); 5250 else if (map->spin_lock_off == -ENOENT) 5251 verbose(env, 5252 "map '%s' doesn't have 'struct bpf_spin_lock'\n", 5253 map->name); 5254 else 5255 verbose(env, 5256 "map '%s' is not a struct type or bpf_spin_lock is mangled\n", 5257 map->name); 5258 return -EINVAL; 5259 } 5260 if (map->spin_lock_off != val + reg->off) { 5261 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", 5262 val + reg->off); 5263 return -EINVAL; 5264 } 5265 if (is_lock) { 5266 if (cur->active_spin_lock) { 5267 verbose(env, 5268 "Locking two bpf_spin_locks are not allowed\n"); 5269 return -EINVAL; 5270 } 5271 cur->active_spin_lock = reg->id; 5272 } else { 5273 if (!cur->active_spin_lock) { 5274 verbose(env, "bpf_spin_unlock without taking a lock\n"); 5275 return -EINVAL; 5276 } 5277 if (cur->active_spin_lock != reg->id) { 5278 verbose(env, "bpf_spin_unlock of different lock\n"); 5279 return -EINVAL; 5280 } 5281 cur->active_spin_lock = 0; 5282 } 5283 return 0; 5284 } 5285 5286 static int process_timer_func(struct bpf_verifier_env *env, int regno, 5287 struct bpf_call_arg_meta *meta) 5288 { 5289 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5290 bool is_const = tnum_is_const(reg->var_off); 5291 struct bpf_map *map = reg->map_ptr; 5292 u64 val = reg->var_off.value; 5293 5294 if (!is_const) { 5295 verbose(env, 5296 "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n", 5297 regno); 5298 return -EINVAL; 5299 } 5300 if (!map->btf) { 5301 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", 5302 map->name); 5303 return -EINVAL; 5304 } 5305 if (!map_value_has_timer(map)) { 5306 if (map->timer_off == -E2BIG) 5307 verbose(env, 5308 "map '%s' has more than one 'struct bpf_timer'\n", 5309 map->name); 5310 else if (map->timer_off == -ENOENT) 5311 verbose(env, 5312 "map '%s' doesn't have 'struct bpf_timer'\n", 5313 map->name); 5314 else 5315 verbose(env, 5316 "map '%s' is not a struct type or bpf_timer is mangled\n", 5317 map->name); 5318 return -EINVAL; 5319 } 5320 if (map->timer_off != val + reg->off) { 5321 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", 5322 val + reg->off, map->timer_off); 5323 return -EINVAL; 5324 } 5325 if (meta->map_ptr) { 5326 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); 5327 return -EFAULT; 5328 } 5329 meta->map_uid = reg->map_uid; 5330 meta->map_ptr = map; 5331 return 0; 5332 } 5333 5334 static int process_kptr_func(struct bpf_verifier_env *env, int regno, 5335 struct bpf_call_arg_meta *meta) 5336 { 5337 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5338 struct bpf_map_value_off_desc *off_desc; 5339 struct bpf_map *map_ptr = reg->map_ptr; 5340 u32 kptr_off; 5341 int ret; 5342 5343 if (!tnum_is_const(reg->var_off)) { 5344 verbose(env, 5345 "R%d doesn't have constant offset. kptr has to be at the constant offset\n", 5346 regno); 5347 return -EINVAL; 5348 } 5349 if (!map_ptr->btf) { 5350 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", 5351 map_ptr->name); 5352 return -EINVAL; 5353 } 5354 if (!map_value_has_kptrs(map_ptr)) { 5355 ret = PTR_ERR(map_ptr->kptr_off_tab); 5356 if (ret == -E2BIG) 5357 verbose(env, "map '%s' has more than %d kptr\n", map_ptr->name, 5358 BPF_MAP_VALUE_OFF_MAX); 5359 else if (ret == -EEXIST) 5360 verbose(env, "map '%s' has repeating kptr BTF tags\n", map_ptr->name); 5361 else 5362 verbose(env, "map '%s' has no valid kptr\n", map_ptr->name); 5363 return -EINVAL; 5364 } 5365 5366 meta->map_ptr = map_ptr; 5367 kptr_off = reg->off + reg->var_off.value; 5368 off_desc = bpf_map_kptr_off_contains(map_ptr, kptr_off); 5369 if (!off_desc) { 5370 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); 5371 return -EACCES; 5372 } 5373 if (off_desc->type != BPF_KPTR_REF) { 5374 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); 5375 return -EACCES; 5376 } 5377 meta->kptr_off_desc = off_desc; 5378 return 0; 5379 } 5380 5381 static bool arg_type_is_mem_ptr(enum bpf_arg_type type) 5382 { 5383 return base_type(type) == ARG_PTR_TO_MEM || 5384 base_type(type) == ARG_PTR_TO_UNINIT_MEM; 5385 } 5386 5387 static bool arg_type_is_mem_size(enum bpf_arg_type type) 5388 { 5389 return type == ARG_CONST_SIZE || 5390 type == ARG_CONST_SIZE_OR_ZERO; 5391 } 5392 5393 static bool arg_type_is_alloc_size(enum bpf_arg_type type) 5394 { 5395 return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; 5396 } 5397 5398 static bool arg_type_is_int_ptr(enum bpf_arg_type type) 5399 { 5400 return type == ARG_PTR_TO_INT || 5401 type == ARG_PTR_TO_LONG; 5402 } 5403 5404 static bool arg_type_is_release(enum bpf_arg_type type) 5405 { 5406 return type & OBJ_RELEASE; 5407 } 5408 5409 static int int_ptr_type_to_size(enum bpf_arg_type type) 5410 { 5411 if (type == ARG_PTR_TO_INT) 5412 return sizeof(u32); 5413 else if (type == ARG_PTR_TO_LONG) 5414 return sizeof(u64); 5415 5416 return -EINVAL; 5417 } 5418 5419 static int resolve_map_arg_type(struct bpf_verifier_env *env, 5420 const struct bpf_call_arg_meta *meta, 5421 enum bpf_arg_type *arg_type) 5422 { 5423 if (!meta->map_ptr) { 5424 /* kernel subsystem misconfigured verifier */ 5425 verbose(env, "invalid map_ptr to access map->type\n"); 5426 return -EACCES; 5427 } 5428 5429 switch (meta->map_ptr->map_type) { 5430 case BPF_MAP_TYPE_SOCKMAP: 5431 case BPF_MAP_TYPE_SOCKHASH: 5432 if (*arg_type == ARG_PTR_TO_MAP_VALUE) { 5433 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; 5434 } else { 5435 verbose(env, "invalid arg_type for sockmap/sockhash\n"); 5436 return -EINVAL; 5437 } 5438 break; 5439 case BPF_MAP_TYPE_BLOOM_FILTER: 5440 if (meta->func_id == BPF_FUNC_map_peek_elem) 5441 *arg_type = ARG_PTR_TO_MAP_VALUE; 5442 break; 5443 default: 5444 break; 5445 } 5446 return 0; 5447 } 5448 5449 struct bpf_reg_types { 5450 const enum bpf_reg_type types[10]; 5451 u32 *btf_id; 5452 }; 5453 5454 static const struct bpf_reg_types map_key_value_types = { 5455 .types = { 5456 PTR_TO_STACK, 5457 PTR_TO_PACKET, 5458 PTR_TO_PACKET_META, 5459 PTR_TO_MAP_KEY, 5460 PTR_TO_MAP_VALUE, 5461 }, 5462 }; 5463 5464 static const struct bpf_reg_types sock_types = { 5465 .types = { 5466 PTR_TO_SOCK_COMMON, 5467 PTR_TO_SOCKET, 5468 PTR_TO_TCP_SOCK, 5469 PTR_TO_XDP_SOCK, 5470 }, 5471 }; 5472 5473 #ifdef CONFIG_NET 5474 static const struct bpf_reg_types btf_id_sock_common_types = { 5475 .types = { 5476 PTR_TO_SOCK_COMMON, 5477 PTR_TO_SOCKET, 5478 PTR_TO_TCP_SOCK, 5479 PTR_TO_XDP_SOCK, 5480 PTR_TO_BTF_ID, 5481 }, 5482 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], 5483 }; 5484 #endif 5485 5486 static const struct bpf_reg_types mem_types = { 5487 .types = { 5488 PTR_TO_STACK, 5489 PTR_TO_PACKET, 5490 PTR_TO_PACKET_META, 5491 PTR_TO_MAP_KEY, 5492 PTR_TO_MAP_VALUE, 5493 PTR_TO_MEM, 5494 PTR_TO_MEM | MEM_ALLOC, 5495 PTR_TO_BUF, 5496 }, 5497 }; 5498 5499 static const struct bpf_reg_types int_ptr_types = { 5500 .types = { 5501 PTR_TO_STACK, 5502 PTR_TO_PACKET, 5503 PTR_TO_PACKET_META, 5504 PTR_TO_MAP_KEY, 5505 PTR_TO_MAP_VALUE, 5506 }, 5507 }; 5508 5509 static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; 5510 static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; 5511 static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; 5512 static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM | MEM_ALLOC } }; 5513 static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; 5514 static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; 5515 static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; 5516 static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_BTF_ID | MEM_PERCPU } }; 5517 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } }; 5518 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } }; 5519 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5520 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; 5521 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; 5522 5523 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { 5524 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, 5525 [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, 5526 [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, 5527 [ARG_CONST_SIZE] = &scalar_types, 5528 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, 5529 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, 5530 [ARG_CONST_MAP_PTR] = &const_map_ptr_types, 5531 [ARG_PTR_TO_CTX] = &context_types, 5532 [ARG_PTR_TO_SOCK_COMMON] = &sock_types, 5533 #ifdef CONFIG_NET 5534 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, 5535 #endif 5536 [ARG_PTR_TO_SOCKET] = &fullsock_types, 5537 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, 5538 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, 5539 [ARG_PTR_TO_MEM] = &mem_types, 5540 [ARG_PTR_TO_UNINIT_MEM] = &mem_types, 5541 [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, 5542 [ARG_PTR_TO_INT] = &int_ptr_types, 5543 [ARG_PTR_TO_LONG] = &int_ptr_types, 5544 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, 5545 [ARG_PTR_TO_FUNC] = &func_ptr_types, 5546 [ARG_PTR_TO_STACK] = &stack_ptr_types, 5547 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, 5548 [ARG_PTR_TO_TIMER] = &timer_types, 5549 [ARG_PTR_TO_KPTR] = &kptr_types, 5550 }; 5551 5552 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, 5553 enum bpf_arg_type arg_type, 5554 const u32 *arg_btf_id, 5555 struct bpf_call_arg_meta *meta) 5556 { 5557 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5558 enum bpf_reg_type expected, type = reg->type; 5559 const struct bpf_reg_types *compatible; 5560 int i, j; 5561 5562 compatible = compatible_reg_types[base_type(arg_type)]; 5563 if (!compatible) { 5564 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); 5565 return -EFAULT; 5566 } 5567 5568 /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY, 5569 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY 5570 * 5571 * Same for MAYBE_NULL: 5572 * 5573 * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL, 5574 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL 5575 * 5576 * Therefore we fold these flags depending on the arg_type before comparison. 5577 */ 5578 if (arg_type & MEM_RDONLY) 5579 type &= ~MEM_RDONLY; 5580 if (arg_type & PTR_MAYBE_NULL) 5581 type &= ~PTR_MAYBE_NULL; 5582 5583 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { 5584 expected = compatible->types[i]; 5585 if (expected == NOT_INIT) 5586 break; 5587 5588 if (type == expected) 5589 goto found; 5590 } 5591 5592 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); 5593 for (j = 0; j + 1 < i; j++) 5594 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); 5595 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); 5596 return -EACCES; 5597 5598 found: 5599 if (reg->type == PTR_TO_BTF_ID) { 5600 /* For bpf_sk_release, it needs to match against first member 5601 * 'struct sock_common', hence make an exception for it. This 5602 * allows bpf_sk_release to work for multiple socket types. 5603 */ 5604 bool strict_type_match = arg_type_is_release(arg_type) && 5605 meta->func_id != BPF_FUNC_sk_release; 5606 5607 if (!arg_btf_id) { 5608 if (!compatible->btf_id) { 5609 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); 5610 return -EFAULT; 5611 } 5612 arg_btf_id = compatible->btf_id; 5613 } 5614 5615 if (meta->func_id == BPF_FUNC_kptr_xchg) { 5616 if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno)) 5617 return -EACCES; 5618 } else if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, 5619 btf_vmlinux, *arg_btf_id, 5620 strict_type_match)) { 5621 verbose(env, "R%d is of type %s but %s is expected\n", 5622 regno, kernel_type_name(reg->btf, reg->btf_id), 5623 kernel_type_name(btf_vmlinux, *arg_btf_id)); 5624 return -EACCES; 5625 } 5626 } 5627 5628 return 0; 5629 } 5630 5631 int check_func_arg_reg_off(struct bpf_verifier_env *env, 5632 const struct bpf_reg_state *reg, int regno, 5633 enum bpf_arg_type arg_type) 5634 { 5635 enum bpf_reg_type type = reg->type; 5636 bool fixed_off_ok = false; 5637 5638 switch ((u32)type) { 5639 case SCALAR_VALUE: 5640 /* Pointer types where reg offset is explicitly allowed: */ 5641 case PTR_TO_PACKET: 5642 case PTR_TO_PACKET_META: 5643 case PTR_TO_MAP_KEY: 5644 case PTR_TO_MAP_VALUE: 5645 case PTR_TO_MEM: 5646 case PTR_TO_MEM | MEM_RDONLY: 5647 case PTR_TO_MEM | MEM_ALLOC: 5648 case PTR_TO_BUF: 5649 case PTR_TO_BUF | MEM_RDONLY: 5650 case PTR_TO_STACK: 5651 /* Some of the argument types nevertheless require a 5652 * zero register offset. 5653 */ 5654 if (base_type(arg_type) != ARG_PTR_TO_ALLOC_MEM) 5655 return 0; 5656 break; 5657 /* All the rest must be rejected, except PTR_TO_BTF_ID which allows 5658 * fixed offset. 5659 */ 5660 case PTR_TO_BTF_ID: 5661 /* When referenced PTR_TO_BTF_ID is passed to release function, 5662 * it's fixed offset must be 0. In the other cases, fixed offset 5663 * can be non-zero. 5664 */ 5665 if (arg_type_is_release(arg_type) && reg->off) { 5666 verbose(env, "R%d must have zero offset when passed to release func\n", 5667 regno); 5668 return -EINVAL; 5669 } 5670 /* For arg is release pointer, fixed_off_ok must be false, but 5671 * we already checked and rejected reg->off != 0 above, so set 5672 * to true to allow fixed offset for all other cases. 5673 */ 5674 fixed_off_ok = true; 5675 break; 5676 default: 5677 break; 5678 } 5679 return __check_ptr_off_reg(env, reg, regno, fixed_off_ok); 5680 } 5681 5682 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, 5683 struct bpf_call_arg_meta *meta, 5684 const struct bpf_func_proto *fn) 5685 { 5686 u32 regno = BPF_REG_1 + arg; 5687 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 5688 enum bpf_arg_type arg_type = fn->arg_type[arg]; 5689 enum bpf_reg_type type = reg->type; 5690 int err = 0; 5691 5692 if (arg_type == ARG_DONTCARE) 5693 return 0; 5694 5695 err = check_reg_arg(env, regno, SRC_OP); 5696 if (err) 5697 return err; 5698 5699 if (arg_type == ARG_ANYTHING) { 5700 if (is_pointer_value(env, regno)) { 5701 verbose(env, "R%d leaks addr into helper function\n", 5702 regno); 5703 return -EACCES; 5704 } 5705 return 0; 5706 } 5707 5708 if (type_is_pkt_pointer(type) && 5709 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 5710 verbose(env, "helper access to the packet is not allowed\n"); 5711 return -EACCES; 5712 } 5713 5714 if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE || 5715 base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) { 5716 err = resolve_map_arg_type(env, meta, &arg_type); 5717 if (err) 5718 return err; 5719 } 5720 5721 if (register_is_null(reg) && type_may_be_null(arg_type)) 5722 /* A NULL register has a SCALAR_VALUE type, so skip 5723 * type checking. 5724 */ 5725 goto skip_type_check; 5726 5727 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg], meta); 5728 if (err) 5729 return err; 5730 5731 err = check_func_arg_reg_off(env, reg, regno, arg_type); 5732 if (err) 5733 return err; 5734 5735 skip_type_check: 5736 if (arg_type_is_release(arg_type)) { 5737 if (!reg->ref_obj_id && !register_is_null(reg)) { 5738 verbose(env, "R%d must be referenced when passed to release function\n", 5739 regno); 5740 return -EINVAL; 5741 } 5742 if (meta->release_regno) { 5743 verbose(env, "verifier internal error: more than one release argument\n"); 5744 return -EFAULT; 5745 } 5746 meta->release_regno = regno; 5747 } 5748 5749 if (reg->ref_obj_id) { 5750 if (meta->ref_obj_id) { 5751 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 5752 regno, reg->ref_obj_id, 5753 meta->ref_obj_id); 5754 return -EFAULT; 5755 } 5756 meta->ref_obj_id = reg->ref_obj_id; 5757 } 5758 5759 if (arg_type == ARG_CONST_MAP_PTR) { 5760 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 5761 if (meta->map_ptr) { 5762 /* Use map_uid (which is unique id of inner map) to reject: 5763 * inner_map1 = bpf_map_lookup_elem(outer_map, key1) 5764 * inner_map2 = bpf_map_lookup_elem(outer_map, key2) 5765 * if (inner_map1 && inner_map2) { 5766 * timer = bpf_map_lookup_elem(inner_map1); 5767 * if (timer) 5768 * // mismatch would have been allowed 5769 * bpf_timer_init(timer, inner_map2); 5770 * } 5771 * 5772 * Comparing map_ptr is enough to distinguish normal and outer maps. 5773 */ 5774 if (meta->map_ptr != reg->map_ptr || 5775 meta->map_uid != reg->map_uid) { 5776 verbose(env, 5777 "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", 5778 meta->map_uid, reg->map_uid); 5779 return -EINVAL; 5780 } 5781 } 5782 meta->map_ptr = reg->map_ptr; 5783 meta->map_uid = reg->map_uid; 5784 } else if (arg_type == ARG_PTR_TO_MAP_KEY) { 5785 /* bpf_map_xxx(..., map_ptr, ..., key) call: 5786 * check that [key, key + map->key_size) are within 5787 * stack limits and initialized 5788 */ 5789 if (!meta->map_ptr) { 5790 /* in function declaration map_ptr must come before 5791 * map_key, so that it's verified and known before 5792 * we have to check map_key here. Otherwise it means 5793 * that kernel subsystem misconfigured verifier 5794 */ 5795 verbose(env, "invalid map_ptr to access map->key\n"); 5796 return -EACCES; 5797 } 5798 err = check_helper_mem_access(env, regno, 5799 meta->map_ptr->key_size, false, 5800 NULL); 5801 } else if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE || 5802 base_type(arg_type) == ARG_PTR_TO_UNINIT_MAP_VALUE) { 5803 if (type_may_be_null(arg_type) && register_is_null(reg)) 5804 return 0; 5805 5806 /* bpf_map_xxx(..., map_ptr, ..., value) call: 5807 * check [value, value + map->value_size) validity 5808 */ 5809 if (!meta->map_ptr) { 5810 /* kernel subsystem misconfigured verifier */ 5811 verbose(env, "invalid map_ptr to access map->value\n"); 5812 return -EACCES; 5813 } 5814 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); 5815 err = check_helper_mem_access(env, regno, 5816 meta->map_ptr->value_size, false, 5817 meta); 5818 } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { 5819 if (!reg->btf_id) { 5820 verbose(env, "Helper has invalid btf_id in R%d\n", regno); 5821 return -EACCES; 5822 } 5823 meta->ret_btf = reg->btf; 5824 meta->ret_btf_id = reg->btf_id; 5825 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { 5826 if (meta->func_id == BPF_FUNC_spin_lock) { 5827 if (process_spin_lock(env, regno, true)) 5828 return -EACCES; 5829 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 5830 if (process_spin_lock(env, regno, false)) 5831 return -EACCES; 5832 } else { 5833 verbose(env, "verifier internal error\n"); 5834 return -EFAULT; 5835 } 5836 } else if (arg_type == ARG_PTR_TO_TIMER) { 5837 if (process_timer_func(env, regno, meta)) 5838 return -EACCES; 5839 } else if (arg_type == ARG_PTR_TO_FUNC) { 5840 meta->subprogno = reg->subprogno; 5841 } else if (arg_type_is_mem_ptr(arg_type)) { 5842 /* The access to this pointer is only checked when we hit the 5843 * next is_mem_size argument below. 5844 */ 5845 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); 5846 } else if (arg_type_is_mem_size(arg_type)) { 5847 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); 5848 5849 err = check_mem_size_reg(env, reg, regno, zero_size_allowed, meta); 5850 } else if (arg_type_is_alloc_size(arg_type)) { 5851 if (!tnum_is_const(reg->var_off)) { 5852 verbose(env, "R%d is not a known constant'\n", 5853 regno); 5854 return -EACCES; 5855 } 5856 meta->mem_size = reg->var_off.value; 5857 } else if (arg_type_is_int_ptr(arg_type)) { 5858 int size = int_ptr_type_to_size(arg_type); 5859 5860 err = check_helper_mem_access(env, regno, size, false, meta); 5861 if (err) 5862 return err; 5863 err = check_ptr_alignment(env, reg, 0, size, true); 5864 } else if (arg_type == ARG_PTR_TO_CONST_STR) { 5865 struct bpf_map *map = reg->map_ptr; 5866 int map_off; 5867 u64 map_addr; 5868 char *str_ptr; 5869 5870 if (!bpf_map_is_rdonly(map)) { 5871 verbose(env, "R%d does not point to a readonly map'\n", regno); 5872 return -EACCES; 5873 } 5874 5875 if (!tnum_is_const(reg->var_off)) { 5876 verbose(env, "R%d is not a constant address'\n", regno); 5877 return -EACCES; 5878 } 5879 5880 if (!map->ops->map_direct_value_addr) { 5881 verbose(env, "no direct value access support for this map type\n"); 5882 return -EACCES; 5883 } 5884 5885 err = check_map_access(env, regno, reg->off, 5886 map->value_size - reg->off, false, 5887 ACCESS_HELPER); 5888 if (err) 5889 return err; 5890 5891 map_off = reg->off + reg->var_off.value; 5892 err = map->ops->map_direct_value_addr(map, &map_addr, map_off); 5893 if (err) { 5894 verbose(env, "direct value access on string failed\n"); 5895 return err; 5896 } 5897 5898 str_ptr = (char *)(long)(map_addr); 5899 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) { 5900 verbose(env, "string is not zero-terminated\n"); 5901 return -EINVAL; 5902 } 5903 } else if (arg_type == ARG_PTR_TO_KPTR) { 5904 if (process_kptr_func(env, regno, meta)) 5905 return -EACCES; 5906 } 5907 5908 return err; 5909 } 5910 5911 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) 5912 { 5913 enum bpf_attach_type eatype = env->prog->expected_attach_type; 5914 enum bpf_prog_type type = resolve_prog_type(env->prog); 5915 5916 if (func_id != BPF_FUNC_map_update_elem) 5917 return false; 5918 5919 /* It's not possible to get access to a locked struct sock in these 5920 * contexts, so updating is safe. 5921 */ 5922 switch (type) { 5923 case BPF_PROG_TYPE_TRACING: 5924 if (eatype == BPF_TRACE_ITER) 5925 return true; 5926 break; 5927 case BPF_PROG_TYPE_SOCKET_FILTER: 5928 case BPF_PROG_TYPE_SCHED_CLS: 5929 case BPF_PROG_TYPE_SCHED_ACT: 5930 case BPF_PROG_TYPE_XDP: 5931 case BPF_PROG_TYPE_SK_REUSEPORT: 5932 case BPF_PROG_TYPE_FLOW_DISSECTOR: 5933 case BPF_PROG_TYPE_SK_LOOKUP: 5934 return true; 5935 default: 5936 break; 5937 } 5938 5939 verbose(env, "cannot update sockmap in this context\n"); 5940 return false; 5941 } 5942 5943 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) 5944 { 5945 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); 5946 } 5947 5948 static int check_map_func_compatibility(struct bpf_verifier_env *env, 5949 struct bpf_map *map, int func_id) 5950 { 5951 if (!map) 5952 return 0; 5953 5954 /* We need a two way check, first is from map perspective ... */ 5955 switch (map->map_type) { 5956 case BPF_MAP_TYPE_PROG_ARRAY: 5957 if (func_id != BPF_FUNC_tail_call) 5958 goto error; 5959 break; 5960 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5961 if (func_id != BPF_FUNC_perf_event_read && 5962 func_id != BPF_FUNC_perf_event_output && 5963 func_id != BPF_FUNC_skb_output && 5964 func_id != BPF_FUNC_perf_event_read_value && 5965 func_id != BPF_FUNC_xdp_output) 5966 goto error; 5967 break; 5968 case BPF_MAP_TYPE_RINGBUF: 5969 if (func_id != BPF_FUNC_ringbuf_output && 5970 func_id != BPF_FUNC_ringbuf_reserve && 5971 func_id != BPF_FUNC_ringbuf_query) 5972 goto error; 5973 break; 5974 case BPF_MAP_TYPE_STACK_TRACE: 5975 if (func_id != BPF_FUNC_get_stackid) 5976 goto error; 5977 break; 5978 case BPF_MAP_TYPE_CGROUP_ARRAY: 5979 if (func_id != BPF_FUNC_skb_under_cgroup && 5980 func_id != BPF_FUNC_current_task_under_cgroup) 5981 goto error; 5982 break; 5983 case BPF_MAP_TYPE_CGROUP_STORAGE: 5984 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 5985 if (func_id != BPF_FUNC_get_local_storage) 5986 goto error; 5987 break; 5988 case BPF_MAP_TYPE_DEVMAP: 5989 case BPF_MAP_TYPE_DEVMAP_HASH: 5990 if (func_id != BPF_FUNC_redirect_map && 5991 func_id != BPF_FUNC_map_lookup_elem) 5992 goto error; 5993 break; 5994 /* Restrict bpf side of cpumap and xskmap, open when use-cases 5995 * appear. 5996 */ 5997 case BPF_MAP_TYPE_CPUMAP: 5998 if (func_id != BPF_FUNC_redirect_map) 5999 goto error; 6000 break; 6001 case BPF_MAP_TYPE_XSKMAP: 6002 if (func_id != BPF_FUNC_redirect_map && 6003 func_id != BPF_FUNC_map_lookup_elem) 6004 goto error; 6005 break; 6006 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 6007 case BPF_MAP_TYPE_HASH_OF_MAPS: 6008 if (func_id != BPF_FUNC_map_lookup_elem) 6009 goto error; 6010 break; 6011 case BPF_MAP_TYPE_SOCKMAP: 6012 if (func_id != BPF_FUNC_sk_redirect_map && 6013 func_id != BPF_FUNC_sock_map_update && 6014 func_id != BPF_FUNC_map_delete_elem && 6015 func_id != BPF_FUNC_msg_redirect_map && 6016 func_id != BPF_FUNC_sk_select_reuseport && 6017 func_id != BPF_FUNC_map_lookup_elem && 6018 !may_update_sockmap(env, func_id)) 6019 goto error; 6020 break; 6021 case BPF_MAP_TYPE_SOCKHASH: 6022 if (func_id != BPF_FUNC_sk_redirect_hash && 6023 func_id != BPF_FUNC_sock_hash_update && 6024 func_id != BPF_FUNC_map_delete_elem && 6025 func_id != BPF_FUNC_msg_redirect_hash && 6026 func_id != BPF_FUNC_sk_select_reuseport && 6027 func_id != BPF_FUNC_map_lookup_elem && 6028 !may_update_sockmap(env, func_id)) 6029 goto error; 6030 break; 6031 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 6032 if (func_id != BPF_FUNC_sk_select_reuseport) 6033 goto error; 6034 break; 6035 case BPF_MAP_TYPE_QUEUE: 6036 case BPF_MAP_TYPE_STACK: 6037 if (func_id != BPF_FUNC_map_peek_elem && 6038 func_id != BPF_FUNC_map_pop_elem && 6039 func_id != BPF_FUNC_map_push_elem) 6040 goto error; 6041 break; 6042 case BPF_MAP_TYPE_SK_STORAGE: 6043 if (func_id != BPF_FUNC_sk_storage_get && 6044 func_id != BPF_FUNC_sk_storage_delete) 6045 goto error; 6046 break; 6047 case BPF_MAP_TYPE_INODE_STORAGE: 6048 if (func_id != BPF_FUNC_inode_storage_get && 6049 func_id != BPF_FUNC_inode_storage_delete) 6050 goto error; 6051 break; 6052 case BPF_MAP_TYPE_TASK_STORAGE: 6053 if (func_id != BPF_FUNC_task_storage_get && 6054 func_id != BPF_FUNC_task_storage_delete) 6055 goto error; 6056 break; 6057 case BPF_MAP_TYPE_BLOOM_FILTER: 6058 if (func_id != BPF_FUNC_map_peek_elem && 6059 func_id != BPF_FUNC_map_push_elem) 6060 goto error; 6061 break; 6062 default: 6063 break; 6064 } 6065 6066 /* ... and second from the function itself. */ 6067 switch (func_id) { 6068 case BPF_FUNC_tail_call: 6069 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 6070 goto error; 6071 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { 6072 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 6073 return -EINVAL; 6074 } 6075 break; 6076 case BPF_FUNC_perf_event_read: 6077 case BPF_FUNC_perf_event_output: 6078 case BPF_FUNC_perf_event_read_value: 6079 case BPF_FUNC_skb_output: 6080 case BPF_FUNC_xdp_output: 6081 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 6082 goto error; 6083 break; 6084 case BPF_FUNC_ringbuf_output: 6085 case BPF_FUNC_ringbuf_reserve: 6086 case BPF_FUNC_ringbuf_query: 6087 if (map->map_type != BPF_MAP_TYPE_RINGBUF) 6088 goto error; 6089 break; 6090 case BPF_FUNC_get_stackid: 6091 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 6092 goto error; 6093 break; 6094 case BPF_FUNC_current_task_under_cgroup: 6095 case BPF_FUNC_skb_under_cgroup: 6096 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 6097 goto error; 6098 break; 6099 case BPF_FUNC_redirect_map: 6100 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 6101 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && 6102 map->map_type != BPF_MAP_TYPE_CPUMAP && 6103 map->map_type != BPF_MAP_TYPE_XSKMAP) 6104 goto error; 6105 break; 6106 case BPF_FUNC_sk_redirect_map: 6107 case BPF_FUNC_msg_redirect_map: 6108 case BPF_FUNC_sock_map_update: 6109 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 6110 goto error; 6111 break; 6112 case BPF_FUNC_sk_redirect_hash: 6113 case BPF_FUNC_msg_redirect_hash: 6114 case BPF_FUNC_sock_hash_update: 6115 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 6116 goto error; 6117 break; 6118 case BPF_FUNC_get_local_storage: 6119 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 6120 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 6121 goto error; 6122 break; 6123 case BPF_FUNC_sk_select_reuseport: 6124 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && 6125 map->map_type != BPF_MAP_TYPE_SOCKMAP && 6126 map->map_type != BPF_MAP_TYPE_SOCKHASH) 6127 goto error; 6128 break; 6129 case BPF_FUNC_map_pop_elem: 6130 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6131 map->map_type != BPF_MAP_TYPE_STACK) 6132 goto error; 6133 break; 6134 case BPF_FUNC_map_peek_elem: 6135 case BPF_FUNC_map_push_elem: 6136 if (map->map_type != BPF_MAP_TYPE_QUEUE && 6137 map->map_type != BPF_MAP_TYPE_STACK && 6138 map->map_type != BPF_MAP_TYPE_BLOOM_FILTER) 6139 goto error; 6140 break; 6141 case BPF_FUNC_sk_storage_get: 6142 case BPF_FUNC_sk_storage_delete: 6143 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 6144 goto error; 6145 break; 6146 case BPF_FUNC_inode_storage_get: 6147 case BPF_FUNC_inode_storage_delete: 6148 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) 6149 goto error; 6150 break; 6151 case BPF_FUNC_task_storage_get: 6152 case BPF_FUNC_task_storage_delete: 6153 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) 6154 goto error; 6155 break; 6156 default: 6157 break; 6158 } 6159 6160 return 0; 6161 error: 6162 verbose(env, "cannot pass map_type %d into func %s#%d\n", 6163 map->map_type, func_id_name(func_id), func_id); 6164 return -EINVAL; 6165 } 6166 6167 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 6168 { 6169 int count = 0; 6170 6171 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 6172 count++; 6173 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 6174 count++; 6175 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 6176 count++; 6177 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 6178 count++; 6179 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 6180 count++; 6181 6182 /* We only support one arg being in raw mode at the moment, 6183 * which is sufficient for the helper functions we have 6184 * right now. 6185 */ 6186 return count <= 1; 6187 } 6188 6189 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, 6190 enum bpf_arg_type arg_next) 6191 { 6192 return (arg_type_is_mem_ptr(arg_curr) && 6193 !arg_type_is_mem_size(arg_next)) || 6194 (!arg_type_is_mem_ptr(arg_curr) && 6195 arg_type_is_mem_size(arg_next)); 6196 } 6197 6198 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 6199 { 6200 /* bpf_xxx(..., buf, len) call will access 'len' 6201 * bytes from memory 'buf'. Both arg types need 6202 * to be paired, so make sure there's no buggy 6203 * helper function specification. 6204 */ 6205 if (arg_type_is_mem_size(fn->arg1_type) || 6206 arg_type_is_mem_ptr(fn->arg5_type) || 6207 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || 6208 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || 6209 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || 6210 check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) 6211 return false; 6212 6213 return true; 6214 } 6215 6216 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) 6217 { 6218 int count = 0; 6219 6220 if (arg_type_may_be_refcounted(fn->arg1_type)) 6221 count++; 6222 if (arg_type_may_be_refcounted(fn->arg2_type)) 6223 count++; 6224 if (arg_type_may_be_refcounted(fn->arg3_type)) 6225 count++; 6226 if (arg_type_may_be_refcounted(fn->arg4_type)) 6227 count++; 6228 if (arg_type_may_be_refcounted(fn->arg5_type)) 6229 count++; 6230 6231 /* A reference acquiring function cannot acquire 6232 * another refcounted ptr. 6233 */ 6234 if (may_be_acquire_function(func_id) && count) 6235 return false; 6236 6237 /* We only support one arg being unreferenced at the moment, 6238 * which is sufficient for the helper functions we have right now. 6239 */ 6240 return count <= 1; 6241 } 6242 6243 static bool check_btf_id_ok(const struct bpf_func_proto *fn) 6244 { 6245 int i; 6246 6247 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { 6248 if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) 6249 return false; 6250 6251 if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) 6252 return false; 6253 } 6254 6255 return true; 6256 } 6257 6258 static int check_func_proto(const struct bpf_func_proto *fn, int func_id, 6259 struct bpf_call_arg_meta *meta) 6260 { 6261 return check_raw_mode_ok(fn) && 6262 check_arg_pair_ok(fn) && 6263 check_btf_id_ok(fn) && 6264 check_refcount_ok(fn, func_id) ? 0 : -EINVAL; 6265 } 6266 6267 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 6268 * are now invalid, so turn them into unknown SCALAR_VALUE. 6269 */ 6270 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, 6271 struct bpf_func_state *state) 6272 { 6273 struct bpf_reg_state *regs = state->regs, *reg; 6274 int i; 6275 6276 for (i = 0; i < MAX_BPF_REG; i++) 6277 if (reg_is_pkt_pointer_any(®s[i])) 6278 mark_reg_unknown(env, regs, i); 6279 6280 bpf_for_each_spilled_reg(i, state, reg) { 6281 if (!reg) 6282 continue; 6283 if (reg_is_pkt_pointer_any(reg)) 6284 __mark_reg_unknown(env, reg); 6285 } 6286 } 6287 6288 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 6289 { 6290 struct bpf_verifier_state *vstate = env->cur_state; 6291 int i; 6292 6293 for (i = 0; i <= vstate->curframe; i++) 6294 __clear_all_pkt_pointers(env, vstate->frame[i]); 6295 } 6296 6297 enum { 6298 AT_PKT_END = -1, 6299 BEYOND_PKT_END = -2, 6300 }; 6301 6302 static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) 6303 { 6304 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 6305 struct bpf_reg_state *reg = &state->regs[regn]; 6306 6307 if (reg->type != PTR_TO_PACKET) 6308 /* PTR_TO_PACKET_META is not supported yet */ 6309 return; 6310 6311 /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. 6312 * How far beyond pkt_end it goes is unknown. 6313 * if (!range_open) it's the case of pkt >= pkt_end 6314 * if (range_open) it's the case of pkt > pkt_end 6315 * hence this pointer is at least 1 byte bigger than pkt_end 6316 */ 6317 if (range_open) 6318 reg->range = BEYOND_PKT_END; 6319 else 6320 reg->range = AT_PKT_END; 6321 } 6322 6323 static void release_reg_references(struct bpf_verifier_env *env, 6324 struct bpf_func_state *state, 6325 int ref_obj_id) 6326 { 6327 struct bpf_reg_state *regs = state->regs, *reg; 6328 int i; 6329 6330 for (i = 0; i < MAX_BPF_REG; i++) 6331 if (regs[i].ref_obj_id == ref_obj_id) 6332 mark_reg_unknown(env, regs, i); 6333 6334 bpf_for_each_spilled_reg(i, state, reg) { 6335 if (!reg) 6336 continue; 6337 if (reg->ref_obj_id == ref_obj_id) 6338 __mark_reg_unknown(env, reg); 6339 } 6340 } 6341 6342 /* The pointer with the specified id has released its reference to kernel 6343 * resources. Identify all copies of the same pointer and clear the reference. 6344 */ 6345 static int release_reference(struct bpf_verifier_env *env, 6346 int ref_obj_id) 6347 { 6348 struct bpf_verifier_state *vstate = env->cur_state; 6349 int err; 6350 int i; 6351 6352 err = release_reference_state(cur_func(env), ref_obj_id); 6353 if (err) 6354 return err; 6355 6356 for (i = 0; i <= vstate->curframe; i++) 6357 release_reg_references(env, vstate->frame[i], ref_obj_id); 6358 6359 return 0; 6360 } 6361 6362 static void clear_caller_saved_regs(struct bpf_verifier_env *env, 6363 struct bpf_reg_state *regs) 6364 { 6365 int i; 6366 6367 /* after the call registers r0 - r5 were scratched */ 6368 for (i = 0; i < CALLER_SAVED_REGS; i++) { 6369 mark_reg_not_init(env, regs, caller_saved[i]); 6370 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 6371 } 6372 } 6373 6374 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, 6375 struct bpf_func_state *caller, 6376 struct bpf_func_state *callee, 6377 int insn_idx); 6378 6379 static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6380 int *insn_idx, int subprog, 6381 set_callee_state_fn set_callee_state_cb) 6382 { 6383 struct bpf_verifier_state *state = env->cur_state; 6384 struct bpf_func_info_aux *func_info_aux; 6385 struct bpf_func_state *caller, *callee; 6386 int err; 6387 bool is_global = false; 6388 6389 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 6390 verbose(env, "the call stack of %d frames is too deep\n", 6391 state->curframe + 2); 6392 return -E2BIG; 6393 } 6394 6395 caller = state->frame[state->curframe]; 6396 if (state->frame[state->curframe + 1]) { 6397 verbose(env, "verifier bug. Frame %d already allocated\n", 6398 state->curframe + 1); 6399 return -EFAULT; 6400 } 6401 6402 func_info_aux = env->prog->aux->func_info_aux; 6403 if (func_info_aux) 6404 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; 6405 err = btf_check_subprog_arg_match(env, subprog, caller->regs); 6406 if (err == -EFAULT) 6407 return err; 6408 if (is_global) { 6409 if (err) { 6410 verbose(env, "Caller passes invalid args into func#%d\n", 6411 subprog); 6412 return err; 6413 } else { 6414 if (env->log.level & BPF_LOG_LEVEL) 6415 verbose(env, 6416 "Func#%d is global and valid. Skipping.\n", 6417 subprog); 6418 clear_caller_saved_regs(env, caller->regs); 6419 6420 /* All global functions return a 64-bit SCALAR_VALUE */ 6421 mark_reg_unknown(env, caller->regs, BPF_REG_0); 6422 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 6423 6424 /* continue with next insn after call */ 6425 return 0; 6426 } 6427 } 6428 6429 if (insn->code == (BPF_JMP | BPF_CALL) && 6430 insn->src_reg == 0 && 6431 insn->imm == BPF_FUNC_timer_set_callback) { 6432 struct bpf_verifier_state *async_cb; 6433 6434 /* there is no real recursion here. timer callbacks are async */ 6435 env->subprog_info[subprog].is_async_cb = true; 6436 async_cb = push_async_cb(env, env->subprog_info[subprog].start, 6437 *insn_idx, subprog); 6438 if (!async_cb) 6439 return -EFAULT; 6440 callee = async_cb->frame[0]; 6441 callee->async_entry_cnt = caller->async_entry_cnt + 1; 6442 6443 /* Convert bpf_timer_set_callback() args into timer callback args */ 6444 err = set_callee_state_cb(env, caller, callee, *insn_idx); 6445 if (err) 6446 return err; 6447 6448 clear_caller_saved_regs(env, caller->regs); 6449 mark_reg_unknown(env, caller->regs, BPF_REG_0); 6450 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 6451 /* continue with next insn after call */ 6452 return 0; 6453 } 6454 6455 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 6456 if (!callee) 6457 return -ENOMEM; 6458 state->frame[state->curframe + 1] = callee; 6459 6460 /* callee cannot access r0, r6 - r9 for reading and has to write 6461 * into its own stack before reading from it. 6462 * callee can read/write into caller's stack 6463 */ 6464 init_func_state(env, callee, 6465 /* remember the callsite, it will be used by bpf_exit */ 6466 *insn_idx /* callsite */, 6467 state->curframe + 1 /* frameno within this callchain */, 6468 subprog /* subprog number within this prog */); 6469 6470 /* Transfer references to the callee */ 6471 err = copy_reference_state(callee, caller); 6472 if (err) 6473 return err; 6474 6475 err = set_callee_state_cb(env, caller, callee, *insn_idx); 6476 if (err) 6477 return err; 6478 6479 clear_caller_saved_regs(env, caller->regs); 6480 6481 /* only increment it after check_reg_arg() finished */ 6482 state->curframe++; 6483 6484 /* and go analyze first insn of the callee */ 6485 *insn_idx = env->subprog_info[subprog].start - 1; 6486 6487 if (env->log.level & BPF_LOG_LEVEL) { 6488 verbose(env, "caller:\n"); 6489 print_verifier_state(env, caller, true); 6490 verbose(env, "callee:\n"); 6491 print_verifier_state(env, callee, true); 6492 } 6493 return 0; 6494 } 6495 6496 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 6497 struct bpf_func_state *caller, 6498 struct bpf_func_state *callee) 6499 { 6500 /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn, 6501 * void *callback_ctx, u64 flags); 6502 * callback_fn(struct bpf_map *map, void *key, void *value, 6503 * void *callback_ctx); 6504 */ 6505 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 6506 6507 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 6508 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 6509 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr; 6510 6511 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 6512 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 6513 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; 6514 6515 /* pointer to stack or null */ 6516 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; 6517 6518 /* unused */ 6519 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 6520 return 0; 6521 } 6522 6523 static int set_callee_state(struct bpf_verifier_env *env, 6524 struct bpf_func_state *caller, 6525 struct bpf_func_state *callee, int insn_idx) 6526 { 6527 int i; 6528 6529 /* copy r1 - r5 args that callee can access. The copy includes parent 6530 * pointers, which connects us up to the liveness chain 6531 */ 6532 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 6533 callee->regs[i] = caller->regs[i]; 6534 return 0; 6535 } 6536 6537 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6538 int *insn_idx) 6539 { 6540 int subprog, target_insn; 6541 6542 target_insn = *insn_idx + insn->imm + 1; 6543 subprog = find_subprog(env, target_insn); 6544 if (subprog < 0) { 6545 verbose(env, "verifier bug. No program starts at insn %d\n", 6546 target_insn); 6547 return -EFAULT; 6548 } 6549 6550 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state); 6551 } 6552 6553 static int set_map_elem_callback_state(struct bpf_verifier_env *env, 6554 struct bpf_func_state *caller, 6555 struct bpf_func_state *callee, 6556 int insn_idx) 6557 { 6558 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; 6559 struct bpf_map *map; 6560 int err; 6561 6562 if (bpf_map_ptr_poisoned(insn_aux)) { 6563 verbose(env, "tail_call abusing map_ptr\n"); 6564 return -EINVAL; 6565 } 6566 6567 map = BPF_MAP_PTR(insn_aux->map_ptr_state); 6568 if (!map->ops->map_set_for_each_callback_args || 6569 !map->ops->map_for_each_callback) { 6570 verbose(env, "callback function not allowed for map\n"); 6571 return -ENOTSUPP; 6572 } 6573 6574 err = map->ops->map_set_for_each_callback_args(env, caller, callee); 6575 if (err) 6576 return err; 6577 6578 callee->in_callback_fn = true; 6579 return 0; 6580 } 6581 6582 static int set_loop_callback_state(struct bpf_verifier_env *env, 6583 struct bpf_func_state *caller, 6584 struct bpf_func_state *callee, 6585 int insn_idx) 6586 { 6587 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx, 6588 * u64 flags); 6589 * callback_fn(u32 index, void *callback_ctx); 6590 */ 6591 callee->regs[BPF_REG_1].type = SCALAR_VALUE; 6592 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; 6593 6594 /* unused */ 6595 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); 6596 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 6597 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 6598 6599 callee->in_callback_fn = true; 6600 return 0; 6601 } 6602 6603 static int set_timer_callback_state(struct bpf_verifier_env *env, 6604 struct bpf_func_state *caller, 6605 struct bpf_func_state *callee, 6606 int insn_idx) 6607 { 6608 struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr; 6609 6610 /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn); 6611 * callback_fn(struct bpf_map *map, void *key, void *value); 6612 */ 6613 callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP; 6614 __mark_reg_known_zero(&callee->regs[BPF_REG_1]); 6615 callee->regs[BPF_REG_1].map_ptr = map_ptr; 6616 6617 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY; 6618 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 6619 callee->regs[BPF_REG_2].map_ptr = map_ptr; 6620 6621 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE; 6622 __mark_reg_known_zero(&callee->regs[BPF_REG_3]); 6623 callee->regs[BPF_REG_3].map_ptr = map_ptr; 6624 6625 /* unused */ 6626 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 6627 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 6628 callee->in_async_callback_fn = true; 6629 return 0; 6630 } 6631 6632 static int set_find_vma_callback_state(struct bpf_verifier_env *env, 6633 struct bpf_func_state *caller, 6634 struct bpf_func_state *callee, 6635 int insn_idx) 6636 { 6637 /* bpf_find_vma(struct task_struct *task, u64 addr, 6638 * void *callback_fn, void *callback_ctx, u64 flags) 6639 * (callback_fn)(struct task_struct *task, 6640 * struct vm_area_struct *vma, void *callback_ctx); 6641 */ 6642 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; 6643 6644 callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; 6645 __mark_reg_known_zero(&callee->regs[BPF_REG_2]); 6646 callee->regs[BPF_REG_2].btf = btf_vmlinux; 6647 callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], 6648 6649 /* pointer to stack or null */ 6650 callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; 6651 6652 /* unused */ 6653 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); 6654 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); 6655 callee->in_callback_fn = true; 6656 return 0; 6657 } 6658 6659 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 6660 { 6661 struct bpf_verifier_state *state = env->cur_state; 6662 struct bpf_func_state *caller, *callee; 6663 struct bpf_reg_state *r0; 6664 int err; 6665 6666 callee = state->frame[state->curframe]; 6667 r0 = &callee->regs[BPF_REG_0]; 6668 if (r0->type == PTR_TO_STACK) { 6669 /* technically it's ok to return caller's stack pointer 6670 * (or caller's caller's pointer) back to the caller, 6671 * since these pointers are valid. Only current stack 6672 * pointer will be invalid as soon as function exits, 6673 * but let's be conservative 6674 */ 6675 verbose(env, "cannot return stack pointer to the caller\n"); 6676 return -EINVAL; 6677 } 6678 6679 state->curframe--; 6680 caller = state->frame[state->curframe]; 6681 if (callee->in_callback_fn) { 6682 /* enforce R0 return value range [0, 1]. */ 6683 struct tnum range = tnum_range(0, 1); 6684 6685 if (r0->type != SCALAR_VALUE) { 6686 verbose(env, "R0 not a scalar value\n"); 6687 return -EACCES; 6688 } 6689 if (!tnum_in(range, r0->var_off)) { 6690 verbose_invalid_scalar(env, r0, &range, "callback return", "R0"); 6691 return -EINVAL; 6692 } 6693 } else { 6694 /* return to the caller whatever r0 had in the callee */ 6695 caller->regs[BPF_REG_0] = *r0; 6696 } 6697 6698 /* Transfer references to the caller */ 6699 err = copy_reference_state(caller, callee); 6700 if (err) 6701 return err; 6702 6703 *insn_idx = callee->callsite + 1; 6704 if (env->log.level & BPF_LOG_LEVEL) { 6705 verbose(env, "returning from callee:\n"); 6706 print_verifier_state(env, callee, true); 6707 verbose(env, "to caller at %d:\n", *insn_idx); 6708 print_verifier_state(env, caller, true); 6709 } 6710 /* clear everything in the callee */ 6711 free_func_state(callee); 6712 state->frame[state->curframe + 1] = NULL; 6713 return 0; 6714 } 6715 6716 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 6717 int func_id, 6718 struct bpf_call_arg_meta *meta) 6719 { 6720 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 6721 6722 if (ret_type != RET_INTEGER || 6723 (func_id != BPF_FUNC_get_stack && 6724 func_id != BPF_FUNC_get_task_stack && 6725 func_id != BPF_FUNC_probe_read_str && 6726 func_id != BPF_FUNC_probe_read_kernel_str && 6727 func_id != BPF_FUNC_probe_read_user_str)) 6728 return; 6729 6730 ret_reg->smax_value = meta->msize_max_value; 6731 ret_reg->s32_max_value = meta->msize_max_value; 6732 ret_reg->smin_value = -MAX_ERRNO; 6733 ret_reg->s32_min_value = -MAX_ERRNO; 6734 __reg_deduce_bounds(ret_reg); 6735 __reg_bound_offset(ret_reg); 6736 __update_reg_bounds(ret_reg); 6737 } 6738 6739 static int 6740 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 6741 int func_id, int insn_idx) 6742 { 6743 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 6744 struct bpf_map *map = meta->map_ptr; 6745 6746 if (func_id != BPF_FUNC_tail_call && 6747 func_id != BPF_FUNC_map_lookup_elem && 6748 func_id != BPF_FUNC_map_update_elem && 6749 func_id != BPF_FUNC_map_delete_elem && 6750 func_id != BPF_FUNC_map_push_elem && 6751 func_id != BPF_FUNC_map_pop_elem && 6752 func_id != BPF_FUNC_map_peek_elem && 6753 func_id != BPF_FUNC_for_each_map_elem && 6754 func_id != BPF_FUNC_redirect_map) 6755 return 0; 6756 6757 if (map == NULL) { 6758 verbose(env, "kernel subsystem misconfigured verifier\n"); 6759 return -EINVAL; 6760 } 6761 6762 /* In case of read-only, some additional restrictions 6763 * need to be applied in order to prevent altering the 6764 * state of the map from program side. 6765 */ 6766 if ((map->map_flags & BPF_F_RDONLY_PROG) && 6767 (func_id == BPF_FUNC_map_delete_elem || 6768 func_id == BPF_FUNC_map_update_elem || 6769 func_id == BPF_FUNC_map_push_elem || 6770 func_id == BPF_FUNC_map_pop_elem)) { 6771 verbose(env, "write into map forbidden\n"); 6772 return -EACCES; 6773 } 6774 6775 if (!BPF_MAP_PTR(aux->map_ptr_state)) 6776 bpf_map_ptr_store(aux, meta->map_ptr, 6777 !meta->map_ptr->bypass_spec_v1); 6778 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) 6779 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 6780 !meta->map_ptr->bypass_spec_v1); 6781 return 0; 6782 } 6783 6784 static int 6785 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 6786 int func_id, int insn_idx) 6787 { 6788 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 6789 struct bpf_reg_state *regs = cur_regs(env), *reg; 6790 struct bpf_map *map = meta->map_ptr; 6791 struct tnum range; 6792 u64 val; 6793 int err; 6794 6795 if (func_id != BPF_FUNC_tail_call) 6796 return 0; 6797 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { 6798 verbose(env, "kernel subsystem misconfigured verifier\n"); 6799 return -EINVAL; 6800 } 6801 6802 range = tnum_range(0, map->max_entries - 1); 6803 reg = ®s[BPF_REG_3]; 6804 6805 if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { 6806 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 6807 return 0; 6808 } 6809 6810 err = mark_chain_precision(env, BPF_REG_3); 6811 if (err) 6812 return err; 6813 6814 val = reg->var_off.value; 6815 if (bpf_map_key_unseen(aux)) 6816 bpf_map_key_store(aux, val); 6817 else if (!bpf_map_key_poisoned(aux) && 6818 bpf_map_key_immediate(aux) != val) 6819 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); 6820 return 0; 6821 } 6822 6823 static int check_reference_leak(struct bpf_verifier_env *env) 6824 { 6825 struct bpf_func_state *state = cur_func(env); 6826 int i; 6827 6828 for (i = 0; i < state->acquired_refs; i++) { 6829 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 6830 state->refs[i].id, state->refs[i].insn_idx); 6831 } 6832 return state->acquired_refs ? -EINVAL : 0; 6833 } 6834 6835 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, 6836 struct bpf_reg_state *regs) 6837 { 6838 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3]; 6839 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5]; 6840 struct bpf_map *fmt_map = fmt_reg->map_ptr; 6841 int err, fmt_map_off, num_args; 6842 u64 fmt_addr; 6843 char *fmt; 6844 6845 /* data must be an array of u64 */ 6846 if (data_len_reg->var_off.value % 8) 6847 return -EINVAL; 6848 num_args = data_len_reg->var_off.value / 8; 6849 6850 /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const 6851 * and map_direct_value_addr is set. 6852 */ 6853 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value; 6854 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr, 6855 fmt_map_off); 6856 if (err) { 6857 verbose(env, "verifier bug\n"); 6858 return -EFAULT; 6859 } 6860 fmt = (char *)(long)fmt_addr + fmt_map_off; 6861 6862 /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we 6863 * can focus on validating the format specifiers. 6864 */ 6865 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args); 6866 if (err < 0) 6867 verbose(env, "Invalid format string\n"); 6868 6869 return err; 6870 } 6871 6872 static int check_get_func_ip(struct bpf_verifier_env *env) 6873 { 6874 enum bpf_prog_type type = resolve_prog_type(env->prog); 6875 int func_id = BPF_FUNC_get_func_ip; 6876 6877 if (type == BPF_PROG_TYPE_TRACING) { 6878 if (!bpf_prog_has_trampoline(env->prog)) { 6879 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", 6880 func_id_name(func_id), func_id); 6881 return -ENOTSUPP; 6882 } 6883 return 0; 6884 } else if (type == BPF_PROG_TYPE_KPROBE) { 6885 return 0; 6886 } 6887 6888 verbose(env, "func %s#%d not supported for program type %d\n", 6889 func_id_name(func_id), func_id, type); 6890 return -ENOTSUPP; 6891 } 6892 6893 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 6894 int *insn_idx_p) 6895 { 6896 const struct bpf_func_proto *fn = NULL; 6897 enum bpf_return_type ret_type; 6898 enum bpf_type_flag ret_flag; 6899 struct bpf_reg_state *regs; 6900 struct bpf_call_arg_meta meta; 6901 int insn_idx = *insn_idx_p; 6902 bool changes_data; 6903 int i, err, func_id; 6904 6905 /* find function prototype */ 6906 func_id = insn->imm; 6907 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 6908 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 6909 func_id); 6910 return -EINVAL; 6911 } 6912 6913 if (env->ops->get_func_proto) 6914 fn = env->ops->get_func_proto(func_id, env->prog); 6915 if (!fn) { 6916 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 6917 func_id); 6918 return -EINVAL; 6919 } 6920 6921 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 6922 if (!env->prog->gpl_compatible && fn->gpl_only) { 6923 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 6924 return -EINVAL; 6925 } 6926 6927 if (fn->allowed && !fn->allowed(env->prog)) { 6928 verbose(env, "helper call is not allowed in probe\n"); 6929 return -EINVAL; 6930 } 6931 6932 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 6933 changes_data = bpf_helper_changes_pkt_data(fn->func); 6934 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 6935 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 6936 func_id_name(func_id), func_id); 6937 return -EINVAL; 6938 } 6939 6940 memset(&meta, 0, sizeof(meta)); 6941 meta.pkt_access = fn->pkt_access; 6942 6943 err = check_func_proto(fn, func_id, &meta); 6944 if (err) { 6945 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 6946 func_id_name(func_id), func_id); 6947 return err; 6948 } 6949 6950 meta.func_id = func_id; 6951 /* check args */ 6952 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { 6953 err = check_func_arg(env, i, &meta, fn); 6954 if (err) 6955 return err; 6956 } 6957 6958 err = record_func_map(env, &meta, func_id, insn_idx); 6959 if (err) 6960 return err; 6961 6962 err = record_func_key(env, &meta, func_id, insn_idx); 6963 if (err) 6964 return err; 6965 6966 /* Mark slots with STACK_MISC in case of raw mode, stack offset 6967 * is inferred from register state. 6968 */ 6969 for (i = 0; i < meta.access_size; i++) { 6970 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 6971 BPF_WRITE, -1, false); 6972 if (err) 6973 return err; 6974 } 6975 6976 regs = cur_regs(env); 6977 6978 if (meta.release_regno) { 6979 err = -EINVAL; 6980 if (meta.ref_obj_id) 6981 err = release_reference(env, meta.ref_obj_id); 6982 /* meta.ref_obj_id can only be 0 if register that is meant to be 6983 * released is NULL, which must be > R0. 6984 */ 6985 else if (register_is_null(®s[meta.release_regno])) 6986 err = 0; 6987 if (err) { 6988 verbose(env, "func %s#%d reference has not been acquired before\n", 6989 func_id_name(func_id), func_id); 6990 return err; 6991 } 6992 } 6993 6994 switch (func_id) { 6995 case BPF_FUNC_tail_call: 6996 err = check_reference_leak(env); 6997 if (err) { 6998 verbose(env, "tail_call would lead to reference leak\n"); 6999 return err; 7000 } 7001 break; 7002 case BPF_FUNC_get_local_storage: 7003 /* check that flags argument in get_local_storage(map, flags) is 0, 7004 * this is required because get_local_storage() can't return an error. 7005 */ 7006 if (!register_is_null(®s[BPF_REG_2])) { 7007 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 7008 return -EINVAL; 7009 } 7010 break; 7011 case BPF_FUNC_for_each_map_elem: 7012 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7013 set_map_elem_callback_state); 7014 break; 7015 case BPF_FUNC_timer_set_callback: 7016 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7017 set_timer_callback_state); 7018 break; 7019 case BPF_FUNC_find_vma: 7020 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7021 set_find_vma_callback_state); 7022 break; 7023 case BPF_FUNC_snprintf: 7024 err = check_bpf_snprintf_call(env, regs); 7025 break; 7026 case BPF_FUNC_loop: 7027 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, 7028 set_loop_callback_state); 7029 break; 7030 } 7031 7032 if (err) 7033 return err; 7034 7035 /* reset caller saved regs */ 7036 for (i = 0; i < CALLER_SAVED_REGS; i++) { 7037 mark_reg_not_init(env, regs, caller_saved[i]); 7038 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 7039 } 7040 7041 /* helper call returns 64-bit value. */ 7042 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; 7043 7044 /* update return register (already marked as written above) */ 7045 ret_type = fn->ret_type; 7046 ret_flag = type_flag(fn->ret_type); 7047 if (ret_type == RET_INTEGER) { 7048 /* sets type to SCALAR_VALUE */ 7049 mark_reg_unknown(env, regs, BPF_REG_0); 7050 } else if (ret_type == RET_VOID) { 7051 regs[BPF_REG_0].type = NOT_INIT; 7052 } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) { 7053 /* There is no offset yet applied, variable or fixed */ 7054 mark_reg_known_zero(env, regs, BPF_REG_0); 7055 /* remember map_ptr, so that check_map_access() 7056 * can check 'value_size' boundary of memory access 7057 * to map element returned from bpf_map_lookup_elem() 7058 */ 7059 if (meta.map_ptr == NULL) { 7060 verbose(env, 7061 "kernel subsystem misconfigured verifier\n"); 7062 return -EINVAL; 7063 } 7064 regs[BPF_REG_0].map_ptr = meta.map_ptr; 7065 regs[BPF_REG_0].map_uid = meta.map_uid; 7066 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; 7067 if (!type_may_be_null(ret_type) && 7068 map_value_has_spin_lock(meta.map_ptr)) { 7069 regs[BPF_REG_0].id = ++env->id_gen; 7070 } 7071 } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) { 7072 mark_reg_known_zero(env, regs, BPF_REG_0); 7073 regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; 7074 } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) { 7075 mark_reg_known_zero(env, regs, BPF_REG_0); 7076 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; 7077 } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) { 7078 mark_reg_known_zero(env, regs, BPF_REG_0); 7079 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; 7080 } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) { 7081 mark_reg_known_zero(env, regs, BPF_REG_0); 7082 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7083 regs[BPF_REG_0].mem_size = meta.mem_size; 7084 } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) { 7085 const struct btf_type *t; 7086 7087 mark_reg_known_zero(env, regs, BPF_REG_0); 7088 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); 7089 if (!btf_type_is_struct(t)) { 7090 u32 tsize; 7091 const struct btf_type *ret; 7092 const char *tname; 7093 7094 /* resolve the type size of ksym. */ 7095 ret = btf_resolve_size(meta.ret_btf, t, &tsize); 7096 if (IS_ERR(ret)) { 7097 tname = btf_name_by_offset(meta.ret_btf, t->name_off); 7098 verbose(env, "unable to resolve the size of type '%s': %ld\n", 7099 tname, PTR_ERR(ret)); 7100 return -EINVAL; 7101 } 7102 regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; 7103 regs[BPF_REG_0].mem_size = tsize; 7104 } else { 7105 /* MEM_RDONLY may be carried from ret_flag, but it 7106 * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise 7107 * it will confuse the check of PTR_TO_BTF_ID in 7108 * check_mem_access(). 7109 */ 7110 ret_flag &= ~MEM_RDONLY; 7111 7112 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7113 regs[BPF_REG_0].btf = meta.ret_btf; 7114 regs[BPF_REG_0].btf_id = meta.ret_btf_id; 7115 } 7116 } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) { 7117 struct btf *ret_btf; 7118 int ret_btf_id; 7119 7120 mark_reg_known_zero(env, regs, BPF_REG_0); 7121 regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag; 7122 if (func_id == BPF_FUNC_kptr_xchg) { 7123 ret_btf = meta.kptr_off_desc->kptr.btf; 7124 ret_btf_id = meta.kptr_off_desc->kptr.btf_id; 7125 } else { 7126 ret_btf = btf_vmlinux; 7127 ret_btf_id = *fn->ret_btf_id; 7128 } 7129 if (ret_btf_id == 0) { 7130 verbose(env, "invalid return type %u of func %s#%d\n", 7131 base_type(ret_type), func_id_name(func_id), 7132 func_id); 7133 return -EINVAL; 7134 } 7135 regs[BPF_REG_0].btf = ret_btf; 7136 regs[BPF_REG_0].btf_id = ret_btf_id; 7137 } else { 7138 verbose(env, "unknown return type %u of func %s#%d\n", 7139 base_type(ret_type), func_id_name(func_id), func_id); 7140 return -EINVAL; 7141 } 7142 7143 if (type_may_be_null(regs[BPF_REG_0].type)) 7144 regs[BPF_REG_0].id = ++env->id_gen; 7145 7146 if (is_ptr_cast_function(func_id)) { 7147 /* For release_reference() */ 7148 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 7149 } else if (is_acquire_function(func_id, meta.map_ptr)) { 7150 int id = acquire_reference_state(env, insn_idx); 7151 7152 if (id < 0) 7153 return id; 7154 /* For mark_ptr_or_null_reg() */ 7155 regs[BPF_REG_0].id = id; 7156 /* For release_reference() */ 7157 regs[BPF_REG_0].ref_obj_id = id; 7158 } 7159 7160 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 7161 7162 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 7163 if (err) 7164 return err; 7165 7166 if ((func_id == BPF_FUNC_get_stack || 7167 func_id == BPF_FUNC_get_task_stack) && 7168 !env->prog->has_callchain_buf) { 7169 const char *err_str; 7170 7171 #ifdef CONFIG_PERF_EVENTS 7172 err = get_callchain_buffers(sysctl_perf_event_max_stack); 7173 err_str = "cannot get callchain buffer for func %s#%d\n"; 7174 #else 7175 err = -ENOTSUPP; 7176 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 7177 #endif 7178 if (err) { 7179 verbose(env, err_str, func_id_name(func_id), func_id); 7180 return err; 7181 } 7182 7183 env->prog->has_callchain_buf = true; 7184 } 7185 7186 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) 7187 env->prog->call_get_stack = true; 7188 7189 if (func_id == BPF_FUNC_get_func_ip) { 7190 if (check_get_func_ip(env)) 7191 return -ENOTSUPP; 7192 env->prog->call_get_func_ip = true; 7193 } 7194 7195 if (changes_data) 7196 clear_all_pkt_pointers(env); 7197 return 0; 7198 } 7199 7200 /* mark_btf_func_reg_size() is used when the reg size is determined by 7201 * the BTF func_proto's return value size and argument. 7202 */ 7203 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, 7204 size_t reg_size) 7205 { 7206 struct bpf_reg_state *reg = &cur_regs(env)[regno]; 7207 7208 if (regno == BPF_REG_0) { 7209 /* Function return value */ 7210 reg->live |= REG_LIVE_WRITTEN; 7211 reg->subreg_def = reg_size == sizeof(u64) ? 7212 DEF_NOT_SUBREG : env->insn_idx + 1; 7213 } else { 7214 /* Function argument */ 7215 if (reg_size == sizeof(u64)) { 7216 mark_insn_zext(env, reg); 7217 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); 7218 } else { 7219 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); 7220 } 7221 } 7222 } 7223 7224 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 7225 int *insn_idx_p) 7226 { 7227 const struct btf_type *t, *func, *func_proto, *ptr_type; 7228 struct bpf_reg_state *regs = cur_regs(env); 7229 const char *func_name, *ptr_type_name; 7230 u32 i, nargs, func_id, ptr_type_id; 7231 int err, insn_idx = *insn_idx_p; 7232 const struct btf_param *args; 7233 struct btf *desc_btf; 7234 bool acq; 7235 7236 /* skip for now, but return error when we find this in fixup_kfunc_call */ 7237 if (!insn->imm) 7238 return 0; 7239 7240 desc_btf = find_kfunc_desc_btf(env, insn->imm, insn->off); 7241 if (IS_ERR(desc_btf)) 7242 return PTR_ERR(desc_btf); 7243 7244 func_id = insn->imm; 7245 func = btf_type_by_id(desc_btf, func_id); 7246 func_name = btf_name_by_offset(desc_btf, func->name_off); 7247 func_proto = btf_type_by_id(desc_btf, func->type); 7248 7249 if (!btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), 7250 BTF_KFUNC_TYPE_CHECK, func_id)) { 7251 verbose(env, "calling kernel function %s is not allowed\n", 7252 func_name); 7253 return -EACCES; 7254 } 7255 7256 acq = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), 7257 BTF_KFUNC_TYPE_ACQUIRE, func_id); 7258 7259 /* Check the arguments */ 7260 err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs); 7261 if (err < 0) 7262 return err; 7263 /* In case of release function, we get register number of refcounted 7264 * PTR_TO_BTF_ID back from btf_check_kfunc_arg_match, do the release now 7265 */ 7266 if (err) { 7267 err = release_reference(env, regs[err].ref_obj_id); 7268 if (err) { 7269 verbose(env, "kfunc %s#%d reference has not been acquired before\n", 7270 func_name, func_id); 7271 return err; 7272 } 7273 } 7274 7275 for (i = 0; i < CALLER_SAVED_REGS; i++) 7276 mark_reg_not_init(env, regs, caller_saved[i]); 7277 7278 /* Check return type */ 7279 t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL); 7280 7281 if (acq && !btf_type_is_ptr(t)) { 7282 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); 7283 return -EINVAL; 7284 } 7285 7286 if (btf_type_is_scalar(t)) { 7287 mark_reg_unknown(env, regs, BPF_REG_0); 7288 mark_btf_func_reg_size(env, BPF_REG_0, t->size); 7289 } else if (btf_type_is_ptr(t)) { 7290 ptr_type = btf_type_skip_modifiers(desc_btf, t->type, 7291 &ptr_type_id); 7292 if (!btf_type_is_struct(ptr_type)) { 7293 ptr_type_name = btf_name_by_offset(desc_btf, 7294 ptr_type->name_off); 7295 verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", 7296 func_name, btf_type_str(ptr_type), 7297 ptr_type_name); 7298 return -EINVAL; 7299 } 7300 mark_reg_known_zero(env, regs, BPF_REG_0); 7301 regs[BPF_REG_0].btf = desc_btf; 7302 regs[BPF_REG_0].type = PTR_TO_BTF_ID; 7303 regs[BPF_REG_0].btf_id = ptr_type_id; 7304 if (btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), 7305 BTF_KFUNC_TYPE_RET_NULL, func_id)) { 7306 regs[BPF_REG_0].type |= PTR_MAYBE_NULL; 7307 /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ 7308 regs[BPF_REG_0].id = ++env->id_gen; 7309 } 7310 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); 7311 if (acq) { 7312 int id = acquire_reference_state(env, insn_idx); 7313 7314 if (id < 0) 7315 return id; 7316 regs[BPF_REG_0].id = id; 7317 regs[BPF_REG_0].ref_obj_id = id; 7318 } 7319 } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */ 7320 7321 nargs = btf_type_vlen(func_proto); 7322 args = (const struct btf_param *)(func_proto + 1); 7323 for (i = 0; i < nargs; i++) { 7324 u32 regno = i + 1; 7325 7326 t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); 7327 if (btf_type_is_ptr(t)) 7328 mark_btf_func_reg_size(env, regno, sizeof(void *)); 7329 else 7330 /* scalar. ensured by btf_check_kfunc_arg_match() */ 7331 mark_btf_func_reg_size(env, regno, t->size); 7332 } 7333 7334 return 0; 7335 } 7336 7337 static bool signed_add_overflows(s64 a, s64 b) 7338 { 7339 /* Do the add in u64, where overflow is well-defined */ 7340 s64 res = (s64)((u64)a + (u64)b); 7341 7342 if (b < 0) 7343 return res > a; 7344 return res < a; 7345 } 7346 7347 static bool signed_add32_overflows(s32 a, s32 b) 7348 { 7349 /* Do the add in u32, where overflow is well-defined */ 7350 s32 res = (s32)((u32)a + (u32)b); 7351 7352 if (b < 0) 7353 return res > a; 7354 return res < a; 7355 } 7356 7357 static bool signed_sub_overflows(s64 a, s64 b) 7358 { 7359 /* Do the sub in u64, where overflow is well-defined */ 7360 s64 res = (s64)((u64)a - (u64)b); 7361 7362 if (b < 0) 7363 return res < a; 7364 return res > a; 7365 } 7366 7367 static bool signed_sub32_overflows(s32 a, s32 b) 7368 { 7369 /* Do the sub in u32, where overflow is well-defined */ 7370 s32 res = (s32)((u32)a - (u32)b); 7371 7372 if (b < 0) 7373 return res < a; 7374 return res > a; 7375 } 7376 7377 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 7378 const struct bpf_reg_state *reg, 7379 enum bpf_reg_type type) 7380 { 7381 bool known = tnum_is_const(reg->var_off); 7382 s64 val = reg->var_off.value; 7383 s64 smin = reg->smin_value; 7384 7385 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 7386 verbose(env, "math between %s pointer and %lld is not allowed\n", 7387 reg_type_str(env, type), val); 7388 return false; 7389 } 7390 7391 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 7392 verbose(env, "%s pointer offset %d is not allowed\n", 7393 reg_type_str(env, type), reg->off); 7394 return false; 7395 } 7396 7397 if (smin == S64_MIN) { 7398 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 7399 reg_type_str(env, type)); 7400 return false; 7401 } 7402 7403 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 7404 verbose(env, "value %lld makes %s pointer be out of bounds\n", 7405 smin, reg_type_str(env, type)); 7406 return false; 7407 } 7408 7409 return true; 7410 } 7411 7412 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 7413 { 7414 return &env->insn_aux_data[env->insn_idx]; 7415 } 7416 7417 enum { 7418 REASON_BOUNDS = -1, 7419 REASON_TYPE = -2, 7420 REASON_PATHS = -3, 7421 REASON_LIMIT = -4, 7422 REASON_STACK = -5, 7423 }; 7424 7425 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 7426 u32 *alu_limit, bool mask_to_left) 7427 { 7428 u32 max = 0, ptr_limit = 0; 7429 7430 switch (ptr_reg->type) { 7431 case PTR_TO_STACK: 7432 /* Offset 0 is out-of-bounds, but acceptable start for the 7433 * left direction, see BPF_REG_FP. Also, unknown scalar 7434 * offset where we would need to deal with min/max bounds is 7435 * currently prohibited for unprivileged. 7436 */ 7437 max = MAX_BPF_STACK + mask_to_left; 7438 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); 7439 break; 7440 case PTR_TO_MAP_VALUE: 7441 max = ptr_reg->map_ptr->value_size; 7442 ptr_limit = (mask_to_left ? 7443 ptr_reg->smin_value : 7444 ptr_reg->umax_value) + ptr_reg->off; 7445 break; 7446 default: 7447 return REASON_TYPE; 7448 } 7449 7450 if (ptr_limit >= max) 7451 return REASON_LIMIT; 7452 *alu_limit = ptr_limit; 7453 return 0; 7454 } 7455 7456 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 7457 const struct bpf_insn *insn) 7458 { 7459 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; 7460 } 7461 7462 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 7463 u32 alu_state, u32 alu_limit) 7464 { 7465 /* If we arrived here from different branches with different 7466 * state or limits to sanitize, then this won't work. 7467 */ 7468 if (aux->alu_state && 7469 (aux->alu_state != alu_state || 7470 aux->alu_limit != alu_limit)) 7471 return REASON_PATHS; 7472 7473 /* Corresponding fixup done in do_misc_fixups(). */ 7474 aux->alu_state = alu_state; 7475 aux->alu_limit = alu_limit; 7476 return 0; 7477 } 7478 7479 static int sanitize_val_alu(struct bpf_verifier_env *env, 7480 struct bpf_insn *insn) 7481 { 7482 struct bpf_insn_aux_data *aux = cur_aux(env); 7483 7484 if (can_skip_alu_sanitation(env, insn)) 7485 return 0; 7486 7487 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 7488 } 7489 7490 static bool sanitize_needed(u8 opcode) 7491 { 7492 return opcode == BPF_ADD || opcode == BPF_SUB; 7493 } 7494 7495 struct bpf_sanitize_info { 7496 struct bpf_insn_aux_data aux; 7497 bool mask_to_left; 7498 }; 7499 7500 static struct bpf_verifier_state * 7501 sanitize_speculative_path(struct bpf_verifier_env *env, 7502 const struct bpf_insn *insn, 7503 u32 next_idx, u32 curr_idx) 7504 { 7505 struct bpf_verifier_state *branch; 7506 struct bpf_reg_state *regs; 7507 7508 branch = push_stack(env, next_idx, curr_idx, true); 7509 if (branch && insn) { 7510 regs = branch->frame[branch->curframe]->regs; 7511 if (BPF_SRC(insn->code) == BPF_K) { 7512 mark_reg_unknown(env, regs, insn->dst_reg); 7513 } else if (BPF_SRC(insn->code) == BPF_X) { 7514 mark_reg_unknown(env, regs, insn->dst_reg); 7515 mark_reg_unknown(env, regs, insn->src_reg); 7516 } 7517 } 7518 return branch; 7519 } 7520 7521 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 7522 struct bpf_insn *insn, 7523 const struct bpf_reg_state *ptr_reg, 7524 const struct bpf_reg_state *off_reg, 7525 struct bpf_reg_state *dst_reg, 7526 struct bpf_sanitize_info *info, 7527 const bool commit_window) 7528 { 7529 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; 7530 struct bpf_verifier_state *vstate = env->cur_state; 7531 bool off_is_imm = tnum_is_const(off_reg->var_off); 7532 bool off_is_neg = off_reg->smin_value < 0; 7533 bool ptr_is_dst_reg = ptr_reg == dst_reg; 7534 u8 opcode = BPF_OP(insn->code); 7535 u32 alu_state, alu_limit; 7536 struct bpf_reg_state tmp; 7537 bool ret; 7538 int err; 7539 7540 if (can_skip_alu_sanitation(env, insn)) 7541 return 0; 7542 7543 /* We already marked aux for masking from non-speculative 7544 * paths, thus we got here in the first place. We only care 7545 * to explore bad access from here. 7546 */ 7547 if (vstate->speculative) 7548 goto do_sim; 7549 7550 if (!commit_window) { 7551 if (!tnum_is_const(off_reg->var_off) && 7552 (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) 7553 return REASON_BOUNDS; 7554 7555 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || 7556 (opcode == BPF_SUB && !off_is_neg); 7557 } 7558 7559 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); 7560 if (err < 0) 7561 return err; 7562 7563 if (commit_window) { 7564 /* In commit phase we narrow the masking window based on 7565 * the observed pointer move after the simulated operation. 7566 */ 7567 alu_state = info->aux.alu_state; 7568 alu_limit = abs(info->aux.alu_limit - alu_limit); 7569 } else { 7570 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 7571 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; 7572 alu_state |= ptr_is_dst_reg ? 7573 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 7574 7575 /* Limit pruning on unknown scalars to enable deep search for 7576 * potential masking differences from other program paths. 7577 */ 7578 if (!off_is_imm) 7579 env->explore_alu_limits = true; 7580 } 7581 7582 err = update_alu_sanitation_state(aux, alu_state, alu_limit); 7583 if (err < 0) 7584 return err; 7585 do_sim: 7586 /* If we're in commit phase, we're done here given we already 7587 * pushed the truncated dst_reg into the speculative verification 7588 * stack. 7589 * 7590 * Also, when register is a known constant, we rewrite register-based 7591 * operation to immediate-based, and thus do not need masking (and as 7592 * a consequence, do not need to simulate the zero-truncation either). 7593 */ 7594 if (commit_window || off_is_imm) 7595 return 0; 7596 7597 /* Simulate and find potential out-of-bounds access under 7598 * speculative execution from truncation as a result of 7599 * masking when off was not within expected range. If off 7600 * sits in dst, then we temporarily need to move ptr there 7601 * to simulate dst (== 0) +/-= ptr. Needed, for example, 7602 * for cases where we use K-based arithmetic in one direction 7603 * and truncated reg-based in the other in order to explore 7604 * bad access. 7605 */ 7606 if (!ptr_is_dst_reg) { 7607 tmp = *dst_reg; 7608 *dst_reg = *ptr_reg; 7609 } 7610 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, 7611 env->insn_idx); 7612 if (!ptr_is_dst_reg && ret) 7613 *dst_reg = tmp; 7614 return !ret ? REASON_STACK : 0; 7615 } 7616 7617 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) 7618 { 7619 struct bpf_verifier_state *vstate = env->cur_state; 7620 7621 /* If we simulate paths under speculation, we don't update the 7622 * insn as 'seen' such that when we verify unreachable paths in 7623 * the non-speculative domain, sanitize_dead_code() can still 7624 * rewrite/sanitize them. 7625 */ 7626 if (!vstate->speculative) 7627 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; 7628 } 7629 7630 static int sanitize_err(struct bpf_verifier_env *env, 7631 const struct bpf_insn *insn, int reason, 7632 const struct bpf_reg_state *off_reg, 7633 const struct bpf_reg_state *dst_reg) 7634 { 7635 static const char *err = "pointer arithmetic with it prohibited for !root"; 7636 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; 7637 u32 dst = insn->dst_reg, src = insn->src_reg; 7638 7639 switch (reason) { 7640 case REASON_BOUNDS: 7641 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", 7642 off_reg == dst_reg ? dst : src, err); 7643 break; 7644 case REASON_TYPE: 7645 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", 7646 off_reg == dst_reg ? src : dst, err); 7647 break; 7648 case REASON_PATHS: 7649 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", 7650 dst, op, err); 7651 break; 7652 case REASON_LIMIT: 7653 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", 7654 dst, op, err); 7655 break; 7656 case REASON_STACK: 7657 verbose(env, "R%d could not be pushed for speculative verification, %s\n", 7658 dst, err); 7659 break; 7660 default: 7661 verbose(env, "verifier internal error: unknown reason (%d)\n", 7662 reason); 7663 break; 7664 } 7665 7666 return -EACCES; 7667 } 7668 7669 /* check that stack access falls within stack limits and that 'reg' doesn't 7670 * have a variable offset. 7671 * 7672 * Variable offset is prohibited for unprivileged mode for simplicity since it 7673 * requires corresponding support in Spectre masking for stack ALU. See also 7674 * retrieve_ptr_limit(). 7675 * 7676 * 7677 * 'off' includes 'reg->off'. 7678 */ 7679 static int check_stack_access_for_ptr_arithmetic( 7680 struct bpf_verifier_env *env, 7681 int regno, 7682 const struct bpf_reg_state *reg, 7683 int off) 7684 { 7685 if (!tnum_is_const(reg->var_off)) { 7686 char tn_buf[48]; 7687 7688 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 7689 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", 7690 regno, tn_buf, off); 7691 return -EACCES; 7692 } 7693 7694 if (off >= 0 || off < -MAX_BPF_STACK) { 7695 verbose(env, "R%d stack pointer arithmetic goes out of range, " 7696 "prohibited for !root; off=%d\n", regno, off); 7697 return -EACCES; 7698 } 7699 7700 return 0; 7701 } 7702 7703 static int sanitize_check_bounds(struct bpf_verifier_env *env, 7704 const struct bpf_insn *insn, 7705 const struct bpf_reg_state *dst_reg) 7706 { 7707 u32 dst = insn->dst_reg; 7708 7709 /* For unprivileged we require that resulting offset must be in bounds 7710 * in order to be able to sanitize access later on. 7711 */ 7712 if (env->bypass_spec_v1) 7713 return 0; 7714 7715 switch (dst_reg->type) { 7716 case PTR_TO_STACK: 7717 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, 7718 dst_reg->off + dst_reg->var_off.value)) 7719 return -EACCES; 7720 break; 7721 case PTR_TO_MAP_VALUE: 7722 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { 7723 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 7724 "prohibited for !root\n", dst); 7725 return -EACCES; 7726 } 7727 break; 7728 default: 7729 break; 7730 } 7731 7732 return 0; 7733 } 7734 7735 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 7736 * Caller should also handle BPF_MOV case separately. 7737 * If we return -EACCES, caller may want to try again treating pointer as a 7738 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 7739 */ 7740 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 7741 struct bpf_insn *insn, 7742 const struct bpf_reg_state *ptr_reg, 7743 const struct bpf_reg_state *off_reg) 7744 { 7745 struct bpf_verifier_state *vstate = env->cur_state; 7746 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 7747 struct bpf_reg_state *regs = state->regs, *dst_reg; 7748 bool known = tnum_is_const(off_reg->var_off); 7749 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 7750 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 7751 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 7752 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 7753 struct bpf_sanitize_info info = {}; 7754 u8 opcode = BPF_OP(insn->code); 7755 u32 dst = insn->dst_reg; 7756 int ret; 7757 7758 dst_reg = ®s[dst]; 7759 7760 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 7761 smin_val > smax_val || umin_val > umax_val) { 7762 /* Taint dst register if offset had invalid bounds derived from 7763 * e.g. dead branches. 7764 */ 7765 __mark_reg_unknown(env, dst_reg); 7766 return 0; 7767 } 7768 7769 if (BPF_CLASS(insn->code) != BPF_ALU64) { 7770 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 7771 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 7772 __mark_reg_unknown(env, dst_reg); 7773 return 0; 7774 } 7775 7776 verbose(env, 7777 "R%d 32-bit pointer arithmetic prohibited\n", 7778 dst); 7779 return -EACCES; 7780 } 7781 7782 if (ptr_reg->type & PTR_MAYBE_NULL) { 7783 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 7784 dst, reg_type_str(env, ptr_reg->type)); 7785 return -EACCES; 7786 } 7787 7788 switch (base_type(ptr_reg->type)) { 7789 case CONST_PTR_TO_MAP: 7790 /* smin_val represents the known value */ 7791 if (known && smin_val == 0 && opcode == BPF_ADD) 7792 break; 7793 fallthrough; 7794 case PTR_TO_PACKET_END: 7795 case PTR_TO_SOCKET: 7796 case PTR_TO_SOCK_COMMON: 7797 case PTR_TO_TCP_SOCK: 7798 case PTR_TO_XDP_SOCK: 7799 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 7800 dst, reg_type_str(env, ptr_reg->type)); 7801 return -EACCES; 7802 default: 7803 break; 7804 } 7805 7806 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 7807 * The id may be overwritten later if we create a new variable offset. 7808 */ 7809 dst_reg->type = ptr_reg->type; 7810 dst_reg->id = ptr_reg->id; 7811 7812 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 7813 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 7814 return -EINVAL; 7815 7816 /* pointer types do not carry 32-bit bounds at the moment. */ 7817 __mark_reg32_unbounded(dst_reg); 7818 7819 if (sanitize_needed(opcode)) { 7820 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, 7821 &info, false); 7822 if (ret < 0) 7823 return sanitize_err(env, insn, ret, off_reg, dst_reg); 7824 } 7825 7826 switch (opcode) { 7827 case BPF_ADD: 7828 /* We can take a fixed offset as long as it doesn't overflow 7829 * the s32 'off' field 7830 */ 7831 if (known && (ptr_reg->off + smin_val == 7832 (s64)(s32)(ptr_reg->off + smin_val))) { 7833 /* pointer += K. Accumulate it into fixed offset */ 7834 dst_reg->smin_value = smin_ptr; 7835 dst_reg->smax_value = smax_ptr; 7836 dst_reg->umin_value = umin_ptr; 7837 dst_reg->umax_value = umax_ptr; 7838 dst_reg->var_off = ptr_reg->var_off; 7839 dst_reg->off = ptr_reg->off + smin_val; 7840 dst_reg->raw = ptr_reg->raw; 7841 break; 7842 } 7843 /* A new variable offset is created. Note that off_reg->off 7844 * == 0, since it's a scalar. 7845 * dst_reg gets the pointer type and since some positive 7846 * integer value was added to the pointer, give it a new 'id' 7847 * if it's a PTR_TO_PACKET. 7848 * this creates a new 'base' pointer, off_reg (variable) gets 7849 * added into the variable offset, and we copy the fixed offset 7850 * from ptr_reg. 7851 */ 7852 if (signed_add_overflows(smin_ptr, smin_val) || 7853 signed_add_overflows(smax_ptr, smax_val)) { 7854 dst_reg->smin_value = S64_MIN; 7855 dst_reg->smax_value = S64_MAX; 7856 } else { 7857 dst_reg->smin_value = smin_ptr + smin_val; 7858 dst_reg->smax_value = smax_ptr + smax_val; 7859 } 7860 if (umin_ptr + umin_val < umin_ptr || 7861 umax_ptr + umax_val < umax_ptr) { 7862 dst_reg->umin_value = 0; 7863 dst_reg->umax_value = U64_MAX; 7864 } else { 7865 dst_reg->umin_value = umin_ptr + umin_val; 7866 dst_reg->umax_value = umax_ptr + umax_val; 7867 } 7868 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 7869 dst_reg->off = ptr_reg->off; 7870 dst_reg->raw = ptr_reg->raw; 7871 if (reg_is_pkt_pointer(ptr_reg)) { 7872 dst_reg->id = ++env->id_gen; 7873 /* something was added to pkt_ptr, set range to zero */ 7874 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 7875 } 7876 break; 7877 case BPF_SUB: 7878 if (dst_reg == off_reg) { 7879 /* scalar -= pointer. Creates an unknown scalar */ 7880 verbose(env, "R%d tried to subtract pointer from scalar\n", 7881 dst); 7882 return -EACCES; 7883 } 7884 /* We don't allow subtraction from FP, because (according to 7885 * test_verifier.c test "invalid fp arithmetic", JITs might not 7886 * be able to deal with it. 7887 */ 7888 if (ptr_reg->type == PTR_TO_STACK) { 7889 verbose(env, "R%d subtraction from stack pointer prohibited\n", 7890 dst); 7891 return -EACCES; 7892 } 7893 if (known && (ptr_reg->off - smin_val == 7894 (s64)(s32)(ptr_reg->off - smin_val))) { 7895 /* pointer -= K. Subtract it from fixed offset */ 7896 dst_reg->smin_value = smin_ptr; 7897 dst_reg->smax_value = smax_ptr; 7898 dst_reg->umin_value = umin_ptr; 7899 dst_reg->umax_value = umax_ptr; 7900 dst_reg->var_off = ptr_reg->var_off; 7901 dst_reg->id = ptr_reg->id; 7902 dst_reg->off = ptr_reg->off - smin_val; 7903 dst_reg->raw = ptr_reg->raw; 7904 break; 7905 } 7906 /* A new variable offset is created. If the subtrahend is known 7907 * nonnegative, then any reg->range we had before is still good. 7908 */ 7909 if (signed_sub_overflows(smin_ptr, smax_val) || 7910 signed_sub_overflows(smax_ptr, smin_val)) { 7911 /* Overflow possible, we know nothing */ 7912 dst_reg->smin_value = S64_MIN; 7913 dst_reg->smax_value = S64_MAX; 7914 } else { 7915 dst_reg->smin_value = smin_ptr - smax_val; 7916 dst_reg->smax_value = smax_ptr - smin_val; 7917 } 7918 if (umin_ptr < umax_val) { 7919 /* Overflow possible, we know nothing */ 7920 dst_reg->umin_value = 0; 7921 dst_reg->umax_value = U64_MAX; 7922 } else { 7923 /* Cannot overflow (as long as bounds are consistent) */ 7924 dst_reg->umin_value = umin_ptr - umax_val; 7925 dst_reg->umax_value = umax_ptr - umin_val; 7926 } 7927 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 7928 dst_reg->off = ptr_reg->off; 7929 dst_reg->raw = ptr_reg->raw; 7930 if (reg_is_pkt_pointer(ptr_reg)) { 7931 dst_reg->id = ++env->id_gen; 7932 /* something was added to pkt_ptr, set range to zero */ 7933 if (smin_val < 0) 7934 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); 7935 } 7936 break; 7937 case BPF_AND: 7938 case BPF_OR: 7939 case BPF_XOR: 7940 /* bitwise ops on pointers are troublesome, prohibit. */ 7941 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 7942 dst, bpf_alu_string[opcode >> 4]); 7943 return -EACCES; 7944 default: 7945 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 7946 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 7947 dst, bpf_alu_string[opcode >> 4]); 7948 return -EACCES; 7949 } 7950 7951 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 7952 return -EINVAL; 7953 7954 __update_reg_bounds(dst_reg); 7955 __reg_deduce_bounds(dst_reg); 7956 __reg_bound_offset(dst_reg); 7957 7958 if (sanitize_check_bounds(env, insn, dst_reg) < 0) 7959 return -EACCES; 7960 if (sanitize_needed(opcode)) { 7961 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, 7962 &info, true); 7963 if (ret < 0) 7964 return sanitize_err(env, insn, ret, off_reg, dst_reg); 7965 } 7966 7967 return 0; 7968 } 7969 7970 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, 7971 struct bpf_reg_state *src_reg) 7972 { 7973 s32 smin_val = src_reg->s32_min_value; 7974 s32 smax_val = src_reg->s32_max_value; 7975 u32 umin_val = src_reg->u32_min_value; 7976 u32 umax_val = src_reg->u32_max_value; 7977 7978 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || 7979 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { 7980 dst_reg->s32_min_value = S32_MIN; 7981 dst_reg->s32_max_value = S32_MAX; 7982 } else { 7983 dst_reg->s32_min_value += smin_val; 7984 dst_reg->s32_max_value += smax_val; 7985 } 7986 if (dst_reg->u32_min_value + umin_val < umin_val || 7987 dst_reg->u32_max_value + umax_val < umax_val) { 7988 dst_reg->u32_min_value = 0; 7989 dst_reg->u32_max_value = U32_MAX; 7990 } else { 7991 dst_reg->u32_min_value += umin_val; 7992 dst_reg->u32_max_value += umax_val; 7993 } 7994 } 7995 7996 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, 7997 struct bpf_reg_state *src_reg) 7998 { 7999 s64 smin_val = src_reg->smin_value; 8000 s64 smax_val = src_reg->smax_value; 8001 u64 umin_val = src_reg->umin_value; 8002 u64 umax_val = src_reg->umax_value; 8003 8004 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 8005 signed_add_overflows(dst_reg->smax_value, smax_val)) { 8006 dst_reg->smin_value = S64_MIN; 8007 dst_reg->smax_value = S64_MAX; 8008 } else { 8009 dst_reg->smin_value += smin_val; 8010 dst_reg->smax_value += smax_val; 8011 } 8012 if (dst_reg->umin_value + umin_val < umin_val || 8013 dst_reg->umax_value + umax_val < umax_val) { 8014 dst_reg->umin_value = 0; 8015 dst_reg->umax_value = U64_MAX; 8016 } else { 8017 dst_reg->umin_value += umin_val; 8018 dst_reg->umax_value += umax_val; 8019 } 8020 } 8021 8022 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, 8023 struct bpf_reg_state *src_reg) 8024 { 8025 s32 smin_val = src_reg->s32_min_value; 8026 s32 smax_val = src_reg->s32_max_value; 8027 u32 umin_val = src_reg->u32_min_value; 8028 u32 umax_val = src_reg->u32_max_value; 8029 8030 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || 8031 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { 8032 /* Overflow possible, we know nothing */ 8033 dst_reg->s32_min_value = S32_MIN; 8034 dst_reg->s32_max_value = S32_MAX; 8035 } else { 8036 dst_reg->s32_min_value -= smax_val; 8037 dst_reg->s32_max_value -= smin_val; 8038 } 8039 if (dst_reg->u32_min_value < umax_val) { 8040 /* Overflow possible, we know nothing */ 8041 dst_reg->u32_min_value = 0; 8042 dst_reg->u32_max_value = U32_MAX; 8043 } else { 8044 /* Cannot overflow (as long as bounds are consistent) */ 8045 dst_reg->u32_min_value -= umax_val; 8046 dst_reg->u32_max_value -= umin_val; 8047 } 8048 } 8049 8050 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, 8051 struct bpf_reg_state *src_reg) 8052 { 8053 s64 smin_val = src_reg->smin_value; 8054 s64 smax_val = src_reg->smax_value; 8055 u64 umin_val = src_reg->umin_value; 8056 u64 umax_val = src_reg->umax_value; 8057 8058 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 8059 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 8060 /* Overflow possible, we know nothing */ 8061 dst_reg->smin_value = S64_MIN; 8062 dst_reg->smax_value = S64_MAX; 8063 } else { 8064 dst_reg->smin_value -= smax_val; 8065 dst_reg->smax_value -= smin_val; 8066 } 8067 if (dst_reg->umin_value < umax_val) { 8068 /* Overflow possible, we know nothing */ 8069 dst_reg->umin_value = 0; 8070 dst_reg->umax_value = U64_MAX; 8071 } else { 8072 /* Cannot overflow (as long as bounds are consistent) */ 8073 dst_reg->umin_value -= umax_val; 8074 dst_reg->umax_value -= umin_val; 8075 } 8076 } 8077 8078 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, 8079 struct bpf_reg_state *src_reg) 8080 { 8081 s32 smin_val = src_reg->s32_min_value; 8082 u32 umin_val = src_reg->u32_min_value; 8083 u32 umax_val = src_reg->u32_max_value; 8084 8085 if (smin_val < 0 || dst_reg->s32_min_value < 0) { 8086 /* Ain't nobody got time to multiply that sign */ 8087 __mark_reg32_unbounded(dst_reg); 8088 return; 8089 } 8090 /* Both values are positive, so we can work with unsigned and 8091 * copy the result to signed (unless it exceeds S32_MAX). 8092 */ 8093 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { 8094 /* Potential overflow, we know nothing */ 8095 __mark_reg32_unbounded(dst_reg); 8096 return; 8097 } 8098 dst_reg->u32_min_value *= umin_val; 8099 dst_reg->u32_max_value *= umax_val; 8100 if (dst_reg->u32_max_value > S32_MAX) { 8101 /* Overflow possible, we know nothing */ 8102 dst_reg->s32_min_value = S32_MIN; 8103 dst_reg->s32_max_value = S32_MAX; 8104 } else { 8105 dst_reg->s32_min_value = dst_reg->u32_min_value; 8106 dst_reg->s32_max_value = dst_reg->u32_max_value; 8107 } 8108 } 8109 8110 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, 8111 struct bpf_reg_state *src_reg) 8112 { 8113 s64 smin_val = src_reg->smin_value; 8114 u64 umin_val = src_reg->umin_value; 8115 u64 umax_val = src_reg->umax_value; 8116 8117 if (smin_val < 0 || dst_reg->smin_value < 0) { 8118 /* Ain't nobody got time to multiply that sign */ 8119 __mark_reg64_unbounded(dst_reg); 8120 return; 8121 } 8122 /* Both values are positive, so we can work with unsigned and 8123 * copy the result to signed (unless it exceeds S64_MAX). 8124 */ 8125 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 8126 /* Potential overflow, we know nothing */ 8127 __mark_reg64_unbounded(dst_reg); 8128 return; 8129 } 8130 dst_reg->umin_value *= umin_val; 8131 dst_reg->umax_value *= umax_val; 8132 if (dst_reg->umax_value > S64_MAX) { 8133 /* Overflow possible, we know nothing */ 8134 dst_reg->smin_value = S64_MIN; 8135 dst_reg->smax_value = S64_MAX; 8136 } else { 8137 dst_reg->smin_value = dst_reg->umin_value; 8138 dst_reg->smax_value = dst_reg->umax_value; 8139 } 8140 } 8141 8142 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, 8143 struct bpf_reg_state *src_reg) 8144 { 8145 bool src_known = tnum_subreg_is_const(src_reg->var_off); 8146 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 8147 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 8148 s32 smin_val = src_reg->s32_min_value; 8149 u32 umax_val = src_reg->u32_max_value; 8150 8151 if (src_known && dst_known) { 8152 __mark_reg32_known(dst_reg, var32_off.value); 8153 return; 8154 } 8155 8156 /* We get our minimum from the var_off, since that's inherently 8157 * bitwise. Our maximum is the minimum of the operands' maxima. 8158 */ 8159 dst_reg->u32_min_value = var32_off.value; 8160 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); 8161 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 8162 /* Lose signed bounds when ANDing negative numbers, 8163 * ain't nobody got time for that. 8164 */ 8165 dst_reg->s32_min_value = S32_MIN; 8166 dst_reg->s32_max_value = S32_MAX; 8167 } else { 8168 /* ANDing two positives gives a positive, so safe to 8169 * cast result into s64. 8170 */ 8171 dst_reg->s32_min_value = dst_reg->u32_min_value; 8172 dst_reg->s32_max_value = dst_reg->u32_max_value; 8173 } 8174 } 8175 8176 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, 8177 struct bpf_reg_state *src_reg) 8178 { 8179 bool src_known = tnum_is_const(src_reg->var_off); 8180 bool dst_known = tnum_is_const(dst_reg->var_off); 8181 s64 smin_val = src_reg->smin_value; 8182 u64 umax_val = src_reg->umax_value; 8183 8184 if (src_known && dst_known) { 8185 __mark_reg_known(dst_reg, dst_reg->var_off.value); 8186 return; 8187 } 8188 8189 /* We get our minimum from the var_off, since that's inherently 8190 * bitwise. Our maximum is the minimum of the operands' maxima. 8191 */ 8192 dst_reg->umin_value = dst_reg->var_off.value; 8193 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 8194 if (dst_reg->smin_value < 0 || smin_val < 0) { 8195 /* Lose signed bounds when ANDing negative numbers, 8196 * ain't nobody got time for that. 8197 */ 8198 dst_reg->smin_value = S64_MIN; 8199 dst_reg->smax_value = S64_MAX; 8200 } else { 8201 /* ANDing two positives gives a positive, so safe to 8202 * cast result into s64. 8203 */ 8204 dst_reg->smin_value = dst_reg->umin_value; 8205 dst_reg->smax_value = dst_reg->umax_value; 8206 } 8207 /* We may learn something more from the var_off */ 8208 __update_reg_bounds(dst_reg); 8209 } 8210 8211 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, 8212 struct bpf_reg_state *src_reg) 8213 { 8214 bool src_known = tnum_subreg_is_const(src_reg->var_off); 8215 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 8216 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 8217 s32 smin_val = src_reg->s32_min_value; 8218 u32 umin_val = src_reg->u32_min_value; 8219 8220 if (src_known && dst_known) { 8221 __mark_reg32_known(dst_reg, var32_off.value); 8222 return; 8223 } 8224 8225 /* We get our maximum from the var_off, and our minimum is the 8226 * maximum of the operands' minima 8227 */ 8228 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); 8229 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 8230 if (dst_reg->s32_min_value < 0 || smin_val < 0) { 8231 /* Lose signed bounds when ORing negative numbers, 8232 * ain't nobody got time for that. 8233 */ 8234 dst_reg->s32_min_value = S32_MIN; 8235 dst_reg->s32_max_value = S32_MAX; 8236 } else { 8237 /* ORing two positives gives a positive, so safe to 8238 * cast result into s64. 8239 */ 8240 dst_reg->s32_min_value = dst_reg->u32_min_value; 8241 dst_reg->s32_max_value = dst_reg->u32_max_value; 8242 } 8243 } 8244 8245 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, 8246 struct bpf_reg_state *src_reg) 8247 { 8248 bool src_known = tnum_is_const(src_reg->var_off); 8249 bool dst_known = tnum_is_const(dst_reg->var_off); 8250 s64 smin_val = src_reg->smin_value; 8251 u64 umin_val = src_reg->umin_value; 8252 8253 if (src_known && dst_known) { 8254 __mark_reg_known(dst_reg, dst_reg->var_off.value); 8255 return; 8256 } 8257 8258 /* We get our maximum from the var_off, and our minimum is the 8259 * maximum of the operands' minima 8260 */ 8261 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 8262 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 8263 if (dst_reg->smin_value < 0 || smin_val < 0) { 8264 /* Lose signed bounds when ORing negative numbers, 8265 * ain't nobody got time for that. 8266 */ 8267 dst_reg->smin_value = S64_MIN; 8268 dst_reg->smax_value = S64_MAX; 8269 } else { 8270 /* ORing two positives gives a positive, so safe to 8271 * cast result into s64. 8272 */ 8273 dst_reg->smin_value = dst_reg->umin_value; 8274 dst_reg->smax_value = dst_reg->umax_value; 8275 } 8276 /* We may learn something more from the var_off */ 8277 __update_reg_bounds(dst_reg); 8278 } 8279 8280 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, 8281 struct bpf_reg_state *src_reg) 8282 { 8283 bool src_known = tnum_subreg_is_const(src_reg->var_off); 8284 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); 8285 struct tnum var32_off = tnum_subreg(dst_reg->var_off); 8286 s32 smin_val = src_reg->s32_min_value; 8287 8288 if (src_known && dst_known) { 8289 __mark_reg32_known(dst_reg, var32_off.value); 8290 return; 8291 } 8292 8293 /* We get both minimum and maximum from the var32_off. */ 8294 dst_reg->u32_min_value = var32_off.value; 8295 dst_reg->u32_max_value = var32_off.value | var32_off.mask; 8296 8297 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { 8298 /* XORing two positive sign numbers gives a positive, 8299 * so safe to cast u32 result into s32. 8300 */ 8301 dst_reg->s32_min_value = dst_reg->u32_min_value; 8302 dst_reg->s32_max_value = dst_reg->u32_max_value; 8303 } else { 8304 dst_reg->s32_min_value = S32_MIN; 8305 dst_reg->s32_max_value = S32_MAX; 8306 } 8307 } 8308 8309 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, 8310 struct bpf_reg_state *src_reg) 8311 { 8312 bool src_known = tnum_is_const(src_reg->var_off); 8313 bool dst_known = tnum_is_const(dst_reg->var_off); 8314 s64 smin_val = src_reg->smin_value; 8315 8316 if (src_known && dst_known) { 8317 /* dst_reg->var_off.value has been updated earlier */ 8318 __mark_reg_known(dst_reg, dst_reg->var_off.value); 8319 return; 8320 } 8321 8322 /* We get both minimum and maximum from the var_off. */ 8323 dst_reg->umin_value = dst_reg->var_off.value; 8324 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; 8325 8326 if (dst_reg->smin_value >= 0 && smin_val >= 0) { 8327 /* XORing two positive sign numbers gives a positive, 8328 * so safe to cast u64 result into s64. 8329 */ 8330 dst_reg->smin_value = dst_reg->umin_value; 8331 dst_reg->smax_value = dst_reg->umax_value; 8332 } else { 8333 dst_reg->smin_value = S64_MIN; 8334 dst_reg->smax_value = S64_MAX; 8335 } 8336 8337 __update_reg_bounds(dst_reg); 8338 } 8339 8340 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 8341 u64 umin_val, u64 umax_val) 8342 { 8343 /* We lose all sign bit information (except what we can pick 8344 * up from var_off) 8345 */ 8346 dst_reg->s32_min_value = S32_MIN; 8347 dst_reg->s32_max_value = S32_MAX; 8348 /* If we might shift our top bit out, then we know nothing */ 8349 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { 8350 dst_reg->u32_min_value = 0; 8351 dst_reg->u32_max_value = U32_MAX; 8352 } else { 8353 dst_reg->u32_min_value <<= umin_val; 8354 dst_reg->u32_max_value <<= umax_val; 8355 } 8356 } 8357 8358 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, 8359 struct bpf_reg_state *src_reg) 8360 { 8361 u32 umax_val = src_reg->u32_max_value; 8362 u32 umin_val = src_reg->u32_min_value; 8363 /* u32 alu operation will zext upper bits */ 8364 struct tnum subreg = tnum_subreg(dst_reg->var_off); 8365 8366 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 8367 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); 8368 /* Not required but being careful mark reg64 bounds as unknown so 8369 * that we are forced to pick them up from tnum and zext later and 8370 * if some path skips this step we are still safe. 8371 */ 8372 __mark_reg64_unbounded(dst_reg); 8373 __update_reg32_bounds(dst_reg); 8374 } 8375 8376 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, 8377 u64 umin_val, u64 umax_val) 8378 { 8379 /* Special case <<32 because it is a common compiler pattern to sign 8380 * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are 8381 * positive we know this shift will also be positive so we can track 8382 * bounds correctly. Otherwise we lose all sign bit information except 8383 * what we can pick up from var_off. Perhaps we can generalize this 8384 * later to shifts of any length. 8385 */ 8386 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) 8387 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; 8388 else 8389 dst_reg->smax_value = S64_MAX; 8390 8391 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) 8392 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; 8393 else 8394 dst_reg->smin_value = S64_MIN; 8395 8396 /* If we might shift our top bit out, then we know nothing */ 8397 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 8398 dst_reg->umin_value = 0; 8399 dst_reg->umax_value = U64_MAX; 8400 } else { 8401 dst_reg->umin_value <<= umin_val; 8402 dst_reg->umax_value <<= umax_val; 8403 } 8404 } 8405 8406 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, 8407 struct bpf_reg_state *src_reg) 8408 { 8409 u64 umax_val = src_reg->umax_value; 8410 u64 umin_val = src_reg->umin_value; 8411 8412 /* scalar64 calc uses 32bit unshifted bounds so must be called first */ 8413 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); 8414 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); 8415 8416 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 8417 /* We may learn something more from the var_off */ 8418 __update_reg_bounds(dst_reg); 8419 } 8420 8421 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, 8422 struct bpf_reg_state *src_reg) 8423 { 8424 struct tnum subreg = tnum_subreg(dst_reg->var_off); 8425 u32 umax_val = src_reg->u32_max_value; 8426 u32 umin_val = src_reg->u32_min_value; 8427 8428 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 8429 * be negative, then either: 8430 * 1) src_reg might be zero, so the sign bit of the result is 8431 * unknown, so we lose our signed bounds 8432 * 2) it's known negative, thus the unsigned bounds capture the 8433 * signed bounds 8434 * 3) the signed bounds cross zero, so they tell us nothing 8435 * about the result 8436 * If the value in dst_reg is known nonnegative, then again the 8437 * unsigned bounds capture the signed bounds. 8438 * Thus, in all cases it suffices to blow away our signed bounds 8439 * and rely on inferring new ones from the unsigned bounds and 8440 * var_off of the result. 8441 */ 8442 dst_reg->s32_min_value = S32_MIN; 8443 dst_reg->s32_max_value = S32_MAX; 8444 8445 dst_reg->var_off = tnum_rshift(subreg, umin_val); 8446 dst_reg->u32_min_value >>= umax_val; 8447 dst_reg->u32_max_value >>= umin_val; 8448 8449 __mark_reg64_unbounded(dst_reg); 8450 __update_reg32_bounds(dst_reg); 8451 } 8452 8453 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, 8454 struct bpf_reg_state *src_reg) 8455 { 8456 u64 umax_val = src_reg->umax_value; 8457 u64 umin_val = src_reg->umin_value; 8458 8459 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 8460 * be negative, then either: 8461 * 1) src_reg might be zero, so the sign bit of the result is 8462 * unknown, so we lose our signed bounds 8463 * 2) it's known negative, thus the unsigned bounds capture the 8464 * signed bounds 8465 * 3) the signed bounds cross zero, so they tell us nothing 8466 * about the result 8467 * If the value in dst_reg is known nonnegative, then again the 8468 * unsigned bounds capture the signed bounds. 8469 * Thus, in all cases it suffices to blow away our signed bounds 8470 * and rely on inferring new ones from the unsigned bounds and 8471 * var_off of the result. 8472 */ 8473 dst_reg->smin_value = S64_MIN; 8474 dst_reg->smax_value = S64_MAX; 8475 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 8476 dst_reg->umin_value >>= umax_val; 8477 dst_reg->umax_value >>= umin_val; 8478 8479 /* Its not easy to operate on alu32 bounds here because it depends 8480 * on bits being shifted in. Take easy way out and mark unbounded 8481 * so we can recalculate later from tnum. 8482 */ 8483 __mark_reg32_unbounded(dst_reg); 8484 __update_reg_bounds(dst_reg); 8485 } 8486 8487 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, 8488 struct bpf_reg_state *src_reg) 8489 { 8490 u64 umin_val = src_reg->u32_min_value; 8491 8492 /* Upon reaching here, src_known is true and 8493 * umax_val is equal to umin_val. 8494 */ 8495 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); 8496 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); 8497 8498 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); 8499 8500 /* blow away the dst_reg umin_value/umax_value and rely on 8501 * dst_reg var_off to refine the result. 8502 */ 8503 dst_reg->u32_min_value = 0; 8504 dst_reg->u32_max_value = U32_MAX; 8505 8506 __mark_reg64_unbounded(dst_reg); 8507 __update_reg32_bounds(dst_reg); 8508 } 8509 8510 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, 8511 struct bpf_reg_state *src_reg) 8512 { 8513 u64 umin_val = src_reg->umin_value; 8514 8515 /* Upon reaching here, src_known is true and umax_val is equal 8516 * to umin_val. 8517 */ 8518 dst_reg->smin_value >>= umin_val; 8519 dst_reg->smax_value >>= umin_val; 8520 8521 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); 8522 8523 /* blow away the dst_reg umin_value/umax_value and rely on 8524 * dst_reg var_off to refine the result. 8525 */ 8526 dst_reg->umin_value = 0; 8527 dst_reg->umax_value = U64_MAX; 8528 8529 /* Its not easy to operate on alu32 bounds here because it depends 8530 * on bits being shifted in from upper 32-bits. Take easy way out 8531 * and mark unbounded so we can recalculate later from tnum. 8532 */ 8533 __mark_reg32_unbounded(dst_reg); 8534 __update_reg_bounds(dst_reg); 8535 } 8536 8537 /* WARNING: This function does calculations on 64-bit values, but the actual 8538 * execution may occur on 32-bit values. Therefore, things like bitshifts 8539 * need extra checks in the 32-bit case. 8540 */ 8541 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 8542 struct bpf_insn *insn, 8543 struct bpf_reg_state *dst_reg, 8544 struct bpf_reg_state src_reg) 8545 { 8546 struct bpf_reg_state *regs = cur_regs(env); 8547 u8 opcode = BPF_OP(insn->code); 8548 bool src_known; 8549 s64 smin_val, smax_val; 8550 u64 umin_val, umax_val; 8551 s32 s32_min_val, s32_max_val; 8552 u32 u32_min_val, u32_max_val; 8553 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 8554 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); 8555 int ret; 8556 8557 smin_val = src_reg.smin_value; 8558 smax_val = src_reg.smax_value; 8559 umin_val = src_reg.umin_value; 8560 umax_val = src_reg.umax_value; 8561 8562 s32_min_val = src_reg.s32_min_value; 8563 s32_max_val = src_reg.s32_max_value; 8564 u32_min_val = src_reg.u32_min_value; 8565 u32_max_val = src_reg.u32_max_value; 8566 8567 if (alu32) { 8568 src_known = tnum_subreg_is_const(src_reg.var_off); 8569 if ((src_known && 8570 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || 8571 s32_min_val > s32_max_val || u32_min_val > u32_max_val) { 8572 /* Taint dst register if offset had invalid bounds 8573 * derived from e.g. dead branches. 8574 */ 8575 __mark_reg_unknown(env, dst_reg); 8576 return 0; 8577 } 8578 } else { 8579 src_known = tnum_is_const(src_reg.var_off); 8580 if ((src_known && 8581 (smin_val != smax_val || umin_val != umax_val)) || 8582 smin_val > smax_val || umin_val > umax_val) { 8583 /* Taint dst register if offset had invalid bounds 8584 * derived from e.g. dead branches. 8585 */ 8586 __mark_reg_unknown(env, dst_reg); 8587 return 0; 8588 } 8589 } 8590 8591 if (!src_known && 8592 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 8593 __mark_reg_unknown(env, dst_reg); 8594 return 0; 8595 } 8596 8597 if (sanitize_needed(opcode)) { 8598 ret = sanitize_val_alu(env, insn); 8599 if (ret < 0) 8600 return sanitize_err(env, insn, ret, NULL, NULL); 8601 } 8602 8603 /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. 8604 * There are two classes of instructions: The first class we track both 8605 * alu32 and alu64 sign/unsigned bounds independently this provides the 8606 * greatest amount of precision when alu operations are mixed with jmp32 8607 * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, 8608 * and BPF_OR. This is possible because these ops have fairly easy to 8609 * understand and calculate behavior in both 32-bit and 64-bit alu ops. 8610 * See alu32 verifier tests for examples. The second class of 8611 * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy 8612 * with regards to tracking sign/unsigned bounds because the bits may 8613 * cross subreg boundaries in the alu64 case. When this happens we mark 8614 * the reg unbounded in the subreg bound space and use the resulting 8615 * tnum to calculate an approximation of the sign/unsigned bounds. 8616 */ 8617 switch (opcode) { 8618 case BPF_ADD: 8619 scalar32_min_max_add(dst_reg, &src_reg); 8620 scalar_min_max_add(dst_reg, &src_reg); 8621 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 8622 break; 8623 case BPF_SUB: 8624 scalar32_min_max_sub(dst_reg, &src_reg); 8625 scalar_min_max_sub(dst_reg, &src_reg); 8626 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 8627 break; 8628 case BPF_MUL: 8629 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 8630 scalar32_min_max_mul(dst_reg, &src_reg); 8631 scalar_min_max_mul(dst_reg, &src_reg); 8632 break; 8633 case BPF_AND: 8634 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 8635 scalar32_min_max_and(dst_reg, &src_reg); 8636 scalar_min_max_and(dst_reg, &src_reg); 8637 break; 8638 case BPF_OR: 8639 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 8640 scalar32_min_max_or(dst_reg, &src_reg); 8641 scalar_min_max_or(dst_reg, &src_reg); 8642 break; 8643 case BPF_XOR: 8644 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); 8645 scalar32_min_max_xor(dst_reg, &src_reg); 8646 scalar_min_max_xor(dst_reg, &src_reg); 8647 break; 8648 case BPF_LSH: 8649 if (umax_val >= insn_bitness) { 8650 /* Shifts greater than 31 or 63 are undefined. 8651 * This includes shifts by a negative number. 8652 */ 8653 mark_reg_unknown(env, regs, insn->dst_reg); 8654 break; 8655 } 8656 if (alu32) 8657 scalar32_min_max_lsh(dst_reg, &src_reg); 8658 else 8659 scalar_min_max_lsh(dst_reg, &src_reg); 8660 break; 8661 case BPF_RSH: 8662 if (umax_val >= insn_bitness) { 8663 /* Shifts greater than 31 or 63 are undefined. 8664 * This includes shifts by a negative number. 8665 */ 8666 mark_reg_unknown(env, regs, insn->dst_reg); 8667 break; 8668 } 8669 if (alu32) 8670 scalar32_min_max_rsh(dst_reg, &src_reg); 8671 else 8672 scalar_min_max_rsh(dst_reg, &src_reg); 8673 break; 8674 case BPF_ARSH: 8675 if (umax_val >= insn_bitness) { 8676 /* Shifts greater than 31 or 63 are undefined. 8677 * This includes shifts by a negative number. 8678 */ 8679 mark_reg_unknown(env, regs, insn->dst_reg); 8680 break; 8681 } 8682 if (alu32) 8683 scalar32_min_max_arsh(dst_reg, &src_reg); 8684 else 8685 scalar_min_max_arsh(dst_reg, &src_reg); 8686 break; 8687 default: 8688 mark_reg_unknown(env, regs, insn->dst_reg); 8689 break; 8690 } 8691 8692 /* ALU32 ops are zero extended into 64bit register */ 8693 if (alu32) 8694 zext_32_to_64(dst_reg); 8695 8696 __update_reg_bounds(dst_reg); 8697 __reg_deduce_bounds(dst_reg); 8698 __reg_bound_offset(dst_reg); 8699 return 0; 8700 } 8701 8702 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 8703 * and var_off. 8704 */ 8705 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 8706 struct bpf_insn *insn) 8707 { 8708 struct bpf_verifier_state *vstate = env->cur_state; 8709 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 8710 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 8711 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 8712 u8 opcode = BPF_OP(insn->code); 8713 int err; 8714 8715 dst_reg = ®s[insn->dst_reg]; 8716 src_reg = NULL; 8717 if (dst_reg->type != SCALAR_VALUE) 8718 ptr_reg = dst_reg; 8719 else 8720 /* Make sure ID is cleared otherwise dst_reg min/max could be 8721 * incorrectly propagated into other registers by find_equal_scalars() 8722 */ 8723 dst_reg->id = 0; 8724 if (BPF_SRC(insn->code) == BPF_X) { 8725 src_reg = ®s[insn->src_reg]; 8726 if (src_reg->type != SCALAR_VALUE) { 8727 if (dst_reg->type != SCALAR_VALUE) { 8728 /* Combining two pointers by any ALU op yields 8729 * an arbitrary scalar. Disallow all math except 8730 * pointer subtraction 8731 */ 8732 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 8733 mark_reg_unknown(env, regs, insn->dst_reg); 8734 return 0; 8735 } 8736 verbose(env, "R%d pointer %s pointer prohibited\n", 8737 insn->dst_reg, 8738 bpf_alu_string[opcode >> 4]); 8739 return -EACCES; 8740 } else { 8741 /* scalar += pointer 8742 * This is legal, but we have to reverse our 8743 * src/dest handling in computing the range 8744 */ 8745 err = mark_chain_precision(env, insn->dst_reg); 8746 if (err) 8747 return err; 8748 return adjust_ptr_min_max_vals(env, insn, 8749 src_reg, dst_reg); 8750 } 8751 } else if (ptr_reg) { 8752 /* pointer += scalar */ 8753 err = mark_chain_precision(env, insn->src_reg); 8754 if (err) 8755 return err; 8756 return adjust_ptr_min_max_vals(env, insn, 8757 dst_reg, src_reg); 8758 } 8759 } else { 8760 /* Pretend the src is a reg with a known value, since we only 8761 * need to be able to read from this state. 8762 */ 8763 off_reg.type = SCALAR_VALUE; 8764 __mark_reg_known(&off_reg, insn->imm); 8765 src_reg = &off_reg; 8766 if (ptr_reg) /* pointer += K */ 8767 return adjust_ptr_min_max_vals(env, insn, 8768 ptr_reg, src_reg); 8769 } 8770 8771 /* Got here implies adding two SCALAR_VALUEs */ 8772 if (WARN_ON_ONCE(ptr_reg)) { 8773 print_verifier_state(env, state, true); 8774 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 8775 return -EINVAL; 8776 } 8777 if (WARN_ON(!src_reg)) { 8778 print_verifier_state(env, state, true); 8779 verbose(env, "verifier internal error: no src_reg\n"); 8780 return -EINVAL; 8781 } 8782 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 8783 } 8784 8785 /* check validity of 32-bit and 64-bit arithmetic operations */ 8786 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 8787 { 8788 struct bpf_reg_state *regs = cur_regs(env); 8789 u8 opcode = BPF_OP(insn->code); 8790 int err; 8791 8792 if (opcode == BPF_END || opcode == BPF_NEG) { 8793 if (opcode == BPF_NEG) { 8794 if (BPF_SRC(insn->code) != 0 || 8795 insn->src_reg != BPF_REG_0 || 8796 insn->off != 0 || insn->imm != 0) { 8797 verbose(env, "BPF_NEG uses reserved fields\n"); 8798 return -EINVAL; 8799 } 8800 } else { 8801 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 8802 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 8803 BPF_CLASS(insn->code) == BPF_ALU64) { 8804 verbose(env, "BPF_END uses reserved fields\n"); 8805 return -EINVAL; 8806 } 8807 } 8808 8809 /* check src operand */ 8810 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8811 if (err) 8812 return err; 8813 8814 if (is_pointer_value(env, insn->dst_reg)) { 8815 verbose(env, "R%d pointer arithmetic prohibited\n", 8816 insn->dst_reg); 8817 return -EACCES; 8818 } 8819 8820 /* check dest operand */ 8821 err = check_reg_arg(env, insn->dst_reg, DST_OP); 8822 if (err) 8823 return err; 8824 8825 } else if (opcode == BPF_MOV) { 8826 8827 if (BPF_SRC(insn->code) == BPF_X) { 8828 if (insn->imm != 0 || insn->off != 0) { 8829 verbose(env, "BPF_MOV uses reserved fields\n"); 8830 return -EINVAL; 8831 } 8832 8833 /* check src operand */ 8834 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8835 if (err) 8836 return err; 8837 } else { 8838 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 8839 verbose(env, "BPF_MOV uses reserved fields\n"); 8840 return -EINVAL; 8841 } 8842 } 8843 8844 /* check dest operand, mark as required later */ 8845 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 8846 if (err) 8847 return err; 8848 8849 if (BPF_SRC(insn->code) == BPF_X) { 8850 struct bpf_reg_state *src_reg = regs + insn->src_reg; 8851 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 8852 8853 if (BPF_CLASS(insn->code) == BPF_ALU64) { 8854 /* case: R1 = R2 8855 * copy register state to dest reg 8856 */ 8857 if (src_reg->type == SCALAR_VALUE && !src_reg->id) 8858 /* Assign src and dst registers the same ID 8859 * that will be used by find_equal_scalars() 8860 * to propagate min/max range. 8861 */ 8862 src_reg->id = ++env->id_gen; 8863 *dst_reg = *src_reg; 8864 dst_reg->live |= REG_LIVE_WRITTEN; 8865 dst_reg->subreg_def = DEF_NOT_SUBREG; 8866 } else { 8867 /* R1 = (u32) R2 */ 8868 if (is_pointer_value(env, insn->src_reg)) { 8869 verbose(env, 8870 "R%d partial copy of pointer\n", 8871 insn->src_reg); 8872 return -EACCES; 8873 } else if (src_reg->type == SCALAR_VALUE) { 8874 *dst_reg = *src_reg; 8875 /* Make sure ID is cleared otherwise 8876 * dst_reg min/max could be incorrectly 8877 * propagated into src_reg by find_equal_scalars() 8878 */ 8879 dst_reg->id = 0; 8880 dst_reg->live |= REG_LIVE_WRITTEN; 8881 dst_reg->subreg_def = env->insn_idx + 1; 8882 } else { 8883 mark_reg_unknown(env, regs, 8884 insn->dst_reg); 8885 } 8886 zext_32_to_64(dst_reg); 8887 8888 __update_reg_bounds(dst_reg); 8889 __reg_deduce_bounds(dst_reg); 8890 __reg_bound_offset(dst_reg); 8891 } 8892 } else { 8893 /* case: R = imm 8894 * remember the value we stored into this reg 8895 */ 8896 /* clear any state __mark_reg_known doesn't set */ 8897 mark_reg_unknown(env, regs, insn->dst_reg); 8898 regs[insn->dst_reg].type = SCALAR_VALUE; 8899 if (BPF_CLASS(insn->code) == BPF_ALU64) { 8900 __mark_reg_known(regs + insn->dst_reg, 8901 insn->imm); 8902 } else { 8903 __mark_reg_known(regs + insn->dst_reg, 8904 (u32)insn->imm); 8905 } 8906 } 8907 8908 } else if (opcode > BPF_END) { 8909 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 8910 return -EINVAL; 8911 8912 } else { /* all other ALU ops: and, sub, xor, add, ... */ 8913 8914 if (BPF_SRC(insn->code) == BPF_X) { 8915 if (insn->imm != 0 || insn->off != 0) { 8916 verbose(env, "BPF_ALU uses reserved fields\n"); 8917 return -EINVAL; 8918 } 8919 /* check src1 operand */ 8920 err = check_reg_arg(env, insn->src_reg, SRC_OP); 8921 if (err) 8922 return err; 8923 } else { 8924 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 8925 verbose(env, "BPF_ALU uses reserved fields\n"); 8926 return -EINVAL; 8927 } 8928 } 8929 8930 /* check src2 operand */ 8931 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 8932 if (err) 8933 return err; 8934 8935 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 8936 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 8937 verbose(env, "div by zero\n"); 8938 return -EINVAL; 8939 } 8940 8941 if ((opcode == BPF_LSH || opcode == BPF_RSH || 8942 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 8943 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 8944 8945 if (insn->imm < 0 || insn->imm >= size) { 8946 verbose(env, "invalid shift %d\n", insn->imm); 8947 return -EINVAL; 8948 } 8949 } 8950 8951 /* check dest operand */ 8952 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 8953 if (err) 8954 return err; 8955 8956 return adjust_reg_min_max_vals(env, insn); 8957 } 8958 8959 return 0; 8960 } 8961 8962 static void __find_good_pkt_pointers(struct bpf_func_state *state, 8963 struct bpf_reg_state *dst_reg, 8964 enum bpf_reg_type type, int new_range) 8965 { 8966 struct bpf_reg_state *reg; 8967 int i; 8968 8969 for (i = 0; i < MAX_BPF_REG; i++) { 8970 reg = &state->regs[i]; 8971 if (reg->type == type && reg->id == dst_reg->id) 8972 /* keep the maximum range already checked */ 8973 reg->range = max(reg->range, new_range); 8974 } 8975 8976 bpf_for_each_spilled_reg(i, state, reg) { 8977 if (!reg) 8978 continue; 8979 if (reg->type == type && reg->id == dst_reg->id) 8980 reg->range = max(reg->range, new_range); 8981 } 8982 } 8983 8984 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 8985 struct bpf_reg_state *dst_reg, 8986 enum bpf_reg_type type, 8987 bool range_right_open) 8988 { 8989 int new_range, i; 8990 8991 if (dst_reg->off < 0 || 8992 (dst_reg->off == 0 && range_right_open)) 8993 /* This doesn't give us any range */ 8994 return; 8995 8996 if (dst_reg->umax_value > MAX_PACKET_OFF || 8997 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 8998 /* Risk of overflow. For instance, ptr + (1<<63) may be less 8999 * than pkt_end, but that's because it's also less than pkt. 9000 */ 9001 return; 9002 9003 new_range = dst_reg->off; 9004 if (range_right_open) 9005 new_range++; 9006 9007 /* Examples for register markings: 9008 * 9009 * pkt_data in dst register: 9010 * 9011 * r2 = r3; 9012 * r2 += 8; 9013 * if (r2 > pkt_end) goto <handle exception> 9014 * <access okay> 9015 * 9016 * r2 = r3; 9017 * r2 += 8; 9018 * if (r2 < pkt_end) goto <access okay> 9019 * <handle exception> 9020 * 9021 * Where: 9022 * r2 == dst_reg, pkt_end == src_reg 9023 * r2=pkt(id=n,off=8,r=0) 9024 * r3=pkt(id=n,off=0,r=0) 9025 * 9026 * pkt_data in src register: 9027 * 9028 * r2 = r3; 9029 * r2 += 8; 9030 * if (pkt_end >= r2) goto <access okay> 9031 * <handle exception> 9032 * 9033 * r2 = r3; 9034 * r2 += 8; 9035 * if (pkt_end <= r2) goto <handle exception> 9036 * <access okay> 9037 * 9038 * Where: 9039 * pkt_end == dst_reg, r2 == src_reg 9040 * r2=pkt(id=n,off=8,r=0) 9041 * r3=pkt(id=n,off=0,r=0) 9042 * 9043 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 9044 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 9045 * and [r3, r3 + 8-1) respectively is safe to access depending on 9046 * the check. 9047 */ 9048 9049 /* If our ids match, then we must have the same max_value. And we 9050 * don't care about the other reg's fixed offset, since if it's too big 9051 * the range won't allow anything. 9052 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 9053 */ 9054 for (i = 0; i <= vstate->curframe; i++) 9055 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, 9056 new_range); 9057 } 9058 9059 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) 9060 { 9061 struct tnum subreg = tnum_subreg(reg->var_off); 9062 s32 sval = (s32)val; 9063 9064 switch (opcode) { 9065 case BPF_JEQ: 9066 if (tnum_is_const(subreg)) 9067 return !!tnum_equals_const(subreg, val); 9068 break; 9069 case BPF_JNE: 9070 if (tnum_is_const(subreg)) 9071 return !tnum_equals_const(subreg, val); 9072 break; 9073 case BPF_JSET: 9074 if ((~subreg.mask & subreg.value) & val) 9075 return 1; 9076 if (!((subreg.mask | subreg.value) & val)) 9077 return 0; 9078 break; 9079 case BPF_JGT: 9080 if (reg->u32_min_value > val) 9081 return 1; 9082 else if (reg->u32_max_value <= val) 9083 return 0; 9084 break; 9085 case BPF_JSGT: 9086 if (reg->s32_min_value > sval) 9087 return 1; 9088 else if (reg->s32_max_value <= sval) 9089 return 0; 9090 break; 9091 case BPF_JLT: 9092 if (reg->u32_max_value < val) 9093 return 1; 9094 else if (reg->u32_min_value >= val) 9095 return 0; 9096 break; 9097 case BPF_JSLT: 9098 if (reg->s32_max_value < sval) 9099 return 1; 9100 else if (reg->s32_min_value >= sval) 9101 return 0; 9102 break; 9103 case BPF_JGE: 9104 if (reg->u32_min_value >= val) 9105 return 1; 9106 else if (reg->u32_max_value < val) 9107 return 0; 9108 break; 9109 case BPF_JSGE: 9110 if (reg->s32_min_value >= sval) 9111 return 1; 9112 else if (reg->s32_max_value < sval) 9113 return 0; 9114 break; 9115 case BPF_JLE: 9116 if (reg->u32_max_value <= val) 9117 return 1; 9118 else if (reg->u32_min_value > val) 9119 return 0; 9120 break; 9121 case BPF_JSLE: 9122 if (reg->s32_max_value <= sval) 9123 return 1; 9124 else if (reg->s32_min_value > sval) 9125 return 0; 9126 break; 9127 } 9128 9129 return -1; 9130 } 9131 9132 9133 static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) 9134 { 9135 s64 sval = (s64)val; 9136 9137 switch (opcode) { 9138 case BPF_JEQ: 9139 if (tnum_is_const(reg->var_off)) 9140 return !!tnum_equals_const(reg->var_off, val); 9141 break; 9142 case BPF_JNE: 9143 if (tnum_is_const(reg->var_off)) 9144 return !tnum_equals_const(reg->var_off, val); 9145 break; 9146 case BPF_JSET: 9147 if ((~reg->var_off.mask & reg->var_off.value) & val) 9148 return 1; 9149 if (!((reg->var_off.mask | reg->var_off.value) & val)) 9150 return 0; 9151 break; 9152 case BPF_JGT: 9153 if (reg->umin_value > val) 9154 return 1; 9155 else if (reg->umax_value <= val) 9156 return 0; 9157 break; 9158 case BPF_JSGT: 9159 if (reg->smin_value > sval) 9160 return 1; 9161 else if (reg->smax_value <= sval) 9162 return 0; 9163 break; 9164 case BPF_JLT: 9165 if (reg->umax_value < val) 9166 return 1; 9167 else if (reg->umin_value >= val) 9168 return 0; 9169 break; 9170 case BPF_JSLT: 9171 if (reg->smax_value < sval) 9172 return 1; 9173 else if (reg->smin_value >= sval) 9174 return 0; 9175 break; 9176 case BPF_JGE: 9177 if (reg->umin_value >= val) 9178 return 1; 9179 else if (reg->umax_value < val) 9180 return 0; 9181 break; 9182 case BPF_JSGE: 9183 if (reg->smin_value >= sval) 9184 return 1; 9185 else if (reg->smax_value < sval) 9186 return 0; 9187 break; 9188 case BPF_JLE: 9189 if (reg->umax_value <= val) 9190 return 1; 9191 else if (reg->umin_value > val) 9192 return 0; 9193 break; 9194 case BPF_JSLE: 9195 if (reg->smax_value <= sval) 9196 return 1; 9197 else if (reg->smin_value > sval) 9198 return 0; 9199 break; 9200 } 9201 9202 return -1; 9203 } 9204 9205 /* compute branch direction of the expression "if (reg opcode val) goto target;" 9206 * and return: 9207 * 1 - branch will be taken and "goto target" will be executed 9208 * 0 - branch will not be taken and fall-through to next insn 9209 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value 9210 * range [0,10] 9211 */ 9212 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 9213 bool is_jmp32) 9214 { 9215 if (__is_pointer_value(false, reg)) { 9216 if (!reg_type_not_null(reg->type)) 9217 return -1; 9218 9219 /* If pointer is valid tests against zero will fail so we can 9220 * use this to direct branch taken. 9221 */ 9222 if (val != 0) 9223 return -1; 9224 9225 switch (opcode) { 9226 case BPF_JEQ: 9227 return 0; 9228 case BPF_JNE: 9229 return 1; 9230 default: 9231 return -1; 9232 } 9233 } 9234 9235 if (is_jmp32) 9236 return is_branch32_taken(reg, val, opcode); 9237 return is_branch64_taken(reg, val, opcode); 9238 } 9239 9240 static int flip_opcode(u32 opcode) 9241 { 9242 /* How can we transform "a <op> b" into "b <op> a"? */ 9243 static const u8 opcode_flip[16] = { 9244 /* these stay the same */ 9245 [BPF_JEQ >> 4] = BPF_JEQ, 9246 [BPF_JNE >> 4] = BPF_JNE, 9247 [BPF_JSET >> 4] = BPF_JSET, 9248 /* these swap "lesser" and "greater" (L and G in the opcodes) */ 9249 [BPF_JGE >> 4] = BPF_JLE, 9250 [BPF_JGT >> 4] = BPF_JLT, 9251 [BPF_JLE >> 4] = BPF_JGE, 9252 [BPF_JLT >> 4] = BPF_JGT, 9253 [BPF_JSGE >> 4] = BPF_JSLE, 9254 [BPF_JSGT >> 4] = BPF_JSLT, 9255 [BPF_JSLE >> 4] = BPF_JSGE, 9256 [BPF_JSLT >> 4] = BPF_JSGT 9257 }; 9258 return opcode_flip[opcode >> 4]; 9259 } 9260 9261 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, 9262 struct bpf_reg_state *src_reg, 9263 u8 opcode) 9264 { 9265 struct bpf_reg_state *pkt; 9266 9267 if (src_reg->type == PTR_TO_PACKET_END) { 9268 pkt = dst_reg; 9269 } else if (dst_reg->type == PTR_TO_PACKET_END) { 9270 pkt = src_reg; 9271 opcode = flip_opcode(opcode); 9272 } else { 9273 return -1; 9274 } 9275 9276 if (pkt->range >= 0) 9277 return -1; 9278 9279 switch (opcode) { 9280 case BPF_JLE: 9281 /* pkt <= pkt_end */ 9282 fallthrough; 9283 case BPF_JGT: 9284 /* pkt > pkt_end */ 9285 if (pkt->range == BEYOND_PKT_END) 9286 /* pkt has at last one extra byte beyond pkt_end */ 9287 return opcode == BPF_JGT; 9288 break; 9289 case BPF_JLT: 9290 /* pkt < pkt_end */ 9291 fallthrough; 9292 case BPF_JGE: 9293 /* pkt >= pkt_end */ 9294 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) 9295 return opcode == BPF_JGE; 9296 break; 9297 } 9298 return -1; 9299 } 9300 9301 /* Adjusts the register min/max values in the case that the dst_reg is the 9302 * variable register that we are working on, and src_reg is a constant or we're 9303 * simply doing a BPF_K check. 9304 * In JEQ/JNE cases we also adjust the var_off values. 9305 */ 9306 static void reg_set_min_max(struct bpf_reg_state *true_reg, 9307 struct bpf_reg_state *false_reg, 9308 u64 val, u32 val32, 9309 u8 opcode, bool is_jmp32) 9310 { 9311 struct tnum false_32off = tnum_subreg(false_reg->var_off); 9312 struct tnum false_64off = false_reg->var_off; 9313 struct tnum true_32off = tnum_subreg(true_reg->var_off); 9314 struct tnum true_64off = true_reg->var_off; 9315 s64 sval = (s64)val; 9316 s32 sval32 = (s32)val32; 9317 9318 /* If the dst_reg is a pointer, we can't learn anything about its 9319 * variable offset from the compare (unless src_reg were a pointer into 9320 * the same object, but we don't bother with that. 9321 * Since false_reg and true_reg have the same type by construction, we 9322 * only need to check one of them for pointerness. 9323 */ 9324 if (__is_pointer_value(false, false_reg)) 9325 return; 9326 9327 switch (opcode) { 9328 case BPF_JEQ: 9329 case BPF_JNE: 9330 { 9331 struct bpf_reg_state *reg = 9332 opcode == BPF_JEQ ? true_reg : false_reg; 9333 9334 /* JEQ/JNE comparison doesn't change the register equivalence. 9335 * r1 = r2; 9336 * if (r1 == 42) goto label; 9337 * ... 9338 * label: // here both r1 and r2 are known to be 42. 9339 * 9340 * Hence when marking register as known preserve it's ID. 9341 */ 9342 if (is_jmp32) 9343 __mark_reg32_known(reg, val32); 9344 else 9345 ___mark_reg_known(reg, val); 9346 break; 9347 } 9348 case BPF_JSET: 9349 if (is_jmp32) { 9350 false_32off = tnum_and(false_32off, tnum_const(~val32)); 9351 if (is_power_of_2(val32)) 9352 true_32off = tnum_or(true_32off, 9353 tnum_const(val32)); 9354 } else { 9355 false_64off = tnum_and(false_64off, tnum_const(~val)); 9356 if (is_power_of_2(val)) 9357 true_64off = tnum_or(true_64off, 9358 tnum_const(val)); 9359 } 9360 break; 9361 case BPF_JGE: 9362 case BPF_JGT: 9363 { 9364 if (is_jmp32) { 9365 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; 9366 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; 9367 9368 false_reg->u32_max_value = min(false_reg->u32_max_value, 9369 false_umax); 9370 true_reg->u32_min_value = max(true_reg->u32_min_value, 9371 true_umin); 9372 } else { 9373 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 9374 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 9375 9376 false_reg->umax_value = min(false_reg->umax_value, false_umax); 9377 true_reg->umin_value = max(true_reg->umin_value, true_umin); 9378 } 9379 break; 9380 } 9381 case BPF_JSGE: 9382 case BPF_JSGT: 9383 { 9384 if (is_jmp32) { 9385 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; 9386 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; 9387 9388 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); 9389 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); 9390 } else { 9391 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 9392 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 9393 9394 false_reg->smax_value = min(false_reg->smax_value, false_smax); 9395 true_reg->smin_value = max(true_reg->smin_value, true_smin); 9396 } 9397 break; 9398 } 9399 case BPF_JLE: 9400 case BPF_JLT: 9401 { 9402 if (is_jmp32) { 9403 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; 9404 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; 9405 9406 false_reg->u32_min_value = max(false_reg->u32_min_value, 9407 false_umin); 9408 true_reg->u32_max_value = min(true_reg->u32_max_value, 9409 true_umax); 9410 } else { 9411 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 9412 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 9413 9414 false_reg->umin_value = max(false_reg->umin_value, false_umin); 9415 true_reg->umax_value = min(true_reg->umax_value, true_umax); 9416 } 9417 break; 9418 } 9419 case BPF_JSLE: 9420 case BPF_JSLT: 9421 { 9422 if (is_jmp32) { 9423 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; 9424 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; 9425 9426 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); 9427 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); 9428 } else { 9429 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 9430 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 9431 9432 false_reg->smin_value = max(false_reg->smin_value, false_smin); 9433 true_reg->smax_value = min(true_reg->smax_value, true_smax); 9434 } 9435 break; 9436 } 9437 default: 9438 return; 9439 } 9440 9441 if (is_jmp32) { 9442 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), 9443 tnum_subreg(false_32off)); 9444 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), 9445 tnum_subreg(true_32off)); 9446 __reg_combine_32_into_64(false_reg); 9447 __reg_combine_32_into_64(true_reg); 9448 } else { 9449 false_reg->var_off = false_64off; 9450 true_reg->var_off = true_64off; 9451 __reg_combine_64_into_32(false_reg); 9452 __reg_combine_64_into_32(true_reg); 9453 } 9454 } 9455 9456 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 9457 * the variable reg. 9458 */ 9459 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 9460 struct bpf_reg_state *false_reg, 9461 u64 val, u32 val32, 9462 u8 opcode, bool is_jmp32) 9463 { 9464 opcode = flip_opcode(opcode); 9465 /* This uses zero as "not present in table"; luckily the zero opcode, 9466 * BPF_JA, can't get here. 9467 */ 9468 if (opcode) 9469 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); 9470 } 9471 9472 /* Regs are known to be equal, so intersect their min/max/var_off */ 9473 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 9474 struct bpf_reg_state *dst_reg) 9475 { 9476 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 9477 dst_reg->umin_value); 9478 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 9479 dst_reg->umax_value); 9480 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 9481 dst_reg->smin_value); 9482 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 9483 dst_reg->smax_value); 9484 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 9485 dst_reg->var_off); 9486 /* We might have learned new bounds from the var_off. */ 9487 __update_reg_bounds(src_reg); 9488 __update_reg_bounds(dst_reg); 9489 /* We might have learned something about the sign bit. */ 9490 __reg_deduce_bounds(src_reg); 9491 __reg_deduce_bounds(dst_reg); 9492 /* We might have learned some bits from the bounds. */ 9493 __reg_bound_offset(src_reg); 9494 __reg_bound_offset(dst_reg); 9495 /* Intersecting with the old var_off might have improved our bounds 9496 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 9497 * then new var_off is (0; 0x7f...fc) which improves our umax. 9498 */ 9499 __update_reg_bounds(src_reg); 9500 __update_reg_bounds(dst_reg); 9501 } 9502 9503 static void reg_combine_min_max(struct bpf_reg_state *true_src, 9504 struct bpf_reg_state *true_dst, 9505 struct bpf_reg_state *false_src, 9506 struct bpf_reg_state *false_dst, 9507 u8 opcode) 9508 { 9509 switch (opcode) { 9510 case BPF_JEQ: 9511 __reg_combine_min_max(true_src, true_dst); 9512 break; 9513 case BPF_JNE: 9514 __reg_combine_min_max(false_src, false_dst); 9515 break; 9516 } 9517 } 9518 9519 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 9520 struct bpf_reg_state *reg, u32 id, 9521 bool is_null) 9522 { 9523 if (type_may_be_null(reg->type) && reg->id == id && 9524 !WARN_ON_ONCE(!reg->id)) { 9525 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || 9526 !tnum_equals_const(reg->var_off, 0) || 9527 reg->off)) { 9528 /* Old offset (both fixed and variable parts) should 9529 * have been known-zero, because we don't allow pointer 9530 * arithmetic on pointers that might be NULL. If we 9531 * see this happening, don't convert the register. 9532 */ 9533 return; 9534 } 9535 if (is_null) { 9536 reg->type = SCALAR_VALUE; 9537 /* We don't need id and ref_obj_id from this point 9538 * onwards anymore, thus we should better reset it, 9539 * so that state pruning has chances to take effect. 9540 */ 9541 reg->id = 0; 9542 reg->ref_obj_id = 0; 9543 9544 return; 9545 } 9546 9547 mark_ptr_not_null_reg(reg); 9548 9549 if (!reg_may_point_to_spin_lock(reg)) { 9550 /* For not-NULL ptr, reg->ref_obj_id will be reset 9551 * in release_reg_references(). 9552 * 9553 * reg->id is still used by spin_lock ptr. Other 9554 * than spin_lock ptr type, reg->id can be reset. 9555 */ 9556 reg->id = 0; 9557 } 9558 } 9559 } 9560 9561 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, 9562 bool is_null) 9563 { 9564 struct bpf_reg_state *reg; 9565 int i; 9566 9567 for (i = 0; i < MAX_BPF_REG; i++) 9568 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); 9569 9570 bpf_for_each_spilled_reg(i, state, reg) { 9571 if (!reg) 9572 continue; 9573 mark_ptr_or_null_reg(state, reg, id, is_null); 9574 } 9575 } 9576 9577 /* The logic is similar to find_good_pkt_pointers(), both could eventually 9578 * be folded together at some point. 9579 */ 9580 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 9581 bool is_null) 9582 { 9583 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 9584 struct bpf_reg_state *regs = state->regs; 9585 u32 ref_obj_id = regs[regno].ref_obj_id; 9586 u32 id = regs[regno].id; 9587 int i; 9588 9589 if (ref_obj_id && ref_obj_id == id && is_null) 9590 /* regs[regno] is in the " == NULL" branch. 9591 * No one could have freed the reference state before 9592 * doing the NULL check. 9593 */ 9594 WARN_ON_ONCE(release_reference_state(state, id)); 9595 9596 for (i = 0; i <= vstate->curframe; i++) 9597 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); 9598 } 9599 9600 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 9601 struct bpf_reg_state *dst_reg, 9602 struct bpf_reg_state *src_reg, 9603 struct bpf_verifier_state *this_branch, 9604 struct bpf_verifier_state *other_branch) 9605 { 9606 if (BPF_SRC(insn->code) != BPF_X) 9607 return false; 9608 9609 /* Pointers are always 64-bit. */ 9610 if (BPF_CLASS(insn->code) == BPF_JMP32) 9611 return false; 9612 9613 switch (BPF_OP(insn->code)) { 9614 case BPF_JGT: 9615 if ((dst_reg->type == PTR_TO_PACKET && 9616 src_reg->type == PTR_TO_PACKET_END) || 9617 (dst_reg->type == PTR_TO_PACKET_META && 9618 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 9619 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 9620 find_good_pkt_pointers(this_branch, dst_reg, 9621 dst_reg->type, false); 9622 mark_pkt_end(other_branch, insn->dst_reg, true); 9623 } else if ((dst_reg->type == PTR_TO_PACKET_END && 9624 src_reg->type == PTR_TO_PACKET) || 9625 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 9626 src_reg->type == PTR_TO_PACKET_META)) { 9627 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 9628 find_good_pkt_pointers(other_branch, src_reg, 9629 src_reg->type, true); 9630 mark_pkt_end(this_branch, insn->src_reg, false); 9631 } else { 9632 return false; 9633 } 9634 break; 9635 case BPF_JLT: 9636 if ((dst_reg->type == PTR_TO_PACKET && 9637 src_reg->type == PTR_TO_PACKET_END) || 9638 (dst_reg->type == PTR_TO_PACKET_META && 9639 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 9640 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 9641 find_good_pkt_pointers(other_branch, dst_reg, 9642 dst_reg->type, true); 9643 mark_pkt_end(this_branch, insn->dst_reg, false); 9644 } else if ((dst_reg->type == PTR_TO_PACKET_END && 9645 src_reg->type == PTR_TO_PACKET) || 9646 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 9647 src_reg->type == PTR_TO_PACKET_META)) { 9648 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 9649 find_good_pkt_pointers(this_branch, src_reg, 9650 src_reg->type, false); 9651 mark_pkt_end(other_branch, insn->src_reg, true); 9652 } else { 9653 return false; 9654 } 9655 break; 9656 case BPF_JGE: 9657 if ((dst_reg->type == PTR_TO_PACKET && 9658 src_reg->type == PTR_TO_PACKET_END) || 9659 (dst_reg->type == PTR_TO_PACKET_META && 9660 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 9661 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 9662 find_good_pkt_pointers(this_branch, dst_reg, 9663 dst_reg->type, true); 9664 mark_pkt_end(other_branch, insn->dst_reg, false); 9665 } else if ((dst_reg->type == PTR_TO_PACKET_END && 9666 src_reg->type == PTR_TO_PACKET) || 9667 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 9668 src_reg->type == PTR_TO_PACKET_META)) { 9669 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 9670 find_good_pkt_pointers(other_branch, src_reg, 9671 src_reg->type, false); 9672 mark_pkt_end(this_branch, insn->src_reg, true); 9673 } else { 9674 return false; 9675 } 9676 break; 9677 case BPF_JLE: 9678 if ((dst_reg->type == PTR_TO_PACKET && 9679 src_reg->type == PTR_TO_PACKET_END) || 9680 (dst_reg->type == PTR_TO_PACKET_META && 9681 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 9682 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 9683 find_good_pkt_pointers(other_branch, dst_reg, 9684 dst_reg->type, false); 9685 mark_pkt_end(this_branch, insn->dst_reg, true); 9686 } else if ((dst_reg->type == PTR_TO_PACKET_END && 9687 src_reg->type == PTR_TO_PACKET) || 9688 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 9689 src_reg->type == PTR_TO_PACKET_META)) { 9690 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 9691 find_good_pkt_pointers(this_branch, src_reg, 9692 src_reg->type, true); 9693 mark_pkt_end(other_branch, insn->src_reg, false); 9694 } else { 9695 return false; 9696 } 9697 break; 9698 default: 9699 return false; 9700 } 9701 9702 return true; 9703 } 9704 9705 static void find_equal_scalars(struct bpf_verifier_state *vstate, 9706 struct bpf_reg_state *known_reg) 9707 { 9708 struct bpf_func_state *state; 9709 struct bpf_reg_state *reg; 9710 int i, j; 9711 9712 for (i = 0; i <= vstate->curframe; i++) { 9713 state = vstate->frame[i]; 9714 for (j = 0; j < MAX_BPF_REG; j++) { 9715 reg = &state->regs[j]; 9716 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 9717 *reg = *known_reg; 9718 } 9719 9720 bpf_for_each_spilled_reg(j, state, reg) { 9721 if (!reg) 9722 continue; 9723 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) 9724 *reg = *known_reg; 9725 } 9726 } 9727 } 9728 9729 static int check_cond_jmp_op(struct bpf_verifier_env *env, 9730 struct bpf_insn *insn, int *insn_idx) 9731 { 9732 struct bpf_verifier_state *this_branch = env->cur_state; 9733 struct bpf_verifier_state *other_branch; 9734 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 9735 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; 9736 u8 opcode = BPF_OP(insn->code); 9737 bool is_jmp32; 9738 int pred = -1; 9739 int err; 9740 9741 /* Only conditional jumps are expected to reach here. */ 9742 if (opcode == BPF_JA || opcode > BPF_JSLE) { 9743 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 9744 return -EINVAL; 9745 } 9746 9747 if (BPF_SRC(insn->code) == BPF_X) { 9748 if (insn->imm != 0) { 9749 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 9750 return -EINVAL; 9751 } 9752 9753 /* check src1 operand */ 9754 err = check_reg_arg(env, insn->src_reg, SRC_OP); 9755 if (err) 9756 return err; 9757 9758 if (is_pointer_value(env, insn->src_reg)) { 9759 verbose(env, "R%d pointer comparison prohibited\n", 9760 insn->src_reg); 9761 return -EACCES; 9762 } 9763 src_reg = ®s[insn->src_reg]; 9764 } else { 9765 if (insn->src_reg != BPF_REG_0) { 9766 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 9767 return -EINVAL; 9768 } 9769 } 9770 9771 /* check src2 operand */ 9772 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 9773 if (err) 9774 return err; 9775 9776 dst_reg = ®s[insn->dst_reg]; 9777 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 9778 9779 if (BPF_SRC(insn->code) == BPF_K) { 9780 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); 9781 } else if (src_reg->type == SCALAR_VALUE && 9782 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { 9783 pred = is_branch_taken(dst_reg, 9784 tnum_subreg(src_reg->var_off).value, 9785 opcode, 9786 is_jmp32); 9787 } else if (src_reg->type == SCALAR_VALUE && 9788 !is_jmp32 && tnum_is_const(src_reg->var_off)) { 9789 pred = is_branch_taken(dst_reg, 9790 src_reg->var_off.value, 9791 opcode, 9792 is_jmp32); 9793 } else if (reg_is_pkt_pointer_any(dst_reg) && 9794 reg_is_pkt_pointer_any(src_reg) && 9795 !is_jmp32) { 9796 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); 9797 } 9798 9799 if (pred >= 0) { 9800 /* If we get here with a dst_reg pointer type it is because 9801 * above is_branch_taken() special cased the 0 comparison. 9802 */ 9803 if (!__is_pointer_value(false, dst_reg)) 9804 err = mark_chain_precision(env, insn->dst_reg); 9805 if (BPF_SRC(insn->code) == BPF_X && !err && 9806 !__is_pointer_value(false, src_reg)) 9807 err = mark_chain_precision(env, insn->src_reg); 9808 if (err) 9809 return err; 9810 } 9811 9812 if (pred == 1) { 9813 /* Only follow the goto, ignore fall-through. If needed, push 9814 * the fall-through branch for simulation under speculative 9815 * execution. 9816 */ 9817 if (!env->bypass_spec_v1 && 9818 !sanitize_speculative_path(env, insn, *insn_idx + 1, 9819 *insn_idx)) 9820 return -EFAULT; 9821 *insn_idx += insn->off; 9822 return 0; 9823 } else if (pred == 0) { 9824 /* Only follow the fall-through branch, since that's where the 9825 * program will go. If needed, push the goto branch for 9826 * simulation under speculative execution. 9827 */ 9828 if (!env->bypass_spec_v1 && 9829 !sanitize_speculative_path(env, insn, 9830 *insn_idx + insn->off + 1, 9831 *insn_idx)) 9832 return -EFAULT; 9833 return 0; 9834 } 9835 9836 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 9837 false); 9838 if (!other_branch) 9839 return -EFAULT; 9840 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 9841 9842 /* detect if we are comparing against a constant value so we can adjust 9843 * our min/max values for our dst register. 9844 * this is only legit if both are scalars (or pointers to the same 9845 * object, I suppose, but we don't support that right now), because 9846 * otherwise the different base pointers mean the offsets aren't 9847 * comparable. 9848 */ 9849 if (BPF_SRC(insn->code) == BPF_X) { 9850 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 9851 9852 if (dst_reg->type == SCALAR_VALUE && 9853 src_reg->type == SCALAR_VALUE) { 9854 if (tnum_is_const(src_reg->var_off) || 9855 (is_jmp32 && 9856 tnum_is_const(tnum_subreg(src_reg->var_off)))) 9857 reg_set_min_max(&other_branch_regs[insn->dst_reg], 9858 dst_reg, 9859 src_reg->var_off.value, 9860 tnum_subreg(src_reg->var_off).value, 9861 opcode, is_jmp32); 9862 else if (tnum_is_const(dst_reg->var_off) || 9863 (is_jmp32 && 9864 tnum_is_const(tnum_subreg(dst_reg->var_off)))) 9865 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 9866 src_reg, 9867 dst_reg->var_off.value, 9868 tnum_subreg(dst_reg->var_off).value, 9869 opcode, is_jmp32); 9870 else if (!is_jmp32 && 9871 (opcode == BPF_JEQ || opcode == BPF_JNE)) 9872 /* Comparing for equality, we can combine knowledge */ 9873 reg_combine_min_max(&other_branch_regs[insn->src_reg], 9874 &other_branch_regs[insn->dst_reg], 9875 src_reg, dst_reg, opcode); 9876 if (src_reg->id && 9877 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { 9878 find_equal_scalars(this_branch, src_reg); 9879 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); 9880 } 9881 9882 } 9883 } else if (dst_reg->type == SCALAR_VALUE) { 9884 reg_set_min_max(&other_branch_regs[insn->dst_reg], 9885 dst_reg, insn->imm, (u32)insn->imm, 9886 opcode, is_jmp32); 9887 } 9888 9889 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && 9890 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { 9891 find_equal_scalars(this_branch, dst_reg); 9892 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); 9893 } 9894 9895 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 9896 * NOTE: these optimizations below are related with pointer comparison 9897 * which will never be JMP32. 9898 */ 9899 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 9900 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 9901 type_may_be_null(dst_reg->type)) { 9902 /* Mark all identical registers in each branch as either 9903 * safe or unknown depending R == 0 or R != 0 conditional. 9904 */ 9905 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 9906 opcode == BPF_JNE); 9907 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 9908 opcode == BPF_JEQ); 9909 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 9910 this_branch, other_branch) && 9911 is_pointer_value(env, insn->dst_reg)) { 9912 verbose(env, "R%d pointer comparison prohibited\n", 9913 insn->dst_reg); 9914 return -EACCES; 9915 } 9916 if (env->log.level & BPF_LOG_LEVEL) 9917 print_insn_state(env, this_branch->frame[this_branch->curframe]); 9918 return 0; 9919 } 9920 9921 /* verify BPF_LD_IMM64 instruction */ 9922 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 9923 { 9924 struct bpf_insn_aux_data *aux = cur_aux(env); 9925 struct bpf_reg_state *regs = cur_regs(env); 9926 struct bpf_reg_state *dst_reg; 9927 struct bpf_map *map; 9928 int err; 9929 9930 if (BPF_SIZE(insn->code) != BPF_DW) { 9931 verbose(env, "invalid BPF_LD_IMM insn\n"); 9932 return -EINVAL; 9933 } 9934 if (insn->off != 0) { 9935 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 9936 return -EINVAL; 9937 } 9938 9939 err = check_reg_arg(env, insn->dst_reg, DST_OP); 9940 if (err) 9941 return err; 9942 9943 dst_reg = ®s[insn->dst_reg]; 9944 if (insn->src_reg == 0) { 9945 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 9946 9947 dst_reg->type = SCALAR_VALUE; 9948 __mark_reg_known(®s[insn->dst_reg], imm); 9949 return 0; 9950 } 9951 9952 /* All special src_reg cases are listed below. From this point onwards 9953 * we either succeed and assign a corresponding dst_reg->type after 9954 * zeroing the offset, or fail and reject the program. 9955 */ 9956 mark_reg_known_zero(env, regs, insn->dst_reg); 9957 9958 if (insn->src_reg == BPF_PSEUDO_BTF_ID) { 9959 dst_reg->type = aux->btf_var.reg_type; 9960 switch (base_type(dst_reg->type)) { 9961 case PTR_TO_MEM: 9962 dst_reg->mem_size = aux->btf_var.mem_size; 9963 break; 9964 case PTR_TO_BTF_ID: 9965 dst_reg->btf = aux->btf_var.btf; 9966 dst_reg->btf_id = aux->btf_var.btf_id; 9967 break; 9968 default: 9969 verbose(env, "bpf verifier is misconfigured\n"); 9970 return -EFAULT; 9971 } 9972 return 0; 9973 } 9974 9975 if (insn->src_reg == BPF_PSEUDO_FUNC) { 9976 struct bpf_prog_aux *aux = env->prog->aux; 9977 u32 subprogno = find_subprog(env, 9978 env->insn_idx + insn->imm + 1); 9979 9980 if (!aux->func_info) { 9981 verbose(env, "missing btf func_info\n"); 9982 return -EINVAL; 9983 } 9984 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { 9985 verbose(env, "callback function not static\n"); 9986 return -EINVAL; 9987 } 9988 9989 dst_reg->type = PTR_TO_FUNC; 9990 dst_reg->subprogno = subprogno; 9991 return 0; 9992 } 9993 9994 map = env->used_maps[aux->map_index]; 9995 dst_reg->map_ptr = map; 9996 9997 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE || 9998 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) { 9999 dst_reg->type = PTR_TO_MAP_VALUE; 10000 dst_reg->off = aux->map_off; 10001 if (map_value_has_spin_lock(map)) 10002 dst_reg->id = ++env->id_gen; 10003 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD || 10004 insn->src_reg == BPF_PSEUDO_MAP_IDX) { 10005 dst_reg->type = CONST_PTR_TO_MAP; 10006 } else { 10007 verbose(env, "bpf verifier is misconfigured\n"); 10008 return -EINVAL; 10009 } 10010 10011 return 0; 10012 } 10013 10014 static bool may_access_skb(enum bpf_prog_type type) 10015 { 10016 switch (type) { 10017 case BPF_PROG_TYPE_SOCKET_FILTER: 10018 case BPF_PROG_TYPE_SCHED_CLS: 10019 case BPF_PROG_TYPE_SCHED_ACT: 10020 return true; 10021 default: 10022 return false; 10023 } 10024 } 10025 10026 /* verify safety of LD_ABS|LD_IND instructions: 10027 * - they can only appear in the programs where ctx == skb 10028 * - since they are wrappers of function calls, they scratch R1-R5 registers, 10029 * preserve R6-R9, and store return value into R0 10030 * 10031 * Implicit input: 10032 * ctx == skb == R6 == CTX 10033 * 10034 * Explicit input: 10035 * SRC == any register 10036 * IMM == 32-bit immediate 10037 * 10038 * Output: 10039 * R0 - 8/16/32-bit skb data converted to cpu endianness 10040 */ 10041 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 10042 { 10043 struct bpf_reg_state *regs = cur_regs(env); 10044 static const int ctx_reg = BPF_REG_6; 10045 u8 mode = BPF_MODE(insn->code); 10046 int i, err; 10047 10048 if (!may_access_skb(resolve_prog_type(env->prog))) { 10049 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 10050 return -EINVAL; 10051 } 10052 10053 if (!env->ops->gen_ld_abs) { 10054 verbose(env, "bpf verifier is misconfigured\n"); 10055 return -EINVAL; 10056 } 10057 10058 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 10059 BPF_SIZE(insn->code) == BPF_DW || 10060 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 10061 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 10062 return -EINVAL; 10063 } 10064 10065 /* check whether implicit source operand (register R6) is readable */ 10066 err = check_reg_arg(env, ctx_reg, SRC_OP); 10067 if (err) 10068 return err; 10069 10070 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 10071 * gen_ld_abs() may terminate the program at runtime, leading to 10072 * reference leak. 10073 */ 10074 err = check_reference_leak(env); 10075 if (err) { 10076 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 10077 return err; 10078 } 10079 10080 if (env->cur_state->active_spin_lock) { 10081 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 10082 return -EINVAL; 10083 } 10084 10085 if (regs[ctx_reg].type != PTR_TO_CTX) { 10086 verbose(env, 10087 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 10088 return -EINVAL; 10089 } 10090 10091 if (mode == BPF_IND) { 10092 /* check explicit source operand */ 10093 err = check_reg_arg(env, insn->src_reg, SRC_OP); 10094 if (err) 10095 return err; 10096 } 10097 10098 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); 10099 if (err < 0) 10100 return err; 10101 10102 /* reset caller saved regs to unreadable */ 10103 for (i = 0; i < CALLER_SAVED_REGS; i++) { 10104 mark_reg_not_init(env, regs, caller_saved[i]); 10105 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 10106 } 10107 10108 /* mark destination R0 register as readable, since it contains 10109 * the value fetched from the packet. 10110 * Already marked as written above. 10111 */ 10112 mark_reg_unknown(env, regs, BPF_REG_0); 10113 /* ld_abs load up to 32-bit skb data. */ 10114 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; 10115 return 0; 10116 } 10117 10118 static int check_return_code(struct bpf_verifier_env *env) 10119 { 10120 struct tnum enforce_attach_type_range = tnum_unknown; 10121 const struct bpf_prog *prog = env->prog; 10122 struct bpf_reg_state *reg; 10123 struct tnum range = tnum_range(0, 1); 10124 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); 10125 int err; 10126 struct bpf_func_state *frame = env->cur_state->frame[0]; 10127 const bool is_subprog = frame->subprogno; 10128 10129 /* LSM and struct_ops func-ptr's return type could be "void" */ 10130 if (!is_subprog && 10131 (prog_type == BPF_PROG_TYPE_STRUCT_OPS || 10132 prog_type == BPF_PROG_TYPE_LSM) && 10133 !prog->aux->attach_func_proto->type) 10134 return 0; 10135 10136 /* eBPF calling convention is such that R0 is used 10137 * to return the value from eBPF program. 10138 * Make sure that it's readable at this time 10139 * of bpf_exit, which means that program wrote 10140 * something into it earlier 10141 */ 10142 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 10143 if (err) 10144 return err; 10145 10146 if (is_pointer_value(env, BPF_REG_0)) { 10147 verbose(env, "R0 leaks addr as return value\n"); 10148 return -EACCES; 10149 } 10150 10151 reg = cur_regs(env) + BPF_REG_0; 10152 10153 if (frame->in_async_callback_fn) { 10154 /* enforce return zero from async callbacks like timer */ 10155 if (reg->type != SCALAR_VALUE) { 10156 verbose(env, "In async callback the register R0 is not a known value (%s)\n", 10157 reg_type_str(env, reg->type)); 10158 return -EINVAL; 10159 } 10160 10161 if (!tnum_in(tnum_const(0), reg->var_off)) { 10162 verbose_invalid_scalar(env, reg, &range, "async callback", "R0"); 10163 return -EINVAL; 10164 } 10165 return 0; 10166 } 10167 10168 if (is_subprog) { 10169 if (reg->type != SCALAR_VALUE) { 10170 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", 10171 reg_type_str(env, reg->type)); 10172 return -EINVAL; 10173 } 10174 return 0; 10175 } 10176 10177 switch (prog_type) { 10178 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 10179 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 10180 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || 10181 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || 10182 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || 10183 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || 10184 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) 10185 range = tnum_range(1, 1); 10186 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || 10187 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) 10188 range = tnum_range(0, 3); 10189 break; 10190 case BPF_PROG_TYPE_CGROUP_SKB: 10191 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { 10192 range = tnum_range(0, 3); 10193 enforce_attach_type_range = tnum_range(2, 3); 10194 } 10195 break; 10196 case BPF_PROG_TYPE_CGROUP_SOCK: 10197 case BPF_PROG_TYPE_SOCK_OPS: 10198 case BPF_PROG_TYPE_CGROUP_DEVICE: 10199 case BPF_PROG_TYPE_CGROUP_SYSCTL: 10200 case BPF_PROG_TYPE_CGROUP_SOCKOPT: 10201 break; 10202 case BPF_PROG_TYPE_RAW_TRACEPOINT: 10203 if (!env->prog->aux->attach_btf_id) 10204 return 0; 10205 range = tnum_const(0); 10206 break; 10207 case BPF_PROG_TYPE_TRACING: 10208 switch (env->prog->expected_attach_type) { 10209 case BPF_TRACE_FENTRY: 10210 case BPF_TRACE_FEXIT: 10211 range = tnum_const(0); 10212 break; 10213 case BPF_TRACE_RAW_TP: 10214 case BPF_MODIFY_RETURN: 10215 return 0; 10216 case BPF_TRACE_ITER: 10217 break; 10218 default: 10219 return -ENOTSUPP; 10220 } 10221 break; 10222 case BPF_PROG_TYPE_SK_LOOKUP: 10223 range = tnum_range(SK_DROP, SK_PASS); 10224 break; 10225 case BPF_PROG_TYPE_EXT: 10226 /* freplace program can return anything as its return value 10227 * depends on the to-be-replaced kernel func or bpf program. 10228 */ 10229 default: 10230 return 0; 10231 } 10232 10233 if (reg->type != SCALAR_VALUE) { 10234 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 10235 reg_type_str(env, reg->type)); 10236 return -EINVAL; 10237 } 10238 10239 if (!tnum_in(range, reg->var_off)) { 10240 verbose_invalid_scalar(env, reg, &range, "program exit", "R0"); 10241 return -EINVAL; 10242 } 10243 10244 if (!tnum_is_unknown(enforce_attach_type_range) && 10245 tnum_in(enforce_attach_type_range, reg->var_off)) 10246 env->prog->enforce_expected_attach_type = 1; 10247 return 0; 10248 } 10249 10250 /* non-recursive DFS pseudo code 10251 * 1 procedure DFS-iterative(G,v): 10252 * 2 label v as discovered 10253 * 3 let S be a stack 10254 * 4 S.push(v) 10255 * 5 while S is not empty 10256 * 6 t <- S.pop() 10257 * 7 if t is what we're looking for: 10258 * 8 return t 10259 * 9 for all edges e in G.adjacentEdges(t) do 10260 * 10 if edge e is already labelled 10261 * 11 continue with the next edge 10262 * 12 w <- G.adjacentVertex(t,e) 10263 * 13 if vertex w is not discovered and not explored 10264 * 14 label e as tree-edge 10265 * 15 label w as discovered 10266 * 16 S.push(w) 10267 * 17 continue at 5 10268 * 18 else if vertex w is discovered 10269 * 19 label e as back-edge 10270 * 20 else 10271 * 21 // vertex w is explored 10272 * 22 label e as forward- or cross-edge 10273 * 23 label t as explored 10274 * 24 S.pop() 10275 * 10276 * convention: 10277 * 0x10 - discovered 10278 * 0x11 - discovered and fall-through edge labelled 10279 * 0x12 - discovered and fall-through and branch edges labelled 10280 * 0x20 - explored 10281 */ 10282 10283 enum { 10284 DISCOVERED = 0x10, 10285 EXPLORED = 0x20, 10286 FALLTHROUGH = 1, 10287 BRANCH = 2, 10288 }; 10289 10290 static u32 state_htab_size(struct bpf_verifier_env *env) 10291 { 10292 return env->prog->len; 10293 } 10294 10295 static struct bpf_verifier_state_list **explored_state( 10296 struct bpf_verifier_env *env, 10297 int idx) 10298 { 10299 struct bpf_verifier_state *cur = env->cur_state; 10300 struct bpf_func_state *state = cur->frame[cur->curframe]; 10301 10302 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; 10303 } 10304 10305 static void init_explored_state(struct bpf_verifier_env *env, int idx) 10306 { 10307 env->insn_aux_data[idx].prune_point = true; 10308 } 10309 10310 enum { 10311 DONE_EXPLORING = 0, 10312 KEEP_EXPLORING = 1, 10313 }; 10314 10315 /* t, w, e - match pseudo-code above: 10316 * t - index of current instruction 10317 * w - next instruction 10318 * e - edge 10319 */ 10320 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, 10321 bool loop_ok) 10322 { 10323 int *insn_stack = env->cfg.insn_stack; 10324 int *insn_state = env->cfg.insn_state; 10325 10326 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 10327 return DONE_EXPLORING; 10328 10329 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 10330 return DONE_EXPLORING; 10331 10332 if (w < 0 || w >= env->prog->len) { 10333 verbose_linfo(env, t, "%d: ", t); 10334 verbose(env, "jump out of range from insn %d to %d\n", t, w); 10335 return -EINVAL; 10336 } 10337 10338 if (e == BRANCH) 10339 /* mark branch target for state pruning */ 10340 init_explored_state(env, w); 10341 10342 if (insn_state[w] == 0) { 10343 /* tree-edge */ 10344 insn_state[t] = DISCOVERED | e; 10345 insn_state[w] = DISCOVERED; 10346 if (env->cfg.cur_stack >= env->prog->len) 10347 return -E2BIG; 10348 insn_stack[env->cfg.cur_stack++] = w; 10349 return KEEP_EXPLORING; 10350 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 10351 if (loop_ok && env->bpf_capable) 10352 return DONE_EXPLORING; 10353 verbose_linfo(env, t, "%d: ", t); 10354 verbose_linfo(env, w, "%d: ", w); 10355 verbose(env, "back-edge from insn %d to %d\n", t, w); 10356 return -EINVAL; 10357 } else if (insn_state[w] == EXPLORED) { 10358 /* forward- or cross-edge */ 10359 insn_state[t] = DISCOVERED | e; 10360 } else { 10361 verbose(env, "insn state internal bug\n"); 10362 return -EFAULT; 10363 } 10364 return DONE_EXPLORING; 10365 } 10366 10367 static int visit_func_call_insn(int t, int insn_cnt, 10368 struct bpf_insn *insns, 10369 struct bpf_verifier_env *env, 10370 bool visit_callee) 10371 { 10372 int ret; 10373 10374 ret = push_insn(t, t + 1, FALLTHROUGH, env, false); 10375 if (ret) 10376 return ret; 10377 10378 if (t + 1 < insn_cnt) 10379 init_explored_state(env, t + 1); 10380 if (visit_callee) { 10381 init_explored_state(env, t); 10382 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, 10383 /* It's ok to allow recursion from CFG point of 10384 * view. __check_func_call() will do the actual 10385 * check. 10386 */ 10387 bpf_pseudo_func(insns + t)); 10388 } 10389 return ret; 10390 } 10391 10392 /* Visits the instruction at index t and returns one of the following: 10393 * < 0 - an error occurred 10394 * DONE_EXPLORING - the instruction was fully explored 10395 * KEEP_EXPLORING - there is still work to be done before it is fully explored 10396 */ 10397 static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) 10398 { 10399 struct bpf_insn *insns = env->prog->insnsi; 10400 int ret; 10401 10402 if (bpf_pseudo_func(insns + t)) 10403 return visit_func_call_insn(t, insn_cnt, insns, env, true); 10404 10405 /* All non-branch instructions have a single fall-through edge. */ 10406 if (BPF_CLASS(insns[t].code) != BPF_JMP && 10407 BPF_CLASS(insns[t].code) != BPF_JMP32) 10408 return push_insn(t, t + 1, FALLTHROUGH, env, false); 10409 10410 switch (BPF_OP(insns[t].code)) { 10411 case BPF_EXIT: 10412 return DONE_EXPLORING; 10413 10414 case BPF_CALL: 10415 if (insns[t].imm == BPF_FUNC_timer_set_callback) 10416 /* Mark this call insn to trigger is_state_visited() check 10417 * before call itself is processed by __check_func_call(). 10418 * Otherwise new async state will be pushed for further 10419 * exploration. 10420 */ 10421 init_explored_state(env, t); 10422 return visit_func_call_insn(t, insn_cnt, insns, env, 10423 insns[t].src_reg == BPF_PSEUDO_CALL); 10424 10425 case BPF_JA: 10426 if (BPF_SRC(insns[t].code) != BPF_K) 10427 return -EINVAL; 10428 10429 /* unconditional jump with single edge */ 10430 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, 10431 true); 10432 if (ret) 10433 return ret; 10434 10435 /* unconditional jmp is not a good pruning point, 10436 * but it's marked, since backtracking needs 10437 * to record jmp history in is_state_visited(). 10438 */ 10439 init_explored_state(env, t + insns[t].off + 1); 10440 /* tell verifier to check for equivalent states 10441 * after every call and jump 10442 */ 10443 if (t + 1 < insn_cnt) 10444 init_explored_state(env, t + 1); 10445 10446 return ret; 10447 10448 default: 10449 /* conditional jump with two edges */ 10450 init_explored_state(env, t); 10451 ret = push_insn(t, t + 1, FALLTHROUGH, env, true); 10452 if (ret) 10453 return ret; 10454 10455 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); 10456 } 10457 } 10458 10459 /* non-recursive depth-first-search to detect loops in BPF program 10460 * loop == back-edge in directed graph 10461 */ 10462 static int check_cfg(struct bpf_verifier_env *env) 10463 { 10464 int insn_cnt = env->prog->len; 10465 int *insn_stack, *insn_state; 10466 int ret = 0; 10467 int i; 10468 10469 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 10470 if (!insn_state) 10471 return -ENOMEM; 10472 10473 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 10474 if (!insn_stack) { 10475 kvfree(insn_state); 10476 return -ENOMEM; 10477 } 10478 10479 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 10480 insn_stack[0] = 0; /* 0 is the first instruction */ 10481 env->cfg.cur_stack = 1; 10482 10483 while (env->cfg.cur_stack > 0) { 10484 int t = insn_stack[env->cfg.cur_stack - 1]; 10485 10486 ret = visit_insn(t, insn_cnt, env); 10487 switch (ret) { 10488 case DONE_EXPLORING: 10489 insn_state[t] = EXPLORED; 10490 env->cfg.cur_stack--; 10491 break; 10492 case KEEP_EXPLORING: 10493 break; 10494 default: 10495 if (ret > 0) { 10496 verbose(env, "visit_insn internal bug\n"); 10497 ret = -EFAULT; 10498 } 10499 goto err_free; 10500 } 10501 } 10502 10503 if (env->cfg.cur_stack < 0) { 10504 verbose(env, "pop stack internal bug\n"); 10505 ret = -EFAULT; 10506 goto err_free; 10507 } 10508 10509 for (i = 0; i < insn_cnt; i++) { 10510 if (insn_state[i] != EXPLORED) { 10511 verbose(env, "unreachable insn %d\n", i); 10512 ret = -EINVAL; 10513 goto err_free; 10514 } 10515 } 10516 ret = 0; /* cfg looks good */ 10517 10518 err_free: 10519 kvfree(insn_state); 10520 kvfree(insn_stack); 10521 env->cfg.insn_state = env->cfg.insn_stack = NULL; 10522 return ret; 10523 } 10524 10525 static int check_abnormal_return(struct bpf_verifier_env *env) 10526 { 10527 int i; 10528 10529 for (i = 1; i < env->subprog_cnt; i++) { 10530 if (env->subprog_info[i].has_ld_abs) { 10531 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); 10532 return -EINVAL; 10533 } 10534 if (env->subprog_info[i].has_tail_call) { 10535 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); 10536 return -EINVAL; 10537 } 10538 } 10539 return 0; 10540 } 10541 10542 /* The minimum supported BTF func info size */ 10543 #define MIN_BPF_FUNCINFO_SIZE 8 10544 #define MAX_FUNCINFO_REC_SIZE 252 10545 10546 static int check_btf_func(struct bpf_verifier_env *env, 10547 const union bpf_attr *attr, 10548 bpfptr_t uattr) 10549 { 10550 const struct btf_type *type, *func_proto, *ret_type; 10551 u32 i, nfuncs, urec_size, min_size; 10552 u32 krec_size = sizeof(struct bpf_func_info); 10553 struct bpf_func_info *krecord; 10554 struct bpf_func_info_aux *info_aux = NULL; 10555 struct bpf_prog *prog; 10556 const struct btf *btf; 10557 bpfptr_t urecord; 10558 u32 prev_offset = 0; 10559 bool scalar_return; 10560 int ret = -ENOMEM; 10561 10562 nfuncs = attr->func_info_cnt; 10563 if (!nfuncs) { 10564 if (check_abnormal_return(env)) 10565 return -EINVAL; 10566 return 0; 10567 } 10568 10569 if (nfuncs != env->subprog_cnt) { 10570 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 10571 return -EINVAL; 10572 } 10573 10574 urec_size = attr->func_info_rec_size; 10575 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 10576 urec_size > MAX_FUNCINFO_REC_SIZE || 10577 urec_size % sizeof(u32)) { 10578 verbose(env, "invalid func info rec size %u\n", urec_size); 10579 return -EINVAL; 10580 } 10581 10582 prog = env->prog; 10583 btf = prog->aux->btf; 10584 10585 urecord = make_bpfptr(attr->func_info, uattr.is_kernel); 10586 min_size = min_t(u32, krec_size, urec_size); 10587 10588 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 10589 if (!krecord) 10590 return -ENOMEM; 10591 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); 10592 if (!info_aux) 10593 goto err_free; 10594 10595 for (i = 0; i < nfuncs; i++) { 10596 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 10597 if (ret) { 10598 if (ret == -E2BIG) { 10599 verbose(env, "nonzero tailing record in func info"); 10600 /* set the size kernel expects so loader can zero 10601 * out the rest of the record. 10602 */ 10603 if (copy_to_bpfptr_offset(uattr, 10604 offsetof(union bpf_attr, func_info_rec_size), 10605 &min_size, sizeof(min_size))) 10606 ret = -EFAULT; 10607 } 10608 goto err_free; 10609 } 10610 10611 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) { 10612 ret = -EFAULT; 10613 goto err_free; 10614 } 10615 10616 /* check insn_off */ 10617 ret = -EINVAL; 10618 if (i == 0) { 10619 if (krecord[i].insn_off) { 10620 verbose(env, 10621 "nonzero insn_off %u for the first func info record", 10622 krecord[i].insn_off); 10623 goto err_free; 10624 } 10625 } else if (krecord[i].insn_off <= prev_offset) { 10626 verbose(env, 10627 "same or smaller insn offset (%u) than previous func info record (%u)", 10628 krecord[i].insn_off, prev_offset); 10629 goto err_free; 10630 } 10631 10632 if (env->subprog_info[i].start != krecord[i].insn_off) { 10633 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 10634 goto err_free; 10635 } 10636 10637 /* check type_id */ 10638 type = btf_type_by_id(btf, krecord[i].type_id); 10639 if (!type || !btf_type_is_func(type)) { 10640 verbose(env, "invalid type id %d in func info", 10641 krecord[i].type_id); 10642 goto err_free; 10643 } 10644 info_aux[i].linkage = BTF_INFO_VLEN(type->info); 10645 10646 func_proto = btf_type_by_id(btf, type->type); 10647 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) 10648 /* btf_func_check() already verified it during BTF load */ 10649 goto err_free; 10650 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); 10651 scalar_return = 10652 btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); 10653 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { 10654 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); 10655 goto err_free; 10656 } 10657 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { 10658 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); 10659 goto err_free; 10660 } 10661 10662 prev_offset = krecord[i].insn_off; 10663 bpfptr_add(&urecord, urec_size); 10664 } 10665 10666 prog->aux->func_info = krecord; 10667 prog->aux->func_info_cnt = nfuncs; 10668 prog->aux->func_info_aux = info_aux; 10669 return 0; 10670 10671 err_free: 10672 kvfree(krecord); 10673 kfree(info_aux); 10674 return ret; 10675 } 10676 10677 static void adjust_btf_func(struct bpf_verifier_env *env) 10678 { 10679 struct bpf_prog_aux *aux = env->prog->aux; 10680 int i; 10681 10682 if (!aux->func_info) 10683 return; 10684 10685 for (i = 0; i < env->subprog_cnt; i++) 10686 aux->func_info[i].insn_off = env->subprog_info[i].start; 10687 } 10688 10689 #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col) 10690 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 10691 10692 static int check_btf_line(struct bpf_verifier_env *env, 10693 const union bpf_attr *attr, 10694 bpfptr_t uattr) 10695 { 10696 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 10697 struct bpf_subprog_info *sub; 10698 struct bpf_line_info *linfo; 10699 struct bpf_prog *prog; 10700 const struct btf *btf; 10701 bpfptr_t ulinfo; 10702 int err; 10703 10704 nr_linfo = attr->line_info_cnt; 10705 if (!nr_linfo) 10706 return 0; 10707 if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info)) 10708 return -EINVAL; 10709 10710 rec_size = attr->line_info_rec_size; 10711 if (rec_size < MIN_BPF_LINEINFO_SIZE || 10712 rec_size > MAX_LINEINFO_REC_SIZE || 10713 rec_size & (sizeof(u32) - 1)) 10714 return -EINVAL; 10715 10716 /* Need to zero it in case the userspace may 10717 * pass in a smaller bpf_line_info object. 10718 */ 10719 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 10720 GFP_KERNEL | __GFP_NOWARN); 10721 if (!linfo) 10722 return -ENOMEM; 10723 10724 prog = env->prog; 10725 btf = prog->aux->btf; 10726 10727 s = 0; 10728 sub = env->subprog_info; 10729 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel); 10730 expected_size = sizeof(struct bpf_line_info); 10731 ncopy = min_t(u32, expected_size, rec_size); 10732 for (i = 0; i < nr_linfo; i++) { 10733 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 10734 if (err) { 10735 if (err == -E2BIG) { 10736 verbose(env, "nonzero tailing record in line_info"); 10737 if (copy_to_bpfptr_offset(uattr, 10738 offsetof(union bpf_attr, line_info_rec_size), 10739 &expected_size, sizeof(expected_size))) 10740 err = -EFAULT; 10741 } 10742 goto err_free; 10743 } 10744 10745 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) { 10746 err = -EFAULT; 10747 goto err_free; 10748 } 10749 10750 /* 10751 * Check insn_off to ensure 10752 * 1) strictly increasing AND 10753 * 2) bounded by prog->len 10754 * 10755 * The linfo[0].insn_off == 0 check logically falls into 10756 * the later "missing bpf_line_info for func..." case 10757 * because the first linfo[0].insn_off must be the 10758 * first sub also and the first sub must have 10759 * subprog_info[0].start == 0. 10760 */ 10761 if ((i && linfo[i].insn_off <= prev_offset) || 10762 linfo[i].insn_off >= prog->len) { 10763 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 10764 i, linfo[i].insn_off, prev_offset, 10765 prog->len); 10766 err = -EINVAL; 10767 goto err_free; 10768 } 10769 10770 if (!prog->insnsi[linfo[i].insn_off].code) { 10771 verbose(env, 10772 "Invalid insn code at line_info[%u].insn_off\n", 10773 i); 10774 err = -EINVAL; 10775 goto err_free; 10776 } 10777 10778 if (!btf_name_by_offset(btf, linfo[i].line_off) || 10779 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 10780 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 10781 err = -EINVAL; 10782 goto err_free; 10783 } 10784 10785 if (s != env->subprog_cnt) { 10786 if (linfo[i].insn_off == sub[s].start) { 10787 sub[s].linfo_idx = i; 10788 s++; 10789 } else if (sub[s].start < linfo[i].insn_off) { 10790 verbose(env, "missing bpf_line_info for func#%u\n", s); 10791 err = -EINVAL; 10792 goto err_free; 10793 } 10794 } 10795 10796 prev_offset = linfo[i].insn_off; 10797 bpfptr_add(&ulinfo, rec_size); 10798 } 10799 10800 if (s != env->subprog_cnt) { 10801 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 10802 env->subprog_cnt - s, s); 10803 err = -EINVAL; 10804 goto err_free; 10805 } 10806 10807 prog->aux->linfo = linfo; 10808 prog->aux->nr_linfo = nr_linfo; 10809 10810 return 0; 10811 10812 err_free: 10813 kvfree(linfo); 10814 return err; 10815 } 10816 10817 #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) 10818 #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE 10819 10820 static int check_core_relo(struct bpf_verifier_env *env, 10821 const union bpf_attr *attr, 10822 bpfptr_t uattr) 10823 { 10824 u32 i, nr_core_relo, ncopy, expected_size, rec_size; 10825 struct bpf_core_relo core_relo = {}; 10826 struct bpf_prog *prog = env->prog; 10827 const struct btf *btf = prog->aux->btf; 10828 struct bpf_core_ctx ctx = { 10829 .log = &env->log, 10830 .btf = btf, 10831 }; 10832 bpfptr_t u_core_relo; 10833 int err; 10834 10835 nr_core_relo = attr->core_relo_cnt; 10836 if (!nr_core_relo) 10837 return 0; 10838 if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo)) 10839 return -EINVAL; 10840 10841 rec_size = attr->core_relo_rec_size; 10842 if (rec_size < MIN_CORE_RELO_SIZE || 10843 rec_size > MAX_CORE_RELO_SIZE || 10844 rec_size % sizeof(u32)) 10845 return -EINVAL; 10846 10847 u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel); 10848 expected_size = sizeof(struct bpf_core_relo); 10849 ncopy = min_t(u32, expected_size, rec_size); 10850 10851 /* Unlike func_info and line_info, copy and apply each CO-RE 10852 * relocation record one at a time. 10853 */ 10854 for (i = 0; i < nr_core_relo; i++) { 10855 /* future proofing when sizeof(bpf_core_relo) changes */ 10856 err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size); 10857 if (err) { 10858 if (err == -E2BIG) { 10859 verbose(env, "nonzero tailing record in core_relo"); 10860 if (copy_to_bpfptr_offset(uattr, 10861 offsetof(union bpf_attr, core_relo_rec_size), 10862 &expected_size, sizeof(expected_size))) 10863 err = -EFAULT; 10864 } 10865 break; 10866 } 10867 10868 if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) { 10869 err = -EFAULT; 10870 break; 10871 } 10872 10873 if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) { 10874 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", 10875 i, core_relo.insn_off, prog->len); 10876 err = -EINVAL; 10877 break; 10878 } 10879 10880 err = bpf_core_apply(&ctx, &core_relo, i, 10881 &prog->insnsi[core_relo.insn_off / 8]); 10882 if (err) 10883 break; 10884 bpfptr_add(&u_core_relo, rec_size); 10885 } 10886 return err; 10887 } 10888 10889 static int check_btf_info(struct bpf_verifier_env *env, 10890 const union bpf_attr *attr, 10891 bpfptr_t uattr) 10892 { 10893 struct btf *btf; 10894 int err; 10895 10896 if (!attr->func_info_cnt && !attr->line_info_cnt) { 10897 if (check_abnormal_return(env)) 10898 return -EINVAL; 10899 return 0; 10900 } 10901 10902 btf = btf_get_by_fd(attr->prog_btf_fd); 10903 if (IS_ERR(btf)) 10904 return PTR_ERR(btf); 10905 if (btf_is_kernel(btf)) { 10906 btf_put(btf); 10907 return -EACCES; 10908 } 10909 env->prog->aux->btf = btf; 10910 10911 err = check_btf_func(env, attr, uattr); 10912 if (err) 10913 return err; 10914 10915 err = check_btf_line(env, attr, uattr); 10916 if (err) 10917 return err; 10918 10919 err = check_core_relo(env, attr, uattr); 10920 if (err) 10921 return err; 10922 10923 return 0; 10924 } 10925 10926 /* check %cur's range satisfies %old's */ 10927 static bool range_within(struct bpf_reg_state *old, 10928 struct bpf_reg_state *cur) 10929 { 10930 return old->umin_value <= cur->umin_value && 10931 old->umax_value >= cur->umax_value && 10932 old->smin_value <= cur->smin_value && 10933 old->smax_value >= cur->smax_value && 10934 old->u32_min_value <= cur->u32_min_value && 10935 old->u32_max_value >= cur->u32_max_value && 10936 old->s32_min_value <= cur->s32_min_value && 10937 old->s32_max_value >= cur->s32_max_value; 10938 } 10939 10940 /* If in the old state two registers had the same id, then they need to have 10941 * the same id in the new state as well. But that id could be different from 10942 * the old state, so we need to track the mapping from old to new ids. 10943 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 10944 * regs with old id 5 must also have new id 9 for the new state to be safe. But 10945 * regs with a different old id could still have new id 9, we don't care about 10946 * that. 10947 * So we look through our idmap to see if this old id has been seen before. If 10948 * so, we require the new id to match; otherwise, we add the id pair to the map. 10949 */ 10950 static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap) 10951 { 10952 unsigned int i; 10953 10954 for (i = 0; i < BPF_ID_MAP_SIZE; i++) { 10955 if (!idmap[i].old) { 10956 /* Reached an empty slot; haven't seen this id before */ 10957 idmap[i].old = old_id; 10958 idmap[i].cur = cur_id; 10959 return true; 10960 } 10961 if (idmap[i].old == old_id) 10962 return idmap[i].cur == cur_id; 10963 } 10964 /* We ran out of idmap slots, which should be impossible */ 10965 WARN_ON_ONCE(1); 10966 return false; 10967 } 10968 10969 static void clean_func_state(struct bpf_verifier_env *env, 10970 struct bpf_func_state *st) 10971 { 10972 enum bpf_reg_liveness live; 10973 int i, j; 10974 10975 for (i = 0; i < BPF_REG_FP; i++) { 10976 live = st->regs[i].live; 10977 /* liveness must not touch this register anymore */ 10978 st->regs[i].live |= REG_LIVE_DONE; 10979 if (!(live & REG_LIVE_READ)) 10980 /* since the register is unused, clear its state 10981 * to make further comparison simpler 10982 */ 10983 __mark_reg_not_init(env, &st->regs[i]); 10984 } 10985 10986 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 10987 live = st->stack[i].spilled_ptr.live; 10988 /* liveness must not touch this stack slot anymore */ 10989 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 10990 if (!(live & REG_LIVE_READ)) { 10991 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); 10992 for (j = 0; j < BPF_REG_SIZE; j++) 10993 st->stack[i].slot_type[j] = STACK_INVALID; 10994 } 10995 } 10996 } 10997 10998 static void clean_verifier_state(struct bpf_verifier_env *env, 10999 struct bpf_verifier_state *st) 11000 { 11001 int i; 11002 11003 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 11004 /* all regs in this state in all frames were already marked */ 11005 return; 11006 11007 for (i = 0; i <= st->curframe; i++) 11008 clean_func_state(env, st->frame[i]); 11009 } 11010 11011 /* the parentage chains form a tree. 11012 * the verifier states are added to state lists at given insn and 11013 * pushed into state stack for future exploration. 11014 * when the verifier reaches bpf_exit insn some of the verifer states 11015 * stored in the state lists have their final liveness state already, 11016 * but a lot of states will get revised from liveness point of view when 11017 * the verifier explores other branches. 11018 * Example: 11019 * 1: r0 = 1 11020 * 2: if r1 == 100 goto pc+1 11021 * 3: r0 = 2 11022 * 4: exit 11023 * when the verifier reaches exit insn the register r0 in the state list of 11024 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 11025 * of insn 2 and goes exploring further. At the insn 4 it will walk the 11026 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 11027 * 11028 * Since the verifier pushes the branch states as it sees them while exploring 11029 * the program the condition of walking the branch instruction for the second 11030 * time means that all states below this branch were already explored and 11031 * their final liveness marks are already propagated. 11032 * Hence when the verifier completes the search of state list in is_state_visited() 11033 * we can call this clean_live_states() function to mark all liveness states 11034 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 11035 * will not be used. 11036 * This function also clears the registers and stack for states that !READ 11037 * to simplify state merging. 11038 * 11039 * Important note here that walking the same branch instruction in the callee 11040 * doesn't meant that the states are DONE. The verifier has to compare 11041 * the callsites 11042 */ 11043 static void clean_live_states(struct bpf_verifier_env *env, int insn, 11044 struct bpf_verifier_state *cur) 11045 { 11046 struct bpf_verifier_state_list *sl; 11047 int i; 11048 11049 sl = *explored_state(env, insn); 11050 while (sl) { 11051 if (sl->state.branches) 11052 goto next; 11053 if (sl->state.insn_idx != insn || 11054 sl->state.curframe != cur->curframe) 11055 goto next; 11056 for (i = 0; i <= cur->curframe; i++) 11057 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 11058 goto next; 11059 clean_verifier_state(env, &sl->state); 11060 next: 11061 sl = sl->next; 11062 } 11063 } 11064 11065 /* Returns true if (rold safe implies rcur safe) */ 11066 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, 11067 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap) 11068 { 11069 bool equal; 11070 11071 if (!(rold->live & REG_LIVE_READ)) 11072 /* explored state didn't use this */ 11073 return true; 11074 11075 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 11076 11077 if (rold->type == PTR_TO_STACK) 11078 /* two stack pointers are equal only if they're pointing to 11079 * the same stack frame, since fp-8 in foo != fp-8 in bar 11080 */ 11081 return equal && rold->frameno == rcur->frameno; 11082 11083 if (equal) 11084 return true; 11085 11086 if (rold->type == NOT_INIT) 11087 /* explored state can't have used this */ 11088 return true; 11089 if (rcur->type == NOT_INIT) 11090 return false; 11091 switch (base_type(rold->type)) { 11092 case SCALAR_VALUE: 11093 if (env->explore_alu_limits) 11094 return false; 11095 if (rcur->type == SCALAR_VALUE) { 11096 if (!rold->precise && !rcur->precise) 11097 return true; 11098 /* new val must satisfy old val knowledge */ 11099 return range_within(rold, rcur) && 11100 tnum_in(rold->var_off, rcur->var_off); 11101 } else { 11102 /* We're trying to use a pointer in place of a scalar. 11103 * Even if the scalar was unbounded, this could lead to 11104 * pointer leaks because scalars are allowed to leak 11105 * while pointers are not. We could make this safe in 11106 * special cases if root is calling us, but it's 11107 * probably not worth the hassle. 11108 */ 11109 return false; 11110 } 11111 case PTR_TO_MAP_KEY: 11112 case PTR_TO_MAP_VALUE: 11113 /* a PTR_TO_MAP_VALUE could be safe to use as a 11114 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 11115 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 11116 * checked, doing so could have affected others with the same 11117 * id, and we can't check for that because we lost the id when 11118 * we converted to a PTR_TO_MAP_VALUE. 11119 */ 11120 if (type_may_be_null(rold->type)) { 11121 if (!type_may_be_null(rcur->type)) 11122 return false; 11123 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 11124 return false; 11125 /* Check our ids match any regs they're supposed to */ 11126 return check_ids(rold->id, rcur->id, idmap); 11127 } 11128 11129 /* If the new min/max/var_off satisfy the old ones and 11130 * everything else matches, we are OK. 11131 * 'id' is not compared, since it's only used for maps with 11132 * bpf_spin_lock inside map element and in such cases if 11133 * the rest of the prog is valid for one map element then 11134 * it's valid for all map elements regardless of the key 11135 * used in bpf_map_lookup() 11136 */ 11137 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 11138 range_within(rold, rcur) && 11139 tnum_in(rold->var_off, rcur->var_off); 11140 case PTR_TO_PACKET_META: 11141 case PTR_TO_PACKET: 11142 if (rcur->type != rold->type) 11143 return false; 11144 /* We must have at least as much range as the old ptr 11145 * did, so that any accesses which were safe before are 11146 * still safe. This is true even if old range < old off, 11147 * since someone could have accessed through (ptr - k), or 11148 * even done ptr -= k in a register, to get a safe access. 11149 */ 11150 if (rold->range > rcur->range) 11151 return false; 11152 /* If the offsets don't match, we can't trust our alignment; 11153 * nor can we be sure that we won't fall out of range. 11154 */ 11155 if (rold->off != rcur->off) 11156 return false; 11157 /* id relations must be preserved */ 11158 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 11159 return false; 11160 /* new val must satisfy old val knowledge */ 11161 return range_within(rold, rcur) && 11162 tnum_in(rold->var_off, rcur->var_off); 11163 case PTR_TO_CTX: 11164 case CONST_PTR_TO_MAP: 11165 case PTR_TO_PACKET_END: 11166 case PTR_TO_FLOW_KEYS: 11167 case PTR_TO_SOCKET: 11168 case PTR_TO_SOCK_COMMON: 11169 case PTR_TO_TCP_SOCK: 11170 case PTR_TO_XDP_SOCK: 11171 /* Only valid matches are exact, which memcmp() above 11172 * would have accepted 11173 */ 11174 default: 11175 /* Don't know what's going on, just say it's not safe */ 11176 return false; 11177 } 11178 11179 /* Shouldn't get here; if we do, say it's not safe */ 11180 WARN_ON_ONCE(1); 11181 return false; 11182 } 11183 11184 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, 11185 struct bpf_func_state *cur, struct bpf_id_pair *idmap) 11186 { 11187 int i, spi; 11188 11189 /* walk slots of the explored stack and ignore any additional 11190 * slots in the current stack, since explored(safe) state 11191 * didn't use them 11192 */ 11193 for (i = 0; i < old->allocated_stack; i++) { 11194 spi = i / BPF_REG_SIZE; 11195 11196 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 11197 i += BPF_REG_SIZE - 1; 11198 /* explored state didn't use this */ 11199 continue; 11200 } 11201 11202 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 11203 continue; 11204 11205 /* explored stack has more populated slots than current stack 11206 * and these slots were used 11207 */ 11208 if (i >= cur->allocated_stack) 11209 return false; 11210 11211 /* if old state was safe with misc data in the stack 11212 * it will be safe with zero-initialized stack. 11213 * The opposite is not true 11214 */ 11215 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 11216 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 11217 continue; 11218 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 11219 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 11220 /* Ex: old explored (safe) state has STACK_SPILL in 11221 * this stack slot, but current has STACK_MISC -> 11222 * this verifier states are not equivalent, 11223 * return false to continue verification of this path 11224 */ 11225 return false; 11226 if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1) 11227 continue; 11228 if (!is_spilled_reg(&old->stack[spi])) 11229 continue; 11230 if (!regsafe(env, &old->stack[spi].spilled_ptr, 11231 &cur->stack[spi].spilled_ptr, idmap)) 11232 /* when explored and current stack slot are both storing 11233 * spilled registers, check that stored pointers types 11234 * are the same as well. 11235 * Ex: explored safe path could have stored 11236 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 11237 * but current path has stored: 11238 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 11239 * such verifier states are not equivalent. 11240 * return false to continue verification of this path 11241 */ 11242 return false; 11243 } 11244 return true; 11245 } 11246 11247 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 11248 { 11249 if (old->acquired_refs != cur->acquired_refs) 11250 return false; 11251 return !memcmp(old->refs, cur->refs, 11252 sizeof(*old->refs) * old->acquired_refs); 11253 } 11254 11255 /* compare two verifier states 11256 * 11257 * all states stored in state_list are known to be valid, since 11258 * verifier reached 'bpf_exit' instruction through them 11259 * 11260 * this function is called when verifier exploring different branches of 11261 * execution popped from the state stack. If it sees an old state that has 11262 * more strict register state and more strict stack state then this execution 11263 * branch doesn't need to be explored further, since verifier already 11264 * concluded that more strict state leads to valid finish. 11265 * 11266 * Therefore two states are equivalent if register state is more conservative 11267 * and explored stack state is more conservative than the current one. 11268 * Example: 11269 * explored current 11270 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 11271 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 11272 * 11273 * In other words if current stack state (one being explored) has more 11274 * valid slots than old one that already passed validation, it means 11275 * the verifier can stop exploring and conclude that current state is valid too 11276 * 11277 * Similarly with registers. If explored state has register type as invalid 11278 * whereas register type in current state is meaningful, it means that 11279 * the current state will reach 'bpf_exit' instruction safely 11280 */ 11281 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, 11282 struct bpf_func_state *cur) 11283 { 11284 int i; 11285 11286 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch)); 11287 for (i = 0; i < MAX_BPF_REG; i++) 11288 if (!regsafe(env, &old->regs[i], &cur->regs[i], 11289 env->idmap_scratch)) 11290 return false; 11291 11292 if (!stacksafe(env, old, cur, env->idmap_scratch)) 11293 return false; 11294 11295 if (!refsafe(old, cur)) 11296 return false; 11297 11298 return true; 11299 } 11300 11301 static bool states_equal(struct bpf_verifier_env *env, 11302 struct bpf_verifier_state *old, 11303 struct bpf_verifier_state *cur) 11304 { 11305 int i; 11306 11307 if (old->curframe != cur->curframe) 11308 return false; 11309 11310 /* Verification state from speculative execution simulation 11311 * must never prune a non-speculative execution one. 11312 */ 11313 if (old->speculative && !cur->speculative) 11314 return false; 11315 11316 if (old->active_spin_lock != cur->active_spin_lock) 11317 return false; 11318 11319 /* for states to be equal callsites have to be the same 11320 * and all frame states need to be equivalent 11321 */ 11322 for (i = 0; i <= old->curframe; i++) { 11323 if (old->frame[i]->callsite != cur->frame[i]->callsite) 11324 return false; 11325 if (!func_states_equal(env, old->frame[i], cur->frame[i])) 11326 return false; 11327 } 11328 return true; 11329 } 11330 11331 /* Return 0 if no propagation happened. Return negative error code if error 11332 * happened. Otherwise, return the propagated bit. 11333 */ 11334 static int propagate_liveness_reg(struct bpf_verifier_env *env, 11335 struct bpf_reg_state *reg, 11336 struct bpf_reg_state *parent_reg) 11337 { 11338 u8 parent_flag = parent_reg->live & REG_LIVE_READ; 11339 u8 flag = reg->live & REG_LIVE_READ; 11340 int err; 11341 11342 /* When comes here, read flags of PARENT_REG or REG could be any of 11343 * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need 11344 * of propagation if PARENT_REG has strongest REG_LIVE_READ64. 11345 */ 11346 if (parent_flag == REG_LIVE_READ64 || 11347 /* Or if there is no read flag from REG. */ 11348 !flag || 11349 /* Or if the read flag from REG is the same as PARENT_REG. */ 11350 parent_flag == flag) 11351 return 0; 11352 11353 err = mark_reg_read(env, reg, parent_reg, flag); 11354 if (err) 11355 return err; 11356 11357 return flag; 11358 } 11359 11360 /* A write screens off any subsequent reads; but write marks come from the 11361 * straight-line code between a state and its parent. When we arrive at an 11362 * equivalent state (jump target or such) we didn't arrive by the straight-line 11363 * code, so read marks in the state must propagate to the parent regardless 11364 * of the state's write marks. That's what 'parent == state->parent' comparison 11365 * in mark_reg_read() is for. 11366 */ 11367 static int propagate_liveness(struct bpf_verifier_env *env, 11368 const struct bpf_verifier_state *vstate, 11369 struct bpf_verifier_state *vparent) 11370 { 11371 struct bpf_reg_state *state_reg, *parent_reg; 11372 struct bpf_func_state *state, *parent; 11373 int i, frame, err = 0; 11374 11375 if (vparent->curframe != vstate->curframe) { 11376 WARN(1, "propagate_live: parent frame %d current frame %d\n", 11377 vparent->curframe, vstate->curframe); 11378 return -EFAULT; 11379 } 11380 /* Propagate read liveness of registers... */ 11381 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 11382 for (frame = 0; frame <= vstate->curframe; frame++) { 11383 parent = vparent->frame[frame]; 11384 state = vstate->frame[frame]; 11385 parent_reg = parent->regs; 11386 state_reg = state->regs; 11387 /* We don't need to worry about FP liveness, it's read-only */ 11388 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 11389 err = propagate_liveness_reg(env, &state_reg[i], 11390 &parent_reg[i]); 11391 if (err < 0) 11392 return err; 11393 if (err == REG_LIVE_READ64) 11394 mark_insn_zext(env, &parent_reg[i]); 11395 } 11396 11397 /* Propagate stack slots. */ 11398 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 11399 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 11400 parent_reg = &parent->stack[i].spilled_ptr; 11401 state_reg = &state->stack[i].spilled_ptr; 11402 err = propagate_liveness_reg(env, state_reg, 11403 parent_reg); 11404 if (err < 0) 11405 return err; 11406 } 11407 } 11408 return 0; 11409 } 11410 11411 /* find precise scalars in the previous equivalent state and 11412 * propagate them into the current state 11413 */ 11414 static int propagate_precision(struct bpf_verifier_env *env, 11415 const struct bpf_verifier_state *old) 11416 { 11417 struct bpf_reg_state *state_reg; 11418 struct bpf_func_state *state; 11419 int i, err = 0; 11420 11421 state = old->frame[old->curframe]; 11422 state_reg = state->regs; 11423 for (i = 0; i < BPF_REG_FP; i++, state_reg++) { 11424 if (state_reg->type != SCALAR_VALUE || 11425 !state_reg->precise) 11426 continue; 11427 if (env->log.level & BPF_LOG_LEVEL2) 11428 verbose(env, "propagating r%d\n", i); 11429 err = mark_chain_precision(env, i); 11430 if (err < 0) 11431 return err; 11432 } 11433 11434 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 11435 if (!is_spilled_reg(&state->stack[i])) 11436 continue; 11437 state_reg = &state->stack[i].spilled_ptr; 11438 if (state_reg->type != SCALAR_VALUE || 11439 !state_reg->precise) 11440 continue; 11441 if (env->log.level & BPF_LOG_LEVEL2) 11442 verbose(env, "propagating fp%d\n", 11443 (-i - 1) * BPF_REG_SIZE); 11444 err = mark_chain_precision_stack(env, i); 11445 if (err < 0) 11446 return err; 11447 } 11448 return 0; 11449 } 11450 11451 static bool states_maybe_looping(struct bpf_verifier_state *old, 11452 struct bpf_verifier_state *cur) 11453 { 11454 struct bpf_func_state *fold, *fcur; 11455 int i, fr = cur->curframe; 11456 11457 if (old->curframe != fr) 11458 return false; 11459 11460 fold = old->frame[fr]; 11461 fcur = cur->frame[fr]; 11462 for (i = 0; i < MAX_BPF_REG; i++) 11463 if (memcmp(&fold->regs[i], &fcur->regs[i], 11464 offsetof(struct bpf_reg_state, parent))) 11465 return false; 11466 return true; 11467 } 11468 11469 11470 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 11471 { 11472 struct bpf_verifier_state_list *new_sl; 11473 struct bpf_verifier_state_list *sl, **pprev; 11474 struct bpf_verifier_state *cur = env->cur_state, *new; 11475 int i, j, err, states_cnt = 0; 11476 bool add_new_state = env->test_state_freq ? true : false; 11477 11478 cur->last_insn_idx = env->prev_insn_idx; 11479 if (!env->insn_aux_data[insn_idx].prune_point) 11480 /* this 'insn_idx' instruction wasn't marked, so we will not 11481 * be doing state search here 11482 */ 11483 return 0; 11484 11485 /* bpf progs typically have pruning point every 4 instructions 11486 * http://vger.kernel.org/bpfconf2019.html#session-1 11487 * Do not add new state for future pruning if the verifier hasn't seen 11488 * at least 2 jumps and at least 8 instructions. 11489 * This heuristics helps decrease 'total_states' and 'peak_states' metric. 11490 * In tests that amounts to up to 50% reduction into total verifier 11491 * memory consumption and 20% verifier time speedup. 11492 */ 11493 if (env->jmps_processed - env->prev_jmps_processed >= 2 && 11494 env->insn_processed - env->prev_insn_processed >= 8) 11495 add_new_state = true; 11496 11497 pprev = explored_state(env, insn_idx); 11498 sl = *pprev; 11499 11500 clean_live_states(env, insn_idx, cur); 11501 11502 while (sl) { 11503 states_cnt++; 11504 if (sl->state.insn_idx != insn_idx) 11505 goto next; 11506 11507 if (sl->state.branches) { 11508 struct bpf_func_state *frame = sl->state.frame[sl->state.curframe]; 11509 11510 if (frame->in_async_callback_fn && 11511 frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) { 11512 /* Different async_entry_cnt means that the verifier is 11513 * processing another entry into async callback. 11514 * Seeing the same state is not an indication of infinite 11515 * loop or infinite recursion. 11516 * But finding the same state doesn't mean that it's safe 11517 * to stop processing the current state. The previous state 11518 * hasn't yet reached bpf_exit, since state.branches > 0. 11519 * Checking in_async_callback_fn alone is not enough either. 11520 * Since the verifier still needs to catch infinite loops 11521 * inside async callbacks. 11522 */ 11523 } else if (states_maybe_looping(&sl->state, cur) && 11524 states_equal(env, &sl->state, cur)) { 11525 verbose_linfo(env, insn_idx, "; "); 11526 verbose(env, "infinite loop detected at insn %d\n", insn_idx); 11527 return -EINVAL; 11528 } 11529 /* if the verifier is processing a loop, avoid adding new state 11530 * too often, since different loop iterations have distinct 11531 * states and may not help future pruning. 11532 * This threshold shouldn't be too low to make sure that 11533 * a loop with large bound will be rejected quickly. 11534 * The most abusive loop will be: 11535 * r1 += 1 11536 * if r1 < 1000000 goto pc-2 11537 * 1M insn_procssed limit / 100 == 10k peak states. 11538 * This threshold shouldn't be too high either, since states 11539 * at the end of the loop are likely to be useful in pruning. 11540 */ 11541 if (env->jmps_processed - env->prev_jmps_processed < 20 && 11542 env->insn_processed - env->prev_insn_processed < 100) 11543 add_new_state = false; 11544 goto miss; 11545 } 11546 if (states_equal(env, &sl->state, cur)) { 11547 sl->hit_cnt++; 11548 /* reached equivalent register/stack state, 11549 * prune the search. 11550 * Registers read by the continuation are read by us. 11551 * If we have any write marks in env->cur_state, they 11552 * will prevent corresponding reads in the continuation 11553 * from reaching our parent (an explored_state). Our 11554 * own state will get the read marks recorded, but 11555 * they'll be immediately forgotten as we're pruning 11556 * this state and will pop a new one. 11557 */ 11558 err = propagate_liveness(env, &sl->state, cur); 11559 11560 /* if previous state reached the exit with precision and 11561 * current state is equivalent to it (except precsion marks) 11562 * the precision needs to be propagated back in 11563 * the current state. 11564 */ 11565 err = err ? : push_jmp_history(env, cur); 11566 err = err ? : propagate_precision(env, &sl->state); 11567 if (err) 11568 return err; 11569 return 1; 11570 } 11571 miss: 11572 /* when new state is not going to be added do not increase miss count. 11573 * Otherwise several loop iterations will remove the state 11574 * recorded earlier. The goal of these heuristics is to have 11575 * states from some iterations of the loop (some in the beginning 11576 * and some at the end) to help pruning. 11577 */ 11578 if (add_new_state) 11579 sl->miss_cnt++; 11580 /* heuristic to determine whether this state is beneficial 11581 * to keep checking from state equivalence point of view. 11582 * Higher numbers increase max_states_per_insn and verification time, 11583 * but do not meaningfully decrease insn_processed. 11584 */ 11585 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 11586 /* the state is unlikely to be useful. Remove it to 11587 * speed up verification 11588 */ 11589 *pprev = sl->next; 11590 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 11591 u32 br = sl->state.branches; 11592 11593 WARN_ONCE(br, 11594 "BUG live_done but branches_to_explore %d\n", 11595 br); 11596 free_verifier_state(&sl->state, false); 11597 kfree(sl); 11598 env->peak_states--; 11599 } else { 11600 /* cannot free this state, since parentage chain may 11601 * walk it later. Add it for free_list instead to 11602 * be freed at the end of verification 11603 */ 11604 sl->next = env->free_list; 11605 env->free_list = sl; 11606 } 11607 sl = *pprev; 11608 continue; 11609 } 11610 next: 11611 pprev = &sl->next; 11612 sl = *pprev; 11613 } 11614 11615 if (env->max_states_per_insn < states_cnt) 11616 env->max_states_per_insn = states_cnt; 11617 11618 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 11619 return push_jmp_history(env, cur); 11620 11621 if (!add_new_state) 11622 return push_jmp_history(env, cur); 11623 11624 /* There were no equivalent states, remember the current one. 11625 * Technically the current state is not proven to be safe yet, 11626 * but it will either reach outer most bpf_exit (which means it's safe) 11627 * or it will be rejected. When there are no loops the verifier won't be 11628 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 11629 * again on the way to bpf_exit. 11630 * When looping the sl->state.branches will be > 0 and this state 11631 * will not be considered for equivalence until branches == 0. 11632 */ 11633 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 11634 if (!new_sl) 11635 return -ENOMEM; 11636 env->total_states++; 11637 env->peak_states++; 11638 env->prev_jmps_processed = env->jmps_processed; 11639 env->prev_insn_processed = env->insn_processed; 11640 11641 /* add new state to the head of linked list */ 11642 new = &new_sl->state; 11643 err = copy_verifier_state(new, cur); 11644 if (err) { 11645 free_verifier_state(new, false); 11646 kfree(new_sl); 11647 return err; 11648 } 11649 new->insn_idx = insn_idx; 11650 WARN_ONCE(new->branches != 1, 11651 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); 11652 11653 cur->parent = new; 11654 cur->first_insn_idx = insn_idx; 11655 clear_jmp_history(cur); 11656 new_sl->next = *explored_state(env, insn_idx); 11657 *explored_state(env, insn_idx) = new_sl; 11658 /* connect new state to parentage chain. Current frame needs all 11659 * registers connected. Only r6 - r9 of the callers are alive (pushed 11660 * to the stack implicitly by JITs) so in callers' frames connect just 11661 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 11662 * the state of the call instruction (with WRITTEN set), and r0 comes 11663 * from callee with its full parentage chain, anyway. 11664 */ 11665 /* clear write marks in current state: the writes we did are not writes 11666 * our child did, so they don't screen off its reads from us. 11667 * (There are no read marks in current state, because reads always mark 11668 * their parent and current state never has children yet. Only 11669 * explored_states can get read marks.) 11670 */ 11671 for (j = 0; j <= cur->curframe; j++) { 11672 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 11673 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 11674 for (i = 0; i < BPF_REG_FP; i++) 11675 cur->frame[j]->regs[i].live = REG_LIVE_NONE; 11676 } 11677 11678 /* all stack frames are accessible from callee, clear them all */ 11679 for (j = 0; j <= cur->curframe; j++) { 11680 struct bpf_func_state *frame = cur->frame[j]; 11681 struct bpf_func_state *newframe = new->frame[j]; 11682 11683 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 11684 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 11685 frame->stack[i].spilled_ptr.parent = 11686 &newframe->stack[i].spilled_ptr; 11687 } 11688 } 11689 return 0; 11690 } 11691 11692 /* Return true if it's OK to have the same insn return a different type. */ 11693 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 11694 { 11695 switch (base_type(type)) { 11696 case PTR_TO_CTX: 11697 case PTR_TO_SOCKET: 11698 case PTR_TO_SOCK_COMMON: 11699 case PTR_TO_TCP_SOCK: 11700 case PTR_TO_XDP_SOCK: 11701 case PTR_TO_BTF_ID: 11702 return false; 11703 default: 11704 return true; 11705 } 11706 } 11707 11708 /* If an instruction was previously used with particular pointer types, then we 11709 * need to be careful to avoid cases such as the below, where it may be ok 11710 * for one branch accessing the pointer, but not ok for the other branch: 11711 * 11712 * R1 = sock_ptr 11713 * goto X; 11714 * ... 11715 * R1 = some_other_valid_ptr; 11716 * goto X; 11717 * ... 11718 * R2 = *(u32 *)(R1 + 0); 11719 */ 11720 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 11721 { 11722 return src != prev && (!reg_type_mismatch_ok(src) || 11723 !reg_type_mismatch_ok(prev)); 11724 } 11725 11726 static int do_check(struct bpf_verifier_env *env) 11727 { 11728 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 11729 struct bpf_verifier_state *state = env->cur_state; 11730 struct bpf_insn *insns = env->prog->insnsi; 11731 struct bpf_reg_state *regs; 11732 int insn_cnt = env->prog->len; 11733 bool do_print_state = false; 11734 int prev_insn_idx = -1; 11735 11736 for (;;) { 11737 struct bpf_insn *insn; 11738 u8 class; 11739 int err; 11740 11741 env->prev_insn_idx = prev_insn_idx; 11742 if (env->insn_idx >= insn_cnt) { 11743 verbose(env, "invalid insn idx %d insn_cnt %d\n", 11744 env->insn_idx, insn_cnt); 11745 return -EFAULT; 11746 } 11747 11748 insn = &insns[env->insn_idx]; 11749 class = BPF_CLASS(insn->code); 11750 11751 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 11752 verbose(env, 11753 "BPF program is too large. Processed %d insn\n", 11754 env->insn_processed); 11755 return -E2BIG; 11756 } 11757 11758 err = is_state_visited(env, env->insn_idx); 11759 if (err < 0) 11760 return err; 11761 if (err == 1) { 11762 /* found equivalent state, can prune the search */ 11763 if (env->log.level & BPF_LOG_LEVEL) { 11764 if (do_print_state) 11765 verbose(env, "\nfrom %d to %d%s: safe\n", 11766 env->prev_insn_idx, env->insn_idx, 11767 env->cur_state->speculative ? 11768 " (speculative execution)" : ""); 11769 else 11770 verbose(env, "%d: safe\n", env->insn_idx); 11771 } 11772 goto process_bpf_exit; 11773 } 11774 11775 if (signal_pending(current)) 11776 return -EAGAIN; 11777 11778 if (need_resched()) 11779 cond_resched(); 11780 11781 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { 11782 verbose(env, "\nfrom %d to %d%s:", 11783 env->prev_insn_idx, env->insn_idx, 11784 env->cur_state->speculative ? 11785 " (speculative execution)" : ""); 11786 print_verifier_state(env, state->frame[state->curframe], true); 11787 do_print_state = false; 11788 } 11789 11790 if (env->log.level & BPF_LOG_LEVEL) { 11791 const struct bpf_insn_cbs cbs = { 11792 .cb_call = disasm_kfunc_name, 11793 .cb_print = verbose, 11794 .private_data = env, 11795 }; 11796 11797 if (verifier_state_scratched(env)) 11798 print_insn_state(env, state->frame[state->curframe]); 11799 11800 verbose_linfo(env, env->insn_idx, "; "); 11801 env->prev_log_len = env->log.len_used; 11802 verbose(env, "%d: ", env->insn_idx); 11803 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 11804 env->prev_insn_print_len = env->log.len_used - env->prev_log_len; 11805 env->prev_log_len = env->log.len_used; 11806 } 11807 11808 if (bpf_prog_is_dev_bound(env->prog->aux)) { 11809 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 11810 env->prev_insn_idx); 11811 if (err) 11812 return err; 11813 } 11814 11815 regs = cur_regs(env); 11816 sanitize_mark_insn_seen(env); 11817 prev_insn_idx = env->insn_idx; 11818 11819 if (class == BPF_ALU || class == BPF_ALU64) { 11820 err = check_alu_op(env, insn); 11821 if (err) 11822 return err; 11823 11824 } else if (class == BPF_LDX) { 11825 enum bpf_reg_type *prev_src_type, src_reg_type; 11826 11827 /* check for reserved fields is already done */ 11828 11829 /* check src operand */ 11830 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11831 if (err) 11832 return err; 11833 11834 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 11835 if (err) 11836 return err; 11837 11838 src_reg_type = regs[insn->src_reg].type; 11839 11840 /* check that memory (src_reg + off) is readable, 11841 * the state of dst_reg will be updated by this func 11842 */ 11843 err = check_mem_access(env, env->insn_idx, insn->src_reg, 11844 insn->off, BPF_SIZE(insn->code), 11845 BPF_READ, insn->dst_reg, false); 11846 if (err) 11847 return err; 11848 11849 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 11850 11851 if (*prev_src_type == NOT_INIT) { 11852 /* saw a valid insn 11853 * dst_reg = *(u32 *)(src_reg + off) 11854 * save type to validate intersecting paths 11855 */ 11856 *prev_src_type = src_reg_type; 11857 11858 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 11859 /* ABuser program is trying to use the same insn 11860 * dst_reg = *(u32*) (src_reg + off) 11861 * with different pointer types: 11862 * src_reg == ctx in one branch and 11863 * src_reg == stack|map in some other branch. 11864 * Reject it. 11865 */ 11866 verbose(env, "same insn cannot be used with different pointers\n"); 11867 return -EINVAL; 11868 } 11869 11870 } else if (class == BPF_STX) { 11871 enum bpf_reg_type *prev_dst_type, dst_reg_type; 11872 11873 if (BPF_MODE(insn->code) == BPF_ATOMIC) { 11874 err = check_atomic(env, env->insn_idx, insn); 11875 if (err) 11876 return err; 11877 env->insn_idx++; 11878 continue; 11879 } 11880 11881 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { 11882 verbose(env, "BPF_STX uses reserved fields\n"); 11883 return -EINVAL; 11884 } 11885 11886 /* check src1 operand */ 11887 err = check_reg_arg(env, insn->src_reg, SRC_OP); 11888 if (err) 11889 return err; 11890 /* check src2 operand */ 11891 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11892 if (err) 11893 return err; 11894 11895 dst_reg_type = regs[insn->dst_reg].type; 11896 11897 /* check that memory (dst_reg + off) is writeable */ 11898 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 11899 insn->off, BPF_SIZE(insn->code), 11900 BPF_WRITE, insn->src_reg, false); 11901 if (err) 11902 return err; 11903 11904 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 11905 11906 if (*prev_dst_type == NOT_INIT) { 11907 *prev_dst_type = dst_reg_type; 11908 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 11909 verbose(env, "same insn cannot be used with different pointers\n"); 11910 return -EINVAL; 11911 } 11912 11913 } else if (class == BPF_ST) { 11914 if (BPF_MODE(insn->code) != BPF_MEM || 11915 insn->src_reg != BPF_REG_0) { 11916 verbose(env, "BPF_ST uses reserved fields\n"); 11917 return -EINVAL; 11918 } 11919 /* check src operand */ 11920 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 11921 if (err) 11922 return err; 11923 11924 if (is_ctx_reg(env, insn->dst_reg)) { 11925 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 11926 insn->dst_reg, 11927 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); 11928 return -EACCES; 11929 } 11930 11931 /* check that memory (dst_reg + off) is writeable */ 11932 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 11933 insn->off, BPF_SIZE(insn->code), 11934 BPF_WRITE, -1, false); 11935 if (err) 11936 return err; 11937 11938 } else if (class == BPF_JMP || class == BPF_JMP32) { 11939 u8 opcode = BPF_OP(insn->code); 11940 11941 env->jmps_processed++; 11942 if (opcode == BPF_CALL) { 11943 if (BPF_SRC(insn->code) != BPF_K || 11944 (insn->src_reg != BPF_PSEUDO_KFUNC_CALL 11945 && insn->off != 0) || 11946 (insn->src_reg != BPF_REG_0 && 11947 insn->src_reg != BPF_PSEUDO_CALL && 11948 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) || 11949 insn->dst_reg != BPF_REG_0 || 11950 class == BPF_JMP32) { 11951 verbose(env, "BPF_CALL uses reserved fields\n"); 11952 return -EINVAL; 11953 } 11954 11955 if (env->cur_state->active_spin_lock && 11956 (insn->src_reg == BPF_PSEUDO_CALL || 11957 insn->imm != BPF_FUNC_spin_unlock)) { 11958 verbose(env, "function calls are not allowed while holding a lock\n"); 11959 return -EINVAL; 11960 } 11961 if (insn->src_reg == BPF_PSEUDO_CALL) 11962 err = check_func_call(env, insn, &env->insn_idx); 11963 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) 11964 err = check_kfunc_call(env, insn, &env->insn_idx); 11965 else 11966 err = check_helper_call(env, insn, &env->insn_idx); 11967 if (err) 11968 return err; 11969 } else if (opcode == BPF_JA) { 11970 if (BPF_SRC(insn->code) != BPF_K || 11971 insn->imm != 0 || 11972 insn->src_reg != BPF_REG_0 || 11973 insn->dst_reg != BPF_REG_0 || 11974 class == BPF_JMP32) { 11975 verbose(env, "BPF_JA uses reserved fields\n"); 11976 return -EINVAL; 11977 } 11978 11979 env->insn_idx += insn->off + 1; 11980 continue; 11981 11982 } else if (opcode == BPF_EXIT) { 11983 if (BPF_SRC(insn->code) != BPF_K || 11984 insn->imm != 0 || 11985 insn->src_reg != BPF_REG_0 || 11986 insn->dst_reg != BPF_REG_0 || 11987 class == BPF_JMP32) { 11988 verbose(env, "BPF_EXIT uses reserved fields\n"); 11989 return -EINVAL; 11990 } 11991 11992 if (env->cur_state->active_spin_lock) { 11993 verbose(env, "bpf_spin_unlock is missing\n"); 11994 return -EINVAL; 11995 } 11996 11997 if (state->curframe) { 11998 /* exit from nested function */ 11999 err = prepare_func_exit(env, &env->insn_idx); 12000 if (err) 12001 return err; 12002 do_print_state = true; 12003 continue; 12004 } 12005 12006 err = check_reference_leak(env); 12007 if (err) 12008 return err; 12009 12010 err = check_return_code(env); 12011 if (err) 12012 return err; 12013 process_bpf_exit: 12014 mark_verifier_state_scratched(env); 12015 update_branch_counts(env, env->cur_state); 12016 err = pop_stack(env, &prev_insn_idx, 12017 &env->insn_idx, pop_log); 12018 if (err < 0) { 12019 if (err != -ENOENT) 12020 return err; 12021 break; 12022 } else { 12023 do_print_state = true; 12024 continue; 12025 } 12026 } else { 12027 err = check_cond_jmp_op(env, insn, &env->insn_idx); 12028 if (err) 12029 return err; 12030 } 12031 } else if (class == BPF_LD) { 12032 u8 mode = BPF_MODE(insn->code); 12033 12034 if (mode == BPF_ABS || mode == BPF_IND) { 12035 err = check_ld_abs(env, insn); 12036 if (err) 12037 return err; 12038 12039 } else if (mode == BPF_IMM) { 12040 err = check_ld_imm(env, insn); 12041 if (err) 12042 return err; 12043 12044 env->insn_idx++; 12045 sanitize_mark_insn_seen(env); 12046 } else { 12047 verbose(env, "invalid BPF_LD mode\n"); 12048 return -EINVAL; 12049 } 12050 } else { 12051 verbose(env, "unknown insn class %d\n", class); 12052 return -EINVAL; 12053 } 12054 12055 env->insn_idx++; 12056 } 12057 12058 return 0; 12059 } 12060 12061 static int find_btf_percpu_datasec(struct btf *btf) 12062 { 12063 const struct btf_type *t; 12064 const char *tname; 12065 int i, n; 12066 12067 /* 12068 * Both vmlinux and module each have their own ".data..percpu" 12069 * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF 12070 * types to look at only module's own BTF types. 12071 */ 12072 n = btf_nr_types(btf); 12073 if (btf_is_module(btf)) 12074 i = btf_nr_types(btf_vmlinux); 12075 else 12076 i = 1; 12077 12078 for(; i < n; i++) { 12079 t = btf_type_by_id(btf, i); 12080 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) 12081 continue; 12082 12083 tname = btf_name_by_offset(btf, t->name_off); 12084 if (!strcmp(tname, ".data..percpu")) 12085 return i; 12086 } 12087 12088 return -ENOENT; 12089 } 12090 12091 /* replace pseudo btf_id with kernel symbol address */ 12092 static int check_pseudo_btf_id(struct bpf_verifier_env *env, 12093 struct bpf_insn *insn, 12094 struct bpf_insn_aux_data *aux) 12095 { 12096 const struct btf_var_secinfo *vsi; 12097 const struct btf_type *datasec; 12098 struct btf_mod_pair *btf_mod; 12099 const struct btf_type *t; 12100 const char *sym_name; 12101 bool percpu = false; 12102 u32 type, id = insn->imm; 12103 struct btf *btf; 12104 s32 datasec_id; 12105 u64 addr; 12106 int i, btf_fd, err; 12107 12108 btf_fd = insn[1].imm; 12109 if (btf_fd) { 12110 btf = btf_get_by_fd(btf_fd); 12111 if (IS_ERR(btf)) { 12112 verbose(env, "invalid module BTF object FD specified.\n"); 12113 return -EINVAL; 12114 } 12115 } else { 12116 if (!btf_vmlinux) { 12117 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); 12118 return -EINVAL; 12119 } 12120 btf = btf_vmlinux; 12121 btf_get(btf); 12122 } 12123 12124 t = btf_type_by_id(btf, id); 12125 if (!t) { 12126 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); 12127 err = -ENOENT; 12128 goto err_put; 12129 } 12130 12131 if (!btf_type_is_var(t)) { 12132 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); 12133 err = -EINVAL; 12134 goto err_put; 12135 } 12136 12137 sym_name = btf_name_by_offset(btf, t->name_off); 12138 addr = kallsyms_lookup_name(sym_name); 12139 if (!addr) { 12140 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", 12141 sym_name); 12142 err = -ENOENT; 12143 goto err_put; 12144 } 12145 12146 datasec_id = find_btf_percpu_datasec(btf); 12147 if (datasec_id > 0) { 12148 datasec = btf_type_by_id(btf, datasec_id); 12149 for_each_vsi(i, datasec, vsi) { 12150 if (vsi->type == id) { 12151 percpu = true; 12152 break; 12153 } 12154 } 12155 } 12156 12157 insn[0].imm = (u32)addr; 12158 insn[1].imm = addr >> 32; 12159 12160 type = t->type; 12161 t = btf_type_skip_modifiers(btf, type, NULL); 12162 if (percpu) { 12163 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; 12164 aux->btf_var.btf = btf; 12165 aux->btf_var.btf_id = type; 12166 } else if (!btf_type_is_struct(t)) { 12167 const struct btf_type *ret; 12168 const char *tname; 12169 u32 tsize; 12170 12171 /* resolve the type size of ksym. */ 12172 ret = btf_resolve_size(btf, t, &tsize); 12173 if (IS_ERR(ret)) { 12174 tname = btf_name_by_offset(btf, t->name_off); 12175 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", 12176 tname, PTR_ERR(ret)); 12177 err = -EINVAL; 12178 goto err_put; 12179 } 12180 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; 12181 aux->btf_var.mem_size = tsize; 12182 } else { 12183 aux->btf_var.reg_type = PTR_TO_BTF_ID; 12184 aux->btf_var.btf = btf; 12185 aux->btf_var.btf_id = type; 12186 } 12187 12188 /* check whether we recorded this BTF (and maybe module) already */ 12189 for (i = 0; i < env->used_btf_cnt; i++) { 12190 if (env->used_btfs[i].btf == btf) { 12191 btf_put(btf); 12192 return 0; 12193 } 12194 } 12195 12196 if (env->used_btf_cnt >= MAX_USED_BTFS) { 12197 err = -E2BIG; 12198 goto err_put; 12199 } 12200 12201 btf_mod = &env->used_btfs[env->used_btf_cnt]; 12202 btf_mod->btf = btf; 12203 btf_mod->module = NULL; 12204 12205 /* if we reference variables from kernel module, bump its refcount */ 12206 if (btf_is_module(btf)) { 12207 btf_mod->module = btf_try_get_module(btf); 12208 if (!btf_mod->module) { 12209 err = -ENXIO; 12210 goto err_put; 12211 } 12212 } 12213 12214 env->used_btf_cnt++; 12215 12216 return 0; 12217 err_put: 12218 btf_put(btf); 12219 return err; 12220 } 12221 12222 static int check_map_prealloc(struct bpf_map *map) 12223 { 12224 return (map->map_type != BPF_MAP_TYPE_HASH && 12225 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 12226 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || 12227 !(map->map_flags & BPF_F_NO_PREALLOC); 12228 } 12229 12230 static bool is_tracing_prog_type(enum bpf_prog_type type) 12231 { 12232 switch (type) { 12233 case BPF_PROG_TYPE_KPROBE: 12234 case BPF_PROG_TYPE_TRACEPOINT: 12235 case BPF_PROG_TYPE_PERF_EVENT: 12236 case BPF_PROG_TYPE_RAW_TRACEPOINT: 12237 return true; 12238 default: 12239 return false; 12240 } 12241 } 12242 12243 static bool is_preallocated_map(struct bpf_map *map) 12244 { 12245 if (!check_map_prealloc(map)) 12246 return false; 12247 if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) 12248 return false; 12249 return true; 12250 } 12251 12252 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 12253 struct bpf_map *map, 12254 struct bpf_prog *prog) 12255 12256 { 12257 enum bpf_prog_type prog_type = resolve_prog_type(prog); 12258 /* 12259 * Validate that trace type programs use preallocated hash maps. 12260 * 12261 * For programs attached to PERF events this is mandatory as the 12262 * perf NMI can hit any arbitrary code sequence. 12263 * 12264 * All other trace types using preallocated hash maps are unsafe as 12265 * well because tracepoint or kprobes can be inside locked regions 12266 * of the memory allocator or at a place where a recursion into the 12267 * memory allocator would see inconsistent state. 12268 * 12269 * On RT enabled kernels run-time allocation of all trace type 12270 * programs is strictly prohibited due to lock type constraints. On 12271 * !RT kernels it is allowed for backwards compatibility reasons for 12272 * now, but warnings are emitted so developers are made aware of 12273 * the unsafety and can fix their programs before this is enforced. 12274 */ 12275 if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { 12276 if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { 12277 verbose(env, "perf_event programs can only use preallocated hash map\n"); 12278 return -EINVAL; 12279 } 12280 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 12281 verbose(env, "trace type programs can only use preallocated hash map\n"); 12282 return -EINVAL; 12283 } 12284 WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); 12285 verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); 12286 } 12287 12288 if (map_value_has_spin_lock(map)) { 12289 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { 12290 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); 12291 return -EINVAL; 12292 } 12293 12294 if (is_tracing_prog_type(prog_type)) { 12295 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 12296 return -EINVAL; 12297 } 12298 12299 if (prog->aux->sleepable) { 12300 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); 12301 return -EINVAL; 12302 } 12303 } 12304 12305 if (map_value_has_timer(map)) { 12306 if (is_tracing_prog_type(prog_type)) { 12307 verbose(env, "tracing progs cannot use bpf_timer yet\n"); 12308 return -EINVAL; 12309 } 12310 } 12311 12312 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 12313 !bpf_offload_prog_map_match(prog, map)) { 12314 verbose(env, "offload device mismatch between prog and map\n"); 12315 return -EINVAL; 12316 } 12317 12318 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { 12319 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); 12320 return -EINVAL; 12321 } 12322 12323 if (prog->aux->sleepable) 12324 switch (map->map_type) { 12325 case BPF_MAP_TYPE_HASH: 12326 case BPF_MAP_TYPE_LRU_HASH: 12327 case BPF_MAP_TYPE_ARRAY: 12328 case BPF_MAP_TYPE_PERCPU_HASH: 12329 case BPF_MAP_TYPE_PERCPU_ARRAY: 12330 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 12331 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 12332 case BPF_MAP_TYPE_HASH_OF_MAPS: 12333 if (!is_preallocated_map(map)) { 12334 verbose(env, 12335 "Sleepable programs can only use preallocated maps\n"); 12336 return -EINVAL; 12337 } 12338 break; 12339 case BPF_MAP_TYPE_RINGBUF: 12340 case BPF_MAP_TYPE_INODE_STORAGE: 12341 case BPF_MAP_TYPE_SK_STORAGE: 12342 case BPF_MAP_TYPE_TASK_STORAGE: 12343 break; 12344 default: 12345 verbose(env, 12346 "Sleepable programs can only use array, hash, and ringbuf maps\n"); 12347 return -EINVAL; 12348 } 12349 12350 return 0; 12351 } 12352 12353 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 12354 { 12355 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 12356 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 12357 } 12358 12359 /* find and rewrite pseudo imm in ld_imm64 instructions: 12360 * 12361 * 1. if it accesses map FD, replace it with actual map pointer. 12362 * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. 12363 * 12364 * NOTE: btf_vmlinux is required for converting pseudo btf_id. 12365 */ 12366 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) 12367 { 12368 struct bpf_insn *insn = env->prog->insnsi; 12369 int insn_cnt = env->prog->len; 12370 int i, j, err; 12371 12372 err = bpf_prog_calc_tag(env->prog); 12373 if (err) 12374 return err; 12375 12376 for (i = 0; i < insn_cnt; i++, insn++) { 12377 if (BPF_CLASS(insn->code) == BPF_LDX && 12378 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 12379 verbose(env, "BPF_LDX uses reserved fields\n"); 12380 return -EINVAL; 12381 } 12382 12383 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 12384 struct bpf_insn_aux_data *aux; 12385 struct bpf_map *map; 12386 struct fd f; 12387 u64 addr; 12388 u32 fd; 12389 12390 if (i == insn_cnt - 1 || insn[1].code != 0 || 12391 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 12392 insn[1].off != 0) { 12393 verbose(env, "invalid bpf_ld_imm64 insn\n"); 12394 return -EINVAL; 12395 } 12396 12397 if (insn[0].src_reg == 0) 12398 /* valid generic load 64-bit imm */ 12399 goto next_insn; 12400 12401 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { 12402 aux = &env->insn_aux_data[i]; 12403 err = check_pseudo_btf_id(env, insn, aux); 12404 if (err) 12405 return err; 12406 goto next_insn; 12407 } 12408 12409 if (insn[0].src_reg == BPF_PSEUDO_FUNC) { 12410 aux = &env->insn_aux_data[i]; 12411 aux->ptr_type = PTR_TO_FUNC; 12412 goto next_insn; 12413 } 12414 12415 /* In final convert_pseudo_ld_imm64() step, this is 12416 * converted into regular 64-bit imm load insn. 12417 */ 12418 switch (insn[0].src_reg) { 12419 case BPF_PSEUDO_MAP_VALUE: 12420 case BPF_PSEUDO_MAP_IDX_VALUE: 12421 break; 12422 case BPF_PSEUDO_MAP_FD: 12423 case BPF_PSEUDO_MAP_IDX: 12424 if (insn[1].imm == 0) 12425 break; 12426 fallthrough; 12427 default: 12428 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); 12429 return -EINVAL; 12430 } 12431 12432 switch (insn[0].src_reg) { 12433 case BPF_PSEUDO_MAP_IDX_VALUE: 12434 case BPF_PSEUDO_MAP_IDX: 12435 if (bpfptr_is_null(env->fd_array)) { 12436 verbose(env, "fd_idx without fd_array is invalid\n"); 12437 return -EPROTO; 12438 } 12439 if (copy_from_bpfptr_offset(&fd, env->fd_array, 12440 insn[0].imm * sizeof(fd), 12441 sizeof(fd))) 12442 return -EFAULT; 12443 break; 12444 default: 12445 fd = insn[0].imm; 12446 break; 12447 } 12448 12449 f = fdget(fd); 12450 map = __bpf_map_get(f); 12451 if (IS_ERR(map)) { 12452 verbose(env, "fd %d is not pointing to valid bpf_map\n", 12453 insn[0].imm); 12454 return PTR_ERR(map); 12455 } 12456 12457 err = check_map_prog_compatibility(env, map, env->prog); 12458 if (err) { 12459 fdput(f); 12460 return err; 12461 } 12462 12463 aux = &env->insn_aux_data[i]; 12464 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD || 12465 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) { 12466 addr = (unsigned long)map; 12467 } else { 12468 u32 off = insn[1].imm; 12469 12470 if (off >= BPF_MAX_VAR_OFF) { 12471 verbose(env, "direct value offset of %u is not allowed\n", off); 12472 fdput(f); 12473 return -EINVAL; 12474 } 12475 12476 if (!map->ops->map_direct_value_addr) { 12477 verbose(env, "no direct value access support for this map type\n"); 12478 fdput(f); 12479 return -EINVAL; 12480 } 12481 12482 err = map->ops->map_direct_value_addr(map, &addr, off); 12483 if (err) { 12484 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 12485 map->value_size, off); 12486 fdput(f); 12487 return err; 12488 } 12489 12490 aux->map_off = off; 12491 addr += off; 12492 } 12493 12494 insn[0].imm = (u32)addr; 12495 insn[1].imm = addr >> 32; 12496 12497 /* check whether we recorded this map already */ 12498 for (j = 0; j < env->used_map_cnt; j++) { 12499 if (env->used_maps[j] == map) { 12500 aux->map_index = j; 12501 fdput(f); 12502 goto next_insn; 12503 } 12504 } 12505 12506 if (env->used_map_cnt >= MAX_USED_MAPS) { 12507 fdput(f); 12508 return -E2BIG; 12509 } 12510 12511 /* hold the map. If the program is rejected by verifier, 12512 * the map will be released by release_maps() or it 12513 * will be used by the valid program until it's unloaded 12514 * and all maps are released in free_used_maps() 12515 */ 12516 bpf_map_inc(map); 12517 12518 aux->map_index = env->used_map_cnt; 12519 env->used_maps[env->used_map_cnt++] = map; 12520 12521 if (bpf_map_is_cgroup_storage(map) && 12522 bpf_cgroup_storage_assign(env->prog->aux, map)) { 12523 verbose(env, "only one cgroup storage of each type is allowed\n"); 12524 fdput(f); 12525 return -EBUSY; 12526 } 12527 12528 fdput(f); 12529 next_insn: 12530 insn++; 12531 i++; 12532 continue; 12533 } 12534 12535 /* Basic sanity check before we invest more work here. */ 12536 if (!bpf_opcode_in_insntable(insn->code)) { 12537 verbose(env, "unknown opcode %02x\n", insn->code); 12538 return -EINVAL; 12539 } 12540 } 12541 12542 /* now all pseudo BPF_LD_IMM64 instructions load valid 12543 * 'struct bpf_map *' into a register instead of user map_fd. 12544 * These pointers will be used later by verifier to validate map access. 12545 */ 12546 return 0; 12547 } 12548 12549 /* drop refcnt of maps used by the rejected program */ 12550 static void release_maps(struct bpf_verifier_env *env) 12551 { 12552 __bpf_free_used_maps(env->prog->aux, env->used_maps, 12553 env->used_map_cnt); 12554 } 12555 12556 /* drop refcnt of maps used by the rejected program */ 12557 static void release_btfs(struct bpf_verifier_env *env) 12558 { 12559 __bpf_free_used_btfs(env->prog->aux, env->used_btfs, 12560 env->used_btf_cnt); 12561 } 12562 12563 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 12564 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 12565 { 12566 struct bpf_insn *insn = env->prog->insnsi; 12567 int insn_cnt = env->prog->len; 12568 int i; 12569 12570 for (i = 0; i < insn_cnt; i++, insn++) { 12571 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) 12572 continue; 12573 if (insn->src_reg == BPF_PSEUDO_FUNC) 12574 continue; 12575 insn->src_reg = 0; 12576 } 12577 } 12578 12579 /* single env->prog->insni[off] instruction was replaced with the range 12580 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 12581 * [0, off) and [off, end) to new locations, so the patched range stays zero 12582 */ 12583 static void adjust_insn_aux_data(struct bpf_verifier_env *env, 12584 struct bpf_insn_aux_data *new_data, 12585 struct bpf_prog *new_prog, u32 off, u32 cnt) 12586 { 12587 struct bpf_insn_aux_data *old_data = env->insn_aux_data; 12588 struct bpf_insn *insn = new_prog->insnsi; 12589 u32 old_seen = old_data[off].seen; 12590 u32 prog_len; 12591 int i; 12592 12593 /* aux info at OFF always needs adjustment, no matter fast path 12594 * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the 12595 * original insn at old prog. 12596 */ 12597 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); 12598 12599 if (cnt == 1) 12600 return; 12601 prog_len = new_prog->len; 12602 12603 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 12604 memcpy(new_data + off + cnt - 1, old_data + off, 12605 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 12606 for (i = off; i < off + cnt - 1; i++) { 12607 /* Expand insni[off]'s seen count to the patched range. */ 12608 new_data[i].seen = old_seen; 12609 new_data[i].zext_dst = insn_has_def32(env, insn + i); 12610 } 12611 env->insn_aux_data = new_data; 12612 vfree(old_data); 12613 } 12614 12615 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 12616 { 12617 int i; 12618 12619 if (len == 1) 12620 return; 12621 /* NOTE: fake 'exit' subprog should be updated as well. */ 12622 for (i = 0; i <= env->subprog_cnt; i++) { 12623 if (env->subprog_info[i].start <= off) 12624 continue; 12625 env->subprog_info[i].start += len - 1; 12626 } 12627 } 12628 12629 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) 12630 { 12631 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; 12632 int i, sz = prog->aux->size_poke_tab; 12633 struct bpf_jit_poke_descriptor *desc; 12634 12635 for (i = 0; i < sz; i++) { 12636 desc = &tab[i]; 12637 if (desc->insn_idx <= off) 12638 continue; 12639 desc->insn_idx += len - 1; 12640 } 12641 } 12642 12643 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 12644 const struct bpf_insn *patch, u32 len) 12645 { 12646 struct bpf_prog *new_prog; 12647 struct bpf_insn_aux_data *new_data = NULL; 12648 12649 if (len > 1) { 12650 new_data = vzalloc(array_size(env->prog->len + len - 1, 12651 sizeof(struct bpf_insn_aux_data))); 12652 if (!new_data) 12653 return NULL; 12654 } 12655 12656 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 12657 if (IS_ERR(new_prog)) { 12658 if (PTR_ERR(new_prog) == -ERANGE) 12659 verbose(env, 12660 "insn %d cannot be patched due to 16-bit range\n", 12661 env->insn_aux_data[off].orig_idx); 12662 vfree(new_data); 12663 return NULL; 12664 } 12665 adjust_insn_aux_data(env, new_data, new_prog, off, len); 12666 adjust_subprog_starts(env, off, len); 12667 adjust_poke_descs(new_prog, off, len); 12668 return new_prog; 12669 } 12670 12671 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 12672 u32 off, u32 cnt) 12673 { 12674 int i, j; 12675 12676 /* find first prog starting at or after off (first to remove) */ 12677 for (i = 0; i < env->subprog_cnt; i++) 12678 if (env->subprog_info[i].start >= off) 12679 break; 12680 /* find first prog starting at or after off + cnt (first to stay) */ 12681 for (j = i; j < env->subprog_cnt; j++) 12682 if (env->subprog_info[j].start >= off + cnt) 12683 break; 12684 /* if j doesn't start exactly at off + cnt, we are just removing 12685 * the front of previous prog 12686 */ 12687 if (env->subprog_info[j].start != off + cnt) 12688 j--; 12689 12690 if (j > i) { 12691 struct bpf_prog_aux *aux = env->prog->aux; 12692 int move; 12693 12694 /* move fake 'exit' subprog as well */ 12695 move = env->subprog_cnt + 1 - j; 12696 12697 memmove(env->subprog_info + i, 12698 env->subprog_info + j, 12699 sizeof(*env->subprog_info) * move); 12700 env->subprog_cnt -= j - i; 12701 12702 /* remove func_info */ 12703 if (aux->func_info) { 12704 move = aux->func_info_cnt - j; 12705 12706 memmove(aux->func_info + i, 12707 aux->func_info + j, 12708 sizeof(*aux->func_info) * move); 12709 aux->func_info_cnt -= j - i; 12710 /* func_info->insn_off is set after all code rewrites, 12711 * in adjust_btf_func() - no need to adjust 12712 */ 12713 } 12714 } else { 12715 /* convert i from "first prog to remove" to "first to adjust" */ 12716 if (env->subprog_info[i].start == off) 12717 i++; 12718 } 12719 12720 /* update fake 'exit' subprog as well */ 12721 for (; i <= env->subprog_cnt; i++) 12722 env->subprog_info[i].start -= cnt; 12723 12724 return 0; 12725 } 12726 12727 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 12728 u32 cnt) 12729 { 12730 struct bpf_prog *prog = env->prog; 12731 u32 i, l_off, l_cnt, nr_linfo; 12732 struct bpf_line_info *linfo; 12733 12734 nr_linfo = prog->aux->nr_linfo; 12735 if (!nr_linfo) 12736 return 0; 12737 12738 linfo = prog->aux->linfo; 12739 12740 /* find first line info to remove, count lines to be removed */ 12741 for (i = 0; i < nr_linfo; i++) 12742 if (linfo[i].insn_off >= off) 12743 break; 12744 12745 l_off = i; 12746 l_cnt = 0; 12747 for (; i < nr_linfo; i++) 12748 if (linfo[i].insn_off < off + cnt) 12749 l_cnt++; 12750 else 12751 break; 12752 12753 /* First live insn doesn't match first live linfo, it needs to "inherit" 12754 * last removed linfo. prog is already modified, so prog->len == off 12755 * means no live instructions after (tail of the program was removed). 12756 */ 12757 if (prog->len != off && l_cnt && 12758 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 12759 l_cnt--; 12760 linfo[--i].insn_off = off + cnt; 12761 } 12762 12763 /* remove the line info which refer to the removed instructions */ 12764 if (l_cnt) { 12765 memmove(linfo + l_off, linfo + i, 12766 sizeof(*linfo) * (nr_linfo - i)); 12767 12768 prog->aux->nr_linfo -= l_cnt; 12769 nr_linfo = prog->aux->nr_linfo; 12770 } 12771 12772 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 12773 for (i = l_off; i < nr_linfo; i++) 12774 linfo[i].insn_off -= cnt; 12775 12776 /* fix up all subprogs (incl. 'exit') which start >= off */ 12777 for (i = 0; i <= env->subprog_cnt; i++) 12778 if (env->subprog_info[i].linfo_idx > l_off) { 12779 /* program may have started in the removed region but 12780 * may not be fully removed 12781 */ 12782 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 12783 env->subprog_info[i].linfo_idx -= l_cnt; 12784 else 12785 env->subprog_info[i].linfo_idx = l_off; 12786 } 12787 12788 return 0; 12789 } 12790 12791 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 12792 { 12793 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 12794 unsigned int orig_prog_len = env->prog->len; 12795 int err; 12796 12797 if (bpf_prog_is_dev_bound(env->prog->aux)) 12798 bpf_prog_offload_remove_insns(env, off, cnt); 12799 12800 err = bpf_remove_insns(env->prog, off, cnt); 12801 if (err) 12802 return err; 12803 12804 err = adjust_subprog_starts_after_remove(env, off, cnt); 12805 if (err) 12806 return err; 12807 12808 err = bpf_adj_linfo_after_remove(env, off, cnt); 12809 if (err) 12810 return err; 12811 12812 memmove(aux_data + off, aux_data + off + cnt, 12813 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 12814 12815 return 0; 12816 } 12817 12818 /* The verifier does more data flow analysis than llvm and will not 12819 * explore branches that are dead at run time. Malicious programs can 12820 * have dead code too. Therefore replace all dead at-run-time code 12821 * with 'ja -1'. 12822 * 12823 * Just nops are not optimal, e.g. if they would sit at the end of the 12824 * program and through another bug we would manage to jump there, then 12825 * we'd execute beyond program memory otherwise. Returning exception 12826 * code also wouldn't work since we can have subprogs where the dead 12827 * code could be located. 12828 */ 12829 static void sanitize_dead_code(struct bpf_verifier_env *env) 12830 { 12831 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 12832 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 12833 struct bpf_insn *insn = env->prog->insnsi; 12834 const int insn_cnt = env->prog->len; 12835 int i; 12836 12837 for (i = 0; i < insn_cnt; i++) { 12838 if (aux_data[i].seen) 12839 continue; 12840 memcpy(insn + i, &trap, sizeof(trap)); 12841 aux_data[i].zext_dst = false; 12842 } 12843 } 12844 12845 static bool insn_is_cond_jump(u8 code) 12846 { 12847 u8 op; 12848 12849 if (BPF_CLASS(code) == BPF_JMP32) 12850 return true; 12851 12852 if (BPF_CLASS(code) != BPF_JMP) 12853 return false; 12854 12855 op = BPF_OP(code); 12856 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 12857 } 12858 12859 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 12860 { 12861 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 12862 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 12863 struct bpf_insn *insn = env->prog->insnsi; 12864 const int insn_cnt = env->prog->len; 12865 int i; 12866 12867 for (i = 0; i < insn_cnt; i++, insn++) { 12868 if (!insn_is_cond_jump(insn->code)) 12869 continue; 12870 12871 if (!aux_data[i + 1].seen) 12872 ja.off = insn->off; 12873 else if (!aux_data[i + 1 + insn->off].seen) 12874 ja.off = 0; 12875 else 12876 continue; 12877 12878 if (bpf_prog_is_dev_bound(env->prog->aux)) 12879 bpf_prog_offload_replace_insn(env, i, &ja); 12880 12881 memcpy(insn, &ja, sizeof(ja)); 12882 } 12883 } 12884 12885 static int opt_remove_dead_code(struct bpf_verifier_env *env) 12886 { 12887 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 12888 int insn_cnt = env->prog->len; 12889 int i, err; 12890 12891 for (i = 0; i < insn_cnt; i++) { 12892 int j; 12893 12894 j = 0; 12895 while (i + j < insn_cnt && !aux_data[i + j].seen) 12896 j++; 12897 if (!j) 12898 continue; 12899 12900 err = verifier_remove_insns(env, i, j); 12901 if (err) 12902 return err; 12903 insn_cnt = env->prog->len; 12904 } 12905 12906 return 0; 12907 } 12908 12909 static int opt_remove_nops(struct bpf_verifier_env *env) 12910 { 12911 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 12912 struct bpf_insn *insn = env->prog->insnsi; 12913 int insn_cnt = env->prog->len; 12914 int i, err; 12915 12916 for (i = 0; i < insn_cnt; i++) { 12917 if (memcmp(&insn[i], &ja, sizeof(ja))) 12918 continue; 12919 12920 err = verifier_remove_insns(env, i, 1); 12921 if (err) 12922 return err; 12923 insn_cnt--; 12924 i--; 12925 } 12926 12927 return 0; 12928 } 12929 12930 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, 12931 const union bpf_attr *attr) 12932 { 12933 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; 12934 struct bpf_insn_aux_data *aux = env->insn_aux_data; 12935 int i, patch_len, delta = 0, len = env->prog->len; 12936 struct bpf_insn *insns = env->prog->insnsi; 12937 struct bpf_prog *new_prog; 12938 bool rnd_hi32; 12939 12940 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; 12941 zext_patch[1] = BPF_ZEXT_REG(0); 12942 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); 12943 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); 12944 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); 12945 for (i = 0; i < len; i++) { 12946 int adj_idx = i + delta; 12947 struct bpf_insn insn; 12948 int load_reg; 12949 12950 insn = insns[adj_idx]; 12951 load_reg = insn_def_regno(&insn); 12952 if (!aux[adj_idx].zext_dst) { 12953 u8 code, class; 12954 u32 imm_rnd; 12955 12956 if (!rnd_hi32) 12957 continue; 12958 12959 code = insn.code; 12960 class = BPF_CLASS(code); 12961 if (load_reg == -1) 12962 continue; 12963 12964 /* NOTE: arg "reg" (the fourth one) is only used for 12965 * BPF_STX + SRC_OP, so it is safe to pass NULL 12966 * here. 12967 */ 12968 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { 12969 if (class == BPF_LD && 12970 BPF_MODE(code) == BPF_IMM) 12971 i++; 12972 continue; 12973 } 12974 12975 /* ctx load could be transformed into wider load. */ 12976 if (class == BPF_LDX && 12977 aux[adj_idx].ptr_type == PTR_TO_CTX) 12978 continue; 12979 12980 imm_rnd = get_random_int(); 12981 rnd_hi32_patch[0] = insn; 12982 rnd_hi32_patch[1].imm = imm_rnd; 12983 rnd_hi32_patch[3].dst_reg = load_reg; 12984 patch = rnd_hi32_patch; 12985 patch_len = 4; 12986 goto apply_patch_buffer; 12987 } 12988 12989 /* Add in an zero-extend instruction if a) the JIT has requested 12990 * it or b) it's a CMPXCHG. 12991 * 12992 * The latter is because: BPF_CMPXCHG always loads a value into 12993 * R0, therefore always zero-extends. However some archs' 12994 * equivalent instruction only does this load when the 12995 * comparison is successful. This detail of CMPXCHG is 12996 * orthogonal to the general zero-extension behaviour of the 12997 * CPU, so it's treated independently of bpf_jit_needs_zext. 12998 */ 12999 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) 13000 continue; 13001 13002 if (WARN_ON(load_reg == -1)) { 13003 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); 13004 return -EFAULT; 13005 } 13006 13007 zext_patch[0] = insn; 13008 zext_patch[1].dst_reg = load_reg; 13009 zext_patch[1].src_reg = load_reg; 13010 patch = zext_patch; 13011 patch_len = 2; 13012 apply_patch_buffer: 13013 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); 13014 if (!new_prog) 13015 return -ENOMEM; 13016 env->prog = new_prog; 13017 insns = new_prog->insnsi; 13018 aux = env->insn_aux_data; 13019 delta += patch_len - 1; 13020 } 13021 13022 return 0; 13023 } 13024 13025 /* convert load instructions that access fields of a context type into a 13026 * sequence of instructions that access fields of the underlying structure: 13027 * struct __sk_buff -> struct sk_buff 13028 * struct bpf_sock_ops -> struct sock 13029 */ 13030 static int convert_ctx_accesses(struct bpf_verifier_env *env) 13031 { 13032 const struct bpf_verifier_ops *ops = env->ops; 13033 int i, cnt, size, ctx_field_size, delta = 0; 13034 const int insn_cnt = env->prog->len; 13035 struct bpf_insn insn_buf[16], *insn; 13036 u32 target_size, size_default, off; 13037 struct bpf_prog *new_prog; 13038 enum bpf_access_type type; 13039 bool is_narrower_load; 13040 13041 if (ops->gen_prologue || env->seen_direct_write) { 13042 if (!ops->gen_prologue) { 13043 verbose(env, "bpf verifier is misconfigured\n"); 13044 return -EINVAL; 13045 } 13046 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 13047 env->prog); 13048 if (cnt >= ARRAY_SIZE(insn_buf)) { 13049 verbose(env, "bpf verifier is misconfigured\n"); 13050 return -EINVAL; 13051 } else if (cnt) { 13052 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 13053 if (!new_prog) 13054 return -ENOMEM; 13055 13056 env->prog = new_prog; 13057 delta += cnt - 1; 13058 } 13059 } 13060 13061 if (bpf_prog_is_dev_bound(env->prog->aux)) 13062 return 0; 13063 13064 insn = env->prog->insnsi + delta; 13065 13066 for (i = 0; i < insn_cnt; i++, insn++) { 13067 bpf_convert_ctx_access_t convert_ctx_access; 13068 bool ctx_access; 13069 13070 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 13071 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 13072 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 13073 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) { 13074 type = BPF_READ; 13075 ctx_access = true; 13076 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 13077 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 13078 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 13079 insn->code == (BPF_STX | BPF_MEM | BPF_DW) || 13080 insn->code == (BPF_ST | BPF_MEM | BPF_B) || 13081 insn->code == (BPF_ST | BPF_MEM | BPF_H) || 13082 insn->code == (BPF_ST | BPF_MEM | BPF_W) || 13083 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) { 13084 type = BPF_WRITE; 13085 ctx_access = BPF_CLASS(insn->code) == BPF_STX; 13086 } else { 13087 continue; 13088 } 13089 13090 if (type == BPF_WRITE && 13091 env->insn_aux_data[i + delta].sanitize_stack_spill) { 13092 struct bpf_insn patch[] = { 13093 *insn, 13094 BPF_ST_NOSPEC(), 13095 }; 13096 13097 cnt = ARRAY_SIZE(patch); 13098 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 13099 if (!new_prog) 13100 return -ENOMEM; 13101 13102 delta += cnt - 1; 13103 env->prog = new_prog; 13104 insn = new_prog->insnsi + i + delta; 13105 continue; 13106 } 13107 13108 if (!ctx_access) 13109 continue; 13110 13111 switch ((int)env->insn_aux_data[i + delta].ptr_type) { 13112 case PTR_TO_CTX: 13113 if (!ops->convert_ctx_access) 13114 continue; 13115 convert_ctx_access = ops->convert_ctx_access; 13116 break; 13117 case PTR_TO_SOCKET: 13118 case PTR_TO_SOCK_COMMON: 13119 convert_ctx_access = bpf_sock_convert_ctx_access; 13120 break; 13121 case PTR_TO_TCP_SOCK: 13122 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 13123 break; 13124 case PTR_TO_XDP_SOCK: 13125 convert_ctx_access = bpf_xdp_sock_convert_ctx_access; 13126 break; 13127 case PTR_TO_BTF_ID: 13128 case PTR_TO_BTF_ID | PTR_UNTRUSTED: 13129 if (type == BPF_READ) { 13130 insn->code = BPF_LDX | BPF_PROBE_MEM | 13131 BPF_SIZE((insn)->code); 13132 env->prog->aux->num_exentries++; 13133 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { 13134 verbose(env, "Writes through BTF pointers are not allowed\n"); 13135 return -EINVAL; 13136 } 13137 continue; 13138 default: 13139 continue; 13140 } 13141 13142 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 13143 size = BPF_LDST_BYTES(insn); 13144 13145 /* If the read access is a narrower load of the field, 13146 * convert to a 4/8-byte load, to minimum program type specific 13147 * convert_ctx_access changes. If conversion is successful, 13148 * we will apply proper mask to the result. 13149 */ 13150 is_narrower_load = size < ctx_field_size; 13151 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 13152 off = insn->off; 13153 if (is_narrower_load) { 13154 u8 size_code; 13155 13156 if (type == BPF_WRITE) { 13157 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 13158 return -EINVAL; 13159 } 13160 13161 size_code = BPF_H; 13162 if (ctx_field_size == 4) 13163 size_code = BPF_W; 13164 else if (ctx_field_size == 8) 13165 size_code = BPF_DW; 13166 13167 insn->off = off & ~(size_default - 1); 13168 insn->code = BPF_LDX | BPF_MEM | size_code; 13169 } 13170 13171 target_size = 0; 13172 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 13173 &target_size); 13174 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 13175 (ctx_field_size && !target_size)) { 13176 verbose(env, "bpf verifier is misconfigured\n"); 13177 return -EINVAL; 13178 } 13179 13180 if (is_narrower_load && size < target_size) { 13181 u8 shift = bpf_ctx_narrow_access_offset( 13182 off, size, size_default) * 8; 13183 if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) { 13184 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); 13185 return -EINVAL; 13186 } 13187 if (ctx_field_size <= 4) { 13188 if (shift) 13189 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 13190 insn->dst_reg, 13191 shift); 13192 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 13193 (1 << size * 8) - 1); 13194 } else { 13195 if (shift) 13196 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 13197 insn->dst_reg, 13198 shift); 13199 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 13200 (1ULL << size * 8) - 1); 13201 } 13202 } 13203 13204 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13205 if (!new_prog) 13206 return -ENOMEM; 13207 13208 delta += cnt - 1; 13209 13210 /* keep walking new program and skip insns we just inserted */ 13211 env->prog = new_prog; 13212 insn = new_prog->insnsi + i + delta; 13213 } 13214 13215 return 0; 13216 } 13217 13218 static int jit_subprogs(struct bpf_verifier_env *env) 13219 { 13220 struct bpf_prog *prog = env->prog, **func, *tmp; 13221 int i, j, subprog_start, subprog_end = 0, len, subprog; 13222 struct bpf_map *map_ptr; 13223 struct bpf_insn *insn; 13224 void *old_bpf_func; 13225 int err, num_exentries; 13226 13227 if (env->subprog_cnt <= 1) 13228 return 0; 13229 13230 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 13231 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) 13232 continue; 13233 13234 /* Upon error here we cannot fall back to interpreter but 13235 * need a hard reject of the program. Thus -EFAULT is 13236 * propagated in any case. 13237 */ 13238 subprog = find_subprog(env, i + insn->imm + 1); 13239 if (subprog < 0) { 13240 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 13241 i + insn->imm + 1); 13242 return -EFAULT; 13243 } 13244 /* temporarily remember subprog id inside insn instead of 13245 * aux_data, since next loop will split up all insns into funcs 13246 */ 13247 insn->off = subprog; 13248 /* remember original imm in case JIT fails and fallback 13249 * to interpreter will be needed 13250 */ 13251 env->insn_aux_data[i].call_imm = insn->imm; 13252 /* point imm to __bpf_call_base+1 from JITs point of view */ 13253 insn->imm = 1; 13254 if (bpf_pseudo_func(insn)) 13255 /* jit (e.g. x86_64) may emit fewer instructions 13256 * if it learns a u32 imm is the same as a u64 imm. 13257 * Force a non zero here. 13258 */ 13259 insn[1].imm = 1; 13260 } 13261 13262 err = bpf_prog_alloc_jited_linfo(prog); 13263 if (err) 13264 goto out_undo_insn; 13265 13266 err = -ENOMEM; 13267 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 13268 if (!func) 13269 goto out_undo_insn; 13270 13271 for (i = 0; i < env->subprog_cnt; i++) { 13272 subprog_start = subprog_end; 13273 subprog_end = env->subprog_info[i + 1].start; 13274 13275 len = subprog_end - subprog_start; 13276 /* bpf_prog_run() doesn't call subprogs directly, 13277 * hence main prog stats include the runtime of subprogs. 13278 * subprogs don't have IDs and not reachable via prog_get_next_id 13279 * func[i]->stats will never be accessed and stays NULL 13280 */ 13281 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 13282 if (!func[i]) 13283 goto out_free; 13284 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 13285 len * sizeof(struct bpf_insn)); 13286 func[i]->type = prog->type; 13287 func[i]->len = len; 13288 if (bpf_prog_calc_tag(func[i])) 13289 goto out_free; 13290 func[i]->is_func = 1; 13291 func[i]->aux->func_idx = i; 13292 /* Below members will be freed only at prog->aux */ 13293 func[i]->aux->btf = prog->aux->btf; 13294 func[i]->aux->func_info = prog->aux->func_info; 13295 func[i]->aux->poke_tab = prog->aux->poke_tab; 13296 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; 13297 13298 for (j = 0; j < prog->aux->size_poke_tab; j++) { 13299 struct bpf_jit_poke_descriptor *poke; 13300 13301 poke = &prog->aux->poke_tab[j]; 13302 if (poke->insn_idx < subprog_end && 13303 poke->insn_idx >= subprog_start) 13304 poke->aux = func[i]->aux; 13305 } 13306 13307 /* Use bpf_prog_F_tag to indicate functions in stack traces. 13308 * Long term would need debug info to populate names 13309 */ 13310 func[i]->aux->name[0] = 'F'; 13311 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 13312 func[i]->jit_requested = 1; 13313 func[i]->blinding_requested = prog->blinding_requested; 13314 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; 13315 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; 13316 func[i]->aux->linfo = prog->aux->linfo; 13317 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 13318 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 13319 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 13320 num_exentries = 0; 13321 insn = func[i]->insnsi; 13322 for (j = 0; j < func[i]->len; j++, insn++) { 13323 if (BPF_CLASS(insn->code) == BPF_LDX && 13324 BPF_MODE(insn->code) == BPF_PROBE_MEM) 13325 num_exentries++; 13326 } 13327 func[i]->aux->num_exentries = num_exentries; 13328 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; 13329 func[i] = bpf_int_jit_compile(func[i]); 13330 if (!func[i]->jited) { 13331 err = -ENOTSUPP; 13332 goto out_free; 13333 } 13334 cond_resched(); 13335 } 13336 13337 /* at this point all bpf functions were successfully JITed 13338 * now populate all bpf_calls with correct addresses and 13339 * run last pass of JIT 13340 */ 13341 for (i = 0; i < env->subprog_cnt; i++) { 13342 insn = func[i]->insnsi; 13343 for (j = 0; j < func[i]->len; j++, insn++) { 13344 if (bpf_pseudo_func(insn)) { 13345 subprog = insn->off; 13346 insn[0].imm = (u32)(long)func[subprog]->bpf_func; 13347 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; 13348 continue; 13349 } 13350 if (!bpf_pseudo_call(insn)) 13351 continue; 13352 subprog = insn->off; 13353 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); 13354 } 13355 13356 /* we use the aux data to keep a list of the start addresses 13357 * of the JITed images for each function in the program 13358 * 13359 * for some architectures, such as powerpc64, the imm field 13360 * might not be large enough to hold the offset of the start 13361 * address of the callee's JITed image from __bpf_call_base 13362 * 13363 * in such cases, we can lookup the start address of a callee 13364 * by using its subprog id, available from the off field of 13365 * the call instruction, as an index for this list 13366 */ 13367 func[i]->aux->func = func; 13368 func[i]->aux->func_cnt = env->subprog_cnt; 13369 } 13370 for (i = 0; i < env->subprog_cnt; i++) { 13371 old_bpf_func = func[i]->bpf_func; 13372 tmp = bpf_int_jit_compile(func[i]); 13373 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 13374 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 13375 err = -ENOTSUPP; 13376 goto out_free; 13377 } 13378 cond_resched(); 13379 } 13380 13381 /* finally lock prog and jit images for all functions and 13382 * populate kallsysm 13383 */ 13384 for (i = 0; i < env->subprog_cnt; i++) { 13385 bpf_prog_lock_ro(func[i]); 13386 bpf_prog_kallsyms_add(func[i]); 13387 } 13388 13389 /* Last step: make now unused interpreter insns from main 13390 * prog consistent for later dump requests, so they can 13391 * later look the same as if they were interpreted only. 13392 */ 13393 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 13394 if (bpf_pseudo_func(insn)) { 13395 insn[0].imm = env->insn_aux_data[i].call_imm; 13396 insn[1].imm = insn->off; 13397 insn->off = 0; 13398 continue; 13399 } 13400 if (!bpf_pseudo_call(insn)) 13401 continue; 13402 insn->off = env->insn_aux_data[i].call_imm; 13403 subprog = find_subprog(env, i + insn->off + 1); 13404 insn->imm = subprog; 13405 } 13406 13407 prog->jited = 1; 13408 prog->bpf_func = func[0]->bpf_func; 13409 prog->jited_len = func[0]->jited_len; 13410 prog->aux->func = func; 13411 prog->aux->func_cnt = env->subprog_cnt; 13412 bpf_prog_jit_attempt_done(prog); 13413 return 0; 13414 out_free: 13415 /* We failed JIT'ing, so at this point we need to unregister poke 13416 * descriptors from subprogs, so that kernel is not attempting to 13417 * patch it anymore as we're freeing the subprog JIT memory. 13418 */ 13419 for (i = 0; i < prog->aux->size_poke_tab; i++) { 13420 map_ptr = prog->aux->poke_tab[i].tail_call.map; 13421 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); 13422 } 13423 /* At this point we're guaranteed that poke descriptors are not 13424 * live anymore. We can just unlink its descriptor table as it's 13425 * released with the main prog. 13426 */ 13427 for (i = 0; i < env->subprog_cnt; i++) { 13428 if (!func[i]) 13429 continue; 13430 func[i]->aux->poke_tab = NULL; 13431 bpf_jit_free(func[i]); 13432 } 13433 kfree(func); 13434 out_undo_insn: 13435 /* cleanup main prog to be interpreted */ 13436 prog->jit_requested = 0; 13437 prog->blinding_requested = 0; 13438 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 13439 if (!bpf_pseudo_call(insn)) 13440 continue; 13441 insn->off = 0; 13442 insn->imm = env->insn_aux_data[i].call_imm; 13443 } 13444 bpf_prog_jit_attempt_done(prog); 13445 return err; 13446 } 13447 13448 static int fixup_call_args(struct bpf_verifier_env *env) 13449 { 13450 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 13451 struct bpf_prog *prog = env->prog; 13452 struct bpf_insn *insn = prog->insnsi; 13453 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); 13454 int i, depth; 13455 #endif 13456 int err = 0; 13457 13458 if (env->prog->jit_requested && 13459 !bpf_prog_is_dev_bound(env->prog->aux)) { 13460 err = jit_subprogs(env); 13461 if (err == 0) 13462 return 0; 13463 if (err == -EFAULT) 13464 return err; 13465 } 13466 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 13467 if (has_kfunc_call) { 13468 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); 13469 return -EINVAL; 13470 } 13471 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { 13472 /* When JIT fails the progs with bpf2bpf calls and tail_calls 13473 * have to be rejected, since interpreter doesn't support them yet. 13474 */ 13475 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); 13476 return -EINVAL; 13477 } 13478 for (i = 0; i < prog->len; i++, insn++) { 13479 if (bpf_pseudo_func(insn)) { 13480 /* When JIT fails the progs with callback calls 13481 * have to be rejected, since interpreter doesn't support them yet. 13482 */ 13483 verbose(env, "callbacks are not allowed in non-JITed programs\n"); 13484 return -EINVAL; 13485 } 13486 13487 if (!bpf_pseudo_call(insn)) 13488 continue; 13489 depth = get_callee_stack_depth(env, insn, i); 13490 if (depth < 0) 13491 return depth; 13492 bpf_patch_call_args(insn, depth); 13493 } 13494 err = 0; 13495 #endif 13496 return err; 13497 } 13498 13499 static int fixup_kfunc_call(struct bpf_verifier_env *env, 13500 struct bpf_insn *insn) 13501 { 13502 const struct bpf_kfunc_desc *desc; 13503 13504 if (!insn->imm) { 13505 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); 13506 return -EINVAL; 13507 } 13508 13509 /* insn->imm has the btf func_id. Replace it with 13510 * an address (relative to __bpf_base_call). 13511 */ 13512 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); 13513 if (!desc) { 13514 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", 13515 insn->imm); 13516 return -EFAULT; 13517 } 13518 13519 insn->imm = desc->imm; 13520 13521 return 0; 13522 } 13523 13524 /* Do various post-verification rewrites in a single program pass. 13525 * These rewrites simplify JIT and interpreter implementations. 13526 */ 13527 static int do_misc_fixups(struct bpf_verifier_env *env) 13528 { 13529 struct bpf_prog *prog = env->prog; 13530 enum bpf_attach_type eatype = prog->expected_attach_type; 13531 enum bpf_prog_type prog_type = resolve_prog_type(prog); 13532 struct bpf_insn *insn = prog->insnsi; 13533 const struct bpf_func_proto *fn; 13534 const int insn_cnt = prog->len; 13535 const struct bpf_map_ops *ops; 13536 struct bpf_insn_aux_data *aux; 13537 struct bpf_insn insn_buf[16]; 13538 struct bpf_prog *new_prog; 13539 struct bpf_map *map_ptr; 13540 int i, ret, cnt, delta = 0; 13541 13542 for (i = 0; i < insn_cnt; i++, insn++) { 13543 /* Make divide-by-zero exceptions impossible. */ 13544 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 13545 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 13546 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 13547 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 13548 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 13549 bool isdiv = BPF_OP(insn->code) == BPF_DIV; 13550 struct bpf_insn *patchlet; 13551 struct bpf_insn chk_and_div[] = { 13552 /* [R,W]x div 0 -> 0 */ 13553 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 13554 BPF_JNE | BPF_K, insn->src_reg, 13555 0, 2, 0), 13556 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 13557 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 13558 *insn, 13559 }; 13560 struct bpf_insn chk_and_mod[] = { 13561 /* [R,W]x mod 0 -> [R,W]x */ 13562 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | 13563 BPF_JEQ | BPF_K, insn->src_reg, 13564 0, 1 + (is64 ? 0 : 1), 0), 13565 *insn, 13566 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 13567 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), 13568 }; 13569 13570 patchlet = isdiv ? chk_and_div : chk_and_mod; 13571 cnt = isdiv ? ARRAY_SIZE(chk_and_div) : 13572 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); 13573 13574 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 13575 if (!new_prog) 13576 return -ENOMEM; 13577 13578 delta += cnt - 1; 13579 env->prog = prog = new_prog; 13580 insn = new_prog->insnsi + i + delta; 13581 continue; 13582 } 13583 13584 /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */ 13585 if (BPF_CLASS(insn->code) == BPF_LD && 13586 (BPF_MODE(insn->code) == BPF_ABS || 13587 BPF_MODE(insn->code) == BPF_IND)) { 13588 cnt = env->ops->gen_ld_abs(insn, insn_buf); 13589 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 13590 verbose(env, "bpf verifier is misconfigured\n"); 13591 return -EINVAL; 13592 } 13593 13594 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13595 if (!new_prog) 13596 return -ENOMEM; 13597 13598 delta += cnt - 1; 13599 env->prog = prog = new_prog; 13600 insn = new_prog->insnsi + i + delta; 13601 continue; 13602 } 13603 13604 /* Rewrite pointer arithmetic to mitigate speculation attacks. */ 13605 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 13606 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 13607 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 13608 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 13609 struct bpf_insn *patch = &insn_buf[0]; 13610 bool issrc, isneg, isimm; 13611 u32 off_reg; 13612 13613 aux = &env->insn_aux_data[i + delta]; 13614 if (!aux->alu_state || 13615 aux->alu_state == BPF_ALU_NON_POINTER) 13616 continue; 13617 13618 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 13619 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 13620 BPF_ALU_SANITIZE_SRC; 13621 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; 13622 13623 off_reg = issrc ? insn->src_reg : insn->dst_reg; 13624 if (isimm) { 13625 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 13626 } else { 13627 if (isneg) 13628 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 13629 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); 13630 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 13631 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 13632 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 13633 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 13634 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); 13635 } 13636 if (!issrc) 13637 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); 13638 insn->src_reg = BPF_REG_AX; 13639 if (isneg) 13640 insn->code = insn->code == code_add ? 13641 code_sub : code_add; 13642 *patch++ = *insn; 13643 if (issrc && isneg && !isimm) 13644 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 13645 cnt = patch - insn_buf; 13646 13647 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13648 if (!new_prog) 13649 return -ENOMEM; 13650 13651 delta += cnt - 1; 13652 env->prog = prog = new_prog; 13653 insn = new_prog->insnsi + i + delta; 13654 continue; 13655 } 13656 13657 if (insn->code != (BPF_JMP | BPF_CALL)) 13658 continue; 13659 if (insn->src_reg == BPF_PSEUDO_CALL) 13660 continue; 13661 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { 13662 ret = fixup_kfunc_call(env, insn); 13663 if (ret) 13664 return ret; 13665 continue; 13666 } 13667 13668 if (insn->imm == BPF_FUNC_get_route_realm) 13669 prog->dst_needed = 1; 13670 if (insn->imm == BPF_FUNC_get_prandom_u32) 13671 bpf_user_rnd_init_once(); 13672 if (insn->imm == BPF_FUNC_override_return) 13673 prog->kprobe_override = 1; 13674 if (insn->imm == BPF_FUNC_tail_call) { 13675 /* If we tail call into other programs, we 13676 * cannot make any assumptions since they can 13677 * be replaced dynamically during runtime in 13678 * the program array. 13679 */ 13680 prog->cb_access = 1; 13681 if (!allow_tail_call_in_subprogs(env)) 13682 prog->aux->stack_depth = MAX_BPF_STACK; 13683 prog->aux->max_pkt_offset = MAX_PACKET_OFF; 13684 13685 /* mark bpf_tail_call as different opcode to avoid 13686 * conditional branch in the interpreter for every normal 13687 * call and to prevent accidental JITing by JIT compiler 13688 * that doesn't support bpf_tail_call yet 13689 */ 13690 insn->imm = 0; 13691 insn->code = BPF_JMP | BPF_TAIL_CALL; 13692 13693 aux = &env->insn_aux_data[i + delta]; 13694 if (env->bpf_capable && !prog->blinding_requested && 13695 prog->jit_requested && 13696 !bpf_map_key_poisoned(aux) && 13697 !bpf_map_ptr_poisoned(aux) && 13698 !bpf_map_ptr_unpriv(aux)) { 13699 struct bpf_jit_poke_descriptor desc = { 13700 .reason = BPF_POKE_REASON_TAIL_CALL, 13701 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), 13702 .tail_call.key = bpf_map_key_immediate(aux), 13703 .insn_idx = i + delta, 13704 }; 13705 13706 ret = bpf_jit_add_poke_descriptor(prog, &desc); 13707 if (ret < 0) { 13708 verbose(env, "adding tail call poke descriptor failed\n"); 13709 return ret; 13710 } 13711 13712 insn->imm = ret + 1; 13713 continue; 13714 } 13715 13716 if (!bpf_map_ptr_unpriv(aux)) 13717 continue; 13718 13719 /* instead of changing every JIT dealing with tail_call 13720 * emit two extra insns: 13721 * if (index >= max_entries) goto out; 13722 * index &= array->index_mask; 13723 * to avoid out-of-bounds cpu speculation 13724 */ 13725 if (bpf_map_ptr_poisoned(aux)) { 13726 verbose(env, "tail_call abusing map_ptr\n"); 13727 return -EINVAL; 13728 } 13729 13730 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 13731 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 13732 map_ptr->max_entries, 2); 13733 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 13734 container_of(map_ptr, 13735 struct bpf_array, 13736 map)->index_mask); 13737 insn_buf[2] = *insn; 13738 cnt = 3; 13739 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13740 if (!new_prog) 13741 return -ENOMEM; 13742 13743 delta += cnt - 1; 13744 env->prog = prog = new_prog; 13745 insn = new_prog->insnsi + i + delta; 13746 continue; 13747 } 13748 13749 if (insn->imm == BPF_FUNC_timer_set_callback) { 13750 /* The verifier will process callback_fn as many times as necessary 13751 * with different maps and the register states prepared by 13752 * set_timer_callback_state will be accurate. 13753 * 13754 * The following use case is valid: 13755 * map1 is shared by prog1, prog2, prog3. 13756 * prog1 calls bpf_timer_init for some map1 elements 13757 * prog2 calls bpf_timer_set_callback for some map1 elements. 13758 * Those that were not bpf_timer_init-ed will return -EINVAL. 13759 * prog3 calls bpf_timer_start for some map1 elements. 13760 * Those that were not both bpf_timer_init-ed and 13761 * bpf_timer_set_callback-ed will return -EINVAL. 13762 */ 13763 struct bpf_insn ld_addrs[2] = { 13764 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), 13765 }; 13766 13767 insn_buf[0] = ld_addrs[0]; 13768 insn_buf[1] = ld_addrs[1]; 13769 insn_buf[2] = *insn; 13770 cnt = 3; 13771 13772 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13773 if (!new_prog) 13774 return -ENOMEM; 13775 13776 delta += cnt - 1; 13777 env->prog = prog = new_prog; 13778 insn = new_prog->insnsi + i + delta; 13779 goto patch_call_imm; 13780 } 13781 13782 if (insn->imm == BPF_FUNC_task_storage_get || 13783 insn->imm == BPF_FUNC_sk_storage_get || 13784 insn->imm == BPF_FUNC_inode_storage_get) { 13785 if (env->prog->aux->sleepable) 13786 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL); 13787 else 13788 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC); 13789 insn_buf[1] = *insn; 13790 cnt = 2; 13791 13792 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13793 if (!new_prog) 13794 return -ENOMEM; 13795 13796 delta += cnt - 1; 13797 env->prog = prog = new_prog; 13798 insn = new_prog->insnsi + i + delta; 13799 goto patch_call_imm; 13800 } 13801 13802 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 13803 * and other inlining handlers are currently limited to 64 bit 13804 * only. 13805 */ 13806 if (prog->jit_requested && BITS_PER_LONG == 64 && 13807 (insn->imm == BPF_FUNC_map_lookup_elem || 13808 insn->imm == BPF_FUNC_map_update_elem || 13809 insn->imm == BPF_FUNC_map_delete_elem || 13810 insn->imm == BPF_FUNC_map_push_elem || 13811 insn->imm == BPF_FUNC_map_pop_elem || 13812 insn->imm == BPF_FUNC_map_peek_elem || 13813 insn->imm == BPF_FUNC_redirect_map || 13814 insn->imm == BPF_FUNC_for_each_map_elem)) { 13815 aux = &env->insn_aux_data[i + delta]; 13816 if (bpf_map_ptr_poisoned(aux)) 13817 goto patch_call_imm; 13818 13819 map_ptr = BPF_MAP_PTR(aux->map_ptr_state); 13820 ops = map_ptr->ops; 13821 if (insn->imm == BPF_FUNC_map_lookup_elem && 13822 ops->map_gen_lookup) { 13823 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 13824 if (cnt == -EOPNOTSUPP) 13825 goto patch_map_ops_generic; 13826 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { 13827 verbose(env, "bpf verifier is misconfigured\n"); 13828 return -EINVAL; 13829 } 13830 13831 new_prog = bpf_patch_insn_data(env, i + delta, 13832 insn_buf, cnt); 13833 if (!new_prog) 13834 return -ENOMEM; 13835 13836 delta += cnt - 1; 13837 env->prog = prog = new_prog; 13838 insn = new_prog->insnsi + i + delta; 13839 continue; 13840 } 13841 13842 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 13843 (void *(*)(struct bpf_map *map, void *key))NULL)); 13844 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 13845 (int (*)(struct bpf_map *map, void *key))NULL)); 13846 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 13847 (int (*)(struct bpf_map *map, void *key, void *value, 13848 u64 flags))NULL)); 13849 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 13850 (int (*)(struct bpf_map *map, void *value, 13851 u64 flags))NULL)); 13852 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 13853 (int (*)(struct bpf_map *map, void *value))NULL)); 13854 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 13855 (int (*)(struct bpf_map *map, void *value))NULL)); 13856 BUILD_BUG_ON(!__same_type(ops->map_redirect, 13857 (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL)); 13858 BUILD_BUG_ON(!__same_type(ops->map_for_each_callback, 13859 (int (*)(struct bpf_map *map, 13860 bpf_callback_t callback_fn, 13861 void *callback_ctx, 13862 u64 flags))NULL)); 13863 13864 patch_map_ops_generic: 13865 switch (insn->imm) { 13866 case BPF_FUNC_map_lookup_elem: 13867 insn->imm = BPF_CALL_IMM(ops->map_lookup_elem); 13868 continue; 13869 case BPF_FUNC_map_update_elem: 13870 insn->imm = BPF_CALL_IMM(ops->map_update_elem); 13871 continue; 13872 case BPF_FUNC_map_delete_elem: 13873 insn->imm = BPF_CALL_IMM(ops->map_delete_elem); 13874 continue; 13875 case BPF_FUNC_map_push_elem: 13876 insn->imm = BPF_CALL_IMM(ops->map_push_elem); 13877 continue; 13878 case BPF_FUNC_map_pop_elem: 13879 insn->imm = BPF_CALL_IMM(ops->map_pop_elem); 13880 continue; 13881 case BPF_FUNC_map_peek_elem: 13882 insn->imm = BPF_CALL_IMM(ops->map_peek_elem); 13883 continue; 13884 case BPF_FUNC_redirect_map: 13885 insn->imm = BPF_CALL_IMM(ops->map_redirect); 13886 continue; 13887 case BPF_FUNC_for_each_map_elem: 13888 insn->imm = BPF_CALL_IMM(ops->map_for_each_callback); 13889 continue; 13890 } 13891 13892 goto patch_call_imm; 13893 } 13894 13895 /* Implement bpf_jiffies64 inline. */ 13896 if (prog->jit_requested && BITS_PER_LONG == 64 && 13897 insn->imm == BPF_FUNC_jiffies64) { 13898 struct bpf_insn ld_jiffies_addr[2] = { 13899 BPF_LD_IMM64(BPF_REG_0, 13900 (unsigned long)&jiffies), 13901 }; 13902 13903 insn_buf[0] = ld_jiffies_addr[0]; 13904 insn_buf[1] = ld_jiffies_addr[1]; 13905 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, 13906 BPF_REG_0, 0); 13907 cnt = 3; 13908 13909 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 13910 cnt); 13911 if (!new_prog) 13912 return -ENOMEM; 13913 13914 delta += cnt - 1; 13915 env->prog = prog = new_prog; 13916 insn = new_prog->insnsi + i + delta; 13917 continue; 13918 } 13919 13920 /* Implement bpf_get_func_arg inline. */ 13921 if (prog_type == BPF_PROG_TYPE_TRACING && 13922 insn->imm == BPF_FUNC_get_func_arg) { 13923 /* Load nr_args from ctx - 8 */ 13924 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 13925 insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6); 13926 insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3); 13927 insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1); 13928 insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0); 13929 insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 13930 insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0); 13931 insn_buf[7] = BPF_JMP_A(1); 13932 insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); 13933 cnt = 9; 13934 13935 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13936 if (!new_prog) 13937 return -ENOMEM; 13938 13939 delta += cnt - 1; 13940 env->prog = prog = new_prog; 13941 insn = new_prog->insnsi + i + delta; 13942 continue; 13943 } 13944 13945 /* Implement bpf_get_func_ret inline. */ 13946 if (prog_type == BPF_PROG_TYPE_TRACING && 13947 insn->imm == BPF_FUNC_get_func_ret) { 13948 if (eatype == BPF_TRACE_FEXIT || 13949 eatype == BPF_MODIFY_RETURN) { 13950 /* Load nr_args from ctx - 8 */ 13951 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 13952 insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3); 13953 insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1); 13954 insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0); 13955 insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0); 13956 insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0); 13957 cnt = 6; 13958 } else { 13959 insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP); 13960 cnt = 1; 13961 } 13962 13963 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 13964 if (!new_prog) 13965 return -ENOMEM; 13966 13967 delta += cnt - 1; 13968 env->prog = prog = new_prog; 13969 insn = new_prog->insnsi + i + delta; 13970 continue; 13971 } 13972 13973 /* Implement get_func_arg_cnt inline. */ 13974 if (prog_type == BPF_PROG_TYPE_TRACING && 13975 insn->imm == BPF_FUNC_get_func_arg_cnt) { 13976 /* Load nr_args from ctx - 8 */ 13977 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8); 13978 13979 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 13980 if (!new_prog) 13981 return -ENOMEM; 13982 13983 env->prog = prog = new_prog; 13984 insn = new_prog->insnsi + i + delta; 13985 continue; 13986 } 13987 13988 /* Implement bpf_get_func_ip inline. */ 13989 if (prog_type == BPF_PROG_TYPE_TRACING && 13990 insn->imm == BPF_FUNC_get_func_ip) { 13991 /* Load IP address from ctx - 16 */ 13992 insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); 13993 13994 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); 13995 if (!new_prog) 13996 return -ENOMEM; 13997 13998 env->prog = prog = new_prog; 13999 insn = new_prog->insnsi + i + delta; 14000 continue; 14001 } 14002 14003 patch_call_imm: 14004 fn = env->ops->get_func_proto(insn->imm, env->prog); 14005 /* all functions that have prototype and verifier allowed 14006 * programs to call them, must be real in-kernel functions 14007 */ 14008 if (!fn->func) { 14009 verbose(env, 14010 "kernel subsystem misconfigured func %s#%d\n", 14011 func_id_name(insn->imm), insn->imm); 14012 return -EFAULT; 14013 } 14014 insn->imm = fn->func - __bpf_call_base; 14015 } 14016 14017 /* Since poke tab is now finalized, publish aux to tracker. */ 14018 for (i = 0; i < prog->aux->size_poke_tab; i++) { 14019 map_ptr = prog->aux->poke_tab[i].tail_call.map; 14020 if (!map_ptr->ops->map_poke_track || 14021 !map_ptr->ops->map_poke_untrack || 14022 !map_ptr->ops->map_poke_run) { 14023 verbose(env, "bpf verifier is misconfigured\n"); 14024 return -EINVAL; 14025 } 14026 14027 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); 14028 if (ret < 0) { 14029 verbose(env, "tracking tail call prog failed\n"); 14030 return ret; 14031 } 14032 } 14033 14034 sort_kfunc_descs_by_imm(env->prog); 14035 14036 return 0; 14037 } 14038 14039 static void free_states(struct bpf_verifier_env *env) 14040 { 14041 struct bpf_verifier_state_list *sl, *sln; 14042 int i; 14043 14044 sl = env->free_list; 14045 while (sl) { 14046 sln = sl->next; 14047 free_verifier_state(&sl->state, false); 14048 kfree(sl); 14049 sl = sln; 14050 } 14051 env->free_list = NULL; 14052 14053 if (!env->explored_states) 14054 return; 14055 14056 for (i = 0; i < state_htab_size(env); i++) { 14057 sl = env->explored_states[i]; 14058 14059 while (sl) { 14060 sln = sl->next; 14061 free_verifier_state(&sl->state, false); 14062 kfree(sl); 14063 sl = sln; 14064 } 14065 env->explored_states[i] = NULL; 14066 } 14067 } 14068 14069 static int do_check_common(struct bpf_verifier_env *env, int subprog) 14070 { 14071 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); 14072 struct bpf_verifier_state *state; 14073 struct bpf_reg_state *regs; 14074 int ret, i; 14075 14076 env->prev_linfo = NULL; 14077 env->pass_cnt++; 14078 14079 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 14080 if (!state) 14081 return -ENOMEM; 14082 state->curframe = 0; 14083 state->speculative = false; 14084 state->branches = 1; 14085 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 14086 if (!state->frame[0]) { 14087 kfree(state); 14088 return -ENOMEM; 14089 } 14090 env->cur_state = state; 14091 init_func_state(env, state->frame[0], 14092 BPF_MAIN_FUNC /* callsite */, 14093 0 /* frameno */, 14094 subprog); 14095 14096 regs = state->frame[state->curframe]->regs; 14097 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { 14098 ret = btf_prepare_func_args(env, subprog, regs); 14099 if (ret) 14100 goto out; 14101 for (i = BPF_REG_1; i <= BPF_REG_5; i++) { 14102 if (regs[i].type == PTR_TO_CTX) 14103 mark_reg_known_zero(env, regs, i); 14104 else if (regs[i].type == SCALAR_VALUE) 14105 mark_reg_unknown(env, regs, i); 14106 else if (base_type(regs[i].type) == PTR_TO_MEM) { 14107 const u32 mem_size = regs[i].mem_size; 14108 14109 mark_reg_known_zero(env, regs, i); 14110 regs[i].mem_size = mem_size; 14111 regs[i].id = ++env->id_gen; 14112 } 14113 } 14114 } else { 14115 /* 1st arg to a function */ 14116 regs[BPF_REG_1].type = PTR_TO_CTX; 14117 mark_reg_known_zero(env, regs, BPF_REG_1); 14118 ret = btf_check_subprog_arg_match(env, subprog, regs); 14119 if (ret == -EFAULT) 14120 /* unlikely verifier bug. abort. 14121 * ret == 0 and ret < 0 are sadly acceptable for 14122 * main() function due to backward compatibility. 14123 * Like socket filter program may be written as: 14124 * int bpf_prog(struct pt_regs *ctx) 14125 * and never dereference that ctx in the program. 14126 * 'struct pt_regs' is a type mismatch for socket 14127 * filter that should be using 'struct __sk_buff'. 14128 */ 14129 goto out; 14130 } 14131 14132 ret = do_check(env); 14133 out: 14134 /* check for NULL is necessary, since cur_state can be freed inside 14135 * do_check() under memory pressure. 14136 */ 14137 if (env->cur_state) { 14138 free_verifier_state(env->cur_state, true); 14139 env->cur_state = NULL; 14140 } 14141 while (!pop_stack(env, NULL, NULL, false)); 14142 if (!ret && pop_log) 14143 bpf_vlog_reset(&env->log, 0); 14144 free_states(env); 14145 return ret; 14146 } 14147 14148 /* Verify all global functions in a BPF program one by one based on their BTF. 14149 * All global functions must pass verification. Otherwise the whole program is rejected. 14150 * Consider: 14151 * int bar(int); 14152 * int foo(int f) 14153 * { 14154 * return bar(f); 14155 * } 14156 * int bar(int b) 14157 * { 14158 * ... 14159 * } 14160 * foo() will be verified first for R1=any_scalar_value. During verification it 14161 * will be assumed that bar() already verified successfully and call to bar() 14162 * from foo() will be checked for type match only. Later bar() will be verified 14163 * independently to check that it's safe for R1=any_scalar_value. 14164 */ 14165 static int do_check_subprogs(struct bpf_verifier_env *env) 14166 { 14167 struct bpf_prog_aux *aux = env->prog->aux; 14168 int i, ret; 14169 14170 if (!aux->func_info) 14171 return 0; 14172 14173 for (i = 1; i < env->subprog_cnt; i++) { 14174 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) 14175 continue; 14176 env->insn_idx = env->subprog_info[i].start; 14177 WARN_ON_ONCE(env->insn_idx == 0); 14178 ret = do_check_common(env, i); 14179 if (ret) { 14180 return ret; 14181 } else if (env->log.level & BPF_LOG_LEVEL) { 14182 verbose(env, 14183 "Func#%d is safe for any args that match its prototype\n", 14184 i); 14185 } 14186 } 14187 return 0; 14188 } 14189 14190 static int do_check_main(struct bpf_verifier_env *env) 14191 { 14192 int ret; 14193 14194 env->insn_idx = 0; 14195 ret = do_check_common(env, 0); 14196 if (!ret) 14197 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 14198 return ret; 14199 } 14200 14201 14202 static void print_verification_stats(struct bpf_verifier_env *env) 14203 { 14204 int i; 14205 14206 if (env->log.level & BPF_LOG_STATS) { 14207 verbose(env, "verification time %lld usec\n", 14208 div_u64(env->verification_time, 1000)); 14209 verbose(env, "stack depth "); 14210 for (i = 0; i < env->subprog_cnt; i++) { 14211 u32 depth = env->subprog_info[i].stack_depth; 14212 14213 verbose(env, "%d", depth); 14214 if (i + 1 < env->subprog_cnt) 14215 verbose(env, "+"); 14216 } 14217 verbose(env, "\n"); 14218 } 14219 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 14220 "total_states %d peak_states %d mark_read %d\n", 14221 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 14222 env->max_states_per_insn, env->total_states, 14223 env->peak_states, env->longest_mark_read_walk); 14224 } 14225 14226 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) 14227 { 14228 const struct btf_type *t, *func_proto; 14229 const struct bpf_struct_ops *st_ops; 14230 const struct btf_member *member; 14231 struct bpf_prog *prog = env->prog; 14232 u32 btf_id, member_idx; 14233 const char *mname; 14234 14235 if (!prog->gpl_compatible) { 14236 verbose(env, "struct ops programs must have a GPL compatible license\n"); 14237 return -EINVAL; 14238 } 14239 14240 btf_id = prog->aux->attach_btf_id; 14241 st_ops = bpf_struct_ops_find(btf_id); 14242 if (!st_ops) { 14243 verbose(env, "attach_btf_id %u is not a supported struct\n", 14244 btf_id); 14245 return -ENOTSUPP; 14246 } 14247 14248 t = st_ops->type; 14249 member_idx = prog->expected_attach_type; 14250 if (member_idx >= btf_type_vlen(t)) { 14251 verbose(env, "attach to invalid member idx %u of struct %s\n", 14252 member_idx, st_ops->name); 14253 return -EINVAL; 14254 } 14255 14256 member = &btf_type_member(t)[member_idx]; 14257 mname = btf_name_by_offset(btf_vmlinux, member->name_off); 14258 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, 14259 NULL); 14260 if (!func_proto) { 14261 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", 14262 mname, member_idx, st_ops->name); 14263 return -EINVAL; 14264 } 14265 14266 if (st_ops->check_member) { 14267 int err = st_ops->check_member(t, member); 14268 14269 if (err) { 14270 verbose(env, "attach to unsupported member %s of struct %s\n", 14271 mname, st_ops->name); 14272 return err; 14273 } 14274 } 14275 14276 prog->aux->attach_func_proto = func_proto; 14277 prog->aux->attach_func_name = mname; 14278 env->ops = st_ops->verifier_ops; 14279 14280 return 0; 14281 } 14282 #define SECURITY_PREFIX "security_" 14283 14284 static int check_attach_modify_return(unsigned long addr, const char *func_name) 14285 { 14286 if (within_error_injection_list(addr) || 14287 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) 14288 return 0; 14289 14290 return -EINVAL; 14291 } 14292 14293 /* list of non-sleepable functions that are otherwise on 14294 * ALLOW_ERROR_INJECTION list 14295 */ 14296 BTF_SET_START(btf_non_sleepable_error_inject) 14297 /* Three functions below can be called from sleepable and non-sleepable context. 14298 * Assume non-sleepable from bpf safety point of view. 14299 */ 14300 BTF_ID(func, __filemap_add_folio) 14301 BTF_ID(func, should_fail_alloc_page) 14302 BTF_ID(func, should_failslab) 14303 BTF_SET_END(btf_non_sleepable_error_inject) 14304 14305 static int check_non_sleepable_error_inject(u32 btf_id) 14306 { 14307 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); 14308 } 14309 14310 int bpf_check_attach_target(struct bpf_verifier_log *log, 14311 const struct bpf_prog *prog, 14312 const struct bpf_prog *tgt_prog, 14313 u32 btf_id, 14314 struct bpf_attach_target_info *tgt_info) 14315 { 14316 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; 14317 const char prefix[] = "btf_trace_"; 14318 int ret = 0, subprog = -1, i; 14319 const struct btf_type *t; 14320 bool conservative = true; 14321 const char *tname; 14322 struct btf *btf; 14323 long addr = 0; 14324 14325 if (!btf_id) { 14326 bpf_log(log, "Tracing programs must provide btf_id\n"); 14327 return -EINVAL; 14328 } 14329 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; 14330 if (!btf) { 14331 bpf_log(log, 14332 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); 14333 return -EINVAL; 14334 } 14335 t = btf_type_by_id(btf, btf_id); 14336 if (!t) { 14337 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); 14338 return -EINVAL; 14339 } 14340 tname = btf_name_by_offset(btf, t->name_off); 14341 if (!tname) { 14342 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); 14343 return -EINVAL; 14344 } 14345 if (tgt_prog) { 14346 struct bpf_prog_aux *aux = tgt_prog->aux; 14347 14348 for (i = 0; i < aux->func_info_cnt; i++) 14349 if (aux->func_info[i].type_id == btf_id) { 14350 subprog = i; 14351 break; 14352 } 14353 if (subprog == -1) { 14354 bpf_log(log, "Subprog %s doesn't exist\n", tname); 14355 return -EINVAL; 14356 } 14357 conservative = aux->func_info_aux[subprog].unreliable; 14358 if (prog_extension) { 14359 if (conservative) { 14360 bpf_log(log, 14361 "Cannot replace static functions\n"); 14362 return -EINVAL; 14363 } 14364 if (!prog->jit_requested) { 14365 bpf_log(log, 14366 "Extension programs should be JITed\n"); 14367 return -EINVAL; 14368 } 14369 } 14370 if (!tgt_prog->jited) { 14371 bpf_log(log, "Can attach to only JITed progs\n"); 14372 return -EINVAL; 14373 } 14374 if (tgt_prog->type == prog->type) { 14375 /* Cannot fentry/fexit another fentry/fexit program. 14376 * Cannot attach program extension to another extension. 14377 * It's ok to attach fentry/fexit to extension program. 14378 */ 14379 bpf_log(log, "Cannot recursively attach\n"); 14380 return -EINVAL; 14381 } 14382 if (tgt_prog->type == BPF_PROG_TYPE_TRACING && 14383 prog_extension && 14384 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || 14385 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { 14386 /* Program extensions can extend all program types 14387 * except fentry/fexit. The reason is the following. 14388 * The fentry/fexit programs are used for performance 14389 * analysis, stats and can be attached to any program 14390 * type except themselves. When extension program is 14391 * replacing XDP function it is necessary to allow 14392 * performance analysis of all functions. Both original 14393 * XDP program and its program extension. Hence 14394 * attaching fentry/fexit to BPF_PROG_TYPE_EXT is 14395 * allowed. If extending of fentry/fexit was allowed it 14396 * would be possible to create long call chain 14397 * fentry->extension->fentry->extension beyond 14398 * reasonable stack size. Hence extending fentry is not 14399 * allowed. 14400 */ 14401 bpf_log(log, "Cannot extend fentry/fexit\n"); 14402 return -EINVAL; 14403 } 14404 } else { 14405 if (prog_extension) { 14406 bpf_log(log, "Cannot replace kernel functions\n"); 14407 return -EINVAL; 14408 } 14409 } 14410 14411 switch (prog->expected_attach_type) { 14412 case BPF_TRACE_RAW_TP: 14413 if (tgt_prog) { 14414 bpf_log(log, 14415 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); 14416 return -EINVAL; 14417 } 14418 if (!btf_type_is_typedef(t)) { 14419 bpf_log(log, "attach_btf_id %u is not a typedef\n", 14420 btf_id); 14421 return -EINVAL; 14422 } 14423 if (strncmp(prefix, tname, sizeof(prefix) - 1)) { 14424 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", 14425 btf_id, tname); 14426 return -EINVAL; 14427 } 14428 tname += sizeof(prefix) - 1; 14429 t = btf_type_by_id(btf, t->type); 14430 if (!btf_type_is_ptr(t)) 14431 /* should never happen in valid vmlinux build */ 14432 return -EINVAL; 14433 t = btf_type_by_id(btf, t->type); 14434 if (!btf_type_is_func_proto(t)) 14435 /* should never happen in valid vmlinux build */ 14436 return -EINVAL; 14437 14438 break; 14439 case BPF_TRACE_ITER: 14440 if (!btf_type_is_func(t)) { 14441 bpf_log(log, "attach_btf_id %u is not a function\n", 14442 btf_id); 14443 return -EINVAL; 14444 } 14445 t = btf_type_by_id(btf, t->type); 14446 if (!btf_type_is_func_proto(t)) 14447 return -EINVAL; 14448 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 14449 if (ret) 14450 return ret; 14451 break; 14452 default: 14453 if (!prog_extension) 14454 return -EINVAL; 14455 fallthrough; 14456 case BPF_MODIFY_RETURN: 14457 case BPF_LSM_MAC: 14458 case BPF_TRACE_FENTRY: 14459 case BPF_TRACE_FEXIT: 14460 if (!btf_type_is_func(t)) { 14461 bpf_log(log, "attach_btf_id %u is not a function\n", 14462 btf_id); 14463 return -EINVAL; 14464 } 14465 if (prog_extension && 14466 btf_check_type_match(log, prog, btf, t)) 14467 return -EINVAL; 14468 t = btf_type_by_id(btf, t->type); 14469 if (!btf_type_is_func_proto(t)) 14470 return -EINVAL; 14471 14472 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && 14473 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || 14474 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) 14475 return -EINVAL; 14476 14477 if (tgt_prog && conservative) 14478 t = NULL; 14479 14480 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); 14481 if (ret < 0) 14482 return ret; 14483 14484 if (tgt_prog) { 14485 if (subprog == 0) 14486 addr = (long) tgt_prog->bpf_func; 14487 else 14488 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; 14489 } else { 14490 addr = kallsyms_lookup_name(tname); 14491 if (!addr) { 14492 bpf_log(log, 14493 "The address of function %s cannot be found\n", 14494 tname); 14495 return -ENOENT; 14496 } 14497 } 14498 14499 if (prog->aux->sleepable) { 14500 ret = -EINVAL; 14501 switch (prog->type) { 14502 case BPF_PROG_TYPE_TRACING: 14503 /* fentry/fexit/fmod_ret progs can be sleepable only if they are 14504 * attached to ALLOW_ERROR_INJECTION and are not in denylist. 14505 */ 14506 if (!check_non_sleepable_error_inject(btf_id) && 14507 within_error_injection_list(addr)) 14508 ret = 0; 14509 break; 14510 case BPF_PROG_TYPE_LSM: 14511 /* LSM progs check that they are attached to bpf_lsm_*() funcs. 14512 * Only some of them are sleepable. 14513 */ 14514 if (bpf_lsm_is_sleepable_hook(btf_id)) 14515 ret = 0; 14516 break; 14517 default: 14518 break; 14519 } 14520 if (ret) { 14521 bpf_log(log, "%s is not sleepable\n", tname); 14522 return ret; 14523 } 14524 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { 14525 if (tgt_prog) { 14526 bpf_log(log, "can't modify return codes of BPF programs\n"); 14527 return -EINVAL; 14528 } 14529 ret = check_attach_modify_return(addr, tname); 14530 if (ret) { 14531 bpf_log(log, "%s() is not modifiable\n", tname); 14532 return ret; 14533 } 14534 } 14535 14536 break; 14537 } 14538 tgt_info->tgt_addr = addr; 14539 tgt_info->tgt_name = tname; 14540 tgt_info->tgt_type = t; 14541 return 0; 14542 } 14543 14544 BTF_SET_START(btf_id_deny) 14545 BTF_ID_UNUSED 14546 #ifdef CONFIG_SMP 14547 BTF_ID(func, migrate_disable) 14548 BTF_ID(func, migrate_enable) 14549 #endif 14550 #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU 14551 BTF_ID(func, rcu_read_unlock_strict) 14552 #endif 14553 BTF_SET_END(btf_id_deny) 14554 14555 static int check_attach_btf_id(struct bpf_verifier_env *env) 14556 { 14557 struct bpf_prog *prog = env->prog; 14558 struct bpf_prog *tgt_prog = prog->aux->dst_prog; 14559 struct bpf_attach_target_info tgt_info = {}; 14560 u32 btf_id = prog->aux->attach_btf_id; 14561 struct bpf_trampoline *tr; 14562 int ret; 14563 u64 key; 14564 14565 if (prog->type == BPF_PROG_TYPE_SYSCALL) { 14566 if (prog->aux->sleepable) 14567 /* attach_btf_id checked to be zero already */ 14568 return 0; 14569 verbose(env, "Syscall programs can only be sleepable\n"); 14570 return -EINVAL; 14571 } 14572 14573 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && 14574 prog->type != BPF_PROG_TYPE_LSM) { 14575 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); 14576 return -EINVAL; 14577 } 14578 14579 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) 14580 return check_struct_ops_btf_id(env); 14581 14582 if (prog->type != BPF_PROG_TYPE_TRACING && 14583 prog->type != BPF_PROG_TYPE_LSM && 14584 prog->type != BPF_PROG_TYPE_EXT) 14585 return 0; 14586 14587 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); 14588 if (ret) 14589 return ret; 14590 14591 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { 14592 /* to make freplace equivalent to their targets, they need to 14593 * inherit env->ops and expected_attach_type for the rest of the 14594 * verification 14595 */ 14596 env->ops = bpf_verifier_ops[tgt_prog->type]; 14597 prog->expected_attach_type = tgt_prog->expected_attach_type; 14598 } 14599 14600 /* store info about the attachment target that will be used later */ 14601 prog->aux->attach_func_proto = tgt_info.tgt_type; 14602 prog->aux->attach_func_name = tgt_info.tgt_name; 14603 14604 if (tgt_prog) { 14605 prog->aux->saved_dst_prog_type = tgt_prog->type; 14606 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; 14607 } 14608 14609 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { 14610 prog->aux->attach_btf_trace = true; 14611 return 0; 14612 } else if (prog->expected_attach_type == BPF_TRACE_ITER) { 14613 if (!bpf_iter_prog_supported(prog)) 14614 return -EINVAL; 14615 return 0; 14616 } 14617 14618 if (prog->type == BPF_PROG_TYPE_LSM) { 14619 ret = bpf_lsm_verify_prog(&env->log, prog); 14620 if (ret < 0) 14621 return ret; 14622 } else if (prog->type == BPF_PROG_TYPE_TRACING && 14623 btf_id_set_contains(&btf_id_deny, btf_id)) { 14624 return -EINVAL; 14625 } 14626 14627 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); 14628 tr = bpf_trampoline_get(key, &tgt_info); 14629 if (!tr) 14630 return -ENOMEM; 14631 14632 prog->aux->dst_trampoline = tr; 14633 return 0; 14634 } 14635 14636 struct btf *bpf_get_btf_vmlinux(void) 14637 { 14638 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { 14639 mutex_lock(&bpf_verifier_lock); 14640 if (!btf_vmlinux) 14641 btf_vmlinux = btf_parse_vmlinux(); 14642 mutex_unlock(&bpf_verifier_lock); 14643 } 14644 return btf_vmlinux; 14645 } 14646 14647 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr) 14648 { 14649 u64 start_time = ktime_get_ns(); 14650 struct bpf_verifier_env *env; 14651 struct bpf_verifier_log *log; 14652 int i, len, ret = -EINVAL; 14653 bool is_priv; 14654 14655 /* no program is valid */ 14656 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 14657 return -EINVAL; 14658 14659 /* 'struct bpf_verifier_env' can be global, but since it's not small, 14660 * allocate/free it every time bpf_check() is called 14661 */ 14662 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 14663 if (!env) 14664 return -ENOMEM; 14665 log = &env->log; 14666 14667 len = (*prog)->len; 14668 env->insn_aux_data = 14669 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 14670 ret = -ENOMEM; 14671 if (!env->insn_aux_data) 14672 goto err_free_env; 14673 for (i = 0; i < len; i++) 14674 env->insn_aux_data[i].orig_idx = i; 14675 env->prog = *prog; 14676 env->ops = bpf_verifier_ops[env->prog->type]; 14677 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); 14678 is_priv = bpf_capable(); 14679 14680 bpf_get_btf_vmlinux(); 14681 14682 /* grab the mutex to protect few globals used by verifier */ 14683 if (!is_priv) 14684 mutex_lock(&bpf_verifier_lock); 14685 14686 if (attr->log_level || attr->log_buf || attr->log_size) { 14687 /* user requested verbose verifier output 14688 * and supplied buffer to store the verification trace 14689 */ 14690 log->level = attr->log_level; 14691 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 14692 log->len_total = attr->log_size; 14693 14694 /* log attributes have to be sane */ 14695 if (!bpf_verifier_log_attr_valid(log)) { 14696 ret = -EINVAL; 14697 goto err_unlock; 14698 } 14699 } 14700 14701 mark_verifier_state_clean(env); 14702 14703 if (IS_ERR(btf_vmlinux)) { 14704 /* Either gcc or pahole or kernel are broken. */ 14705 verbose(env, "in-kernel BTF is malformed\n"); 14706 ret = PTR_ERR(btf_vmlinux); 14707 goto skip_full_check; 14708 } 14709 14710 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 14711 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 14712 env->strict_alignment = true; 14713 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 14714 env->strict_alignment = false; 14715 14716 env->allow_ptr_leaks = bpf_allow_ptr_leaks(); 14717 env->allow_uninit_stack = bpf_allow_uninit_stack(); 14718 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); 14719 env->bypass_spec_v1 = bpf_bypass_spec_v1(); 14720 env->bypass_spec_v4 = bpf_bypass_spec_v4(); 14721 env->bpf_capable = bpf_capable(); 14722 14723 if (is_priv) 14724 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; 14725 14726 env->explored_states = kvcalloc(state_htab_size(env), 14727 sizeof(struct bpf_verifier_state_list *), 14728 GFP_USER); 14729 ret = -ENOMEM; 14730 if (!env->explored_states) 14731 goto skip_full_check; 14732 14733 ret = add_subprog_and_kfunc(env); 14734 if (ret < 0) 14735 goto skip_full_check; 14736 14737 ret = check_subprogs(env); 14738 if (ret < 0) 14739 goto skip_full_check; 14740 14741 ret = check_btf_info(env, attr, uattr); 14742 if (ret < 0) 14743 goto skip_full_check; 14744 14745 ret = check_attach_btf_id(env); 14746 if (ret) 14747 goto skip_full_check; 14748 14749 ret = resolve_pseudo_ldimm64(env); 14750 if (ret < 0) 14751 goto skip_full_check; 14752 14753 if (bpf_prog_is_dev_bound(env->prog->aux)) { 14754 ret = bpf_prog_offload_verifier_prep(env->prog); 14755 if (ret) 14756 goto skip_full_check; 14757 } 14758 14759 ret = check_cfg(env); 14760 if (ret < 0) 14761 goto skip_full_check; 14762 14763 ret = do_check_subprogs(env); 14764 ret = ret ?: do_check_main(env); 14765 14766 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 14767 ret = bpf_prog_offload_finalize(env); 14768 14769 skip_full_check: 14770 kvfree(env->explored_states); 14771 14772 if (ret == 0) 14773 ret = check_max_stack_depth(env); 14774 14775 /* instruction rewrites happen after this point */ 14776 if (is_priv) { 14777 if (ret == 0) 14778 opt_hard_wire_dead_code_branches(env); 14779 if (ret == 0) 14780 ret = opt_remove_dead_code(env); 14781 if (ret == 0) 14782 ret = opt_remove_nops(env); 14783 } else { 14784 if (ret == 0) 14785 sanitize_dead_code(env); 14786 } 14787 14788 if (ret == 0) 14789 /* program is valid, convert *(u32*)(ctx + off) accesses */ 14790 ret = convert_ctx_accesses(env); 14791 14792 if (ret == 0) 14793 ret = do_misc_fixups(env); 14794 14795 /* do 32-bit optimization after insn patching has done so those patched 14796 * insns could be handled correctly. 14797 */ 14798 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { 14799 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); 14800 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret 14801 : false; 14802 } 14803 14804 if (ret == 0) 14805 ret = fixup_call_args(env); 14806 14807 env->verification_time = ktime_get_ns() - start_time; 14808 print_verification_stats(env); 14809 env->prog->aux->verified_insns = env->insn_processed; 14810 14811 if (log->level && bpf_verifier_log_full(log)) 14812 ret = -ENOSPC; 14813 if (log->level && !log->ubuf) { 14814 ret = -EFAULT; 14815 goto err_release_maps; 14816 } 14817 14818 if (ret) 14819 goto err_release_maps; 14820 14821 if (env->used_map_cnt) { 14822 /* if program passed verifier, update used_maps in bpf_prog_info */ 14823 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 14824 sizeof(env->used_maps[0]), 14825 GFP_KERNEL); 14826 14827 if (!env->prog->aux->used_maps) { 14828 ret = -ENOMEM; 14829 goto err_release_maps; 14830 } 14831 14832 memcpy(env->prog->aux->used_maps, env->used_maps, 14833 sizeof(env->used_maps[0]) * env->used_map_cnt); 14834 env->prog->aux->used_map_cnt = env->used_map_cnt; 14835 } 14836 if (env->used_btf_cnt) { 14837 /* if program passed verifier, update used_btfs in bpf_prog_aux */ 14838 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, 14839 sizeof(env->used_btfs[0]), 14840 GFP_KERNEL); 14841 if (!env->prog->aux->used_btfs) { 14842 ret = -ENOMEM; 14843 goto err_release_maps; 14844 } 14845 14846 memcpy(env->prog->aux->used_btfs, env->used_btfs, 14847 sizeof(env->used_btfs[0]) * env->used_btf_cnt); 14848 env->prog->aux->used_btf_cnt = env->used_btf_cnt; 14849 } 14850 if (env->used_map_cnt || env->used_btf_cnt) { 14851 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 14852 * bpf_ld_imm64 instructions 14853 */ 14854 convert_pseudo_ld_imm64(env); 14855 } 14856 14857 adjust_btf_func(env); 14858 14859 err_release_maps: 14860 if (!env->prog->aux->used_maps) 14861 /* if we didn't copy map pointers into bpf_prog_info, release 14862 * them now. Otherwise free_used_maps() will release them. 14863 */ 14864 release_maps(env); 14865 if (!env->prog->aux->used_btfs) 14866 release_btfs(env); 14867 14868 /* extension progs temporarily inherit the attach_type of their targets 14869 for verification purposes, so set it back to zero before returning 14870 */ 14871 if (env->prog->type == BPF_PROG_TYPE_EXT) 14872 env->prog->expected_attach_type = 0; 14873 14874 *prog = env->prog; 14875 err_unlock: 14876 if (!is_priv) 14877 mutex_unlock(&bpf_verifier_lock); 14878 vfree(env->insn_aux_data); 14879 err_free_env: 14880 kfree(env); 14881 return ret; 14882 } 14883